From bbc5d73e0d9b19b1c7a016ddb0c2da6790669c29 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 11:44:11 +0200 Subject: [PATCH 0001/1795] first version of bot/check-result.sh for software-layer - follows what has been implemented in the EESSI/eessi-bot-software-layer for checking the result of a job - re-uses code from a small script that was run manually on a bot instance to check a job's result - also re-uses code from the check-result.sh script in EESSI/compatibility-layer --- bot/check-result.sh | 195 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100755 bot/check-result.sh diff --git a/bot/check-result.sh b/bot/check-result.sh new file mode 100755 index 0000000000..38e76c3abc --- /dev/null +++ b/bot/check-result.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# +# Script to check the result of building the EESSI software layer. +# Intended use is that it is called by a (batch) job running on a compute +# node. +# +# This script is part of the EESSI compatibility layer, see +# https://github.com/EESSI/compatibility-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# result cases + +# - SUCCESS (all of) +# - working directory contains slurm-JOBID.out file +# - working directory contains eessi*tar.gz +# - no message ERROR +# - no message FAILED +# - no message ' required modules missing:' +# - one or more of 'No missing modules!' +# - message regarding created tarball +# - FAILED (one of ... implemented as NOT SUCCESS) +# - no slurm-JOBID.out file +# - no tarball +# - message with ERROR +# - message with FAILED +# - message with ' required modules missing:' +# - no message regarding created tarball + +# stop as soon as something fails +# set -e + +TOPDIR=$(dirname $(realpath $0)) + +source ${TOPDIR}/scripts/utils.sh +source ${TOPDIR}/scripts/cfg_files.sh + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " OPTIONS:" + echo " -h | --help - display this usage information [default: false]" + echo " -v | --verbose - display more information [default: false]" +} + +# set defaults for command line arguments +VERBOSE=0 + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + display_help + exit 0 + ;; + -v|--verbose) + VERBOSE=1 + shift 1 + ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +job_dir=${PWD} + +[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" + +GP_slurm_out="slurm-${SLURM_JOB_ID}.out" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" +job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") +[[ $? -eq 0 ]] && SLURM=1 || SLURM=0 +[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" + +GP_error='ERROR: ' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" +grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && ERROR=1 || ERROR=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_failed='FAILED: ' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" +grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && FAILED=1 || FAILED=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_req_missing=' required modules missing:' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" +grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && MISSING=1 || MISSING=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_no_missing='No missing modules!' +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" +grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +GP_tgz_created="tar.gz created!" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" +grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) +TARBALL= +if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') +else + TGZ=0 +fi +[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + +echo "SUMMARY: ${job_dir}/${job_out}" +echo " test name : result (expected result)" +echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" + +if [[ ${SLURM} -eq 1 ]] && \ + [[ ${ERROR} -eq 0 ]] && \ + [[ ${FAILED} -eq 0 ]] && \ + [[ ${MISSING} -eq 0 ]] && \ + [[ ${NO_MISSING} -eq 1 ]] && \ + [[ ${TGZ} -eq 1 ]] && \ + [[ ! -z ${TARBALL} ]]; then + # SUCCESS + echo "[RESULT]" > ${job_result_file} + echo "summary = :grin: SUCCESS" >> ${job_result_file} + echo "details =" >> ${job_result_file} +else + # FAILURE + echo "[RESULT]" > ${job_result_file} + echo "summary = :cry: FAILURE" >> ${job_result_file} + echo "details =" >> ${job_result_file} +fi + +if [[ ${SLURM} -eq 1 ]]; then + # need to indent by 4 spaces + echo " job output file ${job_out} (pattern: ${GP_slurm_out})" >> ${job_result_file} +else + echo " no job output file matching ${GP_slurm_out}" >> ${job_result_file} +fi + +if [[ ${ERROR} -eq 0 ]]; then + echo " job output lacks message matching ${GP_error}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_error}" >> ${job_result_file} +fi + +if [[ ${FAILED} -eq 0 ]]; then + echo " job output lacks message matching ${GP_failed}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_failed}" >> ${job_result_file} +fi + +if [[ ${MISSING} -eq 0 ]]; then + echo " job output lacks message matching ${GP_req_missing}" >> ${job_result_file} +else + echo " job output contains message matching ${GP_req_missing}" >> ${job_result_file} +fi + +if [[ ${NO_MISSING} -eq 1 ]]; then + echo " found message(s) matching ${GP_no_missing}" >> ${job_result_file} +else + echo " found no message matching ${GP_no_missing}" >> ${job_result_file} +fi + +if [[ ${TGZ} -eq 1 ]]; then + echo " found message matching ${GP_tgz_created}" >> ${job_result_file} +else + echo " found no message matching ${GP_tgz_created}" >> ${job_result_file} +fi + +echo "artefacts =" >> ${job_result_file} + +if [[ ! -z ${TARBALL} ]]; then + echo " ${TARBALL}" >> ${job_result_file} +fi + +exit 0 From fb04876141a464d98eb4cbb1aa3e34e4bc1a321c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 14:34:27 +0200 Subject: [PATCH 0002/1795] fix path to helper scripts + define result file name --- bot/check-result.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 38e76c3abc..15356f10cf 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -35,8 +35,8 @@ TOPDIR=$(dirname $(realpath $0)) -source ${TOPDIR}/scripts/utils.sh -source ${TOPDIR}/scripts/cfg_files.sh +source ${TOPDIR}/../scripts/utils.sh +source ${TOPDIR}/../scripts/cfg_files.sh display_help() { echo "usage: $0 [OPTIONS]" @@ -131,6 +131,8 @@ echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +job_result_file=_bot_job${SLURM_JOB_ID}.result + if [[ ${SLURM} -eq 1 ]] && \ [[ ${ERROR} -eq 0 ]] && \ [[ ${FAILED} -eq 0 ]] && \ From 5ebc5d92014b97738b85102d175edf77d14e2799 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 18 Apr 2023 15:41:45 +0200 Subject: [PATCH 0003/1795] define functions for standardizing result output --- bot/check-result.sh | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 15356f10cf..dbc76e2dc9 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -151,41 +151,48 @@ else echo "details =" >> ${job_result_file} fi +function succeeded() { + echo "    :heavy_check_mark:${1}" +} + +function failed() { + echo "    :heavy_multiplication_x:${1}" +} + if [[ ${SLURM} -eq 1 ]]; then - # need to indent by 4 spaces - echo " job output file ${job_out} (pattern: ${GP_slurm_out})" >> ${job_result_file} + succeeded "job output file ${job_out}" >> ${job_result_file} else - echo " no job output file matching ${GP_slurm_out}" >> ${job_result_file} + failed "no job output file matching ${GP_slurm_out}" >> ${job_result_file} fi if [[ ${ERROR} -eq 0 ]]; then - echo " job output lacks message matching ${GP_error}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_error}" >> ${job_result_file} else - echo " job output contains message matching ${GP_error}" >> ${job_result_file} + failed "job output contains message matching ${GP_error}" >> ${job_result_file} fi if [[ ${FAILED} -eq 0 ]]; then - echo " job output lacks message matching ${GP_failed}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_failed}" >> ${job_result_file} else - echo " job output contains message matching ${GP_failed}" >> ${job_result_file} + failed "job output contains message matching ${GP_failed}" >> ${job_result_file} fi if [[ ${MISSING} -eq 0 ]]; then - echo " job output lacks message matching ${GP_req_missing}" >> ${job_result_file} + succeeded "job output lacks message matching ${GP_req_missing}" >> ${job_result_file} else - echo " job output contains message matching ${GP_req_missing}" >> ${job_result_file} + failed "job output contains message matching ${GP_req_missing}" >> ${job_result_file} fi if [[ ${NO_MISSING} -eq 1 ]]; then - echo " found message(s) matching ${GP_no_missing}" >> ${job_result_file} + succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} else - echo " found no message matching ${GP_no_missing}" >> ${job_result_file} + failed "found no message matching ${GP_no_missing}" >> ${job_result_file} fi if [[ ${TGZ} -eq 1 ]]; then - echo " found message matching ${GP_tgz_created}" >> ${job_result_file} + succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} else - echo " found no message matching ${GP_tgz_created}" >> ${job_result_file} + failed "found no message matching ${GP_tgz_created}" >> ${job_result_file} fi echo "artefacts =" >> ${job_result_file} From b8621545003d3d3e9bd482152a84f1aada2227ed Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 11:29:05 +0200 Subject: [PATCH 0004/1795] polishing formating of status messages --- bot/check-result.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index dbc76e2dc9..9e34f81dd8 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,11 +152,11 @@ else fi function succeeded() { - echo "    :heavy_check_mark:${1}" + echo " :heavy_check_mark:${1}" } function failed() { - echo "    :heavy_multiplication_x:${1}" + echo " :heavy_multiplication_x:${1}" } if [[ ${SLURM} -eq 1 ]]; then From 95e8ff8e55f316852aeb1ec09ce94db5cacb1b62 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 12:29:56 +0200 Subject: [PATCH 0005/1795] more polishing of comment details --- bot/check-result.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 9e34f81dd8..2d2f33be06 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -166,33 +166,33 @@ else fi if [[ ${ERROR} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_error}" >> ${job_result_file} + succeeded "no message matching ${GP_error}" >> ${job_result_file} else - failed "job output contains message matching ${GP_error}" >> ${job_result_file} + failed "found message matching ${GP_error}" >> ${job_result_file} fi if [[ ${FAILED} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_failed}" >> ${job_result_file} + succeeded "no message matching ${GP_failed}" >> ${job_result_file} else - failed "job output contains message matching ${GP_failed}" >> ${job_result_file} + failed "found message matching ${GP_failed}" >> ${job_result_file} fi if [[ ${MISSING} -eq 0 ]]; then - succeeded "job output lacks message matching ${GP_req_missing}" >> ${job_result_file} + succeeded "no message matching ${GP_req_missing}" >> ${job_result_file} else - failed "job output contains message matching ${GP_req_missing}" >> ${job_result_file} + failed "found message matching ${GP_req_missing}" >> ${job_result_file} fi if [[ ${NO_MISSING} -eq 1 ]]; then succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} else - failed "found no message matching ${GP_no_missing}" >> ${job_result_file} + failed "no message matching ${GP_no_missing}" >> ${job_result_file} fi if [[ ${TGZ} -eq 1 ]]; then succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} else - failed "found no message matching ${GP_tgz_created}" >> ${job_result_file} + failed "no message matching ${GP_tgz_created}" >> ${job_result_file} fi echo "artefacts =" >> ${job_result_file} From 6724b49b18dc0a4cabc87c5526e0e8741b148686 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:06:15 +0200 Subject: [PATCH 0006/1795] minor polishing of job result messages --- bot/check-result.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 2d2f33be06..1598e9ca1b 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,11 +152,11 @@ else fi function succeeded() { - echo " :heavy_check_mark:${1}" + echo " :heavy_check_mark: ${1}" } function failed() { - echo " :heavy_multiplication_x:${1}" + echo " :heavy_multiplication_x: ${1}" } if [[ ${SLURM} -eq 1 ]]; then From b70c858d65eadba24366518d30f63a3fdca3b686 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:36:15 +0200 Subject: [PATCH 0007/1795] only log if run with --verbose --- bot/check-result.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 1598e9ca1b..55276c405a 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -123,13 +123,13 @@ else fi [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" -echo "SUMMARY: ${job_dir}/${job_out}" -echo " test name : result (expected result)" -echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" -echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" +[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" +[[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" job_result_file=_bot_job${SLURM_JOB_ID}.result From 677c36296f1ab433964b8caa230ad8ef32c5d625 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 15:22:40 +0200 Subject: [PATCH 0008/1795] fix small bug when checking for tarball msg --- bot/check-result.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 55276c405a..3f0c89ea21 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -112,9 +112,9 @@ grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_tgz_created="tar.gz created!" +TARBALL= [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) -TARBALL= if [[ $? -eq 0 ]]; then TGZ=1 TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') From e4932eefaff86adee6e91265efdfd5268efdf52a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 15:31:36 +0200 Subject: [PATCH 0009/1795] avoid creating false positives --- bot/check-result.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 3f0c89ea21..e28ae4e38e 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -82,38 +82,42 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" GP_slurm_out="slurm-${SLURM_JOB_ID}.out" -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") [[ $? -eq 0 ]] && SLURM=1 || SLURM=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" GP_error='ERROR: ' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_failed='FAILED: ' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_req_missing=' required modules missing:' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_no_missing='No missing modules!' -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" GP_tgz_created="tar.gz created!" TARBALL= -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) if [[ $? -eq 0 ]]; then TGZ=1 @@ -121,6 +125,8 @@ if [[ $? -eq 0 ]]; then else TGZ=0 fi +# have to be careful to not add searched for pattern into slurm out file +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" From 9a0e220db5a21b6285785ab09683bc0bb62e3108 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 5 Jun 2023 18:48:43 +0200 Subject: [PATCH 0010/1795] also filter OpenSSL dependency in EasyBuild configuration --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index 245553f342..5d4e9b5a48 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -26,7 +26,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) # problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,OpenSSL,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From 0a41c207e0ee1646cff53768cc86f83a8f740205 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 08:14:02 +0200 Subject: [PATCH 0011/1795] bump version to 2023.06 --- init/eessi_defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/eessi_defaults b/init/eessi_defaults index f482cbc269..0143dc38ab 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -9,4 +9,4 @@ # export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" -export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" +export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}" From 88a758a4c250111d35ca3c76e1516bd62c3e2620 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 08:14:45 +0200 Subject: [PATCH 0012/1795] configure EasyBuild to allow experimental features (like using easystack files) --- configure_easybuild | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure_easybuild b/configure_easybuild index 245553f342..23f3920154 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -36,3 +36,6 @@ fi export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER export EASYBUILD_MODULE_EXTENSIONS=1 + +# need to enable use of experimental features, since we're using easystack files +export EASYBUILD_EXPERIMENTAL=1 From 8aa1c25b69e84a76ce286029bb320e910e8aff3b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 08:23:22 +0200 Subject: [PATCH 0013/1795] update install script to use EasyBuild 4.7.2 + use easystack file for foss/2021a --- EESSI-pilot-install-software.sh | 244 +++---------------------------- eessi-2023.04-eb-4.7.2-2021a.yml | 6 + 2 files changed, 24 insertions(+), 226 deletions(-) create mode 100644 eessi-2023.04-eb-4.7.2-2021a.yml diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index d9bcf20231..f58903f8cd 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -137,235 +137,27 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -REQ_EB_VERSION='4.5.0' - -# load EasyBuild module (will be installed if it's not available yet) -source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION} - -echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." - -# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, -# see https://github.com/EESSI/software-layer/issues/123 -# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 -ok_msg="Java installed, off to a good (?) start!" -fail_msg="Failed to install Java, woopsie..." -$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install GCC for foss/2020a -export GCC_EC="GCC-9.3.0.eb" -echo ">> Starting slow with ${GCC_EC}..." -ok_msg="${GCC_EC} installed, yippy! Off to a good start..." -fail_msg="Installation of ${GCC_EC} failed!" -# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place -$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install CMake with custom easyblock that patches CMake when --sysroot is used -echo ">> Install CMake with fixed easyblock to take into account --sysroot" -ok_msg="CMake installed!" -fail_msg="Installation of CMake failed, what the ..." -$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 -echo ">> Installing OpenBLAS..." -ok_msg="Done with OpenBLAS!" -fail_msg="Installation of OpenBLAS failed!" -if [[ $GENERIC -eq 1 ]]; then - echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" -else - openblas_include_easyblocks_from_pr='' -fi -$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OpenMPI..." -ok_msg="OpenMPI installed, w00!" -fail_msg="Installation of OpenMPI failed, that's not good..." -$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install Python -echo ">> Install Python 2.7.18 and Python 3.8.2..." -ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" -fail_msg="Installation of Python failed, oh no..." -$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Perl..." -ok_msg="Perl installed, making progress..." -fail_msg="Installation of Perl failed, this never happens..." -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Qt5..." -ok_msg="Qt5 installed, phieuw, that was a big one!" -fail_msg="Installation of Qt5 failed, that's frustrating..." -$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# skip test step when installing SciPy-bundle on aarch64, -# to dance around problem with broken numpy tests; -# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 -echo ">> Installing SciPy-bundle" -ok_msg="SciPy-bundle installed, yihaa!" -fail_msg="SciPy-bundle installation failed, bummer..." -SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb -if [[ "$(uname -m)" == "aarch64" ]]; then - $EB $SCIPY_EC --robot --skip-test-step -else - $EB $SCIPY_EC --robot -fi -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing GROMACS..." -ok_msg="GROMACS installed, wow!" -fail_msg="Installation of GROMACS failed, damned..." -$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! -# 32GB is sufficient to build with 16 cores -echo ">> Installing OpenFOAM (twice!)..." -ok_msg="OpenFOAM installed, now we're talking!" -fail_msg="Installation of OpenFOAM failed, we were so close..." -$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - echo ">> Installing QuantumESPRESSO..." - ok_msg="QuantumESPRESSO installed, let's go quantum!" - fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi +for eb_version in '4.7.2'; do -echo ">> Installing R 4.0.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Bioconductor 3.11 bundle..." -ok_msg="Bioconductor installed, enjoy!" -fail_msg="Installation of Bioconductor failed, that's annoying..." -$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing TensorFlow 2.3.1..." -ok_msg="TensorFlow 2.3.1 installed, w00!" -fail_msg="Installation of TensorFlow failed, why am I not surprised..." -$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Horovod 0.21.3..." -ok_msg="Horovod installed! Go do some parallel training!" -fail_msg="Horovod installation failed. There comes the headache..." -$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - - echo ">> Installing code-server 3.7.3..." - ok_msg="code-server 3.7.3 installed, now you can use VS Code!" - fail_msg="Installation of code-server failed, that's going to be hard to fix..." - $EB code-server-3.7.3.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION} -echo ">> Installing RStudio-Server 1.3.1093..." -ok_msg="RStudio-Server installed, enjoy!" -fail_msg="Installation of RStudio-Server failed, might be OS deps..." -$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Spark 3.1.1..." -ok_msg="Spark installed, set off the fireworks!" -fail_msg="Installation of Spark failed, no fireworks this time..." -$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing IPython 7.15.0..." -ok_msg="IPython installed, launch your Jupyter Notebooks!" -fail_msg="Installation of IPython failed, that's unexpected..." -$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing WRF 3.9.1.1..." -ok_msg="WRF installed, it's getting hot in here!" -fail_msg="Installation of WRF failed, that's unexpected..." -OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing R 4.1.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Nextflow 22.10.1..." -ok_msg="Nextflow installed, the work must flow..." -fail_msg="Installation of Nextflow failed, that's unexpected..." -$EB -r --from-pr 16531 Nextflow-22.10.1.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing EasyBuild 4.5.1..." -ok_msg="EasyBuild v4.5.1 installed" -fail_msg="EasyBuild v4.5.1 failed to install" -$EB --from-pr 14545 --include-easyblocks-from-pr 2805 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 -check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" - -echo ">> Installing SciPy-bundle with foss/2021a..." -ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" -fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." -# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 -# which includes patch to fix installation with recent Linux kernel headers -$EB --from-pr 14454 GCCcore-10.3.0.eb --robot -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 -# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set -# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 -# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 -# that includes patch to fix bootstrap problem when using alternate sysroot -$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot -# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 -# which includes a patch to fix installation on POWER -$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot -# ignore failing FlexiBLAS tests when building on POWER; -# some tests are failing due to a segmentation fault due to "invalid memory reference", -# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; -# using -fstack-protector-strong -fstack-clash-protection should fix that, -# but it doesn't for some reason when building for ppc64le/generic... -if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure -else - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb -fi + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." + + for gen in '2021a'; do + + es="eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-${gen}.yml" -$EB SciPy-bundle-2021.05-foss-2021a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" + if [ -f ${es} ]; then + echo_green "Feeding easystack file ${es} to EasyBuild..." + + ${EB} --easystack ${TOPDIR}/${es} -M && ${EB} --easystack ${TOPDIR}/${es} --robot + else + fatal_error "Easystack file ${es} not found!" + fi + done + +done ### add packages here diff --git a/eessi-2023.04-eb-4.7.2-2021a.yml b/eessi-2023.04-eb-4.7.2-2021a.yml new file mode 100644 index 0000000000..9b1312ba9a --- /dev/null +++ b/eessi-2023.04-eb-4.7.2-2021a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - GCC-10.3.0 + - CMake-3.20.1-GCCcore-10.3.0.eb: + options: + include-easyblocks-from-pr: 2248 + - foss-2021a From b7870afd2e543cd19a07b4cafb94a9e1633b71b0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 08:24:00 +0200 Subject: [PATCH 0014/1795] implement pre-configure hook for OpenBLAS to use DYNAMIC_ARCH=1 when building for a generic CPU target --- eb_hooks.py | 56 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 2fba925b01..787e990e75 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -11,6 +11,13 @@ from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC +# prefer importing LooseVersion from easybuild.tools, but fall back to distuils in case EasyBuild <= 4.7.0 is used +try: + from easybuild.tools import LooseVersion +except ImportError: + from distutils.version import LooseVersion + + EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -56,13 +63,6 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) -def pre_configure_hook(self, *args, **kwargs): - """Main pre-configure hook: trigger custom functions based on software name.""" - - if self.name in PRE_CONFIGURE_HOOKS: - PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) - - def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" @@ -91,7 +91,7 @@ def pre_prepare_hook(self, *args, **kwargs): mpi_family, rpath_override_dirs) -def gcc_postprepare(self, *args, **kwargs): +def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ Post-configure hook for GCCcore: - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' @@ -121,6 +121,7 @@ def gcc_postprepare(self, *args, **kwargs): else: raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!") + def post_prepare_hook(self, *args, **kwargs): """Main post-prepare hook: trigger custom functions.""" @@ -134,7 +135,7 @@ def post_prepare_hook(self, *args, **kwargs): POST_PREPARE_HOOKS[self.name](self, *args, **kwargs) -def cgal_toolchainopts_precise(ec, eprefix): +def parse_hook_cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" if ec.name == 'CGAL': if get_cpu_architecture() == POWER: @@ -147,7 +148,7 @@ def cgal_toolchainopts_precise(ec, eprefix): raise EasyBuildError("CGAL-specific hook triggered for non-CGAL easyconfig?!") -def fontconfig_add_fonts(ec, eprefix): +def parse_hook_fontconfig_add_fonts(ec, eprefix): """Inject --with-add-fonts configure option for fontconfig.""" if ec.name == 'fontconfig': # make fontconfig aware of fonts included with compat layer @@ -158,7 +159,7 @@ def fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") -def ucx_eprefix(ec, eprefix): +def parse_hook_ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) @@ -174,7 +175,19 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) -def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): +def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): + """ + Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC + """ + if self.name == 'OpenBLAS': + if build_option('optarch') == OPTARCH_GENERIC: + for step in ('build', 'test', 'install'): + self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1") + else: + raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") + + +def pre_configure_hook_libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): """Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64.""" if self.name == 'libfabric': if get_cpu_architecture() == X86_64: @@ -187,7 +200,7 @@ def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): raise EasyBuildError("libfabric-specific hook triggered for non-libfabric easyconfig?!") -def metabat_preconfigure(self, *args, **kwargs): +def pre_configure_hook_metabat_filtered_zlib_dep(self, *args, **kwargs): """ Pre-configure hook for MetaBAT: - take into account that zlib is a filtered dependency, @@ -201,7 +214,7 @@ def metabat_preconfigure(self, *args, **kwargs): raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!") -def wrf_preconfigure(self, *args, **kwargs): +def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): """ Pre-configure hook for WRF: - patch arch/configure_new.defaults so building WRF with foss toolchain works on aarch64 @@ -217,17 +230,18 @@ def wrf_preconfigure(self, *args, **kwargs): PARSE_HOOKS = { - 'CGAL': cgal_toolchainopts_precise, - 'fontconfig': fontconfig_add_fonts, - 'UCX': ucx_eprefix, + 'CGAL': parse_hook_cgal_toolchainopts_precise, + 'fontconfig': parse_hook_fontconfig_add_fonts, + 'UCX': parse_hook_ucx_eprefix, } POST_PREPARE_HOOKS = { - 'GCCcore': gcc_postprepare, + 'GCCcore': post_prepare_hook_gcc_prefixed_ld_rpath_wrapper, } PRE_CONFIGURE_HOOKS = { - 'libfabric': libfabric_disable_psm3_x86_64_generic, - 'MetaBAT': metabat_preconfigure, - 'WRF': wrf_preconfigure, + 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, + 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, + 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, + 'WRF': pre_configure_hook_wrf_aarch64, } From a4790b442d2f44b84f3be1bb1f025ed44247586c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 17:20:42 +0200 Subject: [PATCH 0015/1795] consistently use ${eb_version} rather than ${REQ_EB_VERSION} --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index f58903f8cd..74887db93b 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -140,7 +140,7 @@ fi for eb_version in '4.7.2'; do # load EasyBuild module (will be installed if it's not available yet) - source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION} + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." From 9cdc38e34f6b00ce49b2ac1b24b82c473e028433 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 17:24:09 +0200 Subject: [PATCH 0016/1795] rename easystack file for EESSI 2023.06 to use correct EESSI version --- ...2023.04-eb-4.7.2-2021a.yml => eessi-2023.06-eb-4.7.2-2021a.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename eessi-2023.04-eb-4.7.2-2021a.yml => eessi-2023.06-eb-4.7.2-2021a.yml (100%) diff --git a/eessi-2023.04-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml similarity index 100% rename from eessi-2023.04-eb-4.7.2-2021a.yml rename to eessi-2023.06-eb-4.7.2-2021a.yml From 64132649925ce3f040a4551589eeef8f9df9c0c3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 18:57:54 +0200 Subject: [PATCH 0017/1795] update check_missing_installations.sh to take path to easystack file as an argument, and hoist it into for loop that iterates over all easystack files --- EESSI-pilot-install-software.sh | 6 +++--- check_missing_installations.sh | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 74887db93b..f2d50dfe91 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -151,7 +151,9 @@ for eb_version in '4.7.2'; do if [ -f ${es} ]; then echo_green "Feeding easystack file ${es} to EasyBuild..." - ${EB} --easystack ${TOPDIR}/${es} -M && ${EB} --easystack ${TOPDIR}/${es} --robot + ${EB} --easystack ${TOPDIR}/${es} --missing && ${EB} --easystack ${TOPDIR}/${es} --robot + + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es} else fatal_error "Easystack file ${es} not found!" fi @@ -170,7 +172,5 @@ fi $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} -$TOPDIR/check_missing_installations.sh - echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 4a5316c09f..3627d1d0b5 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to check for missing installations in EESSI pilot software stack (version 2021.12) +# Script to check for missing installations in EESSI pilot software stack (version 2023.06) # # author: Kenneth Hoste (@boegel) # author: Thomas Roeblitz (@trz42) @@ -10,10 +10,11 @@ TOPDIR=$(dirname $(realpath $0)) -if [ -z ${EESSI_PILOT_VERSION} ]; then - echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2 +if [ $# -ne 1 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi +easystack=$1 LOCAL_TMPDIR=$(mktemp -d) @@ -23,9 +24,7 @@ source $TOPDIR/configure_easybuild echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used -# PR #16531: Nextflow-22.10.1.eb -${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." From 2b6d630c36eb078a268548b812af98143c15da8f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 14 Jun 2023 19:14:15 +0200 Subject: [PATCH 0018/1795] update test workflow to test with version 2023.06 of EESSI pilot repo --- .github/workflows/test_eessi.yml | 17 ++++++++--------- .github/workflows/tests_scripts.yml | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 04195dd619..65415f87fc 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: EESSI_VERSION: - - 2021.12 + - 2023.06 EESSI_SOFTWARE_SUBDIR: - aarch64/generic - aarch64/graviton2 @@ -20,6 +20,8 @@ jobs: - x86_64/intel/haswell - x86_64/intel/skylake_avx512 - x86_64/generic + EASYSTACK_FILE: + - eessi-2023.06-eb-4.7.2-2021a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 @@ -41,7 +43,7 @@ jobs: export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" - ./check_missing_installations.sh + ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | @@ -52,18 +54,15 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" - echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml - tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml + echo "modify easystack file by adding a missing package (GCC/8.3.0)" + echo " - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}} + tail -n 5 ${{matrix.EASYSTACK_FILE}} # note, check_missing_installations.sh exits 1 if a package was # missing, which is intepreted as false (exit code based, not # boolean logic), hence when the script exits 0 if no package was # missing it is interpreted as true, thus the test did not capture # the missing package - if ./check_missing_installations.sh; then + if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then echo "did NOT capture missing package; test FAILED" exit 1 else diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 5c0b3893ae..8ad4928993 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -92,6 +92,6 @@ jobs: # since create_directory_tarballs.sh must be accessible from within build container cp -a * /tmp/ cd /tmp - ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12 + ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz From a6b5bdef363abf7a35e5e315adfd992ea19b2c33 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 15 Jun 2023 22:46:54 +0200 Subject: [PATCH 0019/1795] update bot configuration for EESSI 2023.06 --- bot/bot-eessi-aws-citc.cfg | 149 +++++++++++++++++++++++++++++-------- 1 file changed, 116 insertions(+), 33 deletions(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 5b3ad34612..2766104d62 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -21,7 +21,23 @@ app_name = eessi-bot-citc-aws installation_id = 33078935 # path to the private key that was generated when the GitHub App was registered -private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem +private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws.2023-01-12.private-key.pem + + +[bot_control] +# which GH accounts have the permission to send commands to the bot +# if value is left/empty everyone can send commands +# value can be a space delimited list of GH accounts +command_permission = boegel trz42 bedroge + +# format of the response when processing bot commands +command_response_fmt = +
Updates by the bot instance {app_name} + (click for details) + + {comment_response} + {comment_result} +
[buildenv] @@ -35,7 +51,7 @@ container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-ca # it may happen that we need to customize some CVMFS configuration # the value of cvmfs_customizations is a dictionary which maps a file # name to an entry that needs to be added to that file -cvmfs_customizations = {} +# cvmfs_customizations = {} # if compute nodes have no internet connection, we need to set http(s)_proxy # or commands such as pip3 cannot download software from package repositories @@ -52,7 +68,7 @@ jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs # useful/needed if some tool is not provided as system-wide package # (read by bot and handed over to build_job_script via parameter # --load-modules) -load_modules = +# load_modules = # PATH to temporary directory on build node ... ends up being used for # for example, EESSI_TMPDIR --> /tmp/$USER/EESSI @@ -78,22 +94,93 @@ submit_command = /usr/bin/sbatch # value can be a space delimited list of GH accounts build_permission = boegel trz42 bedroge +# template for comment when user who set a label has no permission to trigger build jobs +no_build_permission_comment = Label `bot:build` has been set by user `{build_labeler}`, but only users `{build_permission_users}` have permission to trigger the action + + +[deploycfg] +# script for uploading built software packages +tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging + +# URL to S3/minio bucket +# if attribute is set, bucket_base will be constructed as follows +# bucket_base=${endpoint_url}/${bucket_name} +# otherwise, bucket_base will be constructed as follows +# bucket_base=https://${bucket_name}.s3.amazonaws.com +# - The former variant is used for non AWS S3 services, eg, minio, or when +# the bucket name is not provided in the hostname (see latter case). +# - The latter variant is used for AWS S3 services. +# endpoint_url = URL_TO_S3_SERVER + +# bucket name +bucket_name = eessi-staging + +# upload policy: defines what policy is used for uploading built artefacts +# to an S3 bucket +# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) +# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) +# only upload the latest built artefact +# 'once' : only once upload any built artefact for the build target +# 'none' : do not upload any built artefacts +upload_policy = once + +# which GH account has the permission to trigger the deployment (by setting +# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the deployment +# value can be a space delimited list of GH accounts +deploy_permission = boegel trz42 bedroge + +# template for comment when user who set a label has no permission to trigger deploying tarballs +no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_labeler}`, but only users `{deploy_permission_users}` have permission to trigger the action + + [architecturetargets] # defines both for which architectures the bot will build # and what submission parameters shall be used +# 5 c4.2xlarge haswell 8 vCPU, 15 GiB RAM (1 + generic) +# 2 c4.4xlarge haswell 16 vCPU, 30 GiB RAM +# 5 c5.2xlarge skylake_avx512 8 vCPU, 16 GiB RAM (1) +# 1 c5.4xlarge skylake_avx512 16 vCPU, 32 GiB RAM +# 5 c5a.2xlarge zen2 8 vCPU, 16 GiB RAM (1) +# 1 c5a.4xlarge zen2 16 vCPU, 32 GiB RAM +# 5 c5d.2xlarge skylake_avx512 8 vCPU, 16 GiB RAM + 200 GB NVMe +# 5 c6a.2xlarge zen3 8 vCPU, 16 GiB RAM (1) +# 1 c6a.4xlarge zen3 16 vCPU, 32 GiB RAM +# 5 c6g.2xlarge graviton2 8 vCPU, 16 GiB RAM (1 + generic) +# 2 c6g.4xlarge graviton2 16 vCPU, 32 GiB RAM +# 1 c6g.8xlarge graviton2 32 vCPU, 64 GiB RAM +# 1 c6i.2xlarge cascadelake 8 vCPU, 16 GiB RAM (1) +# 5 c7g.2xlarge graviton3 8 vCPU, 16 GiB RAM (1) +# 1 c7g.4xlarge graviton3 16 vCPU, 32 GiB RAM # medium instances (8 cores, 16GB RAM) #arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} # larger instances (16 cores, 32GB RAM) -arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} +arch_target_map = { + "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", + "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", + "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", + "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", + "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", + "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", + "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", + "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge" } [repo_targets] # defines for which repository a arch_target should be build for # # only building for repository EESSI-pilot -repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]} +repo_target_map = { + "linux/x86_64/generic" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/x86_64/intel/haswell" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/x86_64/intel/skylake_avx512" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/x86_64/amd/zen2" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/x86_64/amd/zen3" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/aarch64/generic" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/aarch64/graviton2" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/aarch64/graviton3" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"] } # points to definition of repositories (default EESSI-pilot defined by build container) -repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles +repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/repos # configuration for event handler which receives events from a GitHub repository. [event_handler] @@ -118,34 +205,30 @@ poll_interval = 60 # full path to the command for manipulating existing jobs scontrol_command = /usr/bin/scontrol -[deploycfg] -# script for uploading built software packages -tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging -# URL to S3/minio bucket -# if attribute is set, bucket_base will be constructed as follows -# bucket_base=${endpoint_url}/${bucket_name} -# otherwise, bucket_base will be constructed as follows -# bucket_base=https://${bucket_name}.s3.amazonaws.com -# - The former variant is used for non AWS S3 services, eg, minio, or when -# the bucket name is not provided in the hostname (see latter case). -# - The latter variant is used for AWS S3 services. -#endpoint_url = URL_TO_S3_SERVER +# variable 'comment' under 'submitted_job_comments' should not be changed as there are regular expression patterns matching it +[submitted_job_comments] +initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` +awaits_release = job id `{job_id}` awaits release by job manager -# bucket name -bucket_name = eessi-staging -# upload policy: defines what policy is used for uploading built artefacts -# to an S3 bucket -# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) -# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) -# only upload the latest built artefact -# 'once' : only once upload any built artefact for the build target -# 'none' : do not upload any built artefacts -upload_policy = once +[new_job_comments] +awaits_launch = job awaits launch by Slurm scheduler -# which GH account has the permission to trigger the deployment (by setting -# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) -# if value is left/empty everyone can trigger the deployment -# value can be a space delimited list of GH accounts -deploy_permission = boegel trz42 bedroge + +[running_job_comments] +running_job = job `{job_id}` is running + + +[finished_job_comments] +success = :grin: SUCCESS tarball `{tarball_name}` ({tarball_size} GiB) in job dir +failure = :cry: FAILURE +no_slurm_out = No slurm output `{slurm_out}` in job dir +slurm_out = Found slurm output `{slurm_out}` in job dir +missing_modules = Slurm output lacks message "No missing modules!". +no_tarball_message = Slurm output lacks message about created tarball. +no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. +multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. +job_result_comment_fmt =
{summary} _(click triangle for detailed information)_Details:{details}
Artefacts:{artefacts}
+job_result_details_item_fmt =
    {item} +job_result_artefacts_item_fmt =
  • {item}
  • From cf85694390f4c44d6461e386f8ad052a6d89cc00 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 15 Jun 2023 23:01:06 +0200 Subject: [PATCH 0020/1795] drop 2023.04 repo + use aarch64/neoverse_n1 and aarch64/neoverse_v1 instead of linux/aarch64/graviton* --- bot/bot-eessi-aws-citc.cfg | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 2766104d62..dca81ef2ab 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -146,15 +146,13 @@ no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_ # 5 c5d.2xlarge skylake_avx512 8 vCPU, 16 GiB RAM + 200 GB NVMe # 5 c6a.2xlarge zen3 8 vCPU, 16 GiB RAM (1) # 1 c6a.4xlarge zen3 16 vCPU, 32 GiB RAM -# 5 c6g.2xlarge graviton2 8 vCPU, 16 GiB RAM (1 + generic) -# 2 c6g.4xlarge graviton2 16 vCPU, 32 GiB RAM -# 1 c6g.8xlarge graviton2 32 vCPU, 64 GiB RAM +# 5 c6g.2xlarge neoverse_n1 8 vCPU, 16 GiB RAM (1 + generic) +# 2 c6g.4xlarge neoverse_n1 16 vCPU, 32 GiB RAM +# 1 c6g.8xlarge neoverse_n1 32 vCPU, 64 GiB RAM # 1 c6i.2xlarge cascadelake 8 vCPU, 16 GiB RAM (1) -# 5 c7g.2xlarge graviton3 8 vCPU, 16 GiB RAM (1) -# 1 c7g.4xlarge graviton3 16 vCPU, 32 GiB RAM -# medium instances (8 cores, 16GB RAM) -#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} -# larger instances (16 cores, 32GB RAM) +# 5 c7g.2xlarge neoverse_v1 8 vCPU, 16 GiB RAM (1) +# 1 c7g.4xlarge neoverse_v1 16 vCPU, 32 GiB RAM +# larger instances (*.4xlarge => 16 cores, 32GB RAM) arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", @@ -162,22 +160,22 @@ arch_target_map = { "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", - "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", - "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge" } + "linux/aarch64/neoverse_n1" : "--constraint shape=c6g.4xlarge", + "linux/aarch64/neoverse_v1" : "--constraint shape=c7g.4xlarge" } [repo_targets] # defines for which repository a arch_target should be build for # # only building for repository EESSI-pilot repo_target_map = { - "linux/x86_64/generic" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/intel/haswell" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/intel/skylake_avx512" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/amd/zen2" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/amd/zen3" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/generic" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/graviton2" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/graviton3" : ["eessi-2021.12","eessi-2023.04","eessi-2023.06-compat","eessi-2023.06-software"] } + "linux/x86_64/generic" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/x86_64/intel/haswell" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/x86_64/intel/skylake_avx512" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/x86_64/amd/zen2" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/x86_64/amd/zen3" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/aarch64/generic" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/aarch64/neoverse_n1" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], + "linux/aarch64/neoverse_v1" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"] } # points to definition of repositories (default EESSI-pilot defined by build container) repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/repos From 633de6a2349286a604ec85ecf51af30e4446e9b2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Jun 2023 17:49:23 +0200 Subject: [PATCH 0021/1795] bump EESSI pilot version to 2023.06 + update create_directory_tarballs.sh script to use '2023.06' branch of EESSI/software-layer repo --- create_directory_tarballs.sh | 2 +- init/eessi_defaults | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh index 70e666f871..4465465082 100755 --- a/create_directory_tarballs.sh +++ b/create_directory_tarballs.sh @@ -1,6 +1,6 @@ #!/bin/bash -SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/main +SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/2023.06 set -eo pipefail diff --git a/init/eessi_defaults b/init/eessi_defaults index f482cbc269..0143dc38ab 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -9,4 +9,4 @@ # export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" -export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" +export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}" From ddbaaab7c3386b20656c6d27e0ad654f0d217554 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Jun 2023 17:57:18 +0200 Subject: [PATCH 0022/1795] also run tests for README and scripts for non-main branches --- .github/workflows/tests_readme.yml | 2 -- .github/workflows/tests_scripts.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index 5c6d0318d4..ac906125ab 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -7,8 +7,6 @@ on: - init/eessi_defaults pull_request: - branches: - - main paths: - README.md - init/eessi_defaults diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 5c0b3893ae..81a3633b72 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -13,8 +13,6 @@ on: - update_lmod_cache.sh pull_request: - branches: - - main paths: - build_container.sh - create_directory_tarballs.sh From 60d60a34070f6bf892e8f3451c9078a3c7ed59ee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Jun 2023 17:58:29 +0200 Subject: [PATCH 0023/1795] update README for 2023.06 --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index daf02eebc2..e24de45b0e 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ See also https://eessi.github.io/docs/software_layer. You can set up your environment by sourcing the init script: ``` -$ source /cvmfs/pilot.eessi-hpc.org/versions/2021.12/init/bash -Found EESSI pilot repo @ /cvmfs/pilot.eessi-hpc.org/versions/2021.12! +$ source /cvmfs/pilot.eessi-hpc.org/versions/2023.06/init/bash +Found EESSI pilot repo @ /cvmfs/pilot.eessi-hpc.org/versions/2023.06! Derived subdirectory for software layer: x86_64/intel/haswell Using x86_64/intel/haswell subdirectory for software layer (HARDCODED) Initializing Lmod... -Prepending /cvmfs/pilot.eessi-hpc.org/versions/2021.12/software/x86_64/intel/haswell/modules/all to $MODULEPATH... +Prepending /cvmfs/pilot.eessi-hpc.org/versions/2023.06/software/x86_64/intel/haswell/modules/all to $MODULEPATH... Environment set up to use EESSI pilot software stack, have fun! -[EESSI pilot 2021.12] $ +[EESSI pilot 2023.06] $ ``` ### Accessing EESSI via a container From 074d7a1847ab2353e43045978dec6acd72a95cb1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:06:08 +0200 Subject: [PATCH 0024/1795] update check-result.sh to state tested in NESSI PR125 --- bot/check-result.sh | 429 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 351 insertions(+), 78 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index e28ae4e38e..04a23962e2 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -4,8 +4,8 @@ # Intended use is that it is called by a (batch) job running on a compute # node. # -# This script is part of the EESSI compatibility layer, see -# https://github.com/EESSI/compatibility-layer.git +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git # # author: Thomas Roeblitz (@trz42) # @@ -38,6 +38,20 @@ TOPDIR=$(dirname $(realpath $0)) source ${TOPDIR}/../scripts/utils.sh source ${TOPDIR}/../scripts/cfg_files.sh +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + echo_red "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +else + echo "bot/check-result.sh: showing ${JOB_CFG_FILE} from software-layer side" + cat ${JOB_CFG_FILE} + + echo "bot/check-result.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" + cfg_load ${JOB_CFG_FILE} +fi + display_help() { echo "usage: $0 [OPTIONS]" echo " OPTIONS:" @@ -88,46 +102,61 @@ job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" -GP_error='ERROR: ' -grep_out=$(grep "${GP_error}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && ERROR=1 || ERROR=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +ERROR=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_error='ERROR: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") + [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_failed='FAILED: ' -grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && FAILED=1 || FAILED=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +FAILED=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_failed='FAILED: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") + [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_req_missing=' required modules missing:' -grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && MISSING=1 || MISSING=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +MISSING=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_req_missing=' required modules missing:' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}") + [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_no_missing='No missing modules!' -grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out}) -[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +NO_MISSING=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_no_missing='No missing installations' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}") + [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi -GP_tgz_created="tar.gz created!" +TGZ=-1 TARBALL= -grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out}) -if [[ $? -eq 0 ]]; then - TGZ=1 - TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') -else - TGZ=0 +if [[ ${SLURM} -eq 1 ]]; then + GP_tgz_created="\.tar\.gz created!" + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) + if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') + else + TGZ=0 + fi + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi -# have to be careful to not add searched for pattern into slurm out file -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" -[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" [[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" @@ -147,64 +176,308 @@ if [[ ${SLURM} -eq 1 ]] && \ [[ ${TGZ} -eq 1 ]] && \ [[ ! -z ${TARBALL} ]]; then # SUCCESS - echo "[RESULT]" > ${job_result_file} - echo "summary = :grin: SUCCESS" >> ${job_result_file} - echo "details =" >> ${job_result_file} + status="SUCCESS" + summary=":grin: SUCCESS" else # FAILURE - echo "[RESULT]" > ${job_result_file} - echo "summary = :cry: FAILURE" >> ${job_result_file} - echo "details =" >> ${job_result_file} + status="FAILURE" + summary=":cry: FAILURE" fi -function succeeded() { - echo " :heavy_check_mark: ${1}" +### Example details/descriptions +# Note, final string must not contain any line breaks. Below examples include +# line breaks for the sake of readability. +#
    +# :cry: FAILURE _(click triangle for detailed information)_ +# Details:
    +#     :heavy_check_mark: job output file slurm-470503.out
    +#     :heavy_multiplication_x: found message matching ERROR:
    +#     :heavy_multiplication_x: found message matching FAILED:
    +#     :heavy_multiplication_x: found message matching required modules missing:
    +#     :heavy_check_mark: found message(s) matching No missing installations
    +#     :heavy_check_mark: found message matching tar.gz created!
    +# Artefacts: +#
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
  • +#
    +# +#
    +# :grin: SUCCESS _(click triangle for detailed information)_ +# Details:
    +#     :heavy_check_mark: job output file slurm-470503.out
    +#     :heavy_check_mark: found message matching ERROR:
    +#     :heavy_check_mark: found message matching FAILED:
    +#     :heavy_check_mark: found message matching required modules missing:
    +#     :heavy_check_mark: found message(s) matching No missing installations
    +#     :heavy_check_mark: found message matching tar.gz created!
    +# Artefacts: +#
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
  • +#
    +# +#
    +# :grin: SUCCESS _(click triangle for detailed information)_ +#
    +#
    _Details_
    +#
    +# :heavy_check_mark: job output file slurm-4682.out
    +# :heavy_check_mark: no message matching ERROR:
    +# :heavy_check_mark: no message matching FAILED:
    +# :heavy_check_mark: no message matching required modules missing:
    +# :heavy_check_mark: found message(s) matching No missing installations
    +# :heavy_check_mark: found message matching tar.gz created!
    +#
    +#
    _Artefacts_
    +#
    +#
    +# eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz +# size: 234 MiB (245366784 bytes)
    +# entries: 1234
    +# modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    +#
    +#           GCC/9.3.0.lua
    +# GCC/10.3.0.lua
    +# OpenSSL/1.1.lua +#
    +# software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ +#
    +#           GCC/9.3.0/
    +# CMake/3.20.1-GCCcore-10.3.0/
    +# OpenMPI/4.1.1-GCC-10.3.0/ +#
    +# other under _2023.04/software/linux/x86_64/intel/cascadelake/_ +#
    +#           .lmod/cache/spiderT.lua
    +# .lmod/cache/spiderT.luac_5.1
    +# .lmod/cache/timestamp +#
    +#
    +#
    +#
    +#
    +# +### + +# construct and write complete PR comment details: implements third alternative +comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT____ARTEFACTS_FMT__
    " +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" +comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " +comment_success_item_fmt=":heavy_check_mark: __ITEM__" +comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__" +comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " +comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " + +function print_br_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}
    " } -function failed() { - echo " :heavy_multiplication_x: ${1}" +function print_br_item2() { + format="${1}" + item="${2}" + item2="${3}" + format1="${format//__ITEM__/${item}}" + echo -n "${format1//__ITEM2__/${item2}}
    " } -if [[ ${SLURM} -eq 1 ]]; then - succeeded "job output file ${job_out}" >> ${job_result_file} -else - failed "no job output file matching ${GP_slurm_out}" >> ${job_result_file} -fi +function print_code_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}" +} -if [[ ${ERROR} -eq 0 ]]; then - succeeded "no message matching ${GP_error}" >> ${job_result_file} -else - failed "found message matching ${GP_error}" >> ${job_result_file} -fi +function print_dd_item() { + format="${1}" + item="${2}" + echo -n "
    ${format//__ITEM__/${item}}
    " +} -if [[ ${FAILED} -eq 0 ]]; then - succeeded "no message matching ${GP_failed}" >> ${job_result_file} -else - failed "found message matching ${GP_failed}" >> ${job_result_file} -fi +function print_list_item() { + format="${1}" + item="${2}" + echo -n "
  • ${format//__ITEM__/${item}}
  • " +} -if [[ ${MISSING} -eq 0 ]]; then - succeeded "no message matching ${GP_req_missing}" >> ${job_result_file} -else - failed "found message matching ${GP_req_missing}" >> ${job_result_file} -fi +function print_pre_item() { + format="${1}" + item="${2}" + echo -n "
    ${format//__ITEM__/${item}}
    " +} -if [[ ${NO_MISSING} -eq 1 ]]; then - succeeded "found message(s) matching ${GP_no_missing}" >> ${job_result_file} -else - failed "no message matching ${GP_no_missing}" >> ${job_result_file} -fi +function success() { + format="${comment_success_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} -if [[ ${TGZ} -eq 1 ]]; then - succeeded "found message matching ${GP_tgz_created}" >> ${job_result_file} -else - failed "no message matching ${GP_tgz_created}" >> ${job_result_file} -fi +function failure() { + format="${comment_failure_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + +function add_detail() { + actual=${1} + expected=${2} + success_msg="${3}" + failure_msg="${4}" + if [[ ${actual} -eq ${expected} ]]; then + success "${success_msg}" + else + failure "${failure_msg}" + fi +} + +echo "[RESULT]" > ${job_result_file} +echo -n "comment_description = " >> ${job_result_file} + +# construct values for placeholders in comment_template: +# - __SUMMARY_FMT__ -> variable $comment_summary +# - __DETAILS_FMT__ -> variable $comment_details +# - __ARTEFACTS_FMT__ -> variable $comment_artefacts -echo "artefacts =" >> ${job_result_file} +comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" +# first construct comment_details_list, abbreviated CoDeList +# then use it to set comment_details +CoDeList="" + +success_msg="job output file ${job_out}" +failure_msg="no job output file matching ${GP_slurm_out}" +CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_error}" +failure_msg="found message matching ${GP_error}" +CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_failed}" +failure_msg="found message matching ${GP_failed}" +CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_req_missing}" +failure_msg="found message matching ${GP_req_missing}" +CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") + +success_msg="found message(s) matching ${GP_no_missing}" +failure_msg="no message matching ${GP_no_missing}" +CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") + +success_msg="found message matching ${GP_tgz_created}" +failure_msg="no message matching ${GP_tgz_created}" +CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") + +comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" + + +# first construct comment_artefacts_list, abbreviated CoArList +# then use it to set comment_artefacts +CoArList="" + +# TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then - echo " ${TARBALL}" >> ${job_result_file} + # TODO add tarball details: size, num entries, modules, software pkgs, misc + #
    + #
    + # eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz + # size: 234 MiB (245366784 bytes)
    + # entries: 1234
    + # modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    + #
    +    #       GCC/9.3.0.lua
    + # GCC/10.3.0.lua
    + # OpenSSL/1.1.lua + #
    + # software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ + #
    +    #       GCC/9.3.0/
    + # CMake/3.20.1-GCCcore-10.3.0/
    + # OpenMPI/4.1.1-GCC-10.3.0/ + #
    + # other under _2023.04/software/linux/x86_64/intel/cascadelake/_ + #
    +    #       .lmod/cache/spiderT.lua
    + # .lmod/cache/spiderT.luac_5.1
    + # .lmod/cache/timestamp + #
    + #
    + #
    + size="$(stat --dereference --printf=%s ${TARBALL})" + size_mib=$((${size} >> 20)) + tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) + tar tf ${TARBALL} > ${tmpfile} + entries=$(cat ${tmpfile} | wc -l) + # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE + # 2023.04/software/linux/x86_64/intel/skylake_avx512 + # repo_version = 2022.11 + # software + # os_type = linux + # software_subdir = aarch64/generic + repo_version=$(cfg_get_value "repository" "repo_version") + os_type=$(cfg_get_value "architecture" "os_type") + software_subdir=$(cfg_get_value "architecture" "software_subdir") + prefix="${repo_version}/software/${os_type}/${software_subdir}" + modules_entries=$(grep "${prefix}/modules" ${tmpfile}) + software_entries=$(grep "${prefix}/software" ${tmpfile}) + lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile}) + other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") + other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) + modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) + software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) + lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@") + + artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" + CoArList="" + CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" + CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})" + CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" + CoArList="${CoArList}
    "
    +    if [[ ! -z ${modules} ]]; then
    +        while IFS= read -r mod ; do
    +            CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
    +        done <<< "${modules}"
    +    else
    +        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
    +    fi
    +    CoArList="${CoArList}
    " + CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" + CoArList="${CoArList}
    "
    +    if [[ ! -z ${software_pkgs} ]]; then
    +        while IFS= read -r sw_pkg ; do
    +            CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
    +        done <<< "${software_pkgs}"
    +    else
    +        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
    +    fi
    +    CoArList="${CoArList}
    " + CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})" + CoArList="${CoArList}
    "
    +    if [[ ! -z ${other_shortened} ]]; then
    +        while IFS= read -r other ; do
    +            CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
    +        done <<< "${other_shortened}"
    +    else
    +        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
    +    fi
    +    CoArList="${CoArList}
    " +else + CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')" fi +comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" +comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}" +comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" + +# now put all pieces together creating comment_details from comment_template +comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} +comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} + +echo "${comment_description}" >> ${job_result_file} + +# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts +# - this should make use of subsequent steps such as deploying a tarball more +# efficient +echo "status = ${status}" >> ${job_result_file} +echo "artefacts = " >> ${job_result_file} +echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} + exit 0 From 3c43a92524f4f8f6d5db167c8a58cedc273d51a0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:09:00 +0200 Subject: [PATCH 0025/1795] check for missing installations requires a few updates for check-result.sh --- check_missing_installations.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 926f475903..3627d1d0b5 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,18 +1,20 @@ #!/bin/bash # -# Script to check for missing installations in EESSI pilot software stack (version 2021.12) +# Script to check for missing installations in EESSI pilot software stack (version 2023.06) # # author: Kenneth Hoste (@boegel) +# author: Thomas Roeblitz (@trz42) # # license: GPLv2 # TOPDIR=$(dirname $(realpath $0)) -if [ -z ${EESSI_PILOT_VERSION} ]; then - echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2 +if [ $# -ne 1 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi +easystack=$1 LOCAL_TMPDIR=$(mktemp -d) @@ -21,12 +23,13 @@ source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." -ok_msg="No missing installations, party time!" -fail_msg="On no, some installations are still missing, how did that happen?!" eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used -# PR #16531: Nextflow-22.10.1.eb -${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} +exit_code=${PIPESTATUS[0]} + +ok_msg="Command 'eb --missing ...' succeeded, analysing output..." +fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" +check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" # the above assesses the installed software for each easyconfig provided in # the easystack file and then print messages such as @@ -37,8 +40,11 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim # output does not contain any line with ` required modules missing:` grep " required modules missing:" ${eb_missing_out} > /dev/null +exit_code=$? # if grep returns 1 (` required modules missing:` was NOT found), we set # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 -[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +[[ ${exit_code} -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +ok_msg="No missing installations, party time!" +fail_msg="On no, some installations are still missing, how did that happen?!" check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" From 5a0294e0bf1c553a108ab5ceba5dff07b90be8d6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 20:09:43 +0200 Subject: [PATCH 0026/1795] make sure experimental EasyBuild features can be used --- configure_easybuild | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure_easybuild b/configure_easybuild index 19b2d7454b..8e79f3a35a 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -35,3 +35,6 @@ fi export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER export EASYBUILD_MODULE_EXTENSIONS=1 + +# need to enable use of experimental features, since we're using easystack files +export EASYBUILD_EXPERIMENTAL=1 From 4110548e9280795f32a4c60ed979430217a50771 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:35:44 +0200 Subject: [PATCH 0027/1795] removed OpenSSL from filter deps --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index ad34487902..23f3920154 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -26,7 +26,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) # problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,OpenSSL,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From f709a8a72d425acfa02dd9d3e8114af99b796b1f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:43:56 +0200 Subject: [PATCH 0028/1795] let script's return value reflect SUCCESS (0) or FAILURE (1) --- bot/check-result.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/check-result.sh b/bot/check-result.sh index 04a23962e2..ec58fea557 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -480,4 +480,6 @@ echo "status = ${status}" >> ${job_result_file} echo "artefacts = " >> ${job_result_file} echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} +test "${status}" == "SUCCESS" +exit $? exit 0 From 5cfb4e1510cd8a0c93824af33ff5c4a805e3bd11 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 10:52:06 +0200 Subject: [PATCH 0029/1795] use test to check if slurm-*.out exists + fix reporting message --- bot/check-result.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index ec58fea557..6c23cae680 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -96,11 +96,14 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" GP_slurm_out="slurm-${SLURM_JOB_ID}.out" -job_out=$(ls ${job_dir} | grep "${GP_slurm_out}") -[[ $? -eq 0 ]] && SLURM=1 || SLURM=0 -# have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" -[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" +if [[ -f ${GP_slurm_out} ]]; then + SLURM=1 + [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${GP_slurm_out}"'" +else + SLURM=0 + [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${GP_slurm_out}"' NOT found" +fi ERROR=-1 if [[ ${SLURM} -eq 1 ]]; then From 26a929fd23b5a96d26681d700059f3bb0f00a80b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Jun 2023 11:04:35 +0200 Subject: [PATCH 0030/1795] switch to new dedicated bucket for version 2023.06 --- bot/bot-eessi-aws-citc.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index dca81ef2ab..8329005ae5 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -113,7 +113,7 @@ tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/ee # endpoint_url = URL_TO_S3_SERVER # bucket name -bucket_name = eessi-staging +bucket_name = eessi-staging-2023.06 # upload policy: defines what policy is used for uploading built artefacts # to an S3 bucket From 7478e7d65ab2836135e62fc6f866d84d27a7b0a7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 11:09:34 +0200 Subject: [PATCH 0031/1795] more polishing to address multiple comments/suggestions --- bot/check-result.sh | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 6c23cae680..33542d61ed 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -152,7 +152,7 @@ if [[ ${SLURM} -eq 1 ]]; then grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) if [[ $? -eq 0 ]]; then TGZ=1 - TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@') + TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') else TGZ=0 fi @@ -162,7 +162,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" -[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)" +[[ ${VERBOSE} -ne 0 ]] && echo " : ()" [[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" @@ -377,25 +377,28 @@ CoArList="" # TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then - # TODO add tarball details: size, num entries, modules, software pkgs, misc + # Example of the detailed information for a tarball. The actual result MUST be a + # single line (no '\n') or it would break the structure of the markdown table + # that holds status updates of a bot job. + # #
    #
    - # eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz + # eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz # size: 234 MiB (245366784 bytes)
    # entries: 1234
    - # modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    + # modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_
    #
         #       GCC/9.3.0.lua
    # GCC/10.3.0.lua
    # OpenSSL/1.1.lua #
    - # software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ + # software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_ #
         #       GCC/9.3.0/
    # CMake/3.20.1-GCCcore-10.3.0/
    # OpenMPI/4.1.1-GCC-10.3.0/ #
    - # other under _2023.04/software/linux/x86_64/intel/cascadelake/_ + # other under _2023.06/software/linux/x86_64/intel/cascadelake/_ #
         #       .lmod/cache/spiderT.lua
    # .lmod/cache/spiderT.luac_5.1
    @@ -409,23 +412,25 @@ if [[ ! -z ${TARBALL} ]]; then tar tf ${TARBALL} > ${tmpfile} entries=$(cat ${tmpfile} | wc -l) # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE - # 2023.04/software/linux/x86_64/intel/skylake_avx512 - # repo_version = 2022.11 - # software + # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512 + # cfg/job.cfg contains (only the attributes to be used are shown below): + # [repository] + # repo_version = 2023.06 + # [architecture] # os_type = linux - # software_subdir = aarch64/generic + # software_subdir = x86_64/intel/skylake_avx512 repo_version=$(cfg_get_value "repository" "repo_version") os_type=$(cfg_get_value "architecture" "os_type") software_subdir=$(cfg_get_value "architecture" "software_subdir") prefix="${repo_version}/software/${os_type}/${software_subdir}" + + # extract directories/entries from tarball content modules_entries=$(grep "${prefix}/modules" ${tmpfile}) software_entries=$(grep "${prefix}/software" ${tmpfile}) - lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile}) other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) - lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@") artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" CoArList="" From 827e686edfc1540d82a4bbf5d069ed9460822efc Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 11:14:01 +0200 Subject: [PATCH 0032/1795] remove tmpfile + clarify/improve exit --- bot/check-result.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 33542d61ed..2ffbcd6be6 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -488,6 +488,11 @@ echo "status = ${status}" >> ${job_result_file} echo "artefacts = " >> ${job_result_file} echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} +# remove tmpfile +if [[ -f ${tmpfile} ]]; then + rm ${tmpfile} +fi + +# exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1) test "${status}" == "SUCCESS" exit $? -exit 0 From 9296ed75e52b680bf278aba9ff2fb82ec5318765 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 11:47:19 +0200 Subject: [PATCH 0033/1795] polishing examples + comments --- bot/check-result.sh | 62 ++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 2ffbcd6be6..51e1777bab 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -20,7 +20,7 @@ # - no message ERROR # - no message FAILED # - no message ' required modules missing:' -# - one or more of 'No missing modules!' +# - one or more of 'No missing installations' # - message regarding created tarball # - FAILED (one of ... implemented as NOT SUCCESS) # - no slurm-JOBID.out file @@ -188,36 +188,12 @@ else fi ### Example details/descriptions -# Note, final string must not contain any line breaks. Below examples include -# line breaks for the sake of readability. +# Note, final string must not contain any line breaks. Below example include +# line breaks for the sake of readability. In case of FAILURE, the structure is +# very similar (incl. information about Artefacts if any was produced), however, +# under Details some lines will be marked with :heavy_multiplication_x: #
    -# :cry: FAILURE _(click triangle for detailed information)_ -# Details:
    -#     :heavy_check_mark: job output file slurm-470503.out
    -#     :heavy_multiplication_x: found message matching ERROR:
    -#     :heavy_multiplication_x: found message matching FAILED:
    -#     :heavy_multiplication_x: found message matching required modules missing:
    -#     :heavy_check_mark: found message(s) matching No missing installations
    -#     :heavy_check_mark: found message matching tar.gz created!
    -# Artefacts: -#
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
  • -#
    -# -#
    -# :grin: SUCCESS _(click triangle for detailed information)_ -# Details:
    -#     :heavy_check_mark: job output file slurm-470503.out
    -#     :heavy_check_mark: found message matching ERROR:
    -#     :heavy_check_mark: found message matching FAILED:
    -#     :heavy_check_mark: found message matching required modules missing:
    -#     :heavy_check_mark: found message(s) matching No missing installations
    -#     :heavy_check_mark: found message matching tar.gz created!
    -# Artefacts: -#
  • eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
  • -#
    -# -#
    -# :grin: SUCCESS _(click triangle for detailed information)_ +# :grin: SUCCESS _(click triangle for details)_ #
    #
    _Details_
    #
    @@ -231,22 +207,22 @@ fi #
    _Artefacts_
    #
    #
    -# eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz +# eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz # size: 234 MiB (245366784 bytes)
    # entries: 1234
    -# modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
    +# modules under _2023.06/software/linux/x86_64/generic/modules/all/_
    #
     #           GCC/9.3.0.lua
    # GCC/10.3.0.lua
    # OpenSSL/1.1.lua #
    -# software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_ +# software under _2023.06/software/linux/x86_64/generic/software/_ #
     #           GCC/9.3.0/
    # CMake/3.20.1-GCCcore-10.3.0/
    # OpenMPI/4.1.1-GCC-10.3.0/ #
    -# other under _2023.04/software/linux/x86_64/intel/cascadelake/_ +# other under _2023.06/software/linux/x86_64/generic/_ #
     #           .lmod/cache/spiderT.lua
    # .lmod/cache/spiderT.luac_5.1
    @@ -257,6 +233,24 @@ fi #
    #
    # +#
    +# :cry: FAILURE _(click triangle for details)_ +#
    +#
    _Details_
    +#
    +# :heavy_check_mark: job output file slurm-4682.out
    +# :heavy_multiplication_x: no message matching ERROR:
    +# :heavy_check_mark: no message matching FAILED:
    +# :heavy_multiplication_x: no message matching required modules missing:
    +# :heavy_check_mark: found message(s) matching No missing installations
    +# :heavy_check_mark: found message matching tar.gz created!
    +#
    +#
    _Artefacts_
    +#
    +# No artefacts were created or found. +#
    +#
    +#
    ### # construct and write complete PR comment details: implements third alternative From 78a9dc8e9e67dc998c39e6f74f06b72117e0f22d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Jun 2023 13:50:37 +0200 Subject: [PATCH 0034/1795] make sure that $job_out is defined in bot/check-result.sh --- bot/check-result.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 51e1777bab..182b8555a8 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -95,14 +95,14 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" -GP_slurm_out="slurm-${SLURM_JOB_ID}.out" -[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'" -if [[ -f ${GP_slurm_out} ]]; then +job_out="slurm-${SLURM_JOB_ID}.out" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" +if [[ -f ${job_out} ]]; then SLURM=1 - [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${GP_slurm_out}"'" + [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" else SLURM=0 - [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${GP_slurm_out}"' NOT found" + [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi ERROR=-1 @@ -339,7 +339,7 @@ comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" CoDeList="" success_msg="job output file ${job_out}" -failure_msg="no job output file matching ${GP_slurm_out}" +failure_msg="no job output file ${job_out}" CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_error}" From 4cc8f4135fd030f942950af70559a40aa43143cd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Jun 2023 20:53:32 +0200 Subject: [PATCH 0035/1795] don't run 'eb --missing' in install script, since it confuses the bot's check-result script because of 'required modules missing' messages in job output --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index f2d50dfe91..287688a61f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -151,7 +151,7 @@ for eb_version in '4.7.2'; do if [ -f ${es} ]; then echo_green "Feeding easystack file ${es} to EasyBuild..." - ${EB} --easystack ${TOPDIR}/${es} --missing && ${EB} --easystack ${TOPDIR}/${es} --robot + ${EB} --easystack ${TOPDIR}/${es} --robot $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es} else From 4ecbbbbd09b8f8da8f66d4b99ef08618f4208d91 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Jun 2023 21:05:45 +0200 Subject: [PATCH 0036/1795] add required job_result_unknown_fmt bot config setting --- bot/bot-eessi-aws-citc.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 8329005ae5..de25fc9445 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -230,3 +230,4 @@ multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching ` job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:{details}
    Artefacts:{artefacts}
    job_result_details_item_fmt =
        {item} job_result_artefacts_item_fmt =
  • {item}
  • +job_result_unknown_fmt =
    :shrug: UNKNOWN _(click triangle for detailed information)_
    • Job results file `{filename}` does not exist in job directory or reading it failed.
    • No artefacts were found/reported.
    From 9226b8169b4ffa51587e6de6631ba319aa6310b1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Jun 2023 21:30:55 +0200 Subject: [PATCH 0037/1795] tweak slurm parameters to use 16 cores for jobs --- bot/bot-eessi-aws-citc.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index de25fc9445..a82d4010d3 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -83,7 +83,8 @@ local_tmp = /tmp/$USER/EESSI # NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs # to be initialised as if it is for a login shell. # note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed -slurm_params = --hold --time=24:0:0 +# note: 16 cores which corresponds to *.4xlarge node types, see also arch_target_map +slurm_params = --hold --time=24:0:0 --nodes=1 --ntasks-per-node=16 # full path to the job submission command submit_command = /usr/bin/sbatch From 7a4336348066319ada0b8f17cd6aa365478b5850 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 22:00:29 +0200 Subject: [PATCH 0038/1795] three obsolete settings removed from bot cfg --- bot/bot-eessi-aws-citc.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index a82d4010d3..456ab8a5a3 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -228,7 +228,4 @@ missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:{details}
    Artefacts:{artefacts}
    -job_result_details_item_fmt =
        {item} -job_result_artefacts_item_fmt =
  • {item}
  • job_result_unknown_fmt =
    :shrug: UNKNOWN _(click triangle for detailed information)_
    • Job results file `{filename}` does not exist in job directory or reading it failed.
    • No artefacts were found/reported.
    From 1013f88487006d0456c51bd264e5d4030e6d6358 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 21 Jun 2023 14:37:22 +0200 Subject: [PATCH 0039/1795] only test load_easybuild_module.sh script with EasyBuild v4.5.2 or newer --- .github/workflows/tests_scripts.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 81a3633b72..6ef9d57975 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -40,7 +40,9 @@ jobs: # bind current directory into container as /software-layer export SINGULARITY_BIND="${PWD}:/software-layer" - for EB_VERSION in '4.5.0' '4.5.1' '4.7.2'; do + # can't test with EasyBuild versions older than v4.5.2 when using EESSI pilot 2023.06, + # since Python in compat layer is Python 3.11.x + for EB_VERSION in '4.5.2' '4.6.0' '4.7.2'; do # Create script that uses load_easybuild_module.sh which we can run in compat layer environment # note: Be careful with single vs double quotes below! # ${EB_VERSION} should be expanded, so use double quotes; From 5d7e367e32af6e8d6765b2c3a88451901043237f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 22 Jun 2023 17:41:02 +0200 Subject: [PATCH 0040/1795] use correct aarch64 CPU targets for tests that rely on EESSI pilot 2023.06 --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 65415f87fc..3d18e56816 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -13,8 +13,8 @@ jobs: - 2023.06 EESSI_SOFTWARE_SUBDIR: - aarch64/generic - - aarch64/graviton2 - - aarch64/graviton3 + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 - x86_64/amd/zen2 - x86_64/amd/zen3 - x86_64/intel/haswell From 745151f0f74d9f11824e36f5cbe71f3f13a07a4a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 22 Jun 2023 18:24:27 +0200 Subject: [PATCH 0041/1795] switch to colored emojis in bot/check-result.sh --- bot/check-result.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/bot/check-result.sh b/bot/check-result.sh index 182b8555a8..22ad50a0b7 100755 --- a/bot/check-result.sh +++ b/bot/check-result.sh @@ -191,18 +191,18 @@ fi # Note, final string must not contain any line breaks. Below example include # line breaks for the sake of readability. In case of FAILURE, the structure is # very similar (incl. information about Artefacts if any was produced), however, -# under Details some lines will be marked with :heavy_multiplication_x: +# under Details some lines will be marked with :x: #
    # :grin: SUCCESS _(click triangle for details)_ #
    #
    _Details_
    #
    -# :heavy_check_mark: job output file slurm-4682.out
    -# :heavy_check_mark: no message matching ERROR:
    -# :heavy_check_mark: no message matching FAILED:
    -# :heavy_check_mark: no message matching required modules missing:
    -# :heavy_check_mark: found message(s) matching No missing installations
    -# :heavy_check_mark: found message matching tar.gz created!
    +# :white_check_mark: job output file slurm-4682.out
    +# :white_check_mark: no message matching ERROR:
    +# :white_check_mark: no message matching FAILED:
    +# :white_check_mark: no message matching required modules missing:
    +# :white_check_mark: found message(s) matching No missing installations
    +# :white_check_mark: found message matching tar.gz created!
    #
    #
    _Artefacts_
    #
    @@ -238,12 +238,12 @@ fi #
    #
    _Details_
    #
    -# :heavy_check_mark: job output file slurm-4682.out
    -# :heavy_multiplication_x: no message matching ERROR:
    -# :heavy_check_mark: no message matching FAILED:
    -# :heavy_multiplication_x: no message matching required modules missing:
    -# :heavy_check_mark: found message(s) matching No missing installations
    -# :heavy_check_mark: found message matching tar.gz created!
    +# :white_check_mark: job output file slurm-4682.out
    +# :x: no message matching ERROR:
    +# :white_check_mark: no message matching FAILED:
    +# :x: no message matching required modules missing:
    +# :white_check_mark: found message(s) matching No missing installations
    +# :white_check_mark: found message matching tar.gz created!
    #
    #
    _Artefacts_
    #
    @@ -257,8 +257,8 @@ fi comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT____ARTEFACTS_FMT__
    " comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " -comment_success_item_fmt=":heavy_check_mark: __ITEM__" -comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__" +comment_success_item_fmt=":white_check_mark: __ITEM__" +comment_failure_item_fmt=":x: __ITEM__" comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " From e93d7dfcacf4cd57a5231aa7b435e539299ba3c5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 22 Jun 2023 18:25:32 +0200 Subject: [PATCH 0042/1795] add GCC/11.2.0 to EESSI pilot 2023.06 --- EESSI-pilot-install-software.sh | 4 +--- eessi-2023.06-eb-4.7.2-2021b.yml | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 eessi-2023.06-eb-4.7.2-2021b.yml diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 287688a61f..12878b7e6c 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -144,9 +144,7 @@ for eb_version in '4.7.2'; do echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." - for gen in '2021a'; do - - es="eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-${gen}.yml" + for es in $(ls eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-*.yml); do if [ -f ${es} ]; then echo_green "Feeding easystack file ${es} to EasyBuild..." diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml new file mode 100644 index 0000000000..beffbdca0d --- /dev/null +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GCC-11.2.0 From e7b12c4e1db99208cffd94352bcac3de9106326a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 22 Jun 2023 22:40:01 +0200 Subject: [PATCH 0043/1795] add easystack file for 2021b generation to test workflow --- .github/workflows/test_eessi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 3d18e56816..8e766c3a5c 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -22,6 +22,7 @@ jobs: - x86_64/generic EASYSTACK_FILE: - eessi-2023.06-eb-4.7.2-2021a.yml + - eessi-2023.06-eb-4.7.2-2021b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From d639d27c31b7e581dbdfb285a6cec95a3399aaed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 22 Jun 2023 23:23:44 +0200 Subject: [PATCH 0044/1795] test load_easybuild_module.sh with a single EasyBuild version --- .github/workflows/tests_scripts.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 65224ebb66..74e2ebcffe 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -41,8 +41,9 @@ jobs: export SINGULARITY_BIND="${PWD}:/software-layer" # can't test with EasyBuild versions older than v4.5.2 when using EESSI pilot 2023.06, - # since Python in compat layer is Python 3.11.x - for EB_VERSION in '4.5.2' '4.6.0' '4.7.2'; do + # since Python in compat layer is Python 3.11.x; + # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version + for EB_VERSION in '4.6.0'; do # Create script that uses load_easybuild_module.sh which we can run in compat layer environment # note: Be careful with single vs double quotes below! # ${EB_VERSION} should be expanded, so use double quotes; From cbd2739667cfd23d30cefa9b86963cc6facb280a Mon Sep 17 00:00:00 2001 From: laraPPr Date: Fri, 23 Jun 2023 10:50:49 +0200 Subject: [PATCH 0045/1795] adding foss/2021b with GCC/11.2.0 to EESSI pilot 2023.06 --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index beffbdca0d..4f5c56dfe5 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -1,2 +1,3 @@ easyconfigs: - GCC-11.2.0 + - foss-2021b From 0c32e5dd6289541098c428addc46e0c6b9915818 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 23 Jun 2023 12:03:45 +0200 Subject: [PATCH 0046/1795] grant more build/deploy permissions --- bot/bot-eessi-aws-citc.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 456ab8a5a3..71a674a0db 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -28,7 +28,7 @@ private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws.2 # which GH accounts have the permission to send commands to the bot # if value is left/empty everyone can send commands # value can be a space delimited list of GH accounts -command_permission = boegel trz42 bedroge +command_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 # format of the response when processing bot commands command_response_fmt = @@ -93,7 +93,7 @@ submit_command = /usr/bin/sbatch # the label 'bot:build' (apparently this cannot be restricted on GitHub) # if value is left/empty everyone can trigger the build # value can be a space delimited list of GH accounts -build_permission = boegel trz42 bedroge +build_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 # template for comment when user who set a label has no permission to trigger build jobs no_build_permission_comment = Label `bot:build` has been set by user `{build_labeler}`, but only users `{build_permission_users}` have permission to trigger the action @@ -129,7 +129,7 @@ upload_policy = once # the label 'bot:deploy' (apparently this cannot be restricted on GitHub) # if value is left/empty everyone can trigger the deployment # value can be a space delimited list of GH accounts -deploy_permission = boegel trz42 bedroge +deploy_permission = boegel trz42 bedroge ocaisa casparvl # template for comment when user who set a label has no permission to trigger deploying tarballs no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_labeler}`, but only users `{deploy_permission_users}` have permission to trigger the action From 223c71cc4c144d2a6cb8846d8c3b51dba41c1c00 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 23 Jun 2023 12:22:39 +0200 Subject: [PATCH 0047/1795] Add CMake-3.21.1 and CMake-3.22.1 Also removed foss/2021b --- eessi-2023.06-eb-4.7.2-2021b.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 4f5c56dfe5..ba2a300a52 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -1,3 +1,8 @@ easyconfigs: - GCC-11.2.0 - - foss-2021b + - CMake-3.21.1-GCCcore-11.2.0.eb: + options: + include-easyblocks-from-pr: 2248 + - CMake-3.22.1-GCCcore-11.2.0.eb: + options: + include-easyblocks-from-pr: 2248 From 5b2987c61fcc294a0ffa6487aefb7b599f0f578a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 23 Jun 2023 14:23:47 +0200 Subject: [PATCH 0048/1795] Added libpng --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 9b1312ba9a..eb79be6257 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -4,3 +4,4 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - foss-2021a + - libpng-1.6.37-GCCcore-10.3.0.eb: From 8adb0135cf63f95da8395ff901afacdf2d24f08c Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Fri, 23 Jun 2023 14:52:49 +0200 Subject: [PATCH 0049/1795] Adding libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb to eessi pilot 2023.06. --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 9b1312ba9a..f0ba6aa0b0 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -4,3 +4,4 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - foss-2021a + - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb From 9a36d19918533549f4f9fdf844ae01e1407bfffd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 23 Jun 2023 17:14:01 +0200 Subject: [PATCH 0050/1795] change upload_policy to 'latest' --- bot/bot-eessi-aws-citc.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 71a674a0db..b4381f7e2c 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -123,7 +123,7 @@ bucket_name = eessi-staging-2023.06 # only upload the latest built artefact # 'once' : only once upload any built artefact for the build target # 'none' : do not upload any built artefacts -upload_policy = once +upload_policy = latest # which GH account has the permission to trigger the deployment (by setting # the label 'bot:deploy' (apparently this cannot be restricted on GitHub) From 32ed420fc70750feb71d7054fcd6729b034905c8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 26 Jun 2023 09:19:33 +0200 Subject: [PATCH 0051/1795] {2023.06}[2022a] GCC 11.3.0 --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.7.2-2022a.yml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 eessi-2023.06-eb-4.7.2-2022a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 8e766c3a5c..17ccc42747 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -23,6 +23,7 @@ jobs: EASYSTACK_FILE: - eessi-2023.06-eb-4.7.2-2021a.yml - eessi-2023.06-eb-4.7.2-2021b.yml + - eessi-2023.06-eb-4.7.2-2022a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml new file mode 100644 index 0000000000..b8f0cc17bc --- /dev/null +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GCC-11.3.0 From 3ed2b91597a8777c2922d428ad9313619768be7b Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 26 Jun 2023 10:38:51 +0200 Subject: [PATCH 0052/1795] {2023.06}[2021b] zlib v1.2.11 --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index ba2a300a52..9f06908f0d 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -6,3 +6,4 @@ easyconfigs: - CMake-3.22.1-GCCcore-11.2.0.eb: options: include-easyblocks-from-pr: 2248 + - zlib-1.2.11-GCCcore-11.2.0 From 9417bfb1b71c1114ce4825523cb17399d2e32d64 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 26 Jun 2023 08:50:48 +0000 Subject: [PATCH 0053/1795] adding QuantumESPRESSO/6.7 with GCC/10.3.0 to EESSI pilot 2023.06 --- eessi-2023.06-eb-4.7.2-2021a.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 83dff0ad4e..461c22b7ed 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -5,4 +5,5 @@ easyconfigs: include-easyblocks-from-pr: 2248 - foss-2021a - libpng-1.6.37-GCCcore-10.3.0.eb: - - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb \ No newline at end of file + - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb + - QuantumESPRESSO-6.7-foss-2021a.eb From 01a813c72fd88e9df1a62348ad5c821ce8ae1e3a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 26 Jun 2023 16:08:16 +0200 Subject: [PATCH 0054/1795] {2023.06}[2022b] GCC 12.2.0 --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.7.2-2022b.yml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 eessi-2023.06-eb-4.7.2-2022b.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 8e766c3a5c..32d594fa05 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -23,6 +23,7 @@ jobs: EASYSTACK_FILE: - eessi-2023.06-eb-4.7.2-2021a.yml - eessi-2023.06-eb-4.7.2-2021b.yml + - eessi-2023.06-eb-4.7.2-2022b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml new file mode 100644 index 0000000000..15c99b5e0b --- /dev/null +++ b/eessi-2023.06-eb-4.7.2-2022b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GCC-12.2.0 From 82fe73e19995a5bf70a55d8d5562c07f462fa1cb Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 27 Jun 2023 06:46:37 +0000 Subject: [PATCH 0055/1795] {2023.06}[foss/2021a] Qt5 V5.15.2 --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 461c22b7ed..9468f268c7 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -7,3 +7,4 @@ easyconfigs: - libpng-1.6.37-GCCcore-10.3.0.eb: - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb - QuantumESPRESSO-6.7-foss-2021a.eb + - Qt5-5.15.2-GCCcore-10.3.0.eb From f2b4ede6270bc3481a89086e1c4fb869a9438d18 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 27 Jun 2023 10:10:16 +0000 Subject: [PATCH 0056/1795] added --from-pr option --- eessi-2023.06-eb-4.7.2-2021a.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 9468f268c7..3f8dd5954e 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -7,4 +7,6 @@ easyconfigs: - libpng-1.6.37-GCCcore-10.3.0.eb: - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb - QuantumESPRESSO-6.7-foss-2021a.eb - - Qt5-5.15.2-GCCcore-10.3.0.eb + - Qt5-5.15.2-GCCcore-10.3.0.eb: + options: + -from-pr: 18087 From 237a16f6cefeaeffda24c5cc0b161eab2164572e Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 27 Jun 2023 15:01:20 +0000 Subject: [PATCH 0057/1795] added from-pr option --- eessi-2023.06-eb-4.7.2-2021a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 3f8dd5954e..79fe7a0716 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -9,4 +9,4 @@ easyconfigs: - QuantumESPRESSO-6.7-foss-2021a.eb - Qt5-5.15.2-GCCcore-10.3.0.eb: options: - -from-pr: 18087 + from-pr: 18087 From 77afe94f055a774a70ba50b7eb9e5fccf48f8ac9 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 28 Jun 2023 09:37:00 +0200 Subject: [PATCH 0058/1795] {2023.06}[2021b] OpenMPI v4.1.1 --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 9f06908f0d..5e13dd68e2 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -7,3 +7,4 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - zlib-1.2.11-GCCcore-11.2.0 + - OpenMI-4.1.1-GCC-11.2.0 From b5b780c4ff3d3b68b6c1aeaee8d16f8a0ff55f13 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 28 Jun 2023 09:54:33 +0200 Subject: [PATCH 0059/1795] {2023.06}[foss/2021a] GROMACS v2021.3 --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 461c22b7ed..223bfad6c4 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -7,3 +7,4 @@ easyconfigs: - libpng-1.6.37-GCCcore-10.3.0.eb: - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb - QuantumESPRESSO-6.7-foss-2021a.eb + - GROMACS-2021.3-foss-2021a.eb From a04bbf14346ee4165f46df22f32d4a6dd2cb1563 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:07:43 +0200 Subject: [PATCH 0060/1795] change OpenMPI line --- eessi-2023.06-eb-4.7.2-2021b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 5e13dd68e2..c379896d25 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -7,4 +7,4 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - zlib-1.2.11-GCCcore-11.2.0 - - OpenMI-4.1.1-GCC-11.2.0 + - OpenMPI-4.1.1-GCC-11.2.0 From 9b840928923e4a9ff18afbbf9bf6d032545cf9ee Mon Sep 17 00:00:00 2001 From: TopRichard Date: Wed, 28 Jun 2023 10:54:36 +0000 Subject: [PATCH 0061/1795] {2023.06}[foss/2021b] QuantumESPRESSO V6.8 --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 9f06908f0d..ec83ab018c 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -7,3 +7,4 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - zlib-1.2.11-GCCcore-11.2.0 + - QuantumESPRESSO-6.8-foss-2021b.eb From 7f27f248bdc4df693e07b9ec71aea878cf8ce748 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:08:53 +0200 Subject: [PATCH 0062/1795] Update eessi-2023.06-eb-4.7.2-2021b.yml --- eessi-2023.06-eb-4.7.2-2021b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 5e13dd68e2..d6f22b87d0 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -7,4 +7,4 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - zlib-1.2.11-GCCcore-11.2.0 - - OpenMI-4.1.1-GCC-11.2.0 + From 71122ecfbef0a5295846879eff382b3ea080a15a Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:09:27 +0200 Subject: [PATCH 0063/1795] Update eessi-2023.06-eb-4.7.2-2021b.yml --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index d6f22b87d0..9f06908f0d 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -7,4 +7,3 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - zlib-1.2.11-GCCcore-11.2.0 - From 6fc4c8ee9676991b02110e81d24f85dbdb992f0e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 30 Jun 2023 10:04:11 +0200 Subject: [PATCH 0064/1795] {2023.06}[2021b] FlexiBLAS v3.0.4 (+ OpenBLAS v0.3.18) --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index c379896d25..5857bf8977 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -8,3 +8,4 @@ easyconfigs: include-easyblocks-from-pr: 2248 - zlib-1.2.11-GCCcore-11.2.0 - OpenMPI-4.1.1-GCC-11.2.0 + - FlexiBLAS-3.0.4-GCC-11.2.0.eb From 7b9f662504419327bb02a568f30c4ca2b010492a Mon Sep 17 00:00:00 2001 From: TopRichard Date: Fri, 30 Jun 2023 09:57:06 +0000 Subject: [PATCH 0065/1795] adding the option:ignore-test-failure --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 5db011b245..de0572ed77 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -11,3 +11,4 @@ easyconfigs: - Qt5-5.15.2-GCCcore-10.3.0.eb: options: from-pr: 18087 + ignore-test-failure From 5436f8b507a279cdce598c57cdd7b69b67bb3a27 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Fri, 30 Jun 2023 10:14:53 +0000 Subject: [PATCH 0066/1795] removed the option:ignore-test-failure --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index de0572ed77..5db011b245 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -11,4 +11,3 @@ easyconfigs: - Qt5-5.15.2-GCCcore-10.3.0.eb: options: from-pr: 18087 - ignore-test-failure From c62d164321a45dae9f88129800966b9363260454 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 3 Jul 2023 12:51:27 +0200 Subject: [PATCH 0067/1795] {2023.06}[2021b] gompi --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index c379896d25..6ff917f109 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -8,3 +8,4 @@ easyconfigs: include-easyblocks-from-pr: 2248 - zlib-1.2.11-GCCcore-11.2.0 - OpenMPI-4.1.1-GCC-11.2.0 + - gompi-2021b From 0e6dc4d7c1ab6ebda30c3c1d6f04b43e52f375c3 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Mon, 3 Jul 2023 13:37:17 +0200 Subject: [PATCH 0068/1795] Adding ReFrame for ingestion via the bot. Created a new yml file via which ReFrame should be added since ReFrame depends on the system Python. --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.7.2.yml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 eessi-2023.06-eb-4.7.2.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index a0401d6844..788f9dd7e3 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -25,6 +25,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2021b.yml - eessi-2023.06-eb-4.7.2-2022a.yml - eessi-2023.06-eb-4.7.2-2022b.yml + - eessi-2023.06-eb-4.7.2.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/eessi-2023.06-eb-4.7.2.yml b/eessi-2023.06-eb-4.7.2.yml new file mode 100644 index 0000000000..245271cebf --- /dev/null +++ b/eessi-2023.06-eb-4.7.2.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ReFrame-4.2.0 From ee60b27d0c53d1f0475dbd2cfcb24f63fe8b2fa8 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Tue, 4 Jul 2023 11:26:23 +0200 Subject: [PATCH 0069/1795] Amending file name so that it gets picked up by the bot. --- .github/workflows/test_eessi.yml | 2 +- eessi-2023.06-eb-4.7.2.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 eessi-2023.06-eb-4.7.2.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 788f9dd7e3..e93d437cea 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -25,7 +25,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2021b.yml - eessi-2023.06-eb-4.7.2-2022a.yml - eessi-2023.06-eb-4.7.2-2022b.yml - - eessi-2023.06-eb-4.7.2.yml + - eessi-2023.06-eb-4.7.2-system.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/eessi-2023.06-eb-4.7.2.yml b/eessi-2023.06-eb-4.7.2.yml deleted file mode 100644 index 245271cebf..0000000000 --- a/eessi-2023.06-eb-4.7.2.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - ReFrame-4.2.0 From 78a78214fbba67173b069269fb69ef0d25e0b966 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Tue, 4 Jul 2023 11:33:05 +0200 Subject: [PATCH 0070/1795] Forgot to add this file. The tar balls will be empty if this file is not present. --- eessi-2023.06-eb-4.7.2-system.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 eessi-2023.06-eb-4.7.2-system.yml diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml new file mode 100644 index 0000000000..245271cebf --- /dev/null +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ReFrame-4.2.0 From 2a32626f5b3a38e3b8a0beebbc65e13761830154 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 4 Jul 2023 12:40:47 +0000 Subject: [PATCH 0071/1795] added a hook to set check_qtwebengine to False --- eb_hooks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 787e990e75..55c1e10a5a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -59,9 +59,16 @@ def parse_hook(ec, *args, **kwargs): # determine path to Prefix installation in compat layer via $EPREFIX eprefix = get_eessi_envvar('EPREFIX') +#Disable Qt5/5.15.2 Sanity-check + Qt5_Sanitycheck_disable(ec, eprefix) + if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) +def Qt5_Sanitycheck_disable(ec, eprefix): + if ec.name == 'Qt5'and ec.version == '5.15.2': + ec['check_qtwebengine'] = False + print_msg(NOTE:the value of check_qtwebengine has been set to %s , ec['check_qtwebengine']) def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" From a86d0e6f8c84dd82a22c8f8794ed3fbc945ee3c7 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 4 Jul 2023 12:44:55 +0000 Subject: [PATCH 0072/1795] renamed the function in the hook to Qt5_check_qtwebengine_disable --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 55c1e10a5a..a0d8090d23 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -60,12 +60,12 @@ def parse_hook(ec, *args, **kwargs): eprefix = get_eessi_envvar('EPREFIX') #Disable Qt5/5.15.2 Sanity-check - Qt5_Sanitycheck_disable(ec, eprefix) + Qt5_check_qtwebengine_disable(ec, eprefix) if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) -def Qt5_Sanitycheck_disable(ec, eprefix): +def Qt5_check_qtwebengine_disable(ec, eprefix): if ec.name == 'Qt5'and ec.version == '5.15.2': ec['check_qtwebengine'] = False print_msg(NOTE:the value of check_qtwebengine has been set to %s , ec['check_qtwebengine']) From c46d016dde5a4194821b25c1ed954d13663327ce Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 4 Jul 2023 12:51:26 +0000 Subject: [PATCH 0073/1795] renamed the function in the hook to Qt5_check_qtwebengine_disable 1 --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index a0d8090d23..6592427801 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -59,7 +59,7 @@ def parse_hook(ec, *args, **kwargs): # determine path to Prefix installation in compat layer via $EPREFIX eprefix = get_eessi_envvar('EPREFIX') -#Disable Qt5/5.15.2 Sanity-check +#Disable Qt5/5.15.2 qtwebengine in Sanity-check Qt5_check_qtwebengine_disable(ec, eprefix) if ec.name in PARSE_HOOKS: From 9262538f1420c51df74ae94542a23f130d0343bd Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 4 Jul 2023 15:48:05 +0000 Subject: [PATCH 0074/1795] editted the hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 6592427801..8efd38b05d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -68,7 +68,7 @@ def parse_hook(ec, *args, **kwargs): def Qt5_check_qtwebengine_disable(ec, eprefix): if ec.name == 'Qt5'and ec.version == '5.15.2': ec['check_qtwebengine'] = False - print_msg(NOTE:the value of check_qtwebengine has been set to %s , ec['check_qtwebengine']) + print_msg(The value of check_qtwebengine has been set to %s , ec['check_qtwebengine']) def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" From c1c8e8e0ae4030747ab8752f6d94e40670f62f24 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 4 Jul 2023 15:51:54 +0000 Subject: [PATCH 0075/1795] editted the hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8efd38b05d..aa1b64dc4b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -68,7 +68,7 @@ def parse_hook(ec, *args, **kwargs): def Qt5_check_qtwebengine_disable(ec, eprefix): if ec.name == 'Qt5'and ec.version == '5.15.2': ec['check_qtwebengine'] = False - print_msg(The value of check_qtwebengine has been set to %s , ec['check_qtwebengine']) + print_msg("The value of check_qtwebengine has been set to %s", ec['check_qtwebengine']) def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" From 22245d4b0aec2b21b87c1ea9fc37d2da31fa0af8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jul 2023 08:44:45 +0200 Subject: [PATCH 0076/1795] refactor parse hook for Qt5 to disable check_qtwebengine --- eb_hooks.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index aa1b64dc4b..03ad339fa9 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -59,16 +59,9 @@ def parse_hook(ec, *args, **kwargs): # determine path to Prefix installation in compat layer via $EPREFIX eprefix = get_eessi_envvar('EPREFIX') -#Disable Qt5/5.15.2 qtwebengine in Sanity-check - Qt5_check_qtwebengine_disable(ec, eprefix) - if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) -def Qt5_check_qtwebengine_disable(ec, eprefix): - if ec.name == 'Qt5'and ec.version == '5.15.2': - ec['check_qtwebengine'] = False - print_msg("The value of check_qtwebengine has been set to %s", ec['check_qtwebengine']) def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" @@ -166,6 +159,17 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") +def parse_hook_Qt5_check_qtwebengine_disable(ec, eprefix): + """ + Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. + """ + if ec.name == 'Qt5': + # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, + # see also https://github.com/easybuilders/easybuild-framework/pull/4290 + ec['check_qtwebengine'] = False + print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") + + def parse_hook_ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': @@ -239,6 +243,7 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, + 'Qt5': parse_hook_Qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, } From 913ff5330188f002f2d00746dd6441a2838dd8bf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jul 2023 09:03:37 +0200 Subject: [PATCH 0077/1795] raise error if Qt5-specific hook is triggered for something else than Qt5 --- eb_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 03ad339fa9..ec8e2a2c55 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -168,6 +168,8 @@ def parse_hook_Qt5_check_qtwebengine_disable(ec, eprefix): # see also https://github.com/easybuilders/easybuild-framework/pull/4290 ec['check_qtwebengine'] = False print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") + else: + raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") def parse_hook_ucx_eprefix(ec, eprefix): From a2fda19a6bca18fdf204245fc3984777744a3b16 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jul 2023 09:10:34 +0200 Subject: [PATCH 0078/1795] add hook to relax max_failing_lapack_tests_num_errors for OpenBLAS to 300 on aarch64 --- eb_hooks.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 787e990e75..25cc7bc02a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -159,6 +159,23 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") +def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): + """Relax number of failing numerical LAPACK tests on Arm 64-bit systems.""" + if ec.name == 'OpenBLAS': + # relax maximum number of failed numerical LAPACK tests on Arm 64-bit systems, + # since the default setting of 150 that works well on x86_64 is a bit too strict + cfg_option = 'max_failing_lapack_tests_num_errors' + if get_cpu_architecture() == AARCH64: + orig_value = ec[cfg_option] + ec[cfg_option] = 300 + print_msg("Maximum number of failing LAPACK tests with numerical errors for %s relaxed to %s (was %s)", + ec.name, ec[cfg_option], orig_value) + else: + print_msg("Not changing option %s for %s on non-AARCH64", cfg_option, ec.name) + else: + raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") + + def parse_hook_ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': @@ -232,6 +249,7 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, + 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'UCX': parse_hook_ucx_eprefix, } From ca888ade6e62bb9faed66ae53db92b4630b3c296 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Wed, 5 Jul 2023 07:48:42 +0000 Subject: [PATCH 0079/1795] added some explanation in the eessi-2021a yml file --- eessi-2023.06-eb-4.7.2-2021a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 5db011b245..1c16b1ce72 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -9,5 +9,7 @@ easyconfigs: - QuantumESPRESSO-6.7-foss-2021a.eb - GROMACS-2021.3-foss-2021a.eb - Qt5-5.15.2-GCCcore-10.3.0.eb: +# add missing patch files for Qt5 5.15.2 to fix build problems with glibc 2.34, +# see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 options: from-pr: 18087 From a624cef2055f9f6ff17c0a19bcc24c775bf16f7c Mon Sep 17 00:00:00 2001 From: TopRichard Date: Wed, 5 Jul 2023 08:29:42 +0000 Subject: [PATCH 0080/1795] matching indent --- eessi-2023.06-eb-4.7.2-2021a.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 1c16b1ce72..c0fa353475 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -9,7 +9,7 @@ easyconfigs: - QuantumESPRESSO-6.7-foss-2021a.eb - GROMACS-2021.3-foss-2021a.eb - Qt5-5.15.2-GCCcore-10.3.0.eb: -# add missing patch files for Qt5 5.15.2 to fix build problems with glibc 2.34, -# see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 + # add missing patch files for Qt5 5.15.2 to fix build problems with glibc 2.34, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 options: from-pr: 18087 From ca7e3681d163bb3123c5eae0cea003fecf76fd1c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 5 Jul 2023 15:22:31 +0200 Subject: [PATCH 0081/1795] retain alphabetical order in parse hooks --- eb_hooks.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index ad46dc166a..191e24678b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -159,19 +159,6 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") -def parse_hook_Qt5_check_qtwebengine_disable(ec, eprefix): - """ - Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. - """ - if ec.name == 'Qt5': - # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, - # see also https://github.com/easybuilders/easybuild-framework/pull/4290 - ec['check_qtwebengine'] = False - print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") - else: - raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") - - def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): """Relax number of failing numerical LAPACK tests on Arm 64-bit systems.""" if ec.name == 'OpenBLAS': @@ -189,6 +176,19 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") +def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): + """ + Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. + """ + if ec.name == 'Qt5': + # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, + # see also https://github.com/easybuilders/easybuild-framework/pull/4290 + ec['check_qtwebengine'] = False + print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") + else: + raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") + + def parse_hook_ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': @@ -263,7 +263,7 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, - 'Qt5': parse_hook_Qt5_check_qtwebengine_disable, + 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, } From ab7e4d48e4745f7322654899b8d9e1348dbe514c Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 7 Jul 2023 09:17:53 +0200 Subject: [PATCH 0082/1795] Update eessi-2023.06-eb-4.7.2-2021b.yml Co-authored-by: Kenneth Hoste --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 97adcc3a71..6a467e9064 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -10,4 +10,5 @@ easyconfigs: - OpenMPI-4.1.1-GCC-11.2.0 - gompi-2021b - FlexiBLAS-3.0.4-GCC-11.2.0.eb + - foss-2021b.eb - QuantumESPRESSO-6.8-foss-2021b.eb From 145cbe9c47b1cb89ad40d92d2a36d6a2402c525f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 8 Jul 2023 14:17:32 +0200 Subject: [PATCH 0083/1795] copy build log to build logs dir if build failed --- EESSI-pilot-install-software.sh | 45 +++++++++++++++++++++++++++++++++ bot/build.sh | 20 ++++++++++++--- run_in_compat_layer_env.sh | 3 +++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 12878b7e6c..bc22410b36 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -14,6 +14,36 @@ display_help() { echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" } +function copy_build_log() { + # copy specified build log to specified directory, with some context added + build_log=${1} + build_logs_dir=${2} + + # also copy to build logs directory, if specified + if [ ! -z "${build_logs_dir}" ]; then + log_filename="$(basename ${build_log})" + if [ ! -z "${SLURM_JOB_ID}" ]; then + # use subdirectory for build log in context of a Slurm job + build_log_path="${build_logs_dir}/jobs/${SLURM_JOB_ID}/${log_filename}" + else + build_log_path="${build_logs_dir}/non-jobs/${log_filename}" + fi + mkdir -p $(dirname ${build_log_path}) + cp -a ${build_log} ${build_log_path} + + # add context to end of copied log file + echo >> ${build_log_path} + echo "Context from which build log was copied:" >> ${build_log_path} + echo "- original path of build log: ${build_log}" >> ${build_log_path} + echo "- working directory: ${PWD}" >> ${build_log_path} + echo "- Slurm job ID: ${SLURM_OUT}" >> ${build_log_path} + echo "- EasyBuild version: ${eb_version}" >> ${build_log_path} + echo "- easystack file: ${es}" >> ${build_log_path} + + echo "EasyBuild log file ${build_log} copied to ${build_log_path} (with context appended)" + fi +} + POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do @@ -35,6 +65,10 @@ while [[ $# -gt 0 ]]; do export https_proxy="$2" shift 2 ;; + --build-logs-dir) + export build_logs_dir="${2}" + shift 2 + ;; -*|--*) echo "Error: Unknown option: $1" >&2 exit 1 @@ -150,6 +184,17 @@ for eb_version in '4.7.2'; do echo_green "Feeding easystack file ${es} to EasyBuild..." ${EB} --easystack ${TOPDIR}/${es} --robot + ec=$? + + # copy EasyBuild log file if EasyBuild exited with an error + if [ ${ec} -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + # copy to current working directory + cp -a ${eb_last_log} . + echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}" + # copy to build logs dir (with context added) + copy_build_log "${eb_last_log}" "${build_logs_dir}" + fi $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es} else diff --git a/bot/build.sh b/bot/build.sh index c8def2cdd3..bd7b2fa8d8 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -53,6 +53,17 @@ LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided +# check if path to copy build logs to is specified, so we can copy build logs for failing builds there +BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir") +echo "bot/build.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'" +# if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container +mkdir -p ${BUILD_LOGS_DIR} +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BUILD_LOGS_DIR}" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" +fi + SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then @@ -151,19 +162,20 @@ BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") # prepare arguments to install_software_layer.sh (specific to build step) -GENERIC_OPT= +declare -a INSTALL_SCRIPT_ARGS=() if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then - GENERIC_OPT="--generic" + INSTALL_SCRIPT_ARGS+=("--generic") fi +[[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" -echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" +echo " -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ - -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} + -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} # prepare directory to store tarball of tmp for tarball step TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index c70077bf15..12688e2aed 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -14,6 +14,9 @@ if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then fi INPUT=$(echo "$@") +if [ ! -z ${SLURM_JOB_ID} ]; then + INPUT="export SLURM_JOB_ID=${SLURM_JOB_ID}; ${INPUT}" +fi if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi From 9d9cf1d3a7ee9d3a488716da8d7bdad37f311cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 10 Jul 2023 10:20:39 +0200 Subject: [PATCH 0084/1795] add build_logs_dir setting --- bot/bot-eessi-aws-citc.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 5b3ad34612..3d5f2c77bf 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -28,6 +28,9 @@ private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-p # name of the job script used for building an EESSI stack build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm +# Path (directory) to which build logs for (only) failing builds should be copied by bot/build.sh script +build_logs_dir = /mnt/shared/bot-build-logs + # The container_cachedir may be used to reuse downloaded container image files # across jobs. Thus, jobs can more quickly launch containers. container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir From b662fc08094ff93b99e2fed1839ab56369be3cdd Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 10 Jul 2023 16:34:15 +0200 Subject: [PATCH 0085/1795] {2023.06}[foss/2021b] GROMACS v2021.5 --- eessi-2023.06-eb-4.7.2-2021b.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 6a467e9064..614c22e6a3 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -12,3 +12,5 @@ easyconfigs: - FlexiBLAS-3.0.4-GCC-11.2.0.eb - foss-2021b.eb - QuantumESPRESSO-6.8-foss-2021b.eb + - GROMACS-2021.5-foss-2021b.eb + From 5a382291361d947b37ff16f5013e25a5790c0bc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 12 Jul 2023 12:08:32 +0200 Subject: [PATCH 0086/1795] run chmod 0644 on copied build log --- EESSI-pilot-install-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index bc22410b36..8110135cc0 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -30,6 +30,7 @@ function copy_build_log() { fi mkdir -p $(dirname ${build_log_path}) cp -a ${build_log} ${build_log_path} + chmod 0644 ${build_log_path} # add context to end of copied log file echo >> ${build_log_path} From 1adfe5ea0573d9e6a0583b3f7ff7b3f6145defd3 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 14 Jul 2023 16:12:55 +0200 Subject: [PATCH 0087/1795] [2023.06]{foss/2021b} SciPy-bundle neoverse_v1 --- eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 6a467e9064..1945ce9dde 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -12,3 +12,4 @@ easyconfigs: - FlexiBLAS-3.0.4-GCC-11.2.0.eb - foss-2021b.eb - QuantumESPRESSO-6.8-foss-2021b.eb + - SciPy-bundle-2021.10-foss-2021b From ac9fb4d898991085285d8b1471794574c453681d Mon Sep 17 00:00:00 2001 From: casparvl Date: Tue, 18 Jul 2023 14:39:31 +0000 Subject: [PATCH 0088/1795] Added OpenMPI-4.1.4 with GCC 11.3.0 --- eessi-2023.06-eb-4.7.2-2022a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index b8f0cc17bc..d40ddff261 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,2 +1,3 @@ easyconfigs: - GCC-11.3.0 + - OpenMPI-4.1.4-GCC-11.3.0.eb From 7a840ea3b8ddf6cc13bb2e072eb80a17f58f0ce4 Mon Sep 17 00:00:00 2001 From: casparvl Date: Tue, 18 Jul 2023 15:23:26 +0000 Subject: [PATCH 0089/1795] OpenMPI for foss 2022b --- eessi-2023.06-eb-4.7.2-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml index 15c99b5e0b..359300b1fa 100644 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ b/eessi-2023.06-eb-4.7.2-2022b.yml @@ -1,2 +1,3 @@ easyconfigs: - GCC-12.2.0 + - OpenMPI-4.1.4-GCC-12.2.0 From 94fa4c4bd89fcb0d168dd79c9a369ba1175bd9e5 Mon Sep 17 00:00:00 2001 From: casparvl Date: Tue, 18 Jul 2023 21:24:36 +0000 Subject: [PATCH 0090/1795] Added foss-2022a --- eessi-2023.06-eb-4.7.2-2022a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index b8f0cc17bc..0721a93257 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,2 +1,3 @@ easyconfigs: - GCC-11.3.0 + - foss-2022a From c2160ec99ab1196ef25f7bf9b0890823f1ccb955 Mon Sep 17 00:00:00 2001 From: casparvl Date: Tue, 18 Jul 2023 21:25:43 +0000 Subject: [PATCH 0091/1795] Added foss-2022b --- eessi-2023.06-eb-4.7.2-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml index 15c99b5e0b..125a567af2 100644 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ b/eessi-2023.06-eb-4.7.2-2022b.yml @@ -1,2 +1,3 @@ easyconfigs: - GCC-12.2.0 + - foss-2022b From 45df4625e162830ede5e0dfe15abf40376076697 Mon Sep 17 00:00:00 2001 From: casparvl Date: Tue, 18 Jul 2023 23:33:42 +0000 Subject: [PATCH 0092/1795] Added cmake explicitely, since it needs an include-easyblocks-from-pr --- eessi-2023.06-eb-4.7.2-2022b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml index 00027f3291..6d15ee2b90 100644 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ b/eessi-2023.06-eb-4.7.2-2022b.yml @@ -1,4 +1,7 @@ easyconfigs: - GCC-12.2.0 - OpenMPI-4.1.4-GCC-12.2.0 - - foss-2022b \ No newline at end of file + - CMake-3.24.3-GCCcore-12.2.0.eb: + options: + include-easyblocks-from-pr: 2248 + - foss-2022b From cea6c3b63714e34edca7a39e6fd7ab45ad11e617 Mon Sep 17 00:00:00 2001 From: casparvl Date: Tue, 18 Jul 2023 23:37:33 +0000 Subject: [PATCH 0093/1795] Added Cmake explicitely, since we need to pass an easyblocks-from-pr --- eessi-2023.06-eb-4.7.2-2022a.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index 6c6e0de3f1..f4a3da0143 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,4 +1,10 @@ easyconfigs: - GCC-11.3.0 - OpenMPI-4.1.4-GCC-11.3.0.eb - - foss-2022a \ No newline at end of file + - CMake-3.23.1-GCCcore-11.3.0.eb: + options: + include-easyblocks-from-pr: 2248 + - CMake-3.24.3-GCCcore-11.3.0.eb: + options: + include-easyblocks-from-pr: 2248 + - foss-2022a From b5951a522684a5130bfeaa9d837c75768fd3b2b4 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 20 Jul 2023 17:21:48 +0200 Subject: [PATCH 0094/1795] This is a rebuild of the ReFrame version to include hpctestlib in the PYTHONPATH. * This pull request must not be merged. --- eessi-2023.06-eb-4.7.2-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index 245271cebf..7d785668c2 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -1,2 +1,2 @@ easyconfigs: - - ReFrame-4.2.0 + - ReFrame-4.2.0 --rebuild From e5da6875d024600ab6dff1c9c3fcc6a782d83a9d Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 20 Jul 2023 17:26:16 +0200 Subject: [PATCH 0095/1795] Should also include --from-pr. --- eessi-2023.06-eb-4.7.2-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index 7d785668c2..6693638bfa 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -1,2 +1,2 @@ easyconfigs: - - ReFrame-4.2.0 --rebuild + - ReFrame-4.2.0 ---from-pr=18320 -rebuild From eef33f7fb24d3feb198919e21fa12e94110a81df Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 20 Jul 2023 18:17:16 +0200 Subject: [PATCH 0096/1795] Stupid error in the eb options. --- eessi-2023.06-eb-4.7.2-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index 6693638bfa..da1d7c9692 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -1,2 +1,2 @@ easyconfigs: - - ReFrame-4.2.0 ---from-pr=18320 -rebuild + - ReFrame-4.2.0 --from-pr=18320 --rebuild From 468a9e2fc153c2acfbfe8573ad919426719f4c85 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Fri, 21 Jul 2023 00:08:59 +0200 Subject: [PATCH 0097/1795] Included options in the correct yml format. --- eessi-2023.06-eb-4.7.2-system.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index da1d7c9692..ec1b1982cd 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -1,2 +1,5 @@ easyconfigs: - - ReFrame-4.2.0 --from-pr=18320 --rebuild + - ReFrame-4.2.0 + options: + from-pr: 18320 + rebuild: From d4f5887513a17e60e41192546806a2d8a1c29536 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Fri, 21 Jul 2023 00:31:23 +0200 Subject: [PATCH 0098/1795] Trying this way of specifying the rebuild option without the colon. --- eessi-2023.06-eb-4.7.2-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index ec1b1982cd..ecaa8af088 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -2,4 +2,4 @@ easyconfigs: - ReFrame-4.2.0 options: from-pr: 18320 - rebuild: + rebuild From 89a825b7ccdb03ca8edf4e189cd7f573705c9cd9 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Fri, 21 Jul 2023 15:25:52 +0200 Subject: [PATCH 0099/1795] Trying this, if this does not work then, rebuild: 1 --- eessi-2023.06-eb-4.7.2-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index ecaa8af088..1fcda5034b 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -2,4 +2,4 @@ easyconfigs: - ReFrame-4.2.0 options: from-pr: 18320 - rebuild + rebuild: True From e439db5bba42c0e0cd0be0d81f27bfcb5180e5f2 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Fri, 21 Jul 2023 15:48:01 +0200 Subject: [PATCH 0100/1795] Had made another error in providing a colon in front of the easyconfig file with options. --- eessi-2023.06-eb-4.7.2-system.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index 1fcda5034b..82d0d19de2 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -1,5 +1,5 @@ easyconfigs: - - ReFrame-4.2.0 + - ReFrame-4.2.0.eb: options: - from-pr: 18320 + from-pr: 18320 rebuild: True From c23fd939811a0649812681c6e5a27b1b865b0db0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Aug 2023 12:49:54 +0200 Subject: [PATCH 0101/1795] Increased number of tests that are allowed to fail for OpenBLAS on ARM. See https://github.com/EESSI/software-layer/pull/309/#issuecomment-1643518730 --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 191e24678b..911e2873f9 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -167,7 +167,7 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): cfg_option = 'max_failing_lapack_tests_num_errors' if get_cpu_architecture() == AARCH64: orig_value = ec[cfg_option] - ec[cfg_option] = 300 + ec[cfg_option] = 400 print_msg("Maximum number of failing LAPACK tests with numerical errors for %s relaxed to %s (was %s)", ec.name, ec[cfg_option], orig_value) else: From 36f5b309d709a454920cdeff76b400f388ed43d1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Aug 2023 13:28:41 +0200 Subject: [PATCH 0102/1795] Added reference to ticket for increase in accepted test failures --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index 911e2873f9..3dc6d1efa8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -164,6 +164,7 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): if ec.name == 'OpenBLAS': # relax maximum number of failed numerical LAPACK tests on Arm 64-bit systems, # since the default setting of 150 that works well on x86_64 is a bit too strict + # See https://github.com/EESSI/software-layer/issues/314 cfg_option = 'max_failing_lapack_tests_num_errors' if get_cpu_architecture() == AARCH64: orig_value = ec[cfg_option] From 05d2d2dc3a6b54d61a4af6109c6e2dd4baf42c2b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Aug 2023 17:28:59 +0200 Subject: [PATCH 0103/1795] use shorter 'pilot' job name in workflow to check for missing installations --- .github/workflows/test_eessi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index e93d437cea..65994cab2d 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] permissions: contents: read # to fetch code (actions/checkout) jobs: - eessi_pilot_repo: + pilot: runs-on: ubuntu-20.04 strategy: fail-fast: false From 32ab9818248e5a9d36281c4d9229522791ad3215 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Thu, 10 Aug 2023 07:10:10 +0000 Subject: [PATCH 0104/1795] {2023.06}[foss/2021a] Arrow V6.0.0 --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index c0fa353475..919e02f460 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -13,3 +13,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 options: from-pr: 18087 + - Arrow-6.0.0-foss-2021a.eb From 4f54fd67c4b9c5555c166ce27b8747146a9d37dc Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:01:48 +0200 Subject: [PATCH 0105/1795] Update eb_hooks.py --- eb_hooks.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 191e24678b..6ef1386db0 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -258,6 +258,10 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") +def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): + if self.name == 'SciPy-bundle' and '/neoverse_v1/' in self.installdir: + self.cfg['testopts'] = "|| echo ignoring failing tests" + PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, @@ -277,3 +281,7 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, } + +PRE_TEST_HOOKS = { + 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, +} From 05d85a9d2790e29c59f203ca4f695afeb7f07cec Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 18 Aug 2023 14:41:01 +0200 Subject: [PATCH 0106/1795] initial version for bot/inspect.sh --- bot/inspect.sh | 240 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 bot/inspect.sh diff --git a/bot/inspect.sh b/bot/inspect.sh new file mode 100644 index 0000000000..39ac7ceb99 --- /dev/null +++ b/bot/inspect.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# +# Script to inspect result of a build job for the EESSI software layer. +# Intended use is that it is called with a path to a job directory. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# - Script is executed on the same architecture the job was running on. +# - Initially, we also assume that is run on the same resource with the +# same (compute) node setup (local disk space, HTTP proxies, etc.) +# - The job directory being supplied has been prepared by the bot with a +# checkout of a pull request (OR by some other means) +# - The job directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited. +# - The 'cfg' directory may contain any additional files referenced in +# 'job.cfg' (repos.cfg, etc.). +# - The job produced some tarballs for its state (tmp disk for overlayfs, +# CVMFS cache, etc.) under 'previous_tmp/{build,tarball}_step'. + +# stop as soon as something fails +set -e + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " -h | --help - display this usage information" + echo " -j | --job-dir DIR - inspect job with the given work directory DIR" + echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" + echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" +} + +job_dir= +http_proxy= +https_proxy= + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case ${1} in + -h|--help) + display_help + exit 0 + ;; + -j|--job-dir) + export job_dir="${2}" + shift 2 + ;; + -x|--http-proxy) + export http_proxy="${2}" + shift 2 + ;; + -y|--https-proxy) + export https_proxy="${2}" + shift 2 + ;; + -*|--*) + echo "Error: Unknown option: ${1}" >&2 + exit 1 + ;; + *) # No more options + POSITIONAL_ARGS+=("${1}") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh + +if [[ -z ${job_dir} ]]; then + echo_yellow "path to job directory missing" + display_help + exit 1 +fi + +# defaults +export JOB_CFG_FILE="${job_dir}/cfg/job.cfg}" +HOST_ARCH=$(uname -m) + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "bot/inspect.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/inspect.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} + +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} +echo "bot/inspect.sh: HTTP_PROXY='${HTTP_PROXY}'" + +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} +echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'" + +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") +echo "bot/inspect.sh: LOCAL_TMP='${LOCAL_TMP}'" +# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided + +# check if path to copy build logs to is specified, so we can copy build logs for failing builds there +BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir") +echo "bot/inspect.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'" +# if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container +if [[ ! -z ${BUILD_LOGS_DIR} ]]; then + mkdir -p ${BUILD_LOGS_DIR} + if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BUILD_LOGS_DIR}" + else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" + fi +fi + +SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") +echo "bot/inspect.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + # make sure that separate directories are used for different CPU families + SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH} + export SINGULARITY_CACHEDIR +fi + +echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " +# replace any env variable in ${LOCAL_TMP} with its +# current value (e.g., a value that is local to the job) +STORAGE=$(envsubst <<< ${LOCAL_TMP}) +echo "'${STORAGE}'" + +# make sure ${STORAGE} exists +mkdir -p ${STORAGE} + +# make sure the base tmp storage is unique +JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) +echo "bot/inspect.sh: created unique base tmp storage directory at ${JOB_STORAGE}" + +# obtain list of modules to be loaded +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") +echo "bot/inspect.sh: LOAD_MODULES='${LOAD_MODULES}'" + +# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND +CONTAINER=$(cfg_get_value "repository" "container") +export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" +mkdir -p ${SINGULARITY_TMPDIR} + +# load modules if LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/inspect.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/inspect.sh: no modules to be loaded" +fi + +# determine repository to be used from entry .repository in ${JOB_CFG_FILE} +REPOSITORY=$(cfg_get_value "repository" "repo_id") +EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") +export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} +echo "bot/inspect.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" + +# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# next script (eessi_container.sh) makes use of it via sourcing init scripts +# (e.g., init/eessi_defaults or init/minimal_eessi_env) +export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/inspect.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" + +# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} +# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down +# "source init/eessi_defaults" via sourcing init/minimal_eessi_env +export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +echo "bot/inspect.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" + +# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# fallbacks: +# - ${CPU_TARGET} handed over from bot +# - left empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") +EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} +export EESSI_SOFTWARE_SUBDIR_OVERRIDE +echo "bot/inspect.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" + +# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) +EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") +export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} +echo "bot/inspect.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" + +# prepare arguments to eessi_container.sh common to build and tarball steps +declare -a COMMON_ARGS=() +COMMON_ARGS+=("--verbose") +COMMON_ARGS+=("--access" "rw") +COMMON_ARGS+=("--mode" "run") +[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") +[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") +[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") +[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") + +# make sure to use the same parent dir for storing tarballs of tmp +PREVIOUS_TMP_DIR=${PWD}/previous_tmp + +# prepare directory to store tarball of tmp for build step +TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} + +# prepare arguments to eessi_container.sh specific to build step +declare -a BUILD_STEP_ARGS=() +BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") +BUILD_STEP_ARGS+=("--storage" "${STORAGE}") + +# prepare arguments to install_software_layer.sh (specific to build step) +declare -a INSTALL_SCRIPT_ARGS=() +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then + INSTALL_SCRIPT_ARGS+=("--generic") +fi +[[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") + +# create tmp file for output of build step +build_outerr=$(mktemp build.outerr.XXXX) + +echo "Executing command to build software:" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" +echo " -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" +#./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ +# -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} + + +exit 0 From 90f1018066a88aa25599c34ed861a34f32366b19 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 18 Aug 2023 14:45:44 +0200 Subject: [PATCH 0107/1795] change permissions --- bot/inspect.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bot/inspect.sh diff --git a/bot/inspect.sh b/bot/inspect.sh old mode 100644 new mode 100755 From 0ec406eedc6285a3b7571265892e4a455f51300a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 18 Aug 2023 14:53:56 +0200 Subject: [PATCH 0108/1795] fix cfg file path --- bot/inspect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index 39ac7ceb99..591529a31f 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -83,7 +83,7 @@ if [[ -z ${job_dir} ]]; then fi # defaults -export JOB_CFG_FILE="${job_dir}/cfg/job.cfg}" +export JOB_CFG_FILE="${job_dir}/cfg/job.cfg" HOST_ARCH=$(uname -m) # check if ${JOB_CFG_FILE} exists From d3a0457b7665a41b440369eb7710f4d165c9b95f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 18 Aug 2023 14:58:14 +0200 Subject: [PATCH 0109/1795] report more env var settings --- bot/inspect.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index 591529a31f..f860169d95 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -104,7 +104,7 @@ echo "bot/inspect.sh: HTTP_PROXY='${HTTP_PROXY}'" # if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} -echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'" +echo "bot/inspect.sh: HTTPS_PROXY='${HTTPS_PROXY}'" LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") echo "bot/inspect.sh: LOCAL_TMP='${LOCAL_TMP}'" @@ -150,8 +150,11 @@ echo "bot/inspect.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") +echo "bot/inspect.sh: CONTAINER='${CONTAINER}'" export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +echo "bot/inspect.sh: SINGULARITY_HOME='${SINGULARITY_HOME}'" export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" +echo "bot/inspect.sh: SINGULARITY_TMPDIR='${SINGULARITY_TMPDIR}'" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty @@ -167,6 +170,7 @@ fi # determine repository to be used from entry .repository in ${JOB_CFG_FILE} REPOSITORY=$(cfg_get_value "repository" "repo_id") +echo "bot/inspect.sh: REPOSITORY='${REPOSITORY}'" EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/inspect.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" From 9fb5db25f3e7ce8e2d267a25f225b72911d08794 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Aug 2023 10:49:23 +0200 Subject: [PATCH 0110/1795] simply run startprefix --- bot/inspect.sh | 106 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index f860169d95..9138d6e9b4 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -203,42 +203,80 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} echo "bot/inspect.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" # prepare arguments to eessi_container.sh common to build and tarball steps -declare -a COMMON_ARGS=() -COMMON_ARGS+=("--verbose") -COMMON_ARGS+=("--access" "rw") -COMMON_ARGS+=("--mode" "run") -[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") -[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") -[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") -[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") - -# make sure to use the same parent dir for storing tarballs of tmp -PREVIOUS_TMP_DIR=${PWD}/previous_tmp - -# prepare directory to store tarball of tmp for build step -TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step -mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} - -# prepare arguments to eessi_container.sh specific to build step -declare -a BUILD_STEP_ARGS=() -BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") -BUILD_STEP_ARGS+=("--storage" "${STORAGE}") - -# prepare arguments to install_software_layer.sh (specific to build step) -declare -a INSTALL_SCRIPT_ARGS=() -if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then - INSTALL_SCRIPT_ARGS+=("--generic") +declare -a CMDLINE_ARGS=() +CMDLINE_ARGS+=("--verbose") +CMDLINE_ARGS+=("--access" "rw") +CMDLINE_ARGS+=("--mode" "shell") +[[ ! -z ${CONTAINER} ]] && CMDLINE_ARGS+=("--container" "${CONTAINER}") +[[ ! -z ${HTTP_PROXY} ]] && CMDLINE_ARGS+=("--http-proxy" "${HTTP_PROXY}") +[[ ! -z ${HTTPS_PROXY} ]] && CMDLINE_ARGS+=("--https-proxy" "${HTTPS_PROXY}") +[[ ! -z ${REPOSITORY} ]] && CMDLINE_ARGS+=("--repository" "${REPOSITORY}") + +# create a directory for creating a tarball of the tmp directory +INSPECT_TMP_DIR=$(mktemp -d ${PWD}/inspect.XXX) + +# add arguments for temporary storage and storing a tarball of tmp +CMDLINE_ARGS+=("--save" "${INSPECT_TMP_DIR}") +CMDLINE_ARGS+=("--storage" "${STORAGE}") + +# # prepare arguments to install_software_layer.sh (specific to build step) +# declare -a INSTALL_SCRIPT_ARGS=() +# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then +# INSTALL_SCRIPT_ARGS+=("--generic") +# fi +# [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") + +# make sure some environment settings are available inside the shell started via +# startprefix +base_dir=$(dirname $(realpath $0)) +source ${base_dir}/init/eessi_defaults + +if [ -z $EESSI_PILOT_VERSION ]; then + echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 + exit 1 +fi +EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" + +# NOTE The below requires access to the CVMFS repository. We could make a first +# test run with a container. For now we skip the test. +# if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then +# echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2 +# exit 1 +# fi + +# When we want to run a script with arguments, the next line is ensures to retain +# these arguments. +# INPUT=$(echo "$@") +if [ ! -z ${SLURM_JOB_ID} ]; then + INPUT="export SLURM_JOB_ID=${SLURM_JOB_ID}; ${INPUT}" +fi +if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then + INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then + INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${http_proxy} ]; then + INPUT="export http_proxy=${http_proxy}; ${INPUT}" +fi +if [ ! -z ${https_proxy} ]; then + INPUT="export https_proxy=${https_proxy}; ${INPUT}" fi -[[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") - -# create tmp file for output of build step -build_outerr=$(mktemp build.outerr.XXXX) -echo "Executing command to build software:" -echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" -echo " -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" -#./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ -# -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} +echo "Executing command to start interactive session to inspect build job:" +# TODO possibly add information on how to init session after the prefix is +# entered, initialization consists of +# - environment variable settings (see 'run_in_compat_layer_env.sh') +# - setup steps run in 'EESSI-pilot-install-software.sh' +# These initializations are combined into a single script that is executed when +# the shell in startprefix is started. We set the env variable BASH_ENV here. +echo "./eessi_container.sh ${CMDLINE_ARGS[@]}" +echo " -- ${EESSI_COMPAT_LAYER_DIR}/startprefix" +./eessi_container.sh "${CMDLINE_ARGS[@]}" \ + -- ${EESSI_COMPAT_LAYER_DIR}/startprefix exit 0 From e02c02475884ca87a5ebad32a7e7134e5a4f3475 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Aug 2023 11:12:17 +0200 Subject: [PATCH 0111/1795] fix path to access eessi_defaults --- bot/inspect.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index 9138d6e9b4..f303724d5f 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -229,7 +229,8 @@ CMDLINE_ARGS+=("--storage" "${STORAGE}") # make sure some environment settings are available inside the shell started via # startprefix base_dir=$(dirname $(realpath $0)) -source ${base_dir}/init/eessi_defaults +# base_dir of inspect.sh script is '.../bot', 'init' dir is at the same level +source ${base_dir}/../init/eessi_defaults if [ -z $EESSI_PILOT_VERSION ]; then echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 From 036a6dd4b387a63653381ea700b8cc4c3c6f7985 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Mon, 21 Aug 2023 11:29:25 +0200 Subject: [PATCH 0112/1795] Update eb_hooks.py --- eb_hooks.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 6ef1386db0..e27845e544 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -258,8 +258,18 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") +def pre_test_hook(self,*args, **kwargs): + """Main pre-test hook: trigger custom functions based on software name.""" + if self.name in PRE_TEST_HOOKS: + PRE_TEST_HOOKS[self.name](self, *args, **kwargs) + def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): - if self.name == 'SciPy-bundle' and '/neoverse_v1/' in self.installdir: + """ + Pre-test hook for SciPy-bundle: skip failing tests for SciPy-bundle 2021.10 (currently the only version that is failing). + In previous versions we were not as strict yet on the numpy/SciPy tests + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'SciPy-bundle' and self.version == '2021.10' and cpu_target == 'aarch64/neoverse_v1': self.cfg['testopts'] = "|| echo ignoring failing tests" From 44f7ff5748e1efde0012a0f34600367253378567 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:58:37 +0200 Subject: [PATCH 0113/1795] Update eb_hooks.py Fix indentation line 267 --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index e27845e544..c7435315ab 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -264,7 +264,7 @@ def pre_test_hook(self,*args, **kwargs): PRE_TEST_HOOKS[self.name](self, *args, **kwargs) def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): - """ + """ Pre-test hook for SciPy-bundle: skip failing tests for SciPy-bundle 2021.10 (currently the only version that is failing). In previous versions we were not as strict yet on the numpy/SciPy tests """ From df5fbf09c18948faefcdf7cab26e7bb35ceae688 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 22 Aug 2023 14:52:01 +0200 Subject: [PATCH 0114/1795] tweaks after testing to enter prefix --- bot/inspect.sh | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index f303724d5f..3416426d7d 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -30,12 +30,15 @@ set -e display_help() { echo "usage: $0 [OPTIONS]" echo " -h | --help - display this usage information" - echo " -j | --job-dir DIR - inspect job with the given work directory DIR" + echo " -r | --resume TGZ - inspect job saved in tarball path TGZ; note, we assume the path" + echo " to be something like JOB_DIR/previous_tmp/{build,tarball}_step/TARBALL.tgz" + echo " and thus determine JOB_DIR from the given path" + echo " [default: none]" echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" } -job_dir= +resume_tgz= http_proxy= https_proxy= @@ -47,8 +50,8 @@ while [[ $# -gt 0 ]]; do display_help exit 0 ;; - -j|--job-dir) - export job_dir="${2}" + -r|--resume) + export resume_tgz="${2}" shift 2 ;; -x|--http-proxy) @@ -76,12 +79,21 @@ set -- "${POSITIONAL_ARGS[@]}" source scripts/utils.sh source scripts/cfg_files.sh -if [[ -z ${job_dir} ]]; then - echo_yellow "path to job directory missing" +if [[ -z ${resume_tgz} ]]; then + echo_red "path to tarball for resuming build job is missing" display_help exit 1 fi +job_dir=$(dirname $(dirname $(dirname ${resume_tgz}))) + +if [[ -z ${job_dir} ]]; then + # job directory could be determined + echo_red "job directory could not be determined from '${resume_tgz}'" + display_help + exit 2 +fi + # defaults export JOB_CFG_FILE="${job_dir}/cfg/job.cfg" HOST_ARCH=$(uname -m) @@ -151,7 +163,9 @@ echo "bot/inspect.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") echo "bot/inspect.sh: CONTAINER='${CONTAINER}'" -export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +# instead of using ${PWD} as HOME in the container, we use the job directory +# to have access to output files of the job +export SINGULARITY_HOME="${job_dir}:/eessi_bot_job" echo "bot/inspect.sh: SINGULARITY_HOME='${SINGULARITY_HOME}'" export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" echo "bot/inspect.sh: SINGULARITY_TMPDIR='${SINGULARITY_TMPDIR}'" @@ -171,6 +185,7 @@ fi # determine repository to be used from entry .repository in ${JOB_CFG_FILE} REPOSITORY=$(cfg_get_value "repository" "repo_id") echo "bot/inspect.sh: REPOSITORY='${REPOSITORY}'" +# TODO better to read this from tarball??? EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/inspect.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" @@ -185,7 +200,7 @@ echo "bot/inspect.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRI # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/$(cfg_get_value 'repository' 'repo_name')" echo "bot/inspect.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} @@ -206,12 +221,14 @@ echo "bot/inspect.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" declare -a CMDLINE_ARGS=() CMDLINE_ARGS+=("--verbose") CMDLINE_ARGS+=("--access" "rw") -CMDLINE_ARGS+=("--mode" "shell") +CMDLINE_ARGS+=("--mode" "run") [[ ! -z ${CONTAINER} ]] && CMDLINE_ARGS+=("--container" "${CONTAINER}") [[ ! -z ${HTTP_PROXY} ]] && CMDLINE_ARGS+=("--http-proxy" "${HTTP_PROXY}") [[ ! -z ${HTTPS_PROXY} ]] && CMDLINE_ARGS+=("--https-proxy" "${HTTPS_PROXY}") [[ ! -z ${REPOSITORY} ]] && CMDLINE_ARGS+=("--repository" "${REPOSITORY}") +[[ ! -z ${resume_tgz} ]] && CMDLINE_ARGS+=("--resume" "${resume_tgz}") + # create a directory for creating a tarball of the tmp directory INSPECT_TMP_DIR=$(mktemp -d ${PWD}/inspect.XXX) @@ -230,6 +247,7 @@ CMDLINE_ARGS+=("--storage" "${STORAGE}") # startprefix base_dir=$(dirname $(realpath $0)) # base_dir of inspect.sh script is '.../bot', 'init' dir is at the same level +# TODO better use script from tarball??? source ${base_dir}/../init/eessi_defaults if [ -z $EESSI_PILOT_VERSION ]; then From f34854cf7bc86ac2cfd19d7f589ee684c39fe395 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 23 Aug 2023 10:35:45 +0200 Subject: [PATCH 0115/1795] Trigger CI/CD pipeline From 938cf06f2eebdada59a9b2115432bdab168a0bdf Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 23 Aug 2023 10:47:43 +0200 Subject: [PATCH 0116/1795] Trigger CI/CD pipeline From 45dac8b254e2fd3d0c738f630de9a997f8f86ad5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 24 Aug 2023 15:16:19 +0200 Subject: [PATCH 0117/1795] initializing bot build environment settings incl EasyBuild + information note --- bot/inspect.sh | 149 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 139 insertions(+), 10 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index 3416426d7d..e5e4df5970 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -229,12 +229,17 @@ CMDLINE_ARGS+=("--mode" "run") [[ ! -z ${resume_tgz} ]] && CMDLINE_ARGS+=("--resume" "${resume_tgz}") -# create a directory for creating a tarball of the tmp directory -INSPECT_TMP_DIR=$(mktemp -d ${PWD}/inspect.XXX) +# create a directory for creating temporary data and scripts for the inspection +INSPECT_DIR=$(mktemp --directory --tmpdir=${PWD} inspect.XXX) +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${INSPECT_DIR}:/inspect_eessi_build_job" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${INSPECT_DIR}:/inspect_eessi_build_job" +fi # add arguments for temporary storage and storing a tarball of tmp -CMDLINE_ARGS+=("--save" "${INSPECT_TMP_DIR}") -CMDLINE_ARGS+=("--storage" "${STORAGE}") +CMDLINE_ARGS+=("--save" "${INSPECT_DIR}") +CMDLINE_ARGS+=("--storage" "${JOB_STORAGE}") # # prepare arguments to install_software_layer.sh (specific to build step) # declare -a INSTALL_SCRIPT_ARGS=() @@ -266,23 +271,147 @@ EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/comp # When we want to run a script with arguments, the next line is ensures to retain # these arguments. # INPUT=$(echo "$@") +mkdir -p ${INSPECT_DIR}/scripts +RESUME_SCRIPT=${INSPECT_DIR}/scripts/resume_env.sh +echo "bot/inspect.sh: creating script '${RESUME_SCRIPT}' to resume environment settings" + +cat << EOF > ${RESUME_SCRIPT} +#!${EESSI_COMPAT_LAYER_DIR}/bin/bash +echo "Sourcing '\$BASH_SOURCE' to init bot environment of build job" +EOF if [ ! -z ${SLURM_JOB_ID} ]; then - INPUT="export SLURM_JOB_ID=${SLURM_JOB_ID}; ${INPUT}" + # TODO do we need the value at all? if so which one: current or of the job to + # inspect? + echo "export CURRENT_SLURM_JOB_ID=${SLURM_JOB_ID}" >> ${RESUME_SCRIPT} fi if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then - INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" + echo "export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" >> ${RESUME_SCRIPT} fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then - INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" + echo "export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}" >> ${RESUME_SCRIPT} fi if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then - INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" + echo "export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT} fi if [ ! -z ${http_proxy} ]; then - INPUT="export http_proxy=${http_proxy}; ${INPUT}" + echo "export http_proxy=${http_proxy}" >> ${RESUME_SCRIPT} fi if [ ! -z ${https_proxy} ]; then - INPUT="export https_proxy=${https_proxy}; ${INPUT}" + echo "export https_proxy=${https_proxy}" >> ${RESUME_SCRIPT} +fi +cat << 'EOF' >> ${RESUME_SCRIPT} +TOPDIR=$(dirname $(realpath $BASH_SOURCE)) + +source ${TOPDIR}/scripts/utils.sh + +# honor $TMPDIR if it is already defined, use /tmp otherwise +if [ -z $TMPDIR ]; then + export WORKDIR=/tmp/$USER +else + export WORKDIR=$TMPDIR/$USER +fi + +TMPDIR=$(mktemp -d) + +echo ">> Setting up environment..." + +source $TOPDIR/init/minimal_eessi_env + +if [ -d $EESSI_CVMFS_REPO ]; then + echo_green "$EESSI_CVMFS_REPO available, OK!" +else + fatal_error "$EESSI_CVMFS_REPO is not available!" +fi + +# make sure we're in Prefix environment by checking $SHELL +if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then + echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" +else + fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" +fi + +# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory +export PYTHONPYCACHEPREFIX=$TMPDIR/pycache + +DETECTION_PARAMETERS='' +GENERIC=0 +EB='eb' +if [[ "$EASYBUILD_OPTARCH" == "GENERIC" || "$EESSI_SOFTWARE_SUBDIR_OVERRIDE" == *"/generic" ]]; then + echo_yellow ">> GENERIC build requested, taking appropriate measures!" + DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" + GENERIC=1 + export EASYBUILD_OPTARCH=GENERIC + EB='eb --optarch=GENERIC' +fi + +echo ">> Determining software subdirectory to use for current build host..." +if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" +else + echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" +fi + +# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) +# $EESSI_SILENT - don't print any messages +# $EESSI_BASIC_ENV - give a basic set of environment variables +EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables + +if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then + fatal_error "Failed to determine software subdirectory?!" +elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" +else + echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" +fi + +echo ">> Initializing Lmod..." +source $EPREFIX/usr/share/Lmod/init/bash +ml_version_out=$TMPDIR/ml.out +ml --version &> $ml_version_out +if [[ $? -eq 0 ]]; then + echo_green ">> Found Lmod ${LMOD_VERSION}" +else + fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" +fi + +echo ">> Configuring EasyBuild..." +source $TOPDIR/configure_easybuild + +echo ">> Setting up \$MODULEPATH..." +# make sure no modules are loaded +module --force purge +# ignore current $MODULEPATH entirely +module unuse $MODULEPATH +module use $EASYBUILD_INSTALLPATH/modules/all +if [[ -z ${MODULEPATH} ]]; then + fatal_error "Failed to set up \$MODULEPATH?!" +else + echo_green ">> MODULEPATH set up: ${MODULEPATH}" +fi + +eb_version='4.7.2' + +# load EasyBuild module (will be installed if it's not available yet) +source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + +echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." + +echo "Ready for inspection of build job:" +echo " - job directory is $HOME (\$HOME), check for slurm-*.out file" +echo " - temporary data of job available at /tmp" +echo " - Note, prefix $EESSI_PREFIX is writable" +echo " - EasyBuild v${eb_version} is available" + +EOF +chmod u+x ${RESUME_SCRIPT} + +# try to map it into the container's $HOME/.profile instead +# TODO check if script already exists, if so change its name and source it at the beginning of the RESUME_SCRIPT +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${RESUME_SCRIPT}:/eessi_bot_job/.profile" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${RESUME_SCRIPT}:/eessi_bot_job/.profile" fi echo "Executing command to start interactive session to inspect build job:" From b12c6860bb5224a122bb53f4fafa3c0c9f4e3331 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 28 Aug 2023 13:12:51 +0200 Subject: [PATCH 0118/1795] {2023.06}[SYSTEM] easybuild 4.8.0 --- eessi-2023.06-eb-4.7.2-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index 245271cebf..fe9018b8c0 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -1,2 +1,5 @@ easyconfigs: - ReFrame-4.2.0 + - EasyBuild-4.8.0.eb: + options: + from-pr: 18282 From dd7b26df0950d49c7159ec08e1a2db90ec059fb7 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 4 Sep 2023 15:24:42 +0200 Subject: [PATCH 0119/1795] removing cURL from configure_easybuild --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index 23f3920154..7dca1ce682 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -26,7 +26,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) # problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From 4a8a7a3d76193dc4d031d2271069b92e431c7e92 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 5 Sep 2023 09:40:32 +0200 Subject: [PATCH 0120/1795] add git to stack --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + eessi-2023.06-eb-4.7.2-2021b.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index c0fa353475..be4f1deaa8 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -6,6 +6,7 @@ easyconfigs: - foss-2021a - libpng-1.6.37-GCCcore-10.3.0.eb: - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb + - git-2.32.0-GCCcore-10.3.0-nodocs.eb - QuantumESPRESSO-6.7-foss-2021a.eb - GROMACS-2021.3-foss-2021a.eb - Qt5-5.15.2-GCCcore-10.3.0.eb: diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 7283caa4cc..43a0fbd4bd 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -11,6 +11,7 @@ easyconfigs: - gompi-2021b - FlexiBLAS-3.0.4-GCC-11.2.0.eb - foss-2021b.eb + - git-2.33.1-GCCcore-11.2.0-nodocs.eb - QuantumESPRESSO-6.8-foss-2021b.eb - SciPy-bundle-2021.10-foss-2021b - GROMACS-2021.5-foss-2021b.eb From 2c96e2b8ab87e491e0f9abc363e4fdf50af08c8e Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 5 Sep 2023 15:57:22 +0000 Subject: [PATCH 0121/1795] Added --from-pr 18348 --- eessi-2023.06-eb-4.7.2-2021a.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 919e02f460..c55f1f3976 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -13,4 +13,6 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 options: from-pr: 18087 - - Arrow-6.0.0-foss-2021a.eb + - Arrow-6.0.0-foss-2021a.eb: + options: + from-pr: 18348 From 978da670c998499a4378850beb67463474c1904c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 5 Sep 2023 18:16:10 +0200 Subject: [PATCH 0122/1795] As agreed, ignore test failures for FFTW.MPI test suite --- eb_hooks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 488e53d929..9560d84eff 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -274,6 +274,14 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): self.cfg['testopts'] = "|| echo ignoring failing tests" +def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): + """ + Pre-test hook for FFTW.MPI: skip failing tests for FFTWMPI 3.3.10 + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == 'aarch64/neoverse_v1': + self.cfg['testopts'] = "|| echo ignoring failing tests" + PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, @@ -295,4 +303,5 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): PRE_TEST_HOOKS = { 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, + 'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI, } From d0cfab844b83e12aa84eed8b6da2f32347e90e18 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 5 Sep 2023 18:29:15 +0200 Subject: [PATCH 0123/1795] remove 'rebuild' option for ReFrame 4.2.0 to avoid that ReFrame is re-installed every time --- eessi-2023.06-eb-4.7.2-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml index 7364c57e18..220fd82e87 100644 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -2,7 +2,6 @@ easyconfigs: - ReFrame-4.2.0.eb: options: from-pr: 18320 - rebuild: True - EasyBuild-4.8.0.eb: options: from-pr: 18282 From 02169fc2c104263bbc346ce2bb85a5086f90febb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Sep 2023 10:15:11 +0200 Subject: [PATCH 0124/1795] use RapidJSON easyconfig from PR to avoid hardcoded -march=native being used, which is important for generic builds --- eessi-2023.06-eb-4.7.2-2021a.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index c55f1f3976..288c59d0c7 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -13,6 +13,13 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 options: from-pr: 18087 + - RapidJSON-1.1.0-GCCcore-10.3.0.eb: + # strip out hardcoded -march=native, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18725 + options: + from-pr: 18725 - Arrow-6.0.0-foss-2021a.eb: + # fix installation of pyarrow Python bindings + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 options: from-pr: 18348 From 69d4c019558f8748b5a56f3b0ace7a5a94109eef Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 6 Sep 2023 11:13:57 +0200 Subject: [PATCH 0125/1795] {2023.06}[SYSTEM] Java v11.0.18 --- .github/workflows/test_eessi.yml | 1 + EESSI-pilot-install-software.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 65994cab2d..c51d313490 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -26,6 +26,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2022a.yml - eessi-2023.06-eb-4.7.2-2022b.yml - eessi-2023.06-eb-4.7.2-system.yml + - eessi-2023.06-eb-4.8.0-2021b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 8110135cc0..29fbb7aa62 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,7 +172,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2'; do +for eb_version in '4.7.2' '4.8.0'; do # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} From 41ed8d8485ca018b1657605677e148713efe15a8 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 6 Sep 2023 11:16:30 +0200 Subject: [PATCH 0126/1795] add Java --- eessi-2023.06-eb-4.8.0-system.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.0-system.yml diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml new file mode 100644 index 0000000000..ebd7435eea --- /dev/null +++ b/eessi-2023.06-eb-4.8.0-system.yml @@ -0,0 +1,6 @@ +easyconfigs: + - Java-11.0.18.eb: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/2557 + options: + include-easyblocks-from-pr: 2557 +~ From c0a3f109d03a2a15bc75bf1143ec77dc63dbd5ba Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 6 Sep 2023 11:18:17 +0200 Subject: [PATCH 0127/1795] update eessi-2023.06-eb-4.8.0-system.yml --- eessi-2023.06-eb-4.8.0-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml index ebd7435eea..2cce43b80b 100644 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ b/eessi-2023.06-eb-4.8.0-system.yml @@ -3,4 +3,3 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyblocks/pull/2557 options: include-easyblocks-from-pr: 2557 -~ From c3ce1cb2633f24b8c0d21787d5e8a927ca5dc9e7 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 6 Sep 2023 11:28:30 +0200 Subject: [PATCH 0128/1795] add right easystack-file --- .github/workflows/test_eessi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index c51d313490..7667d02c26 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -26,7 +26,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2022a.yml - eessi-2023.06-eb-4.7.2-2022b.yml - eessi-2023.06-eb-4.7.2-system.yml - - eessi-2023.06-eb-4.8.0-2021b.yml + - eessi-2023.06-eb-4.8.0-system.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From 95370cdfcea6246438e0bace7a2d267707114500 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 6 Sep 2023 13:01:57 +0200 Subject: [PATCH 0129/1795] change Java --- eessi-2023.06-eb-4.8.0-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml index 2cce43b80b..6da9c66483 100644 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ b/eessi-2023.06-eb-4.8.0-system.yml @@ -1,5 +1,5 @@ easyconfigs: - - Java-11.0.18.eb: + - Java-11.eb: # see https://github.com/easybuilders/easybuild-easyblocks/pull/2557 options: include-easyblocks-from-pr: 2557 From 0a1ff49d22fa612aa34b4e6e98dce5efe5a89b36 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:06:04 +0200 Subject: [PATCH 0130/1795] Update eb_hooks.py Co-authored-by: Kenneth Hoste --- eb_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9560d84eff..83bdca4cb5 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -276,7 +276,8 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): """ - Pre-test hook for FFTW.MPI: skip failing tests for FFTWMPI 3.3.10 + Pre-test hook for FFTW.MPI: skip failing tests for FFTW.MPI 3.3.10 on neoverse_v1 + cfr. https://github.com/EESSI/software-layer/issues/325 """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == 'aarch64/neoverse_v1': From 190cb999eb7c166f32b9411fe37ab13b53117721 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Sep 2023 17:40:50 +0200 Subject: [PATCH 0131/1795] use more recent easyblock PR to fix Java/11 installation --- eessi-2023.06-eb-4.8.0-system.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml index 6da9c66483..744a52bde6 100644 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ b/eessi-2023.06-eb-4.8.0-system.yml @@ -1,5 +1,7 @@ easyconfigs: - Java-11.eb: + # patch Java binaries/libraries when using alternate sysroot to ensure correct glibc & co are picked up # see https://github.com/easybuilders/easybuild-easyblocks/pull/2557 + # + https://github.com/easybuilders/easybuild-easyblocks/pull/2995 options: - include-easyblocks-from-pr: 2557 + include-easyblocks-from-pr: 2995 From 25f21696dc4b2cab3ba865bc57bec0b1b24f6bb9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Sep 2023 21:29:44 +0200 Subject: [PATCH 0132/1795] filter out .modulerc.lua (for Java) in create_tarball.sh since there's no corresponding directory under software/ --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index b6c72b341d..65f6efc2dc 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -52,7 +52,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} # module files and symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ - | grep -v '/\.wh\.' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ + | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then From 401a35faba7b43ea5a67d752bede3f5b2067709a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 7 Sep 2023 13:34:25 +0200 Subject: [PATCH 0133/1795] {2023.06}[foss/2021a] R v4.1.0 --- eessi-2023.06-eb-4.7.2-2021a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 42a42f2236..8824c6414f 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -24,3 +24,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 options: from-pr: 18348 + - R-4.1.0-foss-2021a.eb From 77920b267a69299af92359f6da2637188c20a357 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 7 Sep 2023 18:20:08 +0200 Subject: [PATCH 0134/1795] {2023.06}[foss/2021a] GDAL v3.3.0 --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.8.0-2021a.yml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.0-2021a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 7667d02c26..48c41c7ce0 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -26,6 +26,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2022a.yml - eessi-2023.06-eb-4.7.2-2022b.yml - eessi-2023.06-eb-4.7.2-system.yml + - eessi-2023.06-eb-4.8.0-2021a.yml - eessi-2023.06-eb-4.8.0-system.yml steps: - name: Check out software-layer repository diff --git a/eessi-2023.06-eb-4.8.0-2021a.yml b/eessi-2023.06-eb-4.8.0-2021a.yml new file mode 100644 index 0000000000..1adf0572d3 --- /dev/null +++ b/eessi-2023.06-eb-4.8.0-2021a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GDAL-3.3.0-foss-2021a.eb From a9caba346620846eba20fa31e8944a63ffa5a55b Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 8 Sep 2023 10:19:51 +0200 Subject: [PATCH 0135/1795] {2023.06}[foss/2021b] OpenFoam v2112 --- eessi-2023.06-eb-4.8.0-2021b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.0-2021b.yml diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml new file mode 100644 index 0000000000..b2202b5e7d --- /dev/null +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OpenFOAM-v2112-foss-2021b.eb From 1df48dcd78e3c166835c4b47bf70a4ee46ec2f5f Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 8 Sep 2023 10:24:45 +0200 Subject: [PATCH 0136/1795] add new easystack file --- .github/workflows/test_eessi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 7667d02c26..dba7403901 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -27,6 +27,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2022b.yml - eessi-2023.06-eb-4.7.2-system.yml - eessi-2023.06-eb-4.8.0-system.yml + - eessi-2023.06-eb-4.8.0-2021b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From 8c057bc7376d4ce201849262e0fa2a1da65104df Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 8 Sep 2023 12:48:58 +0200 Subject: [PATCH 0137/1795] include patch for tsprintf.c --- eessi-2023.06-eb-4.8.0-2021b.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index b2202b5e7d..ea32b3acbb 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,2 +1,6 @@ easyconfigs: - - OpenFOAM-v2112-foss-2021b.eb + - OpenFOAM-v2112-foss-2021b.eb: + #fix for different output of tsprintf.c test from GNU C library <2.36 + #see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 + options: + from-pr: 18746 From 39eda7ba930570afef702cc834632b99422cd403 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 8 Sep 2023 13:12:32 +0200 Subject: [PATCH 0138/1795] include patch for tsprintf.c --- eessi-2023.06-eb-4.8.0-2021b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index ea32b3acbb..ff15917797 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,6 +1,7 @@ easyconfigs: - - OpenFOAM-v2112-foss-2021b.eb: + - MPFR-4.1.0-GCCcore-11.2.0.eb: #fix for different output of tsprintf.c test from GNU C library <2.36 #see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 options: from-pr: 18746 + - OpenFOAM-v2112-foss-2021b.eb From e0fe6b5c26cd9eb0442ffa5a0496d9d422f04090 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 9 Sep 2023 12:29:40 +0200 Subject: [PATCH 0139/1795] improve comment for use of --from-pr for MPFR 4.1.0 --- eessi-2023.06-eb-4.8.0-2021b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index ff15917797..477ba6320c 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,7 +1,7 @@ easyconfigs: - MPFR-4.1.0-GCCcore-11.2.0.eb: - #fix for different output of tsprintf.c test from GNU C library <2.36 - #see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 + # use patch for MPFR 4.1.0 to fix failing tsprintf test with glibc >= 2.37, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 options: from-pr: 18746 - OpenFOAM-v2112-foss-2021b.eb From 515c0ad8896ca68d87d7eb84c796c33cd3003281 Mon Sep 17 00:00:00 2001 From: Maxim Masterov Date: Mon, 11 Sep 2023 14:13:09 +0200 Subject: [PATCH 0140/1795] Add espresso with GCC/11.3.0 and without CUDA to EESSI pilot 2023.06 --- eessi-2023.06-eb-4.7.2-2022a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index d40ddff261..f690d49192 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,3 +1,6 @@ easyconfigs: - GCC-11.3.0 - OpenMPI-4.1.4-GCC-11.3.0.eb + - ESPResSo-4.2.1-foss-2022a.eb: + options: + from-pr: 18486 From 6317155e0afa3a85b678571ea7c8e2187e85ff5b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 12 Sep 2023 14:44:28 +0200 Subject: [PATCH 0141/1795] {2023.06} EasyBuild v4.8.1 --- eessi-2023.06-eb-4.8.0-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml index 744a52bde6..2928f110b0 100644 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ b/eessi-2023.06-eb-4.8.0-system.yml @@ -5,3 +5,6 @@ easyconfigs: # + https://github.com/easybuilders/easybuild-easyblocks/pull/2995 options: include-easyblocks-from-pr: 2995 + - EasyBuild-4.8.1.eb: + options: + from-pr: 18761 From 26f6c02d6075e1079e49afe73947c0ed2e2cd5fa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 11:05:47 +0200 Subject: [PATCH 0142/1795] trigger rebuild of Java/11 with EasyBuild v4.8.1 --- .github/workflows/test_eessi.yml | 1 + EESSI-pilot-install-software.sh | 2 +- eessi-2023.06-eb-4.8.1-system.yml | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 eessi-2023.06-eb-4.8.1-system.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index da1379043a..2db4809b21 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -29,6 +29,7 @@ jobs: - eessi-2023.06-eb-4.8.0-2021a.yml - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml + - eessi-2023.06-eb-4.8.1-system.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 29fbb7aa62..4605b77873 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,7 +172,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2' '4.8.0'; do +for eb_version in '4.7.2' '4.8.0' '4.8.1'; do # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml new file mode 100644 index 0000000000..bab3755330 --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - Java-11.eb: + options: + rebuild: True From ea318e635ff1e6f9b8c6dc4f26003ab3bd5f720e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 14:38:44 +0200 Subject: [PATCH 0143/1795] remove 'rebuild' option for Java/11 with EasyBuild v4.8.1 --- eessi-2023.06-eb-4.8.1-system.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index bab3755330..8966fbfb7a 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -1,4 +1,3 @@ easyconfigs: - - Java-11.eb: - options: - rebuild: True + # rebuilt with EasyBuild v4.8.1 (now wraps around Java/11.0.20) + - Java-11.eb From 3a99d54bad7a1a7b7d6e8154d4b08c81d4fa0ebe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Sep 2023 18:20:22 +0200 Subject: [PATCH 0144/1795] {2023.03} foss/2023a --- .github/workflows/test_eessi.yml | 1 + EESSI-pilot-install-software.sh | 2 +- eessi-2023.06-eb-4.8.1-2023a.yml | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 eessi-2023.06-eb-4.8.1-2023a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index da1379043a..8c1aede4c9 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -29,6 +29,7 @@ jobs: - eessi-2023.06-eb-4.8.0-2021a.yml - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml + - eessi-2023.06-eb-4.8.1-2023a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 29fbb7aa62..4605b77873 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,7 +172,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2' '4.8.0'; do +for eb_version in '4.7.2' '4.8.0' '4.8.1'; do # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml new file mode 100644 index 0000000000..20aafb11fe --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2023a.yml @@ -0,0 +1,3 @@ +easyconfigs: + - GCC-12.3.0 + - foss-2023a From 76d7e9f6a02b3df29c59f89cf07c4a8de5df74c2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Sep 2023 08:58:12 +0200 Subject: [PATCH 0145/1795] install OpenBLAS for foss/2023a via easyconfig PR that adds patch to fix hang in OpenBLAS test suite --- eessi-2023.06-eb-4.8.1-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml index 20aafb11fe..8c2313b8a4 100644 --- a/eessi-2023.06-eb-4.8.1-2023a.yml +++ b/eessi-2023.06-eb-4.8.1-2023a.yml @@ -1,3 +1,8 @@ easyconfigs: - GCC-12.3.0 + - OpenBLAS-0.3.23-GCC-12.3.0: + # add patch to avoid hang when running OpenBLAS test suite, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18790 + options: + from-pr: 18790 - foss-2023a From 1a28493b4fb379198e2c113edaac07a8a869d079 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 18 Sep 2023 15:59:45 +0200 Subject: [PATCH 0146/1795] {2023.06}[foss/2021b] R 4.2.0 --- eessi-2023.06-eb-4.8.1-2021b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.1-2021b.yml diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml new file mode 100644 index 0000000000..a0ef0fa303 --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - R-4.2.0-foss-2021b.eb From 57be9d9ce95a12c3a98a689d6dfcf32e324264eb Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 18 Sep 2023 16:34:38 +0200 Subject: [PATCH 0147/1795] update .github/workflows/test_eessi.yml --- .github/workflows/test_eessi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 2db4809b21..1a67ad29ea 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -30,6 +30,7 @@ jobs: - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml - eessi-2023.06-eb-4.8.1-system.yml + - eessi-2023.06-eb-4.8.1-2021b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From 65305199a81a5be603f37c39465e9250d045d183 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 19 Sep 2023 15:23:50 +0200 Subject: [PATCH 0148/1795] add fix permission issues when copying xvfb-run script in Xvfb easyconfigs --- eessi-2023.06-eb-4.8.1-2021b.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index a0ef0fa303..c2c50ba27b 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -1,2 +1,7 @@ easyconfigs: + - Xvfb-1.20.13-GCCcore-11.2.0.eb: + # enable exec permissions for xvfb-run after copying; + # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works + options: + from-pr: 18834 - R-4.2.0-foss-2021b.eb From 2bc2d2fcb023141ebfc1f71879158c21eda66860 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 19 Sep 2023 15:34:21 +0200 Subject: [PATCH 0149/1795] use PR to fix xvfb-run permission issue for Xvfb 1.20.11 --- eessi-2023.06-eb-4.7.2-2021a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 8824c6414f..aaa23207d1 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -24,4 +24,9 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 options: from-pr: 18348 + - Xvfb-1.20.11-GCCcore-10.3.0.eb: + # fix permission issues for xvfb-run script, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 + options: + from-pr: 18834 - R-4.1.0-foss-2021a.eb From b8b8c2b6e31cfaff3dbe7160148822cbb90c2878 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 19 Sep 2023 15:59:05 +0200 Subject: [PATCH 0150/1795] resolve parsing error of easystack --- eessi-2023.06-eb-4.8.1-2021b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index c2c50ba27b..62f529563a 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -2,6 +2,6 @@ easyconfigs: - Xvfb-1.20.13-GCCcore-11.2.0.eb: # enable exec permissions for xvfb-run after copying; # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works - options: - from-pr: 18834 + options: + from-pr: 18834 - R-4.2.0-foss-2021b.eb From 3a123b3d1c94179daf81297e8b6965fa0bbbc675 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Thu, 21 Sep 2023 06:05:42 +0000 Subject: [PATCH 0151/1795] {2023.06}[foss/2022b] WRF-dmpar V4.4.1 --- eessi-2023.06-eb-4.7.2-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml index 6d15ee2b90..bbf1c54335 100644 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ b/eessi-2023.06-eb-4.7.2-2022b.yml @@ -5,3 +5,4 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - foss-2022b + - WRF-4.4.1-foss-2022b-dmpar.eb From 08231fa4ad4ca994cb119774e809f45815a70734 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Thu, 21 Sep 2023 06:11:26 +0000 Subject: [PATCH 0152/1795] modified the WRF-hook using LooseVersion --- eb_hooks.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 488e53d929..0cc93312c5 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -254,8 +254,13 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): if get_cpu_architecture() == AARCH64: pattern = "Linux x86_64 ppc64le, gfortran" repl = "Linux x86_64 aarch64 ppc64le, gfortran" - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + if LooseVersion(self.version) <= LooseVersion('3.9.0'): + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + + if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'): + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") From 3c7821d9fd4584aac7da791b5db9d4f9b7a3e233 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Thu, 21 Sep 2023 09:06:22 +0000 Subject: [PATCH 0153/1795] modified the WRF-hook --- eb_hooks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0cc93312c5..7689da24bf 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -244,7 +244,6 @@ def pre_configure_hook_metabat_filtered_zlib_dep(self, *args, **kwargs): else: raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!") - def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): """ Pre-configure hook for WRF: @@ -257,7 +256,7 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): if LooseVersion(self.version) <= LooseVersion('3.9.0'): self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) - + if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'): self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) From f3a1f35b5acf46d241ed3f3262e86ae6ed054771 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Sep 2023 17:34:32 +0200 Subject: [PATCH 0154/1795] fix failing build for testthat R extension on top of recent glibc via pre_single_extension hook that replaces SIGSTKSZ with constant value --- eb_hooks.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 488e53d929..95092fbdc2 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -274,6 +274,22 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): self.cfg['testopts'] = "|| echo ignoring failing tests" +def pre_single_extension_hook(ext, *args, **kwargs): + """Main pre-configure hook: trigger custom functions based on software name.""" + if ext.name in PRE_SINGLE_EXTENSION_HOOKS: + PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) + + +def pre_single_extension_testthat(ext, *args, **kwargs): + """ + Pre-extension hook for testthat, to fix build on top of recent glibc. + """ + if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'): + # use constant value instead of SIGSTKSZ for stack size, + # cfr. https://github.com/r-lib/testthat/issues/1373 + https://github.com/r-lib/testthat/pull/1403 + ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " + + PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, @@ -296,3 +312,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): PRE_TEST_HOOKS = { 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, } + +PRE_SINGLE_EXTENSION_HOOKS = { + 'testthat': pre_single_extension_testthat, +} From 74f74ae32465fd2410cde9c448cc6ccf86d01db2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Sep 2023 18:36:27 +0200 Subject: [PATCH 0155/1795] use easybuild/source subdirectory in shared filesystem path as EasyBuild source path, if specified via shared_fs_path in provided configuration --- EESSI-pilot-install-software.sh | 14 ++++++++++++++ bot/build.sh | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4605b77873..ed5319a6db 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -8,10 +8,12 @@ display_help() { echo "usage: $0 [OPTIONS]" + echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds" echo " -g | --generic - instructs script to build for generic architecture target" echo " -h | --help - display this usage information" echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" + echo " --shared-fs-path - path to directory on shared filesystem that can be used" } function copy_build_log() { @@ -70,6 +72,10 @@ while [[ $# -gt 0 ]]; do export build_logs_dir="${2}" shift 2 ;; + --shared-fs-path) + export shared_fs_path="${2}" + shift 2 + ;; -*|--*) echo "Error: Unknown option: $1" >&2 exit 1 @@ -160,6 +166,14 @@ fi echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild +if [ ! -z "${shared_fs_path}" ]; then + shared_eb_sourcepath=${shared_fs_path}/easybuild/sources + echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" + export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} +fi + +${EB} --show-config + echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge diff --git a/bot/build.sh b/bot/build.sh index bd7b2fa8d8..1e7aac49c1 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -64,6 +64,18 @@ else export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" fi +# check if path to directory on shared filesystem is specified, +# and use it as location for source tarballs used by EasyBuild if so +SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") +echo "bot/build.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'" +# if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container +mkdir -p ${SHARED_FS_PATH} +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${SHARED_FS_PATH}" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}" +fi + SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then @@ -167,6 +179,7 @@ if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then INSTALL_SCRIPT_ARGS+=("--generic") fi [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") +[[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) From 24146ee35424ee5f9e3c621611a1df13118d1315 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Sep 2023 20:18:36 +0200 Subject: [PATCH 0156/1795] {2023.06}[system] ReFrame 4.3.3 --- eessi-2023.06-eb-4.8.1-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index 8966fbfb7a..1e9664e10f 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -1,3 +1,6 @@ easyconfigs: # rebuilt with EasyBuild v4.8.1 (now wraps around Java/11.0.20) - Java-11.eb + - ReFrame-4.3.3.eb: + options: + from-pr: 18851 From 8da226c5eec4aa204e3d522cc322f228b85f4aa5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Sep 2023 10:12:32 +0200 Subject: [PATCH 0157/1795] also fix SIGSTKSZ for isoband R package via pre-extension hook --- eb_hooks.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 95092fbdc2..ce84ba655e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -282,7 +282,7 @@ def pre_single_extension_hook(ext, *args, **kwargs): def pre_single_extension_testthat(ext, *args, **kwargs): """ - Pre-extension hook for testthat, to fix build on top of recent glibc. + Pre-extension hook for testthat R package, to fix build on top of recent glibc. """ if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'): # use constant value instead of SIGSTKSZ for stack size, @@ -290,6 +290,16 @@ def pre_single_extension_testthat(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " +def pre_single_extension_isoband(ext, *args, **kwargs): + """ + Pre-extension hook for isoband R package, to fix build on top of recent glibc. + """ + if ext.name == 'isoband' and LooseVersion(ext.version) < LooseVersion('0.2.5'): + # use constant value instead of SIGSTKSZ for stack size in vendored testthat included in isoband sources, + # cfr. https://github.com/r-lib/isoband/commit/6984e6ce8d977f06e0b5ff73f5d88e5c9a44c027 + ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " + + PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, @@ -314,5 +324,6 @@ def pre_single_extension_testthat(ext, *args, **kwargs): } PRE_SINGLE_EXTENSION_HOOKS = { + 'isoband': pre_single_extension_isoband, 'testthat': pre_single_extension_testthat, } From 18c36d51f4857466c123ac8f6a63ec98c843a078 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 25 Sep 2023 11:58:24 +0200 Subject: [PATCH 0158/1795] {2023.06}[foss/2021b] matplotlib v3.4.3 --- eessi-2023.06-eb-4.8.1-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index 62f529563a..a1b9927e97 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -5,3 +5,4 @@ easyconfigs: options: from-pr: 18834 - R-4.2.0-foss-2021b.eb + - matplotlib-3.4.3-foss-2021b.eb From 39ecb81b31568f82863c9c66709aaf7f8eaf13a8 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 25 Sep 2023 12:48:26 +0200 Subject: [PATCH 0159/1795] add yml for known issues --- eessi-2023.06-missing.yml | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 eessi-2023.06-missing.yml diff --git a/eessi-2023.06-missing.yml b/eessi-2023.06-missing.yml new file mode 100644 index 0000000000..cd7967043d --- /dev/null +++ b/eessi-2023.06-missing.yml @@ -0,0 +1,44 @@ + - aarch64/generic: + - foss/2022b: + - OpenBlas: + - issue: 314 + - foss/2021a: + - OpenBlas: + - issue: 314 + - foss/2021b: + - OpenBlas: + - issue: 314 + - foss/2022a: + - OpenBlas: + - issue: 314 + - aarch64/neoverse_n1: + - foss/2022b: + - OpenBlas: + - issue: 314 + - foss/2021a: + - OpenBlas: + - issue: 314 + - foss/2021b: + - OpenBlas: + - issue: 314 + - foss/2022a: + - OpenBlas: + - issue: 314 + - aarch64/neoverse_v1: + - foss/2022a: + - OpenBlas: + - issue: 314 + - foss/2022b: + - OpenBlas: + - issue: 314 + - foss/2021a: + - OpenBlas: + - issue: 314 + - foss/2021b: + - OpenBlas: + - issue: 314 + - SciPy-bundle: + - issue: 318 + - foss/2022a: + - OpenBlas: + - issue: 314 From b3e6b8acddd5c4afcb453d31b00115a7a007338e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 26 Sep 2023 10:03:15 +0200 Subject: [PATCH 0160/1795] Add the option to pass a command to the inspect script, which will get executed in the container. Useful if you want to e.g. run this in a job, but just submit that job manually (not by the bot). --- bot/inspect.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index e5e4df5970..a97a29b614 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -34,6 +34,7 @@ display_help() { echo " to be something like JOB_DIR/previous_tmp/{build,tarball}_step/TARBALL.tgz" echo " and thus determine JOB_DIR from the given path" echo " [default: none]" + echo " -c | --command COMMAND - command to execute inside the container, in the prefix environment" echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" } @@ -62,6 +63,10 @@ while [[ $# -gt 0 ]]; do export https_proxy="${2}" shift 2 ;; + -c|--command) + export run_in_prefix="${2}" + shift 2 + ;; -*|--*) echo "Error: Unknown option: ${1}" >&2 exit 1 @@ -421,10 +426,16 @@ echo "Executing command to start interactive session to inspect build job:" # - setup steps run in 'EESSI-pilot-install-software.sh' # These initializations are combined into a single script that is executed when # the shell in startprefix is started. We set the env variable BASH_ENV here. -echo "./eessi_container.sh ${CMDLINE_ARGS[@]}" -echo " -- ${EESSI_COMPAT_LAYER_DIR}/startprefix" -./eessi_container.sh "${CMDLINE_ARGS[@]}" \ +if [[ -z ${run_in_prefix} ]]; then + echo "./eessi_container.sh ${CMDLINE_ARGS[@]}" + echo " -- ${EESSI_COMPAT_LAYER_DIR}/startprefix" + ./eessi_container.sh "${CMDLINE_ARGS[@]}" \ -- ${EESSI_COMPAT_LAYER_DIR}/startprefix - +else + echo "./eessi_container.sh ${CMDLINE_ARGS[@]}" + echo " -- ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< ${run_in_prefix}" + ./eessi_container.sh "${CMDLINE_ARGS[@]}" \ + -- ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< ${run_in_prefix} +fi exit 0 From 9d96b667e458b6b0bd12808281379527aa570121 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 26 Sep 2023 10:30:49 +0200 Subject: [PATCH 0161/1795] update known issue to make more compact --- eessi-2023.06-known_issues.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 eessi-2023.06-known_issues.yml diff --git a/eessi-2023.06-known_issues.yml b/eessi-2023.06-known_issues.yml new file mode 100644 index 0000000000..b4a48684a5 --- /dev/null +++ b/eessi-2023.06-known_issues.yml @@ -0,0 +1,16 @@ +- aarch64/generic: + OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 + +- aarch64/generic: + - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 +- aarch64/neoverse_n1: + - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 +- aarch64/neoverse_v1: + - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 + - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 +- SciPy-bundle/2021.10-foss-2021b: https://github.com/EESSI/software-layer/issues/318 From ab7759fbd61e6c708ef8bcd696041312ae458599 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 26 Sep 2023 10:34:14 +0200 Subject: [PATCH 0162/1795] remove missing yml --- eessi-2023.06-missing.yml | 44 --------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 eessi-2023.06-missing.yml diff --git a/eessi-2023.06-missing.yml b/eessi-2023.06-missing.yml deleted file mode 100644 index cd7967043d..0000000000 --- a/eessi-2023.06-missing.yml +++ /dev/null @@ -1,44 +0,0 @@ - - aarch64/generic: - - foss/2022b: - - OpenBlas: - - issue: 314 - - foss/2021a: - - OpenBlas: - - issue: 314 - - foss/2021b: - - OpenBlas: - - issue: 314 - - foss/2022a: - - OpenBlas: - - issue: 314 - - aarch64/neoverse_n1: - - foss/2022b: - - OpenBlas: - - issue: 314 - - foss/2021a: - - OpenBlas: - - issue: 314 - - foss/2021b: - - OpenBlas: - - issue: 314 - - foss/2022a: - - OpenBlas: - - issue: 314 - - aarch64/neoverse_v1: - - foss/2022a: - - OpenBlas: - - issue: 314 - - foss/2022b: - - OpenBlas: - - issue: 314 - - foss/2021a: - - OpenBlas: - - issue: 314 - - foss/2021b: - - OpenBlas: - - issue: 314 - - SciPy-bundle: - - issue: 318 - - foss/2022a: - - OpenBlas: - - issue: 314 From e8ddf5ed20de44548e462e09cc3c02e2c672cb36 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 26 Sep 2023 10:35:31 +0200 Subject: [PATCH 0163/1795] update known issue yml --- eessi-2023.06-known_issues.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/eessi-2023.06-known_issues.yml b/eessi-2023.06-known_issues.yml index b4a48684a5..523ca0d712 100644 --- a/eessi-2023.06-known_issues.yml +++ b/eessi-2023.06-known_issues.yml @@ -1,6 +1,3 @@ -- aarch64/generic: - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - - aarch64/generic: - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 From 63352534ae3a1e1cd437e41ecf371436f2cc569d Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:49:37 +0200 Subject: [PATCH 0164/1795] Update eessi-2023.06-known_issues.yml --- eessi-2023.06-known_issues.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-known_issues.yml b/eessi-2023.06-known_issues.yml index 523ca0d712..de01f1ff89 100644 --- a/eessi-2023.06-known_issues.yml +++ b/eessi-2023.06-known_issues.yml @@ -10,4 +10,4 @@ - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 -- SciPy-bundle/2021.10-foss-2021b: https://github.com/EESSI/software-layer/issues/318 + - SciPy-bundle/2021.10-foss-2021b: https://github.com/EESSI/software-layer/issues/318 From 8e4c6b230e285f43e4d4a40b6a28f8ce16691396 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 26 Sep 2023 13:16:00 +0200 Subject: [PATCH 0165/1795] do not load EasyBuild module, rather provide instructions to load it --- bot/inspect.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index a97a29b614..b6d633744f 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -395,18 +395,22 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -eb_version='4.7.2' - -# load EasyBuild module (will be installed if it's not available yet) -source ${TOPDIR}/load_easybuild_module.sh ${eb_version} - -echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." - -echo "Ready for inspection of build job:" +echo_green "Build environment set up with install path ${EASYBUILD_INSTALLPATH}." +echo +echo "The build job can be inspected with the following resources:" echo " - job directory is $HOME (\$HOME), check for slurm-*.out file" -echo " - temporary data of job available at /tmp" -echo " - Note, prefix $EESSI_PREFIX is writable" -echo " - EasyBuild v${eb_version} is available" +echo " - temporary data of the job is available at /tmp" +echo " - note, the prefix $EESSI_PREFIX is writable" +echo +echo "You may want to load an EasyBuild module. The inspect.sh script does not load" +echo "that automatically, because multiple versions might have been used by the job." +echo "Choose an EasyBuild version (see installed versions with 'module avail')" +echo "and run" +echo +echo "source ${TOPDIR}/load_easybuild_module.sh EasyBuild_version" +echo +echo "Note, if you choose a version that is not installed yet, it will be" +echo "installed first." EOF chmod u+x ${RESUME_SCRIPT} From eb29516eabaf54cb867eb6e331bab94516264061 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 26 Sep 2023 13:56:15 +0200 Subject: [PATCH 0166/1795] tweaking of instructions for setting up EasyBuild --- bot/inspect.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index b6d633744f..d35f11cf35 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -404,13 +404,20 @@ echo " - note, the prefix $EESSI_PREFIX is writable" echo echo "You may want to load an EasyBuild module. The inspect.sh script does not load" echo "that automatically, because multiple versions might have been used by the job." -echo "Choose an EasyBuild version (see installed versions with 'module avail')" +echo "Choose an EasyBuild version (see installed versions with 'module avail EasyBuild')" echo "and run" echo -echo "source ${TOPDIR}/load_easybuild_module.sh EasyBuild_version" +echo "source ${TOPDIR}/load_easybuild_module.sh _EasyBuild_version_" +echo +echo "Replace _EasyBuild_version_ with the version you want to use." echo echo "Note, if you choose a version that is not installed yet, it will be" echo "installed first." +echo +echo "If the version you need is already listed with 'module avail', you can" +echo "simply run" +echo +echo "module load EasyBuild/VERSION_YOU_NEED" EOF chmod u+x ${RESUME_SCRIPT} From eb113c2dae51cada092fd32d849e4b709f43cb26 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 26 Sep 2023 14:03:45 +0200 Subject: [PATCH 0167/1795] minor tweaking --- bot/inspect.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/inspect.sh b/bot/inspect.sh index d35f11cf35..5b89705019 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -395,6 +395,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi +echo echo_green "Build environment set up with install path ${EASYBUILD_INSTALLPATH}." echo echo "The build job can be inspected with the following resources:" From acf6b94cd301619311a4245c7c38b524baf7e52f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 20:27:30 +0200 Subject: [PATCH 0168/1795] install OpenBLAS for foss/2022a via easyconfig PR that adds patch to fix detection of Neoverse V1 CPUs + use EasyBuild v4.8.1 for installing foss/2022a --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.7.2-2022a.yml | 7 ------- eessi-2023.06-eb-4.8.1-2022a.yml | 7 +++++++ 3 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 eessi-2023.06-eb-4.8.1-2022a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 1a67ad29ea..abade73d27 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -31,6 +31,7 @@ jobs: - eessi-2023.06-eb-4.8.0-2021b.yml - eessi-2023.06-eb-4.8.1-system.yml - eessi-2023.06-eb-4.8.1-2021b.yml + - eessi-2023.06-eb-4.8.1-2022a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index f4a3da0143..d40ddff261 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,10 +1,3 @@ easyconfigs: - GCC-11.3.0 - OpenMPI-4.1.4-GCC-11.3.0.eb - - CMake-3.23.1-GCCcore-11.3.0.eb: - options: - include-easyblocks-from-pr: 2248 - - CMake-3.24.3-GCCcore-11.3.0.eb: - options: - include-easyblocks-from-pr: 2248 - - foss-2022a diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml new file mode 100644 index 0000000000..8de4c6169d --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -0,0 +1,7 @@ +easyconfigs: + - OpenBLAS-0.3.20-GCC-11.3.0: + # use easyconfig that includes patch to fix detection of Neoverse V1 CPUs, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18870 + options: + from-pr: 18870 + - foss-2022a From c2b6abae4cc882087c4f78972bfa20426c4a11fc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 20:42:34 +0200 Subject: [PATCH 0169/1795] set shared_fs_path for bot on AWS CitC Slurm cluster --- bot/bot-eessi-aws-citc.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index 0a91b12de2..a5e53d82c1 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -44,6 +44,10 @@ command_response_fmt = # name of the job script used for building an EESSI stack build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm +# path to directory on shared filesystem that can be used for sharing data across build jobs +# (for example source tarballs used by EasyBuild) +shared_fs_path = /mnt/shared/home/bot/shared + # Path (directory) to which build logs for (only) failing builds should be copied by bot/build.sh script build_logs_dir = /mnt/shared/bot-build-logs From e092e6be82177ab6b2c89cbf2405494c7d3ef28b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 20:50:43 +0200 Subject: [PATCH 0170/1795] only create & bind build_logs_dir and shared_fs_path if they are specified --- bot/build.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 1e7aac49c1..cf9d69b65f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -57,11 +57,13 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir") echo "bot/build.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'" # if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container -mkdir -p ${BUILD_LOGS_DIR} -if [[ -z ${SINGULARITY_BIND} ]]; then - export SINGULARITY_BIND="${BUILD_LOGS_DIR}" -else - export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" +if [[ ! -z ${BUILD_LOGS_DIR} ]]; then + mkdir -p ${BUILD_LOGS_DIR} + if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BUILD_LOGS_DIR}" + else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" + fi fi # check if path to directory on shared filesystem is specified, @@ -69,11 +71,13 @@ fi SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") echo "bot/build.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'" # if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container -mkdir -p ${SHARED_FS_PATH} -if [[ -z ${SINGULARITY_BIND} ]]; then - export SINGULARITY_BIND="${SHARED_FS_PATH}" -else - export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}" +if [[ ! -z ${SHARED_FS_PATH} ]]; then + mkdir -p ${SHARED_FS_PATH} + if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${SHARED_FS_PATH}" + else + export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}" + fi fi SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") From 4252a16d86ce45ec76078b414e5f919a8e5297d1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Sep 2023 21:21:38 +0200 Subject: [PATCH 0171/1795] use EasyBuild v4.8.1 to install R 4.1.0 with foss/2021a --- .github/workflows/test_eessi.yml | 1 + eessi-2023.06-eb-4.7.2-2021a.yml | 6 ------ eessi-2023.06-eb-4.8.1-2021a.yml | 7 +++++++ 3 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 eessi-2023.06-eb-4.8.1-2021a.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 1a67ad29ea..387c773885 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -30,6 +30,7 @@ jobs: - eessi-2023.06-eb-4.8.0-system.yml - eessi-2023.06-eb-4.8.0-2021b.yml - eessi-2023.06-eb-4.8.1-system.yml + - eessi-2023.06-eb-4.8.1-2021a.yml - eessi-2023.06-eb-4.8.1-2021b.yml steps: - name: Check out software-layer repository diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index aaa23207d1..42a42f2236 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -24,9 +24,3 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 options: from-pr: 18348 - - Xvfb-1.20.11-GCCcore-10.3.0.eb: - # fix permission issues for xvfb-run script, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 - options: - from-pr: 18834 - - R-4.1.0-foss-2021a.eb diff --git a/eessi-2023.06-eb-4.8.1-2021a.yml b/eessi-2023.06-eb-4.8.1-2021a.yml new file mode 100644 index 0000000000..999eb09925 --- /dev/null +++ b/eessi-2023.06-eb-4.8.1-2021a.yml @@ -0,0 +1,7 @@ +easyconfigs: + - Xvfb-1.20.11-GCCcore-10.3.0.eb: + # fix permission issues for xvfb-run script, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 + options: + from-pr: 18834 + - R-4.1.0-foss-2021a.eb From 2333240285bdd82c95a69bf2f0126b43958aa2c8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Sep 2023 13:57:40 +0200 Subject: [PATCH 0172/1795] install ESPResSo-4.2.1-foss-2022a with EasyBuild v4.8.1 --- eessi-2023.06-eb-4.7.2-2022a.yml | 3 --- eessi-2023.06-eb-4.8.1-2022a.yml | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml index f690d49192..d40ddff261 100644 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ b/eessi-2023.06-eb-4.7.2-2022a.yml @@ -1,6 +1,3 @@ easyconfigs: - GCC-11.3.0 - OpenMPI-4.1.4-GCC-11.3.0.eb - - ESPResSo-4.2.1-foss-2022a.eb: - options: - from-pr: 18486 diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 8de4c6169d..c05316bcf7 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -5,3 +5,4 @@ easyconfigs: options: from-pr: 18870 - foss-2022a + - ESPResSo-4.2.1-foss-2022a From 96704bcb314098a602291af5b56988ca08e4586d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Sep 2023 20:39:51 +0200 Subject: [PATCH 0173/1795] add more info to known issues list, and remove entries for aarch64/generic and aarch64/neoverse_n1, since there we don't actually have problems with OpenBLAS test suite at all --- eessi-2023.06-known-issues.yml | 16 ++++++++++++++++ eessi-2023.06-known_issues.yml | 13 ------------- 2 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 eessi-2023.06-known-issues.yml delete mode 100644 eessi-2023.06-known_issues.yml diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml new file mode 100644 index 0000000000..604cc57555 --- /dev/null +++ b/eessi-2023.06-known-issues.yml @@ -0,0 +1,16 @@ +- aarch64/neoverse_v1: + - OpenBLAS/0.3.18-GCC-11.2.0: + - issue: https://github.com/EESSI/software-layer/issues/314 + - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" + - OpenBLAS/0.3.20-GCC-11.3.0: + - issue: https://github.com/EESSI/software-layer/issues/314 + - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" + - OpenBLAS/0.3.21-GCC-12.2.0: + - issue: https://github.com/EESSI/software-layer/issues/314 + - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (344 vs max. 150 on x86_64/*)" + - SciPy-bundle/2021.05-foss-2021a: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "2 failing tests (vs 30554 passed) in scipy test suite" + - SciPy-bundle/2021.10-foss-2021b: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "20 failing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" diff --git a/eessi-2023.06-known_issues.yml b/eessi-2023.06-known_issues.yml deleted file mode 100644 index de01f1ff89..0000000000 --- a/eessi-2023.06-known_issues.yml +++ /dev/null @@ -1,13 +0,0 @@ -- aarch64/generic: - - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 -- aarch64/neoverse_n1: - - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 -- aarch64/neoverse_v1: - - OpenBLAS/0.3.21-GCC-12.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.18-GCC-11.2.0: https://github.com/EESSI/software-layer/issues/314 - - OpenBLAS/0.3.15-GCC-10.3.0: https://github.com/EESSI/software-layer/issues/314 - - SciPy-bundle/2021.10-foss-2021b: https://github.com/EESSI/software-layer/issues/318 From 4c8745b4b29069f7f6273e9aaaed2df1dbc7d9ce Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Sep 2023 20:44:18 +0200 Subject: [PATCH 0174/1795] only increase limit for numerical test failures for OpenBLAS for aarch64/neoverse_v1 --- eb_hooks.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 83bdca4cb5..004746124e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -18,6 +18,8 @@ from distutils.version import LooseVersion +CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' + EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -160,13 +162,14 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): - """Relax number of failing numerical LAPACK tests on Arm 64-bit systems.""" + """Relax number of failing numerical LAPACK tests on for aarch64/neoverse_v1 CPU target.""" + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.name == 'OpenBLAS': - # relax maximum number of failed numerical LAPACK tests on Arm 64-bit systems, - # since the default setting of 150 that works well on x86_64 is a bit too strict + # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target + # since the default setting of 150 that works well on other aarch64 target and x86_64 is a bit too strict # See https://github.com/EESSI/software-layer/issues/314 cfg_option = 'max_failing_lapack_tests_num_errors' - if get_cpu_architecture() == AARCH64: + if cpu_target == CPU_TARGET_NEOVERSE_V1: orig_value = ec[cfg_option] ec[cfg_option] = 400 print_msg("Maximum number of failing LAPACK tests with numerical errors for %s relaxed to %s (was %s)", @@ -270,7 +273,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): In previous versions we were not as strict yet on the numpy/SciPy tests """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'SciPy-bundle' and self.version == '2021.10' and cpu_target == 'aarch64/neoverse_v1': + if self.name == 'SciPy-bundle' and self.version == '2021.10' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" @@ -280,7 +283,7 @@ def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): cfr. https://github.com/EESSI/software-layer/issues/325 """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == 'aarch64/neoverse_v1': + if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" PARSE_HOOKS = { From 4a026c4e249edd877bb5e4416e3ef47456e6baca Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Sep 2023 20:53:25 +0200 Subject: [PATCH 0175/1795] also mention known issue with flaky FFTW test on aarch64/neoverse_v1 --- eessi-2023.06-known-issues.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 604cc57555..421c43f1cb 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,4 +1,10 @@ - aarch64/neoverse_v1: + - FFTW/3.3.10-3.3.10-GCC-11.3.0: + - issue: https://github.com/EESSI/software-layer/issues/325 + - info: "Flaky FFTW tests, random failures" + - FFTW/3.3.10-3.3.10-gompi-2021b: + - issue: https://github.com/EESSI/software-layer/issues/325 + - info: "Flaky FFTW tests, random failures" - OpenBLAS/0.3.18-GCC-11.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" From 941e0d06bd32f1c7f9fc4c65dcc8ae23323f10ee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Sep 2023 08:31:45 +0200 Subject: [PATCH 0176/1795] install OpenBLAS for foss/2023a via easyconfigs PR #18887 --- eessi-2023.06-eb-4.8.1-2023a.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml index 8c2313b8a4..b015676757 100644 --- a/eessi-2023.06-eb-4.8.1-2023a.yml +++ b/eessi-2023.06-eb-4.8.1-2023a.yml @@ -1,8 +1,10 @@ easyconfigs: - GCC-12.3.0 - OpenBLAS-0.3.23-GCC-12.3.0: - # add patch to avoid hang when running OpenBLAS test suite, + # add patch to disable flaky DDRGES3 LAPACK test, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18887; + # also pulls in patch to avoid hang when running OpenBLAS test suite, # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18790 options: - from-pr: 18790 + from-pr: 18887 - foss-2023a From 1fdbb42dfe74b12db717f8165c11bb0fd6c19352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Sep 2023 09:23:45 +0200 Subject: [PATCH 0177/1795] fix typo --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 004746124e..95abb36e7b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -162,7 +162,7 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): - """Relax number of failing numerical LAPACK tests on for aarch64/neoverse_v1 CPU target.""" + """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target.""" cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.name == 'OpenBLAS': # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target From 40e56c005cd76caafe83be63fc7b3accd1b743f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Sep 2023 09:23:54 +0200 Subject: [PATCH 0178/1795] fix typo --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 95abb36e7b..e5999efcb7 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -166,7 +166,7 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.name == 'OpenBLAS': # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target - # since the default setting of 150 that works well on other aarch64 target and x86_64 is a bit too strict + # since the default setting of 150 that works well on other aarch64 targets and x86_64 is a bit too strict # See https://github.com/EESSI/software-layer/issues/314 cfg_option = 'max_failing_lapack_tests_num_errors' if cpu_target == CPU_TARGET_NEOVERSE_V1: From 15985f950c602b24270b63a98de842e53dd5c01a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Sep 2023 15:08:17 +0200 Subject: [PATCH 0179/1795] fix typo --- eessi-2023.06-known-issues.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 421c43f1cb..7d20c99da4 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -7,13 +7,13 @@ - info: "Flaky FFTW tests, random failures" - OpenBLAS/0.3.18-GCC-11.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" + - info: "Increased number of non-numerical failures in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - OpenBLAS/0.3.20-GCC-11.3.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (238 vs max. 150 on x86_64/*)" + - info: "Increased number of non-numerical failures in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - OpenBLAS/0.3.21-GCC-12.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite compared (344 vs max. 150 on x86_64/*)" + - info: "Increased number of non-numerical failures in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" - SciPy-bundle/2021.05-foss-2021a: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "2 failing tests (vs 30554 passed) in scipy test suite" From dd6c4c0f126fc2572378dc1eb6d34a03decbf47d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 29 Sep 2023 19:50:48 +0200 Subject: [PATCH 0180/1795] change the instructions to simply use module load for getting access to EasyBuild --- bot/inspect.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index 5b89705019..637eaea97f 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -406,19 +406,16 @@ echo echo "You may want to load an EasyBuild module. The inspect.sh script does not load" echo "that automatically, because multiple versions might have been used by the job." echo "Choose an EasyBuild version (see installed versions with 'module avail EasyBuild')" -echo "and run" +echo "and simply run" echo -echo "source ${TOPDIR}/load_easybuild_module.sh _EasyBuild_version_" +echo "module load EasyBuild/_VERSION_" echo -echo "Replace _EasyBuild_version_ with the version you want to use." +echo "Replace _VERSION_ with the version you want to use." echo -echo "Note, if you choose a version that is not installed yet, it will be" -echo "installed first." +echo "Note, you can try to install a newer version with 'eb'. The script" +echo "load_easybuild_module.sh cannot be used currently, because it exits at" +echo "the end which also leaves the container." echo -echo "If the version you need is already listed with 'module avail', you can" -echo "simply run" -echo -echo "module load EasyBuild/VERSION_YOU_NEED" EOF chmod u+x ${RESUME_SCRIPT} From 470a22109d96762e9603739f7fda8152a1ad5dcf Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 29 Sep 2023 21:17:31 +0200 Subject: [PATCH 0181/1795] remove note about installing a new EasyBuild --- bot/inspect.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index 637eaea97f..a3b88e5017 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -412,10 +412,6 @@ echo "module load EasyBuild/_VERSION_" echo echo "Replace _VERSION_ with the version you want to use." echo -echo "Note, you can try to install a newer version with 'eb'. The script" -echo "load_easybuild_module.sh cannot be used currently, because it exits at" -echo "the end which also leaves the container." -echo EOF chmod u+x ${RESUME_SCRIPT} From 0f7ee215e3bf43e5ce31b1d34b83c19197f08b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Oct 2023 11:51:28 +0200 Subject: [PATCH 0182/1795] additional build/command permissions --- bot/bot-eessi-aws-citc.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg index a5e53d82c1..8a911182ca 100644 --- a/bot/bot-eessi-aws-citc.cfg +++ b/bot/bot-eessi-aws-citc.cfg @@ -28,7 +28,7 @@ private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws.2 # which GH accounts have the permission to send commands to the bot # if value is left/empty everyone can send commands # value can be a space delimited list of GH accounts -command_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 +command_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 Neves-P # format of the response when processing bot commands command_response_fmt = @@ -100,7 +100,7 @@ submit_command = /usr/bin/sbatch # the label 'bot:build' (apparently this cannot be restricted on GitHub) # if value is left/empty everyone can trigger the build # value can be a space delimited list of GH accounts -build_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 +build_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 Neves-P # template for comment when user who set a label has no permission to trigger build jobs no_build_permission_comment = Label `bot:build` has been set by user `{build_labeler}`, but only users `{build_permission_users}` have permission to trigger the action From 73f7e42a10481b3a6279ccde227afa030726ae13 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 3 Oct 2023 11:00:43 +0200 Subject: [PATCH 0183/1795] add Pillow to easystack --- eessi-2023.06-eb-4.8.1-2021b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index a1b9927e97..ec019ea879 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -5,4 +5,8 @@ easyconfigs: options: from-pr: 18834 - R-4.2.0-foss-2021b.eb + - Pillow-8.3.2-GCCcore-11.2.0.eb: + # avoid that hardcoded paths like /usr/include are used in build commands + options: + from-pr: 18881 - matplotlib-3.4.3-foss-2021b.eb From 2bed6ad9b55239122067ec93bffd86c20a8e34f1 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 3 Oct 2023 17:16:02 +0200 Subject: [PATCH 0184/1795] {2023.06}[foss/2022a] BAMM v2.5.0 --- eessi-2023.06-eb-4.8.1-2022a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 8de4c6169d..d26ae75411 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -5,3 +5,4 @@ easyconfigs: options: from-pr: 18870 - foss-2022a + - BAMM-2.5.0-foss-2022a.eb From 776e3cd8359ff0dd638b029a5a818cc902e00d18 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Oct 2023 19:57:22 +0200 Subject: [PATCH 0185/1795] determine easystack files to process via PR patch file --- EESSI-pilot-install-software.sh | 50 ++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index ed5319a6db..c6c51e7abc 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -41,7 +41,7 @@ function copy_build_log() { echo "- working directory: ${PWD}" >> ${build_log_path} echo "- Slurm job ID: ${SLURM_OUT}" >> ${build_log_path} echo "- EasyBuild version: ${eb_version}" >> ${build_log_path} - echo "- easystack file: ${es}" >> ${build_log_path} + echo "- easystack file: ${easystack_file}" >> ${build_log_path} echo "EasyBuild log file ${build_log} copied to ${build_log_path} (with context appended)" fi @@ -186,36 +186,42 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2' '4.8.0' '4.8.1'; do +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# use PR patch file to determine in which easystack files stuff was added +for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do + + echo -e "Processing easystack file ${easystack_file}...\n\n" + + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." - for es in $(ls eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-*.yml); do - - if [ -f ${es} ]; then - echo_green "Feeding easystack file ${es} to EasyBuild..." + if [ -f ${easystack_file} ]; then + echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - ${EB} --easystack ${TOPDIR}/${es} --robot - ec=$? + ${EB} --easystack ${TOPDIR}/${easystack_file} --robot + ec=$? - # copy EasyBuild log file if EasyBuild exited with an error - if [ ${ec} -ne 0 ]; then - eb_last_log=$(unset EB_VERBOSE; eb --last-log) - # copy to current working directory - cp -a ${eb_last_log} . - echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}" - # copy to build logs dir (with context added) - copy_build_log "${eb_last_log}" "${build_logs_dir}" - fi - - $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es} - else - fatal_error "Easystack file ${es} not found!" + # copy EasyBuild log file if EasyBuild exited with an error + if [ ${ec} -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + # copy to current working directory + cp -a ${eb_last_log} . + echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}" + # copy to build logs dir (with context added) + copy_build_log "${eb_last_log}" "${build_logs_dir}" fi - done + + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} + else + fatal_error "Easystack file ${easystack_file} not found!" + fi done From 46573849c7030c9ec9c2d6b9baa7cc92ec48b7ab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 4 Oct 2023 08:01:26 +0200 Subject: [PATCH 0186/1795] use 'git diff' to determine which easystack files were changed --- EESSI-pilot-install-software.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index c6c51e7abc..4f473eb82f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -186,11 +186,8 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -# assume there's only one diff file that corresponds to the PR patch file -pr_diff=$(ls [0-9]*.diff | head -1) - -# use PR patch file to determine in which easystack files stuff was added -for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do +# use 'git diff' to determine which easystack files were changed +for easystack_file in $(git diff --name-only | grep '^eessi.*yml$'); do echo -e "Processing easystack file ${easystack_file}...\n\n" From 74563775e89363f2c3490ba6a27cc465a4a381d0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 4 Oct 2023 08:40:38 +0200 Subject: [PATCH 0187/1795] {2023.06}[foss/2022a] SciPy-bundle 2022.05 --- eessi-2023.06-eb-4.8.1-2022a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 8de4c6169d..7286e2f3ee 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -5,3 +5,4 @@ easyconfigs: options: from-pr: 18870 - foss-2022a + - SciPy-bundle-2022.05-foss-2022a From e588608054a35ed1dcef99bc8700119932007a0f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 4 Oct 2023 08:55:10 +0200 Subject: [PATCH 0188/1795] fix typo in FFTW version in known issues YAML file --- eessi-2023.06-known-issues.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 7d20c99da4..f2f96aee34 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,8 +1,8 @@ - aarch64/neoverse_v1: - - FFTW/3.3.10-3.3.10-GCC-11.3.0: + - FFTW/3.3.10-GCC-11.3.0: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" - - FFTW/3.3.10-3.3.10-gompi-2021b: + - FFTW/3.3.10-gompi-2021b: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" - OpenBLAS/0.3.18-GCC-11.2.0: From 19dc39cea7f7236c586d588d73bb0b63e27b3c8d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Oct 2023 23:28:50 +0200 Subject: [PATCH 0189/1795] get changed/new files from PR.diff file --- EESSI-pilot-install-software.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4f473eb82f..c6c51e7abc 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -186,8 +186,11 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -# use 'git diff' to determine which easystack files were changed -for easystack_file in $(git diff --name-only | grep '^eessi.*yml$'); do +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# use PR patch file to determine in which easystack files stuff was added +for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do echo -e "Processing easystack file ${easystack_file}...\n\n" From 08420f1407fad943b6612e6c5b339fe4ee10e8c5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Oct 2023 10:57:59 +0200 Subject: [PATCH 0190/1795] remove bot configuration (2023.06) --- bot/bot-eessi-aws-citc.cfg | 238 ------------------------------------- 1 file changed, 238 deletions(-) delete mode 100644 bot/bot-eessi-aws-citc.cfg diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg deleted file mode 100644 index 8a911182ca..0000000000 --- a/bot/bot-eessi-aws-citc.cfg +++ /dev/null @@ -1,238 +0,0 @@ -# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 - -[github] -# replace '123456' with the ID of your GitHub App -app_id = 281041 - -# a short (!) name for your app instance that can be used for example -# when adding/updating a comment to a PR -# (!) a short yet descriptive name is preferred because it appears in -# comments to the PR -# for example, the name could include the name of the cluster the bot -# runs on and the username which runs the bot -# NOTE avoid putting an actual username here as it will be visible on -# potentially publicly accessible GitHub pages. -app_name = eessi-bot-citc-aws - -# replace '12345678' with the ID of the installation of your GitHub App -# (can be derived by creating an event and then checking for the list -# of sent events and its payload either via the Smee channel's web page -# or via the Advanced section of your GitHub App on github.com) -installation_id = 33078935 - -# path to the private key that was generated when the GitHub App was registered -private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws.2023-01-12.private-key.pem - - -[bot_control] -# which GH accounts have the permission to send commands to the bot -# if value is left/empty everyone can send commands -# value can be a space delimited list of GH accounts -command_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 Neves-P - -# format of the response when processing bot commands -command_response_fmt = -
    Updates by the bot instance {app_name} - (click for details) - - {comment_response} - {comment_result} -
    - - -[buildenv] -# name of the job script used for building an EESSI stack -build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm - -# path to directory on shared filesystem that can be used for sharing data across build jobs -# (for example source tarballs used by EasyBuild) -shared_fs_path = /mnt/shared/home/bot/shared - -# Path (directory) to which build logs for (only) failing builds should be copied by bot/build.sh script -build_logs_dir = /mnt/shared/bot-build-logs - -# The container_cachedir may be used to reuse downloaded container image files -# across jobs. Thus, jobs can more quickly launch containers. -container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir - -# it may happen that we need to customize some CVMFS configuration -# the value of cvmfs_customizations is a dictionary which maps a file -# name to an entry that needs to be added to that file -# cvmfs_customizations = {} - -# if compute nodes have no internet connection, we need to set http(s)_proxy -# or commands such as pip3 cannot download software from package repositories -# for example, the temporary EasyBuild is installed via pip3 first -# http_proxy = http://PROXY_DNS:3128/ -# https_proxy = http://PROXY_DNS:3128/ - -# directory under which the bot prepares directories per job -# structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR -jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs - -# configure environment -# list of comma-separated modules to be loaded by build_job_script -# useful/needed if some tool is not provided as system-wide package -# (read by bot and handed over to build_job_script via parameter -# --load-modules) -# load_modules = - -# PATH to temporary directory on build node ... ends up being used for -# for example, EESSI_TMPDIR --> /tmp/$USER/EESSI -# escaping variables with '\' delays expansion to the start of the -# build_job_script; this can be used for referencing environment -# variables that are only set inside a Slurm job -local_tmp = /tmp/$USER/EESSI - -# parameters to be added to all job submissions -# NOTE do not quote parameter string. Quotes are retained when reading in config and -# then the whole 'string' is recognised as a single parameter. -# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs -# to be initialised as if it is for a login shell. -# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed -# note: 16 cores which corresponds to *.4xlarge node types, see also arch_target_map -slurm_params = --hold --time=24:0:0 --nodes=1 --ntasks-per-node=16 - -# full path to the job submission command -submit_command = /usr/bin/sbatch - -# which GH account has the permission to trigger the build (by setting -# the label 'bot:build' (apparently this cannot be restricted on GitHub) -# if value is left/empty everyone can trigger the build -# value can be a space delimited list of GH accounts -build_permission = boegel trz42 bedroge laraPPr ocaisa casparvl satishskamath maxim-masterov TopRichard xinan1911 Neves-P - -# template for comment when user who set a label has no permission to trigger build jobs -no_build_permission_comment = Label `bot:build` has been set by user `{build_labeler}`, but only users `{build_permission_users}` have permission to trigger the action - - -[deploycfg] -# script for uploading built software packages -tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging - -# URL to S3/minio bucket -# if attribute is set, bucket_base will be constructed as follows -# bucket_base=${endpoint_url}/${bucket_name} -# otherwise, bucket_base will be constructed as follows -# bucket_base=https://${bucket_name}.s3.amazonaws.com -# - The former variant is used for non AWS S3 services, eg, minio, or when -# the bucket name is not provided in the hostname (see latter case). -# - The latter variant is used for AWS S3 services. -# endpoint_url = URL_TO_S3_SERVER - -# bucket name -bucket_name = eessi-staging-2023.06 - -# upload policy: defines what policy is used for uploading built artefacts -# to an S3 bucket -# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) -# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) -# only upload the latest built artefact -# 'once' : only once upload any built artefact for the build target -# 'none' : do not upload any built artefacts -upload_policy = latest - -# which GH account has the permission to trigger the deployment (by setting -# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) -# if value is left/empty everyone can trigger the deployment -# value can be a space delimited list of GH accounts -deploy_permission = boegel trz42 bedroge ocaisa casparvl - -# template for comment when user who set a label has no permission to trigger deploying tarballs -no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_labeler}`, but only users `{deploy_permission_users}` have permission to trigger the action - - -[architecturetargets] -# defines both for which architectures the bot will build -# and what submission parameters shall be used -# 5 c4.2xlarge haswell 8 vCPU, 15 GiB RAM (1 + generic) -# 2 c4.4xlarge haswell 16 vCPU, 30 GiB RAM -# 5 c5.2xlarge skylake_avx512 8 vCPU, 16 GiB RAM (1) -# 1 c5.4xlarge skylake_avx512 16 vCPU, 32 GiB RAM -# 5 c5a.2xlarge zen2 8 vCPU, 16 GiB RAM (1) -# 1 c5a.4xlarge zen2 16 vCPU, 32 GiB RAM -# 5 c5d.2xlarge skylake_avx512 8 vCPU, 16 GiB RAM + 200 GB NVMe -# 5 c6a.2xlarge zen3 8 vCPU, 16 GiB RAM (1) -# 1 c6a.4xlarge zen3 16 vCPU, 32 GiB RAM -# 5 c6g.2xlarge neoverse_n1 8 vCPU, 16 GiB RAM (1 + generic) -# 2 c6g.4xlarge neoverse_n1 16 vCPU, 32 GiB RAM -# 1 c6g.8xlarge neoverse_n1 32 vCPU, 64 GiB RAM -# 1 c6i.2xlarge cascadelake 8 vCPU, 16 GiB RAM (1) -# 5 c7g.2xlarge neoverse_v1 8 vCPU, 16 GiB RAM (1) -# 1 c7g.4xlarge neoverse_v1 16 vCPU, 32 GiB RAM -# larger instances (*.4xlarge => 16 cores, 32GB RAM) -arch_target_map = { - "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", - "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", - "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", - "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", - "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", - "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", - "linux/aarch64/neoverse_n1" : "--constraint shape=c6g.4xlarge", - "linux/aarch64/neoverse_v1" : "--constraint shape=c7g.4xlarge" } - -[repo_targets] -# defines for which repository a arch_target should be build for -# -# only building for repository EESSI-pilot -repo_target_map = { - "linux/x86_64/generic" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/intel/haswell" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/intel/skylake_avx512" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/amd/zen2" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/x86_64/amd/zen3" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/generic" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/neoverse_n1" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"], - "linux/aarch64/neoverse_v1" : ["eessi-2021.12","eessi-2023.06-compat","eessi-2023.06-software"] } - -# points to definition of repositories (default EESSI-pilot defined by build container) -repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/repos - -# configuration for event handler which receives events from a GitHub repository. -[event_handler] -# path to the log file to log messages for event handler -log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log - - -[job_manager] -# path to the log file to log messages for job manager -log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log - -# directory where job manager stores information about jobs to be tracked -# e.g. as symbolic link JOBID -> directory to job -job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs - -# full path to the job status checking command -poll_command = /usr/bin/squeue - -# polling interval in seconds -poll_interval = 60 - -# full path to the command for manipulating existing jobs -scontrol_command = /usr/bin/scontrol - - -# variable 'comment' under 'submitted_job_comments' should not be changed as there are regular expression patterns matching it -[submitted_job_comments] -initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` -awaits_release = job id `{job_id}` awaits release by job manager - - -[new_job_comments] -awaits_launch = job awaits launch by Slurm scheduler - - -[running_job_comments] -running_job = job `{job_id}` is running - - -[finished_job_comments] -success = :grin: SUCCESS tarball `{tarball_name}` ({tarball_size} GiB) in job dir -failure = :cry: FAILURE -no_slurm_out = No slurm output `{slurm_out}` in job dir -slurm_out = Found slurm output `{slurm_out}` in job dir -missing_modules = Slurm output lacks message "No missing modules!". -no_tarball_message = Slurm output lacks message about created tarball. -no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. -multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_unknown_fmt =
    :shrug: UNKNOWN _(click triangle for detailed information)_
    • Job results file `{filename}` does not exist in job directory or reading it failed.
    • No artefacts were found/reported.
    From 2f05bc9c2ff7605663fe7ecf99f5dcd75514b3ee Mon Sep 17 00:00:00 2001 From: TopRichard Date: Thu, 5 Oct 2023 09:26:25 +0000 Subject: [PATCH 0191/1795] added reference to easyblock 3012 --- eessi-2023.06-eb-4.7.2-2022b.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml index bbf1c54335..2998266677 100644 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ b/eessi-2023.06-eb-4.7.2-2022b.yml @@ -5,4 +5,6 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - foss-2022b - - WRF-4.4.1-foss-2022b-dmpar.eb + - WRF-4.4.1-foss-2022b-dmpar.eb: + options: + include-easyblocks-from-pr: 3012 From 38e6af4b0f1009ec1997823ccaf1fa6c72587128 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Oct 2023 15:49:11 +0200 Subject: [PATCH 0192/1795] show active EasyBuild configuration when checking for missing installations --- check_missing_installations.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 3627d1d0b5..e927f14701 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -22,6 +22,9 @@ source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild +echo ">> Active EasyBuild configuration when checking for missing installations:" +${EB:-eb} --show-config + echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out ${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} From c0677ada268ee3c635b7e0dc3748c97609bae5fd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 6 Oct 2023 08:55:35 +0200 Subject: [PATCH 0193/1795] clarify why easyblocks PR #3012 is needed, only use it for netCDF --- eessi-2023.06-eb-4.7.2-2022b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml index 2998266677..b97be34dcf 100644 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ b/eessi-2023.06-eb-4.7.2-2022b.yml @@ -5,6 +5,9 @@ easyconfigs: options: include-easyblocks-from-pr: 2248 - foss-2022b - - WRF-4.4.1-foss-2022b-dmpar.eb: + - netCDF-4.9.0-gompi-2022b.eb: + # use updated CMakeMake easyblock to avoid that -DCMAKE_SKIP_RPATH=ON is used, which breaks the netCDF test step + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3012 options: include-easyblocks-from-pr: 3012 + - WRF-4.4.1-foss-2022b-dmpar.eb From c244bdc92dc6b941ea661c600eb40496946d1501 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 6 Oct 2023 09:20:14 +0200 Subject: [PATCH 0194/1795] simplify CI workflow to check for missing installations, just loop over easystack files per CPU target --- .github/workflows/test_eessi.yml | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 69e988ed02..6dfd78e428 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -20,20 +20,6 @@ jobs: - x86_64/intel/haswell - x86_64/intel/skylake_avx512 - x86_64/generic - EASYSTACK_FILE: - - eessi-2023.06-eb-4.7.2-2021a.yml - - eessi-2023.06-eb-4.7.2-2021b.yml - - eessi-2023.06-eb-4.7.2-2022a.yml - - eessi-2023.06-eb-4.7.2-2022b.yml - - eessi-2023.06-eb-4.7.2-system.yml - - eessi-2023.06-eb-4.8.0-2021a.yml - - eessi-2023.06-eb-4.8.0-system.yml - - eessi-2023.06-eb-4.8.0-2021b.yml - - eessi-2023.06-eb-4.8.1-system.yml - - eessi-2023.06-eb-4.8.1-2021a.yml - - eessi-2023.06-eb-4.8.1-2021b.yml - - eessi-2023.06-eb-4.8.1-2022a.yml - - eessi-2023.06-eb-4.8.1-2023a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 @@ -55,7 +41,12 @@ jobs: export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" - ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} + for easystack_file in $(ls eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "check missing installations for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + done - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | @@ -66,15 +57,18 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify easystack file by adding a missing package (GCC/8.3.0)" - echo " - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}} - tail -n 5 ${{matrix.EASYSTACK_FILE}} + # create dummy easystack file with a single entry (something that is not installed in EESSI) + easystack_file="test.yml" + echo "easyconfigs:" > ${easystack_file} + echo " - GCC-8.3.0:" >> ${easystack_file} + echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):" + cat ${easystack_file} # note, check_missing_installations.sh exits 1 if a package was # missing, which is intepreted as false (exit code based, not # boolean logic), hence when the script exits 0 if no package was # missing it is interpreted as true, thus the test did not capture # the missing package - if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then + if ./check_missing_installations.sh ${easystack_file}; then echo "did NOT capture missing package; test FAILED" exit 1 else From 093c01d04c9279f2c0e65d130f5fce2e668c33ee Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 10 Oct 2023 08:27:03 +0200 Subject: [PATCH 0195/1795] {2023.06}[foss/2021b] LAMMPS 23Jun2022 --- eessi-2023.06-eb-4.8.1-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index 62f529563a..90a62ba2d6 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -5,3 +5,4 @@ easyconfigs: options: from-pr: 18834 - R-4.2.0-foss-2021b.eb + - LAMMPS-23Jun2022-foss-2021b-kokkos.eb From afcd92dff998e039ad8a2d7e5682604f6b1f7ad2 Mon Sep 17 00:00:00 2001 From: Maxim Date: Tue, 10 Oct 2023 11:40:26 +0200 Subject: [PATCH 0196/1795] Update eessi-2023.06-eb-4.8.1-2022a.yml --- eessi-2023.06-eb-4.8.1-2022a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index c05316bcf7..646f28533d 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -6,3 +6,5 @@ easyconfigs: from-pr: 18870 - foss-2022a - ESPResSo-4.2.1-foss-2022a + options: + from-pr: 18963 From d877d7076790e2bd671604df36f2016ac86cbe09 Mon Sep 17 00:00:00 2001 From: maxim-masterov Date: Tue, 10 Oct 2023 11:53:12 +0200 Subject: [PATCH 0197/1795] Add missing colon --- eessi-2023.06-eb-4.8.1-2022a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 3c27495cd1..c9a249872b 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -5,7 +5,7 @@ easyconfigs: options: from-pr: 18870 - foss-2022a - - ESPResSo-4.2.1-foss-2022a + - ESPResSo-4.2.1-foss-2022a: options: from-pr: 18963 - SciPy-bundle-2022.05-foss-2022a From f8b4f42baa9451a2f5fa50d28fdc009e875d414b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 10 Oct 2023 16:47:56 +0200 Subject: [PATCH 0198/1795] add ESPResSo to end of easystack file to preserve installation order --- eessi-2023.06-eb-4.8.1-2022a.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index c9a249872b..8317a40b7a 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -5,8 +5,8 @@ easyconfigs: options: from-pr: 18870 - foss-2022a + - SciPy-bundle-2022.05-foss-2022a + - BAMM-2.5.0-foss-2022a.eb - ESPResSo-4.2.1-foss-2022a: options: from-pr: 18963 - - SciPy-bundle-2022.05-foss-2022a - - BAMM-2.5.0-foss-2022a.eb From 89cc228e5fc0a629a422134e67693b1a60f2fe25 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Wed, 11 Oct 2023 06:42:48 +0000 Subject: [PATCH 0199/1795] {2023.06}[foss/2022a] matplotlib v3.5.2 --- eb_hooks.py | 14 +++++++++++++- eessi-2023.06-eb-4.8.1-2022a.yml | 8 +++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index dbb8415541..2bdead4b73 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -54,13 +54,25 @@ def get_rpath_override_dirs(software_name): return rpath_injection_dirs +def set_Pillow_envvars(ec): + """Get an EESSI_CPATH environment variable from the environment""" + EESSI_CPATH = os.getenv('EESSI_EPREFIX') + '/usr/include' + EESSI_LIB_PATH = os.getenv('EESSI_EPREFIX') + '/usr/lib64' + if ec.name == 'Pillow': + os.environ['CPATH'] = os.pathsep + EESSI_CPATH + os.environ['LIBRARY_PATH'] = os.pathsep + EESSI_LIB_PATH + print_msg("NOTE: For Pillow which has Szip as a dependancy, CPATH has been set to %s", os.getenv('CPATH')) + print_msg("NOTE: For Pillow which has Szip as a dependancy, LIBRARY_PATH has been set to %s", os.getenv('LIBRARY_PATH')) + ec.log.info("NOTE: For Pillow which has Szip as a dependancy, CPATH has been set to %s", os.getenv('CPATH')) + ec.log.info("NOTE: For Pillow which has Szip as a dependancy, LIBRARY_PATH has been set to %s", os.getenv('LIBRARY_PATH')) + def parse_hook(ec, *args, **kwargs): """Main parse hook: trigger custom functions based on software name.""" # determine path to Prefix installation in compat layer via $EPREFIX eprefix = get_eessi_envvar('EPREFIX') - + set_Pillow_envvars(ec) if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 97704ebe30..87ca700d96 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -6,4 +6,10 @@ easyconfigs: from-pr: 18870 - foss-2022a - SciPy-bundle-2022.05-foss-2022a - - BAMM-2.5.0-foss-2022a.eb \ No newline at end of file + - BAMM-2.5.0-foss-2022a.eb + - Pillow-9.1.1-GCCcore-11.3.0.eb: + # avoid that hardcoded paths like /usr/include are used in build commands + # Uses a hook to modify the hardcoded LIBRARY and Header paths. + options: + from-pr: 18881 + - matplotlib-3.5.2-foss-2022a.eb From eae083ff82f540f056c4e7a3a732133d8813314c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Oct 2023 08:58:40 +0200 Subject: [PATCH 0200/1795] add hook to ignore failing tests in ESPResSo v4.2.1 + add corresponding tracker issue to list of known issues --- eb_hooks.py | 35 +++++++++++++++++++++++----------- eessi-2023.06-known-issues.yml | 24 +++++++++++++++-------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index dbb8415541..31f2b9588d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -247,6 +247,7 @@ def pre_configure_hook_metabat_filtered_zlib_dep(self, *args, **kwargs): else: raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!") + def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): """ Pre-configure hook for WRF: @@ -266,11 +267,32 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") + def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: PRE_TEST_HOOKS[self.name](self, *args, **kwargs) + +def pre_test_hook_ignore_failing_tests_ESPResSo(self, *args, **kwargs): + """ + Pre-test hook for ESPResSo: skip failing tests, tests frequently timeout due to known bugs in ESPResSo v4.2.1 + cfr. https://github.com/EESSI/software-layer/issues/363 + """ + if self.name == 'ESPResSo' and self.version == '4.2.1': + self.cfg['testopts'] = "|| echo 'ignoring failing tests (probably due to timeouts)'" + + +def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): + """ + Pre-test hook for FFTW.MPI: skip failing tests for FFTW.MPI 3.3.10 on neoverse_v1 + cfr. https://github.com/EESSI/software-layer/issues/325 + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg['testopts'] = "|| echo ignoring failing tests" + + def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): """ Pre-test hook for SciPy-bundle: skip failing tests for SciPy-bundle 2021.10 (currently the only version that is failing). @@ -307,16 +329,6 @@ def pre_single_extension_isoband(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " -def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): - """ - Pre-test hook for FFTW.MPI: skip failing tests for FFTW.MPI 3.3.10 on neoverse_v1 - cfr. https://github.com/EESSI/software-layer/issues/325 - """ - cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'FFTW.MPI' and self.version == '3.3.10' and cpu_target == CPU_TARGET_NEOVERSE_V1: - self.cfg['testopts'] = "|| echo ignoring failing tests" - - PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, @@ -337,8 +349,9 @@ def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): } PRE_TEST_HOOKS = { - 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, + 'ESPResSo': pre_test_hook_ignore_failing_tests_ESPResSo, 'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI, + 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, } PRE_SINGLE_EXTENSION_HOOKS = { diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index f2f96aee34..846e862983 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,22 +1,30 @@ +- x86_64/*: + - ESPResSo-4.2.1-foss-2022a: + - issue: https://github.com/EESSI/software-layer/issues/363 + - info: "Failing tests in ESPResSo v4.2.1 due to timeouts" +- aarch64/*: + - ESPResSo-4.2.1-foss-2022a: + - issue: https://github.com/EESSI/software-layer/issues/363 + - info: "Failing tests in ESPResSo v4.2.1 due to timeouts" - aarch64/neoverse_v1: - - FFTW/3.3.10-GCC-11.3.0: + - FFTW-3.3.10-GCC-11.3.0: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" - - FFTW/3.3.10-gompi-2021b: + - FFTW-3.3.10-gompi-2021b: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" - - OpenBLAS/0.3.18-GCC-11.2.0: + - OpenBLAS-0.3.18-GCC-11.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - info: "Increased number of non-numerical failures in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - - OpenBLAS/0.3.20-GCC-11.3.0: + - OpenBLAS-0.3.20-GCC-11.3.0: - issue: https://github.com/EESSI/software-layer/issues/314 - info: "Increased number of non-numerical failures in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - - OpenBLAS/0.3.21-GCC-12.2.0: + - OpenBLAS-0.3.21-GCC-12.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - info: "Increased number of non-numerical failures in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" - - SciPy-bundle/2021.05-foss-2021a: + - SciPy-bundle-2021.05-foss-2021a: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "2 failing tests (vs 30554 passed) in scipy test suite" - - SciPy-bundle/2021.10-foss-2021b: + - SciPy-bundle-2021.10-foss-2021b: - issue: https://github.com/EESSI/software-layer/issues/318 - - info: "20 failing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" + - info: "20 ailing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" From 8779e8b350bf2379a36b7671d150fce5f2f8a262 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Oct 2023 12:44:30 +0200 Subject: [PATCH 0201/1795] filter out YAML files that are used to keep track of known issues and missing installations --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index c6c51e7abc..9586abde1d 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -190,7 +190,7 @@ fi pr_diff=$(ls [0-9]*.diff | head -1) # use PR patch file to determine in which easystack files stuff was added -for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do +for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$' | egrep -v 'known-issues|missing'); do echo -e "Processing easystack file ${easystack_file}...\n\n" From cd5cea918c26e67174f278df80a43ef9a2cff045 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Oct 2023 16:30:53 +0200 Subject: [PATCH 0202/1795] fix missing latter in info message in list of known issues --- eessi-2023.06-known-issues.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 846e862983..6cb464cd20 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -27,4 +27,4 @@ - info: "2 failing tests (vs 30554 passed) in scipy test suite" - SciPy-bundle-2021.10-foss-2021b: - issue: https://github.com/EESSI/software-layer/issues/318 - - info: "20 ailing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" + - info: "20 failing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" From ea75facab305f527b81aebff51b593255b54adb3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Oct 2023 21:45:35 +0200 Subject: [PATCH 0203/1795] {2023.06}[2023a] X11 20230603 --- eessi-2023.06-eb-4.8.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml index b015676757..c766815934 100644 --- a/eessi-2023.06-eb-4.8.1-2023a.yml +++ b/eessi-2023.06-eb-4.8.1-2023a.yml @@ -8,3 +8,4 @@ easyconfigs: options: from-pr: 18887 - foss-2023a + - X11-20230603-GCCcore-12.3.0 From 6b0bcb9c00c4768614f5b9087e7f3ce2777e44d3 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 13 Oct 2023 13:59:44 +0200 Subject: [PATCH 0204/1795] add TBB solution for Aarch64 --- eessi-2023.06-eb-4.8.1-2021b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index 90a62ba2d6..1414df5e44 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -5,4 +5,7 @@ easyconfigs: options: from-pr: 18834 - R-4.2.0-foss-2021b.eb - - LAMMPS-23Jun2022-foss-2021b-kokkos.eb + - LAMMPS-23Jun2022-foss-2021b-kokkos.eb: + # TBB is an optional dependency when building on Intel arch + options: + from-pr: 19000 From e5b33b5111447bb663e2cec2ad136c79eaa09b9c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 13 Oct 2023 22:40:31 +0200 Subject: [PATCH 0205/1795] fix info on known issue for OpenBLAS on aarch64/neoverse_v1: increased number of *numerical* errors --- eessi-2023.06-known-issues.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 6cb464cd20..ade1c069df 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -15,13 +15,13 @@ - info: "Flaky FFTW tests, random failures" - OpenBLAS-0.3.18-GCC-11.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" + - info: "Increased number of numerical errors in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - OpenBLAS-0.3.20-GCC-11.3.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" + - info: "Increased number of numerical errors in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - OpenBLAS-0.3.21-GCC-12.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of non-numerical failures in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" + - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" - SciPy-bundle-2021.05-foss-2021a: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "2 failing tests (vs 30554 passed) in scipy test suite" From 6cc2e3ed53708bc07da7932adc0326faac9c670f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 14 Oct 2023 21:09:25 +0200 Subject: [PATCH 0206/1795] first version of bot/test.sh script --- bot/test.sh | 224 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100755 bot/test.sh diff --git a/bot/test.sh b/bot/test.sh new file mode 100755 index 0000000000..6d217caf92 --- /dev/null +++ b/bot/test.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# +# script to run tests or the test suite for the whole EESSI software layer or +# just what has been built in a job. Intended use is that it is called +# at the end of a (batch) job running on a compute node. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# + assumption for the build step (as run through bot/build.sh which is provided +# in this repository too) +# - working directory has been prepared by the bot with a checkout of a +# pull request (OR by some other means) +# - the working directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited +# - the directory may contain any additional files referenced in job.cfg +# + assumptions for the test step +# - temporary storage is still available +# example +# Using /localscratch/9640860/NESSI/eessi.x765Dd8mFh as tmp directory (to resume session add '--resume /localscratch/9640860/NESSI/eessi.x765Dd8mFh'). +# - run test-suite.sh inside build container using tmp storage from build step +# plus possibly additional settings (repo, etc.) +# - needed setup steps may be similar to bot/inspect.sh (PR#317) + +# stop as soon as something fails +set -e + +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +HOST_ARCH=$(uname -m) + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "bot/test.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/test.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} + +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} +echo "bot/test.sh: HTTP_PROXY='${HTTP_PROXY}'" + +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} +echo "bot/test.sh: HTTPS_PROXY='${HTTPS_PROXY}'" + +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") +echo "bot/test.sh: LOCAL_TMP='${LOCAL_TMP}'" +# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided + +# check if path to copy build logs to is specified, so we can copy build logs for failing builds there +BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir") +echo "bot/test.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'" +# if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container +if [[ ! -z ${BUILD_LOGS_DIR} ]]; then + mkdir -p ${BUILD_LOGS_DIR} + if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BUILD_LOGS_DIR}" + else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}" + fi +fi + +# check if path to directory on shared filesystem is specified, +# and use it as location for source tarballs used by EasyBuild if so +SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") +echo "bot/test.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'" +# if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container +if [[ ! -z ${SHARED_FS_PATH} ]]; then + mkdir -p ${SHARED_FS_PATH} + if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${SHARED_FS_PATH}" + else + export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}" + fi +fi + +SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") +echo "bot/test.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + # make sure that separate directories are used for different CPU families + SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH} + export SINGULARITY_CACHEDIR +fi + +# try to determine tmp directory from build job +RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -1 | awk '{print $2}') + +if [[ -z ${RESUME_DIR} ]]; then + echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " + # replace any env variable in ${LOCAL_TMP} with its + # current value (e.g., a value that is local to the job) + STORAGE=$(envsubst <<< ${LOCAL_TMP}) + echo "'${STORAGE}'" + + # make sure ${STORAGE} exists + mkdir -p ${STORAGE} + + # make sure the base tmp storage is unique + JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) + echo "bot/test.sh: created unique base tmp storage directory at ${JOB_STORAGE}" + + RESUME_TGZ=${PWD}/previous_tmp/build_step/$(ls previous_tmp/build_step) + if [[ -z ${RESUME_TGZ} ]]; then + echo "bot/test.sh: no information about tmp directory and tarball of build step; --> giving up" + exit 2 + fi +fi + +# obtain list of modules to be loaded +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") +echo "bot/test.sh: LOAD_MODULES='${LOAD_MODULES}'" + +# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND +CONTAINER=$(cfg_get_value "repository" "container") +export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" +mkdir -p ${SINGULARITY_TMPDIR} + +# load modules if LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/test.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/test.sh: no modules to be loaded" +fi + +# determine repository to be used from entry .repository in ${JOB_CFG_FILE} +REPOSITORY=$(cfg_get_value "repository" "repo_id") +EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") +export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} +echo "bot/test.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" + +# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# next script (eessi_container.sh) makes use of it via sourcing init scripts +# (e.g., init/eessi_defaults or init/minimal_eessi_env) +export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" + +# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} +# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down +# "source init/eessi_defaults" via sourcing init/minimal_eessi_env +export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" + +# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# fallbacks: +# - ${CPU_TARGET} handed over from bot +# - left empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") +EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} +export EESSI_SOFTWARE_SUBDIR_OVERRIDE +echo "bot/test.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" + +# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) +EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") +export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} +echo "bot/test.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" + +# prepare arguments to eessi_container.sh common to build and tarball steps +declare -a COMMON_ARGS=() +COMMON_ARGS+=("--verbose") +COMMON_ARGS+=("--access" "rw") +COMMON_ARGS+=("--mode" "run") +[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") +[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") +[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") +[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") + +# make sure to use the same parent dir for storing tarballs of tmp +PREVIOUS_TMP_DIR=${PWD}/previous_tmp + +# prepare directory to store tarball of tmp for build step +TARBALL_TMP_TEST_STEP_DIR=${PREVIOUS_TMP_DIR}/test_step +mkdir -p ${TARBALL_TMP_TEST_STEP_DIR} + +# prepare arguments to eessi_container.sh specific to test step +declare -a TEST_STEP_ARGS=() +TEST_STEP_ARGS+=("--save" "${TARBALL_TMP_TEST_STEP_DIR}") + +if [[ -z ${RESUME_DIR} ]]; then + TEST_STEP_ARGS+=("--storage" "${STORAGE}") + TEST_STEP_ARGS+=("--resume" "${RESUME_TGZ}") +else + TEST_STEP_ARGS+=("--resume" "${RESUME_DIR}") +fi + +# prepare arguments to test_suite.sh (specific to test step) +declare -a TEST_SUITE_ARGS=() +# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then +# TEST_SUITE_ARGS+=("--generic") +# fi +# [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") +# [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") + +# create tmp file for output of build step +test_outerr=$(mktemp test.outerr.XXXX) + +echo "Executing command to build software:" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}" +echo " -- ./test_suite.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}" +./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \ + -- ./test_suite.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr} + +exit 0 From 9a32dc629b34e4bbc91fda349198611a6a96a082 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 15 Oct 2023 20:29:14 +0200 Subject: [PATCH 0207/1795] scripts to run tests after a build job has finished --- run_tests.sh | 4 ++ test_suite.sh | 184 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 run_tests.sh create mode 100644 test_suite.sh diff --git a/run_tests.sh b/run_tests.sh new file mode 100644 index 0000000000..ccbc751f22 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,4 @@ +#!/bin/bash +base_dir=$(dirname $(realpath $0)) +source ${base_dir}/init/eessi_defaults +./run_in_compat_layer_env.sh ./test_suite.sh "$@" diff --git a/test_suite.sh b/test_suite.sh new file mode 100644 index 0000000000..fcaaa339d2 --- /dev/null +++ b/test_suite.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# +# Run sanity check for all requested modules (and built dependencies)? +# get ec from diff +# set up EasyBuild +# run `eb --sanity-only ec` + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " -g | --generic - instructs script to test for generic architecture target" + echo " -h | --help - display this usage information" + echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" + echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" +} + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -g|--generic) + EASYBUILD_OPTARCH="GENERIC" + shift + ;; + -h|--help) + display_help # Call your function + # no shifting needed here, we're done. + exit 0 + ;; + -x|--http-proxy) + export http_proxy="$2" + shift 2 + ;; + -y|--https-proxy) + export https_proxy="$2" + shift 2 + ;; + --build-logs-dir) + export build_logs_dir="${2}" + shift 2 + ;; + --shared-fs-path) + export shared_fs_path="${2}" + shift 2 + ;; + -*|--*) + echo "Error: Unknown option: $1" >&2 + exit 1 + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +TOPDIR=$(dirname $(realpath $0)) + +source $TOPDIR/scripts/utils.sh + +# honor $TMPDIR if it is already defined, use /tmp otherwise +if [ -z $TMPDIR ]; then + export WORKDIR=/tmp/$USER +else + export WORKDIR=$TMPDIR/$USER +fi + +TMPDIR=$(mktemp -d) + +echo ">> Setting up environment..." + +source $TOPDIR/init/minimal_eessi_env + +if [ -d $EESSI_CVMFS_REPO ]; then + echo_green "$EESSI_CVMFS_REPO available, OK!" +else + fatal_error "$EESSI_CVMFS_REPO is not available!" +fi + +# make sure we're in Prefix environment by checking $SHELL +if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then + echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" +else + fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" +fi + +# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory +export PYTHONPYCACHEPREFIX=$TMPDIR/pycache + +DETECTION_PARAMETERS='' +GENERIC=0 +EB='eb' +if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then + echo_yellow ">> GENERIC build/test requested, taking appropriate measures!" + DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" + GENERIC=1 + EB='eb --optarch=GENERIC' +fi + +echo ">> Determining software subdirectory to use for current build/test host..." +if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" +else + echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" +fi + +# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) +# $EESSI_SILENT - don't print any messages +# $EESSI_BASIC_ENV - give a basic set of environment variables +EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables + +if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then + fatal_error "Failed to determine software subdirectory?!" +elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" +else + echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" +fi + +echo ">> Initializing Lmod..." +source $EPREFIX/usr/share/Lmod/init/bash +ml_version_out=$TMPDIR/ml.out +ml --version &> $ml_version_out +if [[ $? -eq 0 ]]; then + echo_green ">> Found Lmod ${LMOD_VERSION}" +else + fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" +fi + +echo ">> Configuring EasyBuild..." +source $TOPDIR/configure_easybuild + +echo ">> Setting up \$MODULEPATH..." +# make sure no modules are loaded +module --force purge +# ignore current $MODULEPATH entirely +module unuse $MODULEPATH +module use $EASYBUILD_INSTALLPATH/modules/all +if [[ -z ${MODULEPATH} ]]; then + fatal_error "Failed to set up \$MODULEPATH?!" +else + echo_green ">> MODULEPATH set up: ${MODULEPATH}" +fi + +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# "split" the file by prefixing each line belonging to the same file with the +# same number +split_file=$(awk '/^\+\+\+/{n++}{print n, " ", $0 }' ${pr_diff}) + +# determine which easystack files may have changed +changed_es_files=$(echo "${split_file}" | grep '^[0-9 ]*+++ ./eessi.*.yml$' | egrep -v 'known-issues|missing') + +# process all changed easystackfiles +for es_file_num in $(echo "${changed_es_files}" | cut -f1 -d' ') +do + # determine added lines that do not contain a yaml comment only + added_lines=$(echo "${split_file}" | grep "${es_file_num} + " | sed -e "s/^"${es_file_num}" + //" | grep -v "^[ ]*#") + # determine easyconfigs + easyconfigs=$(echo "${added_lines}" | cut -f3 -d' ') + # get easystack file name + easystack_file=$(echo "${changed_es_files}" | grep "^${es_file_num}" | sed -e "s/^"${es_file_num}" ... .\///") + echo -e "Processing easystack file ${easystack_file}...\n\n" + + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + + # load EasyBuild module + module load EasyBuild/${eb_version} + + echo_green "All set, let's run sanity checks for installed packages..." + + for easyconfig in ${easyconfigs}; + do + echo "Running sanity check for '${easyconfig}'..." + eb --sanity-check-only ${easyconfig} + done +done + +echo ">> Cleaning up ${TMPDIR}..." +rm -r ${TMPDIR} From 17cfd02779833bd5f2df96655f3dc19bb5f880e8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 15 Oct 2023 20:30:44 +0200 Subject: [PATCH 0208/1795] small adjustments to test script --- bot/test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index 6d217caf92..89c1a6a8bf 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -206,9 +206,9 @@ fi # prepare arguments to test_suite.sh (specific to test step) declare -a TEST_SUITE_ARGS=() -# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then -# TEST_SUITE_ARGS+=("--generic") -# fi +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then + TEST_SUITE_ARGS+=("--generic") +fi # [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") # [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") @@ -217,8 +217,8 @@ test_outerr=$(mktemp test.outerr.XXXX) echo "Executing command to build software:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}" -echo " -- ./test_suite.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}" +echo " -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \ - -- ./test_suite.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr} + -- ./run_tests.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr} exit 0 From 3be4823474774211581bc2dd85b5fd98e7eec746 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Thu, 19 Oct 2023 18:42:41 +0200 Subject: [PATCH 0209/1795] Add DP3-6.0-foss-2022a and WSClean-3.4-foss-2022a --- eessi-2023.06-eb-4.8.1-2022a.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 8317a40b7a..28b469ee24 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -10,3 +10,9 @@ easyconfigs: - ESPResSo-4.2.1-foss-2022a: options: from-pr: 18963 + - DP3-6.0-foss-2022a: + options: + from-pr: 19010 + - WSClean-3.4-foss-2022a: + options: + from-pr: 19010 From 9166400aa2c1d18c431ea7f409e1357a0823d73b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 20 Oct 2023 21:58:43 +0200 Subject: [PATCH 0210/1795] Easyconfig that can extend EESSI --- EESSI-2023.06-extend.eb | 85 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 EESSI-2023.06-extend.eb diff --git a/EESSI-2023.06-extend.eb b/EESSI-2023.06-extend.eb new file mode 100644 index 0000000000..e4fafb860c --- /dev/null +++ b/EESSI-2023.06-extend.eb @@ -0,0 +1,85 @@ +easyblock = 'Binary' + +name = 'EESSI' +version = '2023.06' +versionsuffix = '-extend' + +homepage = 'https://eessi.github.io/docs/' + +description = """ + The goal of the European Environment for Scientific Software Installations + (EESSI, pronounced as "easy") is to build a common stack of scientific + software installations for HPC systems and beyond, including laptops, + personal workstations and cloud infrastructure. + + This module allows you to extend EESSI using the same configuration for + EasyBuild as EESSI itself uses. It installs the modules in a special location + that is configurable in the CVMFS configuration (so can be on a shared + filesystem), and automatically found when using the default EESSI environment. +""" + +toolchain = SYSTEM + +source_urls = None +sources = ['eb_hooks.py'] +checksums = ['8ae609f99b6953beae89aa1945913686c86c156509dbc55e2b6b8017b13fcf66'] + +# Don't extend the PATH +prepend_to_path = None + +# All the dependencies we filter in EESSI +local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib" +local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''} +local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH] + +modextravars = { + 'EASYBUILD_FILTER_DEPS': local_deps_to_filter, + 'EASYBUILD_IGNORE_OSDEPS': '1', + 'EASYBUILD_DEBUG': '1', + 'EASYBUILD_TRACE': '1', + 'EASYBUILD_ZIP_LOGS': 'bzip2', + 'EASYBUILD_RPATH': '1', + 'EASYBUILD_FILTER_ENV_VARS': 'LD_LIBRARY_PATH', + 'EASYBUILD_READ_ONLY_INSTALLDIR': '1', + 'EASYBUILD_MODULE_EXTENSIONS': '1', + 'EASYBUILD_EXPERIMENTAL': '1', + 'EASYBUILD_HOOKS': '%(installdir)s/eb_hooks.py', +} + +# Need a few other variables, but they are more dynamic +# EASYBUILD_SYSROOT=${EPREFIX} +# EASYBUILD_PREFIX=${WORKDIR}/easybuild +# EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} +# EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH} +modluafooter = """ +if (mode() == "load") then + -- Use a working directory for temporary build files + if (os.getenv("WORKING_DIR") == nil) then + LmodMessage("-- Using /tmp as a temporary working directory for installations, you can override this by setting the environment variable WORKING_DIR and reloading the module (e.g., /dev/shm is a common option)") + end +end +working_dir = os.getenv("WORKING_DIR") or "/tmp" +-- Gather the EPREFIX to use as a sysroot +sysroot = os.getenv("EPREFIX") +-- Use an installation prefix that we _should_ have write access to +easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') +if (mode() == "load") then + LmodMessage("-- To create installations for EESSI, you _must_ have write permissions to " .. easybuild_installpath) + -- Advise them to reuse sources + if (os.getenv("EASYBUILD_SOURCEPATH") == nil) then + LmodMessage("-- You may wish to configure a sources directory for EasyBuild (for example, via setting the environment variable EASYBUILD_SOURCEPATH) to allow you to reuse existing sources for packages.") + end +end +-- Set the relevant environment variables for EasyBuild +setenv ("EASYBUILD_SYSROOT", sysroot) +setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) +setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) +always_load("EasyBuild") +""" + +sanity_check_paths = { + 'files': ['eb_hooks.py'], + 'dirs': [''] +} + +moduleclass = 'devel' From f1a0e71a4b1f01c3b9937a302f50dc87b252c99f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 20 Oct 2023 22:00:21 +0200 Subject: [PATCH 0211/1795] Add extension of EESSI to MODULEPATH --- init/Magic_Castle/bash | 2 +- init/bash | 2 ++ init/eessi_environment_variables | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/init/Magic_Castle/bash b/init/Magic_Castle/bash index 85e4d54241..219c6d3a85 100644 --- a/init/Magic_Castle/bash +++ b/init/Magic_Castle/bash @@ -10,7 +10,7 @@ source $(dirname "$BASH_SOURCE")/../eessi_environment_variables # Provide a clean MODULEPATH export MODULEPATH_ROOT=$EESSI_MODULEPATH -export MODULEPATH=$EESSI_MODULEPATH +export MODULEPATH=$EESSI_EXTEND_MODULEPATH:$EESSI_MODULEPATH # Extensions are too many, let's not print them by default (requires Lmod 8.4.12) export LMOD_AVAIL_EXTENSIONS=no diff --git a/init/bash b/init/bash index ea605db0b5..bf8ada4fd2 100644 --- a/init/bash +++ b/init/bash @@ -26,6 +26,8 @@ if [ $? -eq 0 ]; then # prepend location of modules for EESSI software stack to $MODULEPATH echo "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." >> $output module use $EESSI_MODULEPATH + echo "Prepending $EESSI_EXTEND_MODULEPATH to \$MODULEPATH..." >> $output + module use $EESSI_EXTEND_MODULEPATH #echo >> $output #echo "*** Known problems in the ${EESSI_PILOT_VERSION} pilot software stack ***" >> $output diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 10ac1926f4..7b4f82bbd3 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -60,6 +60,8 @@ if [ -d $EESSI_PREFIX ]; then if [ -d $EESSI_MODULEPATH ]; then export EESSI_MODULEPATH=$EESSI_MODULEPATH echo "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." >> $output + export EESSI_EXTEND_MODULEPATH=${EESSI_MODULEPATH/versions/host_injections} + echo "Using ${EESSI_EXTEND_MODULEPATH} as the extension directory to be added to MODULEPATH." >> $output else error "EESSI module path at $EESSI_MODULEPATH not found!" false From b26dc50b9f3a282a185466e1004071074b1ac345 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 23 Oct 2023 17:40:05 +0200 Subject: [PATCH 0212/1795] add PLUMED hook --- eb_hooks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 31f2b9588d..3147fc9bf5 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -268,6 +268,22 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") +def pre_configure_hook_PLUMED_aarch64(self, *args, **kwargs): + """ + pre-configure hook for PLUMED: + - remove unsupported --enable-asmjit option on aarch64 + """ + + if self.name == 'PLUMED': + if get_cpu_architecture() == AARCH64: + configopts = self.cfg['configopts'] + regex = re.compile(r'--enable-asmjit') + if re.search(regex, configopts): + self.cfg['configopts'] = regex.sub('', configopts) + else: + raise EasyBuildError("PLUMED-specific hook triggered for non-PLUMED easyconfig?!") + + def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: @@ -346,6 +362,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, + 'PLUMED': pre_configure_hook_PLUMED_aarch64, } PRE_TEST_HOOKS = { From 207fb71f0991101edf13ebc1e51387af77087178 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 10:56:24 +0100 Subject: [PATCH 0213/1795] Add EasyBuild v4.8.2 --- eessi-2023.06-eb-4.8.0-system.yml | 3 +++ eessi-2023.06-eb-4.8.1-2022a.yml | 6 ------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml index 2928f110b0..8db29e3c61 100644 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ b/eessi-2023.06-eb-4.8.0-system.yml @@ -8,3 +8,6 @@ easyconfigs: - EasyBuild-4.8.1.eb: options: from-pr: 18761 + - EasyBuild-4.8.2.eb: + options: + from-pr: 19101 diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 28b469ee24..8317a40b7a 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -10,9 +10,3 @@ easyconfigs: - ESPResSo-4.2.1-foss-2022a: options: from-pr: 18963 - - DP3-6.0-foss-2022a: - options: - from-pr: 19010 - - WSClean-3.4-foss-2022a: - options: - from-pr: 19010 From 10830788bdff71a15cc4a51488763b805ef2582f Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 10:57:32 +0100 Subject: [PATCH 0214/1795] Add EasyBuild v4.8.2 --- eessi-2023.06-eb-4.8.2-2022a.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.2-2022a.yml diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml new file mode 100644 index 0000000000..5e812716f2 --- /dev/null +++ b/eessi-2023.06-eb-4.8.2-2022a.yml @@ -0,0 +1,3 @@ +easyconfigs: + - DP3-6.0-foss-2022a: + - WSClean-3.4-foss-2022a: From e2644fa581b67b52acddab01285880b993e17fe0 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 11:23:31 +0100 Subject: [PATCH 0215/1795] Add EasyBuild 4.8.2 to 4.8.1 system file --- eessi-2023.06-eb-4.8.1-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index 1e9664e10f..3d0e1c3c32 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -4,3 +4,6 @@ easyconfigs: - ReFrame-4.3.3.eb: options: from-pr: 18851 + - EasyBuild-4.8.2.eb: + options: + from-pr: 19101 From eb503d9d73904de99ff7d2a3c7d118298ac1461f Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 11:24:59 +0100 Subject: [PATCH 0216/1795] Remove EasyBuild 4.8.2 from 4.8.0 system file --- eessi-2023.06-eb-4.8.0-system.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml index 8db29e3c61..2928f110b0 100644 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ b/eessi-2023.06-eb-4.8.0-system.yml @@ -8,6 +8,3 @@ easyconfigs: - EasyBuild-4.8.1.eb: options: from-pr: 18761 - - EasyBuild-4.8.2.eb: - options: - from-pr: 19101 From 23b23f1a86a038f5ef616799af118cb43c1a972c Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 13:20:47 +0100 Subject: [PATCH 0217/1795] Try loading module without cache before exiting --- load_easybuild_module.sh | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh index 4ff2a3c37c..c027a4b4b1 100755 --- a/load_easybuild_module.sh +++ b/load_easybuild_module.sh @@ -23,14 +23,14 @@ fi EB_VERSION=${1} # make sure that environment variables that we expect to be set are indeed set -if [ -z "${TMPDIR}" ]; then +if [ -z "${TMPDIR}" ]; then echo "\$TMPDIR is not set" >&2 exit 2 fi # ${EB} is used to specify which 'eb' command should be used; # can potentially be more than just 'eb', for example when using 'eb --optarch=GENERIC' -if [ -z "${EB}" ]; then +if [ -z "${EB}" ]; then echo "\$EB is not set" >&2 exit 2 fi @@ -75,17 +75,22 @@ else if [[ $? -eq 0 ]]; then echo_green ">> Module for EasyBuild v${EB_VERSION} found!" else - eb_ec=EasyBuild-${EB_VERSION}.eb - echo_yellow ">> Still no module for EasyBuild v${EB_VERSION}, trying with easyconfig ${eb_ec}..." - ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null + module avail --ignore_cache 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then - echo "Easyconfig ${eb_ec} found for EasyBuild v${EB_VERSION}, so installing it..." - ok_msg="EasyBuild v${EB_VERSION} installed, alright!" - fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})" - ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out} - check_exit_code $? "${ok_msg}" "${fail_msg}" + echo_green ">> Module for EasyBuild v${EB_VERSION} found!" else - fatal_error "No easyconfig found for EasyBuild v${EB_VERSION}" + eb_ec=EasyBuild-${EB_VERSION}.eb + echo_yellow ">> Still no module for EasyBuild v${EB_VERSION}, trying with easyconfig ${eb_ec}..." + ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null + if [[ $? -eq 0 ]]; then + echo "Easyconfig ${eb_ec} found for EasyBuild v${EB_VERSION}, so installing it..." + ok_msg="EasyBuild v${EB_VERSION} installed, alright!" + fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})" + ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out} + check_exit_code $? "${ok_msg}" "${fail_msg}" + else + fatal_error "No easyconfig found for EasyBuild v${EB_VERSION}" + fi fi fi From 48735b571c31fcf443596c16fea87ad1e634a138 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 13:55:18 +0100 Subject: [PATCH 0218/1795] Fix position ignore_cache flag --- load_easybuild_module.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh index c027a4b4b1..9f7716543a 100755 --- a/load_easybuild_module.sh +++ b/load_easybuild_module.sh @@ -75,7 +75,7 @@ else if [[ $? -eq 0 ]]; then echo_green ">> Module for EasyBuild v${EB_VERSION} found!" else - module avail --ignore_cache 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out} + module --ignore_cache avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> Module for EasyBuild v${EB_VERSION} found!" else From d97d23bacf7671d9c8dbc0381d917ddb26a539cd Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 14:56:07 +0100 Subject: [PATCH 0219/1795] Also ignore cache when loading EB module --- load_easybuild_module.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh index 9f7716543a..11c780dcdd 100755 --- a/load_easybuild_module.sh +++ b/load_easybuild_module.sh @@ -71,10 +71,12 @@ else check_exit_code $? "${ok_msg}" "${fail_msg}" # maybe the module obtained with --install-latest-eb-release is exactly the EasyBuild version we wanted? + IGNORE_CACHE='' module avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> Module for EasyBuild v${EB_VERSION} found!" else + IGNORE_CACHE='--ignore_cache' module --ignore_cache avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> Module for EasyBuild v${EB_VERSION} found!" @@ -108,7 +110,7 @@ else fi echo ">> Loading EasyBuild v${EB_VERSION} module..." -module load EasyBuild/${EB_VERSION} +module ${IGNORE_CACHE} load EasyBuild/${EB_VERSION} eb_show_system_info_out=${TMPDIR}/eb_show_system_info.out ${EB} --show-system-info > ${eb_show_system_info_out} if [[ $? -eq 0 ]]; then From a8fdd91817d0bb9a884fb2bf42520f1919a1e51c Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 15:59:53 +0100 Subject: [PATCH 0220/1795] Move IGNORE_CACHE variable down --- load_easybuild_module.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh index 11c780dcdd..d1bfd18bb5 100755 --- a/load_easybuild_module.sh +++ b/load_easybuild_module.sh @@ -76,10 +76,10 @@ else if [[ $? -eq 0 ]]; then echo_green ">> Module for EasyBuild v${EB_VERSION} found!" else - IGNORE_CACHE='--ignore_cache' module --ignore_cache avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> Module for EasyBuild v${EB_VERSION} found!" + IGNORE_CACHE='--ignore_cache' else eb_ec=EasyBuild-${EB_VERSION}.eb echo_yellow ">> Still no module for EasyBuild v${EB_VERSION}, trying with easyconfig ${eb_ec}..." From 9d13aa874e1961029f0d73634351db07f5a19eb4 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 30 Oct 2023 16:12:31 +0100 Subject: [PATCH 0221/1795] replace plumed hook with updated easyconfig and add LAMMPS hook [context](https://gitlab.com/eessi/support/-/issues/10#note_1625972083) --- eb_hooks.py | 14 ++++++-------- eessi-2023.06-eb-4.8.1-2021b.yml | 4 ++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 3147fc9bf5..28ee7907a4 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -268,18 +268,16 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") -def pre_configure_hook_PLUMED_aarch64(self, *args, **kwargs): +def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs): """ pre-configure hook for PLUMED: - remove unsupported --enable-asmjit option on aarch64 """ - if self.name == 'PLUMED': - if get_cpu_architecture() == AARCH64: - configopts = self.cfg['configopts'] - regex = re.compile(r'--enable-asmjit') - if re.search(regex, configopts): - self.cfg['configopts'] = regex.sub('', configopts) + if self.name == 'LAMMPS': + if self.version == '23Jun2022': + if get_cpu_architecture() == AARCH64: + self.cfg['kokkos_arch'] = 'A64FX' else: raise EasyBuildError("PLUMED-specific hook triggered for non-PLUMED easyconfig?!") @@ -362,7 +360,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, - 'PLUMED': pre_configure_hook_PLUMED_aarch64, + 'PLUMED': pre_configure_hook_LAMMPS_aarch64, } PRE_TEST_HOOKS = { diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index 1414df5e44..30035fa837 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -5,6 +5,10 @@ easyconfigs: options: from-pr: 18834 - R-4.2.0-foss-2021b.eb + - PLUMED-2.7.3-foss-2021b.eb: + # the --enable-asmjit is not supported on Aarch64 + options: + from-pr: 19110 - LAMMPS-23Jun2022-foss-2021b-kokkos.eb: # TBB is an optional dependency when building on Intel arch options: From 7537a7f17099c04ebd1520bc6ae093d5585e54b1 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 30 Oct 2023 16:17:13 +0100 Subject: [PATCH 0222/1795] small fix --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 28ee7907a4..546194111d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -279,7 +279,7 @@ def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs): if get_cpu_architecture() == AARCH64: self.cfg['kokkos_arch'] = 'A64FX' else: - raise EasyBuildError("PLUMED-specific hook triggered for non-PLUMED easyconfig?!") + raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") def pre_test_hook(self,*args, **kwargs): From d56e0dd90132f3585f4da2eed1803f0a9191f2c6 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 30 Oct 2023 16:28:36 +0100 Subject: [PATCH 0223/1795] fix PRE_CONFIGURE_HOOKS --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 546194111d..ea9c3bfd48 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -360,7 +360,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, - 'PLUMED': pre_configure_hook_LAMMPS_aarch64, + 'LAMMPS': pre_configure_hook_LAMMPS_aarch64, } PRE_TEST_HOOKS = { From 5c9316dd86a1ceb2d2b2e5d04c8627d0da31abe1 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 30 Oct 2023 16:36:53 +0100 Subject: [PATCH 0224/1795] fix in description --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index ea9c3bfd48..a2890da75e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -270,8 +270,8 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs): """ - pre-configure hook for PLUMED: - - remove unsupported --enable-asmjit option on aarch64 + pre-configure hook for LAMMPS: + - set kokkos_arch on Aarch64 """ if self.name == 'LAMMPS': From cc4e2c03421f52a8be699adeed35dccac04266db Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 17:46:40 +0100 Subject: [PATCH 0225/1795] {2023.06} EasyBuild v4.8.2 + attempt module loads without cache --- eessi-2023.06-eb-4.8.1-system.yml | 3 +++ load_easybuild_module.sh | 31 +++++++++++++++++++------------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index 1e9664e10f..3d0e1c3c32 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -4,3 +4,6 @@ easyconfigs: - ReFrame-4.3.3.eb: options: from-pr: 18851 + - EasyBuild-4.8.2.eb: + options: + from-pr: 19101 diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh index 4ff2a3c37c..d1bfd18bb5 100755 --- a/load_easybuild_module.sh +++ b/load_easybuild_module.sh @@ -23,14 +23,14 @@ fi EB_VERSION=${1} # make sure that environment variables that we expect to be set are indeed set -if [ -z "${TMPDIR}" ]; then +if [ -z "${TMPDIR}" ]; then echo "\$TMPDIR is not set" >&2 exit 2 fi # ${EB} is used to specify which 'eb' command should be used; # can potentially be more than just 'eb', for example when using 'eb --optarch=GENERIC' -if [ -z "${EB}" ]; then +if [ -z "${EB}" ]; then echo "\$EB is not set" >&2 exit 2 fi @@ -71,21 +71,28 @@ else check_exit_code $? "${ok_msg}" "${fail_msg}" # maybe the module obtained with --install-latest-eb-release is exactly the EasyBuild version we wanted? + IGNORE_CACHE='' module avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> Module for EasyBuild v${EB_VERSION} found!" else - eb_ec=EasyBuild-${EB_VERSION}.eb - echo_yellow ">> Still no module for EasyBuild v${EB_VERSION}, trying with easyconfig ${eb_ec}..." - ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null + module --ignore_cache avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then - echo "Easyconfig ${eb_ec} found for EasyBuild v${EB_VERSION}, so installing it..." - ok_msg="EasyBuild v${EB_VERSION} installed, alright!" - fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})" - ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out} - check_exit_code $? "${ok_msg}" "${fail_msg}" + echo_green ">> Module for EasyBuild v${EB_VERSION} found!" + IGNORE_CACHE='--ignore_cache' else - fatal_error "No easyconfig found for EasyBuild v${EB_VERSION}" + eb_ec=EasyBuild-${EB_VERSION}.eb + echo_yellow ">> Still no module for EasyBuild v${EB_VERSION}, trying with easyconfig ${eb_ec}..." + ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null + if [[ $? -eq 0 ]]; then + echo "Easyconfig ${eb_ec} found for EasyBuild v${EB_VERSION}, so installing it..." + ok_msg="EasyBuild v${EB_VERSION} installed, alright!" + fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})" + ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out} + check_exit_code $? "${ok_msg}" "${fail_msg}" + else + fatal_error "No easyconfig found for EasyBuild v${EB_VERSION}" + fi fi fi @@ -103,7 +110,7 @@ else fi echo ">> Loading EasyBuild v${EB_VERSION} module..." -module load EasyBuild/${EB_VERSION} +module ${IGNORE_CACHE} load EasyBuild/${EB_VERSION} eb_show_system_info_out=${TMPDIR}/eb_show_system_info.out ${EB} --show-system-info > ${eb_show_system_info_out} if [[ $? -eq 0 ]]; then From c1b29b3d3384c0a3a570252703d274537437e75c Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 18:09:34 +0100 Subject: [PATCH 0226/1795] Use correct 4.8.2 PR --- eessi-2023.06-eb-4.8.1-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index 3d0e1c3c32..b0731d2534 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -6,4 +6,4 @@ easyconfigs: from-pr: 18851 - EasyBuild-4.8.2.eb: options: - from-pr: 19101 + from-pr: 19105 From 0f803f491d6dfbbeab55cc08d7d70d6e7d3b8e40 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 30 Oct 2023 18:12:41 +0100 Subject: [PATCH 0227/1795] Use correct 4.8.2 PR --- eessi-2023.06-eb-4.8.1-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml index 3d0e1c3c32..b0731d2534 100644 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ b/eessi-2023.06-eb-4.8.1-system.yml @@ -6,4 +6,4 @@ easyconfigs: from-pr: 18851 - EasyBuild-4.8.2.eb: options: - from-pr: 19101 + from-pr: 19105 From 8b457d4dfebeb78f9b1915cb66683ec174f2d638 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 31 Oct 2023 10:50:24 +0100 Subject: [PATCH 0228/1795] use framework fix and Pillow fix in easybuild v4.8.2 --- eessi-2023.06-eb-4.8.1-2021b.yml | 5 ----- eessi-2023.06-eb-4.8.2-2021b.yml | 2 ++ 2 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 eessi-2023.06-eb-4.8.2-2021b.yml diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index ec019ea879..62f529563a 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -5,8 +5,3 @@ easyconfigs: options: from-pr: 18834 - R-4.2.0-foss-2021b.eb - - Pillow-8.3.2-GCCcore-11.2.0.eb: - # avoid that hardcoded paths like /usr/include are used in build commands - options: - from-pr: 18881 - - matplotlib-3.4.3-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.8.2-2021b.yml b/eessi-2023.06-eb-4.8.2-2021b.yml new file mode 100644 index 0000000000..746f8df05f --- /dev/null +++ b/eessi-2023.06-eb-4.8.2-2021b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - matplotlib-3.4.3-foss-2021b.eb From 2aee11e3dac9a9b748e1456aba10d68aca22fce6 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 31 Oct 2023 14:15:20 +0100 Subject: [PATCH 0229/1795] Test excluding filter-deps for AOFlagger and casacore --- eessi-2023.06-eb-4.8.2-2022a.yml | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml index 5e812716f2..8dbfe5c223 100644 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ b/eessi-2023.06-eb-4.8.2-2022a.yml @@ -1,3 +1,55 @@ easyconfigs: + - casacore-3.5.0-foss-2022a: + options: + from-pr: 19119 + # Exclude ncurses + filter-deps: + - Autoconf + - Automake + - Autotools + - binutils + - bzip2 + - DBus + - flex + - gettext + - gperf + - help2man + - intltool + - libreadline + - libtool + - Lua + - M4 + - makeinfo + - util-linux + - XZ + - zlib + - AOFlagger-3.4.0-foss-2022a: + options: + from-pr: 19119 + # Exclude Lua + filter-deps: + - Autoconf + - Automake + - Autotools + - binutils + - bzip2 + - DBus + - flex + - gettext + - gperf + - help2man + - intltool + - libreadline + - libtool + - ncurses + - M4 + - makeinfo + - util-linux + - XZ + - zlib - DP3-6.0-foss-2022a: + options: + from-pr: 19119 - WSClean-3.4-foss-2022a: + options: + from-pr: 19119 From 7259183a4965ff0c83b75a7bc2f04fb47d1c4381 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 31 Oct 2023 14:53:53 +0100 Subject: [PATCH 0230/1795] Remove casacore (ncurses is picked up from compat layer) --- eessi-2023.06-eb-4.8.2-2022a.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml index 8dbfe5c223..6b0dbc0387 100644 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ b/eessi-2023.06-eb-4.8.2-2022a.yml @@ -1,28 +1,4 @@ easyconfigs: - - casacore-3.5.0-foss-2022a: - options: - from-pr: 19119 - # Exclude ncurses - filter-deps: - - Autoconf - - Automake - - Autotools - - binutils - - bzip2 - - DBus - - flex - - gettext - - gperf - - help2man - - intltool - - libreadline - - libtool - - Lua - - M4 - - makeinfo - - util-linux - - XZ - - zlib - AOFlagger-3.4.0-foss-2022a: options: from-pr: 19119 From a7294c51c28d5be8a1871e4e165f708ec3ce0a16 Mon Sep 17 00:00:00 2001 From: t1mk1k <96469032+t1mk1k@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:02:37 +0100 Subject: [PATCH 0231/1795] Explain why Lua is excluded from filter-deps Co-authored-by: ocaisa --- eessi-2023.06-eb-4.8.2-2022a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml index 6b0dbc0387..b0e1992efe 100644 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ b/eessi-2023.06-eb-4.8.2-2022a.yml @@ -2,7 +2,7 @@ easyconfigs: - AOFlagger-3.4.0-foss-2022a: options: from-pr: 19119 - # Exclude Lua + # Exclude Lua from `filter-deps` as the compat layer version is too old for the software filter-deps: - Autoconf - Automake From 31894ec295344e55338be9b225efabff20f9ae9a Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 31 Oct 2023 17:01:17 +0100 Subject: [PATCH 0232/1795] Include casacore and EveryBeam from PR 19119 --- eessi-2023.06-eb-4.8.2-2022a.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml index b0e1992efe..bab796db2b 100644 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ b/eessi-2023.06-eb-4.8.2-2022a.yml @@ -1,4 +1,7 @@ easyconfigs: + - casacore-3.5.0-foss-2022a: + options: + from-pr: 19119 - AOFlagger-3.4.0-foss-2022a: options: from-pr: 19119 @@ -23,6 +26,9 @@ easyconfigs: - util-linux - XZ - zlib + - EveryBeam-0.5.2-foss-2022a: + options: + from-pr: 19119 - DP3-6.0-foss-2022a: options: from-pr: 19119 From a2ab66466d882b9724eb998bf92c136fe514150b Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 7 Nov 2023 11:59:04 +0100 Subject: [PATCH 0233/1795] update easystack --- eessi-2023.06-eb-4.8.1-2021b.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index 30035fa837..478899f672 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -9,7 +9,11 @@ easyconfigs: # the --enable-asmjit is not supported on Aarch64 options: from-pr: 19110 + - ScaFaCoS-1.0.4-foss-2021b.eb: + # Newer version of ScaFaCoS for LAMMPS + options: + from-pr: 19163 - LAMMPS-23Jun2022-foss-2021b-kokkos.eb: # TBB is an optional dependency when building on Intel arch options: - from-pr: 19000 + from-pr: 19164 From 98f20cdd9dd83c481065f30ab6657367d06b869b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 7 Nov 2023 13:14:24 +0100 Subject: [PATCH 0234/1795] Allow for both site and user extensions to EESSI --- ...nd.eb => EESSI-extend-2023.06-easybuild.eb | 30 ++++++++++++------- init/Magic_Castle/bash | 2 +- init/bash | 6 ++-- init/eessi_environment_variables | 6 ++-- 4 files changed, 29 insertions(+), 15 deletions(-) rename EESSI-2023.06-extend.eb => EESSI-extend-2023.06-easybuild.eb (69%) diff --git a/EESSI-2023.06-extend.eb b/EESSI-extend-2023.06-easybuild.eb similarity index 69% rename from EESSI-2023.06-extend.eb rename to EESSI-extend-2023.06-easybuild.eb index e4fafb860c..fd9f897326 100644 --- a/EESSI-2023.06-extend.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -1,8 +1,9 @@ easyblock = 'Binary' -name = 'EESSI' +name = 'EESSI-extend' version = '2023.06' -versionsuffix = '-extend' +# May have different ways to extend EESSI in future (manually, other tools,...) +versionsuffix = '-easybuild' homepage = 'https://eessi.github.io/docs/' @@ -13,9 +14,10 @@ description = """ personal workstations and cloud infrastructure. This module allows you to extend EESSI using the same configuration for - EasyBuild as EESSI itself uses. It installs the modules in a special location - that is configurable in the CVMFS configuration (so can be on a shared - filesystem), and automatically found when using the default EESSI environment. + EasyBuild as EESSI itself uses. The default installation is the users + home directory, but this can be overridden for a site installation (by setting + EESSI_SITE_INSTALL) or for a direct installation under CVMFS (by setting + EESSI_CVMFS_INSTALL). """ toolchain = SYSTEM @@ -55,14 +57,20 @@ modluafooter = """ if (mode() == "load") then -- Use a working directory for temporary build files if (os.getenv("WORKING_DIR") == nil) then - LmodMessage("-- Using /tmp as a temporary working directory for installations, you can override this by setting the environment variable WORKING_DIR and reloading the module (e.g., /dev/shm is a common option)") + LmodMessage("-- Using /tmp/$USER as a temporary working directory for installations, you can override this by setting the environment variable WORKING_DIR and reloading the module (e.g., /dev/shm is a common option)") end end -working_dir = os.getenv("WORKING_DIR") or "/tmp" +working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER")) -- Gather the EPREFIX to use as a sysroot -sysroot = os.getenv("EPREFIX") +sysroot = os.getenv("EESSI_EPREFIX") -- Use an installation prefix that we _should_ have write access to -easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') +if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then + easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") +elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') +else + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), pathJoin(os.getenv("HOME"), "eessi")) +end if (mode() == "load") then LmodMessage("-- To create installations for EESSI, you _must_ have write permissions to " .. easybuild_installpath) -- Advise them to reuse sources @@ -74,7 +82,9 @@ end setenv ("EASYBUILD_SYSROOT", sysroot) setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) -always_load("EasyBuild") +if not ( isloaded("EasyBuild") ) then + load("EasyBuild") +end """ sanity_check_paths = { diff --git a/init/Magic_Castle/bash b/init/Magic_Castle/bash index 219c6d3a85..0ee9d612f3 100644 --- a/init/Magic_Castle/bash +++ b/init/Magic_Castle/bash @@ -10,7 +10,7 @@ source $(dirname "$BASH_SOURCE")/../eessi_environment_variables # Provide a clean MODULEPATH export MODULEPATH_ROOT=$EESSI_MODULEPATH -export MODULEPATH=$EESSI_EXTEND_MODULEPATH:$EESSI_MODULEPATH +export MODULEPATH=$EESSI_USER_MODULEPATH:$EESSI_SITE_MODULEPATH:$EESSI_MODULEPATH # Extensions are too many, let's not print them by default (requires Lmod 8.4.12) export LMOD_AVAIL_EXTENSIONS=no diff --git a/init/bash b/init/bash index bf8ada4fd2..76c1e9154a 100644 --- a/init/bash +++ b/init/bash @@ -26,8 +26,10 @@ if [ $? -eq 0 ]; then # prepend location of modules for EESSI software stack to $MODULEPATH echo "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." >> $output module use $EESSI_MODULEPATH - echo "Prepending $EESSI_EXTEND_MODULEPATH to \$MODULEPATH..." >> $output - module use $EESSI_EXTEND_MODULEPATH + echo "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." >> $output + module use $EESSI_SITE_MODULEPATH + echo "Prepending user path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." >> $output + module use $EESSI_USER_MODULEPATH #echo >> $output #echo "*** Known problems in the ${EESSI_PILOT_VERSION} pilot software stack ***" >> $output diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 7b4f82bbd3..656fbc6637 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -60,8 +60,10 @@ if [ -d $EESSI_PREFIX ]; then if [ -d $EESSI_MODULEPATH ]; then export EESSI_MODULEPATH=$EESSI_MODULEPATH echo "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." >> $output - export EESSI_EXTEND_MODULEPATH=${EESSI_MODULEPATH/versions/host_injections} - echo "Using ${EESSI_EXTEND_MODULEPATH} as the extension directory to be added to MODULEPATH." >> $output + export EESSI_SITE_MODULEPATH=${EESSI_MODULEPATH/versions/host_injections} + echo "Using ${EESSI_SITE_MODULEPATH} as the site extension directory to be added to MODULEPATH." >> $output + export EESSI_USER_MODULEPATH=${EESSI_MODULEPATH/${EESSI_CVMFS_REPO}/${HOME}\/eessi} + echo "Using ${EESSI_USER_MODULEPATH} as the user extension directory to be added to MODULEPATH." >> $output else error "EESSI module path at $EESSI_MODULEPATH not found!" false From 60cb5dc90187b8b8506933a74cc51e07b67cfe0a Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 7 Nov 2023 13:15:20 +0100 Subject: [PATCH 0235/1795] Update EESSI-extend-2023.06-easybuild.eb --- EESSI-extend-2023.06-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index fd9f897326..ce55f619fb 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -30,7 +30,7 @@ checksums = ['8ae609f99b6953beae89aa1945913686c86c156509dbc55e2b6b8017b13fcf66'] prepend_to_path = None # All the dependencies we filter in EESSI -local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib" +local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib" local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''} local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH] From 8fe3892e8bb41a8054bd8cfeea34e96888235db3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 7 Nov 2023 13:48:53 +0100 Subject: [PATCH 0236/1795] Escape hyphen for gsub --- EESSI-extend-2023.06-easybuild.eb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index ce55f619fb..8993c63cc3 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -69,7 +69,8 @@ if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') else - easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), pathJoin(os.getenv("HOME"), "eessi")) + -- Would have liked to use os.getenv("EESSI_CVMFS_REPO") here but the hypen needs to be escaped + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), "/cvmfs/pilot.eessi%-hpc.org", pathJoin(os.getenv("HOME"), "eessi")) end if (mode() == "load") then LmodMessage("-- To create installations for EESSI, you _must_ have write permissions to " .. easybuild_installpath) From 3f5ef9acbef5d3a8f11d15dd27d5b36fc6ff26aa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Nov 2023 15:48:54 +0100 Subject: [PATCH 0237/1795] don't filter Lua dependency --- configure_easybuild | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/configure_easybuild b/configure_easybuild index 7dca1ce682..c67b879cf3 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -25,8 +25,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) -# problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From fd6ac5aa5cb7ad0e5538ebf70335c1c35741e090 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 10 Nov 2023 14:52:14 +0100 Subject: [PATCH 0238/1795] update easystack --- eessi-2023.06-eb-4.8.1-2021b.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index 478899f672..d8a7caff2e 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -9,11 +9,7 @@ easyconfigs: # the --enable-asmjit is not supported on Aarch64 options: from-pr: 19110 - - ScaFaCoS-1.0.4-foss-2021b.eb: - # Newer version of ScaFaCoS for LAMMPS - options: - from-pr: 19163 - LAMMPS-23Jun2022-foss-2021b-kokkos.eb: - # TBB is an optional dependency when building on Intel arch + # TBB and ScaFaCos are optional dependencies when building on Intel arch options: from-pr: 19164 From b03340ac61e2e1d168efe0b56ece7122c06c16a6 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Mon, 13 Nov 2023 16:57:32 +0000 Subject: [PATCH 0239/1795] removed Pillow hook --- eb_hooks.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index ede8401760..c592fcd6ec 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -54,25 +54,12 @@ def get_rpath_override_dirs(software_name): return rpath_injection_dirs -def set_Pillow_envvars(ec): - """Get an EESSI_CPATH environment variable from the environment""" - EESSI_CPATH = os.getenv('EESSI_EPREFIX') + '/usr/include' - EESSI_LIB_PATH = os.getenv('EESSI_EPREFIX') + '/usr/lib64' - if ec.name == 'Pillow': - os.environ['CPATH'] = os.pathsep + EESSI_CPATH - os.environ['LIBRARY_PATH'] = os.pathsep + EESSI_LIB_PATH - print_msg("NOTE: For Pillow which has Szip as a dependancy, CPATH has been set to %s", os.getenv('CPATH')) - print_msg("NOTE: For Pillow which has Szip as a dependancy, LIBRARY_PATH has been set to %s", os.getenv('LIBRARY_PATH')) - ec.log.info("NOTE: For Pillow which has Szip as a dependancy, CPATH has been set to %s", os.getenv('CPATH')) - ec.log.info("NOTE: For Pillow which has Szip as a dependancy, LIBRARY_PATH has been set to %s", os.getenv('LIBRARY_PATH')) - def parse_hook(ec, *args, **kwargs): """Main parse hook: trigger custom functions based on software name.""" # determine path to Prefix installation in compat layer via $EPREFIX eprefix = get_eessi_envvar('EPREFIX') - set_Pillow_envvars(ec) if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) From 09c97bd2b96f1628461f1784d2aaa813d22d9145 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Mon, 13 Nov 2023 16:59:11 +0000 Subject: [PATCH 0240/1795] removed the hook comment --- eessi-2023.06-eb-4.8.1-2022a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index 0aaa66a4ab..b6504915ca 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -12,7 +12,6 @@ easyconfigs: from-pr: 18963 - Pillow-9.1.1-GCCcore-11.3.0.eb: # avoid that hardcoded paths like /usr/include are used in build commands - # Uses a hook to modify the hardcoded LIBRARY and Header paths. options: from-pr: 19266 - matplotlib-3.5.2-foss-2022a.eb From 9b150a73de58592050383adfc7c73b90c2797c69 Mon Sep 17 00:00:00 2001 From: TopRichard Date: Mon, 13 Nov 2023 18:28:17 +0000 Subject: [PATCH 0241/1795] added the eb file in 4.8.2 yml file --- eessi-2023.06-eb-4.8.1-2022a.yml | 5 ----- eessi-2023.06-eb-4.8.2-2022a.yml | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml index b6504915ca..8317a40b7a 100644 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ b/eessi-2023.06-eb-4.8.1-2022a.yml @@ -10,8 +10,3 @@ easyconfigs: - ESPResSo-4.2.1-foss-2022a: options: from-pr: 18963 - - Pillow-9.1.1-GCCcore-11.3.0.eb: - # avoid that hardcoded paths like /usr/include are used in build commands - options: - from-pr: 19266 - - matplotlib-3.5.2-foss-2022a.eb diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml index bab796db2b..81fa757cd7 100644 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ b/eessi-2023.06-eb-4.8.2-2022a.yml @@ -35,3 +35,8 @@ easyconfigs: - WSClean-3.4-foss-2022a: options: from-pr: 19119 + - Pillow-9.1.1-GCCcore-11.3.0.eb: + # avoid that hardcoded paths like /usr/include are used in build commands + options: + from-pr: 19266 + - matplotlib-3.5.2-foss-2022a.eb From b396a56257d42a4a365a97f9e3bcc680404c3fec Mon Sep 17 00:00:00 2001 From: TopRichard Date: Tue, 14 Nov 2023 09:06:56 +0000 Subject: [PATCH 0242/1795] reference to PR#19226 instead --- eessi-2023.06-eb-4.8.2-2022a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml index 81fa757cd7..6406064513 100644 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ b/eessi-2023.06-eb-4.8.2-2022a.yml @@ -38,5 +38,5 @@ easyconfigs: - Pillow-9.1.1-GCCcore-11.3.0.eb: # avoid that hardcoded paths like /usr/include are used in build commands options: - from-pr: 19266 + from-pr: 19226 - matplotlib-3.5.2-foss-2022a.eb From c2e26c159230aa7e06851cd9099368785e189075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 15 Nov 2023 09:33:11 +0100 Subject: [PATCH 0243/1795] restore blank line --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index c592fcd6ec..31f2b9588d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -60,6 +60,7 @@ def parse_hook(ec, *args, **kwargs): # determine path to Prefix installation in compat layer via $EPREFIX eprefix = get_eessi_envvar('EPREFIX') + if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) From da82af74a0f5fd700842cda0f847038ed9469c5e Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 15 Nov 2023 09:49:29 +0100 Subject: [PATCH 0244/1795] change kokkos mapping for Aarch64" --- eb_hooks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index a2890da75e..dfd91ab18b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -19,6 +19,7 @@ CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' +CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -277,7 +278,10 @@ def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs): if self.name == 'LAMMPS': if self.version == '23Jun2022': if get_cpu_architecture() == AARCH64: - self.cfg['kokkos_arch'] = 'A64FX' + if cpu_target == CPU_TARGET_AARCH64_GENERIC: + self.cfg['kokkos_arch'] = 'ARM81' + else: + self.cfg['kokkos_arch'] = 'ARM80' else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") From 151c4924068fe3dd841e3d1380cd2379a50b5d8b Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 15 Nov 2023 09:52:48 +0100 Subject: [PATCH 0245/1795] update kokkos mapping --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index dfd91ab18b..1d01372912 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -279,9 +279,9 @@ def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs): if self.version == '23Jun2022': if get_cpu_architecture() == AARCH64: if cpu_target == CPU_TARGET_AARCH64_GENERIC: - self.cfg['kokkos_arch'] = 'ARM81' - else: self.cfg['kokkos_arch'] = 'ARM80' + else: + self.cfg['kokkos_arch'] = 'ARM81' else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") From ed22feaf78ae50d98dc675f17a3597a8622fc1d1 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 15 Nov 2023 11:25:57 +0100 Subject: [PATCH 0246/1795] update hook --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index 1d01372912..8d05523c2b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -275,6 +275,7 @@ def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs): - set kokkos_arch on Aarch64 """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'LAMMPS': if self.version == '23Jun2022': if get_cpu_architecture() == AARCH64: From 6d138dd2390a8fc9efd843f96c3210640e1535b4 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 16 Nov 2023 17:14:10 +0100 Subject: [PATCH 0247/1795] add fix for sanity check --- eessi-2023.06-eb-4.8.1-2021b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml index d8a7caff2e..6ee44926ba 100644 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ b/eessi-2023.06-eb-4.8.1-2021b.yml @@ -12,4 +12,4 @@ easyconfigs: - LAMMPS-23Jun2022-foss-2021b-kokkos.eb: # TBB and ScaFaCos are optional dependencies when building on Intel arch options: - from-pr: 19164 + from-pr: 19246 From c558a45966cec53d86e4f5a7f5331451c2a19e04 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 18 Nov 2023 10:23:21 +0100 Subject: [PATCH 0248/1795] add gnuplot explicitly (dep for OpenFOAM), to trigger re-install after removing Lua from filtered dependencies in EasyBuild configuration --- eessi-2023.06-eb-4.8.0-2021b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index 477ba6320c..9524dd01df 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -4,4 +4,5 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 options: from-pr: 18746 + - gnuplot-5.4.2-GCCcore-11.2.0.eb - OpenFOAM-v2112-foss-2021b.eb From 73097a32ea0956644adc209ba2097a69340f82e2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 18 Nov 2023 11:25:21 +0100 Subject: [PATCH 0249/1795] remove filter-deps for AOFlagger, since Lua is no longer filtered in general EasyBuild configuration --- eessi-2023.06-eb-4.8.2-2022a.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml index bab796db2b..9921bad045 100644 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ b/eessi-2023.06-eb-4.8.2-2022a.yml @@ -5,27 +5,6 @@ easyconfigs: - AOFlagger-3.4.0-foss-2022a: options: from-pr: 19119 - # Exclude Lua from `filter-deps` as the compat layer version is too old for the software - filter-deps: - - Autoconf - - Automake - - Autotools - - binutils - - bzip2 - - DBus - - flex - - gettext - - gperf - - help2man - - intltool - - libreadline - - libtool - - ncurses - - M4 - - makeinfo - - util-linux - - XZ - - zlib - EveryBeam-0.5.2-foss-2022a: options: from-pr: 19119 From f1ab7f23fbc3bbfa0178c961d8b70248f60db3b3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 18 Nov 2023 12:19:18 +0100 Subject: [PATCH 0250/1795] install gnuplot via easyconfig PR that fixing linking error to Lua dependency --- eessi-2023.06-eb-4.8.0-2021b.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index 9524dd01df..a2562a4ed1 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -4,5 +4,9 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 options: from-pr: 18746 - - gnuplot-5.4.2-GCCcore-11.2.0.eb + - gnuplot-5.4.2-GCCcore-11.2.0.eb: + # make sure that Lua dependency is correctly picked up, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19261 + options: + from-pr: 19261 - OpenFOAM-v2112-foss-2021b.eb From 7c679a03000d5542afde8cf1b34fa6db6335f1ec Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 20 Nov 2023 11:22:24 +0100 Subject: [PATCH 0251/1795] Make Pillow sysroot aware --- eessi-2023.06-eb-4.8.2-2021b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eessi-2023.06-eb-4.8.2-2021b.yml b/eessi-2023.06-eb-4.8.2-2021b.yml index 746f8df05f..00c02adf91 100644 --- a/eessi-2023.06-eb-4.8.2-2021b.yml +++ b/eessi-2023.06-eb-4.8.2-2021b.yml @@ -1,2 +1,6 @@ easyconfigs: + - Pillow-8.3.2-GCCcore-11.2.0.eb: + # avoid that hardcoded paths like /usr/include are used in build commands + options: + from-pr: 19226 - matplotlib-3.4.3-foss-2021b.eb From d451d0622711f7d1900ece73e03596b99e2ffa5e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Nov 2023 22:19:11 +0100 Subject: [PATCH 0252/1795] update init scripts for software.eessi.io (+ drop 'pilot' terminology) --- init/Magic_Castle/bash | 4 ++-- .../{eessi_pilot_python3 => eessi_python3} | 8 ++++---- init/bash | 6 +++--- init/eessi_defaults | 7 +++++-- init/eessi_environment_variables | 10 ++++++---- init/minimal_eessi_env | 4 ++-- 6 files changed, 22 insertions(+), 17 deletions(-) rename init/Magic_Castle/{eessi_pilot_python3 => eessi_python3} (73%) diff --git a/init/Magic_Castle/bash b/init/Magic_Castle/bash index 85e4d54241..dbe1772d3a 100644 --- a/init/Magic_Castle/bash +++ b/init/Magic_Castle/bash @@ -6,7 +6,7 @@ EESSI_SILENT=1 source $(dirname "$BASH_SOURCE")/../eessi_environment_variables # Don't change the default prompt -# export PS1="[EESSI pilot $EESSI_PILOT_VERSION] $ " +# export PS1="[EESSI $EESSI_VERSION] $ " # Provide a clean MODULEPATH export MODULEPATH_ROOT=$EESSI_MODULEPATH @@ -36,4 +36,4 @@ else module reload fi -echo "Environment set up to use EESSI pilot software stack (${EESSI_PILOT_VERSION}), have fun!" +echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!" diff --git a/init/Magic_Castle/eessi_pilot_python3 b/init/Magic_Castle/eessi_python3 similarity index 73% rename from init/Magic_Castle/eessi_pilot_python3 rename to init/Magic_Castle/eessi_python3 index a762ac7b84..b2f7fd4d66 100755 --- a/init/Magic_Castle/eessi_pilot_python3 +++ b/init/Magic_Castle/eessi_python3 @@ -12,10 +12,10 @@ export EESSI_SILENT=1 # for MacOS due to the use of `readlink`) source $(dirname "$(readlink -f "$BASH_SOURCE")")/../eessi_environment_variables -eessi_pilot_python=$(ls ${EESSI_SOFTWARE_PATH}/software/Python/3*GCCcore*/bin/python | sed 1q) -if [ -f "$eessi_pilot_python" ]; then - $eessi_pilot_python "$@" +eessi_python=$(ls ${EESSI_SOFTWARE_PATH}/software/Python/3*GCCcore*/bin/python | sed 1q) +if [ -f "$eessi_python" ]; then + $eessi_python "$@" else - echo "ERROR: No EESSI pilot python 3 available." + echo "ERROR: No EESSI Python 3 available." false fi diff --git a/init/bash b/init/bash index ea605db0b5..84fe783bce 100644 --- a/init/bash +++ b/init/bash @@ -13,7 +13,7 @@ source $(dirname "$BASH_SOURCE")/eessi_environment_variables # only continue if setting EESSI environment variables worked fine if [ $? -eq 0 ]; then - export PS1="[EESSI pilot $EESSI_PILOT_VERSION] $ " + export PS1="[EESSI $EESSI_VERSION] $ " # add location of commands provided by compat layer to $PATH; # see https://github.com/EESSI/software-layer/issues/52 @@ -28,12 +28,12 @@ if [ $? -eq 0 ]; then module use $EESSI_MODULEPATH #echo >> $output - #echo "*** Known problems in the ${EESSI_PILOT_VERSION} pilot software stack ***" >> $output + #echo "*** Known problems in the ${EESSI_VERSION} software stack ***" >> $output #echo >> $output #echo "1) ..." >> $output #echo >> $output #echo >> $output - echo "Environment set up to use EESSI pilot software stack, have fun!" >> $output + echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!" fi diff --git a/init/eessi_defaults b/init/eessi_defaults index 0143dc38ab..d1779a36ae 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -8,5 +8,8 @@ # license: GPLv2 # -export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" -export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}" +export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}" +export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}" +# use archdetect by default, unless otherwise specified +export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=1}" +export EESSI_USE_ARCHSPEC="${EESSI_USE_ARCHSPEC:=0}" diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 10ac1926f4..42f4b6b76a 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -15,11 +15,11 @@ function error() { false } -# set up minimal environment: $EESSI_PREFIX, $EESSI_PILOT_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX +# set up minimal environment: $EESSI_PREFIX, $EESSI_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX source $EESSI_INIT_DIR_PATH/minimal_eessi_env if [ -d $EESSI_PREFIX ]; then - echo "Found EESSI pilot repo @ $EESSI_PREFIX!" >> $output + echo "Found EESSI repo @ $EESSI_PREFIX!" >> $output export EESSI_EPREFIX=$EPREFIX if [ -d $EESSI_EPREFIX ]; then @@ -29,11 +29,13 @@ if [ -d $EESSI_PREFIX ]; then # if archdetect is enabled, use internal code export EESSI_SOFTWARE_SUBDIR=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh cpupath) echo "archdetect says ${EESSI_SOFTWARE_SUBDIR}" >> $output - else + elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined! export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3 export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX) echo "archspec says ${EESSI_SOFTWARE_SUBDIR}" >> $output + else + error "Don't know how to detect host CPU, giving up!" fi if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then @@ -82,5 +84,5 @@ if [ -d $EESSI_PREFIX ]; then error "Compatibility layer directory $EESSI_EPREFIX not found!" fi else - error "EESSI pilot repository at $EESSI_PREFIX not found!" + error "EESSI repository at $EESSI_PREFIX not found!" fi diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index b7cb7c5e9e..5b3562068d 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -4,11 +4,11 @@ # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) -# set up defaults: EESSI_CVMFS_REPO, EESSI_PILOT_VERSION +# set up defaults: EESSI_CVMFS_REPO, EESSI_VERSION # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_PILOT_VERSION +export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From cf6459fe6c978c885417ec489c485dcfe14eb3ab Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Nov 2023 22:23:55 +0100 Subject: [PATCH 0253/1795] also update README for software.eessi.io --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e24de45b0e..ab73ad7579 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,22 @@ # Software layer -The software layer of the EESSI project uses [EasyBuild](https://easybuild.readthedocs.io), [Lmod](https://lmod.readthedocs.io) and [archspec](https://archspec.readthedocs.io). +The software layer of the EESSI project uses [EasyBuild](https://docs.easybuild.io), [Lmod](https://lmod.readthedocs.io) and [archspec](https://archspec.readthedocs.io). -See also https://eessi.github.io/docs/software_layer. +See also https://www.eessi.io/docs/software_layer . ## Pilot software stack You can set up your environment by sourcing the init script: ``` -$ source /cvmfs/pilot.eessi-hpc.org/versions/2023.06/init/bash -Found EESSI pilot repo @ /cvmfs/pilot.eessi-hpc.org/versions/2023.06! +$ source /cvmfs/software.eessi.io/versions/2023.06/init/bash +Found EESSI repo @ /cvmfs/software.eessi.io/versions/2023.06! Derived subdirectory for software layer: x86_64/intel/haswell -Using x86_64/intel/haswell subdirectory for software layer (HARDCODED) +Using x86_64/intel/haswell subdirectory for software layer Initializing Lmod... -Prepending /cvmfs/pilot.eessi-hpc.org/versions/2023.06/software/x86_64/intel/haswell/modules/all to $MODULEPATH... -Environment set up to use EESSI pilot software stack, have fun! -[EESSI pilot 2023.06] $ +Prepending /cvmfs/software.eessi.io/versions/2023.06/software/x86_64/intel/haswell/modules/all to $MODULEPATH... +Environment set up to use EESSI (2023.06), have fun! +[EESSI 2023.06] $ ``` ### Accessing EESSI via a container From 1abade7c24f2bb9077062f4c861619a32f6d2c39 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Nov 2023 22:55:08 +0100 Subject: [PATCH 0254/1795] update install script to only consider easystack files in easystacks/*, which helps to pass tests in CI --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 9586abde1d..2dc9d2b325 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -190,7 +190,7 @@ fi pr_diff=$(ls [0-9]*.diff | head -1) # use PR patch file to determine in which easystack files stuff was added -for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$' | egrep -v 'known-issues|missing'); do +for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do echo -e "Processing easystack file ${easystack_file}...\n\n" From ca7db1551d0c933ba241622b8c141a2ab42eda20 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Nov 2023 23:09:01 +0100 Subject: [PATCH 0255/1795] fix CI for README --- .github/workflows/tests_readme.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index ac906125ab..d229879f67 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -19,10 +19,10 @@ jobs: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults + - name: verify if README.md is consistent with EESSI_VERSION from init/eessi_defaults run: | source init/eessi_defaults - grep "${EESSI_PILOT_VERSION}" README.md + grep "${EESSI_VERSION}" README.md - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults run: | From ad0a5d9c2d26b08484a9ceeb3e27e7fb487b8141 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 09:59:27 +0100 Subject: [PATCH 0256/1795] update script to create init tarball for software.eessi.io --- create_directory_tarballs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh index 4465465082..0270719a73 100755 --- a/create_directory_tarballs.sh +++ b/create_directory_tarballs.sh @@ -1,7 +1,5 @@ #!/bin/bash -SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/2023.06 - set -eo pipefail if [ $# -ne 1 ]; then @@ -11,6 +9,8 @@ fi version=$1 +SOFTWARE_LAYER_TARBALL_URL="https://github.com/EESSI/software-layer/tarball/${version}-software.eessi.io" + TOPDIR=$(dirname $(realpath $0)) source $TOPDIR/scripts/utils.sh @@ -28,9 +28,9 @@ mkdir "${tartmp}/${version}" tarname="eessi-${version}-init-$(date +%s).tar.gz" curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --no-wildcards-match-slash --wildcards '*/init/' source "${tartmp}/${version}/init/minimal_eessi_env" -if [ "${EESSI_PILOT_VERSION}" != "${version}" ] +if [ "${EESSI_VERSION}" != "${version}" ] then - fatal_error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!" + fatal_error "Specified version ${version} does not match version ${EESSI_VERSION} in the init files!" fi tar czf "${tarname}" -C "${tartmp}" "${version}" rm -rf "${tartmp}" From 753296feba5c477a1e90dad84679c0f1775bda27 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 14:27:06 +0100 Subject: [PATCH 0257/1795] update CI workflow for checking for missing installations in /cvmfs/software.eessi.io/versions/2023.06 --- ...st_eessi.yml => test-software.eessi.io.yml} | 18 +++++++++--------- check_missing_installations.sh | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) rename .github/workflows/{test_eessi.yml => test-software.eessi.io.yml} (80%) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test-software.eessi.io.yml similarity index 80% rename from .github/workflows/test_eessi.yml rename to .github/workflows/test-software.eessi.io.yml index 6dfd78e428..86ccbdb663 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -1,11 +1,11 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests relying on having EESSI pilot repo mounted +name: Check for missing software installations in software.eessi.io on: [push, pull_request, workflow_dispatch] permissions: contents: read # to fetch code (actions/checkout) jobs: pilot: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -29,19 +29,19 @@ jobs: with: cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb cvmfs_http_proxy: DIRECT - cvmfs_repositories: pilot.eessi-hpc.org + cvmfs_repositories: software.eessi.io - name: Test check_missing_installations.sh script run: | - source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version - export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" - for easystack_file in $(ls eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? @@ -50,10 +50,10 @@ jobs: - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | - source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version - export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort diff --git a/check_missing_installations.sh b/check_missing_installations.sh index e927f14701..5ea7c5a4f5 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to check for missing installations in EESSI pilot software stack (version 2023.06) +# Script to check for missing installations in EESSI software stack # # author: Kenneth Hoste (@boegel) # author: Thomas Roeblitz (@trz42) From a095a9d02418b42f882d5c468f2eb6fda8cf3e5e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 14:36:59 +0100 Subject: [PATCH 0258/1795] get rid of 'pilot' terminology in scripts --- .github/workflows/tests_scripts.yml | 8 ++++---- ...l-software.sh => EESSI-install-software.sh | 6 +++--- bot/build.sh | 16 +++++++-------- bot/inspect.sh | 20 +++++++++---------- install_software_layer.sh | 2 +- run_in_compat_layer_env.sh | 12 +++++------ 6 files changed, 32 insertions(+), 32 deletions(-) rename EESSI-pilot-install-software.sh => EESSI-install-software.sh (98%) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 74e2ebcffe..589c1458e9 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -5,7 +5,7 @@ on: paths: - build_container.sh - create_directory_tarballs.sh - - EESSI-pilot-install-software.sh + - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh - run_in_compat_layer_env.sh @@ -16,7 +16,7 @@ on: paths: - build_container.sh - create_directory_tarballs.sh - - EESSI-pilot-install-software.sh + - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh - run_in_compat_layer_env.sh @@ -40,7 +40,7 @@ jobs: # bind current directory into container as /software-layer export SINGULARITY_BIND="${PWD}:/software-layer" - # can't test with EasyBuild versions older than v4.5.2 when using EESSI pilot 2023.06, + # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06, # since Python in compat layer is Python 3.11.x; # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version for EB_VERSION in '4.6.0'; do @@ -84,7 +84,7 @@ jobs: cp -a * /tmp/ cd /tmp # force using x86_64/generic, to avoid triggering an installation from scratch - sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh + sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh - name: test create_directory_tarballs.sh script diff --git a/EESSI-pilot-install-software.sh b/EESSI-install-software.sh similarity index 98% rename from EESSI-pilot-install-software.sh rename to EESSI-install-software.sh index 2dc9d2b325..f6087b3cfe 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-install-software.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to install EESSI pilot software stack (version set through init/eessi_defaults) +# Script to install EESSI software stack (version set through init/eessi_defaults) # see example parsing of command line arguments at # https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop @@ -172,8 +172,6 @@ if [ ! -z "${shared_fs_path}" ]; then export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} fi -${EB} --show-config - echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge @@ -200,6 +198,8 @@ for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[ # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + ${EB} --show-config + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." if [ -f ${easystack_file} ]; then diff --git a/bot/build.sh b/bot/build.sh index cf9d69b65f..4af217628e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -128,12 +128,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} -# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") -echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" +export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down @@ -210,9 +210,9 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") timestamp=$(date +%s) -# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now +# to set EESSI_VERSION we need to source init/eessi_defaults now source init/eessi_defaults -export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh @@ -221,8 +221,8 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $ TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/bot/inspect.sh b/bot/inspect.sh index a3b88e5017..9d1fa87e1f 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -195,12 +195,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/inspect.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} -# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") -echo "bot/inspect.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" +export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/inspect.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down @@ -260,11 +260,11 @@ base_dir=$(dirname $(realpath $0)) # TODO better use script from tarball??? source ${base_dir}/../init/eessi_defaults -if [ -z $EESSI_PILOT_VERSION ]; then - echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 +if [ -z $EESSI_VERSION ]; then + echo "ERROR: \$EESSI_VERSION must be set!" >&2 exit 1 fi -EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" +EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" # NOTE The below requires access to the CVMFS repository. We could make a first # test run with a container. For now we skip the test. @@ -295,8 +295,8 @@ fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then echo "export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}" >> ${RESUME_SCRIPT} fi -if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then - echo "export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT} +if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then + echo "export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT} fi if [ ! -z ${http_proxy} ]; then echo "export http_proxy=${http_proxy}" >> ${RESUME_SCRIPT} @@ -428,7 +428,7 @@ echo "Executing command to start interactive session to inspect build job:" # TODO possibly add information on how to init session after the prefix is # entered, initialization consists of # - environment variable settings (see 'run_in_compat_layer_env.sh') -# - setup steps run in 'EESSI-pilot-install-software.sh' +# - setup steps run in 'EESSI-install-software.sh' # These initializations are combined into a single script that is executed when # the shell in startprefix is started. We set the env variable BASH_ENV here. if [[ -z ${run_in_prefix} ]]; then diff --git a/install_software_layer.sh b/install_software_layer.sh index bf3006a4a0..82ca70b73f 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,4 +1,4 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -./run_in_compat_layer_env.sh ./EESSI-pilot-install-software.sh "$@" +./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 12688e2aed..f57c4d0749 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -3,11 +3,11 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -if [ -z $EESSI_PILOT_VERSION ]; then - echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 +if [ -z $EESSI_VERSION ]; then + echo "ERROR: \$EESSI_VERSION must be set!" >&2 exit 1 fi -EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" +EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2 exit 1 @@ -23,8 +23,8 @@ fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" fi -if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then - INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}" fi if [ ! -z ${http_proxy} ]; then INPUT="export http_proxy=${http_proxy}; ${INPUT}" @@ -33,5 +33,5 @@ if [ ! -z ${https_proxy} ]; then INPUT="export https_proxy=${https_proxy}; ${INPUT}" fi -echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." +echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From 50ec01a558c16c91e185fdc96f973e7cb4e6fee4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 14:47:48 +0100 Subject: [PATCH 0259/1795] update eessi_container.sh script for software.eessi.io --- .../workflows/test_eessi_container_script.yml | 16 ++++---- create_tarball.sh | 24 +++++------ eessi_container.sh | 40 ++++++++++--------- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 929fb22cec..33122e6ff4 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -45,7 +45,7 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then outfile=out_listrepos.txt ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI-pilot" ${outfile} + grep "EESSI" ${outfile} # test use of --list-repos with custom repos.cfg elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then @@ -57,7 +57,7 @@ jobs: echo "[EESSI/20HT.TP]" >> cfg/repos.cfg echo "repo_version = 20HT.TP" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI-pilot" ${outfile} + grep "EESSI" ${outfile} export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile2} @@ -90,15 +90,15 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then outfile=out_readwrite.txt fn="test_${RANDOM}.txt" - echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh + echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh chmod u+x test_script.sh export SINGULARITY_BIND="$PWD:/test" ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile} tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") # note: must use '--access rw' again here, since touched file is in overlay upper dir - ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile # test use of --resume elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then @@ -120,12 +120,12 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then outfile=out_save.txt fn="test_${RANDOM}.txt" - test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" + test_cmd="touch /cvmfs/software.eessi.io/${fn}" ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} rm -f ${outfile} - ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile tar tfv test-save.tar | grep "overlay-upper/${fn}" diff --git a/create_tarball.sh b/create_tarball.sh index 65f6efc2dc..8510caebf1 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -5,11 +5,11 @@ set -e base_dir=$(dirname $(realpath $0)) if [ $# -ne 4 ]; then - echo "ERROR: Usage: $0 " >&2 + echo "ERROR: Usage: $0 " >&2 exit 1 fi eessi_tmpdir=$1 -pilot_version=$2 +eessi_version=$2 cpu_arch_subdir=$3 target_tgz=$4 @@ -20,7 +20,7 @@ os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} -software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" +software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 @@ -28,7 +28,7 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" -software_dir_overlay="${overlay_upper_dir}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" +software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 @@ -40,22 +40,22 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}.. files_list=${tmpdir}/files.list.txt module_files_list=${tmpdir}/module_files.list.txt -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} # module files and symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' # using EasyBuild's standard module naming scheme) match the name of the @@ -64,7 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${modu # installation directories), the procedure will likely not work. for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ + ls -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ | grep -v '/\.wh\.' >> ${files_list} done fi diff --git a/eessi_container.sh b/eessi_container.sh index 48c4653ba9..bf0294c7bf 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -79,7 +79,7 @@ display_help() { echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script or command) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot via" + echo " repository to use [default: EESSI via" echo " default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" @@ -111,7 +111,7 @@ VERBOSE=0 STORAGE= LIST_REPOS=0 MODE="shell" -REPOSITORY="EESSI-pilot" +REPOSITORY="EESSI" RESUME= SAVE= HTTP_PROXY=${http_proxy:-} @@ -194,7 +194,7 @@ set -- "${POSITIONAL_ARGS[@]}" if [[ ${LIST_REPOS} -eq 1 ]]; then echo "Listing available repositories with format 'name [source]':" - echo " EESSI-pilot [default]" + echo " EESSI [default]" if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then cfg_load ${EESSI_REPOS_CFG_FILE} sections=$(cfg_sections) @@ -226,7 +226,7 @@ fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE -if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then +if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" fi @@ -403,7 +403,7 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg -if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then +if [[ "${REPOSITORY}" == "EESSI" ]]; then # need to source defaults as late as possible (see other sourcing below) source ${TOPDIR}/init/eessi_defaults @@ -427,7 +427,7 @@ else # map { local_filepath -> container_filepath } # # repo_name_domain is the domain part of the repo_name, e.g., - # eessi-hpc.org for pilot.eessi-hpc.org + # eessi.io for software.eessi.io # # where config bundle includes the files (-> target location in container) # - default.local -> /etc/cvmfs/default.local @@ -479,7 +479,7 @@ else target=${cfg_file_map[${src}]} BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done - export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} + export EESSI_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" # need to source defaults as late as possible (after *_OVERRIDEs) source ${TOPDIR}/init/eessi_defaults @@ -513,10 +513,14 @@ fi # 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() + +# always mount cvmfs-config repo (to get access to software.eessi.io) +EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") + if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" + export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") export EESSI_FUSE_MOUNTS fi @@ -525,18 +529,18 @@ if [[ "${ACCESS}" == "rw" ]]; then mkdir -p ${EESSI_TMPDIR}/overlay-work # set environment variables for fuse mounts in Singularity container - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" - EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_PILOT_WRITABLE_OVERLAY + EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" + export EESSI_WRITABLE_OVERLAY - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") export EESSI_FUSE_MOUNTS fi From ad6604fd0cf8bd6bd3ded84ec69d041e194332b5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 14:57:28 +0100 Subject: [PATCH 0260/1795] remove easystack files used for pilot.eessi-hpc.org --- eessi-2021.06.yml | 53 ------------------------ eessi-2021.12.yml | 69 ------------------------------- eessi-2023.06-eb-4.7.2-2021a.yml | 26 ------------ eessi-2023.06-eb-4.7.2-2021b.yml | 17 -------- eessi-2023.06-eb-4.7.2-2022a.yml | 3 -- eessi-2023.06-eb-4.7.2-2022b.yml | 13 ------ eessi-2023.06-eb-4.7.2-system.yml | 7 ---- eessi-2023.06-eb-4.8.0-2021a.yml | 2 - eessi-2023.06-eb-4.8.0-2021b.yml | 12 ------ eessi-2023.06-eb-4.8.0-system.yml | 10 ----- eessi-2023.06-eb-4.8.1-2021a.yml | 7 ---- eessi-2023.06-eb-4.8.1-2021b.yml | 15 ------- eessi-2023.06-eb-4.8.1-2022a.yml | 12 ------ eessi-2023.06-eb-4.8.1-2023a.yml | 11 ----- eessi-2023.06-eb-4.8.1-system.yml | 9 ---- eessi-2023.06-eb-4.8.2-2021b.yml | 6 --- eessi-2023.06-eb-4.8.2-2022a.yml | 21 ---------- eessi-2023.06-known-issues.yml | 30 -------------- 18 files changed, 323 deletions(-) delete mode 100644 eessi-2021.06.yml delete mode 100644 eessi-2021.12.yml delete mode 100644 eessi-2023.06-eb-4.7.2-2021a.yml delete mode 100644 eessi-2023.06-eb-4.7.2-2021b.yml delete mode 100644 eessi-2023.06-eb-4.7.2-2022a.yml delete mode 100644 eessi-2023.06-eb-4.7.2-2022b.yml delete mode 100644 eessi-2023.06-eb-4.7.2-system.yml delete mode 100644 eessi-2023.06-eb-4.8.0-2021a.yml delete mode 100644 eessi-2023.06-eb-4.8.0-2021b.yml delete mode 100644 eessi-2023.06-eb-4.8.0-system.yml delete mode 100644 eessi-2023.06-eb-4.8.1-2021a.yml delete mode 100644 eessi-2023.06-eb-4.8.1-2021b.yml delete mode 100644 eessi-2023.06-eb-4.8.1-2022a.yml delete mode 100644 eessi-2023.06-eb-4.8.1-2023a.yml delete mode 100644 eessi-2023.06-eb-4.8.1-system.yml delete mode 100644 eessi-2023.06-eb-4.8.2-2021b.yml delete mode 100644 eessi-2023.06-eb-4.8.2-2022a.yml delete mode 100644 eessi-2023.06-known-issues.yml diff --git a/eessi-2021.06.yml b/eessi-2021.06.yml deleted file mode 100644 index 3587827746..0000000000 --- a/eessi-2021.06.yml +++ /dev/null @@ -1,53 +0,0 @@ -software: - R-bundle-Bioconductor: - toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 - GROMACS: - toolchains: - foss-2020a: - versions: - '2020.1': - versionsuffix: -Python-3.8.2 - '2020.4': - versionsuffix: -Python-3.8.2 - Horovod: - toolchains: - foss-2020a: - versions: - '0.21.3': - versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 - OpenFOAM: - toolchains: - foss-2020a: - versions: ['8', 'v2006'] - OSU-Micro-Benchmarks: - toolchains: - gompi-2020a: - versions: ['5.6.3'] - QuantumESPRESSO: - toolchains: - foss-2020a: - versions: ['6.6'] - TensorFlow: - toolchains: - foss-2020a: - versions: - '2.3.1': - versionsuffix: -Python-3.8.2 - RStudio-Server: - toolchains: - foss-2020a: - versions: - '1.3.1093': - versionsuffix: -Java-11-R-4.0.0 - ReFrame: - toolchains: - SYSTEM: - versions: '3.6.2' - code-server: - toolchains: - SYSTEM: - versions: '3.7.3' diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml deleted file mode 100644 index 210bbb2845..0000000000 --- a/eessi-2021.12.yml +++ /dev/null @@ -1,69 +0,0 @@ -software: - code-server: - toolchains: - SYSTEM: - versions: '3.7.3' - GROMACS: - toolchains: - foss-2020a: - versions: - '2020.1': - versionsuffix: -Python-3.8.2 - '2020.4': - versionsuffix: -Python-3.8.2 - Horovod: - toolchains: - foss-2020a: - versions: - '0.21.3': - versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 - Nextflow: - toolchains: - SYSTEM: - versions: '22.10.1' - OpenFOAM: - toolchains: - foss-2020a: - versions: ['8', 'v2006'] - OSU-Micro-Benchmarks: - toolchains: - gompi-2020a: - versions: ['5.6.3'] - gompi-2021a: - versions: ['5.7.1'] - QuantumESPRESSO: - toolchains: - foss-2020a: - versions: ['6.6'] - R: - toolchains: - foss-2021a: - versions: '4.1.0' - R-bundle-Bioconductor: - toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 - RStudio-Server: - toolchains: - foss-2020a: - versions: - '1.3.1093': - versionsuffix: -Java-11-R-4.0.0 - SciPy-bundle: - toolchains: - foss-2021a: - versions: ['2021.05'] - TensorFlow: - toolchains: - foss-2020a: - versions: - '2.3.1': - versionsuffix: -Python-3.8.2 - WRF: - toolchains: - foss-2020a: - versions: - '3.9.1.1': - versionsuffix: -dmpar diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml deleted file mode 100644 index 42a42f2236..0000000000 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ /dev/null @@ -1,26 +0,0 @@ -easyconfigs: - - GCC-10.3.0 - - CMake-3.20.1-GCCcore-10.3.0.eb: - options: - include-easyblocks-from-pr: 2248 - - foss-2021a - - libpng-1.6.37-GCCcore-10.3.0.eb: - - libjpeg-turbo-2.0.6-GCCcore-10.3.0.eb - - git-2.32.0-GCCcore-10.3.0-nodocs.eb - - QuantumESPRESSO-6.7-foss-2021a.eb - - GROMACS-2021.3-foss-2021a.eb - - Qt5-5.15.2-GCCcore-10.3.0.eb: - # add missing patch files for Qt5 5.15.2 to fix build problems with glibc 2.34, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18087 - options: - from-pr: 18087 - - RapidJSON-1.1.0-GCCcore-10.3.0.eb: - # strip out hardcoded -march=native, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18725 - options: - from-pr: 18725 - - Arrow-6.0.0-foss-2021a.eb: - # fix installation of pyarrow Python bindings - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18348 - options: - from-pr: 18348 diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml deleted file mode 100644 index 43a0fbd4bd..0000000000 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ /dev/null @@ -1,17 +0,0 @@ -easyconfigs: - - GCC-11.2.0 - - CMake-3.21.1-GCCcore-11.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - CMake-3.22.1-GCCcore-11.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - zlib-1.2.11-GCCcore-11.2.0 - - OpenMPI-4.1.1-GCC-11.2.0 - - gompi-2021b - - FlexiBLAS-3.0.4-GCC-11.2.0.eb - - foss-2021b.eb - - git-2.33.1-GCCcore-11.2.0-nodocs.eb - - QuantumESPRESSO-6.8-foss-2021b.eb - - SciPy-bundle-2021.10-foss-2021b - - GROMACS-2021.5-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.7.2-2022a.yml b/eessi-2023.06-eb-4.7.2-2022a.yml deleted file mode 100644 index d40ddff261..0000000000 --- a/eessi-2023.06-eb-4.7.2-2022a.yml +++ /dev/null @@ -1,3 +0,0 @@ -easyconfigs: - - GCC-11.3.0 - - OpenMPI-4.1.4-GCC-11.3.0.eb diff --git a/eessi-2023.06-eb-4.7.2-2022b.yml b/eessi-2023.06-eb-4.7.2-2022b.yml deleted file mode 100644 index b97be34dcf..0000000000 --- a/eessi-2023.06-eb-4.7.2-2022b.yml +++ /dev/null @@ -1,13 +0,0 @@ -easyconfigs: - - GCC-12.2.0 - - OpenMPI-4.1.4-GCC-12.2.0 - - CMake-3.24.3-GCCcore-12.2.0.eb: - options: - include-easyblocks-from-pr: 2248 - - foss-2022b - - netCDF-4.9.0-gompi-2022b.eb: - # use updated CMakeMake easyblock to avoid that -DCMAKE_SKIP_RPATH=ON is used, which breaks the netCDF test step - # see https://github.com/easybuilders/easybuild-easyblocks/pull/3012 - options: - include-easyblocks-from-pr: 3012 - - WRF-4.4.1-foss-2022b-dmpar.eb diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml deleted file mode 100644 index 220fd82e87..0000000000 --- a/eessi-2023.06-eb-4.7.2-system.yml +++ /dev/null @@ -1,7 +0,0 @@ -easyconfigs: - - ReFrame-4.2.0.eb: - options: - from-pr: 18320 - - EasyBuild-4.8.0.eb: - options: - from-pr: 18282 diff --git a/eessi-2023.06-eb-4.8.0-2021a.yml b/eessi-2023.06-eb-4.8.0-2021a.yml deleted file mode 100644 index 1adf0572d3..0000000000 --- a/eessi-2023.06-eb-4.8.0-2021a.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - GDAL-3.3.0-foss-2021a.eb diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml deleted file mode 100644 index a2562a4ed1..0000000000 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ /dev/null @@ -1,12 +0,0 @@ -easyconfigs: - - MPFR-4.1.0-GCCcore-11.2.0.eb: - # use patch for MPFR 4.1.0 to fix failing tsprintf test with glibc >= 2.37, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18746 - options: - from-pr: 18746 - - gnuplot-5.4.2-GCCcore-11.2.0.eb: - # make sure that Lua dependency is correctly picked up, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19261 - options: - from-pr: 19261 - - OpenFOAM-v2112-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.8.0-system.yml b/eessi-2023.06-eb-4.8.0-system.yml deleted file mode 100644 index 2928f110b0..0000000000 --- a/eessi-2023.06-eb-4.8.0-system.yml +++ /dev/null @@ -1,10 +0,0 @@ -easyconfigs: - - Java-11.eb: - # patch Java binaries/libraries when using alternate sysroot to ensure correct glibc & co are picked up - # see https://github.com/easybuilders/easybuild-easyblocks/pull/2557 - # + https://github.com/easybuilders/easybuild-easyblocks/pull/2995 - options: - include-easyblocks-from-pr: 2995 - - EasyBuild-4.8.1.eb: - options: - from-pr: 18761 diff --git a/eessi-2023.06-eb-4.8.1-2021a.yml b/eessi-2023.06-eb-4.8.1-2021a.yml deleted file mode 100644 index 999eb09925..0000000000 --- a/eessi-2023.06-eb-4.8.1-2021a.yml +++ /dev/null @@ -1,7 +0,0 @@ -easyconfigs: - - Xvfb-1.20.11-GCCcore-10.3.0.eb: - # fix permission issues for xvfb-run script, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18834 - options: - from-pr: 18834 - - R-4.1.0-foss-2021a.eb diff --git a/eessi-2023.06-eb-4.8.1-2021b.yml b/eessi-2023.06-eb-4.8.1-2021b.yml deleted file mode 100644 index 6ee44926ba..0000000000 --- a/eessi-2023.06-eb-4.8.1-2021b.yml +++ /dev/null @@ -1,15 +0,0 @@ -easyconfigs: - - Xvfb-1.20.13-GCCcore-11.2.0.eb: - # enable exec permissions for xvfb-run after copying; - # need to also enable user write permissions on xvfb-run to ensure that copying with preserved permissions works - options: - from-pr: 18834 - - R-4.2.0-foss-2021b.eb - - PLUMED-2.7.3-foss-2021b.eb: - # the --enable-asmjit is not supported on Aarch64 - options: - from-pr: 19110 - - LAMMPS-23Jun2022-foss-2021b-kokkos.eb: - # TBB and ScaFaCos are optional dependencies when building on Intel arch - options: - from-pr: 19246 diff --git a/eessi-2023.06-eb-4.8.1-2022a.yml b/eessi-2023.06-eb-4.8.1-2022a.yml deleted file mode 100644 index 8317a40b7a..0000000000 --- a/eessi-2023.06-eb-4.8.1-2022a.yml +++ /dev/null @@ -1,12 +0,0 @@ -easyconfigs: - - OpenBLAS-0.3.20-GCC-11.3.0: - # use easyconfig that includes patch to fix detection of Neoverse V1 CPUs, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18870 - options: - from-pr: 18870 - - foss-2022a - - SciPy-bundle-2022.05-foss-2022a - - BAMM-2.5.0-foss-2022a.eb - - ESPResSo-4.2.1-foss-2022a: - options: - from-pr: 18963 diff --git a/eessi-2023.06-eb-4.8.1-2023a.yml b/eessi-2023.06-eb-4.8.1-2023a.yml deleted file mode 100644 index c766815934..0000000000 --- a/eessi-2023.06-eb-4.8.1-2023a.yml +++ /dev/null @@ -1,11 +0,0 @@ -easyconfigs: - - GCC-12.3.0 - - OpenBLAS-0.3.23-GCC-12.3.0: - # add patch to disable flaky DDRGES3 LAPACK test, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18887; - # also pulls in patch to avoid hang when running OpenBLAS test suite, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/18790 - options: - from-pr: 18887 - - foss-2023a - - X11-20230603-GCCcore-12.3.0 diff --git a/eessi-2023.06-eb-4.8.1-system.yml b/eessi-2023.06-eb-4.8.1-system.yml deleted file mode 100644 index b0731d2534..0000000000 --- a/eessi-2023.06-eb-4.8.1-system.yml +++ /dev/null @@ -1,9 +0,0 @@ -easyconfigs: - # rebuilt with EasyBuild v4.8.1 (now wraps around Java/11.0.20) - - Java-11.eb - - ReFrame-4.3.3.eb: - options: - from-pr: 18851 - - EasyBuild-4.8.2.eb: - options: - from-pr: 19105 diff --git a/eessi-2023.06-eb-4.8.2-2021b.yml b/eessi-2023.06-eb-4.8.2-2021b.yml deleted file mode 100644 index 00c02adf91..0000000000 --- a/eessi-2023.06-eb-4.8.2-2021b.yml +++ /dev/null @@ -1,6 +0,0 @@ -easyconfigs: - - Pillow-8.3.2-GCCcore-11.2.0.eb: - # avoid that hardcoded paths like /usr/include are used in build commands - options: - from-pr: 19226 - - matplotlib-3.4.3-foss-2021b.eb diff --git a/eessi-2023.06-eb-4.8.2-2022a.yml b/eessi-2023.06-eb-4.8.2-2022a.yml deleted file mode 100644 index ccd3fcd65d..0000000000 --- a/eessi-2023.06-eb-4.8.2-2022a.yml +++ /dev/null @@ -1,21 +0,0 @@ -easyconfigs: - - casacore-3.5.0-foss-2022a: - options: - from-pr: 19119 - - AOFlagger-3.4.0-foss-2022a: - options: - from-pr: 19119 - - EveryBeam-0.5.2-foss-2022a: - options: - from-pr: 19119 - - DP3-6.0-foss-2022a: - options: - from-pr: 19119 - - WSClean-3.4-foss-2022a: - options: - from-pr: 19119 - - Pillow-9.1.1-GCCcore-11.3.0.eb: - # avoid that hardcoded paths like /usr/include are used in build commands - options: - from-pr: 19226 - - matplotlib-3.5.2-foss-2022a.eb diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml deleted file mode 100644 index ade1c069df..0000000000 --- a/eessi-2023.06-known-issues.yml +++ /dev/null @@ -1,30 +0,0 @@ -- x86_64/*: - - ESPResSo-4.2.1-foss-2022a: - - issue: https://github.com/EESSI/software-layer/issues/363 - - info: "Failing tests in ESPResSo v4.2.1 due to timeouts" -- aarch64/*: - - ESPResSo-4.2.1-foss-2022a: - - issue: https://github.com/EESSI/software-layer/issues/363 - - info: "Failing tests in ESPResSo v4.2.1 due to timeouts" -- aarch64/neoverse_v1: - - FFTW-3.3.10-GCC-11.3.0: - - issue: https://github.com/EESSI/software-layer/issues/325 - - info: "Flaky FFTW tests, random failures" - - FFTW-3.3.10-gompi-2021b: - - issue: https://github.com/EESSI/software-layer/issues/325 - - info: "Flaky FFTW tests, random failures" - - OpenBLAS-0.3.18-GCC-11.2.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - - OpenBLAS-0.3.20-GCC-11.3.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (238 vs max. 150 on x86_64/*)" - - OpenBLAS-0.3.21-GCC-12.2.0: - - issue: https://github.com/EESSI/software-layer/issues/314 - - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" - - SciPy-bundle-2021.05-foss-2021a: - - issue: https://github.com/EESSI/software-layer/issues/318 - - info: "2 failing tests (vs 30554 passed) in scipy test suite" - - SciPy-bundle-2021.10-foss-2021b: - - issue: https://github.com/EESSI/software-layer/issues/318 - - info: "20 failing tests (vs 14429 passed) in numpy test suite + 55 failing tests (vs 32438 passed) in scipy test suite" From 34adbe55ca04aa4464eb726f9173bc98aea6af05 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 16:20:36 +0100 Subject: [PATCH 0261/1795] comment out power9le test for archdetect, since that target is not supported currently in software.eessi.io --- .github/workflows/tests_archdetect.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 37338693c5..01138daad4 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -13,10 +13,12 @@ jobs: - x86_64/intel/skylake_avx512/archspec-linux-6132 - x86_64/amd/zen2/Azure-CentOS7-7V12 - x86_64/amd/zen3/Azure-CentOS7-7V73X - - ppc64le/power9le/unknown-power9le - aarch64/neoverse_n1/Azure-Ubuntu20-Altra - aarch64/neoverse_n1/AWS-awslinux-graviton2 - aarch64/neoverse_v1/AWS-awslinux-graviton3 + # commented out since these targets are currently not supported in software.eessi.io repo + # (and some tests assume that the corresponding subdirectory in software layer is there) + # - ppc64le/power9le/unknown-power9le fail-fast: false steps: - name: checkout From 468c42440ee03678fd5b1188ebdadd4564ab2744 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 16:36:14 +0100 Subject: [PATCH 0262/1795] check that archdetect honors $EESSI_SOFTWARE_SUBDIR_OVERRIDE + only check for directories in software-layer if it has been populated already --- .github/workflows/tests_archdetect.yml | 28 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 01138daad4..9cd2775108 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -30,22 +30,36 @@ jobs: export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*} export EESSI_PROC_CPUINFO=./tests/archdetect/${{matrix.proc_cpuinfo}}.cpuinfo + # check that printing of best match works correctly CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) if [[ $CPU_ARCH == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.output )" ]]; then - echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" >&2 + echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" else echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCH" >&2 exit 1 fi + # check that $EESSI_SOFTWARE_SUBDIR_OVERRIDE is honored + export EESSI_SOFTWARE_SUBDIR_OVERRIDE='dummy/cpu' + CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) + if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE PASSED" + else + echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE FAILED" >&2 + exit 1 + fi + unset EESSI_SOFTWARE_SUBDIR_OVERRIDE + # check that printing of all matches works correctly (-a option for cpupath action) CPU_ARCHES=$(./init/eessi_archdetect.sh -a cpupath) if [[ $CPU_ARCHES == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.all.output )" ]]; then - echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES" >&2 + echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES" else echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2 exit 1 fi - # Check all those architectures actually exist - for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do - # Search all EESSI versions as we may drop support at some point - ls -d "$EESSI_PREFIX"/../*/software/linux/"$dir" - done + # Check all those architectures actually exist (if this EESSI version has been populated already) + if [ -d ${EESSI_PREFIX}/software/linux ]; then + for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do + # Search all EESSI versions as we may drop support at some point + ls -d ${EESSI_PREFIX}/software/linux/${dir} + done + fi From 04a71f761e52e6b83b5d879ae22ebdcf862300fb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 16:38:57 +0100 Subject: [PATCH 0263/1795] fix picking up on $EESSI_SOFTWARE_SUBDIR_OVERRIDE in archdetect --- init/eessi_archdetect.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 58dd99eb6b..58b79b0f2a 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash VERSION="1.1.0" -# Logging -LOG_LEVEL="INFO" +# default log level: only emit warnings or errors +LOG_LEVEL="WARN" # Default result type is a best match CPUPATH_RESULT="best" @@ -69,8 +69,8 @@ check_allinfirst(){ cpupath(){ # If EESSI_SOFTWARE_SUBDIR_OVERRIDE is set, use it - log "DEBUG" "cpupath: Override variable set as '$EESI_SOFTWARE_SUBDIR_OVERRIDE' " - [ $EESI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESI_SOFTWARE_SUBDIR_OVERRIDE} && exit + log "DEBUG" "cpupath: Override variable set as '$EESSI_SOFTWARE_SUBDIR_OVERRIDE' " + [ $EESSI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} && exit # Identify the best matching CPU architecture from a list of supported specifications for the host CPU # Return the path to the installation files in EESSI of the best matching architecture From d0f26f8f38de507e4d13bc5e055be4de91841764 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 16:47:50 +0100 Subject: [PATCH 0264/1795] mount software.eessi.io when testing archdetect --- .github/workflows/tests_archdetect.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 9cd2775108..1e8b830e14 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -23,8 +23,14 @@ jobs: steps: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: Enable EESSI - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + - name: test eessi_archdetect.sh run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} From 5d532ee7f7933de2c4cab11fe717c33593bf1483 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 17:28:14 +0100 Subject: [PATCH 0265/1795] stop using build_container.sh script in .github/workflows/tests_scripts.yml, use eessi_container.sh script instead --- .github/workflows/tests_scripts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 589c1458e9..8128a6da94 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -85,7 +85,7 @@ jobs: cd /tmp # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh - ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh + ./eessi_container.sh --mode run --verbose /tmp/install_software_layer.sh - name: test create_directory_tarballs.sh script run: | @@ -93,6 +93,6 @@ jobs: # since create_directory_tarballs.sh must be accessible from within build container cp -a * /tmp/ cd /tmp - ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06 + ./eessi_container.sh --mode run --verbose /tmp/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz From 51ce189da2426e1f91acb1462821ecf7550d2d90 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 11:59:44 +0100 Subject: [PATCH 0266/1795] fix workflow for scripts by bind-mounting repo to /software-layer --- .github/workflows/tests_scripts.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 8128a6da94..a369f4f187 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -79,20 +79,18 @@ jobs: - name: test install_software_layer.sh script run: | - # scripts need to be copied to /tmp, - # since install_software_layer.sh must be accessible from within build container - cp -a * /tmp/ - cd /tmp + # bind current directory into container as /software-layer + export SINGULARITY_BIND="${PWD}:/software-layer" # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh - ./eessi_container.sh --mode run --verbose /tmp/install_software_layer.sh + ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh - name: test create_directory_tarballs.sh script run: | + # bind current directory into container as /software-layer + export SINGULARITY_BIND="${PWD}:/software-layer" # scripts need to be copied to /tmp, # since create_directory_tarballs.sh must be accessible from within build container - cp -a * /tmp/ - cd /tmp - ./eessi_container.sh --mode run --verbose /tmp/create_directory_tarballs.sh 2023.06 + ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz From c9ab0de1ee746b2ef7be867a529cc239113a968a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 14:23:20 +0100 Subject: [PATCH 0267/1795] {2023.06} foss/2023a --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml new file mode 100644 index 0000000000..75e05a146c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -0,0 +1,7 @@ +easyconfigs: + - Rust-1.70.0-GCCcore-12.3.0.eb: + # fix build of Rust 1.70.0 by disabling download of pre-built LLVM; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19264 + options: + from-pr: 19264 + - foss-2023a.eb From ff615929d282472074531df6b559389f1a06d3d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Nov 2023 15:01:10 +0100 Subject: [PATCH 0268/1795] add initial easystack file for software.eessi.io to check on EasyBuild v4.8.2 installation --- .../2023.06/eessi-2023.06-eb-4.8.2-system.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml new file mode 100644 index 0000000000..5ce6a65913 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - EasyBuild-4.8.2.eb: + options: + from-pr: 19105 From 63a968b088ce3a680bf938687536a70542581703 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 20:48:19 +0100 Subject: [PATCH 0269/1795] add JSON file + Python script to keep track of software licenses --- licenses/README.md | 3 ++ licenses/licenses.json | 10 +++++ licenses/spdx.py | 100 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 licenses/README.md create mode 100644 licenses/licenses.json create mode 100644 licenses/spdx.py diff --git a/licenses/README.md b/licenses/README.md new file mode 100644 index 0000000000..36a7615b21 --- /dev/null +++ b/licenses/README.md @@ -0,0 +1,3 @@ +see https://spdx.org/licenses + +Python function to download SPDX list of licenses is available in `spdx.py` diff --git a/licenses/licenses.json b/licenses/licenses.json new file mode 100644 index 0000000000..8831ed368c --- /dev/null +++ b/licenses/licenses.json @@ -0,0 +1,10 @@ +{ + "EasyBuild": { + "spdx": "GPL-2.0-only", + "license_url": "https://easybuild.io" + }, + "GCCcore": { + "spdx": "GPL-2.0-with-GCC-exception", + "license_url": "https://github.com/gcc-mirror/gcc/blob/master/COPYING" + } +} diff --git a/licenses/spdx.py b/licenses/spdx.py new file mode 100644 index 0000000000..06c3edb4e6 --- /dev/null +++ b/licenses/spdx.py @@ -0,0 +1,100 @@ +import json +import logging +import sys +import urllib.request + +SPDX_LICENSE_LIST_URL = 'https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json' + +LICENSE_URL = 'license_url' +SPDX = 'spdx' + +spdx_license_list = None + +# Configure the logging module +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + + +def get_spdx_license_list(): + """ + Download JSON file with current list of SPDX licenses, parse it, and return it as a Python dictionary. + """ + global spdx_license_list + + if spdx_license_list is None: + with urllib.request.urlopen(SPDX_LICENSE_LIST_URL) as fp: + spdx_license_list = json.load(fp) + version, release_date = spdx_license_list['licenseListVersion'], spdx_license_list['releaseDate'] + logging.info(f"Downloaded version {version} of SPDX license list (release date: {release_date})") + licenses = spdx_license_list['licenses'] + logging.info(f"Found info on {len(licenses)} licenses!") + + return spdx_license_list + + +def license_info(spdx_id): + """Find license with specified SPDX identifier.""" + + spdx_license_list = get_spdx_license_list() + + licenses = spdx_license_list['licenses'] + for lic in licenses: + if lic['licenseId'] == spdx_id: + return lic + + # if no match is found, return None as result + return None + + +def read_licenses(path): + """ + Read software project to license mapping from specified path + """ + with open(path) as fp: + licenses = json.loads(fp.read()) + + return licenses + + +def check_licenses(licenses): + """ + Check mapping of software licenses: make sure SPDX identifiers are valid. + """ + faulty_licenses = {} + + for software_name in licenses: + spdx_lic_id = licenses[software_name][SPDX] + lic_info = license_info(spdx_lic_id) + if lic_info: + lic_url = licenses[software_name][LICENSE_URL] + logging.info(f"License for software '{software_name}': {lic_info['name']} (see {lic_url})") + else: + logging.warning(f"Found faulty SPDX license ID for {software_name}: {spdx_lic_id}") + faulty_licenses[software_name] = spdx_lic_id + + if faulty_licenses: + logging.warning(f"Found {len(faulty_licenses)} faulty SPDIX license IDs (out of {len(licenses)})!") + result = False + else: + logging.info(f"License check passed for {len(licenses)} licenses!") + result = True + + return result + + +def main(args): + if len(args) == 1: + licenses_path = args[0] + else: + logging.error("Usage: python spdx.py ") + sys.exit(1) + + licenses = read_licenses(licenses_path) + if check_licenses(licenses): + logging.info("All license checks PASSED!") + else: + logging.error("One or more licence checks failed!") + sys.exit(2) + + +if __name__ == '__main__': + main(sys.argv[1:]) From fd765bbd41691e49d8efcf044fb07e3632049d85 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 21:04:47 +0100 Subject: [PATCH 0270/1795] add dummy bot/test.sh and bot/check-test.sh scripts --- bot/check-test.sh | 20 ++++++++++++++++++++ bot/test.sh | 13 +++++++++++++ 2 files changed, 33 insertions(+) create mode 100755 bot/check-test.sh create mode 100755 bot/test.sh diff --git a/bot/check-test.sh b/bot/check-test.sh new file mode 100755 index 0000000000..76e0df7f40 --- /dev/null +++ b/bot/check-test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Dummy script that only creates test result file for the bot, without actually checking anything +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Kenneth Hoste (HPC-UGent) +# +# license: GPLv2 +# +job_dir=${PWD} +job_out="slurm-${SLURM_JOB_ID}.out" +job_test_result_file="_bot_job${SLURM_JOB_ID}.test" + +echo "[TEST]" > ${job_test_result_file} +echo "comment_description = (no tests yet)" >> ${job_test_result_file} +echo "status = SUCCESS" >> ${job_test_result_file} + +exit 0 diff --git a/bot/test.sh b/bot/test.sh new file mode 100755 index 0000000000..9d978cdcd0 --- /dev/null +++ b/bot/test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +# Dummy script, no tests yet +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Kenneth Hoste (HPC-UGent) +# +# license: GPLv2 +# + +exit 0 From 0421bae6c78040b95c746fb15bfe3c66b7d874b3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 20:55:38 +0100 Subject: [PATCH 0271/1795] add CI workflow to check specified software licenses --- .github/workflows/test_licenses.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/test_licenses.yml diff --git a/.github/workflows/test_licenses.yml b/.github/workflows/test_licenses.yml new file mode 100644 index 0000000000..00a2c90f6b --- /dev/null +++ b/.github/workflows/test_licenses.yml @@ -0,0 +1,20 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Test software licenses +on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: set up Python + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: '3.9' + + - name: Check software licenses + run: | + python licenses/spdx.py licenses/licenses.json From f882ff35177c8d8ec13721ee0e34325362fe3f0e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 27 Nov 2023 14:31:02 +0100 Subject: [PATCH 0272/1795] make sure that CI workflow for check_missing_installation.sh script uses correct CPU target, by setting $EESSI_SOFTWARE_SUBDIR_OVERRIDE --- .github/workflows/test-software.eessi.io.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 86ccbdb663..6756bd8854 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -11,7 +11,7 @@ jobs: matrix: EESSI_VERSION: - 2023.06 - EESSI_SOFTWARE_SUBDIR: + EESSI_SOFTWARE_SUBDIR_OVERRIDE: - aarch64/generic - aarch64/neoverse_n1 - aarch64/neoverse_v1 @@ -33,12 +33,13 @@ jobs: - name: Test check_missing_installations.sh script run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild + which eb eb --version export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do @@ -50,12 +51,13 @@ jobs: - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild + which eb eb --version export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort # create dummy easystack file with a single entry (something that is not installed in EESSI) easystack_file="test.yml" From 2e085a35468dcbf605f4dfc17b5e761bf2aa3360 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 27 Nov 2023 14:50:51 +0100 Subject: [PATCH 0273/1795] {2023.06}[2023a] GCC v12.3.0 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml new file mode 100644 index 0000000000..44d8ba223d --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GCC-12.3.0.eb From b95eee8b4008821d9af10f7c99c5ce14463fa059 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 27 Nov 2023 14:54:40 +0100 Subject: [PATCH 0274/1795] use merged PR for updated Rust easyblock rather than closed PR for Rust 1.70.0 easyconfig --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 75e05a146c..b2d0902051 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -1,7 +1,7 @@ easyconfigs: - Rust-1.70.0-GCCcore-12.3.0.eb: # fix build of Rust 1.70.0 by disabling download of pre-built LLVM; - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19264 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3038 options: - from-pr: 19264 + include-easyblocks-from-pr: 3038 - foss-2023a.eb From 6fbddc716de7c036f03dc8ed1a2c53a6a641e524 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 28 Nov 2023 08:53:27 +0100 Subject: [PATCH 0275/1795] {2023.06}[foss/2023a] SciPy-bundle v2023.07 --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index a967019b80..e685785f16 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -6,3 +6,9 @@ easyconfigs: options: include-easyblocks-from-pr: 3038 - foss-2023a.eb + - pybind11-2.11.1-GCCcore-12.3.0.eb: + # avoid indirect dependency on old CMake version built with GCCcore/10.2.0 via Catch2 build dependency; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 + options: + from-pr: 19270 + - SciPy-bundle-2023.07-gfbf-2023a.eb From e9756d18c350a471b5ea39cd9b61705480441523 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 28 Nov 2023 09:43:09 +0100 Subject: [PATCH 0276/1795] {2023.06}[foss/2023a] TensorFlow v2.13.0 --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index e685785f16..277fc4ca27 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -12,3 +12,8 @@ easyconfigs: options: from-pr: 19270 - SciPy-bundle-2023.07-gfbf-2023a.eb + - TensorFlow-2.13.0-foss-2023a.eb: + # patch setup.py for grpcio extension in TensorFlow 2.13.0 easyconfigs to take into account alternate sysroot; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19268 + options: + from-pr: 19268 From ce0d5ca7d3386f1f4b37e632017bd2bbba9b1f13 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 28 Nov 2023 11:17:35 +0100 Subject: [PATCH 0277/1795] update hook to also ignore failing tests on neoverse_v1 for SciPy-bundle 2023.07 --- eb_hooks.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8d05523c2b..b0c5fafea0 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -314,11 +314,19 @@ def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): """ - Pre-test hook for SciPy-bundle: skip failing tests for SciPy-bundle 2021.10 (currently the only version that is failing). + Pre-test hook for SciPy-bundle: skip failing tests for selected SciPy-bundle vrsions + In version 2021.10, 2 failing tests in scipy 1.6.3: + FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A... + FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 + = 2 failed, 30554 passed, 2064 skipped, 10992 deselected, 76 xfailed, 7 xpassed, 40 warnings in 380.27s (0:06:20) = + In versions 2023.07, 2 failing tests in scipy 1.11.1: + FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris + FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32 + = 2 failed, 54409 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 892.04s (0:14:52) = In previous versions we were not as strict yet on the numpy/SciPy tests """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'SciPy-bundle' and self.version == '2021.10' and cpu_target == CPU_TARGET_NEOVERSE_V1: + if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07'] and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" From 37898378ca48270f29bab0dab233885e72c92d4c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 28 Nov 2023 11:18:25 +0100 Subject: [PATCH 0278/1795] add parse hook to replace Catch2 build dependency using system toolchain in pybind11 --- eb_hooks.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index b0c5fafea0..583cd446e0 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -4,6 +4,7 @@ import re from easybuild.easyblocks.generic.configuremake import obtain_config_guess +from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which @@ -23,6 +24,8 @@ EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' +SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] + def get_eessi_envvar(eessi_envvar): """Get an EESSI environment variable from the environment""" @@ -181,6 +184,25 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") +def parse_hook_pybind11_replace_catch2(ec, eprefix): + """ + Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain. + cfr. https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 + """ + # tihs is mainly necessary to avoid that --missing keeps reporting Catch2/2.13.9 is missing, + # and to avoid that we need to use "--from-pr 19270" for every easyconfigs that (indirectly) depends on pybind11 + if ec.name == 'pybind11' and ec.version in ['2.10.3', '2.11.1']: + build_deps = ec['builddependencies'] + catch2_build_dep = None + catch2_name, catch2_version = ('Catch2', '2.13.9') + for idx, build_dep in enumerate(build_deps): + if build_dep[0] == catch2_name and build_dep[1] == catch2_version: + catch2_build_dep = build_dep + break + if catch2_build_dep and len(catch2_build_dep) == 4 and catch2_build_dep[3] == SYSTEM: + build_deps[idx] = (catch2_name, catch2_version) + + def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): """ Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. @@ -360,6 +382,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, + 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, } From 22444eceea9fa346460f9d448c30a0a5e548793e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 28 Nov 2023 11:17:35 +0100 Subject: [PATCH 0279/1795] update hook to also ignore failing tests on neoverse_v1 for SciPy-bundle 2023.07 --- eb_hooks.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8d05523c2b..b0c5fafea0 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -314,11 +314,19 @@ def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): """ - Pre-test hook for SciPy-bundle: skip failing tests for SciPy-bundle 2021.10 (currently the only version that is failing). + Pre-test hook for SciPy-bundle: skip failing tests for selected SciPy-bundle vrsions + In version 2021.10, 2 failing tests in scipy 1.6.3: + FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A... + FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 + = 2 failed, 30554 passed, 2064 skipped, 10992 deselected, 76 xfailed, 7 xpassed, 40 warnings in 380.27s (0:06:20) = + In versions 2023.07, 2 failing tests in scipy 1.11.1: + FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris + FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32 + = 2 failed, 54409 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 892.04s (0:14:52) = In previous versions we were not as strict yet on the numpy/SciPy tests """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'SciPy-bundle' and self.version == '2021.10' and cpu_target == CPU_TARGET_NEOVERSE_V1: + if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07'] and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" From 43748eb6518a5b4670c1ae055200db6c21b8b5ec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 28 Nov 2023 11:18:25 +0100 Subject: [PATCH 0280/1795] add parse hook to replace Catch2 build dependency using system toolchain in pybind11 --- eb_hooks.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index b0c5fafea0..583cd446e0 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -4,6 +4,7 @@ import re from easybuild.easyblocks.generic.configuremake import obtain_config_guess +from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which @@ -23,6 +24,8 @@ EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' +SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] + def get_eessi_envvar(eessi_envvar): """Get an EESSI environment variable from the environment""" @@ -181,6 +184,25 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") +def parse_hook_pybind11_replace_catch2(ec, eprefix): + """ + Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain. + cfr. https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 + """ + # tihs is mainly necessary to avoid that --missing keeps reporting Catch2/2.13.9 is missing, + # and to avoid that we need to use "--from-pr 19270" for every easyconfigs that (indirectly) depends on pybind11 + if ec.name == 'pybind11' and ec.version in ['2.10.3', '2.11.1']: + build_deps = ec['builddependencies'] + catch2_build_dep = None + catch2_name, catch2_version = ('Catch2', '2.13.9') + for idx, build_dep in enumerate(build_deps): + if build_dep[0] == catch2_name and build_dep[1] == catch2_version: + catch2_build_dep = build_dep + break + if catch2_build_dep and len(catch2_build_dep) == 4 and catch2_build_dep[3] == SYSTEM: + build_deps[idx] = (catch2_name, catch2_version) + + def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): """ Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. @@ -360,6 +382,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, + 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, } From 7052a9070e34e9125ee200d140c7c9c5c0a2ae22 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 28 Nov 2023 11:49:12 +0100 Subject: [PATCH 0281/1795] TypoUpdate eb_hooks.py --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 583cd446e0..38b30ce684 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -189,7 +189,7 @@ def parse_hook_pybind11_replace_catch2(ec, eprefix): Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain. cfr. https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 """ - # tihs is mainly necessary to avoid that --missing keeps reporting Catch2/2.13.9 is missing, + # this is mainly necessary to avoid that --missing keeps reporting Catch2/2.13.9 is missing, # and to avoid that we need to use "--from-pr 19270" for every easyconfigs that (indirectly) depends on pybind11 if ec.name == 'pybind11' and ec.version in ['2.10.3', '2.11.1']: build_deps = ec['builddependencies'] From bff19929ddf21956b2f42e7461e6fd9c7b0fe6e5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 28 Nov 2023 11:50:13 +0100 Subject: [PATCH 0282/1795] Typo --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 38b30ce684..3d0e63f4c2 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -336,7 +336,7 @@ def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): """ - Pre-test hook for SciPy-bundle: skip failing tests for selected SciPy-bundle vrsions + Pre-test hook for SciPy-bundle: skip failing tests for selected SciPy-bundle versions In version 2021.10, 2 failing tests in scipy 1.6.3: FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A... FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 From 04d39a77d3fcc5b7a919139d44c739428c73fa07 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 28 Nov 2023 21:01:20 +0100 Subject: [PATCH 0283/1795] add post_ready_hook to limit parallellism for selected builds, incl. TensorFlow --- eb_hooks.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 3d0e63f4c2..6fe92c7f7b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -69,6 +69,21 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) +def post_ready_hook(self, *args, **kwargs): + """ + Post-ready hook: limit parallellism for selected builds, because they require a lot of memory per used core. + """ + # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores. + # here we reduce parallellism to only use half of that for selected software, + # to avoid failing builds/tests due to out-of-memory problems + if self.name in ['TensorFlow']: + parallel = self.cfg['parallel'] + if parallel > 1: + self.cfg['parallel'] = parallel // 2 + msg = "limiting parallelism to %s (was %s) for %s to avoid out-of-memory failures during building/testing" + print_msg(msg % (self.cfg['parallel'], parallel, self.name), log=self.log) + + def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" From ab2c260e4d9b9a1b70b7246b10977437ad0fcd15 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:38:57 +0100 Subject: [PATCH 0284/1795] {2023.06}[foss/2023a] OpenFOAM v10 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index e685785f16..b67e894825 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -12,3 +12,4 @@ easyconfigs: options: from-pr: 19270 - SciPy-bundle-2023.07-gfbf-2023a.eb + - OpenFOAM-10-foss-2023a.eb From cae2ceda19a3d95dca6a8c863f9a8a837a3f8046 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 29 Nov 2023 14:28:36 +0100 Subject: [PATCH 0285/1795] {2023.06}[foss/2023a] Qt5 v5.15.10 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 277fc4ca27..b9eb9b4d8f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -17,3 +17,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19268 options: from-pr: 19268 + - Qt5-5.15.10-GCCcore-12.3.0.eb From 6c1e032484df88f8746d2430fd0af1c6915b7071 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 29 Nov 2023 14:32:02 +0100 Subject: [PATCH 0286/1795] {2023.06}[2023a] X11 20230603 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 277fc4ca27..1a9df25df3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -17,3 +17,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19268 options: from-pr: 19268 + - X11-20230603-GCCcore-12.3.0.eb From cb4f7e13fa7b6e213d82f4617be4ff21eda93830 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:39:40 +0100 Subject: [PATCH 0287/1795] option: from-pr: 19339 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index c08abe3bd8..a5a1cdc727 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -19,3 +19,5 @@ easyconfigs: from-pr: 19268 - X11-20230603-GCCcore-12.3.0.eb - Qt5-5.15.10-GCCcore-12.3.0.eb + options: + from-pr: 19339 From e85281ac55519240b7dd34ce690b786c32709584 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:43:59 +0100 Subject: [PATCH 0288/1795] Fix typo --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index a5a1cdc727..0d4fd65ce4 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -18,6 +18,6 @@ easyconfigs: options: from-pr: 19268 - X11-20230603-GCCcore-12.3.0.eb - - Qt5-5.15.10-GCCcore-12.3.0.eb + - Qt5-5.15.10-GCCcore-12.3.0.eb: options: from-pr: 19339 From 6afba5b84d119d3f928c53dc31aa06c8c3b77129 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Thu, 30 Nov 2023 17:15:02 +0100 Subject: [PATCH 0289/1795] Add HarfBuzz-5.3.1-GCCcore-12.3.0.eb to stack with from-pr option --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 0d4fd65ce4..0d5345eacc 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -18,6 +18,7 @@ easyconfigs: options: from-pr: 19268 - X11-20230603-GCCcore-12.3.0.eb - - Qt5-5.15.10-GCCcore-12.3.0.eb: + - HarfBuzz-5.3.1-GCCcore-12.3.0.eb: options: from-pr: 19339 + - Qt5-5.15.10-GCCcore-12.3.0.eb From 1c8131ebbf89da06c6cc12195a7253324fed8903 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 1 Dec 2023 11:55:44 +0100 Subject: [PATCH 0290/1795] {2023.06} Nextflow/23.10.0 --- .../2023.06/eessi-2023.06-eb-4.8.2-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml index 5ce6a65913..f02b9f2802 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml @@ -2,3 +2,6 @@ easyconfigs: - EasyBuild-4.8.2.eb: options: from-pr: 19105 + - Nextflow-23.10.0.eb: + options: + from-pr: 19172 From 3439e8b07981dff466b9efb1af4215bedb603351 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 1 Dec 2023 12:06:39 +0100 Subject: [PATCH 0291/1795] Include gnuplot fix from PR easybuilders/easyconfigs#19261 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 4a302c409b..514ed0b86e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -22,4 +22,7 @@ easyconfigs: options: from-pr: 19339 - Qt5-5.15.10-GCCcore-12.3.0.eb + - gnuplot-5.4.8-GCCcore-12.3.0.eb: + options: + from-pr: 19261 - OpenFOAM-10-foss-2023a.eb From f94430a60940ecfab5d89b789743b4c6b740b983 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 1 Dec 2023 16:43:46 +0100 Subject: [PATCH 0292/1795] Replicated changes that were initially targetted at 2023.06-pilot repo --- eessi_container.sh | 71 +++++- .../nvidia/install_cuda_host_injections.sh | 209 ++++++++++++++++++ .../nvidia/link_nvidia_host_libraries.sh | 136 ++++++++++++ scripts/utils.sh | 56 ++++- 4 files changed, 461 insertions(+), 11 deletions(-) create mode 100755 gpu_support/nvidia/install_cuda_host_injections.sh create mode 100755 gpu_support/nvidia/link_nvidia_host_libraries.sh diff --git a/eessi_container.sh b/eessi_container.sh index bf0294c7bf..81d7be81ad 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -30,8 +30,8 @@ # -. initial settings & exit codes TOPDIR=$(dirname $(realpath $0)) -source ${TOPDIR}/scripts/utils.sh -source ${TOPDIR}/scripts/cfg_files.sh +source "${TOPDIR}"/scripts/utils.sh +source "${TOPDIR}"/scripts/cfg_files.sh # exit codes: bitwise shift codes to allow for combination of exit codes # ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh @@ -46,6 +46,7 @@ SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) +NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12)) # CernVM-FS settings CVMFS_VAR_LIB="var-lib-cvmfs" @@ -72,12 +73,17 @@ display_help() { echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMG - image file or URL defining the container to use" echo " [default: docker://ghcr.io/eessi/build-node:debian11]" - echo " -h | --help - display this usage information [default: false]" echo " -g | --storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -h | --help - display this usage information [default: false]" + echo " -i | --host-injections - directory to link to for host_injections " + echo " [default: /..storage../opt-eessi]" echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script or command) [default: shell]" + echo " -n | --nvidia MODE - configure the container to work with NVIDIA GPUs," + echo " MODE==install for a CUDA installation, MODE==run to" + echo " attach a GPU, MODE==all for both [default: false]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use [default: EESSI via" echo " default container, see --container]" @@ -111,6 +117,8 @@ VERBOSE=0 STORAGE= LIST_REPOS=0 MODE="shell" +SETUP_NVIDIA=0 +ADDITIONAL_SINGULARITY_FLAGS= REPOSITORY="EESSI" RESUME= SAVE= @@ -141,6 +149,10 @@ while [[ $# -gt 0 ]]; do display_help exit 0 ;; + -i|--host-injections) + USER_HOST_INJECTIONS="$2" + shift 2 + ;; -l|--list-repos) LIST_REPOS=1 shift 1 @@ -149,6 +161,11 @@ while [[ $# -gt 0 ]]; do MODE="$2" shift 2 ;; + -n|--nvidia) + SETUP_NVIDIA=1 + NVIDIA_MODE="$2" + shift 2 + ;; -r|--repository) REPOSITORY="$2" shift 2 @@ -224,6 +241,13 @@ if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi +# Also validate the NVIDIA GPU mode (if present) +if [[ ${SETUP_NVIDIA} -eq 1 ]]; then + if [[ "${NVIDIA_MODE}" != "run" && "${NVIDIA_MODE}" != "install" && "${NVIDIA_MODE}" != "all" ]]; then + fatal_error "unknown NVIDIA mode '${NVIDIA_MODE}'" "${NVIDIA_MODE_UNKNOWN_EXITCODE}" + fi +fi + # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then @@ -310,12 +334,25 @@ fi # |-overlay-work # |-home # |-repos_cfg +# |-opt-eessi (unless otherwise specificed for host_injections) # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +# Set host_injections directory and ensure it is a writable directory (if user provided) +if [ -z ${USER_HOST_INJECTIONS+x} ]; then + # Not set, so use our default + HOST_INJECTIONS=${EESSI_TMPDIR}/opt-eessi + mkdir -p $HOST_INJECTIONS +else + # Make sure the host_injections directory specified exists and is a folder + mkdir -p ${USER_HOST_INJECTIONS} || fatal_error "host_injections directory ${USER_HOST_INJECTIONS} is either not a directory or cannot be created" + HOST_INJECTIONS=${USER_HOST_INJECTIONS} +fi +[[ ${VERBOSE} -eq 1 ]] && echo "HOST_INJECTIONS=${HOST_INJECTIONS}" + # configure Singularity: if SINGULARITY_CACHEDIR is already defined, use that # a global SINGULARITY_CACHEDIR would ensure that we don't consume # storage space again and again for the container & also speed-up @@ -394,12 +431,34 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) -BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" +BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" # provide a '/tmp' inside the container BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" +# Configure anything we need for NVIDIA GPUs and CUDA installation +if [[ ${SETUP_NVIDIA} -eq 1 ]]; then + if [[ "${NVIDIA_MODE}" == "run" || "${NVIDIA_MODE}" == "all" ]]; then + # Give singularity the appropriate flag + ADDITIONAL_SINGULARITY_FLAGS="--nv ${ADDITIONAL_SINGULARITY_FLAGS}" + [[ ${VERBOSE} -eq 1 ]] && echo "ADDITIONAL_SINGULARITY_FLAGS=${ADDITIONAL_SINGULARITY_FLAGS}" + fi + if [[ "${NVIDIA_MODE}" == "install" || "${NVIDIA_MODE}" == "all" ]]; then + # Add additional bind mounts to allow CUDA to install within a container + # (Experience tells us that these are necessary, but we don't know _why_ + # as the CUDA installer is a black box. The suspicion is that the CUDA + # installer gets confused by the permissions on these directories when + # inside a container) + EESSI_VAR_LOG=${EESSI_TMPDIR}/var-log + EESSI_USR_LOCAL_CUDA=${EESSI_TMPDIR}/usr-local-cuda + mkdir -p ${EESSI_VAR_LOG} + mkdir -p ${EESSI_USR_LOCAL_CUDA} + BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" + [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" + fi +fi + # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg @@ -562,8 +621,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then fi echo "Launching container with command (next line):" -echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" -singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" +echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_SINGULARITY_FLAGS} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" +singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_SINGULARITY_FLAGS} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" exit_code=$? # 6. save tmp if requested (arg -s|--save) diff --git a/gpu_support/nvidia/install_cuda_host_injections.sh b/gpu_support/nvidia/install_cuda_host_injections.sh new file mode 100755 index 0000000000..f02f0da02e --- /dev/null +++ b/gpu_support/nvidia/install_cuda_host_injections.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +# This script can be used to install CUDA under the `.../host_injections` directory. +# This provides the parts of the CUDA installation that cannot be redistributed as +# part of EESSI due to license limitations. While GPU-based software from EESSI will +# _run_ without these, installation of additional CUDA software requires the CUDA +# installation(s) under `host_injections` to be present. +# +# The `host_injections` directory is a variant symlink that by default points to +# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see +# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the +# installation to be successful, this directory needs to be writeable by the user +# executing this script. + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath $BASH_SOURCE)) +source "$TOPDIR"/../../scripts/utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " --accept-cuda-eula You _must_ accept the CUDA EULA to install" + echo " CUDA, see the EULA at" + echo " https://docs.nvidia.com/cuda/eula/index.html" + echo " -c, --cuda-version CUDA_VERSION Specify a version o CUDA to install (must" + echo " have a corresponding easyconfig in the" + echo " EasyBuild release)" + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the CUDA install" + echo " (must have >10GB available)" +} + +# Initialize variables +install_cuda_version="" +eula_accepted=0 + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + -c|--cuda-version) + if [ -n "$2" ]; then + install_cuda_version="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + --accept-cuda-eula) + eula_accepted=1 + shift 1 + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + CUDA_TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +# Make sure EESSI is initialised +check_eessi_initialised + +# Make sure the CUDA version supplied is a semantic version +is_semantic_version() { + local version=$1 + local regex='^[0-9]+\.[0-9]+\.[0-9]+$' + + if [[ $version =~ $regex ]]; then + return 0 # Return success (0) if it's a semantic version + else + return 1 # Return failure (1) if it's not a semantic version + fi +} +if ! is_semantic_version "$install_cuda_version"; then + show_help + error="\nYou must provide a semantic version for CUDA (e.g., 12.1.1) via the appropriate\n" + error="${error}command line option. This script is intended for use with EESSI so the 'correct'\n" + error="${error}version to provide is probably one of those available under\n" + error="${error}$EESSI_SOFTWARE_PATH/software/CUDA\n" + fatal_error "${error}" +fi + +# Make sure they have accepted the CUDA EULA +if [ "$eula_accepted" -ne 1 ]; then + show_help + error="\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n" + fatal_error "${error}" +fi + +# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` +# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup) +cuda_install_parent=${EESSI_SOFTWARE_PATH/versions/host_injections} + +# Only install CUDA if specified version is not found. +# (existence of easybuild subdir implies a successful install) +if [ -d "${cuda_install_parent}"/software/CUDA/"${install_cuda_version}"/easybuild ]; then + echo_green "CUDA software found! No need to install CUDA again." +else + # We need to be able write to the installation space so let's make sure we can + if ! create_directory_structure "${cuda_install_parent}"/software/CUDA ; then + fatal_error "No write permissions to directory ${cuda_install_parent}/software/CUDA" + fi + + # we need a directory we can use for temporary storage + if [[ -z "${CUDA_TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) + else + tmpdir="${CUDA_TEMP_DIR}"/temp + if ! mkdir "$tmpdir" ; then + fatal_error "Could not create directory ${tmpdir}" + fi + fi + + required_space_in_tmpdir=50000 + # Let's see if we have sources and build locations defined if not, we use the temporary space + if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then + export EASYBUILD_BUILDPATH=${tmpdir}/build + required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000)) + fi + if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then + export EASYBUILD_SOURCEPATH=${tmpdir}/sources + required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000)) + fi + + # The install is pretty fat, you need lots of space for download/unpack/install (~3*5GB), + # need to do a space check before we proceed + avail_space=$(df --output=avail "${cuda_install_parent}"/ | tail -n 1 | awk '{print $1}') + if (( avail_space < 5000000 )); then + fatal_error "Need at least 5GB disk space to install CUDA under ${cuda_install_parent}, exiting now..." + fi + avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') + if (( avail_space < required_space_in_tmpdir )); then + error="Need at least ${required_space_in_tmpdir} disk space under ${tmpdir}.\n" + error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check." + error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH " + error="${error}to reduce this requirement. Exiting now..." + fatal_error "${error}" + fi + + if ! command -v "eb" &>/dev/null; then + echo_yellow "Attempting to load an EasyBuild module to do actual install" + module load EasyBuild + # There are some scenarios where this may fail + if [ $? -ne 0 ]; then + error="'eb' command not found in your environment and\n" + error="${error} module load EasyBuild\n" + error="${error}failed for some reason.\n" + error="${error}Please re-run this script with the 'eb' command available." + fatal_error "${error}" + fi + fi + + cuda_easyconfig="CUDA-${install_cuda_version}.eb" + + # Check the easyconfig file is available in the release + # (eb search always returns 0, so we need a grep to ensure a usable exit code) + eb --search ^${cuda_easyconfig}|grep CUDA > /dev/null 2>&1 + # Check the exit code + if [ $? -ne 0 ]; then + eb_version=$(eb --version) + available_cuda_easyconfigs=$(eb --search ^CUDA-*.eb|grep CUDA) + + error="The easyconfig ${cuda_easyconfig} was not found in EasyBuild version:\n" + error="${error} ${eb_version}\n" + error="${error}You either need to give a different version of CUDA to install _or_ \n" + error="${error}use a different version of EasyBuild for the installation.\n" + error="${error}\nThe versions of available with the current eb command are:\n" + error="${error}${available_cuda_easyconfigs}" + fatal_error "${error}" + fi + + # We need the --rebuild option, as the CUDA module may or may not be on the + # `MODULEPATH` yet. Even if it is, we still want to redo this installation + # since it will provide the symlinked targets for the parts of the CUDA + # installation in the `.../versions/...` prefix + # We install the module in our `tmpdir` since we do not need the modulefile, + # we only care about providing the targets for the symlinks. + extra_args="--rebuild --installpath-modules=${tmpdir}" + + # We don't want hooks used in this install, we need a vanilla CUDA installation + touch "$tmpdir"/none.py + # shellcheck disable=SC2086 # Intended splitting of extra_args + eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}" + ret=$? + if [ $ret -ne 0 ]; then + fatal_error "CUDA installation failed, please check EasyBuild logs..." + else + echo_green "CUDA installation at ${cuda_install_parent}/software/CUDA/${install_cuda_version} succeeded!" + fi + # clean up tmpdir + rm -rf "${tmpdir}" +fi diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/gpu_support/nvidia/link_nvidia_host_libraries.sh new file mode 100755 index 0000000000..26760f0b82 --- /dev/null +++ b/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# This script links host libraries related to GPU drivers to a location where +# they can be found by the EESSI linker + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath $BASH_SOURCE)) +source "$TOPDIR"/../../scripts/utils.sh + +# We rely on ldconfig to give us the location of the libraries on the host +command_name="ldconfig" +# We cannot use a version of ldconfig that's being shipped under CVMFS +exclude_prefix="/cvmfs" + +found_paths=() +# Always attempt to use /sbin/ldconfig +if [ -x "/sbin/$command_name" ]; then + found_paths+=("/sbin/$command_name") +fi +IFS=':' read -ra path_dirs <<< "$PATH" +for dir in "${path_dirs[@]}"; do + if [ "$dir" = "/sbin" ]; then + continue # we've already checked for $command_name in /sbin, don't need to do it twice + fi + if [[ ! "$dir" =~ ^$exclude_prefix ]]; then + if [ -x "$dir/$command_name" ]; then + found_paths+=("$dir/$command_name") + fi + fi +done + +if [ ${#found_paths[@]} -gt 0 ]; then + echo "Found $command_name in the following locations:" + printf -- "- %s\n" "${found_paths[@]}" + echo "Using first version" + host_ldconfig=${found_paths[0]} +else + error="$command_name not found in PATH or only found in paths starting with $exclude_prefix." + fatal_error $error +fi + +# Make sure EESSI is initialised (doesn't matter what version) +check_eessi_initialised + +# Find the CUDA version of the host CUDA drivers +# (making sure that this can still work inside prefix environment inside a container) +export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH +nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader" +if $nvidia_smi_command > /dev/null; then + host_driver_version=$($nvidia_smi_command | tail -n1) + # If the first worked, this should work too + host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}') +else + error="Failed to successfully execute\n $nvidia_smi_command\n" + fatal_error $error +fi + +# Let's make sure the driver libraries are not already in place +link_drivers=1 + +host_injections_nvidia_dir="/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/${EESSI_CPU_FAMILY}" +host_injection_driver_dir="${host_injections_nvidia_dir}/host" +host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt" +if [ -e "$host_injection_driver_version_file" ]; then + if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then + echo_green "The host CUDA driver libraries have already been linked!" + link_drivers=0 + else + # There's something there but it is out of date + echo_yellow "Cleaning out outdated symlinks" + rm $host_injection_driver_dir/* + if [ $? -ne 0 ]; then + error="Unable to remove files under '$host_injection_driver_dir'." + fatal_error $error + fi + fi +fi + +drivers_linked=0 +if [ "$link_drivers" -eq 1 ]; then + if ! create_directory_structure "${host_injection_driver_dir}" ; then + fatal_error "No write permissions to directory ${host_injection_driver_dir}" + fi + cd ${host_injection_driver_dir} + # Need a small temporary space to hold a couple of files + temp_dir=$(mktemp -d) + + # Gather libraries on the host (_must_ be host ldconfig) + $host_ldconfig -p | awk '{print $NF}' > "$temp_dir"/libs.txt + # Allow for the fact that we may be in a container so the CUDA libs might be in there + ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null + + # Leverage singularity to find the full list of libraries we should be linking to + echo_yellow "Downloading latest version of nvliblist.conf from Apptainer" + curl -o "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf + + # Make symlinks to all the interesting libraries + grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {} + + # Inject driver and CUDA versions into dir + echo $host_driver_version > driver_version.txt + echo $host_cuda_version > cuda_version.txt + drivers_linked=1 + + # Remove the temporary directory when done + rm -r "$temp_dir" +fi + +# Make latest symlink for NVIDIA drivers +cd $host_injections_nvidia_dir +symlink="latest" +if [ -L "$symlink" ]; then + # Unless the drivers have been installed, leave the symlink alone + if [ "$drivers_linked" -eq 1 ]; then + ln -sf host latest + fi +else + # No link exists yet + ln -s host latest +fi + +# Make sure the libraries can be found by the EESSI linker +host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections} +if [ -L "$host_injection_linker_dir/lib" ]; then + target_path=$(readlink -f "$host_injection_linker_dir/lib") + if [ "$target_path" != "$$host_injections_nvidia_dir/latest" ]; then + cd $host_injection_linker_dir + ln -sf $host_injections_nvidia_dir/latest lib + fi +else + create_directory_structure $host_injection_linker_dir + cd $host_injection_linker_dir + ln -s $host_injections_nvidia_dir/latest lib +fi + +echo_green "Host NVIDIA gpu drivers linked successfully for EESSI" diff --git a/scripts/utils.sh b/scripts/utils.sh index d0da95e87f..b2be3f6221 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -14,7 +14,7 @@ ANY_ERROR_EXITCODE=1 function fatal_error() { echo_red "ERROR: $1" >&2 if [[ $# -gt 1 ]]; then - exit $2 + exit "$2" else exit "${ANY_ERROR_EXITCODE}" fi @@ -32,11 +32,57 @@ function check_exit_code { fi } +function check_eessi_initialised() { + if [[ -z "${EESSI_SOFTWARE_PATH}" ]]; then + fatal_error "EESSI has not been initialised!" + else + return 0 + fi +} + +function check_in_prefix_shell() { + # Make sure EPREFIX is defined + if [[ -z "${EPREFIX}" ]]; then + fatal_error "This script cannot be used without having first defined EPREFIX" + fi + if [[ ! ${SHELL} = ${EPREFIX}/bin/bash ]]; then + fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" + fi +} + +function create_directory_structure() { + # Ensure we are given a single path argument + if [ $# -ne 1 ]; then + echo_red "Function requires a single (relative or absolute) path argument" >&2 + return $ANY_ERROR_EXITCODE + fi + dir_structure="$1" + + # Attempt to create the directory structure + error_message=$(mkdir -p "$dir_structure" 2>&1) + return_code=$? + # If it fails be explicit about the error + if [ ${return_code} -ne 0 ]; then + real_dir=$(realpath -m "$dir_structure") + echo_red "Creating ${dir_structure} (real path ${real_dir}) failed with:\n ${error_message}" >&2 + else + # If we're creating it, our use case is that we want to be able to write there + # (this is a check in case the directory already existed) + if [ ! -w "${dir_structure}" ]; then + real_dir=$(realpath -m "$dir_structure") + echo_red "You do not have (required) write permissions to ${dir_structure} (real path ${real_dir})!" + return_code=$ANY_ERROR_EXITCODE + fi + fi + + return $return_code +} + function get_path_for_tool { tool_name=$1 tool_envvar_name=$2 - which_out=$(which ${tool_name} 2>&1) + which_out=$(which "${tool_name}" 2>&1) exit_code=$? if [[ ${exit_code} -eq 0 ]]; then echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 @@ -68,7 +114,7 @@ function get_host_from_url { url=$1 re="(http|https)://([^/:]+)" if [[ $url =~ $re ]]; then - echo ${BASH_REMATCH[2]} + echo "${BASH_REMATCH[2]}" return 0 else echo "" @@ -80,7 +126,7 @@ function get_port_from_url { url=$1 re="(http|https)://[^:]+:([0-9]+)" if [[ $url =~ $re ]]; then - echo ${BASH_REMATCH[2]} + echo "${BASH_REMATCH[2]}" return 0 else echo "" @@ -90,7 +136,7 @@ function get_port_from_url { function get_ipv4_address { hname=$1 - hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) + hipv4=$(grep "${hname}" /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) # TODO try other methods if the one above does not work --> tool that verifies # what method can be used? echo "${hipv4}" From c06a7aa392a4d0b719d9ffa86bbafa1e684e54e0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 1 Dec 2023 16:46:19 +0100 Subject: [PATCH 0293/1795] Update nvidia_dir with new repo name --- gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/gpu_support/nvidia/link_nvidia_host_libraries.sh index 26760f0b82..a03ce609a0 100755 --- a/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -58,7 +58,7 @@ fi # Let's make sure the driver libraries are not already in place link_drivers=1 -host_injections_nvidia_dir="/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/${EESSI_CPU_FAMILY}" +host_injections_nvidia_dir="/cvmfs/software.eessi.io/host_injections/nvidia/${EESSI_CPU_FAMILY}" host_injection_driver_dir="${host_injections_nvidia_dir}/host" host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt" if [ -e "$host_injection_driver_version_file" ]; then From 24a4ebca030ea2ea55bb06d9b0d362f1f95ec8fd Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 1 Dec 2023 16:52:40 +0100 Subject: [PATCH 0294/1795] Update link_nvidia_host_libraries.sh --- gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/gpu_support/nvidia/link_nvidia_host_libraries.sh index a03ce609a0..6458be7fae 100755 --- a/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -58,7 +58,7 @@ fi # Let's make sure the driver libraries are not already in place link_drivers=1 -host_injections_nvidia_dir="/cvmfs/software.eessi.io/host_injections/nvidia/${EESSI_CPU_FAMILY}" +host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}" host_injection_driver_dir="${host_injections_nvidia_dir}/host" host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt" if [ -e "$host_injection_driver_version_file" ]; then From 953b17d6dcf774b50acb07af0dfa84371bf68ab5 Mon Sep 17 00:00:00 2001 From: Pablo Escobar Date: Sat, 2 Dec 2023 11:51:12 +0100 Subject: [PATCH 0295/1795] Prepend EESSI version to PS1 instead of overriding PS1 --- init/bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/bash b/init/bash index 84fe783bce..dbd09fdee5 100644 --- a/init/bash +++ b/init/bash @@ -13,7 +13,7 @@ source $(dirname "$BASH_SOURCE")/eessi_environment_variables # only continue if setting EESSI environment variables worked fine if [ $? -eq 0 ]; then - export PS1="[EESSI $EESSI_VERSION] $ " + export PS1="[EESSI $EESSI_VERSION]$PS1" # add location of commands provided by compat layer to $PATH; # see https://github.com/EESSI/software-layer/issues/52 From bf6dd6070215ae2a0df39c4001cf204737197450 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 2 Dec 2023 16:37:41 +0100 Subject: [PATCH 0296/1795] {2023.06}[2023a] OSU-Micro-Benchmarks v7.1-1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 0d5345eacc..8832da8ddd 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -22,3 +22,4 @@ easyconfigs: options: from-pr: 19339 - Qt5-5.15.10-GCCcore-12.3.0.eb + - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb From 645041c31853181b9d46329cc709950b296368b5 Mon Sep 17 00:00:00 2001 From: Pablo Escobar Date: Mon, 4 Dec 2023 09:04:10 +0100 Subject: [PATCH 0297/1795] Use { instead of [ when adding EESSI version to PS1 --- init/bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/bash b/init/bash index dbd09fdee5..c2fd179849 100644 --- a/init/bash +++ b/init/bash @@ -13,7 +13,7 @@ source $(dirname "$BASH_SOURCE")/eessi_environment_variables # only continue if setting EESSI environment variables worked fine if [ $? -eq 0 ]; then - export PS1="[EESSI $EESSI_VERSION]$PS1" + export PS1="{EESSI $EESSI_VERSION}$PS1" # add location of commands provided by compat layer to $PATH; # see https://github.com/EESSI/software-layer/issues/52 From 76ae987cafc858b6ed9ae3c9f4f0a4a209b31d25 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 4 Dec 2023 21:16:14 +0100 Subject: [PATCH 0298/1795] do not use /dev/stdout --- init/bash | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/init/bash b/init/bash index 84fe783bce..0b3bdcb869 100644 --- a/init/bash +++ b/init/bash @@ -1,10 +1,17 @@ -# Allow for a silent mode -if [[ -v EESSI_SILENT ]]; then - # EESSI_SILENT set - output=/dev/null -else - output=/dev/stdout -fi +function show_msg { + msg=$1 + if [[ ! -v EESSI_SILENT ]]; then + echo "$msg" + fi +} + +## Allow for a silent mode +#if [[ -v EESSI_SILENT ]]; then +# # EESSI_SILENT set +# output=/dev/null +#else +# output=/dev/stdout +#fi # The following method should be safe, but might break if file is a symlink # (could switch to $(dirname "$(readlink -f "$BASH_SOURCE")") in that case) @@ -20,11 +27,11 @@ if [ $? -eq 0 ]; then export PATH=$EPREFIX/usr/bin:$EPREFIX/bin:$PATH # init Lmod - echo "Initializing Lmod..." >> $output + show_msg "Initializing Lmod..." source $EESSI_EPREFIX/usr/share/Lmod/init/bash # prepend location of modules for EESSI software stack to $MODULEPATH - echo "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." >> $output + show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." module use $EESSI_MODULEPATH #echo >> $output From 0768ae1bc627ede6bdfe1059d49739437b783b78 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 5 Dec 2023 09:06:43 +0100 Subject: [PATCH 0299/1795] work around permission denied for writing to /dev/stdout --- init/bash | 21 +++++++-------------- init/eessi_environment_variables | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/init/bash b/init/bash index 0b3bdcb869..3adfb4bc9d 100644 --- a/init/bash +++ b/init/bash @@ -1,18 +1,11 @@ function show_msg { + # only echo msg if EESSI_SILENT is unset msg=$1 if [[ ! -v EESSI_SILENT ]]; then echo "$msg" fi } -## Allow for a silent mode -#if [[ -v EESSI_SILENT ]]; then -# # EESSI_SILENT set -# output=/dev/null -#else -# output=/dev/stdout -#fi - # The following method should be safe, but might break if file is a symlink # (could switch to $(dirname "$(readlink -f "$BASH_SOURCE")") in that case) source $(dirname "$BASH_SOURCE")/eessi_environment_variables @@ -34,12 +27,12 @@ if [ $? -eq 0 ]; then show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." module use $EESSI_MODULEPATH - #echo >> $output - #echo "*** Known problems in the ${EESSI_VERSION} software stack ***" >> $output - #echo >> $output - #echo "1) ..." >> $output - #echo >> $output - #echo >> $output + #show_msg "" + #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***" + #show_msg "" + #show_msg "1) ..." + #show_msg "" + #show_msg "" echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!" diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 42f4b6b76a..3584bfaf34 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -2,24 +2,24 @@ # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) -# Allow for a silent mode -if [[ -v EESSI_SILENT ]]; then - # EESSI_SILENT set - output=/dev/null -else - output=/dev/stdout -fi - function error() { echo -e "\e[31mERROR: $1\e[0m" >&2 false } +function show_msg { + # only echo msg if EESSI_SILENT is unset + msg=$1 + if [[ ! -v EESSI_SILENT ]]; then + echo "$msg" + fi +} + # set up minimal environment: $EESSI_PREFIX, $EESSI_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX source $EESSI_INIT_DIR_PATH/minimal_eessi_env if [ -d $EESSI_PREFIX ]; then - echo "Found EESSI repo @ $EESSI_PREFIX!" >> $output + show_msg "Found EESSI repo @ $EESSI_PREFIX!" export EESSI_EPREFIX=$EPREFIX if [ -d $EESSI_EPREFIX ]; then @@ -28,21 +28,21 @@ if [ -d $EESSI_PREFIX ]; then if [ "$EESSI_USE_ARCHDETECT" == "1" ]; then # if archdetect is enabled, use internal code export EESSI_SOFTWARE_SUBDIR=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh cpupath) - echo "archdetect says ${EESSI_SOFTWARE_SUBDIR}" >> $output + show_msg "archdetect says ${EESSI_SOFTWARE_SUBDIR}" elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined! export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3 export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX) - echo "archspec says ${EESSI_SOFTWARE_SUBDIR}" >> $output + show_msg "archspec says ${EESSI_SOFTWARE_SUBDIR}" else error "Don't know how to detect host CPU, giving up!" fi if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then - echo "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." >> $output + show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR if [ ! -z $EESSI_BASIC_ENV ]; then - echo "Only setting up basic environment, so we're done" >> $output + show_msg "Only setting up basic environment, so we're done" elif [ -d $EESSI_SOFTWARE_PATH ]; then # Allow for the use of a custom MNS if [ -z ${EESSI_CUSTOM_MODULEPATH+x} ]; then @@ -55,13 +55,13 @@ if [ -d $EESSI_PREFIX ]; then fi EESSI_MODULEPATH=$EESSI_SOFTWARE_PATH/$EESSI_MODULE_SUBDIR else - echo "Using defined environment variable \$EESSI_CUSTOM_MODULEPATH to set EESSI_MODULEPATH." >> $output + show_msg "Using defined environment variable \$EESSI_CUSTOM_MODULEPATH to set EESSI_MODULEPATH." EESSI_MODULEPATH=$EESSI_CUSTOM_MODULEPATH fi if [ -d $EESSI_MODULEPATH ]; then export EESSI_MODULEPATH=$EESSI_MODULEPATH - echo "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." >> $output + show_msg "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." else error "EESSI module path at $EESSI_MODULEPATH not found!" false @@ -69,7 +69,7 @@ if [ -d $EESSI_PREFIX ]; then export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua" if [ -f $LMOD_RC ]; then - echo "Found Lmod configuration file at $LMOD_RC" >> $output + show_msg "Found Lmod configuration file at $LMOD_RC" else error "Lmod configuration file not found at $LMOD_RC" fi From 30f85b7a388a33d8b8a7f3371d4dfc2def796b76 Mon Sep 17 00:00:00 2001 From: Pablo Escobar Date: Tue, 5 Dec 2023 11:26:14 +0100 Subject: [PATCH 0300/1795] Add space between EESSI version and PS1 --- init/bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/bash b/init/bash index c2fd179849..cd977b3c8e 100644 --- a/init/bash +++ b/init/bash @@ -13,7 +13,7 @@ source $(dirname "$BASH_SOURCE")/eessi_environment_variables # only continue if setting EESSI environment variables worked fine if [ $? -eq 0 ]; then - export PS1="{EESSI $EESSI_VERSION}$PS1" + export PS1="{EESSI $EESSI_VERSION} $PS1" # add location of commands provided by compat layer to $PATH; # see https://github.com/EESSI/software-layer/issues/52 From 7f6896a908fd0b43a6923e1cc8d8cceb0f7f6f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 5 Dec 2023 13:47:47 +0100 Subject: [PATCH 0301/1795] easystack for 2022b, start with foss 2022b --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml new file mode 100644 index 0000000000..39bae9c781 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - foss-2022b.eb From 6f451f9fc70bc04611f85155b38db4e39f5bf445 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Wed, 6 Dec 2023 15:22:54 +0100 Subject: [PATCH 0302/1795] {2023.06}[GCC/12.3.0] LHAPDF 6.5.4 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 8832da8ddd..a5b7af25c7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -23,3 +23,6 @@ easyconfigs: from-pr: 19339 - Qt5-5.15.10-GCCcore-12.3.0.eb - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb + - LHAPDF-6.5.4-GCC-12.3.0.eb + options: + from-pr: 19363 From 5aea0868b4474220e8b2ec6a8c005b4a4a66801b Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Wed, 6 Dec 2023 15:27:48 +0100 Subject: [PATCH 0303/1795] Update eessi-2023.06-eb-4.8.2-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index a5b7af25c7..1a1804e2fb 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -23,6 +23,6 @@ easyconfigs: from-pr: 19339 - Qt5-5.15.10-GCCcore-12.3.0.eb - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb - - LHAPDF-6.5.4-GCC-12.3.0.eb + - LHAPDF-6.5.4-GCC-12.3.0.eb: options: from-pr: 19363 From 91f5459f2b0bb50685e4853ecf37ce863d49a0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Dec 2023 22:31:58 +0100 Subject: [PATCH 0304/1795] add pybind and GDAL --- .../2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index 39bae9c781..a90a34d6ab 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -1,2 +1,8 @@ easyconfigs: - foss-2022b.eb + - pybind11-2.10.3-GCCcore-12.2.0.eb: + # avoid indirect dependency on old CMake version built with GCCcore/10.2.0 via Catch2 build dependency; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 + options: + from-pr: 19270 + - GDAL-3.6.2-foss-2022b.eb From 65a3aace9af3483669893490c2ba614e663e5cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Dec 2023 22:36:45 +0100 Subject: [PATCH 0305/1795] rename pilot to check_missing --- .github/workflows/test-software.eessi.io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 6756bd8854..8c97601611 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] permissions: contents: read # to fetch code (actions/checkout) jobs: - pilot: + check_missing: runs-on: ubuntu-22.04 strategy: fail-fast: false From 6602ed8b40b78407aa5856eeb4882970f7afe331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 9 Dec 2023 22:37:42 +0100 Subject: [PATCH 0306/1795] add LERC with from-pr for sanity check fix --- .../2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index a90a34d6ab..f27e928364 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -5,4 +5,9 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 options: from-pr: 19270 + - LERC-4.0.0-GCCcore-12.2.0.eb: + # avoid RPATH-related sanity check errors by compiling LERC's test binary in postinstallcmds; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19386 + options: + from-pr: 19386 - GDAL-3.6.2-foss-2022b.eb From 3ba0f24de3bd5eec7fcafeea9da28c596aac91bb Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 12 Dec 2023 13:52:38 +0000 Subject: [PATCH 0307/1795] {2023.06}[foss/2022b] Qt5 v5.15.7 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index 39bae9c781..30e89c4485 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -1,2 +1,3 @@ easyconfigs: - foss-2022b.eb + - Qt5-5.15.7-GCCcore-12.2.0.eb From aa572b561a95aefdec8d1d44f9842f66f812600d Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 12 Dec 2023 16:28:25 +0000 Subject: [PATCH 0308/1795] added reference to pr:19339 --- .../2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index 30e89c4485..c618089663 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -1,3 +1,5 @@ easyconfigs: - foss-2022b.eb - - Qt5-5.15.7-GCCcore-12.2.0.eb + - Qt5-5.15.7-GCCcore-12.2.0.eb: + options: + from-pr: 19339 From 70bba82f1132bb45571ff3759ba97f059aa8df82 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Fri, 15 Dec 2023 10:24:06 +0100 Subject: [PATCH 0309/1795] {2023.06}[GCC/12.3.0] LoopTools 2.15 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 1a1804e2fb..fab52b4468 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -26,3 +26,6 @@ easyconfigs: - LHAPDF-6.5.4-GCC-12.3.0.eb: options: from-pr: 19363 + - LoopTools-2.15-GCC-12.3.0.eb: + options: + from-pr: 19397 From 10751d25858239fe78465a5e523fc0077ac4b487 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 15 Dec 2023 15:15:14 +0100 Subject: [PATCH 0310/1795] Add pre test hook to skip netCDF tests --- eb_hooks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 6fe92c7f7b..2081bfedab 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -366,6 +366,18 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07'] and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" +def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): + """ + Pre-test hook for SciPy-bundle: skip failing tests for selected netCDF versions on neoverse_v1 + cfr. https://github.com/EESSI/software-layer/issues/325 + The following tests are problematic: + 163 - nc_test4_run_par_test (Timeout) + 190 - h5_test_run_par_tests (Timeout) + A few other tests are skipped in the easyconfig and patches for similar issues, see above issue for details. + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg['testopts'] = "|| echo ignoring failing tests" def pre_single_extension_hook(ext, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" From 94206acbcc02da084ba4d5d85219d9ae92f885ac Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 15 Dec 2023 15:16:05 +0100 Subject: [PATCH 0311/1795] Add correct issue reference in hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 2081bfedab..3074eb8ee3 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -369,7 +369,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): """ Pre-test hook for SciPy-bundle: skip failing tests for selected netCDF versions on neoverse_v1 - cfr. https://github.com/EESSI/software-layer/issues/325 + cfr. https://github.com/EESSI/software-layer/issues/425 The following tests are problematic: 163 - nc_test4_run_par_test (Timeout) 190 - h5_test_run_par_tests (Timeout) From db150f4838c47d25e7a683ae239c44b25dd88a6d Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 15 Dec 2023 16:34:42 +0100 Subject: [PATCH 0312/1795] Fix hook --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index 3074eb8ee3..bf5da0cc64 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -430,6 +430,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): 'ESPResSo': pre_test_hook_ignore_failing_tests_ESPResSo, 'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI, 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, + 'netCDF': pre_test_hook_ignore_failing_tests_netCDF, } PRE_SINGLE_EXTENSION_HOOKS = { From cf4fc13080e82caaf18694a95875107a04687cf7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Dec 2023 21:43:02 +0100 Subject: [PATCH 0313/1795] {2023.06}[gfbf/2023a] R 4.3.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index fab52b4468..a9323d3df6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -29,3 +29,6 @@ easyconfigs: - LoopTools-2.15-GCC-12.3.0.eb: options: from-pr: 19397 + - R-4.3.2-gfbf-2023a.eb: + options: + from-pr: 19185 From 8ac3f60142d392644cc2d6d3361c8ef91c43c845 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Dec 2023 22:09:29 +0100 Subject: [PATCH 0314/1795] {2023.06}[gompi/2023a] CDO 2.2.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index fab52b4468..06e6258e85 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -29,3 +29,4 @@ easyconfigs: - LoopTools-2.15-GCC-12.3.0.eb: options: from-pr: 19397 + - CDO-2.2.2-gompi-2023a.eb From 7321ffe247dcf69975108480ea8eca9429a20bad Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Sun, 17 Dec 2023 06:49:10 +0100 Subject: [PATCH 0315/1795] Update eessi-2023.06-eb-4.8.2-2022b.yml Trying to build HarfBuzz-5.3.1 before Qt5 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index c618089663..69ffb750a2 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -1,5 +1,6 @@ easyconfigs: - foss-2022b.eb - - Qt5-5.15.7-GCCcore-12.2.0.eb: + - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: options: from-pr: 19339 + - Qt5-5.15.7-GCCcore-12.2.0.eb From 363f42c9bdaa905c0c094f69479b62520ae3a1e0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 17 Dec 2023 10:53:00 +0100 Subject: [PATCH 0316/1795] {2023.06}[foss/2023a] R-bundle-CRAN 2023.12 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index a9323d3df6..f6d01ee828 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -32,3 +32,6 @@ easyconfigs: - R-4.3.2-gfbf-2023a.eb: options: from-pr: 19185 + - R-bundle-CRAN-2023.12-foss-2023a.eb: + options: + from-pr: 19170 From 69fe0569451b708ce13ba506854d44f383b213f2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 17 Dec 2023 18:12:04 +0100 Subject: [PATCH 0317/1795] fix for LERC sanity check RPATH failure --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index f6d01ee828..7371ee2176 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -32,6 +32,11 @@ easyconfigs: - R-4.3.2-gfbf-2023a.eb: options: from-pr: 19185 + - LERC-4.0.0-GCCcore-12.3.0.eb: + # avoid RPATH-related sanity check errors by compiling LERC's test binary in postinstallcmds; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19386 + options: + from-pr: 19386 - R-bundle-CRAN-2023.12-foss-2023a.eb: options: from-pr: 19170 From 34b7e850bd8ece75a0d01fae5ddc62412701f945 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 18 Dec 2023 11:36:14 +0100 Subject: [PATCH 0318/1795] use patch for MPFR --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 7371ee2176..c5fb6a0425 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -32,6 +32,9 @@ easyconfigs: - R-4.3.2-gfbf-2023a.eb: options: from-pr: 19185 + - MPFR-4.2.0-GCCcore-12.3.0.eb: + options: + from-pr: 19438 - LERC-4.0.0-GCCcore-12.3.0.eb: # avoid RPATH-related sanity check errors by compiling LERC's test binary in postinstallcmds; # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19386 From c45f5cab3b0a542d4964cc293c7dac8c9b0686cb Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Tue, 19 Dec 2023 11:51:36 +0100 Subject: [PATCH 0319/1795] Add Boost-1.82.0-GCC-12.3.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index a9323d3df6..bae536ff9b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -32,3 +32,4 @@ easyconfigs: - R-4.3.2-gfbf-2023a.eb: options: from-pr: 19185 + - Boost-1.82.0-GCC-12.3.0.eb From ec94fd6db01e2e765834d80b034be05e197b0c9f Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:03:32 +0100 Subject: [PATCH 0320/1795] Add netCDF-4.9.2-gompi-2023a.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index a9323d3df6..299616b396 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -32,3 +32,4 @@ easyconfigs: - R-4.3.2-gfbf-2023a.eb: options: from-pr: 19185 + - netCDF-4.9.2-gompi-2023a.eb From 004351a037c62dbacfbe41cd2d5a6a0678ce76d9 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:41:07 +0100 Subject: [PATCH 0321/1795] Add FFmpeg-6.0-GCCcore-12.3.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index a9323d3df6..0f538c08d4 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -32,3 +32,4 @@ easyconfigs: - R-4.3.2-gfbf-2023a.eb: options: from-pr: 19185 + - FFmpeg-6.0-GCCcore-12.3.0.eb From c87714f0106bc13415d0d97e8749b37216757baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Dec 2023 15:48:28 +0100 Subject: [PATCH 0322/1795] explicitly set/override EESSI_CPU_FAMILY based on EESSI_SOFTWARE_SUBDIR_OVERRIDE --- .github/workflows/test-software.eessi.io.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 8c97601611..e6c7a49ada 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -35,6 +35,9 @@ jobs: run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash), + # to prevent issues with checks in the Easybuild configuration that use this variable + export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*} module load EasyBuild which eb eb --version From 7e71f59f6aed081367a6aee89d0b11a7f338a205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Dec 2023 15:58:32 +0100 Subject: [PATCH 0323/1795] add the same check for the other check missing installations task --- .github/workflows/test-software.eessi.io.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index e6c7a49ada..3ac341a177 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -56,6 +56,9 @@ jobs: run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash), + # to prevent issues with checks in the Easybuild configuration that use this variable + export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*} module load EasyBuild which eb eb --version From fcc7ddbab8a57d419736d8a0857943569fdb587c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 19 Dec 2023 18:36:31 +0100 Subject: [PATCH 0324/1795] Also recreated lmodrc when it was changed in a PR --- EESSI-install-software.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f6087b3cfe..8170897726 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -229,7 +229,8 @@ done echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" -if [ ! -f $LMOD_RC ]; then +lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?) +if [ ! -f $LMOD_RC ] || [ ${lmodrc_changed} == '0' ]; then python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH} check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC" fi From 2b09d1c474e399cd1fa6634e18a5234a2e16ac7d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 19 Dec 2023 18:41:43 +0100 Subject: [PATCH 0325/1795] Modified lmodrc to add CUDA support. It now checks if you load the CUDA module if a full CUDA SDK was also installed in host_injections (otherwise you have dead links to the non-redistributable parts of the CUDA SDK). Furthermore, for GPU enabled modules, it checks if the drivers have been linked in in the host_injections directory. It also checks if they are new enough to be used with the CUDA version that was used as a dependency for the GPU-enabled module you are trying to load. If any of these checks is not true, it prints an error message with advice on how to proceed. --- create_lmodrc.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/create_lmodrc.py b/create_lmodrc.py index ae65153a20..adf221ecba 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -17,6 +17,81 @@ } """ +GPU_LMOD_RC ="""require("strict") +local hook = require("Hook") +local open = io.open + +local function read_file(path) + local file = open(path, "rb") -- r read mode and b binary mode + if not file then return nil end + local content = file:read "*a" -- *a or *all reads the whole file + file:close() + return content +end + +local function cuda_enabled_load_hook(t) + local frameStk = require("FrameStk"):singleton() + local mt = frameStk:mt() + local simpleName = string.match(t.modFullName, "(.-)/") + -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. + -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse + -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI + if simpleName == 'CUDA' then + -- get the full host_injections path + local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') + -- build final path where the CUDA software should be installed + local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" + local cudaDirExists = isDir(cudaEasyBuildDir) + if not cudaDirExists then + local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " + advice = advice .. "due to licencing. In order to be able to use the CUDA module, please follow the " + advice = advice .. "instructions available under https://www.eessi.io/docs/gpu/ \\n" + LmodError("\\nYou requested to load ", simpleName, " ", advice) + end + end + -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the EESSI linker, + -- otherwise, refuse to load the requested module and print error message + local haveGpu = mt:haveProperty(simpleName,"arch","gpu") + if haveGpu then + local arch = os.getenv("EESSI_CPU_FAMILY") or "" + local cudaVersionFile = "/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" + local cudaDriverFile = "/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" + local cudaDriverExists = isFile(cudaDriverFile) + local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") + if not (cudaDriverExists or singularityCudaExists) then + local advice = "which relies on the CUDA runtime environment and driver libraries. " + advice = advice .. "In order to be able to use the module, please follow the instructions " + advice = advice .. "available under https://www.eessi.io/docs/gpu/ \\n" + LmodError("\\nYou requested to load ", simpleName, " ", advice) + else + -- CUDA driver exists, now we check its version to see if an update is needed + if cudaDriverExists then + local cudaVersion = read_file(cudaVersionFile) + local cudaVersion_req = os.getenv("EESSICUDAVERSION") + -- driver CUDA versions don't give a patch version for CUDA + local major, minor = string.match(cudaVersion, "(%d+)%.(%d+)") + local major_req, minor_req, patch_req = string.match(cudaVersion_req, "(%d+)%.(%d+)%.(%d+)") + local driver_libs_need_update = false + if major < major_req then + driver_libs_need_update = true + elseif major == major_req then + if minor < minor_req then + driver_libs_need_update = true + end + end + if driver_libs_need_update == true then + local advice = "but the module you want to load requires CUDA " .. cudaVersion_req .. ". " + advice = advice .. "Please update your CUDA driver libraries and then follow the instructions " + advice = advice .. "under https://www.eessi.io/docs/gpu/ to let EESSI know about the update.\\n" + LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice) + end + end + end + end +end + +hook.register("load", cuda_enabled_load_hook) +""" def error(msg): sys.stderr.write("ERROR: %s\n" % msg) @@ -36,6 +111,7 @@ def error(msg): 'dot_lmod': DOT_LMOD, 'prefix': prefix, } +lmodrc_txt += '\n' + GPU_LMOD_RC try: os.makedirs(os.path.dirname(lmodrc_path), exist_ok=True) with open(lmodrc_path, 'w') as fp: From 62e70ba9474638b78edeafbb3886e432b7e88e7c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 19 Dec 2023 18:44:50 +0100 Subject: [PATCH 0326/1795] Adapt created_lmodrc.py for the new domain --- create_lmodrc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index adf221ecba..9c007c15e6 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -54,8 +54,8 @@ local haveGpu = mt:haveProperty(simpleName,"arch","gpu") if haveGpu then local arch = os.getenv("EESSI_CPU_FAMILY") or "" - local cudaVersionFile = "/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" - local cudaDriverFile = "/cvmfs/pilot.eessi-hpc.org/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" + local cudaVersionFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" + local cudaDriverFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" local cudaDriverExists = isFile(cudaDriverFile) local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") if not (cudaDriverExists or singularityCudaExists) then From 045c099b3c4eee0a742a5859a32ba2142e21cc9c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 19 Dec 2023 18:52:52 +0100 Subject: [PATCH 0327/1795] Add post_sanitycheck hook for CUDA in order to only ship the files we are allowed to redistribute. It will create symlinks to the host_injections directory for the rest of the files that we are not allowed to redistribute. Additionally, create a hook to inject the GPU lmod property when creating module files for modules that have CUDA as a dependency --- eb_hooks.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 6fe92c7f7b..78580c14b9 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -68,6 +68,9 @@ def parse_hook(ec, *args, **kwargs): if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) + # inject the GPU property (if required) + ec = inject_gpu_property(ec) + def post_ready_hook(self, *args, **kwargs): """ @@ -247,6 +250,12 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) +def post_sanitycheck_hook(self, *args, **kwargs): + """Main post-sanity-check hook: trigger custom functions based on software name.""" + if self.name in POST_SANITYCHECK_HOOKS: + POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) + + def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): """ Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC @@ -393,6 +402,81 @@ def pre_single_extension_isoband(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " +def post_sanitycheck_cuda(self, *args, **kwargs): + """Delete CUDA files we are not allowed to ship and replace them with a symlink to a possible installation under host_injections.""" + print_msg("Replacing CUDA stuff we cannot ship with symlinks...") + # read CUDA EULA + eula_path = os.path.join(self.installdir, "EULA.txt") + tmp_buffer = [] + with open(eula_path) as infile: + copy = False + for line in infile: + if line.strip() == "2.6. Attachment A": + copy = True + continue + elif line.strip() == "2.7. Attachment B": + copy = False + continue + elif copy: + tmp_buffer.append(line) + # create whitelist without file extensions, they're not really needed and they only complicate things + whitelist = ['EULA', 'README'] + file_extensions = [".so", ".a", ".h", ".bc"] + for tmp in tmp_buffer: + for word in tmp.split(): + if any(ext in word for ext in file_extensions): + whitelist.append(word.split(".")[0]) + whitelist = list(set(whitelist)) + # Do some quick checks for things we should or shouldn't have in the list + if "nvcc" in whitelist: + raise EasyBuildError("Found 'nvcc' in whitelist: %s" % whitelist) + if "libcudart" not in whitelist: + raise EasyBuildError("Did not find 'libcudart' in whitelist: %s" % whitelist) + # iterate over all files in the CUDA path + for root, dirs, files in os.walk(self.installdir): + for filename in files: + # we only really care about real files, i.e. not symlinks + if not os.path.islink(os.path.join(root, filename)): + # check if the current file is part of the whitelist + basename = filename.split(".")[0] + if basename not in whitelist: + # if it is not in the whitelist, delete the file and create a symlink to host_injections + source = os.path.join(root, filename) + target = source.replace("versions", "host_injections") + # Make sure source and target are not the same + if source == target: + raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you are" + "using this hook for an EESSI installation?") + os.remove(source) + # Using os.symlink requires the existence of the target directory, so we use os.system + system_command="ln -s '%s' '%s'" % (target, source) + if os.system(system_command) != 0: + raise EasyBuildError("Failed to create symbolic link: %s" % system_command) + + +def inject_gpu_property(ec): + ec_dict = ec.asdict() + # Check if CUDA is in the dependencies, if so add the GPU Lmod tag + if ("CUDA" in [dep[0] for dep in iter(ec_dict["dependencies"])]): + ec.log.info("[parse hook] Injecting gpu as Lmod arch property and envvar with CUDA version") + key = "modluafooter" + value = 'add_property("arch","gpu")' + cuda_version = 0 + for dep in iter(ec_dict["dependencies"]): + # Make CUDA a build dependency only (rpathing saves us from link errors) + if "CUDA" in dep[0]: + cuda_version = dep[1] + ec_dict["dependencies"].remove(dep) + ec_dict["builddependencies"].append(dep) if dep not in ec_dict["builddependencies"] else ec_dict["builddependencies"] + value = "\n".join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) + if key in ec_dict: + if not value in ec_dict[key]: + ec[key] = "\n".join([ec_dict[key], value]) + else: + ec[key] = value + return ec + + PARSE_HOOKS = { 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, @@ -424,3 +508,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): 'isoband': pre_single_extension_isoband, 'testthat': pre_single_extension_testthat, } + +POST_SANITYCHECK_HOOKS = { + 'CUDA': post_sanitycheck_cuda, +} From 4a4c6e768d1b469baf4476a32f9721f789ce341a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 19 Dec 2023 18:53:37 +0100 Subject: [PATCH 0328/1795] Add (the redistributable part of) CUDA to the softare stack --- .../2023.06/eessi-2023.06-eb-4.8.2-system.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml index f02b9f2802..86d6931820 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml @@ -5,3 +5,7 @@ easyconfigs: - Nextflow-23.10.0.eb: options: from-pr: 19172 + - CUDA-12.1.1.eb: + options: + include-easyblocks-from-pr: 3045 + accept-eula-for: CUDA From 0346b22291671d5f03343b11e9a919f13f3ab9c5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 19 Dec 2023 18:55:15 +0100 Subject: [PATCH 0329/1795] Add CUDA-Samples to the build list --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 567db44e42..0537c448e5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -35,3 +35,9 @@ easyconfigs: - Boost-1.82.0-GCC-12.3.0.eb - netCDF-4.9.2-gompi-2023a.eb - FFmpeg-6.0-GCCcore-12.3.0.eb + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb + # use easyconfig that only install subset of CUDA samples, + # to circumvent problem with nvcc linking to glibc of host OS; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189 + options: + from-pr: 19189 From 2b1054f7b393adf590201b5a67ddf5622ccd1b3f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 19 Dec 2023 19:32:44 +0100 Subject: [PATCH 0330/1795] install_scripts.sh should install the scripts required to do install the required GPU components in host_injections. The EESSI-install-software.sh has been modified to run install_scripts.sh early on and then run the actual installed scripts to install a full cuda SDK and drivers. This should enable building and using CUDA software anywhere down the line in this same environment --- EESSI-install-software.sh | 16 ++++++++++ install_scripts.sh | 62 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100755 install_scripts.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f6087b3cfe..4aac3772cf 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -187,6 +187,22 @@ fi # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) +# install any additional required scripts +# order is important: these are needed to install a full CUDA SDK in host_injections +install_scripts_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^install_scripts.sh$' > /dev/null; echo $?) +if [ ${install_scripts_changed} == '0' ]; then + # for now, this just reinstalls all scripts. Note the most elegant, but works + ${TOPDIR}/install_scripts.sh --prefix ${EESSI_CVMFS_REPO} +fi + +# Install full CUDA SDK in host_injections +# Hardcode this for now, see if it works +# TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install +${EESSI_CVMFS_REPO}/gpu_support/nvidia/install_cuda_host_injections.sh 12.1.1 + +# Install drivers in host_injections +${EESSI_CVMFS_REPO}/gpu_support/nvidia/link_nvidia_host_libraries.sh + # use PR patch file to determine in which easystack files stuff was added for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do diff --git a/install_scripts.sh b/install_scripts.sh new file mode 100755 index 0000000000..209d953c88 --- /dev/null +++ b/install_scripts.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Script to install scripts from the software-layer repo into the EESSI software stack + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " -p | --prefix - prefix to copy the scripts to" + echo " -h | --help - display this usage information" +} + + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -o|--prefix) + INSTALL_PREFIX="$2" + shift 2 + ;; + -h|--help) + display_help # Call your function + # no shifting needed here, we're done. + exit 0 + ;; + -*|--*) + echo "Error: Unknown option: $1" >&2 + exit 1 + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +TOPDIR=$(dirname $(realpath $0)) + +# Subdirs for generic scripts +SCRIPTS_DIR_SOURCE=${TOPDIR}/scripts/ # Source dir +SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts/ # Target dir + +# Create target dir +mkdir -p ${SCRIPTS_DIR_TARGET} + +# Copy scripts into this prefix +for file in utils.sh; do + cp ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} +done +# Subdirs for GPU support +NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia/ # Source dir +NVIDIA_GPU_SUPPORT_DIR_TARGET=${INSTALL_PREFIX}/gpu_support/nvidia/ # Target dir + +# Create target dir +mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET} + +# Copy files from this directory into the prefix +# To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add +for file in install_cuda_host_injections.sh link_nvidia_host_injections.sh; do + cp ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file} +done From 5905e727f146480d6472d2f3a1d0ca6ab53105ee Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 20 Dec 2023 01:07:41 +0100 Subject: [PATCH 0331/1795] Tweak GPU support implementation --- EESSI-install-software.sh | 11 ++++------- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 2 +- install_scripts.sh | 16 ++++++++-------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 5e9bd5d472..95fb03e9b7 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -189,19 +189,16 @@ pr_diff=$(ls [0-9]*.diff | head -1) # install any additional required scripts # order is important: these are needed to install a full CUDA SDK in host_injections -install_scripts_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^install_scripts.sh$' > /dev/null; echo $?) -if [ ${install_scripts_changed} == '0' ]; then - # for now, this just reinstalls all scripts. Note the most elegant, but works - ${TOPDIR}/install_scripts.sh --prefix ${EESSI_CVMFS_REPO} -fi +# for now, this just reinstalls all scripts. Note the most elegant, but works +${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} # Install full CUDA SDK in host_injections # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install -${EESSI_CVMFS_REPO}/gpu_support/nvidia/install_cuda_host_injections.sh 12.1.1 +${EESSI_PREFIX}/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula # Install drivers in host_injections -${EESSI_CVMFS_REPO}/gpu_support/nvidia/link_nvidia_host_libraries.sh +${EESSI_PREFIX}/gpu_support/nvidia/link_nvidia_host_libraries.sh # use PR patch file to determine in which easystack files stuff was added for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 0537c448e5..87ccd69e99 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -35,7 +35,7 @@ easyconfigs: - Boost-1.82.0-GCC-12.3.0.eb - netCDF-4.9.2-gompi-2023a.eb - FFmpeg-6.0-GCCcore-12.3.0.eb - - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb: # use easyconfig that only install subset of CUDA samples, # to circumvent problem with nvcc linking to glibc of host OS; # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189 diff --git a/install_scripts.sh b/install_scripts.sh index 209d953c88..8fb27826c6 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -13,7 +13,7 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in - -o|--prefix) + -p|--prefix) INSTALL_PREFIX="$2" shift 2 ;; @@ -38,25 +38,25 @@ set -- "${POSITIONAL_ARGS[@]}" TOPDIR=$(dirname $(realpath $0)) # Subdirs for generic scripts -SCRIPTS_DIR_SOURCE=${TOPDIR}/scripts/ # Source dir -SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts/ # Target dir +SCRIPTS_DIR_SOURCE=${TOPDIR}/scripts # Source dir +SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts # Target dir # Create target dir mkdir -p ${SCRIPTS_DIR_TARGET} # Copy scripts into this prefix for file in utils.sh; do - cp ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} + cp -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} done # Subdirs for GPU support -NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia/ # Source dir -NVIDIA_GPU_SUPPORT_DIR_TARGET=${INSTALL_PREFIX}/gpu_support/nvidia/ # Target dir +NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia # Source dir +NVIDIA_GPU_SUPPORT_DIR_TARGET=${INSTALL_PREFIX}/gpu_support/nvidia # Target dir # Create target dir mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET} # Copy files from this directory into the prefix # To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add -for file in install_cuda_host_injections.sh link_nvidia_host_injections.sh; do - cp ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file} +for file in install_cuda_host_injections.sh link_nvidia_host_libraries.sh; do + cp -u ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file} done From 73618a00ee06889314d6efc0e362882b39f34161 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 20 Dec 2023 01:22:21 +0100 Subject: [PATCH 0332/1795] Add missing quotes on errors --- gpu_support/nvidia/link_nvidia_host_libraries.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/gpu_support/nvidia/link_nvidia_host_libraries.sh index 6458be7fae..cb7420a0e9 100755 --- a/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -36,7 +36,7 @@ if [ ${#found_paths[@]} -gt 0 ]; then host_ldconfig=${found_paths[0]} else error="$command_name not found in PATH or only found in paths starting with $exclude_prefix." - fatal_error $error + fatal_error "$error" fi # Make sure EESSI is initialised (doesn't matter what version) @@ -52,7 +52,7 @@ if $nvidia_smi_command > /dev/null; then host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}') else error="Failed to successfully execute\n $nvidia_smi_command\n" - fatal_error $error + fatal_error "$error" fi # Let's make sure the driver libraries are not already in place @@ -71,7 +71,7 @@ if [ -e "$host_injection_driver_version_file" ]; then rm $host_injection_driver_dir/* if [ $? -ne 0 ]; then error="Unable to remove files under '$host_injection_driver_dir'." - fatal_error $error + fatal_error "$error" fi fi fi From 32925fe17e7576847be87ef3f1786ee374a2356a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 20 Dec 2023 11:12:27 +0100 Subject: [PATCH 0333/1795] Error messages now refer to the scripts that need to be run to install the CUDA SDK or link the CUDA drivers --- create_lmodrc.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index 9c007c15e6..6a72d8dc62 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -43,9 +43,11 @@ local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" local cudaDirExists = isDir(cudaEasyBuildDir) if not cudaDirExists then + local cvmfsRepo = os.getenv('EESSI_CVMFS_REPO') local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " - advice = advice .. "due to licencing. In order to be able to use the CUDA module, please follow the " - advice = advice .. "instructions available under https://www.eessi.io/docs/gpu/ \\n" + advice = advice .. "due to licencing. Please install a full copy of the CUDA SDK using the script " + advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/install_cuda_host_injections.sh.\\n" + advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n" LmodError("\\nYou requested to load ", simpleName, " ", advice) end end @@ -60,8 +62,10 @@ local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") if not (cudaDriverExists or singularityCudaExists) then local advice = "which relies on the CUDA runtime environment and driver libraries. " - advice = advice .. "In order to be able to use the module, please follow the instructions " - advice = advice .. "available under https://www.eessi.io/docs/gpu/ \\n" + advice = advice .. "In order to be able to use the module, please run the script " + advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/link_nvidia_host_libraries.sh " + advice = advice .. "to make sure EESSI can find the drivers from on your host system.\\n" + advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n" LmodError("\\nYou requested to load ", simpleName, " ", advice) else -- CUDA driver exists, now we check its version to see if an update is needed @@ -81,8 +85,10 @@ end if driver_libs_need_update == true then local advice = "but the module you want to load requires CUDA " .. cudaVersion_req .. ". " - advice = advice .. "Please update your CUDA driver libraries and then follow the instructions " - advice = advice .. "under https://www.eessi.io/docs/gpu/ to let EESSI know about the update.\\n" + advice = advice .. "Please update your CUDA driver libraries and rerun the script" + advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/install_cuda_host_injections.sh " + advice = advice .. "to let EESSI know about the update.\\n" + advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n" LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice) end end From a33a0cd204a13be3321336caa18c5b3faa5b29f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 13:50:46 +0100 Subject: [PATCH 0334/1795] make install_scripts a bit more verbose --- install_scripts.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/install_scripts.sh b/install_scripts.sh index 8fb27826c6..d53dbb6a9c 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -45,8 +45,9 @@ SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts # Target dir mkdir -p ${SCRIPTS_DIR_TARGET} # Copy scripts into this prefix +echo "copying scripts from ${SCRIPTS_DIR_SOURCE} to ${SCRIPTS_DIR_TARGET}" for file in utils.sh; do - cp -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} + cp -v -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} done # Subdirs for GPU support NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia # Source dir @@ -57,6 +58,7 @@ mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET} # Copy files from this directory into the prefix # To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add +echo "copying scripts from ${NVIDIA_GPU_SUPPORT_DIR_SOURCE} to ${NVIDIA_GPU_SUPPORT_DIR_TARGET}" for file in install_cuda_host_injections.sh link_nvidia_host_libraries.sh; do - cp -u ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file} + cp -v -u ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file} done From c7b380d7b3d7580c935824fb3f3c4c80200b8301 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 14:07:29 +0100 Subject: [PATCH 0335/1795] use separate easystack file for CUDA + control order in which easystack are processed (where needed) --- easystacks/software.eessi.io/2023.06/README.md | 7 +++++++ ....2-system.yml => eessi-2023.06-eb-4.8.2-001-system.yml} | 4 ---- .../2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml | 5 +++++ 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/README.md rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.8.2-system.yml => eessi-2023.06-eb-4.8.2-001-system.yml} (56%) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/README.md b/easystacks/software.eessi.io/2023.06/README.md new file mode 100644 index 0000000000..733ebf9475 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/README.md @@ -0,0 +1,7 @@ +File naming matters, since it determines the order in which easystack files are processed. + +Software installed with system toolchain should be installed first, +this includes EasyBuild itself, see `eessi-2023.06-eb-4.8.2-001-system.yml` . + +CUDA installations must be done before CUDA is required as dependency for something +built with a non-system toolchain, see `eessi-2023.06-eb-4.8.2-010-CUDA.yml` . diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml similarity index 56% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml rename to easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml index 86d6931820..f02b9f2802 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml @@ -5,7 +5,3 @@ easyconfigs: - Nextflow-23.10.0.eb: options: from-pr: 19172 - - CUDA-12.1.1.eb: - options: - include-easyblocks-from-pr: 3045 - accept-eula-for: CUDA diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml new file mode 100644 index 0000000000..dda274b8db --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-010-CUDA.yml @@ -0,0 +1,5 @@ +easyconfigs: + - CUDA-12.1.1.eb: + options: + include-easyblocks-from-pr: 3045 + accept-eula-for: CUDA From f506566c982818bf0ee985c70d7dbe0d24084035 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 14:10:12 +0100 Subject: [PATCH 0336/1795] copy EasyBuild log file in case CUDA installation failed in install_cuda_host_injections.sh --- gpu_support/nvidia/install_cuda_host_injections.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gpu_support/nvidia/install_cuda_host_injections.sh b/gpu_support/nvidia/install_cuda_host_injections.sh index f02f0da02e..62996fa924 100755 --- a/gpu_support/nvidia/install_cuda_host_injections.sh +++ b/gpu_support/nvidia/install_cuda_host_injections.sh @@ -200,7 +200,9 @@ else eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}" ret=$? if [ $ret -ne 0 ]; then - fatal_error "CUDA installation failed, please check EasyBuild logs..." + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + cp -a ${eb_last_log} . + fatal_error "CUDA installation failed, please check EasyBuild logs $(basename ${eb_last_log})..." else echo_green "CUDA installation at ${cuda_install_parent}/software/CUDA/${install_cuda_version} succeeded!" fi From e3ddaccfc04cca311d40ef69eb55028ccf257dcc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 14:29:00 +0100 Subject: [PATCH 0337/1795] add additional optional options required for handling NVIDIA support to start build container --- bot/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 4af217628e..66f93d523e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -176,6 +176,11 @@ mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} declare -a BUILD_STEP_ARGS=() BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") +# add options required to handle NVIDIA support +BUILD_STEP_ARGS+=("--nvidia" "all") +if [[ ! -z ${SHARED_FS_PATH} ]]; then + BUILD_STEP_ARGS+=("--host-injections ${SHARED_FS_PATH}/host-injections") +fi # prepare arguments to install_software_layer.sh (specific to build step) declare -a INSTALL_SCRIPT_ARGS=() From 16ddf7f8a0d29cfe47c9948933bb2a10b5ee17c0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 14:41:53 +0100 Subject: [PATCH 0338/1795] fix typo when passing --host-injections to container script --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 66f93d523e..1622e757e2 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -179,7 +179,7 @@ BUILD_STEP_ARGS+=("--storage" "${STORAGE}") # add options required to handle NVIDIA support BUILD_STEP_ARGS+=("--nvidia" "all") if [[ ! -z ${SHARED_FS_PATH} ]]; then - BUILD_STEP_ARGS+=("--host-injections ${SHARED_FS_PATH}/host-injections") + BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi # prepare arguments to install_software_layer.sh (specific to build step) From 35d6084a25ea76b75982e9bbac0154236d860d6a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 14:50:04 +0100 Subject: [PATCH 0339/1795] correctly pass --nv to singularity command --- eessi_container.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 81d7be81ad..268f94975e 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -118,7 +118,6 @@ STORAGE= LIST_REPOS=0 MODE="shell" SETUP_NVIDIA=0 -ADDITIONAL_SINGULARITY_FLAGS= REPOSITORY="EESSI" RESUME= SAVE= @@ -437,12 +436,14 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" +declare -a ADDITIONAL_CONTAINER_OPTIONS=() + # Configure anything we need for NVIDIA GPUs and CUDA installation if [[ ${SETUP_NVIDIA} -eq 1 ]]; then if [[ "${NVIDIA_MODE}" == "run" || "${NVIDIA_MODE}" == "all" ]]; then # Give singularity the appropriate flag - ADDITIONAL_SINGULARITY_FLAGS="--nv ${ADDITIONAL_SINGULARITY_FLAGS}" - [[ ${VERBOSE} -eq 1 ]] && echo "ADDITIONAL_SINGULARITY_FLAGS=${ADDITIONAL_SINGULARITY_FLAGS}" + ADDITIONAL_CONTAINER_OPTIONS+=(--nv) + [[ ${VERBOSE} -eq 1 ]] && echo "ADDITIONAL_CONTAINER_OPTIONS=${ADDITIONAL_CONTAINER_OPTIONS[@]}" fi if [[ "${NVIDIA_MODE}" == "install" || "${NVIDIA_MODE}" == "all" ]]; then # Add additional bind mounts to allow CUDA to install within a container @@ -621,8 +622,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then fi echo "Launching container with command (next line):" -echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_SINGULARITY_FLAGS} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" -singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_SINGULARITY_FLAGS} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" +echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" +singularity ${RUN_QUIET} ${MODE} "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" exit_code=$? # 6. save tmp if requested (arg -s|--save) From fd976675c798b1c79205460eec225469fbf28473 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 14:50:55 +0100 Subject: [PATCH 0340/1795] use quotes when adding --nv --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 268f94975e..d6e9558202 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -442,7 +442,7 @@ declare -a ADDITIONAL_CONTAINER_OPTIONS=() if [[ ${SETUP_NVIDIA} -eq 1 ]]; then if [[ "${NVIDIA_MODE}" == "run" || "${NVIDIA_MODE}" == "all" ]]; then # Give singularity the appropriate flag - ADDITIONAL_CONTAINER_OPTIONS+=(--nv) + ADDITIONAL_CONTAINER_OPTIONS+=("--nv") [[ ${VERBOSE} -eq 1 ]] && echo "ADDITIONAL_CONTAINER_OPTIONS=${ADDITIONAL_CONTAINER_OPTIONS[@]}" fi if [[ "${NVIDIA_MODE}" == "install" || "${NVIDIA_MODE}" == "all" ]]; then From 19171465240f52b3ef0f7eba17e4467031e23a58 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 15:22:20 +0100 Subject: [PATCH 0341/1795] comment out running of link_nvidia_host_libraries.sh script, since it requires working nvidia-smi command --- EESSI-install-software.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 95fb03e9b7..4e3f0acee5 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -198,7 +198,9 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} ${EESSI_PREFIX}/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula # Install drivers in host_injections -${EESSI_PREFIX}/gpu_support/nvidia/link_nvidia_host_libraries.sh +# TODO: this is commented out for now, because the script assumes that nvidia-smi is available and works; +# if not, an error is produced, and the bot flags the whole build as failed (even when not installing GPU software) +# ${EESSI_PREFIX}/gpu_support/nvidia/link_nvidia_host_libraries.sh # use PR patch file to determine in which easystack files stuff was added for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do From f80f0fc0f70026e4be1fa980e13a991102503737 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 20:15:58 +0100 Subject: [PATCH 0342/1795] clean up post_sanitycheck_cuda hook and inject_gpu_property function used in parse_hook --- eb_hooks.py | 149 +++++++++++++++++++++++++++++----------------------- 1 file changed, 83 insertions(+), 66 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 78580c14b9..27cc873fa1 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -7,7 +7,7 @@ from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option -from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which +from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -250,12 +250,6 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) -def post_sanitycheck_hook(self, *args, **kwargs): - """Main post-sanity-check hook: trigger custom functions based on software name.""" - if self.name in POST_SANITYCHECK_HOOKS: - POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) - - def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): """ Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC @@ -402,76 +396,99 @@ def pre_single_extension_isoband(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " +def post_sanitycheck_hook(self, *args, **kwargs): + """Main post-sanity-check hook: trigger custom functions based on software name.""" + if self.name in POST_SANITYCHECK_HOOKS: + POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) + + def post_sanitycheck_cuda(self, *args, **kwargs): - """Delete CUDA files we are not allowed to ship and replace them with a symlink to a possible installation under host_injections.""" - print_msg("Replacing CUDA stuff we cannot ship with symlinks...") - # read CUDA EULA - eula_path = os.path.join(self.installdir, "EULA.txt") - tmp_buffer = [] - with open(eula_path) as infile: - copy = False - for line in infile: - if line.strip() == "2.6. Attachment A": - copy = True - continue - elif line.strip() == "2.7. Attachment B": - copy = False - continue - elif copy: - tmp_buffer.append(line) - # create whitelist without file extensions, they're not really needed and they only complicate things - whitelist = ['EULA', 'README'] - file_extensions = [".so", ".a", ".h", ".bc"] - for tmp in tmp_buffer: - for word in tmp.split(): - if any(ext in word for ext in file_extensions): - whitelist.append(word.split(".")[0]) - whitelist = list(set(whitelist)) - # Do some quick checks for things we should or shouldn't have in the list - if "nvcc" in whitelist: - raise EasyBuildError("Found 'nvcc' in whitelist: %s" % whitelist) - if "libcudart" not in whitelist: - raise EasyBuildError("Did not find 'libcudart' in whitelist: %s" % whitelist) - # iterate over all files in the CUDA path - for root, dirs, files in os.walk(self.installdir): - for filename in files: - # we only really care about real files, i.e. not symlinks - if not os.path.islink(os.path.join(root, filename)): - # check if the current file is part of the whitelist - basename = filename.split(".")[0] - if basename not in whitelist: - # if it is not in the whitelist, delete the file and create a symlink to host_injections - source = os.path.join(root, filename) - target = source.replace("versions", "host_injections") - # Make sure source and target are not the same - if source == target: - raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you are" - "using this hook for an EESSI installation?") - os.remove(source) - # Using os.symlink requires the existence of the target directory, so we use os.system - system_command="ln -s '%s' '%s'" % (target, source) - if os.system(system_command) != 0: - raise EasyBuildError("Failed to create symbolic link: %s" % system_command) + """ + Remove files from CUDA installation that we are not allowed to ship, + and replace them with a symlink to a corresponding installation under host_injections. + """ + if self.name == 'CUDA': + print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") + + # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped + eula_path = os.path.join(self.installdir, 'EULA.txt') + relevant_eula_lines = [] + with open(eula_path) as infile: + copy = False + for line in infile: + if line.strip() == "2.6. Attachment A": + copy = True + continue + elif line.strip() == "2.7. Attachment B": + copy = False + continue + elif copy: + relevant_eula_lines.append(line) + + # create list without file extensions, they're not really needed and they only complicate things + allowlist = ['EULA', 'README'] + file_extensions = ['.so', '.a', '.h', '.bc'] + for line in relevant_eula_lines: + for word in line.split(): + if any(ext in word for ext in file_extensions): + allowlist.append(os.path.splitext(word)[0]) + allowlist = sorted(set(allowlist)) + self.log.info("Allowlist for files in CUDA installation that can be redistributed: " + ', '.join(allowlist)) + + # Do some quick sanity checks for things we should or shouldn't have in the list + if 'nvcc' in allowlist: + raise EasyBuildError("Found 'nvcc' in allowlist: %s" % allowlist) + if 'libcudart' not in allowlist: + raise EasyBuildError("Did not find 'libcudart' in allowlist: %s" % allowlist) + + # iterate over all files in the CUDA installation directory + for dir_path, _, files in os.walk(self.installdir): + for filename in files: + full_path = os.path.join(dir_path, filename) + # we only really care about real files, i.e. not symlinks + if not os.path.islink(full_path): + # check if the current file is part of the allowlist + basename = os.path.splitext(filename)[0] + if basename in allowlist: + self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) + else: + self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", + basename, full_path) + # if it is not in the allowlist, delete the file and create a symlink to host_injections + host_inj_path = full_path.replace('versions', 'host_injections') + # make sure source and target of symlink are not the same + if full_path == host_inj_path: + raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " + "are using this hook for an EESSI installation?", + full_path, host_inj_path) + remove_file(full_path) + symlink(host_inj_path, full_path) + else: + raise EasyBuildError("CUDA-specific hook triggered for non-CUDA easyconfig?!") def inject_gpu_property(ec): + """ + Add 'gpu' property, via modluafooter easyconfig parameter + """ ec_dict = ec.asdict() - # Check if CUDA is in the dependencies, if so add the GPU Lmod tag - if ("CUDA" in [dep[0] for dep in iter(ec_dict["dependencies"])]): - ec.log.info("[parse hook] Injecting gpu as Lmod arch property and envvar with CUDA version") - key = "modluafooter" + # Check if CUDA is in the dependencies, if so add the 'gpu' Lmod property + if ('CUDA' in [dep[0] for dep in iter(ec_dict['dependencies'])]): + ec.log.info("Injecting gpu as Lmod arch property and envvar with CUDA version") + key = 'modluafooter' value = 'add_property("arch","gpu")' cuda_version = 0 - for dep in iter(ec_dict["dependencies"]): + for dep in iter(ec_dict['dependencies']): # Make CUDA a build dependency only (rpathing saves us from link errors) - if "CUDA" in dep[0]: + if 'CUDA' in dep[0]: cuda_version = dep[1] - ec_dict["dependencies"].remove(dep) - ec_dict["builddependencies"].append(dep) if dep not in ec_dict["builddependencies"] else ec_dict["builddependencies"] - value = "\n".join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) + ec_dict['dependencies'].remove(dep) + if dep not in ec_dict['builddependencies']: + ec_dict['builddependencies'].append(dep) + value = '\n'.join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) if key in ec_dict: if not value in ec_dict[key]: - ec[key] = "\n".join([ec_dict[key], value]) + ec[key] = '\n'.join([ec_dict[key], value]) else: ec[key] = value return ec From 35b5efb35c5dcbae09d083e25f9cfa46035466ee Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 20 Dec 2023 19:41:12 +0000 Subject: [PATCH 0343/1795] {2023.06}[foss/2022b] QuantumESPRESSO v7.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index 69ffb750a2..fd88fafb0c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -4,3 +4,4 @@ easyconfigs: options: from-pr: 19339 - Qt5-5.15.7-GCCcore-12.2.0.eb + - QuantumESPRESSO-7.2-foss-2022b.eb From 2d378421b697306642467d9019b748a1e67f39bc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 21:22:42 +0100 Subject: [PATCH 0344/1795] remove empty line in eessi-2023.06-eb-4.8.2-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 91efae766f..87ccd69e99 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -35,7 +35,6 @@ easyconfigs: - Boost-1.82.0-GCC-12.3.0.eb - netCDF-4.9.2-gompi-2023a.eb - FFmpeg-6.0-GCCcore-12.3.0.eb - - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb: # use easyconfig that only install subset of CUDA samples, # to circumvent problem with nvcc linking to glibc of host OS; From f007c4061c3104a29b1053e877c61a32a244c30f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 20 Dec 2023 23:24:15 +0100 Subject: [PATCH 0345/1795] use easyconfigs PR 19451 for installing CUDA-Samples v12.1 --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 87ccd69e99..596b9ea21f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -37,7 +37,9 @@ easyconfigs: - FFmpeg-6.0-GCCcore-12.3.0.eb - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb: # use easyconfig that only install subset of CUDA samples, - # to circumvent problem with nvcc linking to glibc of host OS; - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189 + # to circumvent problem with nvcc linking to glibc of host OS, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189; + # and where additional samples are excluded because they fail to build on aarch64, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19451; options: - from-pr: 19189 + from-pr: 19451 From 70fa0f9ac3cd54a582640dc7bdadcd31c210fffc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 21 Dec 2023 00:34:38 +0100 Subject: [PATCH 0346/1795] Ship the scripts, and keep them in a single location --- EESSI-install-software.sh | 4 ++-- create_lmodrc.py | 22 +++++++++---------- create_tarball.sh | 4 ++++ install_scripts.sh | 4 ++-- .../nvidia/install_cuda_host_injections.sh | 2 +- .../nvidia/link_nvidia_host_libraries.sh | 2 +- 6 files changed, 20 insertions(+), 18 deletions(-) rename {gpu_support => scripts/gpu_support}/nvidia/install_cuda_host_injections.sh (99%) rename {gpu_support => scripts/gpu_support}/nvidia/link_nvidia_host_libraries.sh (99%) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 4e3f0acee5..b61ca7a579 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -195,12 +195,12 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} # Install full CUDA SDK in host_injections # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install -${EESSI_PREFIX}/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula +${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula # Install drivers in host_injections # TODO: this is commented out for now, because the script assumes that nvidia-smi is available and works; # if not, an error is produced, and the bot flags the whole build as failed (even when not installing GPU software) -# ${EESSI_PREFIX}/gpu_support/nvidia/link_nvidia_host_libraries.sh +# ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh # use PR patch file to determine in which easystack files stuff was added for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do diff --git a/create_lmodrc.py b/create_lmodrc.py index 6a72d8dc62..80635d78cc 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -36,6 +36,7 @@ -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI + local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" if simpleName == 'CUDA' then -- get the full host_injections path local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') @@ -43,11 +44,10 @@ local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" local cudaDirExists = isDir(cudaEasyBuildDir) if not cudaDirExists then - local cvmfsRepo = os.getenv('EESSI_CVMFS_REPO') local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " - advice = advice .. "due to licencing. Please install a full copy of the CUDA SDK using the script " - advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/install_cuda_host_injections.sh.\\n" - advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n" + advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where EESSI " + advice = advice .. "can find it.\\n" + advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) end end @@ -62,10 +62,9 @@ local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") if not (cudaDriverExists or singularityCudaExists) then local advice = "which relies on the CUDA runtime environment and driver libraries. " - advice = advice .. "In order to be able to use the module, please run the script " - advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/link_nvidia_host_libraries.sh " - advice = advice .. "to make sure EESSI can find the drivers from on your host system.\\n" - advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n" + advice = advice .. "In order to be able to use the module, you will need " + advice = advice .. "to make sure EESSI can find the driver libraries on your host system.\\n" + advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) else -- CUDA driver exists, now we check its version to see if an update is needed @@ -85,10 +84,9 @@ end if driver_libs_need_update == true then local advice = "but the module you want to load requires CUDA " .. cudaVersion_req .. ". " - advice = advice .. "Please update your CUDA driver libraries and rerun the script" - advice = advice .. cvmfsRepo .. "/gpu_support/nvidia/install_cuda_host_injections.sh " - advice = advice .. "to let EESSI know about the update.\\n" - advice = advice .. "More information, see https://www.eessi.io/docs/gpu/.\\n" + advice = advice .. "Please update your CUDA driver libraries and then " + advice = advice .. "let EESSI know about the update.\\n" + advice = advice .. refer_to_docs LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice) end end diff --git a/create_tarball.sh b/create_tarball.sh index 8510caebf1..09ce94c835 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -40,6 +40,10 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}.. files_list=${tmpdir}/files.list.txt module_files_list=${tmpdir}/module_files.list.txt +if [ -d ${eessi_version}/scripts ]; then + # include scripts we wish to ship along with EESSI, + find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list} +fi if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) diff --git a/install_scripts.sh b/install_scripts.sh index d53dbb6a9c..224400db1c 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -50,8 +50,8 @@ for file in utils.sh; do cp -v -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} done # Subdirs for GPU support -NVIDIA_GPU_SUPPORT_DIR_SOURCE=${TOPDIR}/gpu_support/nvidia # Source dir -NVIDIA_GPU_SUPPORT_DIR_TARGET=${INSTALL_PREFIX}/gpu_support/nvidia # Target dir +NVIDIA_GPU_SUPPORT_DIR_SOURCE=${SCRIPTS_DIR_SOURCE}/gpu_support/nvidia # Source dir +NVIDIA_GPU_SUPPORT_DIR_TARGET=${SCRIPTS_DIR_TARGET}/gpu_support/nvidia # Target dir # Create target dir mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET} diff --git a/gpu_support/nvidia/install_cuda_host_injections.sh b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh similarity index 99% rename from gpu_support/nvidia/install_cuda_host_injections.sh rename to scripts/gpu_support/nvidia/install_cuda_host_injections.sh index 62996fa924..a9310d817a 100755 --- a/gpu_support/nvidia/install_cuda_host_injections.sh +++ b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh @@ -14,7 +14,7 @@ # Initialise our bash functions TOPDIR=$(dirname $(realpath $BASH_SOURCE)) -source "$TOPDIR"/../../scripts/utils.sh +source "$TOPDIR"/../../utils.sh # Function to display help message show_help() { diff --git a/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh similarity index 99% rename from gpu_support/nvidia/link_nvidia_host_libraries.sh rename to scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index cb7420a0e9..e6ff110797 100755 --- a/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -5,7 +5,7 @@ # Initialise our bash functions TOPDIR=$(dirname $(realpath $BASH_SOURCE)) -source "$TOPDIR"/../../scripts/utils.sh +source "$TOPDIR"/../../utils.sh # We rely on ldconfig to give us the location of the libraries on the host command_name="ldconfig" From db0c14136f7d4e244e42e374f64357eea0f63789 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 21 Dec 2023 09:27:28 +0000 Subject: [PATCH 0347/1795] Update create_lmodrc.py Co-authored-by: Kenneth Hoste --- create_lmodrc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index 80635d78cc..0e738a530e 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -63,7 +63,7 @@ if not (cudaDriverExists or singularityCudaExists) then local advice = "which relies on the CUDA runtime environment and driver libraries. " advice = advice .. "In order to be able to use the module, you will need " - advice = advice .. "to make sure EESSI can find the driver libraries on your host system.\\n" + advice = advice .. "to make sure EESSI can find the GPU driver libraries on your host system.\\n" advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) else From 293b1075e5619748f043019a5e856e0b52046d0d Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 21 Dec 2023 09:28:08 +0000 Subject: [PATCH 0348/1795] Update create_tarball.sh Co-authored-by: Kenneth Hoste --- create_tarball.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/create_tarball.sh b/create_tarball.sh index 09ce94c835..4d3ad37311 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -40,10 +40,12 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}.. files_list=${tmpdir}/files.list.txt module_files_list=${tmpdir}/module_files.list.txt +# include scripts that were copied by install_scripts.sh if [ -d ${eessi_version}/scripts ]; then # include scripts we wish to ship along with EESSI, find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list} fi + if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) From 73476b2b20c551ab7e6f0947f544f5f44d3f2537 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 21 Dec 2023 12:16:38 +0100 Subject: [PATCH 0349/1795] Only copy scripts if the contents differ --- install_scripts.sh | 21 +- .../nvidia/install_cuda_host_injections.sh | 211 ++++++++++++++++++ .../nvidia/link_nvidia_host_libraries.sh | 136 +++++++++++ temp/scripts/utils.sh | 144 ++++++++++++ 4 files changed, 510 insertions(+), 2 deletions(-) create mode 100755 temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh create mode 100755 temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh create mode 100644 temp/scripts/utils.sh diff --git a/install_scripts.sh b/install_scripts.sh index 224400db1c..588248e8d2 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -8,6 +8,23 @@ display_help() { echo " -h | --help - display this usage information" } +compare_and_copy() { + if [ "$#" -ne 2 ]; then + echo "Usage of function: compare_and_copy " + return 1 + fi + + source_file="$1" + destination_file="$2" + + if [ ! -f "$destination_file" ] || ! diff -q "$source_file" "$destination_file" ; then + cp "$source_file" "$destination_file" + echo "File $1 copied to $2." + else + echo "Files $1 and $2 are identical. No copy needed." + fi +} + POSITIONAL_ARGS=() @@ -47,7 +64,7 @@ mkdir -p ${SCRIPTS_DIR_TARGET} # Copy scripts into this prefix echo "copying scripts from ${SCRIPTS_DIR_SOURCE} to ${SCRIPTS_DIR_TARGET}" for file in utils.sh; do - cp -v -u ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} + compare_and_copy ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} done # Subdirs for GPU support NVIDIA_GPU_SUPPORT_DIR_SOURCE=${SCRIPTS_DIR_SOURCE}/gpu_support/nvidia # Source dir @@ -60,5 +77,5 @@ mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET} # To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add echo "copying scripts from ${NVIDIA_GPU_SUPPORT_DIR_SOURCE} to ${NVIDIA_GPU_SUPPORT_DIR_TARGET}" for file in install_cuda_host_injections.sh link_nvidia_host_libraries.sh; do - cp -v -u ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file} + compare_and_copy ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file} done diff --git a/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh new file mode 100755 index 0000000000..a9310d817a --- /dev/null +++ b/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash + +# This script can be used to install CUDA under the `.../host_injections` directory. +# This provides the parts of the CUDA installation that cannot be redistributed as +# part of EESSI due to license limitations. While GPU-based software from EESSI will +# _run_ without these, installation of additional CUDA software requires the CUDA +# installation(s) under `host_injections` to be present. +# +# The `host_injections` directory is a variant symlink that by default points to +# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see +# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the +# installation to be successful, this directory needs to be writeable by the user +# executing this script. + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath $BASH_SOURCE)) +source "$TOPDIR"/../../utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " --accept-cuda-eula You _must_ accept the CUDA EULA to install" + echo " CUDA, see the EULA at" + echo " https://docs.nvidia.com/cuda/eula/index.html" + echo " -c, --cuda-version CUDA_VERSION Specify a version o CUDA to install (must" + echo " have a corresponding easyconfig in the" + echo " EasyBuild release)" + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the CUDA install" + echo " (must have >10GB available)" +} + +# Initialize variables +install_cuda_version="" +eula_accepted=0 + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + -c|--cuda-version) + if [ -n "$2" ]; then + install_cuda_version="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + --accept-cuda-eula) + eula_accepted=1 + shift 1 + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + CUDA_TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +# Make sure EESSI is initialised +check_eessi_initialised + +# Make sure the CUDA version supplied is a semantic version +is_semantic_version() { + local version=$1 + local regex='^[0-9]+\.[0-9]+\.[0-9]+$' + + if [[ $version =~ $regex ]]; then + return 0 # Return success (0) if it's a semantic version + else + return 1 # Return failure (1) if it's not a semantic version + fi +} +if ! is_semantic_version "$install_cuda_version"; then + show_help + error="\nYou must provide a semantic version for CUDA (e.g., 12.1.1) via the appropriate\n" + error="${error}command line option. This script is intended for use with EESSI so the 'correct'\n" + error="${error}version to provide is probably one of those available under\n" + error="${error}$EESSI_SOFTWARE_PATH/software/CUDA\n" + fatal_error "${error}" +fi + +# Make sure they have accepted the CUDA EULA +if [ "$eula_accepted" -ne 1 ]; then + show_help + error="\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n" + fatal_error "${error}" +fi + +# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` +# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup) +cuda_install_parent=${EESSI_SOFTWARE_PATH/versions/host_injections} + +# Only install CUDA if specified version is not found. +# (existence of easybuild subdir implies a successful install) +if [ -d "${cuda_install_parent}"/software/CUDA/"${install_cuda_version}"/easybuild ]; then + echo_green "CUDA software found! No need to install CUDA again." +else + # We need to be able write to the installation space so let's make sure we can + if ! create_directory_structure "${cuda_install_parent}"/software/CUDA ; then + fatal_error "No write permissions to directory ${cuda_install_parent}/software/CUDA" + fi + + # we need a directory we can use for temporary storage + if [[ -z "${CUDA_TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) + else + tmpdir="${CUDA_TEMP_DIR}"/temp + if ! mkdir "$tmpdir" ; then + fatal_error "Could not create directory ${tmpdir}" + fi + fi + + required_space_in_tmpdir=50000 + # Let's see if we have sources and build locations defined if not, we use the temporary space + if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then + export EASYBUILD_BUILDPATH=${tmpdir}/build + required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000)) + fi + if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then + export EASYBUILD_SOURCEPATH=${tmpdir}/sources + required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000)) + fi + + # The install is pretty fat, you need lots of space for download/unpack/install (~3*5GB), + # need to do a space check before we proceed + avail_space=$(df --output=avail "${cuda_install_parent}"/ | tail -n 1 | awk '{print $1}') + if (( avail_space < 5000000 )); then + fatal_error "Need at least 5GB disk space to install CUDA under ${cuda_install_parent}, exiting now..." + fi + avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') + if (( avail_space < required_space_in_tmpdir )); then + error="Need at least ${required_space_in_tmpdir} disk space under ${tmpdir}.\n" + error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check." + error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH " + error="${error}to reduce this requirement. Exiting now..." + fatal_error "${error}" + fi + + if ! command -v "eb" &>/dev/null; then + echo_yellow "Attempting to load an EasyBuild module to do actual install" + module load EasyBuild + # There are some scenarios where this may fail + if [ $? -ne 0 ]; then + error="'eb' command not found in your environment and\n" + error="${error} module load EasyBuild\n" + error="${error}failed for some reason.\n" + error="${error}Please re-run this script with the 'eb' command available." + fatal_error "${error}" + fi + fi + + cuda_easyconfig="CUDA-${install_cuda_version}.eb" + + # Check the easyconfig file is available in the release + # (eb search always returns 0, so we need a grep to ensure a usable exit code) + eb --search ^${cuda_easyconfig}|grep CUDA > /dev/null 2>&1 + # Check the exit code + if [ $? -ne 0 ]; then + eb_version=$(eb --version) + available_cuda_easyconfigs=$(eb --search ^CUDA-*.eb|grep CUDA) + + error="The easyconfig ${cuda_easyconfig} was not found in EasyBuild version:\n" + error="${error} ${eb_version}\n" + error="${error}You either need to give a different version of CUDA to install _or_ \n" + error="${error}use a different version of EasyBuild for the installation.\n" + error="${error}\nThe versions of available with the current eb command are:\n" + error="${error}${available_cuda_easyconfigs}" + fatal_error "${error}" + fi + + # We need the --rebuild option, as the CUDA module may or may not be on the + # `MODULEPATH` yet. Even if it is, we still want to redo this installation + # since it will provide the symlinked targets for the parts of the CUDA + # installation in the `.../versions/...` prefix + # We install the module in our `tmpdir` since we do not need the modulefile, + # we only care about providing the targets for the symlinks. + extra_args="--rebuild --installpath-modules=${tmpdir}" + + # We don't want hooks used in this install, we need a vanilla CUDA installation + touch "$tmpdir"/none.py + # shellcheck disable=SC2086 # Intended splitting of extra_args + eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}" + ret=$? + if [ $ret -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + cp -a ${eb_last_log} . + fatal_error "CUDA installation failed, please check EasyBuild logs $(basename ${eb_last_log})..." + else + echo_green "CUDA installation at ${cuda_install_parent}/software/CUDA/${install_cuda_version} succeeded!" + fi + # clean up tmpdir + rm -rf "${tmpdir}" +fi diff --git a/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh new file mode 100755 index 0000000000..e6ff110797 --- /dev/null +++ b/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# This script links host libraries related to GPU drivers to a location where +# they can be found by the EESSI linker + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath $BASH_SOURCE)) +source "$TOPDIR"/../../utils.sh + +# We rely on ldconfig to give us the location of the libraries on the host +command_name="ldconfig" +# We cannot use a version of ldconfig that's being shipped under CVMFS +exclude_prefix="/cvmfs" + +found_paths=() +# Always attempt to use /sbin/ldconfig +if [ -x "/sbin/$command_name" ]; then + found_paths+=("/sbin/$command_name") +fi +IFS=':' read -ra path_dirs <<< "$PATH" +for dir in "${path_dirs[@]}"; do + if [ "$dir" = "/sbin" ]; then + continue # we've already checked for $command_name in /sbin, don't need to do it twice + fi + if [[ ! "$dir" =~ ^$exclude_prefix ]]; then + if [ -x "$dir/$command_name" ]; then + found_paths+=("$dir/$command_name") + fi + fi +done + +if [ ${#found_paths[@]} -gt 0 ]; then + echo "Found $command_name in the following locations:" + printf -- "- %s\n" "${found_paths[@]}" + echo "Using first version" + host_ldconfig=${found_paths[0]} +else + error="$command_name not found in PATH or only found in paths starting with $exclude_prefix." + fatal_error "$error" +fi + +# Make sure EESSI is initialised (doesn't matter what version) +check_eessi_initialised + +# Find the CUDA version of the host CUDA drivers +# (making sure that this can still work inside prefix environment inside a container) +export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH +nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader" +if $nvidia_smi_command > /dev/null; then + host_driver_version=$($nvidia_smi_command | tail -n1) + # If the first worked, this should work too + host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}') +else + error="Failed to successfully execute\n $nvidia_smi_command\n" + fatal_error "$error" +fi + +# Let's make sure the driver libraries are not already in place +link_drivers=1 + +host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}" +host_injection_driver_dir="${host_injections_nvidia_dir}/host" +host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt" +if [ -e "$host_injection_driver_version_file" ]; then + if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then + echo_green "The host CUDA driver libraries have already been linked!" + link_drivers=0 + else + # There's something there but it is out of date + echo_yellow "Cleaning out outdated symlinks" + rm $host_injection_driver_dir/* + if [ $? -ne 0 ]; then + error="Unable to remove files under '$host_injection_driver_dir'." + fatal_error "$error" + fi + fi +fi + +drivers_linked=0 +if [ "$link_drivers" -eq 1 ]; then + if ! create_directory_structure "${host_injection_driver_dir}" ; then + fatal_error "No write permissions to directory ${host_injection_driver_dir}" + fi + cd ${host_injection_driver_dir} + # Need a small temporary space to hold a couple of files + temp_dir=$(mktemp -d) + + # Gather libraries on the host (_must_ be host ldconfig) + $host_ldconfig -p | awk '{print $NF}' > "$temp_dir"/libs.txt + # Allow for the fact that we may be in a container so the CUDA libs might be in there + ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null + + # Leverage singularity to find the full list of libraries we should be linking to + echo_yellow "Downloading latest version of nvliblist.conf from Apptainer" + curl -o "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf + + # Make symlinks to all the interesting libraries + grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {} + + # Inject driver and CUDA versions into dir + echo $host_driver_version > driver_version.txt + echo $host_cuda_version > cuda_version.txt + drivers_linked=1 + + # Remove the temporary directory when done + rm -r "$temp_dir" +fi + +# Make latest symlink for NVIDIA drivers +cd $host_injections_nvidia_dir +symlink="latest" +if [ -L "$symlink" ]; then + # Unless the drivers have been installed, leave the symlink alone + if [ "$drivers_linked" -eq 1 ]; then + ln -sf host latest + fi +else + # No link exists yet + ln -s host latest +fi + +# Make sure the libraries can be found by the EESSI linker +host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections} +if [ -L "$host_injection_linker_dir/lib" ]; then + target_path=$(readlink -f "$host_injection_linker_dir/lib") + if [ "$target_path" != "$$host_injections_nvidia_dir/latest" ]; then + cd $host_injection_linker_dir + ln -sf $host_injections_nvidia_dir/latest lib + fi +else + create_directory_structure $host_injection_linker_dir + cd $host_injection_linker_dir + ln -s $host_injections_nvidia_dir/latest lib +fi + +echo_green "Host NVIDIA gpu drivers linked successfully for EESSI" diff --git a/temp/scripts/utils.sh b/temp/scripts/utils.sh new file mode 100644 index 0000000000..b2be3f6221 --- /dev/null +++ b/temp/scripts/utils.sh @@ -0,0 +1,144 @@ +function echo_green() { + echo -e "\e[32m$1\e[0m" +} + +function echo_red() { + echo -e "\e[31m$1\e[0m" +} + +function echo_yellow() { + echo -e "\e[33m$1\e[0m" +} + +ANY_ERROR_EXITCODE=1 +function fatal_error() { + echo_red "ERROR: $1" >&2 + if [[ $# -gt 1 ]]; then + exit "$2" + else + exit "${ANY_ERROR_EXITCODE}" + fi +} + +function check_exit_code { + ec=$1 + ok_msg=$2 + fail_msg=$3 + + if [[ $ec -eq 0 ]]; then + echo_green "${ok_msg}" + else + fatal_error "${fail_msg}" + fi +} + +function check_eessi_initialised() { + if [[ -z "${EESSI_SOFTWARE_PATH}" ]]; then + fatal_error "EESSI has not been initialised!" + else + return 0 + fi +} + +function check_in_prefix_shell() { + # Make sure EPREFIX is defined + if [[ -z "${EPREFIX}" ]]; then + fatal_error "This script cannot be used without having first defined EPREFIX" + fi + if [[ ! ${SHELL} = ${EPREFIX}/bin/bash ]]; then + fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" + fi +} + +function create_directory_structure() { + # Ensure we are given a single path argument + if [ $# -ne 1 ]; then + echo_red "Function requires a single (relative or absolute) path argument" >&2 + return $ANY_ERROR_EXITCODE + fi + dir_structure="$1" + + # Attempt to create the directory structure + error_message=$(mkdir -p "$dir_structure" 2>&1) + return_code=$? + # If it fails be explicit about the error + if [ ${return_code} -ne 0 ]; then + real_dir=$(realpath -m "$dir_structure") + echo_red "Creating ${dir_structure} (real path ${real_dir}) failed with:\n ${error_message}" >&2 + else + # If we're creating it, our use case is that we want to be able to write there + # (this is a check in case the directory already existed) + if [ ! -w "${dir_structure}" ]; then + real_dir=$(realpath -m "$dir_structure") + echo_red "You do not have (required) write permissions to ${dir_structure} (real path ${real_dir})!" + return_code=$ANY_ERROR_EXITCODE + fi + fi + + return $return_code +} + +function get_path_for_tool { + tool_name=$1 + tool_envvar_name=$2 + + which_out=$(which "${tool_name}" 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 + echo "${which_out}" + return 0 + fi + if [[ -z "${tool_envvar_name}" ]]; then + msg="no env var holding the full path to tool '${tool_name}' provided" + echo "${msg}" >&2 + return 1 + else + tool_envvar_value=${!tool_envvar_name} + if [[ -x "${tool_envvar_value}" ]]; then + msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}" + echo "${msg}" >&2 + echo "${tool_envvar_value}" + return 0 + else + msg="ERROR: tool '${tool_name}' not in PATH\n" + msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH" + echo "${msg}" >&2 + echo "" + return 2 + fi + fi +} + +function get_host_from_url { + url=$1 + re="(http|https)://([^/:]+)" + if [[ $url =~ $re ]]; then + echo "${BASH_REMATCH[2]}" + return 0 + else + echo "" + return 1 + fi +} + +function get_port_from_url { + url=$1 + re="(http|https)://[^:]+:([0-9]+)" + if [[ $url =~ $re ]]; then + echo "${BASH_REMATCH[2]}" + return 0 + else + echo "" + return 1 + fi +} + +function get_ipv4_address { + hname=$1 + hipv4=$(grep "${hname}" /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) + # TODO try other methods if the one above does not work --> tool that verifies + # what method can be used? + echo "${hipv4}" + return 0 +} From a333a741bb75a68f6d29cb718472af70e3b5c912 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 21 Dec 2023 12:19:19 +0100 Subject: [PATCH 0350/1795] Remove temporary test directory --- .../nvidia/install_cuda_host_injections.sh | 211 ------------------ .../nvidia/link_nvidia_host_libraries.sh | 136 ----------- temp/scripts/utils.sh | 144 ------------ 3 files changed, 491 deletions(-) delete mode 100755 temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh delete mode 100755 temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh delete mode 100644 temp/scripts/utils.sh diff --git a/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh deleted file mode 100755 index a9310d817a..0000000000 --- a/temp/scripts/gpu_support/nvidia/install_cuda_host_injections.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env bash - -# This script can be used to install CUDA under the `.../host_injections` directory. -# This provides the parts of the CUDA installation that cannot be redistributed as -# part of EESSI due to license limitations. While GPU-based software from EESSI will -# _run_ without these, installation of additional CUDA software requires the CUDA -# installation(s) under `host_injections` to be present. -# -# The `host_injections` directory is a variant symlink that by default points to -# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see -# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the -# installation to be successful, this directory needs to be writeable by the user -# executing this script. - -# Initialise our bash functions -TOPDIR=$(dirname $(realpath $BASH_SOURCE)) -source "$TOPDIR"/../../utils.sh - -# Function to display help message -show_help() { - echo "Usage: $0 [OPTIONS]" - echo "Options:" - echo " --help Display this help message" - echo " --accept-cuda-eula You _must_ accept the CUDA EULA to install" - echo " CUDA, see the EULA at" - echo " https://docs.nvidia.com/cuda/eula/index.html" - echo " -c, --cuda-version CUDA_VERSION Specify a version o CUDA to install (must" - echo " have a corresponding easyconfig in the" - echo " EasyBuild release)" - echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" - echo " storage during the CUDA install" - echo " (must have >10GB available)" -} - -# Initialize variables -install_cuda_version="" -eula_accepted=0 - -# Parse command-line options -while [[ $# -gt 0 ]]; do - case "$1" in - --help) - show_help - exit 0 - ;; - -c|--cuda-version) - if [ -n "$2" ]; then - install_cuda_version="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - --accept-cuda-eula) - eula_accepted=1 - shift 1 - ;; - -t|--temp-dir) - if [ -n "$2" ]; then - CUDA_TEMP_DIR="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - *) - show_help - fatal_error "Error: Unknown option: $1" - ;; - esac -done - -# Make sure EESSI is initialised -check_eessi_initialised - -# Make sure the CUDA version supplied is a semantic version -is_semantic_version() { - local version=$1 - local regex='^[0-9]+\.[0-9]+\.[0-9]+$' - - if [[ $version =~ $regex ]]; then - return 0 # Return success (0) if it's a semantic version - else - return 1 # Return failure (1) if it's not a semantic version - fi -} -if ! is_semantic_version "$install_cuda_version"; then - show_help - error="\nYou must provide a semantic version for CUDA (e.g., 12.1.1) via the appropriate\n" - error="${error}command line option. This script is intended for use with EESSI so the 'correct'\n" - error="${error}version to provide is probably one of those available under\n" - error="${error}$EESSI_SOFTWARE_PATH/software/CUDA\n" - fatal_error "${error}" -fi - -# Make sure they have accepted the CUDA EULA -if [ "$eula_accepted" -ne 1 ]; then - show_help - error="\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n" - fatal_error "${error}" -fi - -# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` -# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup) -cuda_install_parent=${EESSI_SOFTWARE_PATH/versions/host_injections} - -# Only install CUDA if specified version is not found. -# (existence of easybuild subdir implies a successful install) -if [ -d "${cuda_install_parent}"/software/CUDA/"${install_cuda_version}"/easybuild ]; then - echo_green "CUDA software found! No need to install CUDA again." -else - # We need to be able write to the installation space so let's make sure we can - if ! create_directory_structure "${cuda_install_parent}"/software/CUDA ; then - fatal_error "No write permissions to directory ${cuda_install_parent}/software/CUDA" - fi - - # we need a directory we can use for temporary storage - if [[ -z "${CUDA_TEMP_DIR}" ]]; then - tmpdir=$(mktemp -d) - else - tmpdir="${CUDA_TEMP_DIR}"/temp - if ! mkdir "$tmpdir" ; then - fatal_error "Could not create directory ${tmpdir}" - fi - fi - - required_space_in_tmpdir=50000 - # Let's see if we have sources and build locations defined if not, we use the temporary space - if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then - export EASYBUILD_BUILDPATH=${tmpdir}/build - required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000)) - fi - if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then - export EASYBUILD_SOURCEPATH=${tmpdir}/sources - required_space_in_tmpdir=$((required_space_in_tmpdir + 5000000)) - fi - - # The install is pretty fat, you need lots of space for download/unpack/install (~3*5GB), - # need to do a space check before we proceed - avail_space=$(df --output=avail "${cuda_install_parent}"/ | tail -n 1 | awk '{print $1}') - if (( avail_space < 5000000 )); then - fatal_error "Need at least 5GB disk space to install CUDA under ${cuda_install_parent}, exiting now..." - fi - avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') - if (( avail_space < required_space_in_tmpdir )); then - error="Need at least ${required_space_in_tmpdir} disk space under ${tmpdir}.\n" - error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check." - error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH " - error="${error}to reduce this requirement. Exiting now..." - fatal_error "${error}" - fi - - if ! command -v "eb" &>/dev/null; then - echo_yellow "Attempting to load an EasyBuild module to do actual install" - module load EasyBuild - # There are some scenarios where this may fail - if [ $? -ne 0 ]; then - error="'eb' command not found in your environment and\n" - error="${error} module load EasyBuild\n" - error="${error}failed for some reason.\n" - error="${error}Please re-run this script with the 'eb' command available." - fatal_error "${error}" - fi - fi - - cuda_easyconfig="CUDA-${install_cuda_version}.eb" - - # Check the easyconfig file is available in the release - # (eb search always returns 0, so we need a grep to ensure a usable exit code) - eb --search ^${cuda_easyconfig}|grep CUDA > /dev/null 2>&1 - # Check the exit code - if [ $? -ne 0 ]; then - eb_version=$(eb --version) - available_cuda_easyconfigs=$(eb --search ^CUDA-*.eb|grep CUDA) - - error="The easyconfig ${cuda_easyconfig} was not found in EasyBuild version:\n" - error="${error} ${eb_version}\n" - error="${error}You either need to give a different version of CUDA to install _or_ \n" - error="${error}use a different version of EasyBuild for the installation.\n" - error="${error}\nThe versions of available with the current eb command are:\n" - error="${error}${available_cuda_easyconfigs}" - fatal_error "${error}" - fi - - # We need the --rebuild option, as the CUDA module may or may not be on the - # `MODULEPATH` yet. Even if it is, we still want to redo this installation - # since it will provide the symlinked targets for the parts of the CUDA - # installation in the `.../versions/...` prefix - # We install the module in our `tmpdir` since we do not need the modulefile, - # we only care about providing the targets for the symlinks. - extra_args="--rebuild --installpath-modules=${tmpdir}" - - # We don't want hooks used in this install, we need a vanilla CUDA installation - touch "$tmpdir"/none.py - # shellcheck disable=SC2086 # Intended splitting of extra_args - eb --prefix="$tmpdir" ${extra_args} --accept-eula-for=CUDA --hooks="$tmpdir"/none.py --installpath="${cuda_install_parent}"/ "${cuda_easyconfig}" - ret=$? - if [ $ret -ne 0 ]; then - eb_last_log=$(unset EB_VERBOSE; eb --last-log) - cp -a ${eb_last_log} . - fatal_error "CUDA installation failed, please check EasyBuild logs $(basename ${eb_last_log})..." - else - echo_green "CUDA installation at ${cuda_install_parent}/software/CUDA/${install_cuda_version} succeeded!" - fi - # clean up tmpdir - rm -rf "${tmpdir}" -fi diff --git a/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh deleted file mode 100755 index e6ff110797..0000000000 --- a/temp/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash - -# This script links host libraries related to GPU drivers to a location where -# they can be found by the EESSI linker - -# Initialise our bash functions -TOPDIR=$(dirname $(realpath $BASH_SOURCE)) -source "$TOPDIR"/../../utils.sh - -# We rely on ldconfig to give us the location of the libraries on the host -command_name="ldconfig" -# We cannot use a version of ldconfig that's being shipped under CVMFS -exclude_prefix="/cvmfs" - -found_paths=() -# Always attempt to use /sbin/ldconfig -if [ -x "/sbin/$command_name" ]; then - found_paths+=("/sbin/$command_name") -fi -IFS=':' read -ra path_dirs <<< "$PATH" -for dir in "${path_dirs[@]}"; do - if [ "$dir" = "/sbin" ]; then - continue # we've already checked for $command_name in /sbin, don't need to do it twice - fi - if [[ ! "$dir" =~ ^$exclude_prefix ]]; then - if [ -x "$dir/$command_name" ]; then - found_paths+=("$dir/$command_name") - fi - fi -done - -if [ ${#found_paths[@]} -gt 0 ]; then - echo "Found $command_name in the following locations:" - printf -- "- %s\n" "${found_paths[@]}" - echo "Using first version" - host_ldconfig=${found_paths[0]} -else - error="$command_name not found in PATH or only found in paths starting with $exclude_prefix." - fatal_error "$error" -fi - -# Make sure EESSI is initialised (doesn't matter what version) -check_eessi_initialised - -# Find the CUDA version of the host CUDA drivers -# (making sure that this can still work inside prefix environment inside a container) -export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH -nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader" -if $nvidia_smi_command > /dev/null; then - host_driver_version=$($nvidia_smi_command | tail -n1) - # If the first worked, this should work too - host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}') -else - error="Failed to successfully execute\n $nvidia_smi_command\n" - fatal_error "$error" -fi - -# Let's make sure the driver libraries are not already in place -link_drivers=1 - -host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}" -host_injection_driver_dir="${host_injections_nvidia_dir}/host" -host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt" -if [ -e "$host_injection_driver_version_file" ]; then - if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then - echo_green "The host CUDA driver libraries have already been linked!" - link_drivers=0 - else - # There's something there but it is out of date - echo_yellow "Cleaning out outdated symlinks" - rm $host_injection_driver_dir/* - if [ $? -ne 0 ]; then - error="Unable to remove files under '$host_injection_driver_dir'." - fatal_error "$error" - fi - fi -fi - -drivers_linked=0 -if [ "$link_drivers" -eq 1 ]; then - if ! create_directory_structure "${host_injection_driver_dir}" ; then - fatal_error "No write permissions to directory ${host_injection_driver_dir}" - fi - cd ${host_injection_driver_dir} - # Need a small temporary space to hold a couple of files - temp_dir=$(mktemp -d) - - # Gather libraries on the host (_must_ be host ldconfig) - $host_ldconfig -p | awk '{print $NF}' > "$temp_dir"/libs.txt - # Allow for the fact that we may be in a container so the CUDA libs might be in there - ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null - - # Leverage singularity to find the full list of libraries we should be linking to - echo_yellow "Downloading latest version of nvliblist.conf from Apptainer" - curl -o "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf - - # Make symlinks to all the interesting libraries - grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {} - - # Inject driver and CUDA versions into dir - echo $host_driver_version > driver_version.txt - echo $host_cuda_version > cuda_version.txt - drivers_linked=1 - - # Remove the temporary directory when done - rm -r "$temp_dir" -fi - -# Make latest symlink for NVIDIA drivers -cd $host_injections_nvidia_dir -symlink="latest" -if [ -L "$symlink" ]; then - # Unless the drivers have been installed, leave the symlink alone - if [ "$drivers_linked" -eq 1 ]; then - ln -sf host latest - fi -else - # No link exists yet - ln -s host latest -fi - -# Make sure the libraries can be found by the EESSI linker -host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections} -if [ -L "$host_injection_linker_dir/lib" ]; then - target_path=$(readlink -f "$host_injection_linker_dir/lib") - if [ "$target_path" != "$$host_injections_nvidia_dir/latest" ]; then - cd $host_injection_linker_dir - ln -sf $host_injections_nvidia_dir/latest lib - fi -else - create_directory_structure $host_injection_linker_dir - cd $host_injection_linker_dir - ln -s $host_injections_nvidia_dir/latest lib -fi - -echo_green "Host NVIDIA gpu drivers linked successfully for EESSI" diff --git a/temp/scripts/utils.sh b/temp/scripts/utils.sh deleted file mode 100644 index b2be3f6221..0000000000 --- a/temp/scripts/utils.sh +++ /dev/null @@ -1,144 +0,0 @@ -function echo_green() { - echo -e "\e[32m$1\e[0m" -} - -function echo_red() { - echo -e "\e[31m$1\e[0m" -} - -function echo_yellow() { - echo -e "\e[33m$1\e[0m" -} - -ANY_ERROR_EXITCODE=1 -function fatal_error() { - echo_red "ERROR: $1" >&2 - if [[ $# -gt 1 ]]; then - exit "$2" - else - exit "${ANY_ERROR_EXITCODE}" - fi -} - -function check_exit_code { - ec=$1 - ok_msg=$2 - fail_msg=$3 - - if [[ $ec -eq 0 ]]; then - echo_green "${ok_msg}" - else - fatal_error "${fail_msg}" - fi -} - -function check_eessi_initialised() { - if [[ -z "${EESSI_SOFTWARE_PATH}" ]]; then - fatal_error "EESSI has not been initialised!" - else - return 0 - fi -} - -function check_in_prefix_shell() { - # Make sure EPREFIX is defined - if [[ -z "${EPREFIX}" ]]; then - fatal_error "This script cannot be used without having first defined EPREFIX" - fi - if [[ ! ${SHELL} = ${EPREFIX}/bin/bash ]]; then - fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" - fi -} - -function create_directory_structure() { - # Ensure we are given a single path argument - if [ $# -ne 1 ]; then - echo_red "Function requires a single (relative or absolute) path argument" >&2 - return $ANY_ERROR_EXITCODE - fi - dir_structure="$1" - - # Attempt to create the directory structure - error_message=$(mkdir -p "$dir_structure" 2>&1) - return_code=$? - # If it fails be explicit about the error - if [ ${return_code} -ne 0 ]; then - real_dir=$(realpath -m "$dir_structure") - echo_red "Creating ${dir_structure} (real path ${real_dir}) failed with:\n ${error_message}" >&2 - else - # If we're creating it, our use case is that we want to be able to write there - # (this is a check in case the directory already existed) - if [ ! -w "${dir_structure}" ]; then - real_dir=$(realpath -m "$dir_structure") - echo_red "You do not have (required) write permissions to ${dir_structure} (real path ${real_dir})!" - return_code=$ANY_ERROR_EXITCODE - fi - fi - - return $return_code -} - -function get_path_for_tool { - tool_name=$1 - tool_envvar_name=$2 - - which_out=$(which "${tool_name}" 2>&1) - exit_code=$? - if [[ ${exit_code} -eq 0 ]]; then - echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 - echo "${which_out}" - return 0 - fi - if [[ -z "${tool_envvar_name}" ]]; then - msg="no env var holding the full path to tool '${tool_name}' provided" - echo "${msg}" >&2 - return 1 - else - tool_envvar_value=${!tool_envvar_name} - if [[ -x "${tool_envvar_value}" ]]; then - msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}" - echo "${msg}" >&2 - echo "${tool_envvar_value}" - return 0 - else - msg="ERROR: tool '${tool_name}' not in PATH\n" - msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH" - echo "${msg}" >&2 - echo "" - return 2 - fi - fi -} - -function get_host_from_url { - url=$1 - re="(http|https)://([^/:]+)" - if [[ $url =~ $re ]]; then - echo "${BASH_REMATCH[2]}" - return 0 - else - echo "" - return 1 - fi -} - -function get_port_from_url { - url=$1 - re="(http|https)://[^:]+:([0-9]+)" - if [[ $url =~ $re ]]; then - echo "${BASH_REMATCH[2]}" - return 0 - else - echo "" - return 1 - fi -} - -function get_ipv4_address { - hname=$1 - hipv4=$(grep "${hname}" /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) - # TODO try other methods if the one above does not work --> tool that verifies - # what method can be used? - echo "${hipv4}" - return 0 -} From 43c73c07643cf756ff8411b9068bf4cb2a64eec4 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 21 Dec 2023 13:06:53 +0100 Subject: [PATCH 0351/1795] Get rid of copy/paste unfriendly '.' --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index 588248e8d2..6e6cd825ac 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -19,7 +19,7 @@ compare_and_copy() { if [ ! -f "$destination_file" ] || ! diff -q "$source_file" "$destination_file" ; then cp "$source_file" "$destination_file" - echo "File $1 copied to $2." + echo "File $1 copied to $2" else echo "Files $1 and $2 are identical. No copy needed." fi From 3ec3df8f4a91c6684ed83c6b021c231ca74ba7c0 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 21 Dec 2023 14:11:09 +0100 Subject: [PATCH 0352/1795] Update create_tarball.sh --- create_tarball.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 4d3ad37311..a172428af1 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -43,13 +43,13 @@ module_files_list=${tmpdir}/module_files.list.txt # include scripts that were copied by install_scripts.sh if [ -d ${eessi_version}/scripts ]; then # include scripts we wish to ship along with EESSI, - find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' > ${files_list} fi if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) - find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list} + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list} fi if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files From 42e3404a0015d7b4049c5cd3e5a01ee764261d44 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Dec 2023 14:13:23 +0100 Subject: [PATCH 0353/1795] always append to list of files to include in tarball, to avoid overwriting it --- create_tarball.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index a172428af1..faaa9fda6f 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -40,10 +40,10 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}.. files_list=${tmpdir}/files.list.txt module_files_list=${tmpdir}/module_files.list.txt -# include scripts that were copied by install_scripts.sh -if [ -d ${eessi_version}/scripts ]; then - # include scripts we wish to ship along with EESSI, - find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' > ${files_list} +# include Lmod cache and configuration file (lmodrc.lua), +if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then + # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) + find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list} fi if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then @@ -51,6 +51,12 @@ if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list} fi + +# include scripts that were copied by install_scripts.sh, which we want to ship in EESSI repository +if [ -d ${eessi_version}/scripts ]; then + find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list} +fi + if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} @@ -61,6 +67,7 @@ if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi + if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' From 60741ae979757b72c87551a9d0e15afefda4a4a6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Dec 2023 20:48:36 +0100 Subject: [PATCH 0354/1795] make link_nvidia_host_libraries.sh script a bit more robust, in case target of host_injections directory is a non-existing directory --- .../nvidia/link_nvidia_host_libraries.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index e6ff110797..e8d7f0d0a7 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -48,8 +48,10 @@ export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader" if $nvidia_smi_command > /dev/null; then host_driver_version=$($nvidia_smi_command | tail -n1) + echo_green "Found NVIDIA GPU driver version ${host_driver_version}" # If the first worked, this should work too host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}') + echo_green "Found host CUDA version ${host_cuda_version}" else error="Failed to successfully execute\n $nvidia_smi_command\n" fatal_error "$error" @@ -58,12 +60,18 @@ fi # Let's make sure the driver libraries are not already in place link_drivers=1 +# first make sure that target of host_injections variant symlink is an existing directory +host_injections_target=$(realpath -m ${EESSI_CVMFS_REPO}/host_injections) +if [ ! -d ${host_injections_target} ]; then + create_directory_structure ${host_injections_target} +fi + host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}" host_injection_driver_dir="${host_injections_nvidia_dir}/host" host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt" if [ -e "$host_injection_driver_version_file" ]; then if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then - echo_green "The host CUDA driver libraries have already been linked!" + echo_green "The host GPU driver libraries (v${host_driver_version}) have already been linked! (based on ${host_injection_driver_version_file})" link_drivers=0 else # There's something there but it is out of date @@ -91,8 +99,8 @@ if [ "$link_drivers" -eq 1 ]; then ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null # Leverage singularity to find the full list of libraries we should be linking to - echo_yellow "Downloading latest version of nvliblist.conf from Apptainer" - curl -o "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf + echo_yellow "Downloading latest version of nvliblist.conf from Apptainer to ${temp_dir}/nvliblist.conf" + curl --silent --output "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf # Make symlinks to all the interesting libraries grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {} @@ -133,4 +141,4 @@ else ln -s $host_injections_nvidia_dir/latest lib fi -echo_green "Host NVIDIA gpu drivers linked successfully for EESSI" +echo_green "Host NVIDIA GPU drivers linked successfully for EESSI" From 5c248d1dcb844ec11a76f9f368ba523b7edf3e87 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 22 Dec 2023 11:39:31 +0100 Subject: [PATCH 0355/1795] Ensure that bot reports success if no EasyStacks were changed in a PR. Relevant for PRs that e.g. only update things in .../scripts --- EESSI-install-software.sh | 75 +++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index b61ca7a579..edbcf7040b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -203,42 +203,47 @@ ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12 # ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh # use PR patch file to determine in which easystack files stuff was added -for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do - - echo -e "Processing easystack file ${easystack_file}...\n\n" - - # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') - - # load EasyBuild module (will be installed if it's not available yet) - source ${TOPDIR}/load_easybuild_module.sh ${eb_version} - - ${EB} --show-config - - echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." - - if [ -f ${easystack_file} ]; then - echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - - ${EB} --easystack ${TOPDIR}/${easystack_file} --robot - ec=$? - - # copy EasyBuild log file if EasyBuild exited with an error - if [ ${ec} -ne 0 ]; then - eb_last_log=$(unset EB_VERBOSE; eb --last-log) - # copy to current working directory - cp -a ${eb_last_log} . - echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}" - # copy to build logs dir (with context added) - copy_build_log "${eb_last_log}" "${build_logs_dir}" +changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') +if [ -z ${changed_easystacks} ]; then + echo "No missing installations" # Ensure the bot report success, as there was nothing to be build here +else + for easystack_file in ${changed_easystacks}; do + + echo -e "Processing easystack file ${easystack_file}...\n\n" + + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + + ${EB} --show-config + + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." + + if [ -f ${easystack_file} ]; then + echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." + + ${EB} --easystack ${TOPDIR}/${easystack_file} --robot + ec=$? + + # copy EasyBuild log file if EasyBuild exited with an error + if [ ${ec} -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + # copy to current working directory + cp -a ${eb_last_log} . + echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}" + # copy to build logs dir (with context added) + copy_build_log "${eb_last_log}" "${build_logs_dir}" + fi + + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} + else + fatal_error "Easystack file ${easystack_file} not found!" fi - - $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} - else - fatal_error "Easystack file ${easystack_file} not found!" - fi - -done + + done +fi ### add packages here From ac53cf0ea0160d47e302393ef60d2829586708af Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 22 Dec 2023 14:54:46 +0100 Subject: [PATCH 0356/1795] Make the pedantic deploy step of the bot happy... --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index edbcf7040b..69de9d1997 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -205,7 +205,7 @@ ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12 # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z ${changed_easystacks} ]; then - echo "No missing installations" # Ensure the bot report success, as there was nothing to be build here + echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else for easystack_file in ${changed_easystacks}; do From 203cbd104d8189fbc007c88fa505efba3d02826e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 23 Dec 2023 14:49:44 +0100 Subject: [PATCH 0357/1795] add ALL-0.9.2-foss-2023a.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 567db44e42..7d879044d8 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -35,3 +35,6 @@ easyconfigs: - Boost-1.82.0-GCC-12.3.0.eb - netCDF-4.9.2-gompi-2023a.eb - FFmpeg-6.0-GCCcore-12.3.0.eb + - ALL-0.9.2-foss-2023a.eb: + options: + from-pr: 19455 From b1daa1253af0a77804f5bee803886322209db641 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Jan 2024 09:42:46 +0100 Subject: [PATCH 0358/1795] {2023.06}[system] EasyBuild v4.9.0 --- .../2023.06/eessi-2023.06-eb-4.9.0-001-system.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml new file mode 100644 index 0000000000..a7bce002c7 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - EasyBuild-4.9.0.eb: + options: + from-pr: 19464 From d299c7459681facd6316d7a0c7889e217c7b72f6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Jan 2024 14:26:45 +0100 Subject: [PATCH 0359/1795] don't include files in .lmod directory twice in tarball --- create_tarball.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index faaa9fda6f..a619df9439 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -40,12 +40,6 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}.. files_list=${tmpdir}/files.list.txt module_files_list=${tmpdir}/module_files.list.txt -# include Lmod cache and configuration file (lmodrc.lua), -if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then - # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) - find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list} -fi - if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), # skip whiteout files and backup copies of Lmod cache (spiderT.old.*) From 2d0d7e852fe958ff55e1e5a2c1f6e4fb4c3a5f70 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Jan 2024 20:00:13 +0100 Subject: [PATCH 0360/1795] {2023.06} foss/2023b --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml new file mode 100644 index 0000000000..88ab50b8e7 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -0,0 +1,3 @@ +easyconfigs: + - GCC-13.2.0.eb + - foss-2023b.eb From 5c0061510b7ffdcfe733307555db8f9e1bc5ed09 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jan 2024 11:47:09 +0100 Subject: [PATCH 0361/1795] update pre-configure hook for OpenBLAS to use -mtune=generic rather than -mcpu=generic when building with DYNAMIC_ARCH=1 on aarch64 --- eb_hooks.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 27cc873fa1..7a31e87937 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -3,6 +3,7 @@ import os import re +import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg @@ -258,6 +259,13 @@ def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): if build_option('optarch') == OPTARCH_GENERIC: for step in ('build', 'test', 'install'): self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1") + + # use -mtune=generic rather than -mcpu=generic in $CFLAGS on aarch64, + # because -mcpu=generic implies a particular -march=armv* which clashes with those used by OpenBLAS + # when building with DYNAMIC_ARCH=1 + if get_cpu_architecture() == AARCH64: + cflags = os.getenv('CFLAGS').replace('-mcpu=generic', '-mtune=generic') + env.setvar('CFLAGS', cflags) else: raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") From 98b51a90fa62e65ab9d87b096246b4744e8b96ab Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 3 Jan 2024 19:16:34 +0100 Subject: [PATCH 0362/1795] Use only half the cores to build OpenFOAM --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8e29d2c103..edff46221a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -79,7 +79,7 @@ def post_ready_hook(self, *args, **kwargs): # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores. # here we reduce parallellism to only use half of that for selected software, # to avoid failing builds/tests due to out-of-memory problems - if self.name in ['TensorFlow']: + if self.name in ['TensorFlow'] or self.name in ['OpenFOAM']: parallel = self.cfg['parallel'] if parallel > 1: self.cfg['parallel'] = parallel // 2 From 4936618476b3b30572ca8fece14c890d89f4e34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Jan 2024 15:12:16 +0100 Subject: [PATCH 0363/1795] add SciPy-bundle-2023.11-gfbf-2023b.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 88ab50b8e7..b184b88701 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -1,3 +1,4 @@ easyconfigs: - GCC-13.2.0.eb - foss-2023b.eb + - SciPy-bundle-2023.11-gfbf-2023b.eb From 142a65b881183c6c889f3d9cdaef95c0543c4853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Jan 2024 19:22:50 +0100 Subject: [PATCH 0364/1795] also apply SciPy-bundle hook to version 2023.11 --- eb_hooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 7a31e87937..73eac9286d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -367,15 +367,15 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A... FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 = 2 failed, 30554 passed, 2064 skipped, 10992 deselected, 76 xfailed, 7 xpassed, 40 warnings in 380.27s (0:06:20) = - In versions 2023.07, 2 failing tests in scipy 1.11.1: + In versions 2023.07 and 2023.11, 2 failing tests in scipy 1.11.1 and 1.11.4: FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32 = 2 failed, 54409 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 892.04s (0:14:52) = In previous versions we were not as strict yet on the numpy/SciPy tests """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07'] and cpu_target == CPU_TARGET_NEOVERSE_V1: - self.cfg['testopts'] = "|| echo ignoring failing tests" + if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07', '2023.11'] and cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg['testopts'] = "|| echo ignoring failing tests" def pre_single_extension_hook(ext, *args, **kwargs): From d4a831de06fdbae3f81d8f48afdce2f9c7a386ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 6 Jan 2024 11:23:14 +0100 Subject: [PATCH 0365/1795] add new easystack for eb 4.9.0 2023a, add PyTorch 2.1.2 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml new file mode 100644 index 0000000000..1ecf60bdf2 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -0,0 +1,4 @@ +easyconfigs: + - PyTorch-2.1.2-foss-2023a.eb: + options: + from-pr: 19480 From 9c0ca0d408304ece1cdeed0aeabb42b9403bfd94 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Mon, 8 Jan 2024 11:31:46 +0100 Subject: [PATCH 0366/1795] Revert to using all cores in OpenFOAM tests Reducing in half did not solve issue with hanging/crashing in sanity checks --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 3b52b54e9e..00aee51816 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -80,7 +80,7 @@ def post_ready_hook(self, *args, **kwargs): # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores. # here we reduce parallellism to only use half of that for selected software, # to avoid failing builds/tests due to out-of-memory problems - if self.name in ['TensorFlow'] or self.name in ['OpenFOAM']: + if self.name in ['TensorFlow']: parallel = self.cfg['parallel'] if parallel > 1: self.cfg['parallel'] = parallel // 2 From 78466f569ca028d0119c293cd2f1d2409626619a Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:39:40 +0100 Subject: [PATCH 0367/1795] Add OpenFOAM-11-foss-2023a --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index aab910d1cc..803b6138e7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -46,3 +46,6 @@ easyconfigs: - ALL-0.9.2-foss-2023a.eb: options: from-pr: 19455 + - OpenFOAM-11-foss-2023a.eb: + options: + from-pr: 19545 From fdf0d9a1ee712f2d46eaeaaf6614eb15846441d7 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 10 Jan 2024 17:18:22 +0100 Subject: [PATCH 0368/1795] add zen4 / AMD Genoa --- init/arch_specs/eessi_arch_x86.spec | 7 ++--- .../x86_64/amd/zen4/Azure-Alma8-9V33X.cpuinfo | 27 +++++++++++++++++++ .../x86_64/amd/zen4/Azure-Alma8-9V33X.output | 1 + .../x86_64/amd/zen4/Shinx-RHEL8-9654.cpuinfo | 27 +++++++++++++++++++ .../x86_64/amd/zen4/Shinx-RHEL8-9654.output | 1 + 5 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.cpuinfo create mode 100644 tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.output create mode 100644 tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.cpuinfo create mode 100644 tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.output diff --git a/init/arch_specs/eessi_arch_x86.spec b/init/arch_specs/eessi_arch_x86.spec index 8d01cb0c03..bfbc5b4be1 100755 --- a/init/arch_specs/eessi_arch_x86.spec +++ b/init/arch_specs/eessi_arch_x86.spec @@ -1,6 +1,7 @@ # x86_64 CPU architecture specifications # Software path in EESSI | Vendor ID | List of defining CPU features -"x86_64/intel/haswell" "GenuineIntel" "avx2 fma" # Intel Haswell, Broadwell +"x86_64/intel/haswell" "GenuineIntel" "avx2 fma" # Intel Haswell, Broadwell "x86_64/intel/skylake_avx512" "GenuineIntel" "avx2 fma avx512f avx512bw avx512cd avx512dq avx512vl" # Intel Skylake, Cascade Lake -"x86_64/amd/zen2" "AuthenticAMD" "avx2 fma" # AMD Rome -"x86_64/amd/zen3" "AuthenticAMD" "avx2 fma vaes" # AMD Milan, Milan-X +"x86_64/amd/zen2" "AuthenticAMD" "avx2 fma" # AMD Rome +"x86_64/amd/zen3" "AuthenticAMD" "avx2 fma vaes" # AMD Milan, Milan-X +"x86_64/amd/zen4" "AuthenticAMD" "avx2 fma vaes avx512f avx512ifma" # AMD Genoa, Genoa-X diff --git a/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.cpuinfo b/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.cpuinfo new file mode 100644 index 0000000000..4a97da862c --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.cpuinfo @@ -0,0 +1,27 @@ +processor : 0 +vendor_id : AuthenticAMD +cpu family : 25 +model : 17 +model name : AMD EPYC 9V33X 96-Core Processor +stepping : 1 +microcode : 0xffffffff +cpu MHz : 3705.853 +cache size : 1024 KB +physical id : 0 +siblings : 88 +core id : 0 +cpu cores : 88 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves avx512_bf16 clzero xsaveerptr arat npt nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload avx512vbmi umip avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid fsrm +bugs : sysret_ss_attrs null_seg spectre_v1 spectre_v2 spec_store_bypass +bogomips : 5100.08 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: diff --git a/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.output b/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.output new file mode 100644 index 0000000000..950740a78c --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.output @@ -0,0 +1 @@ +x86_64/amd/zen4 diff --git a/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.cpuinfo b/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.cpuinfo new file mode 100644 index 0000000000..f28381d7a2 --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.cpuinfo @@ -0,0 +1,27 @@ +processor : 0 +vendor_id : AuthenticAMD +cpu family : 25 +model : 17 +model name : AMD EPYC 9654 96-Core Processor +stepping : 1 +microcode : 0xa10113e +cpu MHz : 3699.993 +cache size : 1024 KB +physical id : 0 +siblings : 96 +core id : 0 +cpu cores : 96 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local avx512_bf16 clzero irperf xsaveerptr wbnoinvd amd_ppin cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq la57 rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass +bogomips : 4799.99 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 52 bits physical, 57 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] diff --git a/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.output b/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.output new file mode 100644 index 0000000000..950740a78c --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.output @@ -0,0 +1 @@ +x86_64/amd/zen4 From 1ee221bdbfe0ae6015ca3765a8e23b0760a2323f Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 10 Jan 2024 17:24:21 +0100 Subject: [PATCH 0369/1795] add all.output tests --- tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.all.output | 1 + tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.all.output | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.all.output create mode 100644 tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.all.output diff --git a/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.all.output b/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.all.output new file mode 100644 index 0000000000..e1bbd79e4a --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.all.output @@ -0,0 +1 @@ +x86_64/amd/zen4:x86_64/amd/zen3:x86_64/amd/zen2:x86_64/generic diff --git a/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.all.output b/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.all.output new file mode 100644 index 0000000000..e1bbd79e4a --- /dev/null +++ b/tests/archdetect/x86_64/amd/zen4/Shinx-RHEL8-9654.all.output @@ -0,0 +1 @@ +x86_64/amd/zen4:x86_64/amd/zen3:x86_64/amd/zen2:x86_64/generic From 680e7b91836c33caeb1c2c493f0589d4b5a73219 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 10 Jan 2024 17:28:59 +0100 Subject: [PATCH 0370/1795] adding zen4 checks --- .github/workflows/tests_archdetect.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 37338693c5..d1407637e0 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -13,6 +13,8 @@ jobs: - x86_64/intel/skylake_avx512/archspec-linux-6132 - x86_64/amd/zen2/Azure-CentOS7-7V12 - x86_64/amd/zen3/Azure-CentOS7-7V73X + - x86_64/amd/zen4/Azure-Alma8-9V33X + - x86_64/amd/zen4/Shinx-RHEL8-9654 - ppc64le/power9le/unknown-power9le - aarch64/neoverse_n1/Azure-Ubuntu20-Altra - aarch64/neoverse_n1/AWS-awslinux-graviton2 From f6ee972f2a8d1c38fd23fc93fe45415793a85513 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 10 Jan 2024 17:43:10 +0100 Subject: [PATCH 0371/1795] {2023.06}[2022b] SciPy-bundle v2023.02 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml new file mode 100644 index 0000000000..fdacd95c55 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - SciPy-bundle-2023.02-gfbf-2022b.eb From 7070c047ebf57918dd0ae19c11ade011e99cefff Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 10 Jan 2024 18:00:17 +0100 Subject: [PATCH 0372/1795] commenting out power to prevent conflict --- .github/workflows/tests_archdetect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index d1407637e0..45e8dd2e60 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -15,10 +15,10 @@ jobs: - x86_64/amd/zen3/Azure-CentOS7-7V73X - x86_64/amd/zen4/Azure-Alma8-9V33X - x86_64/amd/zen4/Shinx-RHEL8-9654 - - ppc64le/power9le/unknown-power9le - aarch64/neoverse_n1/Azure-Ubuntu20-Altra - aarch64/neoverse_n1/AWS-awslinux-graviton2 - aarch64/neoverse_v1/AWS-awslinux-graviton3 + # - ppc64le/power9le/unknown-power9le fail-fast: false steps: - name: checkout From ad70b6b0857f6e3508a73c4675795de30f6b40eb Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 10 Jan 2024 19:27:14 +0100 Subject: [PATCH 0373/1795] commenting out power9 to prevent conflict --- .github/workflows/tests_archdetect.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 45e8dd2e60..68d25250ca 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -18,6 +18,8 @@ jobs: - aarch64/neoverse_n1/Azure-Ubuntu20-Altra - aarch64/neoverse_n1/AWS-awslinux-graviton2 - aarch64/neoverse_v1/AWS-awslinux-graviton3 + # commented out since these targets are currently not supported in software.eessi.io repo + # (and some tests assume that the corresponding subdirectory in software layer is there) # - ppc64le/power9le/unknown-power9le fail-fast: false steps: From fd3103e1f75157edffcf16023ef4fb6431c6b5eb Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 10 Jan 2024 20:15:18 +0100 Subject: [PATCH 0374/1795] move to software.eessi.io --- .github/workflows/test_eessi.yml | 14 +++++++------- .github/workflows/test_eessi_container_script.yml | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 04195dd619..7c96b091d2 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: EESSI_VERSION: - - 2021.12 + - 2023.06 EESSI_SOFTWARE_SUBDIR: - aarch64/generic - aarch64/graviton2 @@ -24,19 +24,19 @@ jobs: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: Mount EESSI CernVM-FS pilot repository + - name: Mount EESSI CernVM-FS software repository uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 with: cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb cvmfs_http_proxy: DIRECT - cvmfs_repositories: pilot.eessi-hpc.org + cvmfs_repositories: software.eessi.io - name: Test check_missing_installations.sh script run: | - source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version - export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort @@ -45,10 +45,10 @@ jobs: - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | - source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version - export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 929fb22cec..590aa4225b 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -90,15 +90,15 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then outfile=out_readwrite.txt fn="test_${RANDOM}.txt" - echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh + echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh chmod u+x test_script.sh export SINGULARITY_BIND="$PWD:/test" ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile} tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") # note: must use '--access rw' again here, since touched file is in overlay upper dir - ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile # test use of --resume elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then @@ -120,12 +120,12 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then outfile=out_save.txt fn="test_${RANDOM}.txt" - test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" + test_cmd="touch /cvmfs/software.eessi.io/${fn}" ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} rm -f ${outfile} - ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile tar tfv test-save.tar | grep "overlay-upper/${fn}" From a13bce84ca477c9d866ad722993252aad924d5fd Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 10 Jan 2024 20:20:48 +0100 Subject: [PATCH 0375/1795] move from pilot to software --- .github/workflows/tests_readme.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index 5c6d0318d4..034dae3780 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -24,7 +24,7 @@ jobs: - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults run: | source init/eessi_defaults - grep "${EESSI_PILOT_VERSION}" README.md + grep "${EESSI_VERSION}" README.md - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults run: | From 473a5b52b960881f2b4371978279976b9a8bccae Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 11 Jan 2024 03:51:20 +0100 Subject: [PATCH 0376/1795] move EESSI action to v3 --- .github/workflows/tests_archdetect.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 68d25250ca..f09567d125 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -26,7 +26,8 @@ jobs: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Enable EESSI - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 + #uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 + uses: eessi/github-action-eessi@v3 - name: test eessi_archdetect.sh run: | export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}} From 838ceffc09b6007384fc41697c2cbee335c8a234 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 12 Jan 2024 12:07:53 +0100 Subject: [PATCH 0377/1795] Use EasyBuildv4.9.0 for OpenFOAMv11 install --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 803b6138e7..aab910d1cc 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -46,6 +46,3 @@ easyconfigs: - ALL-0.9.2-foss-2023a.eb: options: from-pr: 19455 - - OpenFOAM-11-foss-2023a.eb: - options: - from-pr: 19545 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml new file mode 100644 index 0000000000..e33a8af48c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -0,0 +1,4 @@ +easyconfigs: + - OpenFOAM-11-foss-2023a.eb: + options: + from-pr: 19545 From b55efbf4d6d066dd292240e3863c8028c6336098 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 10 Jan 2024 18:41:52 +0100 Subject: [PATCH 0378/1795] add pre_single_extension_scipy hook to replace -mcpu=native with -march=armv8.4-a when building scipy 1.10.1 on aarch64/neoverse_v1 --- eb_hooks.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 73eac9286d..883d79c6e8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -384,16 +384,6 @@ def pre_single_extension_hook(ext, *args, **kwargs): PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) -def pre_single_extension_testthat(ext, *args, **kwargs): - """ - Pre-extension hook for testthat R package, to fix build on top of recent glibc. - """ - if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'): - # use constant value instead of SIGSTKSZ for stack size, - # cfr. https://github.com/r-lib/testthat/issues/1373 + https://github.com/r-lib/testthat/pull/1403 - ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " - - def pre_single_extension_isoband(ext, *args, **kwargs): """ Pre-extension hook for isoband R package, to fix build on top of recent glibc. @@ -404,6 +394,33 @@ def pre_single_extension_isoband(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " +def pre_single_extension_scipy(ext, *args, **kwargs): + """ + Pre-extension hook for scipy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for + aarch64/neoverse_v1 CPU target. + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if ext.name == 'scipy' and ext.version == '1.10.1' and cpu_target == CPU_TARGET_NEOVERSE_V1: + cflags = os.getenv('CFLAGS') + if '-mcpu=native' in cflags: + cflags = cflags.replace('-mcpu=native', '-march=armv8.4-a') + ext.cfg.update('configopts', ' '.join([ + "-Dc_args='%s'" % cflags, + "-Dcpp_args='%s'" % cflags, + "-Dfortran_args='%s'" % cflags, + ])) + + +def pre_single_extension_testthat(ext, *args, **kwargs): + """ + Pre-extension hook for testthat R package, to fix build on top of recent glibc. + """ + if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'): + # use constant value instead of SIGSTKSZ for stack size, + # cfr. https://github.com/r-lib/testthat/issues/1373 + https://github.com/r-lib/testthat/pull/1403 + ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " + + def post_sanitycheck_hook(self, *args, **kwargs): """Main post-sanity-check hook: trigger custom functions based on software name.""" if self.name in POST_SANITYCHECK_HOOKS: @@ -531,6 +548,7 @@ def inject_gpu_property(ec): PRE_SINGLE_EXTENSION_HOOKS = { 'isoband': pre_single_extension_isoband, + 'scipy': pre_single_extension_scipy, 'testthat': pre_single_extension_testthat, } From 684abebfe2a55cd66aef8ceb31169ea23e12ccc9 Mon Sep 17 00:00:00 2001 From: Xin An Date: Fri, 12 Jan 2024 15:37:13 +0100 Subject: [PATCH 0379/1795] {2023.06}[GCC/12.3.0] ReFrame v4.3.3 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index aab910d1cc..feab9d2c19 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -46,3 +46,4 @@ easyconfigs: - ALL-0.9.2-foss-2023a.eb: options: from-pr: 19455 + - ReFrame-4.3.3.eb From da6dc24615a7892dbea809bae173db588f2140a5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 12 Jan 2024 18:34:08 +0100 Subject: [PATCH 0380/1795] build numpy with -march=armv8.4-a instead of -mcpu=native (instead of scipy) --- eb_hooks.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 883d79c6e8..9d84a9e9f8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -394,21 +394,19 @@ def pre_single_extension_isoband(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " -def pre_single_extension_scipy(ext, *args, **kwargs): +def pre_single_extension_numpy(ext, *args, **kwargs): """ - Pre-extension hook for scipy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for + Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for aarch64/neoverse_v1 CPU target. """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if ext.name == 'scipy' and ext.version == '1.10.1' and cpu_target == CPU_TARGET_NEOVERSE_V1: - cflags = os.getenv('CFLAGS') - if '-mcpu=native' in cflags: - cflags = cflags.replace('-mcpu=native', '-march=armv8.4-a') - ext.cfg.update('configopts', ' '.join([ - "-Dc_args='%s'" % cflags, - "-Dcpp_args='%s'" % cflags, - "-Dfortran_args='%s'" % cflags, - ])) + if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: + # unsure which of these actually matter for numpy, so changing all of them + for envvar in ('CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS'): + value = os.getenv(envvar) + if '-mcpu=native' in value: + value = value.replace('-mcpu=native', '-march=armv8.4-a') + env.setvar(envvar, value) def pre_single_extension_testthat(ext, *args, **kwargs): @@ -548,7 +546,7 @@ def inject_gpu_property(ec): PRE_SINGLE_EXTENSION_HOOKS = { 'isoband': pre_single_extension_isoband, - 'scipy': pre_single_extension_scipy, + 'numpy': pre_single_extension_numpy, 'testthat': pre_single_extension_testthat, } From 7ee64cd33a36cd99b1d5a09d007e14fc868015ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 12 Jan 2024 18:51:11 +0100 Subject: [PATCH 0381/1795] use PyTorch easyconfig from easyconfigs PR #19573 which has extra fixes for non-x86 platforms --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 1ecf60bdf2..b7767c4fd5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -1,4 +1,4 @@ easyconfigs: - PyTorch-2.1.2-foss-2023a.eb: options: - from-pr: 19480 + from-pr: 19573 From 9943e357127f3ecea98517b1c78c09929710626d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 13 Jan 2024 18:23:00 +0100 Subject: [PATCH 0382/1795] update optarch build option to make sure that -march=armv8.4-a is used instead of -mcpu=native for numpy, reset it after installing numpy, and allow 2 failing scipy tests in SciPy-bundle v2023.02 --- eb_hooks.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9d84a9e9f8..5e6ffef823 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -367,23 +367,30 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A... FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 = 2 failed, 30554 passed, 2064 skipped, 10992 deselected, 76 xfailed, 7 xpassed, 40 warnings in 380.27s (0:06:20) = - In versions 2023.07 and 2023.11, 2 failing tests in scipy 1.11.1 and 1.11.4: + In versions 2023.02, 2023.07, and 2023.11, 2 failing tests in scipy (versions 1.10.1, 1.11.1, 1.11.4): FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32 = 2 failed, 54409 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 892.04s (0:14:52) = In previous versions we were not as strict yet on the numpy/SciPy tests """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07', '2023.11'] and cpu_target == CPU_TARGET_NEOVERSE_V1: + scipy_bundle_versions = ('2021.10', '2023.02', '2023.07', '2023.11') + if self.name == 'SciPy-bundle' and self.version in scipy_bundle_versions and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" def pre_single_extension_hook(ext, *args, **kwargs): - """Main pre-configure hook: trigger custom functions based on software name.""" + """Main pre-extension: trigger custom functions based on software name.""" if ext.name in PRE_SINGLE_EXTENSION_HOOKS: PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) +def post_single_extension_hook(ext, *args, **kwargs): + """Main post-extension hook: trigger custom functions based on software name.""" + if ext.name in POST_SINGLE_EXTENSION_HOOKS: + POST_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) + + def pre_single_extension_isoband(ext, *args, **kwargs): """ Pre-extension hook for isoband R package, to fix build on top of recent glibc. @@ -396,17 +403,25 @@ def pre_single_extension_isoband(ext, *args, **kwargs): def pre_single_extension_numpy(ext, *args, **kwargs): """ - Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for - aarch64/neoverse_v1 CPU target. + Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x + when building for aarch64/neoverse_v1 CPU target. """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: + # note: this hook is called before build environment is set up (by calling toolchain.prepare()), + # so environment variables like $CFLAGS are not defined yet # unsure which of these actually matter for numpy, so changing all of them - for envvar in ('CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS'): - value = os.getenv(envvar) - if '-mcpu=native' in value: - value = value.replace('-mcpu=native', '-march=armv8.4-a') - env.setvar(envvar, value) + ext.orig_optarch = build_option('optarch') + update_build_option('optarch', 'march=armv8.4-a') + + +def post_single_extension_numpy(ext, *args, **kwargs): + """ + Post-extension hook for numpy, to reset 'optarch' build option. + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: + update_build_option('optarch', ext.orig_optarch) def pre_single_extension_testthat(ext, *args, **kwargs): @@ -550,6 +565,10 @@ def inject_gpu_property(ec): 'testthat': pre_single_extension_testthat, } +POST_SINGLE_EXTENSION_HOOKS = { + 'numpy': post_single_extension_numpy, +} + POST_SANITYCHECK_HOOKS = { 'CUDA': post_sanitycheck_cuda, } From 258292c99e5aff3c24dc52652f7ad3ddc55af48b Mon Sep 17 00:00:00 2001 From: Xin An Date: Mon, 15 Jan 2024 11:10:23 +0100 Subject: [PATCH 0383/1795] {2023.06} Add ReFrame v4.3.3 to system easystack --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 1 - .../2023.06/eessi-2023.06-eb-4.9.0-001-system.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index feab9d2c19..aab910d1cc 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -46,4 +46,3 @@ easyconfigs: - ALL-0.9.2-foss-2023a.eb: options: from-pr: 19455 - - ReFrame-4.3.3.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml index a7bce002c7..25c13e49c9 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-001-system.yml @@ -2,3 +2,4 @@ easyconfigs: - EasyBuild-4.9.0.eb: options: from-pr: 19464 + - ReFrame-4.3.3.eb From 847eb715bb33825de47c2a90b99d7806c2b1e1d9 Mon Sep 17 00:00:00 2001 From: Xin An Date: Mon, 15 Jan 2024 11:15:22 +0100 Subject: [PATCH 0384/1795] {2023.06} Add ReFrame v4.3.3 to system easystack eb v4.8.2 --- .../2023.06/eessi-2023.06-eb-4.8.2-001-system.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml index f02b9f2802..d85440ec5b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml @@ -5,3 +5,4 @@ easyconfigs: - Nextflow-23.10.0.eb: options: from-pr: 19172 + - ReFrame-4.3.3.eb From 807d78471ad4104d0ff3f590dbc697d6cd2a3968 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 15 Jan 2024 11:27:42 +0100 Subject: [PATCH 0385/1795] fix upstream merge --- .github/workflows/test_eessi.yml | 72 -------------------------------- 1 file changed, 72 deletions(-) delete mode 100644 .github/workflows/test_eessi.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml deleted file mode 100644 index 7c96b091d2..0000000000 --- a/.github/workflows/test_eessi.yml +++ /dev/null @@ -1,72 +0,0 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests relying on having EESSI pilot repo mounted -on: [push, pull_request, workflow_dispatch] -permissions: - contents: read # to fetch code (actions/checkout) -jobs: - eessi_pilot_repo: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - EESSI_VERSION: - - 2023.06 - EESSI_SOFTWARE_SUBDIR: - - aarch64/generic - - aarch64/graviton2 - - aarch64/graviton3 - - x86_64/amd/zen2 - - x86_64/amd/zen3 - - x86_64/intel/haswell - - x86_64/intel/skylake_avx512 - - x86_64/generic - steps: - - name: Check out software-layer repository - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Mount EESSI CernVM-FS software repository - uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 - with: - cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb - cvmfs_http_proxy: DIRECT - cvmfs_repositories: software.eessi.io - - - name: Test check_missing_installations.sh script - run: | - source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash - module load EasyBuild - eb --version - export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} - export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} - env | grep ^EESSI | sort - echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" - ./check_missing_installations.sh - - - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) - run: | - source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash - module load EasyBuild - eb --version - export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} - export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} - env | grep ^EESSI | sort - echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" - echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml - tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml - # note, check_missing_installations.sh exits 1 if a package was - # missing, which is intepreted as false (exit code based, not - # boolean logic), hence when the script exits 0 if no package was - # missing it is interpreted as true, thus the test did not capture - # the missing package - if ./check_missing_installations.sh; then - echo "did NOT capture missing package; test FAILED" - exit 1 - else - echo "captured missing package; test PASSED" - exit 0 - fi From 41f6dea98fa5a4b14b590ba42dffec9eacfaf8a5 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 15 Jan 2024 10:39:09 +0000 Subject: [PATCH 0386/1795] {2023.06}[foss/2022b] R v4.2.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml new file mode 100644 index 0000000000..f8af4b5b45 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - R-4.2.2-foss-2022b.eb From b16dc0a1cc01e054f27940d10a9a038399fc1478 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 15 Jan 2024 12:47:09 +0000 Subject: [PATCH 0387/1795] {2023.06}[gompi/2023b] netCDF v4.9.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index b184b88701..9a3b8efa6d 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -2,3 +2,6 @@ easyconfigs: - GCC-13.2.0.eb - foss-2023b.eb - SciPy-bundle-2023.11-gfbf-2023b.eb + - netCDF-4.9.2-gompi-2023b.eb: + options: + from-pr: 19534 From 937efa6d8520e5af5f8292a21cbd0d3a9c5a8344 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 15 Jan 2024 18:06:00 +0100 Subject: [PATCH 0388/1795] fix docstring in pre_single_extension_numpy hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 5e6ffef823..7c52e24ff1 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -403,7 +403,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): def pre_single_extension_numpy(ext, *args, **kwargs): """ - Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x + Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for numpy 1.24.2 when building for aarch64/neoverse_v1 CPU target. """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') From 9ae0832f722908e9a703c272c1ea7967580a33af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Jan 2024 22:44:49 +0100 Subject: [PATCH 0389/1795] merge main branch --- .../2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index f27e928364..69ffb750a2 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -1,13 +1,6 @@ easyconfigs: - foss-2022b.eb - - pybind11-2.10.3-GCCcore-12.2.0.eb: - # avoid indirect dependency on old CMake version built with GCCcore/10.2.0 via Catch2 build dependency; - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 + - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: options: - from-pr: 19270 - - LERC-4.0.0-GCCcore-12.2.0.eb: - # avoid RPATH-related sanity check errors by compiling LERC's test binary in postinstallcmds; - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19386 - options: - from-pr: 19386 - - GDAL-3.6.2-foss-2022b.eb + from-pr: 19339 + - Qt5-5.15.7-GCCcore-12.2.0.eb From 0e917d51b20458c8e7339f5e859e1121bc026d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Jan 2024 22:47:39 +0100 Subject: [PATCH 0390/1795] add GDAL-3.6.2-foss-2022b.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index fdacd95c55..9e92c79062 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,2 +1,3 @@ easyconfigs: - SciPy-bundle-2023.02-gfbf-2022b.eb + - GDAL-3.6.2-foss-2022b.eb From 15c228b555590b0da28d174ce79d1a8cea44c64b Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 16 Jan 2024 08:53:33 +0000 Subject: [PATCH 0391/1795] added a hook for neoverse_v1 failing tests --- eb_hooks.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 7c52e24ff1..41d196b097 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -378,6 +378,18 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): if self.name == 'SciPy-bundle' and self.version in scipy_bundle_versions and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" +def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): + """ + Pre-test hook for SciPy-bundle: skip failing tests for selected netCDF versions on neoverse_v1 + cfr. https://github.com/EESSI/software-layer/issues/425 + The following tests are problematic: + 163 - nc_test4_run_par_test (Timeout) + 190 - h5_test_run_par_tests (Timeout) + A few other tests are skipped in the easyconfig and patches for similar issues, see above issue for details. + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg['testopts'] = "|| echo ignoring failing tests" def pre_single_extension_hook(ext, *args, **kwargs): """Main pre-extension: trigger custom functions based on software name.""" @@ -557,6 +569,7 @@ def inject_gpu_property(ec): 'ESPResSo': pre_test_hook_ignore_failing_tests_ESPResSo, 'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI, 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, + 'netCDF': pre_test_hook_ignore_failing_tests_netCDF, } PRE_SINGLE_EXTENSION_HOOKS = { From 90e25455b6e1525165d42425ee6035825d040aec Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 16 Jan 2024 12:26:37 +0000 Subject: [PATCH 0392/1795] added eessi-2023.06-known-issues.yml --- eessi-2023.06-known-issues.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 eessi-2023.06-known-issues.yml diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml new file mode 100644 index 0000000000..cb2d945af2 --- /dev/null +++ b/eessi-2023.06-known-issues.yml @@ -0,0 +1,4 @@ +- aarch64/neoverse_v1: + - netCDF-4.9.2-GCC-12.3.0/GCC-13.2.0: + - issue: https://github.com/EESSI/software-layer/issues/425 + - info: "netCDF intermittent test failures" From 67e6a258860325c3242e22939c0df9f91c07334e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Jan 2024 14:02:14 +0100 Subject: [PATCH 0393/1795] add at-spi2-core-2.49.91-GCCcore-12.3.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index e33a8af48c..1586c16827 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -2,3 +2,4 @@ easyconfigs: - OpenFOAM-11-foss-2023a.eb: options: from-pr: 19545 + - at-spi2-core-2.49.91-GCCcore-12.3.0.eb From b49ce5d7929c64687acb8e114ae4938f232647db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Jan 2024 14:12:40 +0100 Subject: [PATCH 0394/1795] add pre-configure hook for at-spi2-core --- eb_hooks.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 7c52e24ff1..ff9072c793 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -335,6 +335,19 @@ def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") +def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): + """ + pre-configure hook for at-spi2-core: + - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH + when EasyBuild is configured to filter it + """ + if self.name == 'at-spi2-core': + if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'): + self.cfg.update('preconfigopts', 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ') + else: + raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!") + + def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: @@ -551,6 +564,7 @@ def inject_gpu_property(ec): 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, 'LAMMPS': pre_configure_hook_LAMMPS_aarch64, + 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, } PRE_TEST_HOOKS = { From 23aa7ce5e0e59f2280cbc5feda7e3f213788b9b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Jan 2024 14:18:42 +0100 Subject: [PATCH 0395/1795] split long line --- eb_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index ff9072c793..c63121afad 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -343,7 +343,8 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): """ if self.name == 'at-spi2-core': if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'): - self.cfg.update('preconfigopts', 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ') + sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ') + self.cfg.update('preconfigopts', sed_cmd) else: raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!") From 75bf2dec2d333df9bf0bee84116f0ea9bdce5d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Jan 2024 14:58:29 +0100 Subject: [PATCH 0396/1795] fix syntax, remove ) --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index c63121afad..a9169d60b8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -343,7 +343,7 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): """ if self.name == 'at-spi2-core': if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'): - sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ') + sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ' self.cfg.update('preconfigopts', sed_cmd) else: raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!") From 9a1b4c82583d9022fb9a50bff9ce7d2e080cc290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Jan 2024 15:53:57 +0100 Subject: [PATCH 0397/1795] add link to github issue --- eb_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index a9169d60b8..b3cbc4b28e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -339,7 +339,8 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): """ pre-configure hook for at-spi2-core: - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH - when EasyBuild is configured to filter it + when EasyBuild is configured to filter it, see: + https://github.com/EESSI/software-layer/issues/196 """ if self.name == 'at-spi2-core': if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'): From 9944ab6546d856acdf0d92231dac56af8aab82b8 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:17:29 +0100 Subject: [PATCH 0398/1795] Update eb_hooks.py Co-authored-by: ocaisa --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 41d196b097..3b18f330ed 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -380,7 +380,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): """ - Pre-test hook for SciPy-bundle: skip failing tests for selected netCDF versions on neoverse_v1 + Pre-test hook for netCDF: skip failing tests for selected netCDF versions on neoverse_v1 cfr. https://github.com/EESSI/software-layer/issues/425 The following tests are problematic: 163 - nc_test4_run_par_test (Timeout) From 5577d243ccb43df34256036a617b0ab1b53665e0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Jan 2024 20:14:26 +0100 Subject: [PATCH 0399/1795] {2023.06}[foss/2023a] ESPResSo v4.2.1 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index e33a8af48c..0569affbca 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -1,4 +1,9 @@ easyconfigs: - OpenFOAM-11-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19545 options: from-pr: 19545 + - ESPResSo-4.2.1-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19592 + options: + from-pr: 19592 From 07a99b43b7b7577e881095a73b74300a00793567 Mon Sep 17 00:00:00 2001 From: Xin An Date: Wed, 17 Jan 2024 11:01:27 +0100 Subject: [PATCH 0400/1795] remove reframe v433 from eb v4.8.2 --- .../2023.06/eessi-2023.06-eb-4.8.2-001-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml index d85440ec5b..f02b9f2802 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-001-system.yml @@ -5,4 +5,3 @@ easyconfigs: - Nextflow-23.10.0.eb: options: from-pr: 19172 - - ReFrame-4.3.3.eb From 34f4c453044ea6acd324b462f47de38db33c3b61 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 18 Jan 2024 12:03:25 +0100 Subject: [PATCH 0401/1795] fix known issues YAML file for netCDF --- eessi-2023.06-known-issues.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index cb2d945af2..0ee661be73 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,4 +1,7 @@ - aarch64/neoverse_v1: - - netCDF-4.9.2-GCC-12.3.0/GCC-13.2.0: + - netCDF-4.9.2-gompi-2023a.eb: + - issue: https://github.com/EESSI/software-layer/issues/425 + - info: "netCDF intermittent test failures" + - netCDF-4.9.2-gompi-2023b.eb: - issue: https://github.com/EESSI/software-layer/issues/425 - info: "netCDF intermittent test failures" From 13ea5e7affb71e73546c963e4136591f213f3193 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Thu, 18 Jan 2024 13:30:46 +0100 Subject: [PATCH 0402/1795] {2023.06}[2023a] Rivet 3.1.9 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index eba776ab1f..f27c9a316f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -8,3 +8,6 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19592 options: from-pr: 19592 + - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb: + options: + from-pr: 19631 From abfc7ae9f5b8f52c9cb5a348bcf263d3d0bf650a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 09:19:59 +0100 Subject: [PATCH 0403/1795] complete overview of known issues (so far) for software.eessi.io version 2023.06 --- eessi-2023.06-known-issues.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 0ee661be73..8d8cfe4d07 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,7 +1,28 @@ - aarch64/neoverse_v1: + - ESPResSo-4.2.1-foss-2023a: + - issue: https://github.com/EESSI/software-layer/issues/363 + - info: "ESPResSo tests failing due to timeouts" + - FFTW.MPI-3.3.10-gompi-2023a: + - issue: https://github.com/EESSI/software-layer/issues/325 + - info: "Flaky FFTW tests, random failures" + - FFTW.MPI-3.3.10-gompi-2023b: + - issue: https://github.com/EESSI/software-layer/issues/325 + - info: "Flaky FFTW tests, random failures" - netCDF-4.9.2-gompi-2023a.eb: - issue: https://github.com/EESSI/software-layer/issues/425 - info: "netCDF intermittent test failures" - netCDF-4.9.2-gompi-2023b.eb: - issue: https://github.com/EESSI/software-layer/issues/425 - info: "netCDF intermittent test failures" + - OpenBLAS-0.3.21-GCC-12.2.0: + - issue: https://github.com/EESSI/software-layer/issues/314 + - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" + - SciPy-bundle-2023.11-gfbf-2023b: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "numpy built with -march=armv8.4-a instead of -mcpu=native (no SVE) + 2 failing tests (vs 50005 passed) in scipy test suite" + - SciPy-bundle-2023.11-gfbf-2023b: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "2 failing tests (vs 54409 passed) in scipy test suite" + - SciPy-bundle-2023.11-gfbf-2023b: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "2 failing tests (vs 54876 passed) in scipy test suite" From efbc538248634d06dc8e09374af87188dd373f0e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 09:20:32 +0100 Subject: [PATCH 0404/1795] limit hook to increase max failing numerical tests on neoverse_v1 to OpenBLAS < 0.3.23 --- eb_hooks.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 14742678bb..2bc2fdebd5 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -185,20 +185,21 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): - """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target.""" + """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target for OpenBLAS < 0.3.23""" cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.name == 'OpenBLAS': - # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target - # since the default setting of 150 that works well on other aarch64 targets and x86_64 is a bit too strict - # See https://github.com/EESSI/software-layer/issues/314 - cfg_option = 'max_failing_lapack_tests_num_errors' - if cpu_target == CPU_TARGET_NEOVERSE_V1: - orig_value = ec[cfg_option] - ec[cfg_option] = 400 - print_msg("Maximum number of failing LAPACK tests with numerical errors for %s relaxed to %s (was %s)", - ec.name, ec[cfg_option], orig_value) - else: - print_msg("Not changing option %s for %s on non-AARCH64", cfg_option, ec.name) + if LooseVersion(ec.version) < LooseVersion('0.3.23'): + # relax maximum number of failed numerical LAPACK tests for aarch64/neoverse_v1 CPU target + # since the default setting of 150 that works well on other aarch64 targets and x86_64 is a bit too strict + # See https://github.com/EESSI/software-layer/issues/314 + cfg_option = 'max_failing_lapack_tests_num_errors' + if cpu_target == CPU_TARGET_NEOVERSE_V1: + orig_value = ec[cfg_option] + ec[cfg_option] = 400 + print_msg("Maximum number of failing LAPACK tests with numerical errors for %s relaxed to %s (was %s)", + ec.name, ec[cfg_option], orig_value) + else: + print_msg("Not changing option %s for %s on non-AARCH64", cfg_option, ec.name) else: raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") From 855251262cb228c8adf1ea37ad22af6e71ddb8c7 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 19 Jan 2024 11:07:59 +0000 Subject: [PATCH 0405/1795] {2023.06}[gfbf/2023b] matplotlib v3.8.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 9a3b8efa6d..4dd31dbd5d 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -5,3 +5,6 @@ easyconfigs: - netCDF-4.9.2-gompi-2023b.eb: options: from-pr: 19534 + - matplotlib-3.8.2-gfbf-2023b.eb: + options: + from-pr: 19552 From 6705bfc57c35bf4ee4a1c3e8c08356c57c0b91c5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 19 Jan 2024 16:31:12 +0100 Subject: [PATCH 0406/1795] fix entries in known issues for SciPy-bundle --- eessi-2023.06-known-issues.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 8d8cfe4d07..475ee2c1d7 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -17,10 +17,10 @@ - OpenBLAS-0.3.21-GCC-12.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" - - SciPy-bundle-2023.11-gfbf-2023b: + - SciPy-bundle-2023.02-gfbf-2022b: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "numpy built with -march=armv8.4-a instead of -mcpu=native (no SVE) + 2 failing tests (vs 50005 passed) in scipy test suite" - - SciPy-bundle-2023.11-gfbf-2023b: + - SciPy-bundle-2023.07-gfbf-2023a: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "2 failing tests (vs 54409 passed) in scipy test suite" - SciPy-bundle-2023.11-gfbf-2023b: From a94759aee70d581e63f46200b8fdeb148c546590 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 23 Jan 2024 13:52:47 +0000 Subject: [PATCH 0407/1795] Trying with parallel 1 which was removed in PR18467 --- .../2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index fd88fafb0c..18fb0a742f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -4,4 +4,6 @@ easyconfigs: options: from-pr: 19339 - Qt5-5.15.7-GCCcore-12.2.0.eb - - QuantumESPRESSO-7.2-foss-2022b.eb + - QuantumESPRESSO-7.2-foss-2022b.eb: + options: + parallel: 1 From 832b152c5296fc05487aa48a7e9d52a69f700e2e Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 23 Jan 2024 14:00:24 +0000 Subject: [PATCH 0408/1795] Trying with parallel 1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index 18fb0a742f..24fe340bf2 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -6,4 +6,4 @@ easyconfigs: - Qt5-5.15.7-GCCcore-12.2.0.eb - QuantumESPRESSO-7.2-foss-2022b.eb: options: - parallel: 1 + parallel 1 From 4d691eca00db20c4e948459973b660dde51d3d58 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 23 Jan 2024 16:49:24 +0000 Subject: [PATCH 0409/1795] Trying with parallel 1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index 24fe340bf2..6c08ff774c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -6,4 +6,4 @@ easyconfigs: - Qt5-5.15.7-GCCcore-12.2.0.eb - QuantumESPRESSO-7.2-foss-2022b.eb: options: - parallel 1 + parallel: '1' From 699052b0ba0145c22051838303548804ab2559d0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 23 Jan 2024 18:21:23 +0100 Subject: [PATCH 0410/1795] only pick a software subdir that is present in the CVMFS repository --- init/eessi_environment_variables | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 3584bfaf34..af5222e7b9 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -27,8 +27,17 @@ if [ -d $EESSI_PREFIX ]; then # determine subdirectory in software layer if [ "$EESSI_USE_ARCHDETECT" == "1" ]; then # if archdetect is enabled, use internal code - export EESSI_SOFTWARE_SUBDIR=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh cpupath) - show_msg "archdetect says ${EESSI_SOFTWARE_SUBDIR}" + all_cpupaths=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh -a cpupath) + # iterate over colon-separated list verifying if the architecture is present + # under $EESSI_PREFIX/software/$EESSI_OS_TYPE; if so use the architecture as best match + IFS=: read -r -a archs <<< "${all_cpupaths}" + for arch in "${archs[@]}"; do + if [ -d ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${arch} ]; then + export EESSI_SOFTWARE_SUBDIR=${arch} + show_msg "archdetect says ${EESSI_SOFTWARE_SUBDIR}" + break + fi + done elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined! export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3 From 97572ac5aaf3d83fe6f81d03c84e7e6161465186 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Tue, 23 Jan 2024 22:01:12 +0100 Subject: [PATCH 0411/1795] Use MR with fixed zlib --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index f27c9a316f..b9f51cff48 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -10,4 +10,4 @@ easyconfigs: from-pr: 19592 - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb: options: - from-pr: 19631 + from-pr: 19679 From abfc5b01d40c1bc8492483682b0ea5f8767f58ef Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 24 Jan 2024 16:21:14 +0100 Subject: [PATCH 0412/1795] Add hook to accept higher number of failing tests (10) for PyTorch on ARM --- eb_hooks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 7a31e87937..ef50fa8ba9 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -20,6 +20,7 @@ from distutils.version import LooseVersion +CPU_TARGET_NEOVERSE_N1 = 'aarch64/neoverse_n1' CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' @@ -378,6 +379,21 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): self.cfg['testopts'] = "|| echo ignoring failing tests" +def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): + """ + Pre-test hook for PyTorch: increase max failing tests for ARM for PyTorch 2.1.2 + See https://github.com/EESSI/software-layer/pull/444#issuecomment-1890416171 + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + arm_target = ( + cpu_target == CPU_TARGET_NEOVERSE_V1 or + cpu_target == CPU_TARGET_NEOVERSE_N1 or + cpu_target == CPU_TARGET_AARCH64_GENERIC + ) + if self.name == 'PyTorch' and self.version == '2.1.2' and arm_target: + self.cfg['max_failed_tests'] = 10 + + def pre_single_extension_hook(ext, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" if ext.name in PRE_SINGLE_EXTENSION_HOOKS: @@ -527,6 +543,7 @@ def inject_gpu_property(ec): 'ESPResSo': pre_test_hook_ignore_failing_tests_ESPResSo, 'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI, 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, + 'PyTorch': pre_test_hook_increase_max_failed_tests_arm_PyTorch, } PRE_SINGLE_EXTENSION_HOOKS = { From 9fc553944e7322030b69ef472d02aec87739c5b9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 24 Jan 2024 16:51:08 +0100 Subject: [PATCH 0413/1795] do general check for ARM, like done in the kokkos hook --- eb_hooks.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index ef50fa8ba9..f949d02272 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -20,7 +20,6 @@ from distutils.version import LooseVersion -CPU_TARGET_NEOVERSE_N1 = 'aarch64/neoverse_n1' CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' @@ -385,12 +384,7 @@ def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): See https://github.com/EESSI/software-layer/pull/444#issuecomment-1890416171 """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - arm_target = ( - cpu_target == CPU_TARGET_NEOVERSE_V1 or - cpu_target == CPU_TARGET_NEOVERSE_N1 or - cpu_target == CPU_TARGET_AARCH64_GENERIC - ) - if self.name == 'PyTorch' and self.version == '2.1.2' and arm_target: + if self.name == 'PyTorch' and self.version == '2.1.2' and get_cpu_architecture() == AARCH64:: self.cfg['max_failed_tests'] = 10 From ed45a743b26605dcf67f54da11de1dda6d015145 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 24 Jan 2024 16:18:38 +0000 Subject: [PATCH 0414/1795] Added libxc to post_ready_hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 2bc2fdebd5..4d791c7c3d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -80,7 +80,7 @@ def post_ready_hook(self, *args, **kwargs): # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores. # here we reduce parallellism to only use half of that for selected software, # to avoid failing builds/tests due to out-of-memory problems - if self.name in ['TensorFlow']: + if self.name in ['TensorFlow', 'libxc']: parallel = self.cfg['parallel'] if parallel > 1: self.cfg['parallel'] = parallel // 2 From 63a9c98720f6773c1d01c7a92a45972a5e8ce00a Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 24 Jan 2024 16:21:18 +0000 Subject: [PATCH 0415/1795] Reverted to default easyconfig setting --- .../2023.06/eessi-2023.06-eb-4.8.2-2022b.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml index 6c08ff774c..fd88fafb0c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2022b.yml @@ -4,6 +4,4 @@ easyconfigs: options: from-pr: 19339 - Qt5-5.15.7-GCCcore-12.2.0.eb - - QuantumESPRESSO-7.2-foss-2022b.eb: - options: - parallel: '1' + - QuantumESPRESSO-7.2-foss-2022b.eb From a4ade847f2bba2f0118eccbc17cefd4fbf296052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 24 Jan 2024 18:47:44 +0100 Subject: [PATCH 0416/1795] remove double colon --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index f949d02272..4544c7d398 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -384,7 +384,7 @@ def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): See https://github.com/EESSI/software-layer/pull/444#issuecomment-1890416171 """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'PyTorch' and self.version == '2.1.2' and get_cpu_architecture() == AARCH64:: + if self.name == 'PyTorch' and self.version == '2.1.2' and get_cpu_architecture() == AARCH64: self.cfg['max_failed_tests'] = 10 From 5ac329bf0b31f508969db150d30514e8f5f7eb50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 24 Jan 2024 18:47:57 +0100 Subject: [PATCH 0417/1795] remove redundant line --- eb_hooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4544c7d398..91591f121f 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -383,7 +383,6 @@ def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): Pre-test hook for PyTorch: increase max failing tests for ARM for PyTorch 2.1.2 See https://github.com/EESSI/software-layer/pull/444#issuecomment-1890416171 """ - cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'PyTorch' and self.version == '2.1.2' and get_cpu_architecture() == AARCH64: self.cfg['max_failed_tests'] = 10 From 128fea77374fc47babcdf591a91eddcbd8d94ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 25 Jan 2024 12:10:50 +0100 Subject: [PATCH 0418/1795] add item for PyTorch test failures on aarch64 --- eessi-2023.06-known-issues.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 475ee2c1d7..5ca68e4f44 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -17,6 +17,9 @@ - OpenBLAS-0.3.21-GCC-12.2.0: - issue: https://github.com/EESSI/software-layer/issues/314 - info: "Increased number of numerical errors in OpenBLAS test suite (344 vs max. 150 on x86_64/*)" + - PyTorch-2.1.2-foss-2023a: + - issue: https://github.com/EESSI/software-layer/issues/461 + - info: "8 failing tests (out of 209539) on aarch64/*" - SciPy-bundle-2023.02-gfbf-2022b: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "numpy built with -march=armv8.4-a instead of -mcpu=native (no SVE) + 2 failing tests (vs 50005 passed) in scipy test suite" From 5955adcf75954b2fd66680af051f5feb7c9a1fc9 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 25 Jan 2024 20:56:54 +0000 Subject: [PATCH 0419/1795] {2023.06}[GCCcore/13.2.0] Qt5 v5.15.11 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 4dd31dbd5d..0d1863f5f3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -8,3 +8,6 @@ easyconfigs: - matplotlib-3.8.2-gfbf-2023b.eb: options: from-pr: 19552 + - Qt5-5.15.11-GCCcore-13.2.0.eb: + options: + from-pr: 19320 From bec06f5998d01d20584e7ab6a3b1654dbaed2faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 26 Jan 2024 13:31:12 +0100 Subject: [PATCH 0420/1795] add pytorch issue for aarch64/generic and aarch64/neoverse_n1 --- eessi-2023.06-known-issues.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 5ca68e4f44..569a0d9f56 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,3 +1,11 @@ +- aarch64/generic: + - PyTorch-2.1.2-foss-2023a: + - issue: https://github.com/EESSI/software-layer/issues/461 + - info: "8 failing tests (out of 209539) on aarch64/*" +- aarch64/neoverse_n1: + - PyTorch-2.1.2-foss-2023a: + - issue: https://github.com/EESSI/software-layer/issues/461 + - info: "8 failing tests (out of 209539) on aarch64/*" - aarch64/neoverse_v1: - ESPResSo-4.2.1-foss-2023a: - issue: https://github.com/EESSI/software-layer/issues/363 From 69a7010feafa11a38e654c62907438aa37bec2f3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 26 Jan 2024 16:47:44 +0100 Subject: [PATCH 0421/1795] {2023.09}[gfbf/2023a] scikit-learn 1.3.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index b9f51cff48..43d9944b8a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -11,3 +11,4 @@ easyconfigs: - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb: options: from-pr: 19679 + - scikit-learn-1.3.1-gfbf-2023a.eb From b82c7bdb8999c62f4b953c9bb2ebc6d55624e500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 26 Jan 2024 21:51:58 +0100 Subject: [PATCH 0422/1795] use CDO from pr 19735 --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 2f25925a3a..b0205f8a85 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -46,4 +46,6 @@ easyconfigs: - ALL-0.9.2-foss-2023a.eb: options: from-pr: 19455 - - CDO-2.2.2-gompi-2023a.eb \ No newline at end of file + - CDO-2.2.2-gompi-2023a.eb: + options: + from-pr: 19735 From 737455a35385b38ded34af3a5bdcc70edb1c2c50 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 28 Jan 2024 00:38:40 +0100 Subject: [PATCH 0423/1795] bump versions of GH actions to address Node.js deprecation note --- .github/workflows/scorecards.yml | 2 +- .github/workflows/test-software.eessi.io.yml | 2 +- .github/workflows/test_eessi_container_script.yml | 2 +- .github/workflows/test_licenses.yml | 4 ++-- .github/workflows/tests.yml | 4 ++-- .github/workflows/tests_archdetect.yml | 2 +- .github/workflows/tests_init.yml | 4 ++-- .github/workflows/tests_readme.yml | 2 +- .github/workflows/tests_scripts.yml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index ec018bd049..dc18fd584a 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -35,7 +35,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: persist-credentials: false diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 3ac341a177..df801ca16e 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -22,7 +22,7 @@ jobs: - x86_64/generic steps: - name: Check out software-layer repository - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Mount EESSI CernVM-FS pilot repository uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 33122e6ff4..32120d0087 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -22,7 +22,7 @@ jobs: #- save steps: - name: Check out software-layer repository - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: install Apptainer run: | diff --git a/.github/workflows/test_licenses.yml b/.github/workflows/test_licenses.yml index 00a2c90f6b..3b9675d523 100644 --- a/.github/workflows/test_licenses.yml +++ b/.github/workflows/test_licenses.yml @@ -8,10 +8,10 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Check out software-layer repository - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: set up Python - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: '3.9' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cc00685a40..8e74a4e844 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,10 +12,10 @@ jobs: fail-fast: false steps: - name: checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: set up Python - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: ${{matrix.python}} architecture: x64 diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 1e8b830e14..5b63efaed3 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -22,7 +22,7 @@ jobs: fail-fast: false steps: - name: checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Mount EESSI CernVM-FS pilot repository uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 417b7851f1..38ccbbad31 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -12,10 +12,10 @@ jobs: fail-fast: false steps: - name: checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: set up Python - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 with: python-version: ${{matrix.python}} architecture: x64 diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index d229879f67..efdb796e5e 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Check out software-layer repository - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: verify if README.md is consistent with EESSI_VERSION from init/eessi_defaults run: | diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index a369f4f187..df1884dd8c 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: install Apptainer run: | From 8daea34e875bb0fa77e28db4f2ca46761afe4757 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 28 Jan 2024 00:49:45 +0100 Subject: [PATCH 0424/1795] bump version of cvmfs action to address Node.js deprecation note --- .github/workflows/test-software.eessi.io.yml | 2 +- .github/workflows/tests_archdetect.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index df801ca16e..8cfb023bc6 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 with: cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb cvmfs_http_proxy: DIRECT diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 5b63efaed3..922c9a1bf0 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 with: cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb cvmfs_http_proxy: DIRECT From 0f3646ef0eec6a89aa1a8a5479c8040264af5af3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 5 Feb 2024 17:28:18 +0100 Subject: [PATCH 0425/1795] Second attempt at having the bot run the test suite, now based on Thomas' PR 366 --- bot/check-test.sh | 66 ++++++++++++++++++++++++++++++++++ run_tests.sh | 10 +++++- test_suite.sh | 92 +++++++++++++++++++++++++++++------------------ 3 files changed, 132 insertions(+), 36 deletions(-) create mode 100644 bot/check-test.sh diff --git a/bot/check-test.sh b/bot/check-test.sh new file mode 100644 index 0000000000..9fc783aa99 --- /dev/null +++ b/bot/check-test.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Dummy script that only creates test result file for the bot, without actually checking anything +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Kenneth Hoste (HPC-UGent) +# +# license: GPLv2 +# +job_dir=${PWD} +job_out="slurm-${SLURM_JOB_ID}.out" +job_test_result_file="_bot_job${SLURM_JOB_ID}.test" + +# ReFrame prints e.g. +#[----------] start processing checks +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default +#[ FAIL ] (1/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default +#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_d597cff4' +#[ OK ] (2/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default +#P: perf: 8.441 ns/day (r:0, l:None, u:None) +#[ FAIL ] (3/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default +#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_f4194106' +#[----------] all spawned checks have finished +#[ FAILED ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted) + +# We will grep for the last and final line, since this reflects the overall result +# Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails +FAILED=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_failed='\[\s*FAILED\s*\]' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") + [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + + +# Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step +# I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests +ERROR=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_error='ERROR: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") + [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + +echo "[TEST]" > ${job_test_result_file} +if [[ ${ERROR} -eq 1 ]]; then + echo "comment_description = Failure to execute test step" >> ${job_test_result_file} + echo "status = FAILURE" >> ${job_test_result_file} +elif [[ ${FAILED} -eq 1 ]]; then + echo "comment_description = EESSI test suite produced failures" >> ${job_test_result_file} +else + echo "comment_description = Test step run succesfully" >> ${job_test_result_file} + echo "status = SUCCESS" >> ${job_test_result_file} +fi + +exit 0 diff --git a/run_tests.sh b/run_tests.sh index ccbc751f22..69672f18f3 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,4 +1,12 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -./run_in_compat_layer_env.sh ./test_suite.sh "$@" + +# Note: for these tests, we _don't_ run in the compat layer env +# These tests should mimic what users do, and they are typically not in a prefix environment + +# Run eb --sanity-check-only on changed easyconfigs +# TODO: in the future we may implement this as a light first check. + +# Run the test suite +./test_suite.sh "$@" diff --git a/test_suite.sh b/test_suite.sh index fcaaa339d2..36262a6cbf 100644 --- a/test_suite.sh +++ b/test_suite.sh @@ -144,41 +144,63 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -# assume there's only one diff file that corresponds to the PR patch file -pr_diff=$(ls [0-9]*.diff | head -1) - -# "split" the file by prefixing each line belonging to the same file with the -# same number -split_file=$(awk '/^\+\+\+/{n++}{print n, " ", $0 }' ${pr_diff}) - -# determine which easystack files may have changed -changed_es_files=$(echo "${split_file}" | grep '^[0-9 ]*+++ ./eessi.*.yml$' | egrep -v 'known-issues|missing') - -# process all changed easystackfiles -for es_file_num in $(echo "${changed_es_files}" | cut -f1 -d' ') -do - # determine added lines that do not contain a yaml comment only - added_lines=$(echo "${split_file}" | grep "${es_file_num} + " | sed -e "s/^"${es_file_num}" + //" | grep -v "^[ ]*#") - # determine easyconfigs - easyconfigs=$(echo "${added_lines}" | cut -f3 -d' ') - # get easystack file name - easystack_file=$(echo "${changed_es_files}" | grep "^${es_file_num}" | sed -e "s/^"${es_file_num}" ... .\///") - echo -e "Processing easystack file ${easystack_file}...\n\n" - - # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') - - # load EasyBuild module - module load EasyBuild/${eb_version} - - echo_green "All set, let's run sanity checks for installed packages..." - - for easyconfig in ${easyconfigs}; - do - echo "Running sanity check for '${easyconfig}'..." - eb --sanity-check-only ${easyconfig} - done -done +# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version +# of the ReFrame module available in the current environment +module load ReFrame/4.3.3 +if [[ $? -eq 0 ]]; then + echo_green ">> Loaded ReFrame/4.3.3" +else + fatal_error "Failed to load the ReFrame module" +fi + +# Check ReFrame came with the hpctestlib and we can import it +python3 -c 'import hpctestlib.sciapps.gromacs' +if [[ $? -eq 0 ]]; then + echo_green "Succesfully found and imported hpctestlib.sciapps.gromas" +else + fatal_error "Failed to load hpctestlib" +fi + +# Clone the EESSI test suite +git clone https://github.com/EESSI/test-suite EESSI-test-suite +export TESTSUITEPREFIX=$PWD/EESSI-test-suite +export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH + +# Check that we can import from the testsuite +python3 -c 'import eessi.testsuite' +if [[ $? -eq 0 ]]; then + echo_green "Succesfully found and imported eessi.testsuite" +else + fatal_error "FAILED to import from eessi.testsuite in Python" +fi + +# Configure ReFrame +export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py +export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests +export RFM_CHECK_SEARCH_RECURSIVE=1 +export RFM_PREFIX=$PWD/reframe_runs + +# Check we can run reframe +reframe --version +if [[ $? -eq 0 ]]; then + echo_green "Succesfully ran reframe --version" +else + fatal_error "Failed to run ReFrame --version" +fi + +# List the tests we want to run +export REFRAME_ARGS='--tag CI --tag 1_nodes' +reframe "${REFRAME_ARGS}" --list +if [[ $? -eq 0 ]]; then + echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list" +else + fatal_error "Failed to list ReFrame tests with command: reframe ${REFRAME_ARGS} --list" +fi + +# Run all tests +reframe "${REFRAME_ARGS}" --run echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} + +exit 0 From 34fca15420fb37eeb136da55542b44b0a474a63d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 5 Feb 2024 17:35:29 +0100 Subject: [PATCH 0426/1795] Make a very simple change to an easystack file so we are able to trigger the bot build and test procedures and see if this PR actually works --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 82190071ab..4c73b5887a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -15,3 +15,4 @@ easyconfigs: options: from-pr: 19573 - scikit-learn-1.3.1-gfbf-2023a.eb + - patchelf-0.18.0-GCCcore-12.3.0.eb From 3faae9de814b96cd07896bd4042ae7b5b4b50f63 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 10:00:25 +0100 Subject: [PATCH 0427/1795] Make scripts executable --- bot/check-test.sh | 0 run_tests.sh | 0 test_suite.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bot/check-test.sh mode change 100644 => 100755 run_tests.sh mode change 100644 => 100755 test_suite.sh diff --git a/bot/check-test.sh b/bot/check-test.sh old mode 100644 new mode 100755 diff --git a/run_tests.sh b/run_tests.sh old mode 100644 new mode 100755 diff --git a/test_suite.sh b/test_suite.sh old mode 100644 new mode 100755 From 081e2b195b1dd948ae86d4f8e1634d368db3b467 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 10:30:25 +0100 Subject: [PATCH 0428/1795] Check for existence of SLURM output first --- bot/check-test.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 9fc783aa99..9308a87e8e 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -13,6 +13,17 @@ job_dir=${PWD} job_out="slurm-${SLURM_JOB_ID}.out" job_test_result_file="_bot_job${SLURM_JOB_ID}.test" +# Check that job output file is found +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" +if [[ -f ${job_out} ]]; then + SLURM=1 + [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" +else + SLURM=0 + [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" +fi + + # ReFrame prints e.g. #[----------] start processing checks #[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default @@ -39,7 +50,6 @@ if [[ ${SLURM} -eq 1 ]]; then [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi - # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step # I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests ERROR=-1 @@ -53,11 +63,13 @@ if [[ ${SLURM} -eq 1 ]]; then fi echo "[TEST]" > ${job_test_result_file} -if [[ ${ERROR} -eq 1 ]]; then - echo "comment_description = Failure to execute test step" >> ${job_test_result_file} +if [[ ${SLURM} -eq 0 ]]; then + echo "comment_description = FAILED (job output file not found)" >> ${job_test_result_file} +elif [[ ${ERROR} -eq 1 ]]; then + echo "comment_description = FAILED (test step failed to execute)" >> ${job_test_result_file} echo "status = FAILURE" >> ${job_test_result_file} elif [[ ${FAILED} -eq 1 ]]; then - echo "comment_description = EESSI test suite produced failures" >> ${job_test_result_file} + echo "comment_description = FAILED (EESSI test suite produced failures)" >> ${job_test_result_file} else echo "comment_description = Test step run succesfully" >> ${job_test_result_file} echo "status = SUCCESS" >> ${job_test_result_file} From 9b6fa731d6ac55e3e57538eda01c9c47d73300b1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 10:43:20 +0100 Subject: [PATCH 0429/1795] Clarify return messages from the bot --- bot/check-test.sh | 8 ++++---- test_suite.sh | 20 -------------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 9308a87e8e..05c7589640 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -64,14 +64,14 @@ fi echo "[TEST]" > ${job_test_result_file} if [[ ${SLURM} -eq 0 ]]; then - echo "comment_description = FAILED (job output file not found)" >> ${job_test_result_file} + echo "comment_description = :cry: FAILED (job output file not found, cannot check test results)" >> ${job_test_result_file} elif [[ ${ERROR} -eq 1 ]]; then - echo "comment_description = FAILED (test step failed to execute)" >> ${job_test_result_file} + echo "comment_description = :cry: FAILED (EESSI test suite was not run, test step itself failed to execute)" >> ${job_test_result_file} echo "status = FAILURE" >> ${job_test_result_file} elif [[ ${FAILED} -eq 1 ]]; then - echo "comment_description = FAILED (EESSI test suite produced failures)" >> ${job_test_result_file} + echo "comment_description = :cry: FAILED (EESSI test suite produced failures)" >> ${job_test_result_file} else - echo "comment_description = Test step run succesfully" >> ${job_test_result_file} + echo "comment_description = :grin: SUCCESS" >> ${job_test_result_file} echo "status = SUCCESS" >> ${job_test_result_file} fi diff --git a/test_suite.sh b/test_suite.sh index 36262a6cbf..7995f8aa3a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -78,26 +78,9 @@ else fatal_error "$EESSI_CVMFS_REPO is not available!" fi -# make sure we're in Prefix environment by checking $SHELL -if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then - echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" -else - fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" -fi - # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory export PYTHONPYCACHEPREFIX=$TMPDIR/pycache -DETECTION_PARAMETERS='' -GENERIC=0 -EB='eb' -if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then - echo_yellow ">> GENERIC build/test requested, taking appropriate measures!" - DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" - GENERIC=1 - EB='eb --optarch=GENERIC' -fi - echo ">> Determining software subdirectory to use for current build/test host..." if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) @@ -129,9 +112,6 @@ else fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" fi -echo ">> Configuring EasyBuild..." -source $TOPDIR/configure_easybuild - echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge From cb48b36e590f1d36d7a8d08df22eb538b716c70b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 11:32:15 +0100 Subject: [PATCH 0430/1795] Use EESSI_SOFTWARE_PATH instead of EASYBUILD_INSTALLPATH to set the modulepath. It should be the same, and we no longer configure EasyBuild, since we don't use it when running the test suite --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 7995f8aa3a..1464a3a5d4 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -117,7 +117,7 @@ echo ">> Setting up \$MODULEPATH..." module --force purge # ignore current $MODULEPATH entirely module unuse $MODULEPATH -module use $EASYBUILD_INSTALLPATH/modules/all +module use ${EESSI_SOFTWARE_PATH}/modules/all if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" else From d1a0219486fe4a4c1ad94039b4b74b41ebf7da1f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 12:02:12 +0100 Subject: [PATCH 0431/1795] Do git clone in a seperate script, so that that can be run in the prefix layer --- clone_eessi_test_suite.sh | 1 + run_tests.sh | 3 ++- test_suite.sh | 9 +++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100755 clone_eessi_test_suite.sh diff --git a/clone_eessi_test_suite.sh b/clone_eessi_test_suite.sh new file mode 100755 index 0000000000..f7684c8cfd --- /dev/null +++ b/clone_eessi_test_suite.sh @@ -0,0 +1 @@ +git clone https://github.com/EESSI/test-suite EESSI-test-suite diff --git a/run_tests.sh b/run_tests.sh index 69672f18f3..90185fea17 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,4 +9,5 @@ source ${base_dir}/init/eessi_defaults # TODO: in the future we may implement this as a light first check. # Run the test suite -./test_suite.sh "$@" +./run_in_compat_layer_env.sh clone_eessi_test_suite.sh +./test_suite.sh diff --git a/test_suite.sh b/test_suite.sh index 1464a3a5d4..6177f0a253 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -141,9 +141,14 @@ else fatal_error "Failed to load hpctestlib" fi -# Clone the EESSI test suite -git clone https://github.com/EESSI/test-suite EESSI-test-suite +# Cloning should already be done by clone_eessi_test_suite.sh, which runs in compat layer to have 'git' available +# git clone https://github.com/EESSI/test-suite EESSI-test-suite export TESTSUITEPREFIX=$PWD/EESSI-test-suite +if [ -d $TESTSUITEPREFIX ]; then + echo_green "Clone of the test suite $TESTSUITEPREFIX available, OK!" +else + fatal_error "Clone of the test suite $TESTSUITEPREFIX is not available!" +fi export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH # Check that we can import from the testsuite From 86d5d3d47d076fbdf0a61c57ca83fee8a1c87123 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 12:06:09 +0100 Subject: [PATCH 0432/1795] Should use the one from current dir --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 90185fea17..bcca1ee417 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,5 +9,5 @@ source ${base_dir}/init/eessi_defaults # TODO: in the future we may implement this as a light first check. # Run the test suite -./run_in_compat_layer_env.sh clone_eessi_test_suite.sh +./run_in_compat_layer_env.sh ./clone_eessi_test_suite.sh ./test_suite.sh From 7861aec18c0081c8ff24e7ed46d34f9ea52c2dae Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 12:10:56 +0100 Subject: [PATCH 0433/1795] Remove quotes, see if that helps --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 6177f0a253..1ac82a5e66 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -175,7 +175,7 @@ fi # List the tests we want to run export REFRAME_ARGS='--tag CI --tag 1_nodes' -reframe "${REFRAME_ARGS}" --list +reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list" else From a7b5ee13c841e940ab136ee2d014b4426a23baa7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 12:52:41 +0100 Subject: [PATCH 0434/1795] Apparently, we need to remove curly braces too --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 1ac82a5e66..46c3493f41 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -175,7 +175,7 @@ fi # List the tests we want to run export REFRAME_ARGS='--tag CI --tag 1_nodes' -reframe ${REFRAME_ARGS} --list +reframe $REFRAME_ARGS --list if [[ $? -eq 0 ]]; then echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list" else From f50e463e9d0d0f5d91b161805279b6a8a764b990 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 13:03:31 +0100 Subject: [PATCH 0435/1795] Make sure this actually gets reported as failure, as the ReFrame runtime fails to run the test suite (it is not an indication of tests failing themselves). --- test_suite.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 46c3493f41..8a69bdee8d 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -175,7 +175,7 @@ fi # List the tests we want to run export REFRAME_ARGS='--tag CI --tag 1_nodes' -reframe $REFRAME_ARGS --list +reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list" else @@ -184,6 +184,12 @@ fi # Run all tests reframe "${REFRAME_ARGS}" --run +if [[ $? -eq 0 ]]; then + echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." +else + fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run." +fi + echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} From f67df9bc1319053901aa506668533cbb8b99716f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 13:04:49 +0100 Subject: [PATCH 0436/1795] Fix the actual issue: make sure that two arguments are interpreted seperately by not quoting them --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 8a69bdee8d..633333890c 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -183,7 +183,7 @@ else fi # Run all tests -reframe "${REFRAME_ARGS}" --run +reframe ${REFRAME_ARGS} --run if [[ $? -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." else From 3394851c5347a045431d7826f579ad5f35f01edf Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 13:32:27 +0100 Subject: [PATCH 0437/1795] See if we can make check-test formatting more fancy --- bot/check-test.sh | 84 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 05c7589640..3fa7c1d694 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -64,15 +64,87 @@ fi echo "[TEST]" > ${job_test_result_file} if [[ ${SLURM} -eq 0 ]]; then - echo "comment_description = :cry: FAILED (job output file not found, cannot check test results)" >> ${job_test_result_file} + summary=":cry: FAILURE" + summary_details="(job output file not found, cannot check test results)"# >> ${job_test_result_file} + status="FAILURE" elif [[ ${ERROR} -eq 1 ]]; then - echo "comment_description = :cry: FAILED (EESSI test suite was not run, test step itself failed to execute)" >> ${job_test_result_file} - echo "status = FAILURE" >> ${job_test_result_file} + summary=":cry: FAILURE" + summary_details="(EESSI test suite was not run, test step itself failed to execute)"# >> ${job_test_result_file} + status="FAILURE" +# echo "status = FAILURE" >> ${job_test_result_file} elif [[ ${FAILED} -eq 1 ]]; then - echo "comment_description = :cry: FAILED (EESSI test suite produced failures)" >> ${job_test_result_file} + summary=":cry: FAILURE" + summary_details="(EESSI test suite produced failures)"# >> ${job_test_result_file} + status="FAILURE" else - echo "comment_description = :grin: SUCCESS" >> ${job_test_result_file} - echo "status = SUCCESS" >> ${job_test_result_file} + summary=":grin: SUCCESS"# >> ${job_test_result_file} + summary_details="" + status="SUCCESS" +# echo "status = SUCCESS" >> ${job_test_result_file} fi +function add_detail() { + actual=${1} + expected=${2} + success_msg="${3}" + failure_msg="${4}" + if [[ ${actual} -eq ${expected} ]]; then + success "${success_msg}" + else + failure "${failure_msg}" + fi +} + +echo "[TEST]" > ${job_result_file} +echo -n "comment_description = " >> ${job_result_file} + +# Use template for writing PR comment with details +# construct and write complete PR comment details: implements third alternative +comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT____ARTEFACTS_FMT__
    " +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" +comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " +comment_success_item_fmt=":white_check_mark: __ITEM__" +comment_failure_item_fmt=":x: __ITEM__" +comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " +comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " + +comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" + +# first construct comment_details_list, abbreviated CoDeList +# then use it to set comment_details +CoDeList="" + +success_msg="job output file ${job_out}" +failure_msg="no job output file ${job_out}" +CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_error}" +failure_msg="found message matching ${GP_error}" +CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_failed}" +failure_msg="found message matching ${GP_failed}" +CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") + +# Should not be needed for testing, I think? Maybe for loading ReFrame module... +# success_msg="no message matching ${GP_req_missing}" +# failure_msg="found message matching ${GP_req_missing}" +# CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") +# +# success_msg="found message(s) matching ${GP_no_missing}" +# failure_msg="no message matching ${GP_no_missing}" +# CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") +# +# success_msg="found message matching ${GP_tgz_created}" +# failure_msg="no message matching ${GP_tgz_created}" +# CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") + +comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" + +comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} + +# Actually writing the comment description to the result file +echo "${comment_description}" >> ${job_result_file} + exit 0 From 8e5682a47a589a2e251852ed674b71f16980ffc7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 14:51:46 +0100 Subject: [PATCH 0438/1795] Replace the env var for the result file --- bot/check-test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 3fa7c1d694..89d3d6a3c6 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -95,8 +95,8 @@ function add_detail() { fi } -echo "[TEST]" > ${job_result_file} -echo -n "comment_description = " >> ${job_result_file} +echo "[TEST]" > ${job_test_result_file} +echo -n "comment_description = " >> ${job_test_result_file} # Use template for writing PR comment with details # construct and write complete PR comment details: implements third alternative @@ -145,6 +145,6 @@ comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} # Actually writing the comment description to the result file -echo "${comment_description}" >> ${job_result_file} +echo "${comment_description}" >> ${job_test_result_file} exit 0 From 6cf0cf959844081b004595bd3a6c21329ccf256d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 14:53:21 +0100 Subject: [PATCH 0439/1795] Add missing functions --- bot/check-test.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 89d3d6a3c6..87c79b3b30 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -83,6 +83,24 @@ else # echo "status = SUCCESS" >> ${job_test_result_file} fi +function print_br_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}
    " +} + +function success() { + format="${comment_success_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + +function failure() { + format="${comment_failure_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + function add_detail() { actual=${1} expected=${2} From 7fe24e13cdef3f703b51930423a2296e3dc07d59 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 15:14:50 +0100 Subject: [PATCH 0440/1795] Remove artefacts from reporting --- bot/check-test.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 87c79b3b30..da06cd630b 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -118,13 +118,11 @@ echo -n "comment_description = " >> ${job_test_result_file} # Use template for writing PR comment with details # construct and write complete PR comment details: implements third alternative -comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT____ARTEFACTS_FMT__
    " +comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT__
    " comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " comment_success_item_fmt=":white_check_mark: __ITEM__" comment_failure_item_fmt=":x: __ITEM__" -comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " -comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" From afc309a168048e54b625ec857a5d09a14ca1d585 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 15:25:33 +0100 Subject: [PATCH 0441/1795] Add a reason for the failure --- bot/check-test.sh | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index da06cd630b..38e86efccd 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -65,24 +65,37 @@ fi echo "[TEST]" > ${job_test_result_file} if [[ ${SLURM} -eq 0 ]]; then summary=":cry: FAILURE" - summary_details="(job output file not found, cannot check test results)"# >> ${job_test_result_file} + summary_details="Reason: job output file not found, cannot check test results." status="FAILURE" elif [[ ${ERROR} -eq 1 ]]; then summary=":cry: FAILURE" - summary_details="(EESSI test suite was not run, test step itself failed to execute)"# >> ${job_test_result_file} + summary_details="Reason: EESSI test suite was not run, test step itself failed to execute." status="FAILURE" -# echo "status = FAILURE" >> ${job_test_result_file} elif [[ ${FAILED} -eq 1 ]]; then summary=":cry: FAILURE" - summary_details="(EESSI test suite produced failures)"# >> ${job_test_result_file} + summary_details="Reason: EESSI test suite produced failures." status="FAILURE" else - summary=":grin: SUCCESS"# >> ${job_test_result_file} + summary=":grin: SUCCESS" summary_details="" status="SUCCESS" -# echo "status = SUCCESS" >> ${job_test_result_file} fi + +echo "[TEST]" > ${job_test_result_file} +echo -n "comment_description = " >> ${job_test_result_file} + +# Use template for writing PR comment with details +# construct and write complete PR comment details: implements third alternative +comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT__
    " +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" +comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " +comment_success_item_fmt=":white_check_mark: __ITEM__" +comment_failure_item_fmt=":x: __ITEM__" + +comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" + +# Declare functions function print_br_item() { format="${1}" item="${2}" @@ -113,22 +126,11 @@ function add_detail() { fi } -echo "[TEST]" > ${job_test_result_file} -echo -n "comment_description = " >> ${job_test_result_file} - -# Use template for writing PR comment with details -# construct and write complete PR comment details: implements third alternative -comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT__
    " -comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" -comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " -comment_success_item_fmt=":white_check_mark: __ITEM__" -comment_failure_item_fmt=":x: __ITEM__" - -comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" - # first construct comment_details_list, abbreviated CoDeList # then use it to set comment_details -CoDeList="" + +# Initialize with summary_details, which elaborates on the reason for failure +CoDeList=$(print_br_item "__ITEM__" "${summary_details}" success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" From b84e48755dfb582a5a4bae3480e8865df179331e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 15:32:07 +0100 Subject: [PATCH 0442/1795] Fixed missing bracket --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 38e86efccd..e340907bb3 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -130,7 +130,7 @@ function add_detail() { # then use it to set comment_details # Initialize with summary_details, which elaborates on the reason for failure -CoDeList=$(print_br_item "__ITEM__" "${summary_details}" +CoDeList=$(print_br_item "__ITEM__" "${summary_details}") success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" From ef7bc0142f0ff518e82121371403c4da9bc0444d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 15:48:44 +0100 Subject: [PATCH 0443/1795] Add reporting of the ReFrame result --- bot/check-test.sh | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index e340907bb3..78c153e5dc 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -43,11 +43,11 @@ fi FAILED=-1 if [[ ${SLURM} -eq 1 ]]; then GP_failed='\[\s*FAILED\s*\]' - grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") + grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" - [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}" fi # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step @@ -62,6 +62,15 @@ if [[ ${SLURM} -eq 1 ]]; then [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi +# Grep for the success pattern, so we can report the amount of tests run +if [[ ${SLURM} -eq 1 ]]; then + GP_success='\[\s*PASSED\s*\]' + grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}" +fi + echo "[TEST]" > ${job_test_result_file} if [[ ${SLURM} -eq 0 ]]; then summary=":cry: FAILURE" @@ -132,6 +141,11 @@ function add_detail() { # Initialize with summary_details, which elaborates on the reason for failure CoDeList=$(print_br_item "__ITEM__" "${summary_details}") +# Add final ReFrame line as line +if [[ ! -z ${grep_reframe_result} ]]; then + CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}" +fi + success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") @@ -140,8 +154,8 @@ success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") -success_msg="no message matching ${GP_failed}" -failure_msg="found message matching ${GP_failed}" +success_msg="no message matching ""${GP_failed}""" +failure_msg="found message matching ""${GP_failed}""" CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") # Should not be needed for testing, I think? Maybe for loading ReFrame module... From 867681a2ed7413b8ecdbf646d05a20d7d2553adc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 16:01:40 +0100 Subject: [PATCH 0444/1795] Add missing bracket. Again --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 78c153e5dc..7ff5f8a3ea 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -143,7 +143,7 @@ CoDeList=$(print_br_item "__ITEM__" "${summary_details}") # Add final ReFrame line as line if [[ ! -z ${grep_reframe_result} ]]; then - CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}" + CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}") fi success_msg="job output file ${job_out}" From 47e30cac825362f85138e1d6cb3fe8ba02f13c6e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 16:17:25 +0100 Subject: [PATCH 0445/1795] Add some echo for debugging --- bot/check-test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 7ff5f8a3ea..1a2d8b5d5d 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -171,6 +171,9 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") # failure_msg="no message matching ${GP_tgz_created}" # CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +echo "Finale CoDeList is:" +echo ${CoDeList} + comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} From 1bdadf884e8f18ec911f8961a894125a6c5cccce Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 16:24:34 +0100 Subject: [PATCH 0446/1795] more debugging output --- bot/check-test.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 1a2d8b5d5d..841b8ed01b 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -137,14 +137,20 @@ function add_detail() { # first construct comment_details_list, abbreviated CoDeList # then use it to set comment_details +CoDeList="" # Initialize with summary_details, which elaborates on the reason for failure -CoDeList=$(print_br_item "__ITEM__" "${summary_details}") +if [[ ! -z ${summary_details} ]]; then + CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}") +fi # Add final ReFrame line as line if [[ ! -z ${grep_reframe_result} ]]; then CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}") fi +echo "CoDeList up here is" +echo ${CoDeList} + success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" From 5152f5e47380c462c234566f5fc3c29922694ef1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 16:54:02 +0100 Subject: [PATCH 0447/1795] Lets try some nicer formatting with headers etc. See if this actually prints something. --- bot/check-test.sh | 62 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 841b8ed01b..8f8eb66f5e 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -74,19 +74,19 @@ fi echo "[TEST]" > ${job_test_result_file} if [[ ${SLURM} -eq 0 ]]; then summary=":cry: FAILURE" - summary_details="Reason: job output file not found, cannot check test results." + reason="Reason: job output file not found, cannot check test results." status="FAILURE" elif [[ ${ERROR} -eq 1 ]]; then summary=":cry: FAILURE" - summary_details="Reason: EESSI test suite was not run, test step itself failed to execute." + reason="Reason: EESSI test suite was not run, test step itself failed to execute." status="FAILURE" elif [[ ${FAILED} -eq 1 ]]; then summary=":cry: FAILURE" - summary_details="Reason: EESSI test suite produced failures." + reason="Reason: EESSI test suite produced failures." status="FAILURE" else summary=":grin: SUCCESS" - summary_details="" + reason="" status="SUCCESS" fi @@ -96,13 +96,36 @@ echo -n "comment_description = " >> ${job_test_result_file} # Use template for writing PR comment with details # construct and write complete PR comment details: implements third alternative -comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT__
    " -comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" -comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " +comment_template="
    __SUMMARY_FMT__
    __REASON_FMT____REFRAME_FMT____DETAILS_FMT__
    " comment_success_item_fmt=":white_check_mark: __ITEM__" comment_failure_item_fmt=":x: __ITEM__" +# Initialize comment_description +comment_description=${comment_template} + +# Now, start replacing template items one by one +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" +comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}} + + +# Omit this if there is no reason (e.g. because it was succesful) +if [[ -z ${reason} ]]; then + comment_reason_fmt="
    _Reason_
    __REASONS__
    " + reason_details="${comment_reason_fmt/__REASONS__/${reason}}" + comment_description=${comment_description/__REASON_FMT__/${reason_details}} +else + comment_description=${comment_description/__REASON_FMT__/""} +fi + +# Omit this if there is no reframe summary (i.e. the workflow didn't run succesfully) +if [[ -z ${grep_reframe_result} ]]; then + comment_reframe_fmt="
    _ReFrame Summary_
    __REFRAME_SUMMARY__
    " + reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}} + comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}} +else + comment_description=${comment_description/__REFRAME_FMT__/""} +fi # Declare functions function print_br_item() { @@ -139,17 +162,17 @@ function add_detail() { # then use it to set comment_details CoDeList="" -# Initialize with summary_details, which elaborates on the reason for failure -if [[ ! -z ${summary_details} ]]; then - CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}") -fi - -# Add final ReFrame line as line -if [[ ! -z ${grep_reframe_result} ]]; then - CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}") -fi -echo "CoDeList up here is" -echo ${CoDeList} +# # Initialize with summary_details, which elaborates on the reason for failure +# if [[ ! -z ${summary_details} ]]; then +# CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}") +# fi +# +# # Add final ReFrame line as line +# if [[ ! -z ${grep_reframe_result} ]]; then +# CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}") +# fi +# echo "CoDeList up here is" +# echo ${CoDeList} success_msg="job output file ${job_out}" @@ -180,9 +203,8 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") echo "Finale CoDeList is:" echo ${CoDeList} +comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" - -comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} # Actually writing the comment description to the result file From ea38c66114af601071e1edc626cf9952219a7a18 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 17:07:28 +0100 Subject: [PATCH 0448/1795] Inverse check, it was incorrect --- bot/check-test.sh | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 8f8eb66f5e..01769bbac3 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -109,8 +109,8 @@ comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}} -# Omit this if there is no reason (e.g. because it was succesful) -if [[ -z ${reason} ]]; then +# Only add if there is a reason (e.g. no reason for successful runs) +if [[ ! -z ${reason} ]]; then comment_reason_fmt="
    _Reason_
    __REASONS__
    " reason_details="${comment_reason_fmt/__REASONS__/${reason}}" comment_description=${comment_description/__REASON_FMT__/${reason_details}} @@ -118,8 +118,8 @@ else comment_description=${comment_description/__REASON_FMT__/""} fi -# Omit this if there is no reframe summary (i.e. the workflow didn't run succesfully) -if [[ -z ${grep_reframe_result} ]]; then +# Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully) +if [[ ! -z ${grep_reframe_result} ]]; then comment_reframe_fmt="
    _ReFrame Summary_
    __REFRAME_SUMMARY__
    " reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}} comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}} @@ -162,19 +162,6 @@ function add_detail() { # then use it to set comment_details CoDeList="" -# # Initialize with summary_details, which elaborates on the reason for failure -# if [[ ! -z ${summary_details} ]]; then -# CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}") -# fi -# -# # Add final ReFrame line as line -# if [[ ! -z ${grep_reframe_result} ]]; then -# CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}") -# fi -# echo "CoDeList up here is" -# echo ${CoDeList} - - success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") From 8edb6fffb401f0e74e0b975b5585d3f9257f88f8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 17:11:41 +0100 Subject: [PATCH 0449/1795] Remove debugging output --- bot/check-test.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 01769bbac3..7a663f35d1 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -187,9 +187,6 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") # failure_msg="no message matching ${GP_tgz_created}" # CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") -echo "Finale CoDeList is:" -echo ${CoDeList} - comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} From 26bd2cb04af381af580805b094f69c096ea40410 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 17:33:12 +0100 Subject: [PATCH 0450/1795] Correct the regex pattern to account for the fact that the slurm output file contains special characters in that pattern that are NOT whitespace --- bot/check-test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 7a663f35d1..aebd56d333 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -42,7 +42,7 @@ fi # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails FAILED=-1 if [[ ${SLURM} -eq 1 ]]; then - GP_failed='\[\s*FAILED\s*\]' + GP_failed='\[.*FAILED.*\].*Ran .* test case' grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 # have to be careful to not add searched for pattern into slurm out file @@ -64,7 +64,7 @@ fi # Grep for the success pattern, so we can report the amount of tests run if [[ ${SLURM} -eq 1 ]]; then - GP_success='\[\s*PASSED\s*\]' + GP_success='\[.*PASSED.*\].*Ran .* test case' grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" @@ -119,6 +119,8 @@ else fi # Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully) +echo "ReFrame result:" +echo "${grep_reframe_result}" if [[ ! -z ${grep_reframe_result} ]]; then comment_reframe_fmt="
    _ReFrame Summary_
    __REFRAME_SUMMARY__
    " reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}} From 416b72d3cb18d6150e0ef8760cd5b66867e29496 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 17:45:30 +0100 Subject: [PATCH 0451/1795] Corrected typo --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index aebd56d333..702add848d 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -123,7 +123,7 @@ echo "ReFrame result:" echo "${grep_reframe_result}" if [[ ! -z ${grep_reframe_result} ]]; then comment_reframe_fmt="
    _ReFrame Summary_
    __REFRAME_SUMMARY__
    " - reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}} + reframe_summary=${comment_reframe_fmt/__REFRAME_SUMMARY__/${grep_reframe_result}} comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}} else comment_description=${comment_description/__REFRAME_FMT__/""} From 19900937361e16259dce714c9e4acda4b09e0a7b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 17:51:27 +0100 Subject: [PATCH 0452/1795] Trigger failure in the workflow on purpose, to show what that looks like --- test_suite.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 633333890c..4439821d56 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -166,7 +166,7 @@ export RFM_CHECK_SEARCH_RECURSIVE=1 export RFM_PREFIX=$PWD/reframe_runs # Check we can run reframe -reframe --version +reframe --version BLABLABLA if [[ $? -eq 0 ]]; then echo_green "Succesfully ran reframe --version" else @@ -174,7 +174,7 @@ else fi # List the tests we want to run -export REFRAME_ARGS='--tag CI --tag 1_nodes' +export REFRAME_ARGS='--tag CI --tag 1_node' reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list" From 4b9885612fa13d2d1f45522ca1aa04ec333863b8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 6 Feb 2024 18:11:30 +0100 Subject: [PATCH 0453/1795] Change order of checking error codes. If the ReFrame tests ran and we find that pattern, that should take priority. Other errors might have come from errors in the build phase... --- bot/check-test.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 702add848d..7b7ada98e7 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -62,10 +62,12 @@ if [[ ${SLURM} -eq 1 ]]; then [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi +SUCCESS=-1 # Grep for the success pattern, so we can report the amount of tests run if [[ ${SLURM} -eq 1 ]]; then GP_success='\[.*PASSED.*\].*Ran .* test case' grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") + [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}" @@ -74,19 +76,27 @@ fi echo "[TEST]" > ${job_test_result_file} if [[ ${SLURM} -eq 0 ]]; then summary=":cry: FAILURE" - reason="Reason: job output file not found, cannot check test results." + reason="Job output file not found, cannot check test results." status="FAILURE" -elif [[ ${ERROR} -eq 1 ]]; then +# Should come before general errors: if SUCCESS==1, it indicates the test suite ran succesfully +# regardless of other things that might have gone wrong +elif [[ ${SUCCESS} -eq 1 ]]; then + summary=":grin: SUCCESS" + reason="" + status="SUCCESS" +# Should come before general errors: if FAILED==1, it indicates the test suite ran +# otherwise the pattern wouldn't have been there +elif [[ ${FAILED} -eq 1 ]]; then summary=":cry: FAILURE" - reason="Reason: EESSI test suite was not run, test step itself failed to execute." + reason="EESSI test suite produced failures." status="FAILURE" -elif [[ ${FAILED} -eq 1 ]]; then +elif [[ ${ERROR} -eq 1 ]]; then summary=":cry: FAILURE" - reason="Reason: EESSI test suite produced failures." + reason="EESSI test suite was not run, test step itself failed to execute." status="FAILURE" else - summary=":grin: SUCCESS" - reason="" + summary=":grin: FAILURE" + reason="Failed for unknown reason" status="SUCCESS" fi From 0f261420106862049091494d6a640a6c46de5dd8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 12:11:25 +0100 Subject: [PATCH 0454/1795] Make sure we don't overwrite the grepped result from failures --- bot/check-test.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 7b7ada98e7..8af45d26a6 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -43,11 +43,11 @@ fi FAILED=-1 if [[ ${SLURM} -eq 1 ]]; then GP_failed='\[.*FAILED.*\].*Ran .* test case' - grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") + grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" - [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_failed}" fi # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step @@ -66,11 +66,17 @@ SUCCESS=-1 # Grep for the success pattern, so we can report the amount of tests run if [[ ${SLURM} -eq 1 ]]; then GP_success='\[.*PASSED.*\].*Ran .* test case' - grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") + grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" - [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_success}" +fi + +if [[ ! -z ${grep_reframe_failed} ]]; then + grep_reframe_results=${grep_reframe_failed} +else + grep_reframe_results=${grep_reframe_success} fi echo "[TEST]" > ${job_test_result_file} From 94e0bb5e3b886192508d95485b221b3ec7a6ff5e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 13:09:31 +0100 Subject: [PATCH 0455/1795] Fix plural/singular typo --- bot/check-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 8af45d26a6..39901f0691 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -74,9 +74,9 @@ if [[ ${SLURM} -eq 1 ]]; then fi if [[ ! -z ${grep_reframe_failed} ]]; then - grep_reframe_results=${grep_reframe_failed} + grep_reframe_result=${grep_reframe_failed} else - grep_reframe_results=${grep_reframe_success} + grep_reframe_result=${grep_reframe_success} fi echo "[TEST]" > ${job_test_result_file} From 9e7ac1c7c4f2f64250ef504810d057e0cb3e5225 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 13:25:45 +0100 Subject: [PATCH 0456/1795] Be a bit more expressive about what we are doing --- test_suite.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 4439821d56..1610d87738 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -165,8 +165,11 @@ export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests export RFM_CHECK_SEARCH_RECURSIVE=1 export RFM_PREFIX=$PWD/reframe_runs +echo "Configured reframe with the following environment variables:" +env | grep "RFM_" + # Check we can run reframe -reframe --version BLABLABLA +reframe --version if [[ $? -eq 0 ]]; then echo_green "Succesfully ran reframe --version" else @@ -175,6 +178,7 @@ fi # List the tests we want to run export REFRAME_ARGS='--tag CI --tag 1_node' +echo "Listing tests: reframe ${REFRAME_ARGS} --list" reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list" @@ -183,6 +187,7 @@ else fi # Run all tests +echo "Running tests: reframe ${REFRAME_ARGS} --run" reframe ${REFRAME_ARGS} --run if [[ $? -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." @@ -190,7 +195,6 @@ else fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run." fi - echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} From 3a52bde23e5de6cfe4d5da170060c07d8dfc0009 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 14:28:08 +0100 Subject: [PATCH 0457/1795] Change to dedicated config file for testing by the bot in the build job --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 1610d87738..d6699878f9 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -160,7 +160,7 @@ else fi # Configure ReFrame -export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py +export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/software_layer_bot.py export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests export RFM_CHECK_SEARCH_RECURSIVE=1 export RFM_PREFIX=$PWD/reframe_runs From 159e384bdef117265e09a0a420cce3fa6d8842c8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 14:47:54 +0100 Subject: [PATCH 0458/1795] Make it verbose, see if we can see why autodetect fails --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index d6699878f9..a8e1b2d97a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -177,7 +177,7 @@ else fi # List the tests we want to run -export REFRAME_ARGS='--tag CI --tag 1_node' +export REFRAME_ARGS='--tag CI --tag 1_node -vvv' echo "Listing tests: reframe ${REFRAME_ARGS} --list" reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then From b92044e5f913ce0a6c7082c0ebfb4e238d47f95d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 16:23:53 +0100 Subject: [PATCH 0459/1795] Add ReFrame config template. Not that four template variables need to be replaced for this to be a valid ReFrame config file --- reframe_config_bot.py | 58 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 reframe_config_bot.py diff --git a/reframe_config_bot.py b/reframe_config_bot.py new file mode 100644 index 0000000000..65ccf9f43e --- /dev/null +++ b/reframe_config_bot.py @@ -0,0 +1,58 @@ +# WARNING: this file is intended as template and the __X__ template variables need to be replaced +# before it can act as a configuration file +# Once replaced, this is a config file for running tests after the build phase, by the bot + +from eessi.testsuite.common_config import common_logging_config +from eessi.testsuite.constants import * # noqa: F403 + + +site_configuration = { + 'systems': [ + { + 'name': 'Testing in bot Build jobs for EESSI software Layer', + 'descr': 'Software-layer bot', + 'hostnames': ['.*'], + 'modules_system': 'lmod', + 'partitions': [ + { + 'name': 'default', + 'scheduler': 'local', + 'launcher': 'mpirun', + 'environs': ['default'], + 'features': [FEATURES[CPU]], + 'processor': { + 'num_cpus': __NUM_CPUS__, + 'num_sockets': __NUM_SOCKETS__, + 'num_cpus_per_core': __NUM_CPUS_PER_CORE__, + 'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__, + } + 'resources': [ + { + 'name': 'memory', + 'options': ['--mem={size}'], + } + ], + 'max_jobs': 1 + } + ] + } + ], + 'environments': [ + { + 'name': 'default', + 'cc': 'cc', + 'cxx': '', + 'ftn': '' + } + ], + 'general': [ + { + 'purge_environment': True, + 'resolve_module_conflicts': False, # avoid loading the module before submitting the job + # Enable automatic detection of CPU architecture + # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information + 'remote_detect': True, + } + ], + 'logging': common_logging_config(), +} From 02b0e3115dfc81387554f0cb9fe2c292e60098ab Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 16:30:18 +0100 Subject: [PATCH 0460/1795] Use template config file and make replacements based on lscpu --- test_suite.sh | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index a8e1b2d97a..f5cc7fabfd 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -156,11 +156,11 @@ python3 -c 'import eessi.testsuite' if [[ $? -eq 0 ]]; then echo_green "Succesfully found and imported eessi.testsuite" else - fatal_error "FAILED to import from eessi.testsuite in Python" + fatal_error "Failed to import from eessi.testsuite in Python" fi # Configure ReFrame -export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/software_layer_bot.py +export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests export RFM_CHECK_SEARCH_RECURSIVE=1 export RFM_PREFIX=$PWD/reframe_runs @@ -168,6 +168,33 @@ export RFM_PREFIX=$PWD/reframe_runs echo "Configured reframe with the following environment variables:" env | grep "RFM_" +# Inject correct CPU properties into the ReFrame config file +cpuinfo=$(lscpu) +if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then + cpu_count=${BASH_REMATCH[1]} +else + fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu." +fi +if [[ "${text}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then + socket_count=${BASH_REMATCH[1]} +else + fatal_error "Failed to get the number of sockets for the current test hardware with lscpu." +fi +if [[ "${text}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then + threads_per_core=${BASH_REMATCH[2]} +else + fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu." +fi +if [[ "${text}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then + cores_per_socket=${BASH_REMATCH[2]} +else + fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." +fi +sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES +sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES +sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES +sed -i "s/__NUM_CPUS_PER_SOCKETS__/${cores_per_socket}/g" $RFM_CONFIG_FILES + # Check we can run reframe reframe --version if [[ $? -eq 0 ]]; then From 58c4cfa772454405c456503c29a6fc8d59956910 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 16:39:46 +0100 Subject: [PATCH 0461/1795] Fix incorrect variable name --- test_suite.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index f5cc7fabfd..7ba5282327 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -175,17 +175,17 @@ if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then else fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu." fi -if [[ "${text}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then +if [[ "${cpuinfo}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then socket_count=${BASH_REMATCH[1]} else fatal_error "Failed to get the number of sockets for the current test hardware with lscpu." fi -if [[ "${text}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then +if [[ "${cpuinfo}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then threads_per_core=${BASH_REMATCH[2]} else fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu." fi -if [[ "${text}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then +if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then cores_per_socket=${BASH_REMATCH[2]} else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." From 2fc121e86399b7c7c3cab6092b609a91a88db43f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 16:57:48 +0100 Subject: [PATCH 0462/1795] Missing comma --- reframe_config_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe_config_bot.py b/reframe_config_bot.py index 65ccf9f43e..bbe8a7e2df 100644 --- a/reframe_config_bot.py +++ b/reframe_config_bot.py @@ -25,7 +25,7 @@ 'num_sockets': __NUM_SOCKETS__, 'num_cpus_per_core': __NUM_CPUS_PER_CORE__, 'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__, - } + }, 'resources': [ { 'name': 'memory', From db08f4f39a618450c70f1d52f43c46f244011545 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 21:30:43 +0100 Subject: [PATCH 0463/1795] Corrected typo --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 7ba5282327..9bd95c6749 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -193,7 +193,7 @@ fi sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES -sed -i "s/__NUM_CPUS_PER_SOCKETS__/${cores_per_socket}/g" $RFM_CONFIG_FILES +sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES # Check we can run reframe reframe --version From 6b3a6ccc985314e5c66c8097e8eb26e1c1fb3546 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 21:47:43 +0100 Subject: [PATCH 0464/1795] Remove verbose flags --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 9bd95c6749..99643c42b8 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -204,7 +204,7 @@ else fi # List the tests we want to run -export REFRAME_ARGS='--tag CI --tag 1_node -vvv' +export REFRAME_ARGS='--tag CI --tag 1_node' echo "Listing tests: reframe ${REFRAME_ARGS} --list" reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then From a70b4f132e36f078cb3b4f0d802f96a877699050 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 7 Feb 2024 21:55:16 +0100 Subject: [PATCH 0465/1795] Pick name without spaces, to avoid issues with directories --- reframe_config_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe_config_bot.py b/reframe_config_bot.py index bbe8a7e2df..ad66a72041 100644 --- a/reframe_config_bot.py +++ b/reframe_config_bot.py @@ -9,7 +9,7 @@ site_configuration = { 'systems': [ { - 'name': 'Testing in bot Build jobs for EESSI software Layer', + 'name': 'BotBuildTests', 'descr': 'Software-layer bot', 'hostnames': ['.*'], 'modules_system': 'lmod', From cee4ab62308a33a59b0837b33707870d2a2d2c6c Mon Sep 17 00:00:00 2001 From: torri Date: Thu, 8 Feb 2024 14:33:50 +0100 Subject: [PATCH 0466/1795] {2023.06}{GCCcore/12.3.0} BWA 0.7.17.20220923 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index b0205f8a85..cbffb4b2b6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -49,3 +49,6 @@ easyconfigs: - CDO-2.2.2-gompi-2023a.eb: options: from-pr: 19735 + - BWA-0.7.17.20220923-GCCcore-12.3.0.eb: + options: + from-pr: 19820 From ce24e6d6de196ca191371b186b449b7a9d4aa5c9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 17:50:39 +0100 Subject: [PATCH 0467/1795] Pick more meaningful name for SLURM variable, namely SLURM_OUTPUT_FOUND --- bot/check-build.sh | 22 +++++++++++----------- bot/check-test.sh | 18 +++++++++--------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index ec1ca56bba..42dca46a08 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -95,18 +95,18 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" -job_out="slurm-${SLURM_JOB_ID}.out" +job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out" [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" if [[ -f ${job_out} ]]; then - SLURM=1 + SLURM_OUTPUT_FOUND=1 [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" else - SLURM=0 + SLURM_OUTPUT_FOUND=0 [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi ERROR=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_error='ERROR: ' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 @@ -116,7 +116,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi FAILED=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_failed='FAILED: ' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 @@ -126,7 +126,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi MISSING=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_req_missing=' required modules missing:' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}") [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 @@ -136,7 +136,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi NO_MISSING=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_no_missing='No missing installations' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}") [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 @@ -147,7 +147,7 @@ fi TGZ=-1 TARBALL= -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_tgz_created="\.tar\.gz created!" grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) if [[ $? -eq 0 ]]; then @@ -169,9 +169,9 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" [[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -job_result_file=_bot_job${SLURM_JOB_ID}.result +job_result_file=_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.result -if [[ ${SLURM} -eq 1 ]] && \ +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ [[ ${ERROR} -eq 0 ]] && \ [[ ${FAILED} -eq 0 ]] && \ [[ ${MISSING} -eq 0 ]] && \ @@ -340,7 +340,7 @@ CoDeList="" success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" -CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") +CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" diff --git a/bot/check-test.sh b/bot/check-test.sh index 39901f0691..549afd2f65 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -10,16 +10,16 @@ # license: GPLv2 # job_dir=${PWD} -job_out="slurm-${SLURM_JOB_ID}.out" -job_test_result_file="_bot_job${SLURM_JOB_ID}.test" +job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out" +job_test_result_file="_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.test" # Check that job output file is found [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" if [[ -f ${job_out} ]]; then - SLURM=1 + SLURM_OUTPUT_FOUND=1 [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" else - SLURM=0 + SLURM_OUTPUT_FOUND=0 [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi @@ -41,7 +41,7 @@ fi # We will grep for the last and final line, since this reflects the overall result # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails FAILED=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_failed='\[.*FAILED.*\].*Ran .* test case' grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 @@ -53,7 +53,7 @@ fi # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step # I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests ERROR=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_error='ERROR: ' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 @@ -64,7 +64,7 @@ fi SUCCESS=-1 # Grep for the success pattern, so we can report the amount of tests run -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_success='\[.*PASSED.*\].*Ran .* test case' grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 @@ -80,7 +80,7 @@ else fi echo "[TEST]" > ${job_test_result_file} -if [[ ${SLURM} -eq 0 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 0 ]]; then summary=":cry: FAILURE" reason="Job output file not found, cannot check test results." status="FAILURE" @@ -182,7 +182,7 @@ CoDeList="" success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" -CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") +CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" From 9881c2ad5f93c617c075e8df81d770033ec2a121 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 17:51:36 +0100 Subject: [PATCH 0468/1795] Correct erroneous status --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 549afd2f65..9724f45832 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -103,7 +103,7 @@ elif [[ ${ERROR} -eq 1 ]]; then else summary=":grin: FAILURE" reason="Failed for unknown reason" - status="SUCCESS" + status="FAILURE" fi From 1446055cb5cd4306d61809c67b5dcda4ce930c8b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 17:53:09 +0100 Subject: [PATCH 0469/1795] Rename CoDeList variable to comment_details_list --- bot/check-build.sh | 18 +++++++++--------- bot/check-test.sh | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 42dca46a08..584d4ff828 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -334,35 +334,35 @@ echo -n "comment_description = " >> ${job_result_file} comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" -# first construct comment_details_list, abbreviated CoDeList +# first construct comment_details_list, abbreviated comment_details_list # then use it to set comment_details -CoDeList="" +comment_details_list="" success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" -CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" -CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_failed}" failure_msg="found message matching ${GP_failed}" -CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_req_missing}" failure_msg="found message matching ${GP_req_missing}" -CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") success_msg="found message(s) matching ${GP_no_missing}" failure_msg="no message matching ${GP_no_missing}" -CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") success_msg="found message matching ${GP_tgz_created}" failure_msg="no message matching ${GP_tgz_created}" -CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") -comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" +comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" # first construct comment_artefacts_list, abbreviated CoArList diff --git a/bot/check-test.sh b/bot/check-test.sh index 9724f45832..fe1fcfe85d 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -176,37 +176,37 @@ function add_detail() { fi } -# first construct comment_details_list, abbreviated CoDeList +# first construct comment_details_list, abbreviated comment_details_list # then use it to set comment_details -CoDeList="" +comment_details_list="" success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" -CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" -CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ""${GP_failed}""" failure_msg="found message matching ""${GP_failed}""" -CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") # Should not be needed for testing, I think? Maybe for loading ReFrame module... # success_msg="no message matching ${GP_req_missing}" # failure_msg="found message matching ${GP_req_missing}" -# CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") +# comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") # # success_msg="found message(s) matching ${GP_no_missing}" # failure_msg="no message matching ${GP_no_missing}" -# CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") +# comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") # # success_msg="found message matching ${GP_tgz_created}" # failure_msg="no message matching ${GP_tgz_created}" -# CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +# comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " -comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" +comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} # Actually writing the comment description to the result file From f80c0eb42f891c686cd1818ccbfc353e8bb29f05 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 17:54:00 +0100 Subject: [PATCH 0470/1795] Remove commented section, it is not relevant --- bot/check-test.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index fe1fcfe85d..9e7a524607 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -192,19 +192,6 @@ success_msg="no message matching ""${GP_failed}""" failure_msg="found message matching ""${GP_failed}""" comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") -# Should not be needed for testing, I think? Maybe for loading ReFrame module... -# success_msg="no message matching ${GP_req_missing}" -# failure_msg="found message matching ${GP_req_missing}" -# comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") -# -# success_msg="found message(s) matching ${GP_no_missing}" -# failure_msg="no message matching ${GP_no_missing}" -# comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") -# -# success_msg="found message matching ${GP_tgz_created}" -# failure_msg="no message matching ${GP_tgz_created}" -# comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") - comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} From ff5baace136c589becaf42adab32b4a0eed7afc6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 8 Feb 2024 17:57:36 +0100 Subject: [PATCH 0471/1795] Update run_tests.sh Avoid a one-line script (if possible) Co-authored-by: Kenneth Hoste --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index bcca1ee417..5e9fcb543d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,5 +9,5 @@ source ${base_dir}/init/eessi_defaults # TODO: in the future we may implement this as a light first check. # Run the test suite -./run_in_compat_layer_env.sh ./clone_eessi_test_suite.sh +./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite" ./test_suite.sh From 4adae1838edd1146d1a4a8c4322bffeb5e49db64 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 18:42:47 +0100 Subject: [PATCH 0472/1795] Fix mistake in replacing SLURM with SLURM_OUTPUT_FOUND --- bot/check-build.sh | 4 ++-- bot/check-test.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 584d4ff828..1b46652049 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -95,7 +95,7 @@ job_dir=${PWD} [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" -job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out" +job_out="slurm-${SLURM_JOB_ID}.out" [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" if [[ -f ${job_out} ]]; then SLURM_OUTPUT_FOUND=1 @@ -169,7 +169,7 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" [[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -job_result_file=_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.result +job_result_file=_bot_job${SLURM_JOB_ID}.result if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ [[ ${ERROR} -eq 0 ]] && \ diff --git a/bot/check-test.sh b/bot/check-test.sh index 9e7a524607..faf6e440fa 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -10,8 +10,8 @@ # license: GPLv2 # job_dir=${PWD} -job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out" -job_test_result_file="_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.test" +job_out="slurm-${SLURM_JOB_ID}.out" +job_test_result_file="_bot_job${SLURM__JOB_ID}.test" # Check that job output file is found [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" From 9e6955f53e70d3848ca2199756150993c36212ec Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 18:52:44 +0100 Subject: [PATCH 0473/1795] Change name of the template for the ReFrame config file to make clear it is a template. Then, alter the sed commands so that they output to a new file: the actual config file --- reframe_config_bot.py => reframe_config_bot.py.tmpl | 0 test_suite.sh | 9 +++++---- 2 files changed, 5 insertions(+), 4 deletions(-) rename reframe_config_bot.py => reframe_config_bot.py.tmpl (100%) diff --git a/reframe_config_bot.py b/reframe_config_bot.py.tmpl similarity index 100% rename from reframe_config_bot.py rename to reframe_config_bot.py.tmpl diff --git a/test_suite.sh b/test_suite.sh index 99643c42b8..b3884c3b81 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -161,6 +161,7 @@ fi # Configure ReFrame export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py +export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests export RFM_CHECK_SEARCH_RECURSIVE=1 export RFM_PREFIX=$PWD/reframe_runs @@ -190,10 +191,10 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi -sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES -sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES -sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES -sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES +sed "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES +sed "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES +sed "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES +sed "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES # Check we can run reframe reframe --version From 08a7fd8eaa64dce298624e1d4f6054d2938a7fcc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 18:55:20 +0100 Subject: [PATCH 0474/1795] Fix typo in variable --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index faf6e440fa..ef52206346 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -11,7 +11,7 @@ # job_dir=${PWD} job_out="slurm-${SLURM_JOB_ID}.out" -job_test_result_file="_bot_job${SLURM__JOB_ID}.test" +job_test_result_file="_bot_job${SLURM_JOB_ID}.test" # Check that job output file is found [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" From 8b5bde5413553c03f58ce788ef03e78ae87f343b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 18:56:07 +0100 Subject: [PATCH 0475/1795] No longer needed --- clone_eessi_test_suite.sh | 1 - 1 file changed, 1 deletion(-) delete mode 100755 clone_eessi_test_suite.sh diff --git a/clone_eessi_test_suite.sh b/clone_eessi_test_suite.sh deleted file mode 100755 index f7684c8cfd..0000000000 --- a/clone_eessi_test_suite.sh +++ /dev/null @@ -1 +0,0 @@ -git clone https://github.com/EESSI/test-suite EESSI-test-suite From 5d383eb8bc4c729cfa85d1aeae9ea10d82000c5c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 19:03:29 +0100 Subject: [PATCH 0476/1795] Added header, rephrased some comments --- run_tests.sh | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 5e9fcb543d..0c51a14807 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,13 +1,24 @@ #!/bin/bash +# +# This script gets invoked by the bot/test.sh script to run within the EESSI container +# Thus, this script defines all of the steps that should run for the tests. +# Note that, unless we have good reason, we don't run test steps in the prefix environment: +# users also typically don't run in the prefix environment, and we want to check if the +# software works well in that specific setup. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Caspar van Leeuwe (@casparvl) +# +# license: GPLv2 +# + base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -# Note: for these tests, we _don't_ run in the compat layer env -# These tests should mimic what users do, and they are typically not in a prefix environment - -# Run eb --sanity-check-only on changed easyconfigs -# TODO: in the future we may implement this as a light first check. +# Git clone has to be run in compat layer, to make the git command available +./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite" # Run the test suite -./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite" ./test_suite.sh From cdf79712184e89a31afaaff3880fc7cf0387b2e5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 19:13:11 +0100 Subject: [PATCH 0477/1795] Fix headers --- run_tests.sh | 2 +- test_suite.sh | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 0c51a14807..de7fd8c2e1 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,7 +9,7 @@ # This script is part of the EESSI software layer, see # https://github.com/EESSI/software-layer.git # -# author: Caspar van Leeuwe (@casparvl) +# author: Caspar van Leeuwen (@casparvl) # # license: GPLv2 # diff --git a/test_suite.sh b/test_suite.sh index b3884c3b81..ffe8955f1e 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -1,9 +1,14 @@ #!/bin/bash # -# Run sanity check for all requested modules (and built dependencies)? -# get ec from diff -# set up EasyBuild -# run `eb --sanity-only ec` +# This script creates a ReFrame config file from a template, in which CPU properties get replaced +# based on where this script is run (typically: a build node). Then, it runs the EESSI test suite. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Caspar van Leeuwen (@casparvl) +# +# license: GPLv2 display_help() { echo "usage: $0 [OPTIONS]" From 79a9ad320aec47ac368ed96f72e0f59a0cfbe9f5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 19:16:10 +0100 Subject: [PATCH 0478/1795] Load default ReFrame version --- test_suite.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index ffe8955f1e..1f6d8ae44d 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -129,11 +129,11 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version -# of the ReFrame module available in the current environment -module load ReFrame/4.3.3 +# Load the ReFrame module +# Currently, we load the default version. Maybe we should somehow make this configurable in the future? +module load ReFrame if [[ $? -eq 0 ]]; then - echo_green ">> Loaded ReFrame/4.3.3" + echo_green ">> Loaded ReFrame module" else fatal_error "Failed to load the ReFrame module" fi From 2f280e4304b520de295fdb408218e3125b153853 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 19:21:24 +0100 Subject: [PATCH 0479/1795] Point to docs for ReFrame config --- test_suite.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 1f6d8ae44d..d2403b91d8 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -130,7 +130,7 @@ else fi # Load the ReFrame module -# Currently, we load the default version. Maybe we should somehow make this configurable in the future? + Currently, we load the default version. Maybe we should somehow make this configurable in the future? module load ReFrame if [[ $? -eq 0 ]]; then echo_green ">> Loaded ReFrame module" @@ -139,15 +139,16 @@ else fi # Check ReFrame came with the hpctestlib and we can import it -python3 -c 'import hpctestlib.sciapps.gromacs' +reframe_import="hpctestlib.sciapps.gromacs" +python3 -c "import ${reframe_import}" if [[ $? -eq 0 ]]; then - echo_green "Succesfully found and imported hpctestlib.sciapps.gromas" + echo_green "Succesfully found and imported ${reframe_import}" else - fatal_error "Failed to load hpctestlib" + fatal_error "Failed to import ${reframe_import}" fi -# Cloning should already be done by clone_eessi_test_suite.sh, which runs in compat layer to have 'git' available -# git clone https://github.com/EESSI/test-suite EESSI-test-suite +# Cloning should already be done in run_tests.sh before test_suite.sh is invoked +# Check if that succeeded export TESTSUITEPREFIX=$PWD/EESSI-test-suite if [ -d $TESTSUITEPREFIX ]; then echo_green "Clone of the test suite $TESTSUITEPREFIX available, OK!" @@ -157,14 +158,15 @@ fi export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH # Check that we can import from the testsuite -python3 -c 'import eessi.testsuite' +testsuite_import="eessi.testsuite" +python3 -c "import ${testsuite_import}" if [[ $? -eq 0 ]]; then - echo_green "Succesfully found and imported eessi.testsuite" + echo_green "Succesfully found and imported ${testsuite_import}" else - fatal_error "Failed to import from eessi.testsuite in Python" + fatal_error "Failed to import ${testsuite_import}" fi -# Configure ReFrame +# Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests From e5d60b9c6bd2b3a39a32454a9ee6707320693c11 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 19:26:17 +0100 Subject: [PATCH 0480/1795] Fix silly mistake: I was overwriting the ReFrame config file with every sed call --- test_suite.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index d2403b91d8..755102e6bc 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -198,17 +198,18 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi -sed "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES -sed "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES -sed "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES -sed "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES +cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} +sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES +sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES +sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES +sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES # Check we can run reframe reframe --version if [[ $? -eq 0 ]]; then - echo_green "Succesfully ran reframe --version" + echo_green "Succesfully ran 'reframe --version'" else - fatal_error "Failed to run ReFrame --version" + fatal_error "Failed to run 'reframe --version'" fi # List the tests we want to run @@ -224,7 +225,8 @@ fi # Run all tests echo "Running tests: reframe ${REFRAME_ARGS} --run" reframe ${REFRAME_ARGS} --run -if [[ $? -eq 0 ]]; then +reframe_exit_code=$? +if [[ ${reframe_exit_code} -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." else fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run." @@ -233,4 +235,4 @@ fi echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} -exit 0 +exit ${reframe_exit_code} From b5c60ccfc483602eb6cebb9e1f580c9d99cd964b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 8 Feb 2024 19:28:32 +0100 Subject: [PATCH 0481/1795] See if @boegel is correct and if we don't actually need ANY software change to have the bot run... --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 4c73b5887a..82190071ab 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -15,4 +15,3 @@ easyconfigs: options: from-pr: 19573 - scikit-learn-1.3.1-gfbf-2023a.eb - - patchelf-0.18.0-GCCcore-12.3.0.eb From f925c7f9eba3c07b57c5d9276e6d2d02a3ce3e1f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 8 Feb 2024 19:50:43 +0100 Subject: [PATCH 0482/1795] rename BWA easyconfig in easystack file (uses '-' to separate version and datestamp) --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index cbffb4b2b6..7244219dc3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -49,6 +49,6 @@ easyconfigs: - CDO-2.2.2-gompi-2023a.eb: options: from-pr: 19735 - - BWA-0.7.17.20220923-GCCcore-12.3.0.eb: + - BWA-0.7.17-20220923-GCCcore-12.3.0.eb: options: from-pr: 19820 From 67232a17d0bd26debc9a584700b93ce3fc56cc16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 9 Feb 2024 16:14:13 +0100 Subject: [PATCH 0483/1795] add hook to exclude failing Highway tests on neoverse_v1 --- eb_hooks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 7c52e24ff1..f2a60c1845 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -341,6 +341,15 @@ def pre_test_hook(self,*args, **kwargs): PRE_TEST_HOOKS[self.name](self, *args, **kwargs) +def pre_test_hook_exclude_failing_tests_Highway(self, *args, **kwargs): + """ + Pre-test hook for Highway: exclude failing TestAllShiftRightLanes/SVE tests on neoverse_v1 + cfr. https://github.com/google/highway/issues/1938 + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'Highway' and self.version in ['1.0.3'] and cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg['runtest'] += ' ARGS="-E TestAllShiftRightLanes/SVE"' + def pre_test_hook_ignore_failing_tests_ESPResSo(self, *args, **kwargs): """ Pre-test hook for ESPResSo: skip failing tests, tests frequently timeout due to known bugs in ESPResSo v4.2.1 @@ -556,6 +565,7 @@ def inject_gpu_property(ec): PRE_TEST_HOOKS = { 'ESPResSo': pre_test_hook_ignore_failing_tests_ESPResSo, 'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI, + 'Highway': pre_test_hook_exclude_failing_tests_Highway, 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, } From 0d48dcabcf0360bedfaa49f2c5d711c65a08ab38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 9 Feb 2024 16:23:33 +0100 Subject: [PATCH 0484/1795] add extra blank line --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index f2a60c1845..f2a9c83d77 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -350,6 +350,7 @@ def pre_test_hook_exclude_failing_tests_Highway(self, *args, **kwargs): if self.name == 'Highway' and self.version in ['1.0.3'] and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['runtest'] += ' ARGS="-E TestAllShiftRightLanes/SVE"' + def pre_test_hook_ignore_failing_tests_ESPResSo(self, *args, **kwargs): """ Pre-test hook for ESPResSo: skip failing tests, tests frequently timeout due to known bugs in ESPResSo v4.2.1 From 8b38c322f7e76974799d658a2b9b02e65e0ca7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 9 Feb 2024 16:41:03 +0100 Subject: [PATCH 0485/1795] add known issue for Highway --- eessi-2023.06-known-issues.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 569a0d9f56..875757559e 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -16,6 +16,9 @@ - FFTW.MPI-3.3.10-gompi-2023b: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" + - Highway-1.0.3-GCCcore-12.2.0.eb: + - issue: https://github.com/EESSI/software-layer/issues/469 + - info: "failing SVE test due to wrong expected value" - netCDF-4.9.2-gompi-2023a.eb: - issue: https://github.com/EESSI/software-layer/issues/425 - info: "netCDF intermittent test failures" From 669e6e42e8a641011a7ff4a7f1c686128de461f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 9 Feb 2024 16:42:09 +0100 Subject: [PATCH 0486/1795] link to internal issue for Highway, and only exclude a single SVE test --- eb_hooks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index c7ef6de1b6..488c25c412 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -357,14 +357,14 @@ def pre_test_hook(self,*args, **kwargs): PRE_TEST_HOOKS[self.name](self, *args, **kwargs) -def pre_test_hook_exclude_failing_tests_Highway(self, *args, **kwargs): +def pre_test_hook_exclude_failing_test_Highway(self, *args, **kwargs): """ - Pre-test hook for Highway: exclude failing TestAllShiftRightLanes/SVE tests on neoverse_v1 - cfr. https://github.com/google/highway/issues/1938 + Pre-test hook for Highway: exclude failing TestAllShiftRightLanes/SVE_256 test on neoverse_v1 + cfr. https://github.com/EESSI/software-layer/issues/469 """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'Highway' and self.version in ['1.0.3'] and cpu_target == CPU_TARGET_NEOVERSE_V1: - self.cfg['runtest'] += ' ARGS="-E TestAllShiftRightLanes/SVE"' + self.cfg['runtest'] += ' ARGS="-E TestAllShiftRightLanes/SVE_256"' def pre_test_hook_ignore_failing_tests_ESPResSo(self, *args, **kwargs): @@ -604,7 +604,7 @@ def inject_gpu_property(ec): PRE_TEST_HOOKS = { 'ESPResSo': pre_test_hook_ignore_failing_tests_ESPResSo, 'FFTW.MPI': pre_test_hook_ignore_failing_tests_FFTWMPI, - 'Highway': pre_test_hook_exclude_failing_tests_Highway, + 'Highway': pre_test_hook_exclude_failing_test_Highway, 'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle, 'netCDF': pre_test_hook_ignore_failing_tests_netCDF, 'PyTorch': pre_test_hook_increase_max_failed_tests_arm_PyTorch, From 7fc04d830d372abbebcf9ec3701e38d7b3fec4b8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 11 Feb 2024 21:46:02 +0100 Subject: [PATCH 0487/1795] {2023.06}[foss/2022b] ImageMagick v7.1.0-53 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 9e92c79062..b8fddd02bb 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,3 +1,4 @@ easyconfigs: - SciPy-bundle-2023.02-gfbf-2022b.eb - GDAL-3.6.2-foss-2022b.eb + - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb From 3336f29795e9b1aca48065f5bcb0e96de996d08f Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 12 Feb 2024 16:17:47 +0100 Subject: [PATCH 0488/1795] {2023.06}[foss/2023a] LAMMPS 2Aug2023 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 82190071ab..4c17a049fd 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -15,3 +15,9 @@ easyconfigs: options: from-pr: 19573 - scikit-learn-1.3.1-gfbf-2023a.eb + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19471 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3036 + options: + from-pr: 19471 + include-easyblocks-from-pr: 3036 From 0d0385271ca0b0bcf6aef755147e1ed53df64e0c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 12 Feb 2024 21:33:55 +0100 Subject: [PATCH 0489/1795] add Lmod hook to set $OMPI_MCA_btl to '^smcuda' when loading OpenMPI module --- create_lmodrc.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/create_lmodrc.py b/create_lmodrc.py index 0e738a530e..bc69dd4396 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -94,7 +94,28 @@ end end +local function openmpi_load_hook(t) + -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1, + -- to work around hang/crash due to bug in OpenMPI; + -- see https://gitlab.com/eessi/support/-/issues/41 + local frameStk = require("FrameStk"):singleton() + local mt = frameStk:mt() + local moduleName = string.match(t.modFullName, "(.-)/") + local cpuTarget = os.getenv("EESSI_SOFTWARE_SUBDIR") or "" + if (moduleName == "OpenMPI") and (cpuTarget == "aarch64/neoverse_v1") then + local msg = "Adding '^smcuda' to $OMPI_MCA_btl to work around bug in OpenMPI" + LmodMessage(msg .. " (see https://gitlab.com/eessi/support/-/issues/41)") + local ompiMcaBtl = os.getenv("OMPI_MCA_btl") + if ompiMcaBtl == nil then + setenv("OMPI_MCA_btl", "^smcuda") + else + setenv("OMPI_MCA_btl", ompiMcaBtl .. ",^smcuda") + end + end +end + hook.register("load", cuda_enabled_load_hook) +hook.register("load", openmpi_load_hook) """ def error(msg): From a721c4b5c1ab6b53c2338bed239a9a3c5299a388 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 13 Feb 2024 17:10:59 +0100 Subject: [PATCH 0490/1795] Add snakemake 8.4.2 to foss/2023a --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 82190071ab..0a83e7be77 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -15,3 +15,6 @@ easyconfigs: options: from-pr: 19573 - scikit-learn-1.3.1-gfbf-2023a.eb + - snakemake-8.4.2-foss-2023a.eb: + options: + from-pr: 19646 From 69d6d2e227c6f2ceb0ad39068d6fe4a99d7fd24e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 14 Feb 2024 14:55:14 +0100 Subject: [PATCH 0491/1795] Add escape character to star, so that the regex pattern prints correctly on GitHub in the report --- bot/check-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index ef52206346..bc5ba84101 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -188,6 +188,8 @@ success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") +# Add an escape character to every *, for it to be printed correctly in the comment on GitHub +GP_failed="${GP_failed//\*/\\*}" success_msg="no message matching ""${GP_failed}""" failure_msg="found message matching ""${GP_failed}""" comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") From 8151ef15b47f7fdb88e646b0d5a3fd08c86c7c69 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 15 Feb 2024 11:15:19 +0100 Subject: [PATCH 0492/1795] add parse hook for LAMMPS CI + remove pre_configure_hook_LAMMPS_aarch64 see: https://github.com/easybuilders/easybuild-easyblocks/pull/3036 --- eb_hooks.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 488c25c412..c3ba03acfd 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -246,6 +246,19 @@ def parse_hook_ucx_eprefix(ec, eprefix): raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!") +def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): + """ + Remove x86_64 specific dependencies for the CI to pass on aarch64 + """ + if ec.name == 'LAMMPS': + if ec.version == '2Aug2023_update2': + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': + ec['dependencies'].remove(('ScaFaCoS', '1.0.4')) + ec['dependencies'].remove(('tbb', '2021.11.0')) + else: + raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") + + def pre_configure_hook(self, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" if self.name in PRE_CONFIGURE_HOOKS: @@ -318,24 +331,6 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") -def pre_configure_hook_LAMMPS_aarch64(self, *args, **kwargs): - """ - pre-configure hook for LAMMPS: - - set kokkos_arch on Aarch64 - """ - - cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'LAMMPS': - if self.version == '23Jun2022': - if get_cpu_architecture() == AARCH64: - if cpu_target == CPU_TARGET_AARCH64_GENERIC: - self.cfg['kokkos_arch'] = 'ARM80' - else: - self.cfg['kokkos_arch'] = 'ARM81' - else: - raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") - - def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): """ pre-configure hook for at-spi2-core: @@ -586,6 +581,7 @@ def inject_gpu_property(ec): 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, + 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, } POST_PREPARE_HOOKS = { @@ -597,7 +593,6 @@ def inject_gpu_property(ec): 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, - 'LAMMPS': pre_configure_hook_LAMMPS_aarch64, 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, } From 8491f9f090f5a55441904cd1d3b4c7ba5dacab4f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 15 Feb 2024 11:58:49 +0100 Subject: [PATCH 0493/1795] Always use LMOD_RC file, and make sure loading CUDA apps always works when building --- eessi_container.sh | 6 ++++++ init/eessi_environment_variables | 16 +++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index d6e9558202..143cbcc6c8 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -457,6 +457,12 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then mkdir -p ${EESSI_USR_LOCAL_CUDA} BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" + if [[ "${NVIDIA_MODE}" == "install" ]] ; then + # We need to "trick" our LMOD_RC file to allow us to load CUDA modules even without a CUDA driver + # (this works because we build within a container and the LMOD_RC recognises that) + touch ${EESSI_TMPDIR}/libcuda.so + export SINGULARITY_CONTAINLIBS="${EESSI_TMPDIR}/libcuda.so" + fi fi fi diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index af5222e7b9..e042e8575a 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -50,6 +50,15 @@ if [ -d $EESSI_PREFIX ]; then show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR + + # Configure our LMOD_RC file + export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua" + if [ -f $LMOD_RC ]; then + show_msg "Found Lmod configuration file at $LMOD_RC" + else + error "Lmod configuration file not found at $LMOD_RC" + fi + if [ ! -z $EESSI_BASIC_ENV ]; then show_msg "Only setting up basic environment, so we're done" elif [ -d $EESSI_SOFTWARE_PATH ]; then @@ -76,13 +85,6 @@ if [ -d $EESSI_PREFIX ]; then false fi - export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua" - if [ -f $LMOD_RC ]; then - show_msg "Found Lmod configuration file at $LMOD_RC" - else - error "Lmod configuration file not found at $LMOD_RC" - fi - else error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!" fi From 2dd048e96cef85b203b76a9ca330c5c72813dbbc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Feb 2024 15:44:15 +0100 Subject: [PATCH 0494/1795] Add the list(SCALES.keys()) feature (though technically we could limit to singlenode already here...). Also, make sure color is not used for printing the ReFrame results, to avoid color-characters making it into the slurm output file --- reframe_config_bot.py.tmpl | 4 +++- test_suite.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl index ad66a72041..b2f7916e79 100644 --- a/reframe_config_bot.py.tmpl +++ b/reframe_config_bot.py.tmpl @@ -19,7 +19,9 @@ site_configuration = { 'scheduler': 'local', 'launcher': 'mpirun', 'environs': ['default'], - 'features': [FEATURES[CPU]], + 'features': [ + FEATURES[CPU] + ] + list(SCALES.keys()), 'processor': { 'num_cpus': __NUM_CPUS__, 'num_sockets': __NUM_SOCKETS__, diff --git a/test_suite.sh b/test_suite.sh index 755102e6bc..f8c46ef764 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -213,7 +213,7 @@ else fi # List the tests we want to run -export REFRAME_ARGS='--tag CI --tag 1_node' +export REFRAME_ARGS='--tag CI --tag 1_node --nocolor' echo "Listing tests: reframe ${REFRAME_ARGS} --list" reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then From 0492b9e6d836ef75126ca5e8f613c7ad7487dc2b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Feb 2024 15:45:25 +0100 Subject: [PATCH 0495/1795] Try the original regex patterns, capturing spaces more specifically, now that we got rid of color characters --- bot/check-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index bc5ba84101..4b5f7575e5 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -42,7 +42,7 @@ fi # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails FAILED=-1 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then - GP_failed='\[.*FAILED.*\].*Ran .* test case' + GP_failed='\[\s*FAILED\s*\].*Ran .* test case' grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 # have to be careful to not add searched for pattern into slurm out file @@ -65,7 +65,7 @@ fi SUCCESS=-1 # Grep for the success pattern, so we can report the amount of tests run if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then - GP_success='\[.*PASSED.*\].*Ran .* test case' + GP_success='\[\s*PASSED\s*\].*Ran .* test case' grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 # have to be careful to not add searched for pattern into slurm out file From 4b5228a2d7671ebfce15d22ca93e80bd92a38000 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 15 Feb 2024 19:35:46 +0100 Subject: [PATCH 0496/1795] {2023.06}[foss/2023a] matplotlib v3.7.2 + PyQt5 v5.15.10 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index c2b33ee91e..157a47a49e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -24,3 +24,8 @@ easyconfigs: options: from-pr: 19471 include-easyblocks-from-pr: 3036 + - matplotlib-3.7.2-gfbf-2023a.eb + - PyQt5-5.15.10-GCCcore-12.3.0.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 + options: + from-pr: 19554 From 4ec7c704d4caedd7b680b3f20ceecd39eeae33b8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 16 Feb 2024 07:29:00 +0100 Subject: [PATCH 0497/1795] fix hook for LAMMPS to filter out ScaFaCoS and tbb dependencies on aarch64/* --- eb_hooks.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index c3ba03acfd..d29a837339 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -250,11 +250,15 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI to pass on aarch64 """ - if ec.name == 'LAMMPS': - if ec.version == '2Aug2023_update2': - if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - ec['dependencies'].remove(('ScaFaCoS', '1.0.4')) - ec['dependencies'].remove(('tbb', '2021.11.0')) + if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2',): + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': + # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets, + # so remove them as dependencies for LAMMPS (they're optional); + # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/19164 + + # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000; + # we need this hook because we check for missing installations for all CPU targets + # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) + ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb')] else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") From 0f6dc766c4770e160f193aac35be546094dc7c73 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 16 Feb 2024 20:37:53 +0100 Subject: [PATCH 0498/1795] Update eessi-2023.06-eb-4.9.0-2022b.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 9e92c79062..903364a289 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,3 +1,6 @@ easyconfigs: - SciPy-bundle-2023.02-gfbf-2022b.eb - GDAL-3.6.2-foss-2022b.eb + - waLBerla-6.1-foss-2022b.eb 19324 + options: + from-pr: 19252 From 3bac7114115028c6d08b808d4201da010671c854 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 16 Feb 2024 20:41:27 +0100 Subject: [PATCH 0499/1795] Add waLBerla v6.1 foss2022b Fix easybuild PR number --- .../2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 903364a289..3bfb3588d8 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,6 +1,6 @@ easyconfigs: - SciPy-bundle-2023.02-gfbf-2022b.eb - GDAL-3.6.2-foss-2022b.eb - - waLBerla-6.1-foss-2022b.eb 19324 + - waLBerla-6.1-foss-2022b.eb options: - from-pr: 19252 + from-pr: 19324 From d6e4cecd1b791633443073e6d357ae35e9f26cc6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 17 Feb 2024 09:42:30 +0100 Subject: [PATCH 0500/1795] add missing ':' for waLBerla entry --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 3bfb3588d8..2a07133770 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,6 +1,6 @@ easyconfigs: - SciPy-bundle-2023.02-gfbf-2022b.eb - GDAL-3.6.2-foss-2022b.eb - - waLBerla-6.1-foss-2022b.eb + - waLBerla-6.1-foss-2022b.eb: options: from-pr: 19324 From ec61759dc7b8a2a39613aff55f6ae74a55dd2620 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Feb 2024 12:11:54 +0100 Subject: [PATCH 0501/1795] Remove manual setup of MODULEPATH etc, and source the regular EESSI init script to do this --- test_suite.sh | 68 +++++++++++---------------------------------------- 1 file changed, 14 insertions(+), 54 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index f8c46ef764..8a98fd59a8 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -74,63 +74,14 @@ fi TMPDIR=$(mktemp -d) echo ">> Setting up environment..." - -source $TOPDIR/init/minimal_eessi_env - -if [ -d $EESSI_CVMFS_REPO ]; then - echo_green "$EESSI_CVMFS_REPO available, OK!" -else - fatal_error "$EESSI_CVMFS_REPO is not available!" -fi - -# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory -export PYTHONPYCACHEPREFIX=$TMPDIR/pycache - -echo ">> Determining software subdirectory to use for current build/test host..." -if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" -else - echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" -fi - -# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) -# $EESSI_SILENT - don't print any messages -# $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables - -if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then - fatal_error "Failed to determine software subdirectory?!" -elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then - fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" -else - echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" -fi - -echo ">> Initializing Lmod..." -source $EPREFIX/usr/share/Lmod/init/bash -ml_version_out=$TMPDIR/ml.out -ml --version &> $ml_version_out -if [[ $? -eq 0 ]]; then - echo_green ">> Found Lmod ${LMOD_VERSION}" -else - fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" -fi - -echo ">> Setting up \$MODULEPATH..." -# make sure no modules are loaded module --force purge -# ignore current $MODULEPATH entirely -module unuse $MODULEPATH -module use ${EESSI_SOFTWARE_PATH}/modules/all -if [[ -z ${MODULEPATH} ]]; then - fatal_error "Failed to set up \$MODULEPATH?!" -else - echo_green ">> MODULEPATH set up: ${MODULEPATH}" -fi +# Make sure defaults are set for EESSI_CVMFS_REPO and EESSI_VERSION, before initializing EESSI +source $TOPDIR/init/eessi_defaults +# Initialize EESSI +source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash # Load the ReFrame module - Currently, we load the default version. Maybe we should somehow make this configurable in the future? +# Currently, we load the default version. Maybe we should somehow make this configurable in the future? module load ReFrame if [[ $? -eq 0 ]]; then echo_green ">> Loaded ReFrame module" @@ -138,6 +89,15 @@ else fatal_error "Failed to load the ReFrame module" fi +# Check that a system python3 is available +python3_found=$(which python3) +if [ -z ${python3_found} ]; then + fatal_error "No system python3 found" +else + echo_green "System python3 found:" + python3 -V +fi + # Check ReFrame came with the hpctestlib and we can import it reframe_import="hpctestlib.sciapps.gromacs" python3 -c "import ${reframe_import}" From f3f9721dac3f590768cb9dcb6fbcf26535f049d8 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 20 Feb 2024 15:12:53 +0100 Subject: [PATCH 0502/1795] {2023.06}[foss/2023b] WSClean 3.4 + DP3 6.0 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 4dd31dbd5d..fea9107dc0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -8,3 +8,33 @@ easyconfigs: - matplotlib-3.8.2-gfbf-2023b.eb: options: from-pr: 19552 + - AOFlagger-3.4.0-foss-2023b.eb: + options: + from-pr: 19840 + # Exclude Lua from `filter-deps` as the compat layer version is too old for the software + filter-deps: + - Autoconf + - Automake + - Autotools + - binutils + - bzip2 + - DBus + - flex + - gettext + - gperf + - help2man + - intltool + - libreadline + - libtool + - ncurses + - M4 + - makeinfo + - util-linux + - XZ + - zlib + - DP3-6.0-foss-2023b.eb: + options: + from-pr: 19840 + - WSClean-3.4-foss-2023b.eb: + options: + from-pr: 19840 From 9bae7f590a6bfe34e48fe20bc0be066ee407fb25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 20 Feb 2024 15:26:18 +0100 Subject: [PATCH 0503/1795] add OpenMPI from PR 19940 (and rebuild: True) --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 75dc86c31a..70fb5bfbe0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -22,10 +22,6 @@ easyconfigs: options: from-pr: 19339 - Qt5-5.15.10-GCCcore-12.3.0.eb - - gnuplot-5.4.8-GCCcore-12.3.0.eb: - options: - from-pr: 19261 - - OpenFOAM-10-foss-2023a.eb - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb - LHAPDF-6.5.4-GCC-12.3.0.eb: options: @@ -56,3 +52,11 @@ easyconfigs: - BWA-0.7.17-20220923-GCCcore-12.3.0.eb: options: from-pr: 19820 + - gnuplot-5.4.8-GCCcore-12.3.0.eb: + options: + from-pr: 19261 + - OpenMPI-4.1.5-GCC-12.3.0.eb: + options: + from-pr: 19940 + rebuild: True + - OpenFOAM-10-foss-2023a.eb From d3c1f654371126a5187d3a9e07b6def690f8725d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 20 Feb 2024 15:41:24 +0100 Subject: [PATCH 0504/1795] fix indentation --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 70fb5bfbe0..b971c5adad 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -57,6 +57,6 @@ easyconfigs: from-pr: 19261 - OpenMPI-4.1.5-GCC-12.3.0.eb: options: - from-pr: 19940 - rebuild: True + from-pr: 19940 + rebuild: True - OpenFOAM-10-foss-2023a.eb From f8cd2aa673187d6c1b346d53658932368118bd2a Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 20 Feb 2024 15:41:51 +0100 Subject: [PATCH 0505/1795] Remove AOFlagger (Lua is no longer part of filter-deps) --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index fea9107dc0..bf7e34932b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -8,30 +8,6 @@ easyconfigs: - matplotlib-3.8.2-gfbf-2023b.eb: options: from-pr: 19552 - - AOFlagger-3.4.0-foss-2023b.eb: - options: - from-pr: 19840 - # Exclude Lua from `filter-deps` as the compat layer version is too old for the software - filter-deps: - - Autoconf - - Automake - - Autotools - - binutils - - bzip2 - - DBus - - flex - - gettext - - gperf - - help2man - - intltool - - libreadline - - libtool - - ncurses - - M4 - - makeinfo - - util-linux - - XZ - - zlib - DP3-6.0-foss-2023b.eb: options: from-pr: 19840 From eafbb6b757320ee003d678989c05bba6eb28ce2f Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 20 Feb 2024 17:29:55 +0100 Subject: [PATCH 0506/1795] Include easyblocks from pr 3088 to fix Python path --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index bf7e34932b..e8496f371a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -8,9 +8,27 @@ easyconfigs: - matplotlib-3.8.2-gfbf-2023b.eb: options: from-pr: 19552 + - AOFlagger-3.4.0-foss-2023b.eb: + options: + from:pr: 19840 + include-easyblocks-from-pr: 3088 + - casacore-3.5.0-foss-2023b.eb: + options: + from:pr: 19840 + include-easyblocks-from-pr: 3088 - DP3-6.0-foss-2023b.eb: options: from-pr: 19840 + include-easyblocks-from-pr: 3088 + - IDG-1.2.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 + - EveryBeam-0.5.2-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 - WSClean-3.4-foss-2023b.eb: options: from-pr: 19840 + include-easyblocks-from-pr: 3088 From df089de7c137ecb3232eca5f4266ca7d7cf577ed Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 20 Feb 2024 17:42:44 +0100 Subject: [PATCH 0507/1795] Fix typo --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index e8496f371a..c874d7753f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -10,11 +10,11 @@ easyconfigs: from-pr: 19552 - AOFlagger-3.4.0-foss-2023b.eb: options: - from:pr: 19840 + from-pr: 19840 include-easyblocks-from-pr: 3088 - casacore-3.5.0-foss-2023b.eb: options: - from:pr: 19840 + from-pr: 19840 include-easyblocks-from-pr: 3088 - DP3-6.0-foss-2023b.eb: options: From 2ec41e17b8e6701323e2a1a2070bc7b82c417980 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Wed, 21 Feb 2024 07:49:52 +0100 Subject: [PATCH 0508/1795] List dependencies first --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index c874d7753f..d89bf6b1b0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -16,15 +16,15 @@ easyconfigs: options: from-pr: 19840 include-easyblocks-from-pr: 3088 - - DP3-6.0-foss-2023b.eb: + - IDG-1.2.e0-foss-2023b.eb: options: from-pr: 19840 include-easyblocks-from-pr: 3088 - - IDG-1.2.0-foss-2023b.eb: + - EveryBeam-0.5.2-foss-2023b.eb: options: from-pr: 19840 include-easyblocks-from-pr: 3088 - - EveryBeam-0.5.2-foss-2023b.eb: + - DP3-6.0-foss-2023b.eb: options: from-pr: 19840 include-easyblocks-from-pr: 3088 From d0ba0b724fb8ac8a4516d40731023e31fb190b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 21 Feb 2024 09:42:52 +0100 Subject: [PATCH 0509/1795] fix typo in IDG version --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index d89bf6b1b0..33acff2e23 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -16,7 +16,7 @@ easyconfigs: options: from-pr: 19840 include-easyblocks-from-pr: 3088 - - IDG-1.2.e0-foss-2023b.eb: + - IDG-1.2.0-foss-2023b.eb: options: from-pr: 19840 include-easyblocks-from-pr: 3088 From 5d797a93f21bf7e898898af0e15bc063849c0bd8 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Wed, 21 Feb 2024 11:24:06 +0100 Subject: [PATCH 0510/1795] Include Armadillo-12.8.0-foss-2023b.eb --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 33acff2e23..7d5ce34a83 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -12,6 +12,10 @@ easyconfigs: options: from-pr: 19840 include-easyblocks-from-pr: 3088 + - Armadillo-12.8.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 - casacore-3.5.0-foss-2023b.eb: options: from-pr: 19840 From a55826ddc0f4175e6fb94b7eb355a263d7a4b759 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Wed, 21 Feb 2024 11:24:41 +0100 Subject: [PATCH 0511/1795] Revert "Include Armadillo-12.8.0-foss-2023b.eb" This reverts commit b72730eecd5265829b58e8b09c7476d460bb011a. --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 7d5ce34a83..33acff2e23 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -12,10 +12,6 @@ easyconfigs: options: from-pr: 19840 include-easyblocks-from-pr: 3088 - - Armadillo-12.8.0-foss-2023b.eb: - options: - from-pr: 19840 - include-easyblocks-from-pr: 3088 - casacore-3.5.0-foss-2023b.eb: options: from-pr: 19840 From f63a4d83c0f307f7a7928f8deaab28f09f3ecf21 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Wed, 21 Feb 2024 11:24:06 +0100 Subject: [PATCH 0512/1795] Include Armadillo-12.8.0-foss-2023b.eb --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 33acff2e23..7d5ce34a83 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -12,6 +12,10 @@ easyconfigs: options: from-pr: 19840 include-easyblocks-from-pr: 3088 + - Armadillo-12.8.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 - casacore-3.5.0-foss-2023b.eb: options: from-pr: 19840 From 7a0acebeb3826ba3bbb6172da344d906c60f2a1c Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Wed, 21 Feb 2024 15:35:54 +0100 Subject: [PATCH 0513/1795] Include arpack-ng-3.9.0-foss-2023b.eb --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 7d5ce34a83..07c2af56a7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -12,6 +12,10 @@ easyconfigs: options: from-pr: 19840 include-easyblocks-from-pr: 3088 + - arpack-ng-3.9.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 - Armadillo-12.8.0-foss-2023b.eb: options: from-pr: 19840 From 5c546ea8bff9692f4f6fbd68c0951bc29ff78f05 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Thu, 22 Feb 2024 18:05:46 +0100 Subject: [PATCH 0514/1795] Disable 'vectorize' toolchain option for casacore 3.5.0 on aarch64/neoverse_v1 with eb_hook --- eb_hooks.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index d29a837339..42f67278a8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -21,7 +21,7 @@ CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' -CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' +CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -159,6 +159,28 @@ def post_prepare_hook(self, *args, **kwargs): if self.name in POST_PREPARE_HOOKS: POST_PREPARE_HOOKS[self.name](self, *args, **kwargs) +def parse_hook_casacore_disable_vectorize(ec, eprefix): + """ + Disable 'vectorize' toolchain option for casacore 3.5.0 on aarch64/neoverse_v1 + Compiling casacore 3.5.0 with GCC 13.2.0 (foss-2023) gives an error when building for aarch64/neoverse_v1. + See also, https://github.com/EESSI/software-layer/pull/479 + """ + if ec.name == 'casacore': + tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] + if ( + LooseVersion(ec.version) == LooseVersion('3.5.0') and + tcname == 'foss' and tcversion == '2023b' + ): + if get_cpu_architecture() == CPU_TARGET_NEOVERSE_V1: + ec['toolchainopts']['vectorize'] = False + print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) + else: + print_msg("Not changing option vectorize for %s on non-AARCH64", ec.name) + else: + print_msg("Not changing option vectorize for %s %s %s", ec.name, ec.version, ec.toolchain) + + else: + raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!") def parse_hook_cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" @@ -327,7 +349,7 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): if LooseVersion(self.version) <= LooseVersion('3.9.0'): self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) - + if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'): self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) @@ -414,7 +436,7 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: - self.cfg['testopts'] = "|| echo ignoring failing tests" + self.cfg['testopts'] = "|| echo ignoring failing tests" def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): """ @@ -579,6 +601,7 @@ def inject_gpu_property(ec): PARSE_HOOKS = { + 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, From 5b822a3e2292e80077956b0a3f7c9e453a1d25cd Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Thu, 22 Feb 2024 18:09:02 +0100 Subject: [PATCH 0515/1795] Remove empty line --- eb_hooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 42f67278a8..2cc9f63ad4 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -178,7 +178,6 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): print_msg("Not changing option vectorize for %s on non-AARCH64", ec.name) else: print_msg("Not changing option vectorize for %s %s %s", ec.name, ec.version, ec.toolchain) - else: raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!") From 889243779f41ee43c68079c2bd98133857d75e98 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Thu, 22 Feb 2024 18:11:03 +0100 Subject: [PATCH 0516/1795] Cleanup code --- eb_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 2cc9f63ad4..448d6fe62c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -159,10 +159,11 @@ def post_prepare_hook(self, *args, **kwargs): if self.name in POST_PREPARE_HOOKS: POST_PREPARE_HOOKS[self.name](self, *args, **kwargs) + def parse_hook_casacore_disable_vectorize(ec, eprefix): """ Disable 'vectorize' toolchain option for casacore 3.5.0 on aarch64/neoverse_v1 - Compiling casacore 3.5.0 with GCC 13.2.0 (foss-2023) gives an error when building for aarch64/neoverse_v1. + Compiling casacore 3.5.0 with GCC 13.2.0 (foss-2023b) gives an error when building for aarch64/neoverse_v1. See also, https://github.com/EESSI/software-layer/pull/479 """ if ec.name == 'casacore': @@ -181,6 +182,7 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): else: raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!") + def parse_hook_cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" if ec.name == 'CGAL': From 3dcf5fae07e154858eec0f7ff866a27d00268e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 22 Feb 2024 22:38:15 +0100 Subject: [PATCH 0517/1795] add WRF-4.4.1-foss-2022b-dmpar.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 2a07133770..ac50c77b67 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -4,3 +4,4 @@ easyconfigs: - waLBerla-6.1-foss-2022b.eb: options: from-pr: 19324 + - WRF-4.4.1-foss-2022b-dmpar.eb From 4e5f2db05384b43af4ebbe4bca3b787660b21875 Mon Sep 17 00:00:00 2001 From: t1mk1k <96469032+t1mk1k@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:26:14 +0100 Subject: [PATCH 0518/1795] Retrieve cpu_target from EESSI env vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- eb_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 448d6fe62c..747ca5f601 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -172,7 +172,8 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): LooseVersion(ec.version) == LooseVersion('3.5.0') and tcname == 'foss' and tcversion == '2023b' ): - if get_cpu_architecture() == CPU_TARGET_NEOVERSE_V1: + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if cpu_target == CPU_TARGET_NEOVERSE_V1: ec['toolchainopts']['vectorize'] = False print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) else: From 354407375cc65087c5850f162077b7fa22f785f2 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Fri, 23 Feb 2024 15:30:27 +0100 Subject: [PATCH 0519/1795] Specify print message for skipping non-neoverse_v1 --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 747ca5f601..15813ec354 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -177,7 +177,7 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): ec['toolchainopts']['vectorize'] = False print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) else: - print_msg("Not changing option vectorize for %s on non-AARCH64", ec.name) + print_msg("Not changing option vectorize for %s on non-neoverse_v1", ec.name) else: print_msg("Not changing option vectorize for %s %s %s", ec.name, ec.version, ec.toolchain) else: From 0aea4b44691d237d9359e0a35991b3874b2a092b Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 26 Feb 2024 19:33:34 +0100 Subject: [PATCH 0520/1795] Create toolchainopts dict if it does not exist --- eb_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 15813ec354..d93ee37067 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -174,6 +174,8 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): ): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if cpu_target == CPU_TARGET_NEOVERSE_V1: + if not hasattr(ec, 'toolchainopts'): + ec['toolchainopts'] = {} ec['toolchainopts']['vectorize'] = False print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) else: From 07296d3589587cf1090e574513185ad6e50b3d69 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 27 Feb 2024 11:17:33 +0100 Subject: [PATCH 0521/1795] Enable lax vector conversion for DP3 --- eb_hooks.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index d93ee37067..1478f1a508 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -186,6 +186,32 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!") +def parse_hook_dp3_enable_relaxed_vector_conversions(ec, eprefix): + """ + Enable lax vector conversion for DP3 on aarch64/neoverse_v1 + Compiling DP3 6.0 with GCC 13.2.0 (foss-2023b) gives a conversion error when building for aarch64/neoverse_v1. + See also, https://github.com/EESSI/software-layer/pull/479 + """ + if ec.name == 'DP3': + tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] + if ( + LooseVersion(ec.version) == LooseVersion('6.0') and + tcname == 'foss' and tcversion == '2023b' + ): + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if cpu_target == CPU_TARGET_NEOVERSE_V1: + if not hasattr(ec, 'configopts'): + ec['configopts'] = "" + ec['configopts'] += ' -DCMAKE_CXX_FLAGS="$CXXFLAGS -flax-vector-conversions" ' + print_msg("Changed configopts for %s: %s", ec.name, ec['configopts']) + else: + print_msg("Not changing configopts for %s on non-neoverse_v1", ec.name) + else: + print_msg("Not changing configopts for %s %s %s", ec.name, ec.version, ec.toolchain) + else: + raise EasyBuildError("DP3-specific hook triggered for non-DP3 easyconfig?!") + + def parse_hook_cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" if ec.name == 'CGAL': @@ -607,6 +633,7 @@ def inject_gpu_property(ec): PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, + 'DP3': parse_hook_dp3_enable_relaxed_vector_conversions, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, From 793e9f8ec925aa09b020f72d4acffa1ce0b400eb Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 28 Feb 2024 13:20:54 +0000 Subject: [PATCH 0522/1795] {2023.06}[foss/2023a] dask v2023.9.2 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + eb_hooks.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 157a47a49e..d444b1f8ae 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -29,3 +29,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 options: from-pr: 19554 + - dask-2023.9.2-foss-2023a.eb diff --git a/eb_hooks.py b/eb_hooks.py index d29a837339..4ed72a12ac 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -204,6 +204,18 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") +def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix): + # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template + # (which will be empty if EasyBuild is not configured to use an alternate sysroot); + # see also https://gitlab.com/eessi/support/-/issues/9 + if ec.name == 'Pillow-SIMD': + ec.update('preinstallopts', """sed -i 's@"/usr/@"%(sysroot)s/usr/@g' setup.py && """) + ec.update('preinstallopts', """sed -i 's@"/lib@"%(sysroot)s/lib@g' setup.py && """) + print_msg("Using custom configure options for %s: %s", ec.name, ec['preinstallopts']) + else: + raise EasyBuildError("Pillow-SIMD-specific hook triggered for non-Pillow-SIMD easyconfig?!") + + def parse_hook_pybind11_replace_catch2(ec, eprefix): """ Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain. @@ -237,7 +249,6 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): def parse_hook_ucx_eprefix(ec, eprefix): - """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr')) @@ -582,6 +593,7 @@ def inject_gpu_property(ec): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, + 'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths, 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, From 68a766fe8637c7c90b7c667aa0f738b8807f115f Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 28 Feb 2024 13:27:38 +0000 Subject: [PATCH 0523/1795] fixed info --- eb_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 4ed72a12ac..d30700267b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -205,6 +205,7 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix): + """Make Pillow-SIMD aware of sysroot.""" # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template # (which will be empty if EasyBuild is not configured to use an alternate sysroot); # see also https://gitlab.com/eessi/support/-/issues/9 @@ -249,6 +250,7 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): def parse_hook_ucx_eprefix(ec, eprefix): + """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr')) From 3cdf80390fa3150bcde82427537776c22bf7bb25 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 28 Feb 2024 17:33:43 +0000 Subject: [PATCH 0524/1795] removed the hook and added --from-pr --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 5 ++++- eb_hooks.py | 14 -------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index d444b1f8ae..983d59cc04 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -29,4 +29,7 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 options: from-pr: 19554 - - dask-2023.9.2-foss-2023a.eb + - dask-2023.9.2-foss-2023a.eb: + options: + from-pr: 19996 + diff --git a/eb_hooks.py b/eb_hooks.py index d30700267b..d29a837339 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -204,19 +204,6 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") -def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix): - """Make Pillow-SIMD aware of sysroot.""" - # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template - # (which will be empty if EasyBuild is not configured to use an alternate sysroot); - # see also https://gitlab.com/eessi/support/-/issues/9 - if ec.name == 'Pillow-SIMD': - ec.update('preinstallopts', """sed -i 's@"/usr/@"%(sysroot)s/usr/@g' setup.py && """) - ec.update('preinstallopts', """sed -i 's@"/lib@"%(sysroot)s/lib@g' setup.py && """) - print_msg("Using custom configure options for %s: %s", ec.name, ec['preinstallopts']) - else: - raise EasyBuildError("Pillow-SIMD-specific hook triggered for non-Pillow-SIMD easyconfig?!") - - def parse_hook_pybind11_replace_catch2(ec, eprefix): """ Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain. @@ -595,7 +582,6 @@ def inject_gpu_property(ec): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, - 'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths, 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, From 117081843c09ae69e9b25eb18ecc54f6736cc16f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 10:01:17 +0100 Subject: [PATCH 0525/1795] Temporary fix for CI by pinning archspec version --- .github/workflows/tests.yml | 2 +- .github/workflows/tests_init.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e74a4e844..faa7eb82ff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,7 @@ jobs: - name: install Python packages run: | - pip install archspec + pip install archspec==0.2.2 - name: test eessi_software_subdir.py script run: | diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 38ccbbad31..053acb9730 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -22,7 +22,7 @@ jobs: - name: install Python packages run: | - pip install archspec pytest + pip install archspec==0.2.2 pytest - name: unit tests for eessi_software_subdir_for_host.py script run: From cf7705643865f85b0e73f5185952a400f41c95f6 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:39:14 +0100 Subject: [PATCH 0526/1795] remove-blanks --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 983d59cc04..7d9c331fcf 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -32,4 +32,3 @@ easyconfigs: - dask-2023.9.2-foss-2023a.eb: options: from-pr: 19996 - From 887e102631ef5fbfadffef744c7724e28a01217f Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 29 Feb 2024 13:50:41 +0100 Subject: [PATCH 0527/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 7d9c331fcf..d187d8e92c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -29,6 +29,7 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 options: from-pr: 19554 - - dask-2023.9.2-foss-2023a.eb: + - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb: options: from-pr: 19996 + - dask-2023.9.2-foss-2023a.eb From 938e52e66b529458fad9984f79bce156dc43d023 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 29 Feb 2024 14:16:17 +0100 Subject: [PATCH 0528/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index d187d8e92c..0a0a625c35 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -31,5 +31,6 @@ easyconfigs: from-pr: 19554 - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb: options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996 from-pr: 19996 - dask-2023.9.2-foss-2023a.eb From 301ab593867335700b576cc9949fe58c408325e7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 18:12:22 +0100 Subject: [PATCH 0529/1795] Make check_missing_installations.sh check against develop branch of EasyBuild --- check_missing_installations.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 5ea7c5a4f5..8845adb161 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -18,6 +18,13 @@ easystack=$1 LOCAL_TMPDIR=$(mktemp -d) +# Clone the develop branch of EasyBuild and use that to search for easyconfigs +git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs +EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs + +# All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr +grep -v from-pr ${easystack} > ${LOCAL_TMPDIR}/${easystack} + source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild @@ -27,7 +34,7 @@ ${EB:-eb} --show-config echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${LOCAL_TMPDIR}/${easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." From 93d1d7df3c0659e9886bac3cd0d4c994e7a82d09 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 18:19:46 +0100 Subject: [PATCH 0530/1795] Use filename rather than full path --- check_missing_installations.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 8845adb161..6c38e8e37f 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -23,7 +23,8 @@ git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $ EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr -grep -v from-pr ${easystack} > ${LOCAL_TMPDIR}/${easystack} +tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) +grep -v from-pr ${easystack} > ${tmp_easystack} source $TOPDIR/scripts/utils.sh @@ -34,7 +35,7 @@ ${EB:-eb} --show-config echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -${EB:-eb} --easystack ${LOCAL_TMPDIR}/${easystack} --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." From f6259dc563bcb1cad3ee962f56501872e53c0cba Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 18:26:16 +0100 Subject: [PATCH 0531/1795] export envvar --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 6c38e8e37f..c902fa8184 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -20,7 +20,7 @@ LOCAL_TMPDIR=$(mktemp -d) # Clone the develop branch of EasyBuild and use that to search for easyconfigs git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs -EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs +export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) From 78ba8d8e304ec0ae4c82721e3e5fae7e25402f91 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 1 Mar 2024 12:53:07 +0100 Subject: [PATCH 0532/1795] Adding OSU microbenchmarks with GPU support for 2023a, and one without GPU support for 2023b --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 0a0a625c35..0d8e71e86c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -34,3 +34,5 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996 from-pr: 19996 - dask-2023.9.2-foss-2023a.eb + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb + - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb From e666e2c4058a38b251697730a1551ba0ca34601a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 13:42:12 +0100 Subject: [PATCH 0533/1795] launch with --fakeroot for build jobs triggered by bot --- eessi_container.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index d6e9558202..55ce409e2b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -585,6 +585,11 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then + if [[ ! -z ${JOB_CFG_FILE} ]]; then + # always launch build jobs triggered by the job with --fakeroot, + # we drop back to a regular user in the build script + ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") + fi mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work From 34bc1c25bb0cdc926d05bcc0354069658bf73592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 13:42:36 +0100 Subject: [PATCH 0534/1795] remove existing dirs for rebuilds --- EESSI-install-software.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 69de9d1997..2803880547 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -222,6 +222,23 @@ else echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." if [ -f ${easystack_file} ]; then + if [ $(basename $(dirname ${easystack_file} = 'rebuilds' ]; then + echo_green "Software rebuild(s) requested, so determining which existing installation have to be removed..." + # we need to remove existing installation directories first, + # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) + # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) + rebuild_apps=$(${EB} --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + for app in ${rebuild_apps}; do + app_dir=${EASYBUILD_INSTALLPATH}/software/${app} + app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua + echo_yellow "Removing ${appdir} and {app_module}..." + rm -rf ${appdir} + rm -rf ${app_module} + done + fi + # drop back to a regular user + su - eessi + echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." ${EB} --easystack ${TOPDIR}/${easystack_file} --robot From 46e753208352710344e9c3f6ebb859c9e3975168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 13:46:55 +0100 Subject: [PATCH 0535/1795] rebuild all OpenMPI 4.1.x versions --- .../20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml new file mode 100644 index 0000000000..7a6d3c63a3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml @@ -0,0 +1,10 @@ +easyconfigs: + - OpenMPI-4.1.4-GCC-12.2.0.eb: + options: + from-pr: 19940 + - OpenMPI-4.1.5-GCC-12.3.0: + options: + from-pr: 19940 + - OpenMPI-4.1.6-GCC-13.2.0: + options: + from-pr: 19940 From 21959d651c35c0cf70b68c239deed30f55bb95d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 13:58:22 +0100 Subject: [PATCH 0536/1795] add some comments/links --- .../rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml index 7a6d3c63a3..042d0a214c 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml @@ -1,3 +1,8 @@ +# 2024-03-01 +# Rebuild all OpenMPI 4.1.x versions due to an issue with smcuda: +# https://github.com/open-mpi/ompi/issues/12270 +# https://github.com/open-mpi/ompi/pull/12344 +# https://github.com/easybuilders/easybuild-easyconfigs/pull/19940 easyconfigs: - OpenMPI-4.1.4-GCC-12.2.0.eb: options: From 7054a587884b2cd56a44bf871d0cd052633a6e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 13:59:35 +0100 Subject: [PATCH 0537/1795] fix syntax, add missing parentheses --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2803880547..2e3641a3ae 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -222,7 +222,7 @@ else echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." if [ -f ${easystack_file} ]; then - if [ $(basename $(dirname ${easystack_file} = 'rebuilds' ]; then + if [ $(basename $(dirname ${easystack_file})) = 'rebuilds' ]; then echo_green "Software rebuild(s) requested, so determining which existing installation have to be removed..." # we need to remove existing installation directories first, # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) From 1fd1609ec960e3b713e940076241f118677a3181 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 1 Mar 2024 14:00:03 +0100 Subject: [PATCH 0538/1795] 'which' is not working in the container, command -v is --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 8a98fd59a8..e9168184bc 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -90,7 +90,7 @@ else fi # Check that a system python3 is available -python3_found=$(which python3) +python3_found=$(command -v python3) if [ -z ${python3_found} ]; then fatal_error "No system python3 found" else From 7b78861fe6ab49547adbcf22df8c044039fbcc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:01:30 +0100 Subject: [PATCH 0539/1795] remove spaces from blank lines --- EESSI-install-software.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2e3641a3ae..25069689cc 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -208,19 +208,19 @@ if [ -z ${changed_easystacks} ]; then echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else for easystack_file in ${changed_easystacks}; do - + echo -e "Processing easystack file ${easystack_file}...\n\n" - + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') - + # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} - + ${EB} --show-config - + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." - + if [ -f ${easystack_file} ]; then if [ $(basename $(dirname ${easystack_file})) = 'rebuilds' ]; then echo_green "Software rebuild(s) requested, so determining which existing installation have to be removed..." @@ -240,10 +240,10 @@ else su - eessi echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - + ${EB} --easystack ${TOPDIR}/${easystack_file} --robot ec=$? - + # copy EasyBuild log file if EasyBuild exited with an error if [ ${ec} -ne 0 ]; then eb_last_log=$(unset EB_VERBOSE; eb --last-log) @@ -253,12 +253,12 @@ else # copy to build logs dir (with context added) copy_build_log "${eb_last_log}" "${build_logs_dir}" fi - + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} else fatal_error "Easystack file ${easystack_file} not found!" fi - + done fi From 87e5ff09fbd8720d4cad0bf60647d970cbea0c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:03:49 +0100 Subject: [PATCH 0540/1795] rename file to be able to find EB version --- ...b-4.9.0.yml => 20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/rebuilds/{20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml => 20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240301-OpenMPI-4.1.x-fix-smcuda-eb-4.9.0.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml From d07879f4c28a798e84d4e22e30e874ce766c6deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:06:48 +0100 Subject: [PATCH 0541/1795] clarify comment about fakeroot --- eessi_container.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 55ce409e2b..bedb269cfd 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -586,7 +586,8 @@ fi if [[ "${ACCESS}" == "rw" ]]; then if [[ ! -z ${JOB_CFG_FILE} ]]; then - # always launch build jobs triggered by the job with --fakeroot, + # always launch build jobs triggered by the job with --fakeroot in order to be able to remove existing installations, see: + # https://github.com/EESSI/software-layer/issues/312 # we drop back to a regular user in the build script ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") fi From 135c3c890f00ca282af5dff53fa8f48c302c7247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:08:27 +0100 Subject: [PATCH 0542/1795] add --allow-use-as-root-and-accept-consequences --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 25069689cc..85cdcbcbbf 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -227,7 +227,7 @@ else # we need to remove existing installation directories first, # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - rebuild_apps=$(${EB} --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') for app in ${rebuild_apps}; do app_dir=${EASYBUILD_INSTALLPATH}/software/${app} app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua From f1831b44d53ecd525717c2c0b1049c0c4da1bca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:14:21 +0100 Subject: [PATCH 0543/1795] add missing $ --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 85cdcbcbbf..4b87a8f0cd 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -231,7 +231,7 @@ else for app in ${rebuild_apps}; do app_dir=${EASYBUILD_INSTALLPATH}/software/${app} app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua - echo_yellow "Removing ${appdir} and {app_module}..." + echo_yellow "Removing ${appdir} and ${app_module}..." rm -rf ${appdir} rm -rf ${app_module} done From 68212a41298d9a90e95214ccbd971b6d9e2a626a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:14:51 +0100 Subject: [PATCH 0544/1795] fix app_dir variable --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 4b87a8f0cd..6b5aeb39bc 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -231,7 +231,7 @@ else for app in ${rebuild_apps}; do app_dir=${EASYBUILD_INSTALLPATH}/software/${app} app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua - echo_yellow "Removing ${appdir} and ${app_module}..." + echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${appdir} rm -rf ${app_module} done From 6bcd0e3a8e35613979ed7e45b4ca546c7a46759f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:28:15 +0100 Subject: [PATCH 0545/1795] split loop, first remove everything that needs to be removed, then drop back to regular user and do installations --- EESSI-install-software.sh | 43 +++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 6b5aeb39bc..b7350f9fde 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -207,6 +207,32 @@ changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z if [ -z ${changed_easystacks} ]; then echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else + changed_easystacks_rebuilds=$(grep "/rebuilds/" <<< ${changed_easystacks}) + for easystack_file in ${changed_easystacks_rebuilds}; do + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + + if [ -f ${easystack_file} ]; then + echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." + # we need to remove existing installation directories first, + # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) + # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) + rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + for app in ${rebuild_apps}; do + app_dir=${EASYBUILD_INSTALLPATH}/software/${app} + app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua + echo_yellow "Removing ${app_dir} and ${app_module}..." + rm -rf ${appdir} + rm -rf ${app_module} + done + done + + # drop back to a regular user + su - eessi + for easystack_file in ${changed_easystacks}; do echo -e "Processing easystack file ${easystack_file}...\n\n" @@ -222,23 +248,6 @@ else echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." if [ -f ${easystack_file} ]; then - if [ $(basename $(dirname ${easystack_file})) = 'rebuilds' ]; then - echo_green "Software rebuild(s) requested, so determining which existing installation have to be removed..." - # we need to remove existing installation directories first, - # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) - # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') - for app in ${rebuild_apps}; do - app_dir=${EASYBUILD_INSTALLPATH}/software/${app} - app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua - echo_yellow "Removing ${app_dir} and ${app_module}..." - rm -rf ${appdir} - rm -rf ${app_module} - done - fi - # drop back to a regular user - su - eessi - echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." ${EB} --easystack ${TOPDIR}/${easystack_file} --robot From c81c0dec1709cd683c571bd50537999da1bdd8ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:31:28 +0100 Subject: [PATCH 0546/1795] add missing else and fi --- EESSI-install-software.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index b7350f9fde..62cc1a3df2 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -228,6 +228,9 @@ else rm -rf ${appdir} rm -rf ${app_module} done + else + fatal_error "Easystack file ${easystack_file} not found!" + fi done # drop back to a regular user From f99458395e44b9b063ad6f785e21a1f16edc16ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 14:40:54 +0100 Subject: [PATCH 0547/1795] su without hypen --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 62cc1a3df2..01ca19336c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -234,7 +234,7 @@ else done # drop back to a regular user - su - eessi + su eessi for easystack_file in ${changed_easystacks}; do From 21e3735712a7679f385a20fa767d60a261854568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 15:28:28 +0100 Subject: [PATCH 0548/1795] rerun the script with a regular user after removing installations --- EESSI-install-software.sh | 63 ++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 01ca19336c..1e517bc243 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -187,6 +187,41 @@ fi # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) +# if this script is run as root, use PR patch file to determine if software needs to be removed first +if [ $UID -eq 0 ]; then + changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') | grep "/rebuilds/ + if [ -z ${changed_easystacks} ]; then + echo "No software needs to be removed." + else + for easystack_file in ${changed_easystacks_rebuilds}; do + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + + if [ -f ${easystack_file} ]; then + echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." + # we need to remove existing installation directories first, + # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) + # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) + rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + for app in ${rebuild_apps}; do + app_dir=${EASYBUILD_INSTALLPATH}/software/${app} + app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua + echo_yellow "Removing ${app_dir} and ${app_module}..." + rm -rf ${appdir} + rm -rf ${app_module} + done + else + fatal_error "Easystack file ${easystack_file} not found!" + fi + done + fi + # now rerun the script with a regular user account to do the software installations + exec su eessi $( readlink -f "$0" ) -- "$@" +fi + # install any additional required scripts # order is important: these are needed to install a full CUDA SDK in host_injections # for now, this just reinstalls all scripts. Note the most elegant, but works @@ -207,34 +242,6 @@ changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z if [ -z ${changed_easystacks} ]; then echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else - changed_easystacks_rebuilds=$(grep "/rebuilds/" <<< ${changed_easystacks}) - for easystack_file in ${changed_easystacks_rebuilds}; do - # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') - - # load EasyBuild module (will be installed if it's not available yet) - source ${TOPDIR}/load_easybuild_module.sh ${eb_version} - - if [ -f ${easystack_file} ]; then - echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." - # we need to remove existing installation directories first, - # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) - # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') - for app in ${rebuild_apps}; do - app_dir=${EASYBUILD_INSTALLPATH}/software/${app} - app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua - echo_yellow "Removing ${app_dir} and ${app_module}..." - rm -rf ${appdir} - rm -rf ${app_module} - done - else - fatal_error "Easystack file ${easystack_file} not found!" - fi - done - - # drop back to a regular user - su eessi for easystack_file in ${changed_easystacks}; do From 4f436b346bbba403fb8c23c939423ba12b8e1f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 15:38:24 +0100 Subject: [PATCH 0549/1795] fix typo and add missing quote --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 1e517bc243..28bfb8534c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -189,8 +189,8 @@ pr_diff=$(ls [0-9]*.diff | head -1) # if this script is run as root, use PR patch file to determine if software needs to be removed first if [ $UID -eq 0 ]; then - changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') | grep "/rebuilds/ - if [ -z ${changed_easystacks} ]; then + changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') | grep "/rebuilds/" + if [ -z ${changed_easystacks_rebuilds} ]; then echo "No software needs to be removed." else for easystack_file in ${changed_easystacks_rebuilds}; do From 0676fdab6d576d0af0b2f107bbd6f92e6cbde146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 15:44:04 +0100 Subject: [PATCH 0550/1795] fix syntax --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 28bfb8534c..0f1856891e 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -189,7 +189,7 @@ pr_diff=$(ls [0-9]*.diff | head -1) # if this script is run as root, use PR patch file to determine if software needs to be removed first if [ $UID -eq 0 ]; then - changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') | grep "/rebuilds/" + changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") if [ -z ${changed_easystacks_rebuilds} ]; then echo "No software needs to be removed." else From 95d6aaa983c9a59e42ff28a6feab75f493dc2e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 15:44:23 +0100 Subject: [PATCH 0551/1795] fix typo in app_dir variable --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 0f1856891e..7f44403c73 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -210,7 +210,7 @@ if [ $UID -eq 0 ]; then app_dir=${EASYBUILD_INSTALLPATH}/software/${app} app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - rm -rf ${appdir} + rm -rf ${app_dir} rm -rf ${app_module} done else From 314178937a1876eff79300fc0b562031ffc410a8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 1 Mar 2024 15:58:01 +0100 Subject: [PATCH 0552/1795] Workaround for 'PSM3 can't open nic unit' error --- test_suite.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_suite.sh b/test_suite.sh index e9168184bc..e3bab04aec 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -164,6 +164,9 @@ sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES +# Workaround for https://github.com/EESSI/software-layer/pull/467#issuecomment-1973341966 +export PSM3_DEVICES='self,shm' # this is enough, since we only run single node for now + # Check we can run reframe reframe --version if [[ $? -eq 0 ]]; then From e52afc192fc900db8476cb4763a5655228ef9d0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 16:17:49 +0100 Subject: [PATCH 0553/1795] run removal step without Prefix --- EESSI-install-software.sh | 3 ++- install_software_layer.sh | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7f44403c73..097d0992e5 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -219,7 +219,8 @@ if [ $UID -eq 0 ]; then done fi # now rerun the script with a regular user account to do the software installations - exec su eessi $( readlink -f "$0" ) -- "$@" + #exec su eessi $( readlink -f "$0" ) -- "$@" + exit fi # install any additional required scripts diff --git a/install_software_layer.sh b/install_software_layer.sh index 82ca70b73f..85f856e86a 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,4 +1,8 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults +if [ $UID -eq 0 ]; then + ./EESSI-install-software.sh "$@" + exec runuser -u eessi $( readlink -f "$0" ) -- "$@" +fi ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" From be4c59280fdf7de49adcc8029970987201f1b551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 1 Mar 2024 16:36:38 +0100 Subject: [PATCH 0554/1795] move removal part to its own script, at least for now --- EESSI-install-software.sh | 36 -------- EESSI-remove-software.sh | 189 ++++++++++++++++++++++++++++++++++++++ install_software_layer.sh | 2 +- 3 files changed, 190 insertions(+), 37 deletions(-) create mode 100755 EESSI-remove-software.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 097d0992e5..ab91d2c6de 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -187,42 +187,6 @@ fi # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) -# if this script is run as root, use PR patch file to determine if software needs to be removed first -if [ $UID -eq 0 ]; then - changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") - if [ -z ${changed_easystacks_rebuilds} ]; then - echo "No software needs to be removed." - else - for easystack_file in ${changed_easystacks_rebuilds}; do - # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') - - # load EasyBuild module (will be installed if it's not available yet) - source ${TOPDIR}/load_easybuild_module.sh ${eb_version} - - if [ -f ${easystack_file} ]; then - echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." - # we need to remove existing installation directories first, - # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) - # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') - for app in ${rebuild_apps}; do - app_dir=${EASYBUILD_INSTALLPATH}/software/${app} - app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua - echo_yellow "Removing ${app_dir} and ${app_module}..." - rm -rf ${app_dir} - rm -rf ${app_module} - done - else - fatal_error "Easystack file ${easystack_file} not found!" - fi - done - fi - # now rerun the script with a regular user account to do the software installations - #exec su eessi $( readlink -f "$0" ) -- "$@" - exit -fi - # install any additional required scripts # order is important: these are needed to install a full CUDA SDK in host_injections # for now, this just reinstalls all scripts. Note the most elegant, but works diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh new file mode 100755 index 0000000000..89e5be977e --- /dev/null +++ b/EESSI-remove-software.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# +# Script to install EESSI software stack (version set through init/eessi_defaults) + +# see example parsing of command line arguments at +# https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds" + echo " -g | --generic - instructs script to build for generic architecture target" + echo " -h | --help - display this usage information" + echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" + echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" + echo " --shared-fs-path - path to directory on shared filesystem that can be used" +} + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -g|--generic) + EASYBUILD_OPTARCH="GENERIC" + shift + ;; + -h|--help) + display_help # Call your function + # no shifting needed here, we're done. + exit 0 + ;; + -x|--http-proxy) + export http_proxy="$2" + shift 2 + ;; + -y|--https-proxy) + export https_proxy="$2" + shift 2 + ;; + --build-logs-dir) + export build_logs_dir="${2}" + shift 2 + ;; + --shared-fs-path) + export shared_fs_path="${2}" + shift 2 + ;; + -*|--*) + echo "Error: Unknown option: $1" >&2 + exit 1 + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +TOPDIR=$(dirname $(realpath $0)) + +source $TOPDIR/scripts/utils.sh + +# honor $TMPDIR if it is already defined, use /tmp otherwise +if [ -z $TMPDIR ]; then + export WORKDIR=/tmp/$USER +else + export WORKDIR=$TMPDIR/$USER +fi + +TMPDIR=$(mktemp -d) + +echo ">> Setting up environment..." + +source $TOPDIR/init/minimal_eessi_env + +if [ -d $EESSI_CVMFS_REPO ]; then + echo_green "$EESSI_CVMFS_REPO available, OK!" +else + fatal_error "$EESSI_CVMFS_REPO is not available!" +fi + +# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory +export PYTHONPYCACHEPREFIX=$TMPDIR/pycache + +DETECTION_PARAMETERS='' +GENERIC=0 +EB='eb' +if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then + echo_yellow ">> GENERIC build requested, taking appropriate measures!" + DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" + GENERIC=1 + EB='eb --optarch=GENERIC' +fi + +echo ">> Determining software subdirectory to use for current build host..." +if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" +else + echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" +fi + +# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) +# $EESSI_SILENT - don't print any messages +# $EESSI_BASIC_ENV - give a basic set of environment variables +EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables + +if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then + fatal_error "Failed to determine software subdirectory?!" +elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" +else + echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" +fi + +echo ">> Initializing Lmod..." +source $EPREFIX/usr/share/Lmod/init/bash +ml_version_out=$TMPDIR/ml.out +ml --version &> $ml_version_out +if [[ $? -eq 0 ]]; then + echo_green ">> Found Lmod ${LMOD_VERSION}" +else + fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" +fi + +echo ">> Configuring EasyBuild..." +source $TOPDIR/configure_easybuild + +if [ ! -z "${shared_fs_path}" ]; then + shared_eb_sourcepath=${shared_fs_path}/easybuild/sources + echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" + export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} +fi + +echo ">> Setting up \$MODULEPATH..." +# make sure no modules are loaded +module --force purge +# ignore current $MODULEPATH entirely +module unuse $MODULEPATH +module use $EASYBUILD_INSTALLPATH/modules/all +if [[ -z ${MODULEPATH} ]]; then + fatal_error "Failed to set up \$MODULEPATH?!" +else + echo_green ">> MODULEPATH set up: ${MODULEPATH}" +fi + +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# if this script is run as root, use PR patch file to determine if software needs to be removed first +if [ $UID -eq 0 ]; then + changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") + if [ -z ${changed_easystacks_rebuilds} ]; then + echo "No software needs to be removed." + else + for easystack_file in ${changed_easystacks_rebuilds}; do + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + + if [ -f ${easystack_file} ]; then + echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." + # we need to remove existing installation directories first, + # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) + # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) + rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + for app in ${rebuild_apps}; do + app_dir=${EASYBUILD_INSTALLPATH}/software/${app} + app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua + echo_yellow "Removing ${app_dir} and ${app_module}..." + rm -rf ${app_dir} + rm -rf ${app_module} + done + else + fatal_error "Easystack file ${easystack_file} not found!" + fi + done + fi + # now rerun the script with a regular user account to do the software installations + #exec su eessi $( readlink -f "$0" ) -- "$@" + exit +fi + +echo ">> Cleaning up ${TMPDIR}..." +rm -r ${TMPDIR} diff --git a/install_software_layer.sh b/install_software_layer.sh index 85f856e86a..249140a503 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -2,7 +2,7 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults if [ $UID -eq 0 ]; then - ./EESSI-install-software.sh "$@" + ./EESSI-remove-software.sh "$@" exec runuser -u eessi $( readlink -f "$0" ) -- "$@" fi ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" From efbeb561740f8ef5c066e1b064c16f45087c668f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 4 Mar 2024 16:59:00 +0100 Subject: [PATCH 0555/1795] Use LMOD_CONFIG_DIR instead of LMOD_RC to allow user overrides of what we specify in our RC file, see search order on https://lmod.readthedocs.io/en/latest/145_properties.html?highlight=lmod_rc#the-properties-file-lmodrc-lua . Also, prefix our hooks with eessi_ to avoid unintentional clashes in namespace between site lmod configuration and eessi lmod configuration --- create_lmodrc.py | 8 ++++---- init/eessi_environment_variables | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index bc69dd4396..ff34906c0b 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -29,7 +29,7 @@ return content end -local function cuda_enabled_load_hook(t) +local function eessi_cuda_enabled_load_hook(t) local frameStk = require("FrameStk"):singleton() local mt = frameStk:mt() local simpleName = string.match(t.modFullName, "(.-)/") @@ -94,7 +94,7 @@ end end -local function openmpi_load_hook(t) +local function eessi_openmpi_load_hook(t) -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1, -- to work around hang/crash due to bug in OpenMPI; -- see https://gitlab.com/eessi/support/-/issues/41 @@ -114,8 +114,8 @@ end end -hook.register("load", cuda_enabled_load_hook) -hook.register("load", openmpi_load_hook) +hook.register("load", eessi_cuda_enabled_load_hook) +hook.register("load", eessi_openmpi_load_hook) """ def error(msg): diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index af5222e7b9..d3a209630e 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -76,8 +76,8 @@ if [ -d $EESSI_PREFIX ]; then false fi - export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua" - if [ -f $LMOD_RC ]; then + export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod" + if [ -f $LMOD_CONFIG_DIR/lmodrc.lua ]; then show_msg "Found Lmod configuration file at $LMOD_RC" else error "Lmod configuration file not found at $LMOD_RC" From 590cabd7c068a9986f0eba33792005a7b9ce883a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 5 Mar 2024 10:42:27 +0100 Subject: [PATCH 0556/1795] Combine functions into a single function, before registering this as a hook. Lmod only permits the registration of a single function, trying to register a second function will just overwrite the first, as can be seen from https://github.com/TACC/Lmod/blob/a97d68193a6f32ebca4b4bd70dac0f6ac8fddefe/src/Hook.lua#L87 --- create_lmodrc.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index ff34906c0b..621c8e271a 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -114,8 +114,15 @@ end end -hook.register("load", eessi_cuda_enabled_load_hook) -hook.register("load", eessi_openmpi_load_hook) +-- Combine both functions into a single one, as we can only register one function as load hook in lmod +-- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed +function eessi_load_hook(t) + eessi_cuda_enabled_load_hook(t) + eessi_openmpi_load_hook(t) +end + + +hook.register("load", eessi_load_hook) """ def error(msg): From 340b8332ffb858ba43613f38bd4ac23d260cc31e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 5 Mar 2024 14:55:32 +0100 Subject: [PATCH 0557/1795] remove su, add error message --- EESSI-remove-software.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 89e5be977e..e55efe823e 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -180,10 +180,10 @@ if [ $UID -eq 0 ]; then fi done fi - # now rerun the script with a regular user account to do the software installations - #exec su eessi $( readlink -f "$0" ) -- "$@" - exit +else + fatal_error "This script can only be run by root!" fi -echo ">> Cleaning up ${TMPDIR}..." -rm -r ${TMPDIR} + +#echo ">> Cleaning up ${TMPDIR}..." +#rm -r ${TMPDIR} From 88d21d9487272abbe951477be8727165d1c01795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 5 Mar 2024 15:13:27 +0100 Subject: [PATCH 0558/1795] first run the EESSI-remove-software.sh script in a separate container session --- eessi_container.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index bedb269cfd..09cec8964f 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -585,12 +585,12 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - if [[ ! -z ${JOB_CFG_FILE} ]]; then + #if [[ ! -z ${JOB_CFG_FILE} ]]; then # always launch build jobs triggered by the job with --fakeroot in order to be able to remove existing installations, see: # https://github.com/EESSI/software-layer/issues/312 # we drop back to a regular user in the build script - ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") - fi + #ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") + #fi mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work @@ -627,6 +627,12 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi +if [[ ! -z ${JOB_CFG_FILE} ]]; then + echo "Removing software by launching container with command (next line):" + echo "singularity ${RUN_QUIET} exec --fakeroot ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ./EESSI-remove-software.sh" + singularity ${RUN_QUIET} exec --fakeroot "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ./EESSI-remove-software.sh +fi + echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" singularity ${RUN_QUIET} ${MODE} "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" From 499328d70e3c6275eec9dcaf717a34afaf75a6e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 5 Mar 2024 15:14:47 +0100 Subject: [PATCH 0559/1795] undo --- install_software_layer.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/install_software_layer.sh b/install_software_layer.sh index 249140a503..82ca70b73f 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,8 +1,4 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -if [ $UID -eq 0 ]; then - ./EESSI-remove-software.sh "$@" - exec runuser -u eessi $( readlink -f "$0" ) -- "$@" -fi ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" From 8100fed0e6a7105a4f1ffc08f6fca89f35145e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 5 Mar 2024 15:32:27 +0100 Subject: [PATCH 0560/1795] dont use TMPDIR --- EESSI-remove-software.sh | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index e55efe823e..91490507c6 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -62,15 +62,6 @@ TOPDIR=$(dirname $(realpath $0)) source $TOPDIR/scripts/utils.sh -# honor $TMPDIR if it is already defined, use /tmp otherwise -if [ -z $TMPDIR ]; then - export WORKDIR=/tmp/$USER -else - export WORKDIR=$TMPDIR/$USER -fi - -TMPDIR=$(mktemp -d) - echo ">> Setting up environment..." source $TOPDIR/init/minimal_eessi_env @@ -82,7 +73,7 @@ else fi # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory -export PYTHONPYCACHEPREFIX=$TMPDIR/pycache +#export PYTHONPYCACHEPREFIX=$TMPDIR/pycache DETECTION_PARAMETERS='' GENERIC=0 @@ -117,12 +108,11 @@ fi echo ">> Initializing Lmod..." source $EPREFIX/usr/share/Lmod/init/bash -ml_version_out=$TMPDIR/ml.out -ml --version &> $ml_version_out +ml --version if [[ $? -eq 0 ]]; then echo_green ">> Found Lmod ${LMOD_VERSION}" else - fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" + fatal_error "Failed to initialize Lmod?!" fi echo ">> Configuring EasyBuild..." @@ -183,7 +173,3 @@ if [ $UID -eq 0 ]; then else fatal_error "This script can only be run by root!" fi - - -#echo ">> Cleaning up ${TMPDIR}..." -#rm -r ${TMPDIR} From c758d74bc70d386ea8c1134d5f9edf500b48db5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 5 Mar 2024 15:42:15 +0100 Subject: [PATCH 0561/1795] use EUID instead of UID --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 91490507c6..4cbc27be98 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -140,7 +140,7 @@ fi pr_diff=$(ls [0-9]*.diff | head -1) # if this script is run as root, use PR patch file to determine if software needs to be removed first -if [ $UID -eq 0 ]; then +if [ $EUID -eq 0 ]; then changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") if [ -z ${changed_easystacks_rebuilds} ]; then echo "No software needs to be removed." From 78cc503e43cd4731fb5a8c10c4d3499613cb09f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 5 Mar 2024 15:48:15 +0100 Subject: [PATCH 0562/1795] do set/override TMPDIR --- EESSI-remove-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 4cbc27be98..819c08f8ce 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -60,6 +60,8 @@ set -- "${POSITIONAL_ARGS[@]}" TOPDIR=$(dirname $(realpath $0)) +export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX) + source $TOPDIR/scripts/utils.sh echo ">> Setting up environment..." From 0de249e32f6cb2cd1cbb694cbddf37fb93987561 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 5 Mar 2024 18:15:10 +0100 Subject: [PATCH 0563/1795] Look for site-specific lmodrc file in host_injections. If present, set LMOD_RC --- init/eessi_environment_variables | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index d3a209630e..1c8a68b1d5 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -82,6 +82,11 @@ if [ -d $EESSI_PREFIX ]; then else error "Lmod configuration file not found at $LMOD_RC" fi + # Allow for site-specific lmod configuration + host_lmod_rc="$EESSI_CVMFS_REPO"/host_injections/.lmod/lmodrc.lua + if [ -f $host_lmod_rc ]; then + export LMOD_RC="$host_lmod_rc" + fi else error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!" From c1840dd55f306b20e6d26201990313ad4dc18a3a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 6 Mar 2024 15:05:07 +0100 Subject: [PATCH 0564/1795] Next the local lmodrc within the versioned directory, e.g. /cvmfs/software.eessi.io/host_injections/2023.06/.lmod/lmodrc.lua --- init/eessi_environment_variables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 1c8a68b1d5..5407034191 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -83,7 +83,7 @@ if [ -d $EESSI_PREFIX ]; then error "Lmod configuration file not found at $LMOD_RC" fi # Allow for site-specific lmod configuration - host_lmod_rc="$EESSI_CVMFS_REPO"/host_injections/.lmod/lmodrc.lua + host_lmod_rc="${EESSI_PREFIX/versions/host_injections}/.lmod/lmodrc.lua if [ -f $host_lmod_rc ]; then export LMOD_RC="$host_lmod_rc" fi From df4140a74bad2f0af022c10a2d8ada12fa1627df Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 6 Mar 2024 15:54:28 +0100 Subject: [PATCH 0565/1795] Update init/eessi_environment_variables --- init/eessi_environment_variables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 5407034191..f9ca6de408 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -83,7 +83,7 @@ if [ -d $EESSI_PREFIX ]; then error "Lmod configuration file not found at $LMOD_RC" fi # Allow for site-specific lmod configuration - host_lmod_rc="${EESSI_PREFIX/versions/host_injections}/.lmod/lmodrc.lua + host_lmod_rc="${EESSI_PREFIX/versions/host_injections}"/.lmod/lmodrc.lua if [ -f $host_lmod_rc ]; then export LMOD_RC="$host_lmod_rc" fi From 78e582ba2c5fd5de813836151a3d256fc6ba5c94 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 6 Mar 2024 15:59:31 +0100 Subject: [PATCH 0566/1795] Update init/eessi_environment_variables --- init/eessi_environment_variables | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index f9ca6de408..b41dd60924 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -78,9 +78,9 @@ if [ -d $EESSI_PREFIX ]; then export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod" if [ -f $LMOD_CONFIG_DIR/lmodrc.lua ]; then - show_msg "Found Lmod configuration file at $LMOD_RC" + show_msg "Found Lmod configuration file at $LMOD_CONFIG_DIR/lmodrc.lua" else - error "Lmod configuration file not found at $LMOD_RC" + error "Lmod configuration file not found at $LMOD_CONFIG_DIR/lmodrc.lua" fi # Allow for site-specific lmod configuration host_lmod_rc="${EESSI_PREFIX/versions/host_injections}"/.lmod/lmodrc.lua From 77c97235f0291dff7b98f2f1958e345a7afb48ee Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 6 Mar 2024 16:00:42 +0100 Subject: [PATCH 0567/1795] Update init/eessi_environment_variables --- init/eessi_environment_variables | 1 + 1 file changed, 1 insertion(+) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index b41dd60924..d4c113c67a 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -85,6 +85,7 @@ if [ -d $EESSI_PREFIX ]; then # Allow for site-specific lmod configuration host_lmod_rc="${EESSI_PREFIX/versions/host_injections}"/.lmod/lmodrc.lua if [ -f $host_lmod_rc ]; then + show_msg "Found site Lmod configuration file at $LMOD_RC" export LMOD_RC="$host_lmod_rc" fi From 276a64c66347e24b94ace71914547cc0abd75381 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 7 Mar 2024 07:54:23 +0000 Subject: [PATCH 0568/1795] {2023.06}[gompi/2023b] CDO v2.2.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 4dd31dbd5d..2664d8f417 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -8,3 +8,6 @@ easyconfigs: - matplotlib-3.8.2-gfbf-2023b.eb: options: from-pr: 19552 + - CDO-2.2.2-gompi-2023b.eb: + options: + from-pr: 19792 From e33938f1a8424d28966827c4a8942f82b8043bed Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 7 Mar 2024 14:52:25 +0100 Subject: [PATCH 0569/1795] Revert changes in eessi_environment_variables. Limit this PR to the bug fix only. We can follow up properly using SitePackage.lua instead --- init/eessi_environment_variables | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index d4c113c67a..af5222e7b9 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -76,17 +76,11 @@ if [ -d $EESSI_PREFIX ]; then false fi - export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod" - if [ -f $LMOD_CONFIG_DIR/lmodrc.lua ]; then - show_msg "Found Lmod configuration file at $LMOD_CONFIG_DIR/lmodrc.lua" + export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua" + if [ -f $LMOD_RC ]; then + show_msg "Found Lmod configuration file at $LMOD_RC" else - error "Lmod configuration file not found at $LMOD_CONFIG_DIR/lmodrc.lua" - fi - # Allow for site-specific lmod configuration - host_lmod_rc="${EESSI_PREFIX/versions/host_injections}"/.lmod/lmodrc.lua - if [ -f $host_lmod_rc ]; then - show_msg "Found site Lmod configuration file at $LMOD_RC" - export LMOD_RC="$host_lmod_rc" + error "Lmod configuration file not found at $LMOD_RC" fi else From c660a4259c7e391502167203e4a9bc31bbac692e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 10:40:40 +0100 Subject: [PATCH 0570/1795] dont set fakeroot here --- eessi_container.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 09cec8964f..ad6d68640c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -627,11 +627,11 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -if [[ ! -z ${JOB_CFG_FILE} ]]; then - echo "Removing software by launching container with command (next line):" - echo "singularity ${RUN_QUIET} exec --fakeroot ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ./EESSI-remove-software.sh" - singularity ${RUN_QUIET} exec --fakeroot "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ./EESSI-remove-software.sh -fi +#if [[ ! -z ${JOB_CFG_FILE} ]]; then +# echo "Removing software by launching container with command (next line):" +# echo "singularity ${RUN_QUIET} exec --fakeroot ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ./EESSI-remove-software.sh" +# singularity ${RUN_QUIET} exec --fakeroot "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ./EESSI-remove-software.sh +#fi echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" From be5aa374e0aea043f97a709afae6fba58c556a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 10:40:51 +0100 Subject: [PATCH 0571/1795] run build step with fakeroot --- bot/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 1622e757e2..ef9b34ab88 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -181,6 +181,8 @@ BUILD_STEP_ARGS+=("--nvidia" "all") if [[ ! -z ${SHARED_FS_PATH} ]]; then BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi +# add fakeroot option in order to be able to remove software +BUILD_STEP_ARGS+=("--fakeroot") # prepare arguments to install_software_layer.sh (specific to build step) declare -a INSTALL_SCRIPT_ARGS=() From 4946b085eb6bff9515aa8e2257d3320f778ab6f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 10:41:21 +0100 Subject: [PATCH 0572/1795] if run as root, run the removal script --- install_software_layer.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install_software_layer.sh b/install_software_layer.sh index 82ca70b73f..8137ec78e3 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,4 +1,8 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults +if [ $EUID -eq 0 ]; then + ./EESSI-remove-software.sh "$@" + exec runuser -u eessi $( readlink -f "$0" ) -- "$@" +fi ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" From 123c47a84ff20e35c2fbe7dba6a7f371b9a1c694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 10:51:16 +0100 Subject: [PATCH 0573/1795] add fakeroot option --- eessi_container.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index ad6d68640c..af8618f4b8 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -73,6 +73,7 @@ display_help() { echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMG - image file or URL defining the container to use" echo " [default: docker://ghcr.io/eessi/build-node:debian11]" + echo " -f | --fakeroot - run the container with --fakeroot [default: false]" echo " -g | --storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -h | --help - display this usage information [default: false]" @@ -113,6 +114,7 @@ display_help() { ACCESS="ro" CONTAINER="docker://ghcr.io/eessi/build-node:debian11" #DRY_RUN=0 +FAKEROOT=0 VERBOSE=0 STORAGE= LIST_REPOS=0 @@ -140,6 +142,10 @@ while [[ $# -gt 0 ]]; do # DRY_RUN=1 # shift 1 # ;; + -f|--fakeroot) + FAKEROOT=1 + shift 1 + ;; -g|--storage) STORAGE="$2" shift 2 @@ -460,6 +466,11 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then fi fi +# Configure the fakeroot setting for the container +if [[ ${FAKEROOT} -eq 1 ]]; then + ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") +fi + # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg From 93ccf68c0fcb5f48aeba2339f09bdf892d4547d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 11:13:46 +0100 Subject: [PATCH 0574/1795] add echo --- install_software_layer.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_software_layer.sh b/install_software_layer.sh index 8137ec78e3..c5cf40bb5a 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,6 +1,9 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults + +echo "User ID: $EUID" + if [ $EUID -eq 0 ]; then ./EESSI-remove-software.sh "$@" exec runuser -u eessi $( readlink -f "$0" ) -- "$@" From 017803bbf1705045b84a4bf8c8b4e479fffada5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 11:18:57 +0100 Subject: [PATCH 0575/1795] add echo --- EESSI-install-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index ab91d2c6de..4d26b30b53 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -87,6 +87,8 @@ while [[ $# -gt 0 ]]; do esac done +echo "Software installation user: $EUID $UID" + set -- "${POSITIONAL_ARGS[@]}" TOPDIR=$(dirname $(realpath $0)) From c49240680190081b1e0cc6ac8b9e57bffde415c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 11:51:42 +0100 Subject: [PATCH 0576/1795] add removal step --- bot/build.sh | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index ef9b34ab88..bc1cd27770 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -168,9 +168,34 @@ COMMON_ARGS+=("--mode" "run") # make sure to use the same parent dir for storing tarballs of tmp PREVIOUS_TMP_DIR=${PWD}/previous_tmp -# prepare directory to store tarball of tmp for build step +# prepare directory to store tarball of tmp for removal and build steps +TARBALL_TMP_REMOVE_STEP_DIR=${PREVIOUS_TMP_DIR}/remove_step TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step -mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} ${TARBALL_TMP_REMOVE_STEP_DIR} + +# prepare arguments to install_software_layer.sh (specific to build step) +declare -a INSTALL_SCRIPT_ARGS=() +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then + INSTALL_SCRIPT_ARGS+=("--generic") +fi +[[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") +[[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") + +# prepare arguments to eessi_container.sh specific to remove step +declare -a REMOVE_STEP_ARGS=() +REMOVE_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") +REMOVE_STEP_ARGS+=("--storage" "${STORAGE}") +# add fakeroot option in order to be able to remove software +REMOVE_STEP_ARGS+=("--fakeroot") + +# create tmp file for output of removal step +remove_outerr=$(mktemp remove.outerr.XXXX) + +echo "Executing command to remove software:" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVE_STEP_ARGS[@]}" +echo " -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${remove_outerr}" +./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVE_STEP_ARGS[@]}" \ + -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${remove_outerr} # prepare arguments to eessi_container.sh specific to build step declare -a BUILD_STEP_ARGS=() @@ -181,16 +206,10 @@ BUILD_STEP_ARGS+=("--nvidia" "all") if [[ ! -z ${SHARED_FS_PATH} ]]; then BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi -# add fakeroot option in order to be able to remove software -BUILD_STEP_ARGS+=("--fakeroot") -# prepare arguments to install_software_layer.sh (specific to build step) -declare -a INSTALL_SCRIPT_ARGS=() -if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then - INSTALL_SCRIPT_ARGS+=("--generic") -fi -[[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") -[[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") +# determine temporary directory to resume from; this is important in case software was removed for a rebuild +REMOVE_TMPDIR=$(grep ' as tmp directory ' ${remove_outerr} | cut -d ' ' -f 2) +BUILD_STEP_ARGS+=("--resume" "${REMOVE_TMPDIR}") # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) From 7b57e8130ca9a03cc2fab3a8b74bda0c59d1c6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 11:52:05 +0100 Subject: [PATCH 0577/1795] remove removal step --- install_software_layer.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/install_software_layer.sh b/install_software_layer.sh index c5cf40bb5a..82ca70b73f 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,11 +1,4 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults - -echo "User ID: $EUID" - -if [ $EUID -eq 0 ]; then - ./EESSI-remove-software.sh "$@" - exec runuser -u eessi $( readlink -f "$0" ) -- "$@" -fi ./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" From 485ff46bd4e87d2c9e35d1e42b12d8c884e782f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 11:52:50 +0100 Subject: [PATCH 0578/1795] call EESSI-remove-software.sh for removal step --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index bc1cd27770..208209ab28 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -193,9 +193,9 @@ remove_outerr=$(mktemp remove.outerr.XXXX) echo "Executing command to remove software:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVE_STEP_ARGS[@]}" -echo " -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${remove_outerr}" +echo " -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${remove_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVE_STEP_ARGS[@]}" \ - -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${remove_outerr} + -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${remove_outerr} # prepare arguments to eessi_container.sh specific to build step declare -a BUILD_STEP_ARGS=() From 8204a529b4d73aa082bcc0a785555a3f1119fe70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 12:00:49 +0100 Subject: [PATCH 0579/1795] remove echo command --- EESSI-install-software.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 4d26b30b53..ab91d2c6de 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -87,8 +87,6 @@ while [[ $# -gt 0 ]]; do esac done -echo "Software installation user: $EUID $UID" - set -- "${POSITIONAL_ARGS[@]}" TOPDIR=$(dirname $(realpath $0)) From 15eb765405f65966f7c16c2756d7062bbcb4d4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 12:02:43 +0100 Subject: [PATCH 0580/1795] remove unused code --- eessi_container.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index af8618f4b8..d084fe5fab 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -596,12 +596,6 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - #if [[ ! -z ${JOB_CFG_FILE} ]]; then - # always launch build jobs triggered by the job with --fakeroot in order to be able to remove existing installations, see: - # https://github.com/EESSI/software-layer/issues/312 - # we drop back to a regular user in the build script - #ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") - #fi mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work @@ -638,12 +632,6 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -#if [[ ! -z ${JOB_CFG_FILE} ]]; then -# echo "Removing software by launching container with command (next line):" -# echo "singularity ${RUN_QUIET} exec --fakeroot ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ./EESSI-remove-software.sh" -# singularity ${RUN_QUIET} exec --fakeroot "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ./EESSI-remove-software.sh -#fi - echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" singularity ${RUN_QUIET} ${MODE} "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" From 1f7501f2e1aad58abb5d85430d8fed95aaa55576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Mar 2024 13:02:14 +0100 Subject: [PATCH 0581/1795] rename remove to removal, resume tarball step from removal dir --- bot/build.sh | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 208209ab28..8f3439b3ad 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -169,9 +169,9 @@ COMMON_ARGS+=("--mode" "run") PREVIOUS_TMP_DIR=${PWD}/previous_tmp # prepare directory to store tarball of tmp for removal and build steps -TARBALL_TMP_REMOVE_STEP_DIR=${PREVIOUS_TMP_DIR}/remove_step +TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step -mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} ${TARBALL_TMP_REMOVE_STEP_DIR} +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} ${TARBALL_TMP_REMOVAL_STEP_DIR} # prepare arguments to install_software_layer.sh (specific to build step) declare -a INSTALL_SCRIPT_ARGS=() @@ -182,20 +182,20 @@ fi [[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") # prepare arguments to eessi_container.sh specific to remove step -declare -a REMOVE_STEP_ARGS=() -REMOVE_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") -REMOVE_STEP_ARGS+=("--storage" "${STORAGE}") +declare -a REMOVAL_STEP_ARGS=() +REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") +REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") # add fakeroot option in order to be able to remove software -REMOVE_STEP_ARGS+=("--fakeroot") +REMOVAL_STEP_ARGS+=("--fakeroot") # create tmp file for output of removal step -remove_outerr=$(mktemp remove.outerr.XXXX) +removal_outerr=$(mktemp remove.outerr.XXXX) echo "Executing command to remove software:" -echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVE_STEP_ARGS[@]}" -echo " -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${remove_outerr}" -./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVE_STEP_ARGS[@]}" \ - -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${remove_outerr} +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" +echo " -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" +./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ + -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} # prepare arguments to eessi_container.sh specific to build step declare -a BUILD_STEP_ARGS=() @@ -208,8 +208,8 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then fi # determine temporary directory to resume from; this is important in case software was removed for a rebuild -REMOVE_TMPDIR=$(grep ' as tmp directory ' ${remove_outerr} | cut -d ' ' -f 2) -BUILD_STEP_ARGS+=("--resume" "${REMOVE_TMPDIR}") +REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2) +BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) @@ -232,8 +232,7 @@ declare -a TARBALL_STEP_ARGS=() TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}") # determine temporary directory to resume from -BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) -TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") +TARBALL_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") timestamp=$(date +%s) # to set EESSI_VERSION we need to source init/eessi_defaults now From 45022b6b47c3046ea3e2d4033197fdd80dc63850 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 8 Mar 2024 17:30:55 +0100 Subject: [PATCH 0582/1795] Allow for open PRs when checking missing installations in build script --- EESSI-install-software.sh | 2 +- check_missing_installations.sh | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 69de9d1997..75b4f71178 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -237,7 +237,7 @@ else copy_build_log "${eb_last_log}" "${build_logs_dir}" fi - $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} ${TOPDIR}/${pr_diff} else fatal_error "Easystack file ${easystack_file} not found!" fi diff --git a/check_missing_installations.sh b/check_missing_installations.sh index c902fa8184..623c46c74f 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -10,8 +10,15 @@ TOPDIR=$(dirname $(realpath $0)) -if [ $# -ne 1 ]; then - echo "ERROR: Usage: $0 " >&2 +if [ "$#" -eq 1 ]; then + true +elif [ "$#" -eq 2 ]; then + echo "Using $2 to give create exceptions for PR filtering of easystack" + # Find lines that are added and use from-pr, make them unique, grab the + # PR numbers and use them to construct something we can use within awk + pr_exceptions=$(grep ^+ $2 | grep from-pr | uniq | awk '{print $3}' | xargs -i echo " || /'{}'/") +else + echo "ERROR: Usage: $0 ()" >&2 exit 1 fi easystack=$1 @@ -24,7 +31,10 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) -grep -v from-pr ${easystack} > ${tmp_easystack} +# Let's use awk so we can allow for exceptions if we are given a PR diff file +awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" +awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! +eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack} source $TOPDIR/scripts/utils.sh @@ -40,6 +50,10 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" +if [ "$#" -eq 1 ]; then + fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" +fi + check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" # the above assesses the installed software for each easyconfig provided in From 723fce82b272cccd75a1d1841ef301be419461e6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 11:39:33 +0100 Subject: [PATCH 0583/1795] Check for unmerged EasyConfig output pattern --- bot/check-build.sh | 143 ++++++++++++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 53 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index ec1ca56bba..e737363f35 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -98,15 +98,15 @@ job_dir=${PWD} job_out="slurm-${SLURM_JOB_ID}.out" [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" if [[ -f ${job_out} ]]; then - SLURM=1 + SLURM_OUTPUT_FOUND=1 [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" else - SLURM=0 + SLURM_OUTPUT_FOUND=0 [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi ERROR=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_error='ERROR: ' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 @@ -116,7 +116,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi FAILED=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_failed='FAILED: ' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 @@ -126,7 +126,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi MISSING=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_req_missing=' required modules missing:' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}") [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 @@ -136,7 +136,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi NO_MISSING=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_no_missing='No missing installations' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}") [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 @@ -147,7 +147,7 @@ fi TGZ=-1 TARBALL= -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_tgz_created="\.tar\.gz created!" grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) if [[ $? -eq 0 ]]; then @@ -169,9 +169,27 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" [[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +# Here, we try to do some additional analysis on the output file +# to see if we can print a more clear 'reason' for the failure +# For now, we only analyse unmerged EasyConfigs as potential cause, but we can easily add checks for other +# specific scenarios below + +# Check for the pattern being added here https://github.com/EESSI/software-layer/pull/493 to the output to +# see if EasyConfigs might have been unmerged, and that's causing a failure +UNMERGED_EASYCONFIG=-1 +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + gp_unmerged="are you sure all PRs referenced have been merged in EasyBuild" + grep_unmerged=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${gp_unmerged}") + [[ $? -eq 0 ]] && UNMERGED_EASYCONFIG=1 || UNMERGED_EASYCONFIG=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${gp_unmerged}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_unmerged}" +fi + job_result_file=_bot_job${SLURM_JOB_ID}.result -if [[ ${SLURM} -eq 1 ]] && \ +# Default reason: +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ [[ ${ERROR} -eq 0 ]] && \ [[ ${FAILED} -eq 0 ]] && \ [[ ${MISSING} -eq 0 ]] && \ @@ -180,7 +198,12 @@ if [[ ${SLURM} -eq 1 ]] && \ [[ ! -z ${TARBALL} ]]; then # SUCCESS status="SUCCESS" + reason="" summary=":grin: SUCCESS" +elif [[ ${UNMERGED_EASYCONFIG} -eq 1 ]]; then + status="FAILURE" + reason="EasyConfig not found during missing installation check. Are you sure all PRs referenced have been merged in EasyBuild?" + summary=":cry: FAILURE" else # FAILURE status="FAILURE" @@ -253,14 +276,6 @@ fi #
    ### -# construct and write complete PR comment details: implements third alternative -comment_template="
    __SUMMARY_FMT__
    __DETAILS_FMT____ARTEFACTS_FMT__
    " -comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" -comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " -comment_success_item_fmt=":white_check_mark: __ITEM__" -comment_failure_item_fmt=":x: __ITEM__" -comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " -comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " function print_br_item() { format="${1}" @@ -332,42 +347,65 @@ echo -n "comment_description = " >> ${job_result_file} # - __DETAILS_FMT__ -> variable $comment_details # - __ARTEFACTS_FMT__ -> variable $comment_artefacts +# construct and write complete PR comment details: implements third alternative +comment_template="
    __SUMMARY_FMT__
    __REASON_FMT____DETAILS_FMT____ARTEFACTS_FMT__
    " +comment_success_item_fmt=":white_check_mark: __ITEM__" +comment_failure_item_fmt=":x: __ITEM__" + +# Initialize comment_description +comment_description=${comment_template} + +# Now, start replacing template items one by one +# Replace the summary template (__SUMMARY_FMT__) +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" +comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} + +# Only add if there is a reason (e.g. no reason for successful runs) +if [[ ! -z ${reason} ]]; then + comment_reason_fmt="
    _Reason_
    __REASONS__
    " + reason_details="${comment_reason_fmt/__REASONS__/${reason}}" + comment_description=${comment_description/__REASON_FMT__/${reason_details}} +else + comment_description=${comment_description/__REASON_FMT__/""} +fi -# first construct comment_details_list, abbreviated CoDeList +# Replace the details template (__DETAILS_FMT__) +# first construct comment_details_list, abbreviated comment_details_list # then use it to set comment_details -CoDeList="" +comment_details_list="" success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" -CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" -CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_failed}" failure_msg="found message matching ${GP_failed}" -CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_req_missing}" failure_msg="found message matching ${GP_req_missing}" -CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") success_msg="found message(s) matching ${GP_no_missing}" failure_msg="no message matching ${GP_no_missing}" -CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") success_msg="found message matching ${GP_tgz_created}" failure_msg="no message matching ${GP_tgz_created}" -CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") - -comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" - +comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +# Now, do the actual repalcement of __DETAILS_FMT__ +comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " +comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} -# first construct comment_artefacts_list, abbreviated CoArList +# first construct comment_artefacts_list # then use it to set comment_artefacts -CoArList="" +comment_artifacts_list="" # TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then @@ -427,50 +465,49 @@ if [[ ! -z ${TARBALL} ]]; then software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" - CoArList="" - CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" - CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})" - CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" - CoArList="${CoArList}
    "
    +    comment_artifacts_list=""
    +    comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
    +    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})"
    +    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
    +    comment_artifacts_list="${comment_artifacts_list}
    "
         if [[ ! -z ${modules} ]]; then
             while IFS= read -r mod ; do
    -            CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
    +            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
             done <<< "${modules}"
         else
    -        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
    +        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
         fi
    -    CoArList="${CoArList}
    " - CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" - CoArList="${CoArList}
    "
    +    comment_artifacts_list="${comment_artifacts_list}
    " + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" + comment_artifacts_list="${comment_artifacts_list}
    "
         if [[ ! -z ${software_pkgs} ]]; then
             while IFS= read -r sw_pkg ; do
    -            CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
    +            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
             done <<< "${software_pkgs}"
         else
    -        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
    +        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
         fi
    -    CoArList="${CoArList}
    " - CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})" - CoArList="${CoArList}
    "
    +    comment_artifacts_list="${comment_artifacts_list}
    " + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})" + comment_artifacts_list="${comment_artifacts_list}
    "
         if [[ ! -z ${other_shortened} ]]; then
             while IFS= read -r other ; do
    -            CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
    +            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
             done <<< "${other_shortened}"
         else
    -        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
    +        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
         fi
    -    CoArList="${CoArList}
    " + comment_artifacts_list="${comment_artifacts_list}
    " else - CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')" + comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')" fi +comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" -comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}" -comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" +comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}" -# now put all pieces together creating comment_details from comment_template -comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} -comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} +comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " +comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} echo "${comment_description}" >> ${job_result_file} From f4f277605a17aea824f940ae1de0727b190be47d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:22:18 +0100 Subject: [PATCH 0584/1795] First, try without --from-pr. If that fails, try with --from-pr, but only for items added to the EasyStack file in _this_ PR to software-layer. If that succeeds, the reason must be that one of the EasyBuild PRs wasn't merged yet, and we can add that information to the fail_msg to make it more explicit --- check_missing_installations.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 623c46c74f..fd2c0d34bb 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -31,6 +31,8 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) +grep -v from-pr ${easystack} > ${tmp_easystack} + # Let's use awk so we can allow for exceptions if we are given a PR diff file awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! @@ -51,7 +53,25 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" if [ "$#" -eq 1 ]; then - fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" + # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR + # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg + + # Let's use awk so we can allow for exceptions if we are given a PR diff file + awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" + awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! + eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack} + + msg=">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}," + msg="${msg} allowing for --from-pr's that were added in this PR..." + echo ${msg} + eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out + ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} + exit_code=${PIPESTATUS[0]} + + # If now we succeeded, the reason must be that we originally stripped the --from-pr's + if [ "$#" -eq 0 ]; then + fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" + fi fi check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" From b7891087072ab05b50d93295b0f8990e27b50edf Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:24:08 +0100 Subject: [PATCH 0585/1795] Add CFITSIO as a test example, see if we can get a clear error message printed on a build failure here --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 2664d8f417..51e96f1fac 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -10,4 +10,7 @@ easyconfigs: from-pr: 19552 - CDO-2.2.2-gompi-2023b.eb: options: - from-pr: 19792 + from-pr: 19792 + - CFITSIO-4.3.1-GCCcore-13.2.0.eb: + options: + from-pr: 19840 From d178a8f14d6e16f5bcddcd7af0bb7978f647d823 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:34:33 +0100 Subject: [PATCH 0586/1795] Added clarifying comment --- bot/check-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/check-build.sh b/bot/check-build.sh index e737363f35..24c608b04a 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -207,6 +207,8 @@ elif [[ ${UNMERGED_EASYCONFIG} -eq 1 ]]; then else # FAILURE status="FAILURE" + # General failure, we don't know a more specific reason + reason="" summary=":cry: FAILURE" fi From 9123c9c8f483cf5f6b1e931cfe5a4484fc978faa Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:37:08 +0100 Subject: [PATCH 0587/1795] Correct mistake: I meant to _move_ the awk part, not duplicate it --- check_missing_installations.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index fd2c0d34bb..4781ecc72a 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -33,11 +33,6 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) grep -v from-pr ${easystack} > ${tmp_easystack} -# Let's use awk so we can allow for exceptions if we are given a PR diff file -awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" -awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! -eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack} - source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild From 7e4422335f1c2f3ad682362a060906866b9b90f4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:54:07 +0100 Subject: [PATCH 0588/1795] Now actually check against the exit code of the pipe... --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 4781ecc72a..51a328b50c 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -47,7 +47,7 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$#" -eq 1 ]; then +if [ "$exit_code" -eq 1 ]; then # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg From da749a111320720773d648d4a55caa2ba1a826e3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 14:08:27 +0100 Subject: [PATCH 0589/1795] Make seperate exit code for the run that allows --from-pr --- check_missing_installations.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 51a328b50c..6d84384fd7 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -61,10 +61,10 @@ if [ "$exit_code" -eq 1 ]; then echo ${msg} eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} - exit_code=${PIPESTATUS[0]} + exit_code_with_from_pr=${PIPESTATUS[0]} # If now we succeeded, the reason must be that we originally stripped the --from-pr's - if [ "$#" -eq 0 ]; then + if [ "$exit_code_with_from_pr" -eq 0 ]; then fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" fi fi From 86d8a36c52a9a3dfe9af25662bc6c95968804024 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Mon, 11 Mar 2024 11:46:32 +0100 Subject: [PATCH 0590/1795] Remove ineffective DP3 hook --- eb_hooks.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 1478f1a508..d93ee37067 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -186,32 +186,6 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!") -def parse_hook_dp3_enable_relaxed_vector_conversions(ec, eprefix): - """ - Enable lax vector conversion for DP3 on aarch64/neoverse_v1 - Compiling DP3 6.0 with GCC 13.2.0 (foss-2023b) gives a conversion error when building for aarch64/neoverse_v1. - See also, https://github.com/EESSI/software-layer/pull/479 - """ - if ec.name == 'DP3': - tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] - if ( - LooseVersion(ec.version) == LooseVersion('6.0') and - tcname == 'foss' and tcversion == '2023b' - ): - cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if cpu_target == CPU_TARGET_NEOVERSE_V1: - if not hasattr(ec, 'configopts'): - ec['configopts'] = "" - ec['configopts'] += ' -DCMAKE_CXX_FLAGS="$CXXFLAGS -flax-vector-conversions" ' - print_msg("Changed configopts for %s: %s", ec.name, ec['configopts']) - else: - print_msg("Not changing configopts for %s on non-neoverse_v1", ec.name) - else: - print_msg("Not changing configopts for %s %s %s", ec.name, ec.version, ec.toolchain) - else: - raise EasyBuildError("DP3-specific hook triggered for non-DP3 easyconfig?!") - - def parse_hook_cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" if ec.name == 'CGAL': @@ -633,7 +607,6 @@ def inject_gpu_property(ec): PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, - 'DP3': parse_hook_dp3_enable_relaxed_vector_conversions, 'fontconfig': parse_hook_fontconfig_add_fonts, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, From f8816acf3c0c0022bbba3e123f6d0239469728d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 11 Mar 2024 14:19:15 +0100 Subject: [PATCH 0591/1795] {2023.06}[2023a] JupyterNotebook v7.0.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 0d8e71e86c..cb207e50ea 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -36,3 +36,4 @@ easyconfigs: - dask-2023.9.2-foss-2023a.eb - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb + - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb From 310e1e3a36df67c5d51b9bb0dbe80fc97abf1fa7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 11 Mar 2024 15:34:50 +0100 Subject: [PATCH 0592/1795] Initial attempt at splitting off the LMOD hooks into a seperate SitePackage.lua file --- EESSI-install-software.sh | 15 +++- create_lmodrc.py | 108 ----------------------- create_lmodsitepackage.py | 141 +++++++++++++++++++++++++++++++ init/eessi_environment_variables | 17 +++- 4 files changed, 166 insertions(+), 115 deletions(-) create mode 100755 create_lmodsitepackage.py diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 69de9d1997..ed137c522d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -248,11 +248,20 @@ fi ### add packages here echo ">> Creating/updating Lmod cache..." -export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" +export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" +lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?) -if [ ! -f $LMOD_RC ] || [ ${lmodrc_changed} == '0' ]; then +if [ ! -f $lmod_rc_file ] || [ ${lmodrc_changed} == '0' ]; then python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH} - check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC" + check_exit_code $? "$lmod_rc_file created" "Failed to create $lmod_rc_file" +fi + +echo ">> Creating/updating Lmod SitePackage.lua ..." +export LMOD_PACKAGE_PATH=${EASYBUILD_INSTALLPATH}/.lmod" +lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" +if [ ! -f $lmod_sitepackage_file ] || [ ${sitepackage_changed} == '0' ]; then + python3 $TOPDIR/create_lmodsitepackage.py ${EASYBUILD_INSTALLPATH} + check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file" fi $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} diff --git a/create_lmodrc.py b/create_lmodrc.py index 621c8e271a..ae65153a20 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -17,113 +17,6 @@ } """ -GPU_LMOD_RC ="""require("strict") -local hook = require("Hook") -local open = io.open - -local function read_file(path) - local file = open(path, "rb") -- r read mode and b binary mode - if not file then return nil end - local content = file:read "*a" -- *a or *all reads the whole file - file:close() - return content -end - -local function eessi_cuda_enabled_load_hook(t) - local frameStk = require("FrameStk"):singleton() - local mt = frameStk:mt() - local simpleName = string.match(t.modFullName, "(.-)/") - -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. - -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse - -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI - local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" - if simpleName == 'CUDA' then - -- get the full host_injections path - local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') - -- build final path where the CUDA software should be installed - local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" - local cudaDirExists = isDir(cudaEasyBuildDir) - if not cudaDirExists then - local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " - advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where EESSI " - advice = advice .. "can find it.\\n" - advice = advice .. refer_to_docs - LmodError("\\nYou requested to load ", simpleName, " ", advice) - end - end - -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the EESSI linker, - -- otherwise, refuse to load the requested module and print error message - local haveGpu = mt:haveProperty(simpleName,"arch","gpu") - if haveGpu then - local arch = os.getenv("EESSI_CPU_FAMILY") or "" - local cudaVersionFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" - local cudaDriverFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" - local cudaDriverExists = isFile(cudaDriverFile) - local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") - if not (cudaDriverExists or singularityCudaExists) then - local advice = "which relies on the CUDA runtime environment and driver libraries. " - advice = advice .. "In order to be able to use the module, you will need " - advice = advice .. "to make sure EESSI can find the GPU driver libraries on your host system.\\n" - advice = advice .. refer_to_docs - LmodError("\\nYou requested to load ", simpleName, " ", advice) - else - -- CUDA driver exists, now we check its version to see if an update is needed - if cudaDriverExists then - local cudaVersion = read_file(cudaVersionFile) - local cudaVersion_req = os.getenv("EESSICUDAVERSION") - -- driver CUDA versions don't give a patch version for CUDA - local major, minor = string.match(cudaVersion, "(%d+)%.(%d+)") - local major_req, minor_req, patch_req = string.match(cudaVersion_req, "(%d+)%.(%d+)%.(%d+)") - local driver_libs_need_update = false - if major < major_req then - driver_libs_need_update = true - elseif major == major_req then - if minor < minor_req then - driver_libs_need_update = true - end - end - if driver_libs_need_update == true then - local advice = "but the module you want to load requires CUDA " .. cudaVersion_req .. ". " - advice = advice .. "Please update your CUDA driver libraries and then " - advice = advice .. "let EESSI know about the update.\\n" - advice = advice .. refer_to_docs - LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice) - end - end - end - end -end - -local function eessi_openmpi_load_hook(t) - -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1, - -- to work around hang/crash due to bug in OpenMPI; - -- see https://gitlab.com/eessi/support/-/issues/41 - local frameStk = require("FrameStk"):singleton() - local mt = frameStk:mt() - local moduleName = string.match(t.modFullName, "(.-)/") - local cpuTarget = os.getenv("EESSI_SOFTWARE_SUBDIR") or "" - if (moduleName == "OpenMPI") and (cpuTarget == "aarch64/neoverse_v1") then - local msg = "Adding '^smcuda' to $OMPI_MCA_btl to work around bug in OpenMPI" - LmodMessage(msg .. " (see https://gitlab.com/eessi/support/-/issues/41)") - local ompiMcaBtl = os.getenv("OMPI_MCA_btl") - if ompiMcaBtl == nil then - setenv("OMPI_MCA_btl", "^smcuda") - else - setenv("OMPI_MCA_btl", ompiMcaBtl .. ",^smcuda") - end - end -end - --- Combine both functions into a single one, as we can only register one function as load hook in lmod --- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed -function eessi_load_hook(t) - eessi_cuda_enabled_load_hook(t) - eessi_openmpi_load_hook(t) -end - - -hook.register("load", eessi_load_hook) -""" def error(msg): sys.stderr.write("ERROR: %s\n" % msg) @@ -143,7 +36,6 @@ def error(msg): 'dot_lmod': DOT_LMOD, 'prefix': prefix, } -lmodrc_txt += '\n' + GPU_LMOD_RC try: os.makedirs(os.path.dirname(lmodrc_path), exist_ok=True) with open(lmodrc_path, 'w') as fp: diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py new file mode 100755 index 0000000000..9e77eafecf --- /dev/null +++ b/create_lmodsitepackage.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# +# Create SitePackage.lua configuration file for Lmod. +# +import os +import sys + +DOT_LMOD = '.lmod' + +hook_txt ="""require("strict") +local hook = require("Hook") +local open = io.open + +local function read_file(path) + local file = open(path, "rb") -- r read mode and b binary mode + if not file then return nil end + local content = file:read "*a" -- *a or *all reads the whole file + file:close() + return content +end + +local function eessi_cuda_enabled_load_hook(t) + local frameStk = require("FrameStk"):singleton() + local mt = frameStk:mt() + local simpleName = string.match(t.modFullName, "(.-)/") + -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. + -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse + -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI + local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" + if simpleName == 'CUDA' then + -- get the full host_injections path + local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') + -- build final path where the CUDA software should be installed + local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" + local cudaDirExists = isDir(cudaEasyBuildDir) + if not cudaDirExists then + local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " + advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where EESSI " + advice = advice .. "can find it.\\n" + advice = advice .. refer_to_docs + LmodError("\\nYou requested to load ", simpleName, " ", advice) + end + end + -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the EESSI linker, + -- otherwise, refuse to load the requested module and print error message + local haveGpu = mt:haveProperty(simpleName,"arch","gpu") + if haveGpu then + local arch = os.getenv("EESSI_CPU_FAMILY") or "" + local cudaVersionFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" + local cudaDriverFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" + local cudaDriverExists = isFile(cudaDriverFile) + local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") + if not (cudaDriverExists or singularityCudaExists) then + local advice = "which relies on the CUDA runtime environment and driver libraries. " + advice = advice .. "In order to be able to use the module, you will need " + advice = advice .. "to make sure EESSI can find the GPU driver libraries on your host system.\\n" + advice = advice .. refer_to_docs + LmodError("\\nYou requested to load ", simpleName, " ", advice) + else + -- CUDA driver exists, now we check its version to see if an update is needed + if cudaDriverExists then + local cudaVersion = read_file(cudaVersionFile) + local cudaVersion_req = os.getenv("EESSICUDAVERSION") + -- driver CUDA versions don't give a patch version for CUDA + local major, minor = string.match(cudaVersion, "(%d+)%.(%d+)") + local major_req, minor_req, patch_req = string.match(cudaVersion_req, "(%d+)%.(%d+)%.(%d+)") + local driver_libs_need_update = false + if major < major_req then + driver_libs_need_update = true + elseif major == major_req then + if minor < minor_req then + driver_libs_need_update = true + end + end + if driver_libs_need_update == true then + local advice = "but the module you want to load requires CUDA " .. cudaVersion_req .. ". " + advice = advice .. "Please update your CUDA driver libraries and then " + advice = advice .. "let EESSI know about the update.\\n" + advice = advice .. refer_to_docs + LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice) + end + end + end + end +end + +local function eessi_openmpi_load_hook(t) + -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1, + -- to work around hang/crash due to bug in OpenMPI; + -- see https://gitlab.com/eessi/support/-/issues/41 + local frameStk = require("FrameStk"):singleton() + local mt = frameStk:mt() + local moduleName = string.match(t.modFullName, "(.-)/") + local cpuTarget = os.getenv("EESSI_SOFTWARE_SUBDIR") or "" + if (moduleName == "OpenMPI") and (cpuTarget == "aarch64/neoverse_v1") then + local msg = "Adding '^smcuda' to $OMPI_MCA_btl to work around bug in OpenMPI" + LmodMessage(msg .. " (see https://gitlab.com/eessi/support/-/issues/41)") + local ompiMcaBtl = os.getenv("OMPI_MCA_btl") + if ompiMcaBtl == nil then + setenv("OMPI_MCA_btl", "^smcuda") + else + setenv("OMPI_MCA_btl", ompiMcaBtl .. ",^smcuda") + end + end +end + +-- Combine both functions into a single one, as we can only register one function as load hook in lmod +-- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed +function eessi_load_hook(t) + eessi_cuda_enabled_load_hook(t) + eessi_openmpi_load_hook(t) +end + + +hook.register("load", eessi_load_hook) +""" + +def error(msg): + sys.stderr.write("ERROR: %s\n" % msg) + sys.exit(1) + + +if len(sys.argv) != 2: + error("Usage: %s " % sys.argv[0]) + +prefix = sys.argv[1] + +if not os.path.exists(prefix): + error("Prefix directory %s does not exist!" % prefix) + +sitepackage_path = os.path.join(prefix, DOT_LMOD, 'SitePackage.lua') +sitepackage_txt += hook_txt +try: + os.makedirs(os.path.dirname(sitepackage_path), exist_ok=True) + with open(sitepackage_path, 'w') as fp: + fp.write(sitepackage_txt) + +except (IOError, OSError) as err: + error("Failed to create %s: %s" % (sitepackage_path, err)) + +print(sitepackage_path) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index af5222e7b9..78f01b1921 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -76,11 +76,20 @@ if [ -d $EESSI_PREFIX ]; then false fi - export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua" - if [ -f $LMOD_RC ]; then - show_msg "Found Lmod configuration file at $LMOD_RC" + export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod" + lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" + if [ -f $lmod_rc_file ]; then + show_msg "Found Lmod configuration file at $lmod_rc_file" else - error "Lmod configuration file not found at $LMOD_RC" + error "Lmod configuration file not found at $lmod_rc_file" + fi + + export LMOD_PACKAGE_PATH="$EESSI_SOFTWARE_PATH/.lmod" + lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" + if [ -f $lmod_sitepackage_file ]; then + show_msg "Found Lmod SitePackage.lua file at $lmod_sitepackage_file" + else + error "Lmod SitePackage.lua file not found at $lmod_sitepackage_file" fi else From a8c2883f0c73af0537a834221c3ae439f44c1b2f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Mon, 11 Mar 2024 15:59:32 +0100 Subject: [PATCH 0593/1795] Correct typo Co-authored-by: ocaisa --- bot/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 24c608b04a..10b1ed6a11 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -400,7 +400,7 @@ comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${succ success_msg="found message matching ${GP_tgz_created}" failure_msg="no message matching ${GP_tgz_created}" comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") -# Now, do the actual repalcement of __DETAILS_FMT__ +# Now, do the actual replacement of __DETAILS_FMT__ comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} From 6553e294163fb9e117803a2433acba3c14e135f2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 11 Mar 2024 16:05:16 +0100 Subject: [PATCH 0594/1795] Skip the if-condition also if pr_exceptions is empty. There is no point in running the eb --missing again in that case, since there were no --from-pr's in this PR to begin with, so unmerged PRs cannot be the reason for failing --- check_missing_installations.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 6d84384fd7..0ee6ec4385 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -47,9 +47,11 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$exit_code" -eq 1 ]; then +if [ "$exit_code" -eq 1 ] && [ ! -z $pr_exceptions ]; then # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg + # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as + # well skip this check - unmerged PRs can not be the reason for the non-zero exit code in that scenario # Let's use awk so we can allow for exceptions if we are given a PR diff file awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" From 27a12bd2e4a88c486f3dd5b5be57ac16eefd2650 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 11 Mar 2024 16:09:46 +0100 Subject: [PATCH 0595/1795] Added missing quote --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index ed137c522d..219fa8680a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -257,7 +257,7 @@ if [ ! -f $lmod_rc_file ] || [ ${lmodrc_changed} == '0' ]; then fi echo ">> Creating/updating Lmod SitePackage.lua ..." -export LMOD_PACKAGE_PATH=${EASYBUILD_INSTALLPATH}/.lmod" +export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" if [ ! -f $lmod_sitepackage_file ] || [ ${sitepackage_changed} == '0' ]; then python3 $TOPDIR/create_lmodsitepackage.py ${EASYBUILD_INSTALLPATH} From 81e544f15333b3d3b3ed70f4954f414f262ba6cd Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 11 Mar 2024 16:11:31 +0100 Subject: [PATCH 0596/1795] Cant add to a variable that doesnt exist... We don't need this intermediate assignment anyway --- create_lmodsitepackage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 9e77eafecf..9a4a232863 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -129,11 +129,10 @@ def error(msg): error("Prefix directory %s does not exist!" % prefix) sitepackage_path = os.path.join(prefix, DOT_LMOD, 'SitePackage.lua') -sitepackage_txt += hook_txt try: os.makedirs(os.path.dirname(sitepackage_path), exist_ok=True) with open(sitepackage_path, 'w') as fp: - fp.write(sitepackage_txt) + fp.write(hook_txt) except (IOError, OSError) as err: error("Failed to create %s: %s" % (sitepackage_path, err)) From 333762b9c1ddab70721826b70a1d432f6d043c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 11 Mar 2024 16:41:19 +0100 Subject: [PATCH 0597/1795] determine whether or not the removal step has to be run --- bot/build.sh | 60 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 8f3439b3ad..31d9b576fe 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -168,12 +168,8 @@ COMMON_ARGS+=("--mode" "run") # make sure to use the same parent dir for storing tarballs of tmp PREVIOUS_TMP_DIR=${PWD}/previous_tmp -# prepare directory to store tarball of tmp for removal and build steps -TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step -TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step -mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} ${TARBALL_TMP_REMOVAL_STEP_DIR} - # prepare arguments to install_software_layer.sh (specific to build step) +declare -a BUILD_STEP_ARGS=() declare -a INSTALL_SCRIPT_ARGS=() if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then INSTALL_SCRIPT_ARGS+=("--generic") @@ -181,24 +177,44 @@ fi [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") [[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") -# prepare arguments to eessi_container.sh specific to remove step -declare -a REMOVAL_STEP_ARGS=() -REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") -REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") -# add fakeroot option in order to be able to remove software -REMOVAL_STEP_ARGS+=("--fakeroot") - -# create tmp file for output of removal step -removal_outerr=$(mktemp remove.outerr.XXXX) +# determine if the removal step has to be run +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) +changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/") +if [[ -z ${changed_easystacks_rebuilds} ]]; then + echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." +else + # prepare directory to store tarball of tmp for removal and build steps + TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step + mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR} + + # prepare arguments to eessi_container.sh specific to remove step + declare -a REMOVAL_STEP_ARGS=() + REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") + REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") + # add fakeroot option in order to be able to remove software + REMOVAL_STEP_ARGS+=("--fakeroot") + + # create tmp file for output of removal step + removal_outerr=$(mktemp remove.outerr.XXXX) + + echo "Executing command to remove software:" + echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" + echo " -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" + ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ + -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} + + # make sure that the build step resumes from the same temporary directory + # this is important, as otherwise the removed software will still be there + REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2) + BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") +fi -echo "Executing command to remove software:" -echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" -echo " -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" -./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ - -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} +# prepare directory to store tarball of tmp for build step +TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} # prepare arguments to eessi_container.sh specific to build step -declare -a BUILD_STEP_ARGS=() BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") # add options required to handle NVIDIA support @@ -207,10 +223,6 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi -# determine temporary directory to resume from; this is important in case software was removed for a rebuild -REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2) -BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") - # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) From 0343684cc8e7b73fc25e160007dc458c2c54e6d0 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 12 Mar 2024 10:19:33 +0100 Subject: [PATCH 0598/1795] {2023.06}[foss/2023b] python-casacore 3.5.2 + libspatialindex 1.9.3 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 54011c262e..186f4f86b3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -43,3 +43,9 @@ easyconfigs: - CDO-2.2.2-gompi-2023b.eb: options: from-pr: 19792 + - python-casacore-3.5.2-foss-2023b.eb: + options: + from-pr: 20084 + - libspatialindex-1.9.3-GCCcore-13.2.0.eb: + options: + from-pr: 19922 From 3e43b061314bdde685b585804a047802cb95f81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 12 Mar 2024 14:00:38 +0100 Subject: [PATCH 0599/1795] add ImageMagick-7.1.1-15-GCCcore-12.3.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index cb207e50ea..8165755865 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -37,3 +37,6 @@ easyconfigs: - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb + - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb: + options: + from-pr: 20086 From 0b826ae64a31e0b2463c6ea65c9c12e1701922ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 12 Mar 2024 16:53:26 +0100 Subject: [PATCH 0600/1795] remove proxy arguments Co-authored-by: Kenneth Hoste --- EESSI-remove-software.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 819c08f8ce..4786d60fc0 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -11,8 +11,6 @@ display_help() { echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds" echo " -g | --generic - instructs script to build for generic architecture target" echo " -h | --help - display this usage information" - echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" - echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" echo " --shared-fs-path - path to directory on shared filesystem that can be used" } From 0ed12492ed91e0ada5dd142536927a5edc4a817f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 12 Mar 2024 16:58:17 +0100 Subject: [PATCH 0601/1795] remove code for setting http(s)_proxy Co-authored-by: Kenneth Hoste --- EESSI-remove-software.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 4786d60fc0..e995f55a45 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -27,14 +27,6 @@ while [[ $# -gt 0 ]]; do # no shifting needed here, we're done. exit 0 ;; - -x|--http-proxy) - export http_proxy="$2" - shift 2 - ;; - -y|--https-proxy) - export https_proxy="$2" - shift 2 - ;; --build-logs-dir) export build_logs_dir="${2}" shift 2 From bc4fd890f9ae68b094b164dc85ed217eef420622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 12 Mar 2024 16:58:42 +0100 Subject: [PATCH 0602/1795] no need to set build logs dir Co-authored-by: Kenneth Hoste --- EESSI-remove-software.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index e995f55a45..d3823c7036 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -27,10 +27,6 @@ while [[ $# -gt 0 ]]; do # no shifting needed here, we're done. exit 0 ;; - --build-logs-dir) - export build_logs_dir="${2}" - shift 2 - ;; --shared-fs-path) export shared_fs_path="${2}" shift 2 From fd7a9eadd21fd386beaceb0af211dd2f523a6669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 12 Mar 2024 16:59:52 +0100 Subject: [PATCH 0603/1795] remove commented code Co-authored-by: ocaisa --- EESSI-remove-software.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index d3823c7036..02e91a43d1 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -60,9 +60,6 @@ else fatal_error "$EESSI_CVMFS_REPO is not available!" fi -# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory -#export PYTHONPYCACHEPREFIX=$TMPDIR/pycache - DETECTION_PARAMETERS='' GENERIC=0 EB='eb' From 994d573759c1c7707952a9d9a125cc6e1d3df47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 12 Mar 2024 17:00:27 +0100 Subject: [PATCH 0604/1795] fix comment about purpose of script Co-authored-by: ocaisa --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 02e91a43d1..f111be339e 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to install EESSI software stack (version set through init/eessi_defaults) +# Script to remove part of the EESSI software stack (version set through init/eessi_defaults) # see example parsing of command line arguments at # https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop From 3d7a959d0a1e62eb4c29c7b6742b82d5eaff4321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 12 Mar 2024 17:02:10 +0100 Subject: [PATCH 0605/1795] remove code and parameter for setting shared fs path --- EESSI-remove-software.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index f111be339e..1779a7ed58 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -11,7 +11,6 @@ display_help() { echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds" echo " -g | --generic - instructs script to build for generic architecture target" echo " -h | --help - display this usage information" - echo " --shared-fs-path - path to directory on shared filesystem that can be used" } POSITIONAL_ARGS=() @@ -27,10 +26,6 @@ while [[ $# -gt 0 ]]; do # no shifting needed here, we're done. exit 0 ;; - --shared-fs-path) - export shared_fs_path="${2}" - shift 2 - ;; -*|--*) echo "Error: Unknown option: $1" >&2 exit 1 @@ -103,12 +98,6 @@ fi echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild -if [ ! -z "${shared_fs_path}" ]; then - shared_eb_sourcepath=${shared_fs_path}/easybuild/sources - echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" - export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} -fi - echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge From 02955711c5600305d8feedece3e85978626ff962 Mon Sep 17 00:00:00 2001 From: Tim Kok Date: Tue, 12 Mar 2024 18:33:07 +0100 Subject: [PATCH 0606/1795] Use PR that sets LD_LIBRARY_PATH for python-casacore --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 186f4f86b3..e2d35276df 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -45,7 +45,7 @@ easyconfigs: from-pr: 19792 - python-casacore-3.5.2-foss-2023b.eb: options: - from-pr: 20084 + from-pr: 20089 - libspatialindex-1.9.3-GCCcore-13.2.0.eb: options: from-pr: 19922 From dab1ac0f66c6164c9ea9bc3798be7a0673465ac5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 19:12:15 +0100 Subject: [PATCH 0607/1795] Create generic function to compare (and if needed copy) files based on an explicit list from one dir to another. Use that generic function to replace the existing code copying the scripts dir, and the scripts/gpu_support/nvidia dir. Add to that a copy of the init dir and the current subdirs, again listing the files to copy explicitely for safety. --- install_scripts.sh | 80 +++++++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 23 deletions(-) diff --git a/install_scripts.sh b/install_scripts.sh index 6e6cd825ac..ac1741428e 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -25,6 +25,35 @@ compare_and_copy() { fi } +copy_files_by_list() { +# Compares and copies listed files from a source to a target directory + if [ "$#" -ne 3 ]; then + echo "Usage of function: copy_files_by_list " + echo "Here, file_list is an (expanded) bash array" + echo "Example:" + echo "my_files=(file1 file2)" + echo 'copy_files_by_list /my/source /my/target "${my_files[@]}"' + return 1 + fi + source_dir="$1" + target_dir="$2" + # Need to shift all arguments to the left twice. Then, rebuild the array with the rest of the arguments + shift + shift + file_list=("$@") + + # Create target dir + mkdir -p ${target_dir} + + # Copy from source to target + echo "Copying files: ${file_list[@]}" + echo "From directory: ${source_dir}" + echo "To directory: ${target_dir}" + + for file in ${file_list[@]}; do + compare_and_copy ${source_dir}/${file} ${target_dir}/${file} + done +} POSITIONAL_ARGS=() @@ -54,28 +83,33 @@ set -- "${POSITIONAL_ARGS[@]}" TOPDIR=$(dirname $(realpath $0)) -# Subdirs for generic scripts -SCRIPTS_DIR_SOURCE=${TOPDIR}/scripts # Source dir -SCRIPTS_DIR_TARGET=${INSTALL_PREFIX}/scripts # Target dir +# Copy for init directory +init_files=( + bash eessi_archdetect.sh eessi_defaults eessi_environment_variables eessi_software_subdir_for_host.py + minimal_eessi_env README.md test.py +) +copy_files_by_list ${TOPDIR}/init ${INSTALL_PREFIX}/init "${init_files[@]}" -# Create target dir -mkdir -p ${SCRIPTS_DIR_TARGET} +# Copy for the init/arch_specs directory +arch_specs_files=( + eessi_arch_arm.spec eessi_arch_ppc.spec eessi_arch_x86.spec +) +copy_files_by_list ${TOPDIR}/init/arch_specs ${INSTALL_PREFIX}/init/arch_specs "${arch_specs_files[@]}" -# Copy scripts into this prefix -echo "copying scripts from ${SCRIPTS_DIR_SOURCE} to ${SCRIPTS_DIR_TARGET}" -for file in utils.sh; do - compare_and_copy ${SCRIPTS_DIR_SOURCE}/${file} ${SCRIPTS_DIR_TARGET}/${file} -done -# Subdirs for GPU support -NVIDIA_GPU_SUPPORT_DIR_SOURCE=${SCRIPTS_DIR_SOURCE}/gpu_support/nvidia # Source dir -NVIDIA_GPU_SUPPORT_DIR_TARGET=${SCRIPTS_DIR_TARGET}/gpu_support/nvidia # Target dir - -# Create target dir -mkdir -p ${NVIDIA_GPU_SUPPORT_DIR_TARGET} - -# Copy files from this directory into the prefix -# To be on the safe side, we dont do recursive copies, but we are explicitely copying each individual file we want to add -echo "copying scripts from ${NVIDIA_GPU_SUPPORT_DIR_SOURCE} to ${NVIDIA_GPU_SUPPORT_DIR_TARGET}" -for file in install_cuda_host_injections.sh link_nvidia_host_libraries.sh; do - compare_and_copy ${NVIDIA_GPU_SUPPORT_DIR_SOURCE}/${file} ${NVIDIA_GPU_SUPPORT_DIR_TARGET}/${file} -done +# Copy for init/Magic_castle directory +mc_files=( + bash eessi_python3 +) +copy_files_by_list ${TOPDIR}/init/Magic_Castle ${INSTALL_PREFIX}/init/Magic_Castle "${mc_files[@]}" + +# Copy for the scripts directory +script_files=( + utils.sh +) +copy_files_by_list ${TOPDIR}/scripts ${INSTALL_PREFIX}/scripts "${script_files[@]}" + +# Copy files for the scripts/gpu_support/nvidia directory +nvidia_files=( + install_cuda_host_injections.sh link_nvidia_host_libraries.sh +) +copy_files_by_list ${TOPDIR}/gpu_support/nvidia ${INSTALL_PREFIX}/gpu_support/nvidia "${nvidia_files[@]}" From 0abedef29ffceb22a5cafa88869bf48b38f3fbb3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 19:17:47 +0100 Subject: [PATCH 0608/1795] An expanded bash array counts as extra arguments. Thus, we cannot check for three arguments, it can be anything greater than 2 --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index ac1741428e..e8077aa242 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -27,7 +27,7 @@ compare_and_copy() { copy_files_by_list() { # Compares and copies listed files from a source to a target directory - if [ "$#" -ne 3 ]; then + if [ ! "$#" -ge 3 ]; then echo "Usage of function: copy_files_by_list " echo "Here, file_list is an (expanded) bash array" echo "Example:" From 11c7525e03069c0a5aa85d36a9cca5a1aeaff381 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 19:21:06 +0100 Subject: [PATCH 0609/1795] Forgot to check the diff if the create_lmodsitepackage.py actually changed --- EESSI-install-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 219fa8680a..2d5912736b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -259,6 +259,7 @@ fi echo ">> Creating/updating Lmod SitePackage.lua ..." export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" +sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?) if [ ! -f $lmod_sitepackage_file ] || [ ${sitepackage_changed} == '0' ]; then python3 $TOPDIR/create_lmodsitepackage.py ${EASYBUILD_INSTALLPATH} check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file" From a07b8a8accc23666be71f36ce0d09e82e9ffdc30 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 19:22:01 +0100 Subject: [PATCH 0610/1795] Is quoting the problem --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2d5912736b..c51459967d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -260,7 +260,7 @@ echo ">> Creating/updating Lmod SitePackage.lua ..." export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?) -if [ ! -f $lmod_sitepackage_file ] || [ ${sitepackage_changed} == '0' ]; then +if [ ! -f $lmod_sitepackage_file ] || [ "${sitepackage_changed}" == '0' ]; then python3 $TOPDIR/create_lmodsitepackage.py ${EASYBUILD_INSTALLPATH} check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file" fi From 90b3ce068aa787715d82a20656c99f656f561a44 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 19:23:57 +0100 Subject: [PATCH 0611/1795] Try quoting --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index c51459967d..2a4f7182cd 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -260,7 +260,7 @@ echo ">> Creating/updating Lmod SitePackage.lua ..." export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?) -if [ ! -f $lmod_sitepackage_file ] || [ "${sitepackage_changed}" == '0' ]; then +if [ ! -f "$lmod_sitepackage_file" ] || [ "${sitepackage_changed}" == '0' ]; then python3 $TOPDIR/create_lmodsitepackage.py ${EASYBUILD_INSTALLPATH} check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file" fi From 9c707e4a69ad385474067f2adee599632ffda882 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 19:25:30 +0100 Subject: [PATCH 0612/1795] Correct prefix for the gpu_support scripts --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index e8077aa242..508735975c 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -112,4 +112,4 @@ copy_files_by_list ${TOPDIR}/scripts ${INSTALL_PREFIX}/scripts "${script_files[@ nvidia_files=( install_cuda_host_injections.sh link_nvidia_host_libraries.sh ) -copy_files_by_list ${TOPDIR}/gpu_support/nvidia ${INSTALL_PREFIX}/gpu_support/nvidia "${nvidia_files[@]}" +copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" From 743860fc06d5be09e9b15d62514f91ac2fb4aebc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 13 Mar 2024 11:29:07 +0100 Subject: [PATCH 0613/1795] {2023.06}[foss/2023b] GROMACS 2024.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index e2d35276df..f906811a73 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -49,3 +49,6 @@ easyconfigs: - libspatialindex-1.9.3-GCCcore-13.2.0.eb: options: from-pr: 19922 + - GROMACS-2024.1-foss-2023b.eb: + options: + from-pr: 20102 From c08831ee060d153bb1da466cfaa00e4103c37e31 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 13 Mar 2024 15:46:24 +0100 Subject: [PATCH 0614/1795] Update bot/check-build.sh Co-authored-by: Kenneth Hoste --- bot/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 10b1ed6a11..e075518421 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -174,7 +174,7 @@ fi # For now, we only analyse unmerged EasyConfigs as potential cause, but we can easily add checks for other # specific scenarios below -# Check for the pattern being added here https://github.com/EESSI/software-layer/pull/493 to the output to +# Check for the pattern being added here by check_missing_installations.sh to the output to # see if EasyConfigs might have been unmerged, and that's causing a failure UNMERGED_EASYCONFIG=-1 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then From 992b11dc782d391899862f80033f08b1efba4634 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 13 Mar 2024 15:47:37 +0100 Subject: [PATCH 0615/1795] Remove PR to trigger comment Co-authored-by: Kenneth Hoste --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 2b5844289d..e2d35276df 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -49,6 +49,3 @@ easyconfigs: - libspatialindex-1.9.3-GCCcore-13.2.0.eb: options: from-pr: 19922 - - CFITSIO-4.3.1-GCCcore-13.2.0.eb: - options: - from-pr: 19840 From 8d6493ea05422ee443f10bc9230c3ff3884af998 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 13 Mar 2024 15:48:08 +0100 Subject: [PATCH 0616/1795] Trigger on any non-zero exit code Co-authored-by: Kenneth Hoste --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 0ee6ec4385..7c29228416 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -47,7 +47,7 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$exit_code" -eq 1 ] && [ ! -z $pr_exceptions ]; then +if [ "$exit_code" -ne 0 ] && [ ! -z $pr_exceptions ]; then # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as From a148920b35a4a70d03ce5f0175b3c730ae279db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 13 Mar 2024 22:35:09 +0100 Subject: [PATCH 0617/1795] remove openmpi hook that provided a temporary workaround --- create_lmodrc.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index bc69dd4396..0e738a530e 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -94,28 +94,7 @@ end end -local function openmpi_load_hook(t) - -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1, - -- to work around hang/crash due to bug in OpenMPI; - -- see https://gitlab.com/eessi/support/-/issues/41 - local frameStk = require("FrameStk"):singleton() - local mt = frameStk:mt() - local moduleName = string.match(t.modFullName, "(.-)/") - local cpuTarget = os.getenv("EESSI_SOFTWARE_SUBDIR") or "" - if (moduleName == "OpenMPI") and (cpuTarget == "aarch64/neoverse_v1") then - local msg = "Adding '^smcuda' to $OMPI_MCA_btl to work around bug in OpenMPI" - LmodMessage(msg .. " (see https://gitlab.com/eessi/support/-/issues/41)") - local ompiMcaBtl = os.getenv("OMPI_MCA_btl") - if ompiMcaBtl == nil then - setenv("OMPI_MCA_btl", "^smcuda") - else - setenv("OMPI_MCA_btl", ompiMcaBtl .. ",^smcuda") - end - end -end - hook.register("load", cuda_enabled_load_hook) -hook.register("load", openmpi_load_hook) """ def error(msg): From 5d4542e5a73962da0032427bba7ee7ed19b79ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 13 Mar 2024 22:38:33 +0100 Subject: [PATCH 0618/1795] remove hook.register for cuda hook --- create_lmodrc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index 2175f885d2..133743c844 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -94,8 +94,6 @@ end end -hook.register("load", cuda_enabled_load_hook) - -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) From 54bb3ec1cdd6cae823068a67cdfdb5d443d4e69b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 14 Mar 2024 09:38:29 +0100 Subject: [PATCH 0619/1795] Make sure the changed init scripts also end up in the tarball --- create_tarball.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/create_tarball.sh b/create_tarball.sh index a619df9439..5bd8eeb110 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -51,6 +51,13 @@ if [ -d ${eessi_version}/scripts ]; then find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list} fi +# also include init, which is also copied by install_scripts.sh +if [ -d ${eessi_version}/init ]; then + find ${eessi_version}/init -type f | grep -v '/\.wh\.' >> ${files_list} +fi + + + if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} From 0fdbd3a3be714c116b76f72c694406ea03e513fe Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 14 Mar 2024 11:35:35 +0100 Subject: [PATCH 0620/1795] Add Z3 with new module name Via https://github.com/easybuilders/easybuild-easyconfigs/pull/20050 , the different Z3 easyconfigs have been consolidated. This means the version used as a dependency for PyTorch has been "corrected" so we need to dance around that for our CI purposes --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 8165755865..5a46cf5fb2 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -40,3 +40,9 @@ easyconfigs: - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb: options: from-pr: 20086 + - Z3-4.12.2-GCCcore-12.3.0.eb: + options: + # The Z3 dependency of PyTorch had it's versionsuffix removed + # and we need to workaround the problem this creates, + # see https://github.com/EESSI/software-layer/pull/501 for details + from-pr: 20050 From 5a6bfd149bd59894e51adec5b2e8e9dc00873977 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 14 Mar 2024 17:47:18 +0100 Subject: [PATCH 0621/1795] Remove whitspace --- create_tarball.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 5bd8eeb110..2d77acfc43 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -56,8 +56,6 @@ if [ -d ${eessi_version}/init ]; then find ${eessi_version}/init -type f | grep -v '/\.wh\.' >> ${files_list} fi - - if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} From d097189aa07f2d323779212e4451a2df2466193d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 14 Mar 2024 22:55:43 +0100 Subject: [PATCH 0622/1795] {2023.06}[2023a] PyOpenGL 3.1.7 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 5a46cf5fb2..1d3119e5c2 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -46,3 +46,4 @@ easyconfigs: # and we need to workaround the problem this creates, # see https://github.com/EESSI/software-layer/pull/501 for details from-pr: 20050 + - PyOpenGL-3.1.7-GCCcore-12.3.0.eb From 8245aff4b7f4dd178924cd42eda76c21eed146fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 15 Mar 2024 00:02:09 +0100 Subject: [PATCH 0623/1795] add from-pr=20007 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 1d3119e5c2..65b4265ff7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -46,4 +46,6 @@ easyconfigs: # and we need to workaround the problem this creates, # see https://github.com/EESSI/software-layer/pull/501 for details from-pr: 20050 - - PyOpenGL-3.1.7-GCCcore-12.3.0.eb + - PyOpenGL-3.1.7-GCCcore-12.3.0.eb: + options: + from-pr: 20007 From af58e1a5fc00ab311eab185a91302d0c735c8978 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Mar 2024 20:48:12 +0100 Subject: [PATCH 0624/1795] {2023.06}[2023a] ipympl 0.9.3 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 65b4265ff7..384aa04a9c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -49,3 +49,6 @@ easyconfigs: - PyOpenGL-3.1.7-GCCcore-12.3.0.eb: options: from-pr: 20007 + - ipympl-0.9.3-foss-2023a.eb: + options: + from-pr: 20126 From a12b9a340e17953a8e6fe741245663c69ece813b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 15 Mar 2024 21:13:32 +0100 Subject: [PATCH 0625/1795] fix check_missing_installations.sh when unmerged PR is used --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 7c29228416..d8135ea3cb 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -47,7 +47,7 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$exit_code" -ne 0 ] && [ ! -z $pr_exceptions ]; then +if [ "$exit_code" -ne 0 ] && [ ! -z "$pr_exceptions" ]; then # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as From 54e30859248035fb70694f68a6b6c6c695b196c9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 20 Mar 2024 23:27:16 +0100 Subject: [PATCH 0626/1795] Add libxc and ELPA --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 384aa04a9c..4fa3d8a29e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -52,3 +52,5 @@ easyconfigs: - ipympl-0.9.3-foss-2023a.eb: options: from-pr: 20126 + - ELPA-2023.05.001-foss-2023a.eb + - libxc-6.2.2-GCC-12.3.0.eb From 9d62787eb90dcd6a42e5c9e8b9f51f87c4d84d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 09:19:44 +0100 Subject: [PATCH 0627/1795] source init/bash instead of minimal eessi env --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 1779a7ed58..010ee365af 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -47,7 +47,7 @@ source $TOPDIR/scripts/utils.sh echo ">> Setting up environment..." -source $TOPDIR/init/minimal_eessi_env +source $TOPDIR/init/bash if [ -d $EESSI_CVMFS_REPO ]; then echo_green "$EESSI_CVMFS_REPO available, OK!" From 229ce93ac4c2db86b8fbd29da38f1e78f2d5c167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 09:20:35 +0100 Subject: [PATCH 0628/1795] remove GENERIC variable --- EESSI-remove-software.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 010ee365af..6d0c9e3800 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -56,12 +56,10 @@ else fi DETECTION_PARAMETERS='' -GENERIC=0 EB='eb' if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then echo_yellow ">> GENERIC build requested, taking appropriate measures!" DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" - GENERIC=1 EB='eb --optarch=GENERIC' fi From 977bcaad3f5ba29d5d38ef95dc5f2acc782c573b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 09:26:41 +0100 Subject: [PATCH 0629/1795] clean up eb optarch=generic settings --- EESSI-remove-software.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 6d0c9e3800..cdf4ae3b7b 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -18,7 +18,9 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in -g|--generic) + echo_yellow ">> GENERIC build requested, taking appropriate measures!" EASYBUILD_OPTARCH="GENERIC" + DETECTION_PARAMETERS="--generic" shift ;; -h|--help) @@ -55,14 +57,6 @@ else fatal_error "$EESSI_CVMFS_REPO is not available!" fi -DETECTION_PARAMETERS='' -EB='eb' -if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then - echo_yellow ">> GENERIC build requested, taking appropriate measures!" - DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" - EB='eb --optarch=GENERIC' -fi - echo ">> Determining software subdirectory to use for current build host..." if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) @@ -129,7 +123,7 @@ if [ $EUID -eq 0 ]; then # we need to remove existing installation directories first, # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - rebuild_apps=$(${EB} --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') for app in ${rebuild_apps}; do app_dir=${EASYBUILD_INSTALLPATH}/software/${app} app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua From f1cfc1b1f4f9f32d3669b46af781b2ab648d9c52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 09:38:48 +0100 Subject: [PATCH 0630/1795] source init/bash and clean up redundant code --- EESSI-remove-software.sh | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index cdf4ae3b7b..c1f0bcb3d0 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -47,6 +47,14 @@ export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX) source $TOPDIR/scripts/utils.sh +echo ">> Determining software subdirectory to use for current build host..." +if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" +else + echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" +fi + echo ">> Setting up environment..." source $TOPDIR/init/bash @@ -57,19 +65,6 @@ else fatal_error "$EESSI_CVMFS_REPO is not available!" fi -echo ">> Determining software subdirectory to use for current build host..." -if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" -else - echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" -fi - -# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) -# $EESSI_SILENT - don't print any messages -# $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables - if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then fatal_error "Failed to determine software subdirectory?!" elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then @@ -78,15 +73,6 @@ else echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" fi -echo ">> Initializing Lmod..." -source $EPREFIX/usr/share/Lmod/init/bash -ml --version -if [[ $? -eq 0 ]]; then - echo_green ">> Found Lmod ${LMOD_VERSION}" -else - fatal_error "Failed to initialize Lmod?!" -fi - echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild From aa4a82dd3c87eccf155715eee9797532f9ed8dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 09:48:48 +0100 Subject: [PATCH 0631/1795] set EB variable --- EESSI-remove-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index c1f0bcb3d0..9993986904 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -74,6 +74,7 @@ else fi echo ">> Configuring EasyBuild..." +EB="eb" source $TOPDIR/configure_easybuild echo ">> Setting up \$MODULEPATH..." From 6ca6a70671a019db0e83f54cb74e6049cba2a0f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 09:57:32 +0100 Subject: [PATCH 0632/1795] remove build-dir argument from display_help --- EESSI-remove-software.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 9993986904..10cea933db 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -8,7 +8,6 @@ display_help() { echo "usage: $0 [OPTIONS]" - echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds" echo " -g | --generic - instructs script to build for generic architecture target" echo " -h | --help - display this usage information" } From e89cb05d3bdcdb8df9c16cfdcea24afa81f8abf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 10:08:12 +0100 Subject: [PATCH 0633/1795] export the EASYBUILD_OPTARCH variable Co-authored-by: Kenneth Hoste --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 10cea933db..f465aff304 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -18,7 +18,7 @@ while [[ $# -gt 0 ]]; do case $1 in -g|--generic) echo_yellow ">> GENERIC build requested, taking appropriate measures!" - EASYBUILD_OPTARCH="GENERIC" + export EASYBUILD_OPTARCH="GENERIC" DETECTION_PARAMETERS="--generic" shift ;; From 502c02311cae893ef7983b340481c47684716997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 10:19:46 +0100 Subject: [PATCH 0634/1795] remove EASYBUILD_OPTARCH --- EESSI-remove-software.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index f465aff304..1cd291b16c 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -18,7 +18,6 @@ while [[ $# -gt 0 ]]; do case $1 in -g|--generic) echo_yellow ">> GENERIC build requested, taking appropriate measures!" - export EASYBUILD_OPTARCH="GENERIC" DETECTION_PARAMETERS="--generic" shift ;; From 3206a0ffa0b8759d8db5d67b5c3525d8b25ed674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 10:24:19 +0100 Subject: [PATCH 0635/1795] remove echo_yellow call for generic builds --- EESSI-remove-software.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 1cd291b16c..446a156cb8 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -17,7 +17,6 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in -g|--generic) - echo_yellow ">> GENERIC build requested, taking appropriate measures!" DETECTION_PARAMETERS="--generic" shift ;; From ee1c2cefd1b6461aa675288bef0cd2faa1163d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 10:27:20 +0100 Subject: [PATCH 0636/1795] use separate variable for removal script arguments --- bot/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 31d9b576fe..c76285faf4 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -171,8 +171,10 @@ PREVIOUS_TMP_DIR=${PWD}/previous_tmp # prepare arguments to install_software_layer.sh (specific to build step) declare -a BUILD_STEP_ARGS=() declare -a INSTALL_SCRIPT_ARGS=() +declare -a REMOVAL_SCRIPT_ARGS=() if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then INSTALL_SCRIPT_ARGS+=("--generic") + REMOVAL_SCRIPT_ARGS+=("--generic") fi [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") [[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") @@ -200,9 +202,9 @@ else echo "Executing command to remove software:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" - echo " -- ./EESSI-remove-software.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" + echo " -- ./EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ - -- ./EESSI-remove-software.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} + -- ./EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} # make sure that the build step resumes from the same temporary directory # this is important, as otherwise the removed software will still be there From 330db7a02cf011c2115babf598991217f87bd022 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Mar 2024 10:32:49 +0100 Subject: [PATCH 0637/1795] also dump status to test result file --- bot/check-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 4b5f7575e5..f045b9500a 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -200,5 +200,6 @@ comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} # Actually writing the comment description to the result file echo "${comment_description}" >> ${job_test_result_file} +echo "status = ${status}" >> ${job_test_result_file} exit 0 From f3c62077c40e299bb6a3f157adfd2c1f5a32d067 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Mar 2024 10:33:24 +0100 Subject: [PATCH 0638/1795] add Caspar as author in `bot/test.sh` script --- bot/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/test.sh b/bot/test.sh index 89c1a6a8bf..66c2cef129 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -8,6 +8,7 @@ # https://github.com/EESSI/software-layer.git # # author: Thomas Roeblitz (@trz42) +# author: Caspar van Leeuwen (@casparvl) # # license: GPLv2 # From c55ee6b6075786fd0dfd657ba3e2e667a215b9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 10:33:44 +0100 Subject: [PATCH 0639/1795] clarify why we need fakeroot, add link to issue --- bot/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index c76285faf4..b007b80172 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -194,7 +194,8 @@ else declare -a REMOVAL_STEP_ARGS=() REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") - # add fakeroot option in order to be able to remove software + # add fakeroot option in order to be able to remove software, see: + # https://github.com/EESSI/software-layer/issues/312 REMOVAL_STEP_ARGS+=("--fakeroot") # create tmp file for output of removal step From 13072df707b96cad40b4d21cbedd6666af756b0d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Mar 2024 10:34:09 +0100 Subject: [PATCH 0640/1795] use read-only mode to access software installations via container --- bot/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/test.sh b/bot/test.sh index 66c2cef129..925b91d49a 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -180,7 +180,7 @@ echo "bot/test.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" # prepare arguments to eessi_container.sh common to build and tarball steps declare -a COMMON_ARGS=() COMMON_ARGS+=("--verbose") -COMMON_ARGS+=("--access" "rw") +COMMON_ARGS+=("--access" "ro") COMMON_ARGS+=("--mode" "run") [[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") [[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") From 1fa323ada6d0fddc312ad102cba8619d073f9fb0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Mar 2024 10:34:53 +0100 Subject: [PATCH 0641/1795] fix typo: build -> test --- bot/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index 925b91d49a..4984340e6e 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -190,7 +190,7 @@ COMMON_ARGS+=("--mode" "run") # make sure to use the same parent dir for storing tarballs of tmp PREVIOUS_TMP_DIR=${PWD}/previous_tmp -# prepare directory to store tarball of tmp for build step +# prepare directory to store tarball of tmp for test step TARBALL_TMP_TEST_STEP_DIR=${PREVIOUS_TMP_DIR}/test_step mkdir -p ${TARBALL_TMP_TEST_STEP_DIR} @@ -216,7 +216,7 @@ fi # create tmp file for output of build step test_outerr=$(mktemp test.outerr.XXXX) -echo "Executing command to build software:" +echo "Executing command to test software:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}" echo " -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \ From 8d05473eb5eedf0cdaba75f6b4d4a40f38a3182d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Mar 2024 10:35:30 +0100 Subject: [PATCH 0642/1795] disable CPU auto-detection in template ReFrame configuration script used for running test suite --- reframe_config_bot.py.tmpl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl index b2f7916e79..0cc3e9f530 100644 --- a/reframe_config_bot.py.tmpl +++ b/reframe_config_bot.py.tmpl @@ -51,9 +51,8 @@ site_configuration = { { 'purge_environment': True, 'resolve_module_conflicts': False, # avoid loading the module before submitting the job - # Enable automatic detection of CPU architecture - # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information - 'remote_detect': True, + # disable automatic detection of CPU architecture (since we're using local scheduler) + 'remote_detect': False, } ], 'logging': common_logging_config(), From c53f578687e43ae0c7876c3a9d5c48fc2432ad49 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Mar 2024 10:37:09 +0100 Subject: [PATCH 0643/1795] pass down --generic to script used for running test suite, so tests are run with generic software installations when intended --- run_tests.sh | 2 +- test_suite.sh | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index de7fd8c2e1..1dbb47db9d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -21,4 +21,4 @@ source ${base_dir}/init/eessi_defaults ./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite" # Run the test suite -./test_suite.sh +./test_suite.sh "$@" diff --git a/test_suite.sh b/test_suite.sh index e3bab04aec..95eb9daa2a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -23,7 +23,7 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in -g|--generic) - EASYBUILD_OPTARCH="GENERIC" + DETECTION_PARAMETERS="--generic" shift ;; -h|--help) @@ -75,10 +75,9 @@ TMPDIR=$(mktemp -d) echo ">> Setting up environment..." module --force purge -# Make sure defaults are set for EESSI_CVMFS_REPO and EESSI_VERSION, before initializing EESSI -source $TOPDIR/init/eessi_defaults -# Initialize EESSI -source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash +export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) + +source $TOPDIR/init/bash # Load the ReFrame module # Currently, we load the default version. Maybe we should somehow make this configurable in the future? From 57d8b6fd4eceaf951c69cac9f325d5aa5eb113ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 10:54:13 +0100 Subject: [PATCH 0644/1795] remove openmpi hook for smcuda workaround --- create_lmodsitepackage.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 9a4a232863..5a7a915494 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -84,31 +84,10 @@ end end -local function eessi_openmpi_load_hook(t) - -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1, - -- to work around hang/crash due to bug in OpenMPI; - -- see https://gitlab.com/eessi/support/-/issues/41 - local frameStk = require("FrameStk"):singleton() - local mt = frameStk:mt() - local moduleName = string.match(t.modFullName, "(.-)/") - local cpuTarget = os.getenv("EESSI_SOFTWARE_SUBDIR") or "" - if (moduleName == "OpenMPI") and (cpuTarget == "aarch64/neoverse_v1") then - local msg = "Adding '^smcuda' to $OMPI_MCA_btl to work around bug in OpenMPI" - LmodMessage(msg .. " (see https://gitlab.com/eessi/support/-/issues/41)") - local ompiMcaBtl = os.getenv("OMPI_MCA_btl") - if ompiMcaBtl == nil then - setenv("OMPI_MCA_btl", "^smcuda") - else - setenv("OMPI_MCA_btl", ompiMcaBtl .. ",^smcuda") - end - end -end - -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) eessi_cuda_enabled_load_hook(t) - eessi_openmpi_load_hook(t) end From bef38bafea82a15300e6c9ded9d2dd2843c3dadf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Mar 2024 10:55:57 +0100 Subject: [PATCH 0645/1795] {2023.06}[2023a] OpenJPEG 2.5.0 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 384aa04a9c..0a1343295e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -52,3 +52,4 @@ easyconfigs: - ipympl-0.9.3-foss-2023a.eb: options: from-pr: 20126 + - OpenJPEG-2.5.0-GCCcore-12.3.0.eb From a0dc2815c4f45162aed56bf2c5e7bfcdc8b0eaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 11:03:33 +0100 Subject: [PATCH 0646/1795] fix comment for creating lmod rc code block --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 31ce30d4fc..b2984af856 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -247,7 +247,7 @@ fi ### add packages here -echo ">> Creating/updating Lmod cache..." +echo ">> Creating/updating Lmod RC file..." export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?) From b2570ed8cca9d7f87d3c506ec33d7b0c7b033671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 11:03:50 +0100 Subject: [PATCH 0647/1795] don't update lmod cache here anymore --- EESSI-install-software.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index b2984af856..7b7a60686c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -265,7 +265,5 @@ if [ ! -f "$lmod_sitepackage_file" ] || [ "${sitepackage_changed}" == '0' ]; the check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file" fi -$TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} - echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} From 56e4b87d1c0de54f374ca77d7c316be173d8bb9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 14:56:15 +0100 Subject: [PATCH 0648/1795] add LittleCMS, giflib, OpenJPEG to fix check_missing CI --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index e2d35276df..380c0107c6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -49,3 +49,6 @@ easyconfigs: - libspatialindex-1.9.3-GCCcore-13.2.0.eb: options: from-pr: 19922 + - LittleCMS-2.15-GCCcore-13.2.0.eb + - giflib-5.2.1-GCCcore-13.2.0.eb + - OpenJPEG-2.5.0-GCCcore-13.2.0.eb From 32f04020a1b3e3c7061107d221dc4faf155c7fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 15:38:05 +0100 Subject: [PATCH 0649/1795] also add libwebp-1.3.2-GCCcore-13.2.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 380c0107c6..25cdde27e6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -52,3 +52,5 @@ easyconfigs: - LittleCMS-2.15-GCCcore-13.2.0.eb - giflib-5.2.1-GCCcore-13.2.0.eb - OpenJPEG-2.5.0-GCCcore-13.2.0.eb + - libwebp-1.3.2-GCCcore-13.2.0.eb + From f2844fb210aa865155989811c88d85b7d8b3c1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Mar 2024 17:09:49 +0100 Subject: [PATCH 0650/1795] remove OpenMPI, which has been rebuilt in #488 --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index b971c5adad..1c3ea8fd78 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -55,8 +55,4 @@ easyconfigs: - gnuplot-5.4.8-GCCcore-12.3.0.eb: options: from-pr: 19261 - - OpenMPI-4.1.5-GCC-12.3.0.eb: - options: - from-pr: 19940 - rebuild: True - OpenFOAM-10-foss-2023a.eb From 5a43f9ef8d8d66dfa2bd0226f28a8bdf7521892b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 26 Mar 2024 19:16:00 +0100 Subject: [PATCH 0651/1795] Don't need gnuplot, install with EasyBuild 4.9.0 --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 4 ---- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 1c3ea8fd78..7244219dc3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -52,7 +52,3 @@ easyconfigs: - BWA-0.7.17-20220923-GCCcore-12.3.0.eb: options: from-pr: 19820 - - gnuplot-5.4.8-GCCcore-12.3.0.eb: - options: - from-pr: 19261 - - OpenFOAM-10-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 157a47a49e..13d7ebdc02 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -29,3 +29,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 options: from-pr: 19554 + - OpenFOAM-10-foss-2023a.eb From 7c7e1b1ff18c960db5f1233438295d9c60dd6431 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 26 Mar 2024 20:18:03 +0100 Subject: [PATCH 0652/1795] use older apptainer (1.2.4) --- install_apptainer_ubuntu.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index c35c34cda6..6cf9a6f48e 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -5,8 +5,11 @@ set -e # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 sudo apt-get install alien alien --version -apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') -curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm} +#apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') +#curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm} +#/pub/archive/epel/8.8/Everything/x86_64/Packages/a +apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') +curl -OL https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/${apptainer_rpm} sudo alien -d ${apptainer_rpm} sudo apt install ./apptainer*.deb apptainer --version From deb12e9fef3103b8c07504e745b14fe366ed618d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Mar 2024 20:35:40 +0100 Subject: [PATCH 0653/1795] clean up downloading RPM for Apptainer < 1.3.0 RPM from EPEL 8.8 archive, for now --- install_apptainer_ubuntu.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index 6cf9a6f48e..5eb513db2c 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -5,11 +5,13 @@ set -e # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 sudo apt-get install alien alien --version -#apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') -#curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm} -#/pub/archive/epel/8.8/Everything/x86_64/Packages/a -apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') -curl -OL https://dl.fedoraproject.org/pub/archive/epel/8.8/Everything/x86_64/Packages/a/${apptainer_rpm} +# stick to Apptainer < 1.3.0 by downloading from EPEL 8.8 archive, +# since CI workflow for testing scripts hangs/fails when using Apptainer 1.3.0 +# cfr. https://github.com/EESSI/software-layer/pull/514 +epel_subdir="pub/epel/8" +epel_subdir="pub/archive/epel/8.8" +apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') +curl -OL https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/${apptainer_rpm} sudo alien -d ${apptainer_rpm} sudo apt install ./apptainer*.deb apptainer --version From 547386f9180687627ce506cab0cacc91ed808025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 10:08:14 +0100 Subject: [PATCH 0654/1795] dont fail if grep doesnt find rebuild easystack files --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index b007b80172..2438adbcd9 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -182,7 +182,7 @@ fi # determine if the removal step has to be run # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) -changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/") +changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true) if [[ -z ${changed_easystacks_rebuilds} ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else From c2fad0032008d0ca73f6603ad7fbae6c2391a6a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 10:08:32 +0100 Subject: [PATCH 0655/1795] add quotes around variable in if statement --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 2438adbcd9..e9f8ecfbb9 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -183,7 +183,7 @@ fi # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true) -if [[ -z ${changed_easystacks_rebuilds} ]]; then +if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else # prepare directory to store tarball of tmp for removal and build steps From bce1067e0b9962c591804c87ea150e38123822be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 10:13:32 +0100 Subject: [PATCH 0656/1795] use proper tmp dir for removal step --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index e9f8ecfbb9..0ebd55692e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -192,7 +192,7 @@ else # prepare arguments to eessi_container.sh specific to remove step declare -a REMOVAL_STEP_ARGS=() - REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") + REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}") REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") # add fakeroot option in order to be able to remove software, see: # https://github.com/EESSI/software-layer/issues/312 From 4a9d9aeedd5fc7dc719bbc9612c6b5e2213a673b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 10:20:16 +0100 Subject: [PATCH 0657/1795] let the tarball step resume from the right directory --- bot/build.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 0ebd55692e..0d23f59b66 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -247,7 +247,14 @@ declare -a TARBALL_STEP_ARGS=() TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}") # determine temporary directory to resume from -TARBALL_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") +if [[ -z ${REMOVAL_TMPDIR} ]]; then + # no rebuild step was done, so the tarball step should resume from the build directory + BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) + TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") +else + # a removal step was done, so resume from its temporary directory (which was also used for the build step) + TARBALL_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") +fi timestamp=$(date +%s) # to set EESSI_VERSION we need to source init/eessi_defaults now From 2ba8513f770f2f23961a05bc64b98488e4aaaa8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 11:14:05 +0100 Subject: [PATCH 0658/1795] group grep and true commands Co-authored-by: Kenneth Hoste --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 0d23f59b66..ff933a447d 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -182,7 +182,7 @@ fi # determine if the removal step has to be run # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) -changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | grep "/rebuilds/" || true) +changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | (grep "/rebuilds/" || true)) if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else From 0e38a4769d7db0e893f9a4eb058ec9333a0e648e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 11:16:35 +0100 Subject: [PATCH 0659/1795] add comment about true command --- bot/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index ff933a447d..dcc61c19d4 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -182,6 +182,8 @@ fi # determine if the removal step has to be run # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) +# the true at the end of the next command is important: grep will expectedly return 1 if there is no easystack file being added under rebuilds, +# but due to "set -e" the entire script would otherwise fail changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | (grep "/rebuilds/" || true)) if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." From e9bb5b8da5e7634396fca503bdafbe11861c0f42 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 27 Mar 2024 14:25:47 +0000 Subject: [PATCH 0660/1795] {2023.06}[GCCcore/13.2.0] Wayland v1.22.0 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 25cdde27e6..6710c07b49 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -53,4 +53,5 @@ easyconfigs: - giflib-5.2.1-GCCcore-13.2.0.eb - OpenJPEG-2.5.0-GCCcore-13.2.0.eb - libwebp-1.3.2-GCCcore-13.2.0.eb + - Wayland-1.22.0-GCCcore-13.2.0.eb From 80609663bf16cf013df9922c2229976c19e821df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 16:08:52 +0100 Subject: [PATCH 0661/1795] add quotes around changed_easystacks variable --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 470663e45a..e2d7cf8c51 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -204,7 +204,7 @@ ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12 # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') -if [ -z ${changed_easystacks} ]; then +if [ -z "${changed_easystacks}" ]; then echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else From 70935ea4293d0e7a94b99f573a77fe74b4f68fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 27 Mar 2024 16:13:07 +0100 Subject: [PATCH 0662/1795] first process rebuilds, then new installations --- EESSI-install-software.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index e2d7cf8c51..a905d966f6 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -208,7 +208,11 @@ if [ -z "${changed_easystacks}" ]; then echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else - for easystack_file in ${changed_easystacks}; do + # first process rebuilds, if any, then easystack files for new installations + # "|| true" is used to make sure that the grep command always returns success + rebuild_easystacks=$(echo "${changed_easystacks}" | (grep "/rebuilds/" || true)) + new_easystacks=$(echo "${changed_easystacks}" | (grep -v "/rebuilds/" || true)) + for easystack_file in ${rebuild_easystacks} ${new_easystacks}; do echo -e "Processing easystack file ${easystack_file}...\n\n" From 005c02c4341f02aedb5a0dc7cf3a3c1e96ca2b56 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 28 Mar 2024 00:18:12 +0100 Subject: [PATCH 0663/1795] Rebuild GCCcore-12.3.0 and 13.2.0 to fix aarch64 vectorization issue --- ...eb-4.9.0-GCCcore-fix-aarch64-vectorization.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240328-eb-4.9.0-GCCcore-fix-aarch64-vectorization.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240328-eb-4.9.0-GCCcore-fix-aarch64-vectorization.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240328-eb-4.9.0-GCCcore-fix-aarch64-vectorization.yml new file mode 100644 index 0000000000..f63aa421f5 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240328-eb-4.9.0-GCCcore-fix-aarch64-vectorization.yml @@ -0,0 +1,15 @@ +# 2024-03-28 +# Rebuild GCCcore to fix a compiler bug in the tree-vectorizer +# We encountered it in https://github.com/EESSI/software-layer/pull/479#issuecomment-1957091774 +# and https://github.com/EESSI/software-layer/pull/507#issuecomment-2011724613 +# Upstream issue: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111478 +# Upstream fix: https://gcc.gnu.org/git/gitweb.cgi?p=gcc.git;h=e5f1956498251a4973d52c8aad3faf34d0443169 +# Fix in EasyBuild https://github.com/easybuilders/easybuild-easyconfigs/pull/19974 +# https://github.com/easybuilders/easybuild-easyconfigs/pull/20218 +easyconfigs: + - GCCcore-12.3.0.eb: + options: + from-pr: 20218 + - GCCcore-13.2.0.eb: + options: + from-pr: 19974 From 1c1dd06db1a9b52b827b7adc87f412db586714cd Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 28 Mar 2024 10:27:33 +0100 Subject: [PATCH 0664/1795] Added Modflow and deps --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 6710c07b49..d0f1f33914 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -54,4 +54,13 @@ easyconfigs: - OpenJPEG-2.5.0-GCCcore-13.2.0.eb - libwebp-1.3.2-GCCcore-13.2.0.eb - Wayland-1.22.0-GCCcore-13.2.0.eb - + - SuperLU_DIST-8.1.2-foss-2023a.eb: + options: + from-pr: 20162 + - PETSc-3.20.3-foss-2023a.eb: + options: + include-easyblocks-from-pr: 3086 + from-pr: 19686 + - MODFLOW-6.4.4-foss-2023a.eb: + options: + from-pr: 20142 From 1489742eae3de68def89474d40945177cc331cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 28 Mar 2024 11:11:07 +0100 Subject: [PATCH 0665/1795] use easyconfigs PR #20238 for R 4.2.2 --- .../2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 629023a1cd..3962f63bda 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -8,4 +8,6 @@ easyconfigs: - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb: options: from-pr: 20086 - - R-4.2.2-foss-2022b.eb + - R-4.2.2-foss-2022b.eb: + options: + from-pr: 20238 From 6e806faef8879bad179c852aa9ec8dc76eca4a1a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 15:12:06 +0100 Subject: [PATCH 0666/1795] Make sure EESSI's SitePackage.lua loads host's SitePackage.lua's if they exist --- create_lmodsitepackage.py | 68 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 5a7a915494..4f99f1a614 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -19,6 +19,71 @@ return content end +local function load_sitespecific_hooks() + -- This function will be run after the EESSI hooks are registered + -- It will load a local SitePackage.lua that is architecture independent (if it exists) from e.g. + -- /cvmfs/software.eessi.io/host_injections/2023.06/.lmod/SitePackage.lua + -- That can define a new hook + -- + -- function site_specific_load_hook(t) + -- + -- end + -- + -- And the either append to the existing hook: + -- + -- local function final_load_hook(t) + -- eessi_load_hook(t) + -- site_specific_load_hook(t) + -- end + -- + -- Over overwrite the EESSI hook entirely: + -- + -- hook.register("load", final_load_hook) + -- + -- Note that the appending procedure can be simplified once we have an lmod >= 8.7.36 + -- See https://github.com/TACC/Lmod/pull/696#issuecomment-1998765722 + -- + -- Subsequently, this function will look for an architecture-specific SitePackage.lua, e.g. from + -- /cvmfs/software.eessi.io/host_injections/2023.06/software/linux/x86_64/amd/zen2/.lmod/SitePackage.lua + -- This can then register an additional hook, e.g. + -- + -- function arch_specific_load_hook(t) + -- + -- end + -- + -- local function final_load_hook(t) + -- eessi_load_hook(t) + -- site_specific_load_hook(t) + -- arch_specific_load_hook(t) + -- end + -- + -- hook.register("load", final_load_hook) + -- + -- Again, the host site could also decide to overwrite by simply doing + -- + -- hook.register("load", arch_specific_load_hook) + + -- get path to to architecture independent SitePackage.lua + local prefixHostInjections = string.gsub(os.getenv('EESSI_PREFIX'), 'versions', 'host_injections') + local hostSitePackage = prefixHostInjections .. "/.lmod/SitePackage.lua" + + -- If the file exists, run it + if isDir(hostSitePackage) then + dofile(hostSitePackage) + end + + -- build the full architecture specific path in host_injections + local archHostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') + local archSitePackage = archHostInjections .. "/.lmod/SitePackage.lua" + + -- If the file exists, run it + if isDir(archSitePackage) + dofile(archSitePackage) + end + +end + + local function eessi_cuda_enabled_load_hook(t) local frameStk = require("FrameStk"):singleton() local mt = frameStk:mt() @@ -92,6 +157,9 @@ hook.register("load", eessi_load_hook) + +-- Note that this needs to happen at the end, so that any EESSI specific hooks can be overwritten by the site +load_site_specific_hooks() """ def error(msg): From 4bc0369d3505347441167436fb2f8de1a005b6b1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 15:14:17 +0100 Subject: [PATCH 0667/1795] Forgot 'then' --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 4f99f1a614..6b67b4beca 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -77,7 +77,7 @@ local archSitePackage = archHostInjections .. "/.lmod/SitePackage.lua" -- If the file exists, run it - if isDir(archSitePackage) + if isDir(archSitePackage) then dofile(archSitePackage) end From cd9b2061a2d83669bf42a3c8d4f64de97afae3cc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 15:49:51 +0100 Subject: [PATCH 0668/1795] Don't set LMOD_RC any longer, we set LMOD_CONFIG_DIR now, so that sites can add their own properties via LMOD_RC if they want. See https://lmod.readthedocs.io/en/latest/145_properties.html --- init/eessi_environment_variables | 34 +++++++++++++------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index a3ff265e77..5450b2bfb4 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -51,14 +51,23 @@ if [ -d $EESSI_PREFIX ]; then show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR - # Configure our LMOD_RC file - export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua" - if [ -f $LMOD_RC ]; then - show_msg "Found Lmod configuration file at $LMOD_RC" + # Configure our LMOD + export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod" + lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" + if [ -f $lmod_rc_file ]; then + show_msg "Found Lmod configuration file at $lmod_rc_file" else - error "Lmod configuration file not found at $LMOD_RC" + error "Lmod configuration file not found at $lmod_rc_file" fi + export LMOD_PACKAGE_PATH="$EESSI_SOFTWARE_PATH/.lmod" + lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" + if [ -f $lmod_sitepackage_file ]; then + show_msg "Found Lmod SitePackage.lua file at $lmod_sitepackage_file" + else + error "Lmod SitePackage.lua file not found at $lmod_sitepackage_file" + fi + if [ ! -z $EESSI_BASIC_ENV ]; then show_msg "Only setting up basic environment, so we're done" elif [ -d $EESSI_SOFTWARE_PATH ]; then @@ -85,21 +94,6 @@ if [ -d $EESSI_PREFIX ]; then false fi - export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod" - lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" - if [ -f $lmod_rc_file ]; then - show_msg "Found Lmod configuration file at $lmod_rc_file" - else - error "Lmod configuration file not found at $lmod_rc_file" - fi - - export LMOD_PACKAGE_PATH="$EESSI_SOFTWARE_PATH/.lmod" - lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" - if [ -f $lmod_sitepackage_file ]; then - show_msg "Found Lmod SitePackage.lua file at $lmod_sitepackage_file" - else - error "Lmod SitePackage.lua file not found at $lmod_sitepackage_file" - fi else error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!" From d2c5dabb735709ef4ab50c3643d5620a7ea97f12 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 15:52:44 +0100 Subject: [PATCH 0669/1795] Add some comment, the lmodrc file did not get properly deployed in https://github.com/EESSI/software-layer/pull/496 . We need to have a change to redeploy it --- create_lmodrc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/create_lmodrc.py b/create_lmodrc.py index ae65153a20..28ad2a1915 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -7,6 +7,7 @@ DOT_LMOD = '.lmod' +# LMOD_RC file is the place to define properties, see https://lmod.readthedocs.io/en/latest/145_properties.html TEMPLATE_LMOD_RC = """propT = { } scDescriptT = { From 70d94a1361eaa02a8dea4c64495480f0a4cc6910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 30 Mar 2024 22:58:48 +0100 Subject: [PATCH 0670/1795] add R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 3962f63bda..8a3e82d760 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -11,3 +11,4 @@ easyconfigs: - R-4.2.2-foss-2022b.eb: options: from-pr: 20238 + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb From 173332f479d74cd220cb25976a17035317e613ae Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 13:02:46 +0200 Subject: [PATCH 0671/1795] Try to retrigger issue 517 --- install_apptainer_ubuntu.sh | 6 ++---- scripts/gpu_support/nvidia/install_cuda_host_injections.sh | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index 5eb513db2c..8642d62999 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -5,11 +5,9 @@ set -e # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 sudo apt-get install alien alien --version -# stick to Apptainer < 1.3.0 by downloading from EPEL 8.8 archive, -# since CI workflow for testing scripts hangs/fails when using Apptainer 1.3.0 -# cfr. https://github.com/EESSI/software-layer/pull/514 +# Switch back to Apptainer >= 1.3.0 to retrigger +# https://github.com/EESSI/software-layer/pull/514 epel_subdir="pub/epel/8" -epel_subdir="pub/archive/epel/8.8" apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') curl -OL https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/${apptainer_rpm} sudo alien -d ${apptainer_rpm} diff --git a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh index a9310d817a..1f1548f7dd 100755 --- a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh +++ b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh @@ -12,6 +12,9 @@ # installation to be successful, this directory needs to be writeable by the user # executing this script. +# TRIGGER SCRIPT CHANGE TO REPRODUCE https://github.com/EESSI/software-layer/issues/517 +# THIS COMMENT SHOULD NEVER BE MERGED! + # Initialise our bash functions TOPDIR=$(dirname $(realpath $BASH_SOURCE)) source "$TOPDIR"/../../utils.sh From 2ace856380267a9e35c44326d5ad2dd2bc61c9a2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 13:10:28 +0200 Subject: [PATCH 0672/1795] Undo change --- scripts/gpu_support/nvidia/install_cuda_host_injections.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh index 1f1548f7dd..a9310d817a 100755 --- a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh +++ b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh @@ -12,9 +12,6 @@ # installation to be successful, this directory needs to be writeable by the user # executing this script. -# TRIGGER SCRIPT CHANGE TO REPRODUCE https://github.com/EESSI/software-layer/issues/517 -# THIS COMMENT SHOULD NEVER BE MERGED! - # Initialise our bash functions TOPDIR=$(dirname $(realpath $BASH_SOURCE)) source "$TOPDIR"/../../utils.sh From dc986adc8b46f622d2d4d899922f5ef4731c305b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 13:11:11 +0200 Subject: [PATCH 0673/1795] Retrigger 517 --- scripts/utils.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/utils.sh b/scripts/utils.sh index b2be3f6221..bbe17584b8 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -1,3 +1,6 @@ +## THIS COMMENT IS ONLY TO RETRIGGER #517 +## IT SHOULD NEVER BE MERGED + function echo_green() { echo -e "\e[32m$1\e[0m" } From 6d5a6ae36e86253b4811e9a9f2ec2ca028558e77 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 14:05:30 +0200 Subject: [PATCH 0674/1795] Add option to skip CUDA SDK installation, e.g. for CI environments --- EESSI-install-software.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index a905d966f6..2fff281ff5 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -14,6 +14,7 @@ display_help() { echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" echo " --shared-fs-path - path to directory on shared filesystem that can be used" + echo " --skip-cuda-install - disable installing a full CUDA SDK in the host_injections prefix (e.g. in CI)" } function copy_build_log() { @@ -76,6 +77,10 @@ while [[ $# -gt 0 ]]; do export shared_fs_path="${2}" shift 2 ;; + --skip-cuda-install) + export skip_cuda_install=True + shift 1 + ;; -*|--*) echo "Error: Unknown option: $1" >&2 exit 1 @@ -195,7 +200,10 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} # Install full CUDA SDK in host_injections # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install -${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula +# Allow skipping CUDA SDK install in e.g. CI environments +if [ ! -z "${skip_cuda_install}" ] && [ ! "${skip_cuda_install}" ]; then + ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula +fi # Install drivers in host_injections # TODO: this is commented out for now, because the script assumes that nvidia-smi is available and works; From 6155e02b317624f287f5bccb694c7b4285d7e388 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 14:18:40 +0200 Subject: [PATCH 0675/1795] Make sure the install_software_layer.sh test skips installing the CUDA SDK - that is simply too heave for CI --- .github/workflows/tests_scripts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index df1884dd8c..b1d2426774 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -83,6 +83,8 @@ jobs: export SINGULARITY_BIND="${PWD}:/software-layer" # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh + # skip installation of CUDA SDKs, since this is too heavy for CI + sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh - name: test create_directory_tarballs.sh script From 877f364e677a31f8218a973640269d003f95fa4b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 14:23:23 +0200 Subject: [PATCH 0676/1795] Fix logic in EESSI-install-software.sh and prove that logic is ok by now first retriggering the issue --- .github/workflows/tests_scripts.yml | 2 +- EESSI-install-software.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index b1d2426774..a245ee1ad5 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -84,7 +84,7 @@ jobs: # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh # skip installation of CUDA SDKs, since this is too heavy for CI - sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh + # sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh - name: test create_directory_tarballs.sh script diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2fff281ff5..0374d8e758 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -201,7 +201,7 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install # Allow skipping CUDA SDK install in e.g. CI environments -if [ ! -z "${skip_cuda_install}" ] && [ ! "${skip_cuda_install}" ]; then +if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula fi From ba142caeea7d6223354b3aa7554f605ef4468fe4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 14:28:16 +0200 Subject: [PATCH 0677/1795] Now make sure CI skips CUDA SDK installation, to prove that this fixes the CI run --- .github/workflows/tests_scripts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index a245ee1ad5..b1d2426774 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -84,7 +84,7 @@ jobs: # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh # skip installation of CUDA SDKs, since this is too heavy for CI - # sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh + sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh - name: test create_directory_tarballs.sh script From dc915fb0b9ec5577787fa58cd618a44a77ff1ff3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 16:01:39 +0200 Subject: [PATCH 0678/1795] Print sensible message to log when the cuda installation is skipped --- EESSI-install-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 0374d8e758..567fed8e79 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -203,6 +203,8 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} # Allow skipping CUDA SDK install in e.g. CI environments if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula +else + echo "Skipping installation of CUDA SDK in host_injections, since the --skip-cuda-install flag was passed" fi # Install drivers in host_injections From 4cf4d76cc80ef5e5aa5df73cef628fca3ad56b99 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 16:03:14 +0200 Subject: [PATCH 0679/1795] Issue is resolved, so we can use Apptainer 1.3.0 --- install_apptainer_ubuntu.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index 8642d62999..5c4f37ac2d 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -5,8 +5,6 @@ set -e # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 sudo apt-get install alien alien --version -# Switch back to Apptainer >= 1.3.0 to retrigger -# https://github.com/EESSI/software-layer/pull/514 epel_subdir="pub/epel/8" apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') curl -OL https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/${apptainer_rpm} From c92ceb2e07ddf37fc12ec86b0f6d90ded0423482 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 16:04:12 +0200 Subject: [PATCH 0680/1795] Revert changes to utils.sh --- scripts/utils.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/utils.sh b/scripts/utils.sh index bbe17584b8..b2be3f6221 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -1,6 +1,3 @@ -## THIS COMMENT IS ONLY TO RETRIGGER #517 -## IT SHOULD NEVER BE MERGED - function echo_green() { echo -e "\e[32m$1\e[0m" } From b59d327ecce05ad1e5904fe2581c45c3e0b68d10 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 17:19:14 +0200 Subject: [PATCH 0681/1795] Changed isDir to isFile --- create_lmodsitepackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 6b67b4beca..cddb40da5c 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -68,7 +68,7 @@ local hostSitePackage = prefixHostInjections .. "/.lmod/SitePackage.lua" -- If the file exists, run it - if isDir(hostSitePackage) then + if isFile(hostSitePackage) then dofile(hostSitePackage) end @@ -77,7 +77,7 @@ local archSitePackage = archHostInjections .. "/.lmod/SitePackage.lua" -- If the file exists, run it - if isDir(archSitePackage) then + if isFile(archSitePackage) then dofile(archSitePackage) end From 9ff3cfab728cf66177b903aaa0e9d63099da2c2d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 17:20:23 +0200 Subject: [PATCH 0682/1795] Account for the possibility that EESSI_PREFIX isn't set --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index cddb40da5c..30f11c9c6f 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -64,7 +64,7 @@ -- hook.register("load", arch_specific_load_hook) -- get path to to architecture independent SitePackage.lua - local prefixHostInjections = string.gsub(os.getenv('EESSI_PREFIX'), 'versions', 'host_injections') + local prefixHostInjections = string.gsub(os.getenv('EESSI_PREFIX') or "", 'versions', 'host_injections') local hostSitePackage = prefixHostInjections .. "/.lmod/SitePackage.lua" -- If the file exists, run it From f25bfe09cf46f15d1869c32019640005aab2006b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 17:31:17 +0200 Subject: [PATCH 0683/1795] Corrected function name --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 30f11c9c6f..29b2d39bd7 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -19,7 +19,7 @@ return content end -local function load_sitespecific_hooks() +local function load_site_specific_hooks() -- This function will be run after the EESSI hooks are registered -- It will load a local SitePackage.lua that is architecture independent (if it exists) from e.g. -- /cvmfs/software.eessi.io/host_injections/2023.06/.lmod/SitePackage.lua From 12d17a4777b2e2aa4f9ddaa1433d75d9c0012c59 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Apr 2024 17:43:01 +0200 Subject: [PATCH 0684/1795] also test create_lmodsitepackage.py in CI --- .github/workflows/tests_scripts.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index df1884dd8c..476be3dc4d 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -5,6 +5,7 @@ on: paths: - build_container.sh - create_directory_tarballs.sh + - create_lmodsitepackage.py - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh @@ -16,6 +17,7 @@ on: paths: - build_container.sh - create_directory_tarballs.sh + - create_lmodsitepackage.py - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh @@ -94,3 +96,22 @@ jobs: ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz + + - name: test create_lmodsitepackage.py script + run: | + # bind current directory into container as /software-layer + export SINGULARITY_BIND="${PWD}:/software-layer" + + python3 create_lmodsitepackage.py . + export LMOD_PACKAGE_PATH="$PWD/.lmod" + # run some commands to make sure that generated Lmod SitePackage file works + test_script="${PWD}/test_lmod_sitepackage.sh" + echo '#!/bin/bash' > ${test_script} + echo 'ml --config' >> ${test_script} + + chmod u+x ${test_script} + + ./eessi_container.sh --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/test_lmod_sitepackage.sh 2>&1 | tee ${out} + for pattern in "^Site Pkg location.*$PWD/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*${PWD}/.lmod/SitePackage.lua"; do + grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) + done From 60c46ce702948e3c97e1dc432f5651f837efb39c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 18:38:23 +0200 Subject: [PATCH 0685/1795] Make sure export is done inside test script, as environment is lost when running in compat layer env --- .github/workflows/tests_scripts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 476be3dc4d..95baedac08 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -102,11 +102,12 @@ jobs: # bind current directory into container as /software-layer export SINGULARITY_BIND="${PWD}:/software-layer" + # Creates .lmod/SitePackage.lua in current dir, which then gets bind-mounted into /software-layer python3 create_lmodsitepackage.py . - export LMOD_PACKAGE_PATH="$PWD/.lmod" # run some commands to make sure that generated Lmod SitePackage file works test_script="${PWD}/test_lmod_sitepackage.sh" echo '#!/bin/bash' > ${test_script} + echo 'export LMOD_PACKAGE_PATH="/software-layer/.lmod"' > ${test_script} echo 'ml --config' >> ${test_script} chmod u+x ${test_script} From d59238b9643de5d34a301122afef133c231747fa Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 18:46:58 +0200 Subject: [PATCH 0686/1795] Change expected output - we need to account for the fact that this is a bind-mounted location --- .github/workflows/tests_scripts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 95baedac08..96838b99df 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -113,6 +113,6 @@ jobs: chmod u+x ${test_script} ./eessi_container.sh --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/test_lmod_sitepackage.sh 2>&1 | tee ${out} - for pattern in "^Site Pkg location.*$PWD/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*${PWD}/.lmod/SitePackage.lua"; do + for pattern in "^Site Pkg location.*/software-layer/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer/.lmod/SitePackage.lua"; do grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) done From 1e8eb6eac7a040cf16d2f97d4126b93300da05c7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 2 Apr 2024 19:12:20 +0200 Subject: [PATCH 0687/1795] Output variable was undefined --- .github/workflows/tests_scripts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 96838b99df..ac55231bf0 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -112,6 +112,7 @@ jobs: chmod u+x ${test_script} + out="${PWD}/test_create_lmodsitepackage.out" ./eessi_container.sh --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/test_lmod_sitepackage.sh 2>&1 | tee ${out} for pattern in "^Site Pkg location.*/software-layer/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer/.lmod/SitePackage.lua"; do grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1) From 542022c9e5a4e2109416c4476c621e1b7bb6ed82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 10:12:50 +0200 Subject: [PATCH 0688/1795] add hook that disables tests for Highway 1.0.4 on neoverse_v1 --- eb_hooks.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index d93ee37067..94591ca91f 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -376,6 +376,18 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!") +def pre_configure_hook_highway_disable_tests((self, *args, **kwargs): + """ + pre-configure hook for Highway: disable tests on neoverse_v1 + cfr. https://github.com/EESSI/software-layer/issues/469 + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'Highway ' and cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') + else: + raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") + + def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: @@ -625,6 +637,7 @@ def inject_gpu_property(ec): 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, + 'Highway': pre_configure_hook_highway_disable_tests, } PRE_TEST_HOOKS = { From a770536d7122c0bf9da89573082c180988798d93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 10:12:59 +0200 Subject: [PATCH 0689/1795] add Highway-1.0.4-GCCcore-12.3.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 9b0de81b35..ac9c36fe82 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -54,3 +54,4 @@ easyconfigs: from-pr: 20126 - OpenJPEG-2.5.0-GCCcore-12.3.0.eb - OpenFOAM-10-foss-2023a.eb + - Highway-1.0.4-GCCcore-12.3.0.eb From 3cc125d7996b04cea0bcd71e1fff767113666310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 10:15:40 +0200 Subject: [PATCH 0690/1795] add known issue about Highway 1.0.4 on neoverse_v1 --- eessi-2023.06-known-issues.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 875757559e..d818c85952 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -19,6 +19,9 @@ - Highway-1.0.3-GCCcore-12.2.0.eb: - issue: https://github.com/EESSI/software-layer/issues/469 - info: "failing SVE test due to wrong expected value" + - Highway-1.0.4-GCCcore-12.3.0.eb: + - issue: https://github.com/EESSI/software-layer/issues/469 + - info: "failing to build the tests" - netCDF-4.9.2-gompi-2023a.eb: - issue: https://github.com/EESSI/software-layer/issues/425 - info: "netCDF intermittent test failures" From 098d5253901c4e32e2c5f9f4e331be29efcc04ea Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 3 Apr 2024 10:46:16 +0200 Subject: [PATCH 0691/1795] Update eb_hooks.py --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 94591ca91f..4b29c84eb2 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -382,7 +382,7 @@ def pre_configure_hook_highway_disable_tests((self, *args, **kwargs): cfr. https://github.com/EESSI/software-layer/issues/469 """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'Highway ' and cpu_target == CPU_TARGET_NEOVERSE_V1: + if self.name == 'Highway' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') else: raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") From 10e1d00ebce2fe8753b2e638e12a8b0f01a33b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 10:47:41 +0200 Subject: [PATCH 0692/1795] fix syntax, remove ( --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 94591ca91f..78432c7a00 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -376,7 +376,7 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!") -def pre_configure_hook_highway_disable_tests((self, *args, **kwargs): +def pre_configure_hook_highway_disable_tests(self, *args, **kwargs): """ pre-configure hook for Highway: disable tests on neoverse_v1 cfr. https://github.com/EESSI/software-layer/issues/469 From 24d5bcfe80061958699ea8376f8ed08780c630a1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 3 Apr 2024 11:33:10 +0200 Subject: [PATCH 0693/1795] Move software builds to the easystack file for the right toolchain... --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 10 ++++++++++ .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 11 +---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 9b0de81b35..0c260cea5c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -54,3 +54,13 @@ easyconfigs: from-pr: 20126 - OpenJPEG-2.5.0-GCCcore-12.3.0.eb - OpenFOAM-10-foss-2023a.eb + - SuperLU_DIST-8.1.2-foss-2023a.eb: + options: + from-pr: 20162 + - PETSc-3.20.3-foss-2023a.eb: + options: + include-easyblocks-from-pr: 3086 + from-pr: 19686 + - MODFLOW-6.4.4-foss-2023a.eb: + options: + from-pr: 20142 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 94303fd2b4..1e94f7be78 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -57,13 +57,4 @@ easyconfigs: - Qt5-5.15.13-GCCcore-13.2.0.eb: options: from-pr: 20201 - - SuperLU_DIST-8.1.2-foss-2023a.eb: - options: - from-pr: 20162 - - PETSc-3.20.3-foss-2023a.eb: - options: - include-easyblocks-from-pr: 3086 - from-pr: 19686 - - MODFLOW-6.4.4-foss-2023a.eb: - options: - from-pr: 20142 + From 5509f6c33fd15c6f7b6dfdf7f182fe839a44c758 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 3 Apr 2024 11:36:19 +0200 Subject: [PATCH 0694/1795] Remove stray blank line --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 1e94f7be78..86c5106c85 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -57,4 +57,3 @@ easyconfigs: - Qt5-5.15.13-GCCcore-13.2.0.eb: options: from-pr: 20201 - From abfe1bf536e91938b4fba50c33585962e8d07042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 12:04:33 +0200 Subject: [PATCH 0695/1795] add toolchain and version check to Highway hook that disables the tests --- eb_hooks.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index eee2a4dfde..9ccd984de4 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -289,6 +289,21 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") +def parse_hook_highway_disable_tests(ec, eprefix):): + """ + pre-configure hook for Highway: disable tests on neoverse_v1 for Highway 1.0.4 and GCC 12.3.0 + cfr. https://github.com/EESSI/software-layer/issues/469 + """ + if ec.name == 'Highway': + tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if ec.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': + if cpu_target == CPU_TARGET_NEOVERSE_V1: + ec.update('configopts', '-DHWY_ENABLE_TESTS=OFF') + else: + raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") + + def pre_configure_hook(self, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" if self.name in PRE_CONFIGURE_HOOKS: @@ -376,18 +391,6 @@ def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!") -def pre_configure_hook_highway_disable_tests(self, *args, **kwargs): - """ - pre-configure hook for Highway: disable tests on neoverse_v1 - cfr. https://github.com/EESSI/software-layer/issues/469 - """ - cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'Highway' and cpu_target == CPU_TARGET_NEOVERSE_V1: - self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') - else: - raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") - - def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: @@ -620,11 +623,12 @@ def inject_gpu_property(ec): 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, + 'Highway': parse_hook_highway_disable_tests, + 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'UCX': parse_hook_ucx_eprefix, - 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, } POST_PREPARE_HOOKS = { @@ -637,7 +641,6 @@ def inject_gpu_property(ec): 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, - 'Highway': pre_configure_hook_highway_disable_tests, } PRE_TEST_HOOKS = { From ae74707573fea4f1f210955e311c391f982c1f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 12:15:49 +0200 Subject: [PATCH 0696/1795] syntax fix --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9ccd984de4..902a37a599 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -289,7 +289,7 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") -def parse_hook_highway_disable_tests(ec, eprefix):): +def parse_hook_highway_disable_tests(ec, eprefix): """ pre-configure hook for Highway: disable tests on neoverse_v1 for Highway 1.0.4 and GCC 12.3.0 cfr. https://github.com/EESSI/software-layer/issues/469 From 3ccecf97e9008ddf98640d3a0ab970607c5b5779 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 3 Apr 2024 12:58:22 +0200 Subject: [PATCH 0697/1795] Add possibility to ignore certain hooks on local modules. E.g. we don't want to be prevented from loading local CUDA modules because of the EESSI hook. See https://github.com/EESSI/software-layer/issues/523 --- create_lmodsitepackage.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 29b2d39bd7..ffbe7a1531 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -19,6 +19,18 @@ return content end +local function from_eessi_prefix(t) + local eessi_prefix = os.getenv("EESSI_PREFIX") + -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment + -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set + if eessi_prefix == nil then + return False + else + -- Check if the full modulepath starts with the eessi_prefix + return t.fn:find( "^" .. eessi_prefix) ~= nil + end +end + local function load_site_specific_hooks() -- This function will be run after the EESSI hooks are registered -- It will load a local SitePackage.lua that is architecture independent (if it exists) from e.g. @@ -152,10 +164,13 @@ -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) - eessi_cuda_enabled_load_hook(t) + -- Only apply CUDA hooks if the loaded module is in the EESSI prefix + -- This avoids getting an Lmod Error when trying to load a CUDA module from a local software stack + if from_eessi_prefix(t) then + eessi_cuda_enabled_load_hook(t) + end end - hook.register("load", eessi_load_hook) -- Note that this needs to happen at the end, so that any EESSI specific hooks can be overwritten by the site From 92aad04bfa1810d313e476be5d89e13217cce9d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 13:02:33 +0200 Subject: [PATCH 0698/1795] also disable Highway 1.0.4 tests for neoverse_n1 --- eb_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 902a37a599..64671a5889 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -20,6 +20,7 @@ from distutils.version import LooseVersion +CPU_TARGET_NEOVERSE_N1 = 'aarch64/neoverse_n1' CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' @@ -173,7 +174,7 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): tcname == 'foss' and tcversion == '2023b' ): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if cpu_target == CPU_TARGET_NEOVERSE_V1: + if cpu_target in [CPU_TARGET_NEOVERSE_V1, CPU_TARGET_NEOVERSE_N1]: if not hasattr(ec, 'toolchainopts'): ec['toolchainopts'] = {} ec['toolchainopts']['vectorize'] = False From f2954ef909353b8c0a782b745324b523a612a1cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 13:26:11 +0200 Subject: [PATCH 0699/1795] oops, changed the wrong hook... fixed --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 64671a5889..7b1eb5cec2 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -174,7 +174,7 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): tcname == 'foss' and tcversion == '2023b' ): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if cpu_target in [CPU_TARGET_NEOVERSE_V1, CPU_TARGET_NEOVERSE_N1]: + if cpu_target == CPU_TARGET_NEOVERSE_V1: if not hasattr(ec, 'toolchainopts'): ec['toolchainopts'] = {} ec['toolchainopts']['vectorize'] = False @@ -299,7 +299,7 @@ def parse_hook_highway_disable_tests(ec, eprefix): tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': - if cpu_target == CPU_TARGET_NEOVERSE_V1: + if cpu_target in [CPU_TARGET_NEOVERSE_V1, CPU_TARGET_NEOVERSE_N1]: ec.update('configopts', '-DHWY_ENABLE_TESTS=OFF') else: raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") From bb20ab2078b25041011f242673a848b199aedb74 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 3 Apr 2024 16:36:39 +0200 Subject: [PATCH 0700/1795] Also use hooks for site extensions in host_injections AND for user extensions in /home/casparl/eessi/... --- create_lmodsitepackage.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index ffbe7a1531..f7d4d06bb9 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -20,14 +20,24 @@ end local function from_eessi_prefix(t) + -- eessi_prefix is the prefix with official EESSI modules + -- e.g. /cvmfs/software.eessi.io/versions/2023.06 local eessi_prefix = os.getenv("EESSI_PREFIX") + -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules) + -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06 + local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections') + -- eessi_prefix_user_home is the prefix with user-extensions (i.e. additional modules) + -- to the offocial EESSI modules, e.g. $HOME/eessi/versions/2023.06 + local eessi_prefix_user_home = string.gsub(eessi_prefix, os.getenv("EESSI_CVMFS_REPO"), pathJoin(os.getenv("HOME"), "eessi")) -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set if eessi_prefix == nil then return False else - -- Check if the full modulepath starts with the eessi_prefix - return t.fn:find( "^" .. eessi_prefix) ~= nil + -- Check if the full modulepath starts with the eessi_prefix_* + return string.find(t.fn, "^" .. eessi_prefix) ~= nil or + string.find(t.fn, "^" .. eessi_prefix_host_injections) ~= nil or + string.find(t.fn, "^" .. eessi_prefix_user_home) ~= nil end end From 838920bd332819b1e50706873ee53f2469cf3794 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 3 Apr 2024 16:38:24 +0200 Subject: [PATCH 0701/1795] Leave breadcrumb to notify that paths may need to be changed in the future --- create_lmodsitepackage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index f7d4d06bb9..ce94a7e311 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -23,6 +23,10 @@ -- eessi_prefix is the prefix with official EESSI modules -- e.g. /cvmfs/software.eessi.io/versions/2023.06 local eessi_prefix = os.getenv("EESSI_PREFIX") + + -- NOTE: exact paths for site and user extensions aren't final, so may need to be updated later. + -- See https://github.com/EESSI/software-layer/pull/371 + -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules) -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06 local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections') From 11b7bc2a41d228079f238e361683f2c9da1b28fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 3 Apr 2024 17:38:06 +0200 Subject: [PATCH 0702/1795] different actions for neoverse_n1 and _v1 in highway hook --- eb_hooks.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 7b1eb5cec2..8bf66427a9 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -290,17 +290,20 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") -def parse_hook_highway_disable_tests(ec, eprefix): +def parse_hook_highway_handle_test_compilation_issues(ec, eprefix): """ - pre-configure hook for Highway: disable tests on neoverse_v1 for Highway 1.0.4 and GCC 12.3.0 + pre-configure hook for Highway that solves issue that arise during the compilation of the tests + on both neoverse_n1 and neoverse_v1 with Highway 1.0.4 and GCC 12.3.0 cfr. https://github.com/EESSI/software-layer/issues/469 """ if ec.name == 'Highway': tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ec.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': - if cpu_target in [CPU_TARGET_NEOVERSE_V1, CPU_TARGET_NEOVERSE_N1]: + if cpu_target == CPU_TARGET_NEOVERSE_V1: ec.update('configopts', '-DHWY_ENABLE_TESTS=OFF') + if cpu_target == CPU_TARGET_NEOVERSE_N1: + update_build_option('optarch', OPTARCH_GENERIC) else: raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") @@ -624,7 +627,7 @@ def inject_gpu_property(ec): 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, - 'Highway': parse_hook_highway_disable_tests, + 'Highway': parse_hook_highway_handle_test_compilation_issues, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, From 69d0e9d132dcb6123ca000ead3580fd8f59da8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Apr 2024 13:51:08 +0200 Subject: [PATCH 0703/1795] update docstring of parse_hook_highway_handle_test_compilation_issues --- eb_hooks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8bf66427a9..0221e98b2a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -292,8 +292,10 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): def parse_hook_highway_handle_test_compilation_issues(ec, eprefix): """ - pre-configure hook for Highway that solves issue that arise during the compilation of the tests - on both neoverse_n1 and neoverse_v1 with Highway 1.0.4 and GCC 12.3.0 + Solve issues with compiling or running the tests on both + neoverse_n1 and neoverse_v1 with Highway 1.0.4 and GCC 12.3.0: + - for neoverse_n1 we set optarch to GENERIC + - for neoverse_v1 we completely disable the tests cfr. https://github.com/EESSI/software-layer/issues/469 """ if ec.name == 'Highway': From 7dba052cfee0996a1bce79984e79c37d4caeac64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Apr 2024 13:52:06 +0200 Subject: [PATCH 0704/1795] add issue with Highway 1.0.4 on neoverse_n1 --- eessi-2023.06-known-issues.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index d818c85952..011cb2dc08 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -3,6 +3,9 @@ - issue: https://github.com/EESSI/software-layer/issues/461 - info: "8 failing tests (out of 209539) on aarch64/*" - aarch64/neoverse_n1: + - Highway-1.0.4-GCCcore-12.3.0.eb: + - issue: https://github.com/EESSI/software-layer/issues/469 + - info: "failing to build the tests" - PyTorch-2.1.2-foss-2023a: - issue: https://github.com/EESSI/software-layer/issues/461 - info: "8 failing tests (out of 209539) on aarch64/*" From 6fa335b3868a79dc6fda347b841b2bfe9192509f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Apr 2024 19:58:57 +0200 Subject: [PATCH 0705/1795] change Highway hook in pre_prepare hook, and reset optarch in a post-prepare hook --- eb_hooks.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0221e98b2a..e8b890a954 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -116,6 +116,9 @@ def pre_prepare_hook(self, *args, **kwargs): print_msg("Updated rpath_override_dirs (to allow overriding MPI family %s): %s", mpi_family, rpath_override_dirs) + if self.name in PRE_PREPARE_HOOKS: + PRE_PREPARE_HOOKS[self.name](self, *args, **kwargs) + def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ @@ -290,7 +293,7 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") -def parse_hook_highway_handle_test_compilation_issues(ec, eprefix): +def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): """ Solve issues with compiling or running the tests on both neoverse_n1 and neoverse_v1 with Highway 1.0.4 and GCC 12.3.0: @@ -298,18 +301,30 @@ def parse_hook_highway_handle_test_compilation_issues(ec, eprefix): - for neoverse_v1 we completely disable the tests cfr. https://github.com/EESSI/software-layer/issues/469 """ - if ec.name == 'Highway': - tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] + if self.name == 'Highway': + tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if ec.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': + if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target == CPU_TARGET_NEOVERSE_V1: - ec.update('configopts', '-DHWY_ENABLE_TESTS=OFF') + self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') if cpu_target == CPU_TARGET_NEOVERSE_N1: + self.orig_optarch = build_option('optarch') update_build_option('optarch', OPTARCH_GENERIC) else: raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") +def post_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): + """ + Post-prepare hook for Highway to reset optarch build option. + """ + if self.name == 'Highway': + tcname, tcversion = self.toolchain.name, self.toolchain.version + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': + if cpu_target == CPU_TARGET_NEOVERSE_N1: + update_build_option('optarch', self.orig_optarch) + def pre_configure_hook(self, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" if self.name in PRE_CONFIGURE_HOOKS: @@ -629,7 +644,6 @@ def inject_gpu_property(ec): 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, - 'Highway': parse_hook_highway_handle_test_compilation_issues, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, @@ -637,8 +651,13 @@ def inject_gpu_property(ec): 'UCX': parse_hook_ucx_eprefix, } +PRE_PREPARE_HOOKS = { + 'Highway': pre_prepare_hook_highway_handle_test_compilation_issues, +} + POST_PREPARE_HOOKS = { 'GCCcore': post_prepare_hook_gcc_prefixed_ld_rpath_wrapper, + 'Highway': post_prepare_hook_highway_handle_test_compilation_issues, } PRE_CONFIGURE_HOOKS = { From 42e580bf17d24b6a3736dd620dd5206d1d7507a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Apr 2024 20:33:49 +0200 Subject: [PATCH 0706/1795] Update eb_hooks.py Co-authored-by: Kenneth Hoste --- eb_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index e8b890a954..c2075c8574 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -304,6 +304,8 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + # note: keep condition in sync with the one used in + # post_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') From 3c4572cb03c5d5dde7df1a16282c8bb70286918c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Apr 2024 20:33:54 +0200 Subject: [PATCH 0707/1795] Update eb_hooks.py Co-authored-by: Kenneth Hoste --- eb_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index c2075c8574..20b5f76cfc 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -323,6 +323,8 @@ def post_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwar if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + # note: keep condition in sync with the one used in + # pre_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target == CPU_TARGET_NEOVERSE_N1: update_build_option('optarch', self.orig_optarch) From 7283d086780b7dc9ae224a70bb4f6236a52e94d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Apr 2024 22:52:41 +0200 Subject: [PATCH 0708/1795] add from-pr 20238 for R-bundle-CRAN --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 1674bd76d9..a7cabc110a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -55,4 +55,6 @@ easyconfigs: - OpenJPEG-2.5.0-GCCcore-12.3.0.eb - OpenFOAM-10-foss-2023a.eb - Highway-1.0.4-GCCcore-12.3.0.eb - - R-bundle-CRAN-2023.12-foss-2023a.eb + - R-bundle-CRAN-2023.12-foss-2023a.eb: + options: + from-pr: 20238 From f6012f7ae91bfdf862f46e93cf6942067c8b580d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 11:42:40 +0200 Subject: [PATCH 0709/1795] add comment, retrigger CI --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 7ab9cf7c9b..44952e2765 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -69,4 +69,5 @@ easyconfigs: from-pr: 20142 - R-bundle-CRAN-2023.12-foss-2023a.eb: options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 From 145706e751ea7a73202676894cd58be65bea97d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 11:53:36 +0200 Subject: [PATCH 0710/1795] add R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index ad3cfddfb6..2dadc5ab28 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -57,3 +57,4 @@ easyconfigs: - Highway-1.0.4-GCCcore-12.3.0.eb - ELPA-2023.05.001-foss-2023a.eb - libxc-6.2.2-GCC-12.3.0.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb From cef7a277ac4092f8ff1256290ad6bb8b244c2443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 13:03:35 +0200 Subject: [PATCH 0711/1795] add from-pr to bioconductor --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index facd4230ab..60b9600589 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -67,4 +67,6 @@ easyconfigs: - MODFLOW-6.4.4-foss-2023a.eb: options: from-pr: 20142 - - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + from-pr: 19949 From 03e66d87cb07fb9d7c2f7f0daff93928f6b25f2d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 5 Apr 2024 20:51:55 +0200 Subject: [PATCH 0712/1795] {2023.06}[foss/2023a] QuantumESPRESSO 7.3.1 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 44952e2765..6cb243556a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -71,3 +71,9 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 + - QuantumESPRESSO-7.3.1-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3257 + options: + from-pr: 20138 + include-easyblocks-from-pr: 3257 From 3745adee2c09aa73d44c7b693baca3ac4c96e70c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Apr 2024 13:52:36 +0200 Subject: [PATCH 0713/1795] use pr 20316 for Bioconductor --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index d3ac5f1c5b..bfc408e045 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -73,4 +73,4 @@ easyconfigs: from-pr: 20238 - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: options: - from-pr: 19949 + from-pr: 20316 From 03b708fe7af1fc1aa330bf17bdf3f25a601f093e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Apr 2024 13:54:13 +0200 Subject: [PATCH 0714/1795] use PR 20316 for Bioconductor --- .../2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 8a3e82d760..da89e01a51 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -11,4 +11,6 @@ easyconfigs: - R-4.2.2-foss-2022b.eb: options: from-pr: 20238 - - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: + options: + from-pr: 20316 From 1998a7cfe2904e1b8dacdba4d6eca1fe67eb6354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Apr 2024 15:16:08 +0200 Subject: [PATCH 0715/1795] add EB 4.9.1 --- .../2023.06/eessi-2023.06-eb-4.9.1-001-system.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml new file mode 100644 index 0000000000..c5a08b5209 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - EasyBuild-4.9.1.eb: + options: + from-pr: 20299 From 1910871e228de824978f00b1d995879330745dec Mon Sep 17 00:00:00 2001 From: maxim-masterov Date: Tue, 9 Apr 2024 12:12:18 +0200 Subject: [PATCH 0716/1795] Add fix for permissions set on Lmod SitePackage --- create_lmodsitepackage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 29b2d39bd7..38600cb471 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -4,6 +4,7 @@ # import os import sys +from stat import S_IREAD, S_IWRITE, S_IRGRP, S_IWGRP, S_IROTH DOT_LMOD = '.lmod' @@ -180,6 +181,8 @@ def error(msg): os.makedirs(os.path.dirname(sitepackage_path), exist_ok=True) with open(sitepackage_path, 'w') as fp: fp.write(hook_txt) + # Make sure that the created Lmod file has "read" permissions for the "other" UNIX group + os.chmod(sitepackage_path, S_IREAD|S_IWRITE|S_IRGRP|S_IWGRP|S_IROTH) except (IOError, OSError) as err: error("Failed to create %s: %s" % (sitepackage_path, err)) From 8f656a4dced151499045dd9c1c49dc6438fb7471 Mon Sep 17 00:00:00 2001 From: Maxim Date: Wed, 10 Apr 2024 16:19:37 +0200 Subject: [PATCH 0717/1795] Update create_lmodsitepackage.py Co-authored-by: ocaisa --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 38600cb471..0761f2cdf1 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -181,7 +181,7 @@ def error(msg): os.makedirs(os.path.dirname(sitepackage_path), exist_ok=True) with open(sitepackage_path, 'w') as fp: fp.write(hook_txt) - # Make sure that the created Lmod file has "read" permissions for the "other" UNIX group + # Make sure that the created Lmod file has "read/write" for the user/group and "read" permissions for others os.chmod(sitepackage_path, S_IREAD|S_IWRITE|S_IRGRP|S_IWGRP|S_IROTH) except (IOError, OSError) as err: From 1a396e03badc537ddf901257bba7efb1cbdc8281 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 12 Apr 2024 13:23:53 +0200 Subject: [PATCH 0718/1795] Ensure we use instructions introduced with ARM v8.2 for Neoverse N1 --- init/arch_specs/eessi_arch_arm.spec | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index b5c9275043..1400871f93 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -1,6 +1,6 @@ # ARM CPU architecture specifications # Software path in EESSI | Vendor ID | List of defining CPU features -"aarch64/neoverse_n1" "ARM" "asimd" # Ampere Altra -"aarch64/neoverse_n1" "" "asimd" # AWS Graviton2 -"aarch64/neoverse_v1" "ARM" "asimd svei8mm" -"aarch64/neoverse_v1" "" "asimd svei8mm" # AWS Graviton3 +"aarch64/neoverse_n1" "ARM" "asimddp" # Ampere Altra +"aarch64/neoverse_n1" "" "asimddp" # AWS Graviton2 +"aarch64/neoverse_v1" "ARM" "asimddp svei8mm" +"aarch64/neoverse_v1" "" "asimddp svei8mm" # AWS Graviton3 From cedcf73c5da8d51d47c8d8d209cab76222e82820 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 12 Apr 2024 16:19:43 +0200 Subject: [PATCH 0719/1795] Update eessi_arch_arm.spec --- init/arch_specs/eessi_arch_arm.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index 1400871f93..8c1bc34d20 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -1,4 +1,4 @@ -# ARM CPU architecture specifications +# ARM CPU architecture specifications (see https://gpages.juszkiewicz.com.pl/arm-socs-table/arm-socs.html for guidance) # Software path in EESSI | Vendor ID | List of defining CPU features "aarch64/neoverse_n1" "ARM" "asimddp" # Ampere Altra "aarch64/neoverse_n1" "" "asimddp" # AWS Graviton2 From 61a788281119974079712a3ee9b1beeaa7fb24b9 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Sun, 14 Apr 2024 13:53:43 +0200 Subject: [PATCH 0720/1795] Update eessi-2023.06-eb-4.9.0-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 6cb243556a..1e1e7fee14 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -76,4 +76,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyblocks/pull/3257 options: from-pr: 20138 - include-easyblocks-from-pr: 3257 + include-easyblocks-from-pr: 3306 From 04e04a308829b49f4b925d4420363739c728a58b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 15 Apr 2024 11:33:08 +0200 Subject: [PATCH 0721/1795] Added ncdu --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml new file mode 100644 index 0000000000..3b51db3b11 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ncdu-1.18-GCC-12.3.0.eb From 8f81944b29dfb48edaf30b9466ce9edde4aa748f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Apr 2024 09:04:47 +0200 Subject: [PATCH 0722/1795] make sure that software directory that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE exists --- EESSI-install-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 567fed8e79..91effe4aba 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -143,6 +143,8 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" + # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) + mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) From 8331e4485a5734730b0929be69c3cd3c6beaa4b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Apr 2024 09:30:02 +0200 Subject: [PATCH 0723/1795] remove build_container.sh, as we now use eessi_container.sh --- build_container.sh | 69 ---------------------------------------------- 1 file changed, 69 deletions(-) delete mode 100755 build_container.sh diff --git a/build_container.sh b/build_container.sh deleted file mode 100755 index 23a9e665c9..0000000000 --- a/build_container.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -base_dir=$(dirname $(realpath $0)) - -BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian11" - -if [ $# -lt 2 ]; then - echo "Usage: $0 " >&2 - exit 1 -fi -SHELL_OR_RUN=$1 -EESSI_TMPDIR=$2 -shift 2 - -if [ "$SHELL_OR_RUN" == "run" ] && [ $# -eq 0 ]; then - echo "ERROR: No command specified to run?!" >&2 - exit 1 -fi - -# make sure specified temporary directory exists -mkdir -p $EESSI_TMPDIR - -echo "Using $EESSI_TMPDIR as parent for temporary directories..." - -# create temporary directories -mkdir -p $EESSI_TMPDIR/{home,overlay-upper,overlay-work} -mkdir -p $EESSI_TMPDIR/{var-lib-cvmfs,var-run-cvmfs} -# configure Singularity -export SINGULARITY_CACHEDIR=$EESSI_TMPDIR/singularity_cache - -# take into account that $SINGULARITY_BIND may be defined already, to bind additional paths into the build container -BIND_PATHS="$EESSI_TMPDIR/var-run-cvmfs:/var/run/cvmfs,$EESSI_TMPDIR/var-lib-cvmfs:/var/lib/cvmfs,$EESSI_TMPDIR" -if [ -z $SINGULARITY_BIND ]; then - export SINGULARITY_BIND="$BIND_PATHS" -else - export SINGULARITY_BIND="$SINGULARITY_BIND,$BIND_PATHS" -fi - -# allow that SINGULARITY_HOME is defined before script is run -if [ -z $SINGULARITY_HOME ]; then - export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER" -fi - -source ${base_dir}/init/eessi_defaults -# strip "/cvmfs/" from default setting -repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} - -# set environment variables for fuse mounts in Singularity container -export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" -export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/${repo_name} -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work ${EESSI_CVMFS_REPO}" - -# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) -if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then - export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility) - export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -fi - -if [ "$SHELL_OR_RUN" == "shell" ]; then - # start shell in Singularity container, with EESSI repository mounted with writable overlay - echo "Starting Singularity build container..." - singularity shell --fusemount "$EESSI_PILOT_READONLY" --fusemount "$EESSI_PILOT_WRITABLE_OVERLAY" $BUILD_CONTAINER -elif [ "$SHELL_OR_RUN" == "run" ]; then - echo "Running '$@' in Singularity build container..." - singularity exec --fusemount "$EESSI_PILOT_READONLY" --fusemount "$EESSI_PILOT_WRITABLE_OVERLAY" $BUILD_CONTAINER "$@" -else - echo "ERROR: Unknown action specified: $SHELL_OR_RUN (should be either 'shell' or 'run')" >&2 - exit 1 -fi From a7305456f10ca0289d1cbea8b212aa7cb4fb114a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Apr 2024 09:32:23 +0200 Subject: [PATCH 0724/1795] replace build_container.sh by eessi_container.sh --- .github/workflows/tests_scripts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 4944d9beaa..76d19d29fe 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -3,9 +3,9 @@ name: Tests for scripts on: push: paths: - - build_container.sh - create_directory_tarballs.sh - create_lmodsitepackage.py + - eessi_container.sh - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh @@ -15,9 +15,9 @@ on: pull_request: paths: - - build_container.sh - create_directory_tarballs.sh - create_lmodsitepackage.py + - eessi_container.sh - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh From 8593e595ee382a26c8f9621dd39ccb338e937259 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 18 Apr 2024 23:51:24 +0200 Subject: [PATCH 0725/1795] No hyphen any more so no need for lua escaping --- EESSI-extend-2023.06-easybuild.eb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 8993c63cc3..ce55f619fb 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -69,8 +69,7 @@ if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') else - -- Would have liked to use os.getenv("EESSI_CVMFS_REPO") here but the hypen needs to be escaped - easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), "/cvmfs/pilot.eessi%-hpc.org", pathJoin(os.getenv("HOME"), "eessi")) + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), pathJoin(os.getenv("HOME"), "eessi")) end if (mode() == "load") then LmodMessage("-- To create installations for EESSI, you _must_ have write permissions to " .. easybuild_installpath) From 8e976c6a2c8fbef7751dbc3f0c434ac431db3c4d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 19 Apr 2024 00:58:16 +0200 Subject: [PATCH 0726/1795] Make the hooks part of the init dir --- EESSI-extend-2023.06-easybuild.eb | 17 +++-------------- EESSI-install-software.sh | 8 ++++++++ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index ce55f619fb..6d71693db8 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -1,4 +1,4 @@ -easyblock = 'Binary' +easyblock = 'Bundle' name = 'EESSI-extend' version = '2023.06' @@ -22,13 +22,6 @@ description = """ toolchain = SYSTEM -source_urls = None -sources = ['eb_hooks.py'] -checksums = ['8ae609f99b6953beae89aa1945913686c86c156509dbc55e2b6b8017b13fcf66'] - -# Don't extend the PATH -prepend_to_path = None - # All the dependencies we filter in EESSI local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib" local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''} @@ -45,12 +38,12 @@ modextravars = { 'EASYBUILD_READ_ONLY_INSTALLDIR': '1', 'EASYBUILD_MODULE_EXTENSIONS': '1', 'EASYBUILD_EXPERIMENTAL': '1', - 'EASYBUILD_HOOKS': '%(installdir)s/eb_hooks.py', } # Need a few other variables, but they are more dynamic # EASYBUILD_SYSROOT=${EPREFIX} # EASYBUILD_PREFIX=${WORKDIR}/easybuild +# EASYBUILD_HOOKS=${EESSI_PREFIX}/init/easybuild/eb_hooks.py # EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} # EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH} modluafooter = """ @@ -82,14 +75,10 @@ end setenv ("EASYBUILD_SYSROOT", sysroot) setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) +setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "hooks.py")) if not ( isloaded("EasyBuild") ) then load("EasyBuild") end """ -sanity_check_paths = { - 'files': ['eb_hooks.py'], - 'dirs': [''] -} - moduleclass = 'devel' diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 91effe4aba..9766c20aa1 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -264,6 +264,14 @@ fi ### add packages here +# use PR patch file to determine if the EasyBuild hooks have changed +changed_eb_hooks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eb_hooks.py$') +if [ -n "${changed_eb_hooks}" ]; then + # If the hooks have been changed we need to copy them over to the init directory + mkdir -p ${EESSI_PREFIX}/init/easybuild + cp eb_hooks.py ${EESSI_PREFIX}/init/easybuild +fi + echo ">> Creating/updating Lmod RC file..." export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" From dbe6d6bde6f7e6e498fe0e337836c4114d4bb0c3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 19 Apr 2024 01:00:43 +0200 Subject: [PATCH 0727/1795] Actually install the EESSI extension module --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 3b51db3b11..b7d4370b84 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,2 +1,3 @@ easyconfigs: - ncdu-1.18-GCC-12.3.0.eb + - EESSI-extend-2023.06-easybuild.eb From 5ca2c4a25d702372f0993b57025a7f01459b21d9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 19 Apr 2024 09:52:39 +0200 Subject: [PATCH 0728/1795] Copy over hooks along with initi files --- EESSI-install-software.sh | 8 -------- install_scripts.sh | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9766c20aa1..91effe4aba 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -264,14 +264,6 @@ fi ### add packages here -# use PR patch file to determine if the EasyBuild hooks have changed -changed_eb_hooks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eb_hooks.py$') -if [ -n "${changed_eb_hooks}" ]; then - # If the hooks have been changed we need to copy them over to the init directory - mkdir -p ${EESSI_PREFIX}/init/easybuild - cp eb_hooks.py ${EESSI_PREFIX}/init/easybuild -fi - echo ">> Creating/updating Lmod RC file..." export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" diff --git a/install_scripts.sh b/install_scripts.sh index 508735975c..e6aecf4513 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -113,3 +113,9 @@ nvidia_files=( install_cuda_host_injections.sh link_nvidia_host_libraries.sh ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" + +# Copy over EasyBuild hooks file used for installations +hook_files=( + eb_hooks.py +) +copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/easybuild "${hook_files[@]}" From 148534a3922272b7f6f4f7a3e1b9d210b69ca933 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 19 Apr 2024 09:58:10 +0200 Subject: [PATCH 0729/1795] Update install_scripts.sh --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index e6aecf4513..17f0b81008 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -118,4 +118,4 @@ copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/script hook_files=( eb_hooks.py ) -copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/easybuild "${hook_files[@]}" +copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}" From dee801bbf1dc31c61a51a6e4b6dde702bf6337f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Apr 2024 10:25:51 +0200 Subject: [PATCH 0730/1795] move bioconductor to 4.9.1 easystack --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml index 8a3e82d760..3962f63bda 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -11,4 +11,3 @@ easyconfigs: - R-4.2.2-foss-2022b.eb: options: from-pr: 20238 - - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb From cbad690f01b417a9a185685d20f7e160808242ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Apr 2024 10:26:05 +0200 Subject: [PATCH 0731/1795] add bioconductor --- .../2023.06/eessi-2023.06-eb-4.9.1-2022b.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml new file mode 100644 index 0000000000..f80b99f187 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml @@ -0,0 +1,5 @@ +easyconfigs: + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: + options: + from-pr: 20379 + From b6a222639341313caca5bebb621590f8d0862373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Apr 2024 10:30:08 +0200 Subject: [PATCH 0732/1795] move biocondutor to 4.9.1 stack, use pr 20379 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index bfc408e045..44952e2765 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -71,6 +71,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 - - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: - options: - from-pr: 20316 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 3b51db3b11..57cfc00297 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,2 +1,5 @@ easyconfigs: - ncdu-1.18-GCC-12.3.0.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + from-pr: 20379 From 786e1e8189d75682c37150d2d7d58e465ec7e9ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Apr 2024 10:31:08 +0200 Subject: [PATCH 0733/1795] remove blank line --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml index f80b99f187..e547e45f2b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml @@ -2,4 +2,3 @@ easyconfigs: - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: options: from-pr: 20379 - From d6336e181efb320a74b895e0baf73373e49875f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Apr 2024 11:50:15 +0200 Subject: [PATCH 0734/1795] move bioconductor back to 4.9.0 file --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 +++ .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 44952e2765..23d64036f1 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -71,3 +71,6 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + from-pr: 20379 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 57cfc00297..3b51db3b11 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,5 +1,2 @@ easyconfigs: - ncdu-1.18-GCC-12.3.0.eb - - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: - options: - from-pr: 20379 From 339f358e46dd338649939c31042a0faf15f9e532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Apr 2024 13:59:23 +0200 Subject: [PATCH 0735/1795] add rebuild easystack for setuptools_scm fix --- ...move-setuptools_scm-from-hatchling-to-Python.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml new file mode 100644 index 0000000000..56ea7a46a1 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml @@ -0,0 +1,13 @@ +# 2024-04-19 +# Move setuptools_scm extension from hatchling to Python by rebuilding +# all affected modules with EasyBuild 4.9.1. +# This solves an issue with pyarrow, which is part of the Arrow installation. +# https://github.com/easybuilders/easybuild-easyconfigs/pull/19777 +# https://github.com/easybuilders/easybuild-easyconfigs/issues/19849 +easyconfigs: + - hatchling-1.18.0-GCCcore-12.3.0.eb + - hatchling-1.18.0-GCCcore-13.2.0.eb + - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb + - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb + - Python-3.11.3-GCCcore-12.3.0.eb + - Python-3.11.5-GCCcore-13.2.0.eb From 4ad271e4891e9c4e1c533e1a04505f16e505bf0a Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 19 Apr 2024 19:30:27 +0200 Subject: [PATCH 0736/1795] Update EESSI-extend-2023.06-easybuild.eb --- EESSI-extend-2023.06-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 6d71693db8..7acd0f0d36 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -75,7 +75,7 @@ end setenv ("EASYBUILD_SYSROOT", sysroot) setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) -setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "hooks.py")) +setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py")) if not ( isloaded("EasyBuild") ) then load("EasyBuild") end From db81fd36358c0430bfc6d3ae2784b7e0958d097a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 20 Apr 2024 07:05:40 +0200 Subject: [PATCH 0737/1795] zen4-only {2023.06}[system] EasyBuild v4.9.1 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.1-001-system.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-001-system.yml new file mode 100644 index 0000000000..c5a08b5209 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-001-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - EasyBuild-4.9.1.eb: + options: + from-pr: 20299 From 8cb9547b0c2b1b3bd1736b57d754fa9cad01516b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 20 Apr 2024 14:10:55 +0200 Subject: [PATCH 0738/1795] make sure lmod cfg files exists early in build environment --- EESSI-install-software.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 91effe4aba..7f67b883f2 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -147,6 +147,24 @@ else mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi +# if we run the script for the first time, e.g., to start building for a new +# stack, we need to ensure certain files are present in +# ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +# - .lmod/lmodrc.lua +# - .lmod/SitePackage.lua +_eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +_lmod_cfg_dir=${_eessi_software_path}/.lmod +_lmod_rc_file=${_lmod_cfg_dir}/lmodrc.lua +if [ ! -f ${_lmod_rc_file} ]; then + command -V python3 + python3 ${TOPDIR}/create_lmodrc.py ${_eessi_software_path} +fi +_lmod_sitepackage_file=${_lmod_cfg_dir}/SitePackage.lua +if [ ! -f ${_lmod_sitepackage_file} ]; then + command -V python3 + python3 ${TOPDIR}/create_lmodrc.py ${_eessi_software_path} +fi + # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages # $EESSI_BASIC_ENV - give a basic set of environment variables From bde1b728c6c39cfffcffbf0480323d5b7348346a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sat, 20 Apr 2024 14:16:44 +0200 Subject: [PATCH 0739/1795] Allow for default site installations, and configurable project and/or user installations --- EESSI-extend-2023.06-easybuild.eb | 93 ++++++++++++++++++++++++++++--- init/Magic_Castle/bash | 2 +- init/bash | 2 - init/eessi_environment_variables | 2 - 4 files changed, 87 insertions(+), 12 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 7acd0f0d36..3f9b62a494 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -14,10 +14,26 @@ description = """ personal workstations and cloud infrastructure. This module allows you to extend EESSI using the same configuration for - EasyBuild as EESSI itself uses. The default installation is the users - home directory, but this can be overridden for a site installation (by setting - EESSI_SITE_INSTALL) or for a direct installation under CVMFS (by setting - EESSI_CVMFS_INSTALL). + EasyBuild as EESSI itself uses. A number of environment variables control the + behaviour of the module: + - EESSI_USER_INSTALL can be set to a location to install modules for use by + the user only. The location must already exist on the filesystem. + - EESSI_PROJECT_INSTALL can be set to a location to install modules for use by + a project. The location must already exist on the filesystem and you should + ensure that the location has the correct Linux group and the SGID permission + is set on that directory (`chmod g+s $EESSI_PROJECT_INSTALL`) so that all + members of the group have permission to read and write installations. + - EESSI_SITE_INSTALL is either defined or not and cannot be used with another + environment variable. A site installation is done in a defined location and + any installations there are (by default) world readable. + - EESSI_CVMFS_INSTALL is either defined or not and cannot be used with another + environment variable. A CVMFS installation targets a defined location which + will be ingested into CVMFS and is only useful for CVMFS administrators. + - If none of the environment variables above are defined, an EESSI_USER_INSTALL + is assumed with a value of $HOME/EESSI + If both EESSI_USER_INSTALL and EESSI_PROJECT_INSTALL are defined, both sets of + installations are exposed, but new installations are created as user + installations. """ toolchain = SYSTEM @@ -27,6 +43,7 @@ local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,get local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''} local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH] +# Set the universal EasyBuild variables modextravars = { 'EASYBUILD_FILTER_DEPS': local_deps_to_filter, 'EASYBUILD_IGNORE_OSDEPS': '1', @@ -46,6 +63,12 @@ modextravars = { # EASYBUILD_HOOKS=${EESSI_PREFIX}/init/easybuild/eb_hooks.py # EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} # EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH} +# +# And also some optional ones based on the kind of installation +# EASYBUILD_SET_GID_BIT +# EASYBUILD_GROUP_WRITABLE_INSTALLDIR +# EASYBUILD_UMASK +# EASYBUILD_STICKY_BIT modluafooter = """ if (mode() == "load") then -- Use a working directory for temporary build files @@ -58,11 +81,48 @@ working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER")) sysroot = os.getenv("EESSI_EPREFIX") -- Use an installation prefix that we _should_ have write access to if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then + -- Make sure no other EESSI install environment variables are set + if (os.getenv("EESSI_SITE_INSTALL") ~= nil) or (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") then + LmodError("You cannot use EESSI_CVMFS_INSTALL in combination with any other EESSI_*_INSTALL environment variables") + end + eessi_cvmfs_install = true easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") -elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then +elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then + -- Make sure no other EESSI install environment variables are set + if (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") then + LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables") + end easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') else - easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), pathJoin(os.getenv("HOME"), "eessi")) + -- Deal with user and project installs + project_install = os.getenv("EESSI_PROJECT_INSTALL") + project_modulepath = nil + if (project_install ~= nil) then + -- Check the folder exists + if not isDir(project_install) then + LmodError("The location of EESSI_PROJECT_INSTALL (" .. project_install .. ") does not exist or is not a folder") + end + LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install) + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) + project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') + end + user_install = os.getenv("EESSI_USER_INSTALL") + user_modulepath = nil + if (project_install == nil) then + if (user_install ~= nil) then + user_install = pathJoin(os.getenv("HOME"), "eessi") + else + -- Check the folder exists + if not isDir(user_install) then + LmodError("The location of EESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder") + end + end + end + if (user_install ~= nil) then + LmodMessage("Configuring for use of EESSI_USER_INSTALL under " .. user_install) + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), user_install) + user_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') + end end if (mode() == "load") then LmodMessage("-- To create installations for EESSI, you _must_ have write permissions to " .. easybuild_installpath) @@ -71,11 +131,30 @@ if (mode() == "load") then LmodMessage("-- You may wish to configure a sources directory for EasyBuild (for example, via setting the environment variable EASYBUILD_SOURCEPATH) to allow you to reuse existing sources for packages.") end end --- Set the relevant environment variables for EasyBuild +-- Set the relevant universal environment variables for EasyBuild setenv ("EASYBUILD_SYSROOT", sysroot) setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py")) + +-- Set all related environment variables if we have project or user installations (including extending MODULEPATH) +if (user_modulepath ~= nil) then + setenv ("EASYBUILD_SET_GID_BIT", "1") + setenv ("EASYBUILD_GROUP_WRITABLE_INSTALLDIR", "1") + setenv ("EASYBUILD_UMASK", "002") + setenv ("EASYBUILD_STICKY_BIT", "0") + -- configure MODULEPATH + if (project_modulepath ~= nil) then + prepend_path("MODULEPATH", project_modulepath) + end + prepend_path("MODULEPATH", user_modulepath) +elseif (project_modulepath ~= nil) then + setenv ("EASYBUILD_UMASK", "022") + setenv ("EASYBUILD_STICKY_BIT", "1") + -- configure MODULEPATH + prepend_path("MODULEPATH", project_modulepath) +end +-- Make aure EasyBuild itself is loaded if not ( isloaded("EasyBuild") ) then load("EasyBuild") end diff --git a/init/Magic_Castle/bash b/init/Magic_Castle/bash index e6c7ff294c..bf625e7e7b 100644 --- a/init/Magic_Castle/bash +++ b/init/Magic_Castle/bash @@ -10,7 +10,7 @@ source $(dirname "$BASH_SOURCE")/../eessi_environment_variables # Provide a clean MODULEPATH export MODULEPATH_ROOT=$EESSI_MODULEPATH -export MODULEPATH=$EESSI_USER_MODULEPATH:$EESSI_SITE_MODULEPATH:$EESSI_MODULEPATH +export MODULEPATH=$EESSI_SITE_MODULEPATH:$EESSI_MODULEPATH # Extensions are too many, let's not print them by default (requires Lmod 8.4.12) export LMOD_AVAIL_EXTENSIONS=no diff --git a/init/bash b/init/bash index 3c6aad47f1..565c37cc65 100644 --- a/init/bash +++ b/init/bash @@ -28,8 +28,6 @@ if [ $? -eq 0 ]; then module use $EESSI_MODULEPATH echo "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." >> $output module use $EESSI_SITE_MODULEPATH - echo "Prepending user path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." >> $output - module use $EESSI_USER_MODULEPATH #show_msg "" #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***" diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 5ae2c5c3c6..dbf01cc9e4 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -91,8 +91,6 @@ if [ -d $EESSI_PREFIX ]; then show_msg "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." export EESSI_SITE_MODULEPATH=${EESSI_MODULEPATH/versions/host_injections} show_msg "Using ${EESSI_SITE_MODULEPATH} as the site extension directory to be added to MODULEPATH." - export EESSI_USER_MODULEPATH=${EESSI_MODULEPATH/${EESSI_CVMFS_REPO}/${HOME}\/eessi} - show_msg "Using ${EESSI_USER_MODULEPATH} as the user extension directory to be added to MODULEPATH." else error "EESSI module path at $EESSI_MODULEPATH not found!" false From da9f206f216c70b8559bdd002235a754125aba2d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 20 Apr 2024 14:18:58 +0200 Subject: [PATCH 0740/1795] use create_lmodsitepackage.py --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7f67b883f2..13e94fb72d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -162,7 +162,7 @@ fi _lmod_sitepackage_file=${_lmod_cfg_dir}/SitePackage.lua if [ ! -f ${_lmod_sitepackage_file} ]; then command -V python3 - python3 ${TOPDIR}/create_lmodrc.py ${_eessi_software_path} + python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path} fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) From 82f58b24ae4cb2ada19c79ad6c8e7cef2727ed56 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sat, 20 Apr 2024 14:30:57 +0200 Subject: [PATCH 0741/1795] Fix conditions --- EESSI-extend-2023.06-easybuild.eb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 3f9b62a494..0f0b4bc9f9 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -82,14 +82,14 @@ sysroot = os.getenv("EESSI_EPREFIX") -- Use an installation prefix that we _should_ have write access to if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then -- Make sure no other EESSI install environment variables are set - if (os.getenv("EESSI_SITE_INSTALL") ~= nil) or (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") then + if ((os.getenv("EESSI_SITE_INSTALL") ~= nil) or (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then LmodError("You cannot use EESSI_CVMFS_INSTALL in combination with any other EESSI_*_INSTALL environment variables") end eessi_cvmfs_install = true easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then -- Make sure no other EESSI install environment variables are set - if (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") then + if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables") end easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') From 4ebfc2057d08056a978743f0e63ce630e5912fef Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sat, 20 Apr 2024 14:35:32 +0200 Subject: [PATCH 0742/1795] Fix condition --- EESSI-extend-2023.06-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 0f0b4bc9f9..900d584720 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -109,7 +109,7 @@ else user_install = os.getenv("EESSI_USER_INSTALL") user_modulepath = nil if (project_install == nil) then - if (user_install ~= nil) then + if (user_install == nil) then user_install = pathJoin(os.getenv("HOME"), "eessi") else -- Check the folder exists From 5c5899b3230c64cb40f57e91fa3521b74d83bdd2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sat, 20 Apr 2024 14:39:06 +0200 Subject: [PATCH 0743/1795] Messages only on module load --- EESSI-extend-2023.06-easybuild.eb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 900d584720..303abe0410 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -102,7 +102,9 @@ else if not isDir(project_install) then LmodError("The location of EESSI_PROJECT_INSTALL (" .. project_install .. ") does not exist or is not a folder") end - LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install) + if (mode() == "load") then + LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install) + end easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') end @@ -119,7 +121,9 @@ else end end if (user_install ~= nil) then - LmodMessage("Configuring for use of EESSI_USER_INSTALL under " .. user_install) + if (mode() == "load") then + LmodMessage("Configuring for use of EESSI_USER_INSTALL under " .. user_install) + end easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), user_install) user_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') end From c4cbcd8b9b6212c830e371aa3013eee8977ff776 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 20 Apr 2024 14:41:02 +0200 Subject: [PATCH 0744/1795] skip CUDA install if no EasyBuild module is found --- EESSI-install-software.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 13e94fb72d..2d401e40ce 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -221,10 +221,21 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install # Allow skipping CUDA SDK install in e.g. CI environments +# The install_cuda... script uses EasyBuild. So, we need to check if we have EB +# or skip this step. +module_avail_out=$TMPDIR/ml.out +module avail 2>&1 | grep EasyBuild &> ${module_avail_out} +if [[ $? -eq 0 ]]; then + echo_green ">> Found an EasyBuild module" +else + echo_yellow ">> No EasyBuild module found: skipping step to install CUDA (see output in ${module_avail_out})" + export skip_cuda_install=True +fi +if if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula else - echo "Skipping installation of CUDA SDK in host_injections, since the --skip-cuda-install flag was passed" + echo "Skipping installation of CUDA SDK in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found" fi # Install drivers in host_injections From eeb5537e4b8889cbbf183ffd17e4743961357f3c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 20 Apr 2024 14:45:17 +0200 Subject: [PATCH 0745/1795] remove extra if --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2d401e40ce..8a5789c2b2 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -231,7 +231,7 @@ else echo_yellow ">> No EasyBuild module found: skipping step to install CUDA (see output in ${module_avail_out})" export skip_cuda_install=True fi -if + if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula else From 8122bbd6bd7f6c44f4db9c955dcbd0bc7d736ad4 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sat, 20 Apr 2024 14:56:52 +0200 Subject: [PATCH 0746/1795] fix location of settings --- EESSI-extend-2023.06-easybuild.eb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 303abe0410..ddf56648b9 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -143,18 +143,18 @@ setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuil -- Set all related environment variables if we have project or user installations (including extending MODULEPATH) if (user_modulepath ~= nil) then - setenv ("EASYBUILD_SET_GID_BIT", "1") - setenv ("EASYBUILD_GROUP_WRITABLE_INSTALLDIR", "1") - setenv ("EASYBUILD_UMASK", "002") - setenv ("EASYBUILD_STICKY_BIT", "0") + setenv ("EASYBUILD_UMASK", "022") + setenv ("EASYBUILD_STICKY_BIT", "1") -- configure MODULEPATH if (project_modulepath ~= nil) then prepend_path("MODULEPATH", project_modulepath) end prepend_path("MODULEPATH", user_modulepath) elseif (project_modulepath ~= nil) then - setenv ("EASYBUILD_UMASK", "022") - setenv ("EASYBUILD_STICKY_BIT", "1") + setenv ("EASYBUILD_SET_GID_BIT", "1") + setenv ("EASYBUILD_GROUP_WRITABLE_INSTALLDIR", "1") + setenv ("EASYBUILD_UMASK", "002") + setenv ("EASYBUILD_STICKY_BIT", "0") -- configure MODULEPATH prepend_path("MODULEPATH", project_modulepath) end From 4d465e9996b61bfb2f2643710bf944ee88bdd355 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Sun, 21 Apr 2024 17:47:40 +0200 Subject: [PATCH 0747/1795] Add SAMtools-1.18-GCC-12.3.0 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 3b51db3b11..ec5fd3c638 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,2 +1,3 @@ easyconfigs: - ncdu-1.18-GCC-12.3.0.eb + - SAMtools-1.18-GCC-12.3.0.eb From 04c6f0ee56bde553968da02f7daba36cbb363d78 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 22 Apr 2024 09:30:11 +0200 Subject: [PATCH 0748/1795] Always check for dir existence when using EESSI_USER_INSTALL --- EESSI-extend-2023.06-easybuild.eb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index ddf56648b9..c06087a940 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -110,15 +110,14 @@ else end user_install = os.getenv("EESSI_USER_INSTALL") user_modulepath = nil - if (project_install == nil) then - if (user_install == nil) then - user_install = pathJoin(os.getenv("HOME"), "eessi") - else - -- Check the folder exists - if not isDir(user_install) then - LmodError("The location of EESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder") - end + if (user_install ~= nil) then + -- Check the folder exists + if not isDir(user_install) then + LmodError("The location of EESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder") end + elseif (user_install == nil) and (project_install == nil) then + -- No need to check for existence when we use a HOME subdir + user_install = pathJoin(os.getenv("HOME"), "eessi") end if (user_install ~= nil) then if (mode() == "load") then From 8cd5d844e872fe58efcdbb49e38689e656f3f9a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 22 Apr 2024 16:39:45 +0200 Subject: [PATCH 0749/1795] attempt to fix the hatchling reinstallation issue --- EESSI-remove-software.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 446a156cb8..d8a6dae63e 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -114,6 +114,11 @@ if [ $EUID -eq 0 ]; then echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${app_dir} rm -rf ${app_module} + # if the parent dir of this application is now empty, remove it too to work around a weird issue with the overlay + # see https://github.com/EESSI/software-layer/pull/546#issuecomment-2067018216 + if [ ! -n "$(dirname ${app_dir})" ]; then + rmdir "$(dirname ${app_dir})" + fi done else fatal_error "Easystack file ${easystack_file} not found!" From 76ecccf629f6b4dda1b65e4ee1f3f2d9d9b9457c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 22 Apr 2024 17:13:37 +0200 Subject: [PATCH 0750/1795] use right if condition for checking if dir is empty --- EESSI-remove-software.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index d8a6dae63e..f3c4f9e618 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -116,8 +116,9 @@ if [ $EUID -eq 0 ]; then rm -rf ${app_module} # if the parent dir of this application is now empty, remove it too to work around a weird issue with the overlay # see https://github.com/EESSI/software-layer/pull/546#issuecomment-2067018216 - if [ ! -n "$(dirname ${app_dir})" ]; then - rmdir "$(dirname ${app_dir})" + app_dir_parent=$(dirname "${app_dir}") + if [ ! -n "$(ls -A ${app_dir_parent})" ]; then + rmdir "${app_dir_parent}" fi done else From 1642621d1b47b804f689280d7c797824d1662a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 26 Apr 2024 14:24:46 +0200 Subject: [PATCH 0751/1795] undo parent dir removal, workaround didn't work --- EESSI-remove-software.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index f3c4f9e618..446a156cb8 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -114,12 +114,6 @@ if [ $EUID -eq 0 ]; then echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${app_dir} rm -rf ${app_module} - # if the parent dir of this application is now empty, remove it too to work around a weird issue with the overlay - # see https://github.com/EESSI/software-layer/pull/546#issuecomment-2067018216 - app_dir_parent=$(dirname "${app_dir}") - if [ ! -n "$(ls -A ${app_dir_parent})" ]; then - rmdir "${app_dir_parent}" - fi done else fatal_error "Easystack file ${easystack_file} not found!" From a5bf9d3da294fad127aef0dc972af323a0488077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 26 Apr 2024 14:26:25 +0200 Subject: [PATCH 0752/1795] only rebuild 2023a easyconfigs --- ...eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml index 56ea7a46a1..76dd37e870 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml @@ -1,13 +1,10 @@ # 2024-04-19 # Move setuptools_scm extension from hatchling to Python by rebuilding -# all affected modules with EasyBuild 4.9.1. +# the affected modules in the 2023a toolchain with EasyBuild 4.9.1. # This solves an issue with pyarrow, which is part of the Arrow installation. # https://github.com/easybuilders/easybuild-easyconfigs/pull/19777 # https://github.com/easybuilders/easybuild-easyconfigs/issues/19849 easyconfigs: - hatchling-1.18.0-GCCcore-12.3.0.eb - - hatchling-1.18.0-GCCcore-13.2.0.eb - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb - - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb - Python-3.11.3-GCCcore-12.3.0.eb - - Python-3.11.5-GCCcore-13.2.0.eb From c4af218d2c7dd604f10cea7454d1f672b00498ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 26 Apr 2024 15:04:02 +0200 Subject: [PATCH 0753/1795] only try 2023b versions --- ...4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml index 76dd37e870..f3484a3486 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml @@ -1,10 +1,10 @@ # 2024-04-19 # Move setuptools_scm extension from hatchling to Python by rebuilding -# the affected modules in the 2023a toolchain with EasyBuild 4.9.1. +# the affected modules in the 2023b toolchain with EasyBuild 4.9.1. # This solves an issue with pyarrow, which is part of the Arrow installation. # https://github.com/easybuilders/easybuild-easyconfigs/pull/19777 # https://github.com/easybuilders/easybuild-easyconfigs/issues/19849 easyconfigs: - - hatchling-1.18.0-GCCcore-12.3.0.eb - - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb - - Python-3.11.3-GCCcore-12.3.0.eb + - hatchling-1.18.0-GCCcore-13.2.0.eb + - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb + - Python-3.11.5-GCCcore-13.2.0.eb From 33cb74f5e2f4f63cd286e0d1b71818ccb1eca1a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 26 Apr 2024 15:33:15 +0200 Subject: [PATCH 0754/1795] try adding write permissions instead of removing --- EESSI-remove-software.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 446a156cb8..1a488dcb67 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -112,8 +112,9 @@ if [ $EUID -eq 0 ]; then app_dir=${EASYBUILD_INSTALLPATH}/software/${app} app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - rm -rf ${app_dir} - rm -rf ${app_module} + #rm -rf ${app_dir} + #rm -rf ${app_module} + chmod -R u+w ${app_dir} ${app_module} done else fatal_error "Easystack file ${easystack_file} not found!" From 1e92daf31bca8a879b2888cefc5156c76774f7ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 26 Apr 2024 16:12:21 +0200 Subject: [PATCH 0755/1795] do remove the module to make sure it gets rebuilt --- EESSI-remove-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 1a488dcb67..9d82dd8aa6 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -113,8 +113,8 @@ if [ $EUID -eq 0 ]; then app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." #rm -rf ${app_dir} - #rm -rf ${app_module} - chmod -R u+w ${app_dir} ${app_module} + rm -rf ${app_module} + chmod -R u+w ${app_dir} done else fatal_error "Easystack file ${easystack_file} not found!" From d0229ccb3adb03eac5cdddde76858b00d20832bf Mon Sep 17 00:00:00 2001 From: ocaisa Date: Mon, 29 Apr 2024 14:40:50 +0200 Subject: [PATCH 0756/1795] Update create_lmodsitepackage.py --- create_lmodsitepackage.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index ce94a7e311..862902d80e 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -24,24 +24,20 @@ -- e.g. /cvmfs/software.eessi.io/versions/2023.06 local eessi_prefix = os.getenv("EESSI_PREFIX") - -- NOTE: exact paths for site and user extensions aren't final, so may need to be updated later. - -- See https://github.com/EESSI/software-layer/pull/371 - - -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules) - -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06 - local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections') - -- eessi_prefix_user_home is the prefix with user-extensions (i.e. additional modules) - -- to the offocial EESSI modules, e.g. $HOME/eessi/versions/2023.06 - local eessi_prefix_user_home = string.gsub(eessi_prefix, os.getenv("EESSI_CVMFS_REPO"), pathJoin(os.getenv("HOME"), "eessi")) -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set if eessi_prefix == nil then return False else - -- Check if the full modulepath starts with the eessi_prefix_* - return string.find(t.fn, "^" .. eessi_prefix) ~= nil or - string.find(t.fn, "^" .. eessi_prefix_host_injections) ~= nil or - string.find(t.fn, "^" .. eessi_prefix_user_home) ~= nil + -- NOTE: exact paths for site so may need to be updated later. + -- See https://github.com/EESSI/software-layer/pull/371 + + -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules) + -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06 + local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections') + + -- Check if the full modulepath starts with the eessi_prefix_* + return string.find(t.fn, "^" .. eessi_prefix) ~= nil or string.find(t.fn, "^" .. eessi_prefix_host_injections) ~= nil end end From b6371d0e2766720a8fbccdb2ed322b5858dc834f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 30 Apr 2024 11:06:30 +0200 Subject: [PATCH 0757/1795] use easyconfig PR for GROMACS 2024.1 that uses backported patch to fix problems with tests --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 --- .../2023.06/eessi-2023.06-eb-4.9.1-2023b.yml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index d45ebfb9ba..86c5106c85 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -57,6 +57,3 @@ easyconfigs: - Qt5-5.15.13-GCCcore-13.2.0.eb: options: from-pr: 20201 - - GROMACS-2024.1-foss-2023b.eb: - options: - from-pr: 20102 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml new file mode 100644 index 0000000000..1392fa2f5a --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml @@ -0,0 +1,4 @@ +easyconfigs: + - GROMACS-2024.1-foss-2023b.eb: + options: + from-pr: 20439 From b94fa6262a2845a03d58a9095b5da4fb84b8e710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 30 Apr 2024 15:13:53 +0200 Subject: [PATCH 0758/1795] restore easystack, rebuild all required versions --- ...eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml index f3484a3486..56ea7a46a1 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml @@ -1,10 +1,13 @@ # 2024-04-19 # Move setuptools_scm extension from hatchling to Python by rebuilding -# the affected modules in the 2023b toolchain with EasyBuild 4.9.1. +# all affected modules with EasyBuild 4.9.1. # This solves an issue with pyarrow, which is part of the Arrow installation. # https://github.com/easybuilders/easybuild-easyconfigs/pull/19777 # https://github.com/easybuilders/easybuild-easyconfigs/issues/19849 easyconfigs: + - hatchling-1.18.0-GCCcore-12.3.0.eb - hatchling-1.18.0-GCCcore-13.2.0.eb + - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb + - Python-3.11.3-GCCcore-12.3.0.eb - Python-3.11.5-GCCcore-13.2.0.eb From 66724f282e405579021811813721aaf79dc35a75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 30 Apr 2024 15:14:50 +0200 Subject: [PATCH 0759/1795] undo changes --- EESSI-remove-software.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 9d82dd8aa6..446a156cb8 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -112,9 +112,8 @@ if [ $EUID -eq 0 ]; then app_dir=${EASYBUILD_INSTALLPATH}/software/${app} app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - #rm -rf ${app_dir} + rm -rf ${app_dir} rm -rf ${app_module} - chmod -R u+w ${app_dir} done else fatal_error "Easystack file ${easystack_file} not found!" From 01c49de45a802aafdc5d0bcf7cd577c0213cfac0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 30 Apr 2024 20:54:28 +0200 Subject: [PATCH 0760/1795] avoid building GROMACS 2024.1 with SVE on Neoverse V1, stick to Neon for now (cfr. https://gitlab.com/gromacs/gromacs/-/issues/5057) --- eb_hooks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 20b5f76cfc..f02c8db48c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -335,6 +335,21 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) +def pre_configure_hook_gromacs(self, *args, **kwargs): + """ + Pre-configure hook for GROMACS: + - avoid building with SVE instructions on Neoverse V1 as workaround for failing tests, + see https://gitlab.com/gromacs/gromacs/-/issues/5057 + https://gitlab.com/eessi/support/-/issues/47 + """ + if self.name == 'GROMACS': + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg.update('configopts', '-DGMX_SIMD=ARM_NEON_ASIMD') + print_msg("Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", self.version) + else: + raise EasyBuildError("GROMACS-specific hook triggered for non-GROMACS easyconfig?!") + + def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): """ Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC @@ -665,6 +680,7 @@ def inject_gpu_property(ec): } PRE_CONFIGURE_HOOKS = { + 'GROMACS': pre_configure_hook_gromacs, 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, From ac82cc840bfbdae0638438fb85f9a5004e020f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 30 Apr 2024 21:21:32 +0200 Subject: [PATCH 0761/1795] move bioconductor to eb 4.9.1 file --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 23d64036f1..44952e2765 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -71,6 +71,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 - - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: - options: - from-pr: 20379 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index ec5fd3c638..e65747e4a5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,3 +1,6 @@ easyconfigs: - ncdu-1.18-GCC-12.3.0.eb - SAMtools-1.18-GCC-12.3.0.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + from-pr: 20379 From 3eee364b47b4cae85aa675d020a7bd7a362b47f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 1 May 2024 11:11:41 +0200 Subject: [PATCH 0762/1795] register known issue for GROMACS 2024.1 on neoverse_v1 --- eessi-2023.06-known-issues.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 011cb2dc08..2d1256354f 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -19,6 +19,9 @@ - FFTW.MPI-3.3.10-gompi-2023b: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" + - GROMACS-2024.1-foss-2023b: + - issue: https://github.com/EESSI/software-layer/issues/557 + - info: "SVE disabled due to known bug which causes test failures" - Highway-1.0.3-GCCcore-12.2.0.eb: - issue: https://github.com/EESSI/software-layer/issues/469 - info: "failing SVE test due to wrong expected value" From fe0c7c534db6ce8d369aafaa84b4d0afaf7b3cc2 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 1 May 2024 17:14:47 +0200 Subject: [PATCH 0763/1795] Add Lmod hook for ESPReSso --- create_lmodsitepackage.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 0761f2cdf1..ef17448f0c 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +z#!/usr/bin/env python3 # # Create SitePackage.lua configuration file for Lmod. # @@ -150,10 +150,24 @@ end end +local function eessi_espresso_deprecated_warning(t) + local frameStk = require("FrameStk"):singleton() + local mt = frameStk:mt() + local simpleName = string.match(t.modFullName, "(.-)/") + local version = string.match(t.modFullName, "%d.%d.%d") + if simpleName == 'ESPResSo' and version == '4.2.1' + local advice = 'Prefer versions >= 4.2.2 which include important bugfixes.\\n' + advice = advice .. 'For details see https://github.com/espressomd/espresso/issues/4856\\n' + advice = advice .. 'Use version 4.2.1 at your own risk!\\nn' + LmodWarning("\\nESPReSso v4.2.1 has known issues and has been deprecated. ", advice) + end +end + -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) eessi_cuda_enabled_load_hook(t) + eessi_espresso_deprecated_warning(t) end From 8f253276033d83d08ea770c1dd8ef946d6675d63 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 6 May 2024 09:36:04 +0200 Subject: [PATCH 0764/1795] Fix typo --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index ef17448f0c..61b40c4252 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -1,4 +1,4 @@ -z#!/usr/bin/env python3 +#!/usr/bin/env python3 # # Create SitePackage.lua configuration file for Lmod. # From a9e8ae8192e916b3b6bcbfa6c47206a748d362b8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 6 May 2024 16:06:22 +0200 Subject: [PATCH 0765/1795] Match name stub of files to CUDA allow list to ship --- .../2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml | 8 ++++++++ eb_hooks.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml b/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml new file mode 100644 index 0000000000..2a9daf7ba6 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml @@ -0,0 +1,8 @@ +# 2024.05.06 +# Original matching of files we could ship was not done correctly. We were +# matching the basename for files (e.g., libcudart.so from libcudart.so.12) +# rather than the name stub (libcudart) +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA diff --git a/eb_hooks.py b/eb_hooks.py index f02c8db48c..8b0a11b0ed 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -612,8 +612,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs): full_path = os.path.join(dir_path, filename) # we only really care about real files, i.e. not symlinks if not os.path.islink(full_path): - # check if the current file is part of the allowlist - basename = os.path.splitext(filename)[0] + # check if the current file name stub is part of the allowlist + basename = filename.split('.')[0] if basename in allowlist: self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) else: From 28e5e9c52005cb46855e7fe965cd41997e9c48d6 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Mon, 6 May 2024 18:49:34 +0200 Subject: [PATCH 0766/1795] Update 2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml --- .../2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml b/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml index 2a9daf7ba6..058ab75e80 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml @@ -2,6 +2,7 @@ # Original matching of files we could ship was not done correctly. We were # matching the basename for files (e.g., libcudart.so from libcudart.so.12) # rather than the name stub (libcudart) +# See https://github.com/EESSI/software-layer/pull/559 easyconfigs: - CUDA-12.1.1.eb: options: From cc91f3411df6010ff29247a94bfe9356e70cf342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Tue, 7 May 2024 10:50:48 +0200 Subject: [PATCH 0767/1795] fix tiny typo --- EESSI-extend-2023.06-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index c06087a940..e6e540bf45 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -157,7 +157,7 @@ elseif (project_modulepath ~= nil) then -- configure MODULEPATH prepend_path("MODULEPATH", project_modulepath) end --- Make aure EasyBuild itself is loaded +-- Make sure EasyBuild itself is loaded if not ( isloaded("EasyBuild") ) then load("EasyBuild") end From 38a0581480ac284c2802e5e63a0541fae31f786a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 7 May 2024 10:55:28 +0200 Subject: [PATCH 0768/1795] use eessi.io/docs and system easystack file --- EESSI-extend-2023.06-easybuild.eb | 2 +- .../2023.06/eessi-2023.06-eb-4.9.1-001-system.yml | 1 + .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index e6e540bf45..241cde264b 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -5,7 +5,7 @@ version = '2023.06' # May have different ways to extend EESSI in future (manually, other tools,...) versionsuffix = '-easybuild' -homepage = 'https://eessi.github.io/docs/' +homepage = 'https://eessi.io/docs/' description = """ The goal of the European Environment for Scientific Software Installations diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml index c5a08b5209..46ac979719 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -2,3 +2,4 @@ easyconfigs: - EasyBuild-4.9.1.eb: options: from-pr: 20299 + - EESSI-extend-2023.06-easybuild.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index b7d4370b84..3b51db3b11 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,3 +1,2 @@ easyconfigs: - ncdu-1.18-GCC-12.3.0.eb - - EESSI-extend-2023.06-easybuild.eb From 0ff23347d31156ed4501f49f1a7f187f51257ecf Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 7 May 2024 12:05:38 +0200 Subject: [PATCH 0769/1795] Update EESSI-extend-2023.06-easybuild.eb --- EESSI-extend-2023.06-easybuild.eb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 241cde264b..76e96766f9 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -139,9 +139,11 @@ setenv ("EASYBUILD_SYSROOT", sysroot) setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py")) +setenv ("EASYBUILD_UMASK", "002") -- Set all related environment variables if we have project or user installations (including extending MODULEPATH) if (user_modulepath ~= nil) then + -- Use a more restrictive umask for this case setenv ("EASYBUILD_UMASK", "022") setenv ("EASYBUILD_STICKY_BIT", "1") -- configure MODULEPATH @@ -152,7 +154,6 @@ if (user_modulepath ~= nil) then elseif (project_modulepath ~= nil) then setenv ("EASYBUILD_SET_GID_BIT", "1") setenv ("EASYBUILD_GROUP_WRITABLE_INSTALLDIR", "1") - setenv ("EASYBUILD_UMASK", "002") setenv ("EASYBUILD_STICKY_BIT", "0") -- configure MODULEPATH prepend_path("MODULEPATH", project_modulepath) From 6682c7d36d54641aeb6a5cd9eeaf2cd981ef89d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 7 May 2024 13:32:25 +0200 Subject: [PATCH 0770/1795] move OSU-Micro-Benchmarks 2023b to 2023b easystack --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 - .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 44952e2765..8f5608d881 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -35,7 +35,6 @@ easyconfigs: from-pr: 19996 - dask-2023.9.2-foss-2023a.eb - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb: options: diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 86c5106c85..15c02951d7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -57,3 +57,4 @@ easyconfigs: - Qt5-5.15.13-GCCcore-13.2.0.eb: options: from-pr: 20201 + - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb From 298159e516db7c696b12ca57bf51a5328588ebf9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 7 May 2024 14:27:24 +0200 Subject: [PATCH 0771/1795] bumping scorecard-action version to 2.3.1 --- .github/workflows/scorecards.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index dc18fd584a..7eff557094 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6 + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 with: results_file: results.sarif results_format: sarif From ae924e0c589d546651725abf87dc59877cfb3bfb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 May 2024 15:21:29 +0200 Subject: [PATCH 0772/1795] {2023.06}[foss/2023a] OpenFOAM v2312 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index e65747e4a5..674cc49a39 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -4,3 +4,4 @@ easyconfigs: - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: options: from-pr: 20379 + - OpenFOAM-v2312-foss-2023a.eb From 462adc5cdd5c3f2255f0275e45a7f6a82da584bf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 May 2024 15:28:38 +0200 Subject: [PATCH 0773/1795] {2023.06,zen4} foss/2023a --- .../2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml new file mode 100644 index 0000000000..c821df1afb --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - foss-2023a.eb From 06f88dbbc5f90654fb1d48c888a9575cb768be59 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 May 2024 18:15:55 +0200 Subject: [PATCH 0774/1795] stick to x86_64/amd/zen3 when AMD Genoa (Zen4) is detected, until optimized software installations are available for Zen4 --- init/eessi_environment_variables | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 5450b2bfb4..e7f8f2faaa 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -48,6 +48,14 @@ if [ -d $EESSI_PREFIX ]; then fi if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then + # use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, + # since optimized software installations for Zen4 are a work-in-progress, + # see https://gitlab.com/eessi/support/-/issues/37 + if [[ "${EESSI_SOFTWARE_SUBDIR}" == "x86_64/amd/zen4" ]]; then + export EESSI_SOFTWARE_SUBDIR="x86_64/amd/zen3" + echo -e "\e[33mSticking to ${EESSI_SOFTWARE_SUBDIR} for now, since optimized installations for AMD Genoa (Zen4) are a work in progress, see https://gitlab.com/eessi/support/-/issues/37 for more information\e[0m" + fi + show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR From 8cce17c83ba3230774a56de285de3fe369792595 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 May 2024 19:19:54 +0200 Subject: [PATCH 0775/1795] relax check for existence of software overlay dir in script to create tarball to deploy --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 2d77acfc43..0a7669f73f 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -28,7 +28,7 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" -software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" +software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 From 39dddf641af1d59498076154c77c9de500451d9d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 7 May 2024 19:29:25 +0200 Subject: [PATCH 0776/1795] Update init/bash Co-authored-by: Kenneth Hoste --- init/bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/bash b/init/bash index 565c37cc65..655260841d 100644 --- a/init/bash +++ b/init/bash @@ -26,7 +26,7 @@ if [ $? -eq 0 ]; then # prepend location of modules for EESSI software stack to $MODULEPATH show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." module use $EESSI_MODULEPATH - echo "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." >> $output + echo "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." module use $EESSI_SITE_MODULEPATH #show_msg "" From 500b1de874a93e523ec308721f85a5824b8260e1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 7 May 2024 19:32:03 +0200 Subject: [PATCH 0777/1795] Update init/bash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Röblitz --- init/bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/bash b/init/bash index 655260841d..93471e075e 100644 --- a/init/bash +++ b/init/bash @@ -26,7 +26,7 @@ if [ $? -eq 0 ]; then # prepend location of modules for EESSI software stack to $MODULEPATH show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." module use $EESSI_MODULEPATH - echo "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." + show_msg "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." module use $EESSI_SITE_MODULEPATH #show_msg "" From 999cbd359528edf4feedd822859b9e2d0f922d9b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 7 May 2024 19:45:16 +0200 Subject: [PATCH 0778/1795] relax check or tarball is not created unless also software was built --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 2d77acfc43..0a7669f73f 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -28,7 +28,7 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" -software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" +software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 From 78b4e13a230b0b95cb4f41df013bf016c9b286b6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 May 2024 15:30:44 +0200 Subject: [PATCH 0779/1795] {2023.06,zen4} foss/2023b --- .../2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml new file mode 100644 index 0000000000..75dc4e6602 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - foss-2023b.eb From 40e269194483ec2574542f34ab55026390117df1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 12 May 2024 15:32:32 +0200 Subject: [PATCH 0780/1795] {2023.06}[2023a] Paraver 4.11.4 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index e65747e4a5..d6a71d0eee 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -4,3 +4,6 @@ easyconfigs: - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: options: from-pr: 20379 + - Paraver-4.11.4-foss-2023a.eb: + options: + from-pr: 20515 From ac3e74795c735eb99acf93e0fe9c024d6d81291a Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 15 May 2024 11:16:49 +0200 Subject: [PATCH 0781/1795] Only correct the Zen4 software subdirectory if it has not been overridden --- init/eessi_environment_variables | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 71e70ee071..78851a9c95 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -52,8 +52,10 @@ if [ -d $EESSI_PREFIX ]; then # since optimized software installations for Zen4 are a work-in-progress, # see https://gitlab.com/eessi/support/-/issues/37 if [[ "${EESSI_SOFTWARE_SUBDIR}" == "x86_64/amd/zen4" ]]; then - export EESSI_SOFTWARE_SUBDIR="x86_64/amd/zen3" - echo -e "\e[33mSticking to ${EESSI_SOFTWARE_SUBDIR} for now, since optimized installations for AMD Genoa (Zen4) are a work in progress, see https://gitlab.com/eessi/support/-/issues/37 for more information\e[0m" + if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then + export EESSI_SOFTWARE_SUBDIR="x86_64/amd/zen3" + echo -e "\e[33mSticking to ${EESSI_SOFTWARE_SUBDIR} for now, since optimized installations for AMD Genoa (Zen4) are a work in progress, see https://gitlab.com/eessi/support/-/issues/37 for more information\e[0m" + fi fi show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." From d043d867a3dccce1b978ee2531923e970b6e6176 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 15 May 2024 12:10:19 +0200 Subject: [PATCH 0782/1795] Add `psm2` to filtered dependencies for `2023.06` `PSM2` was introduced as a dependency of `libfabric` in https://github.com/easybuilders/easybuild-easyconfigs/pull/20501. We already have PSM2 in the compat layer, so we can filter this dependency out, but longer term we probably actually want it since it should be built with accelerator support. --- configure_easybuild | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure_easybuild b/configure_easybuild index c67b879cf3..60e3744603 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -32,6 +32,12 @@ if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then DEPS_TO_FILTER="${DEPS_TO_FILTER},Yasm" fi +# Version 23.06 of EESSI ships PSM2 in the compat layer, so we can filter this out while retaining support for OFA fabric +# (longer term this is probably not the right move as PSM2 should be configured with accelerator support, hence the restricted version) +if [[ "$EESSI_VERSION" == "2023.06" ]]; then + DEPS_TO_FILTER="${DEPS_TO_FILTER},psm2" +fi + export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER export EASYBUILD_MODULE_EXTENSIONS=1 From ec2e8bdf04456b85bf9dae3a46ea132acba90df3 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 15 May 2024 12:14:20 +0200 Subject: [PATCH 0783/1795] Update configure_easybuild --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index 60e3744603..ed3e651a4c 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -35,7 +35,7 @@ fi # Version 23.06 of EESSI ships PSM2 in the compat layer, so we can filter this out while retaining support for OFA fabric # (longer term this is probably not the right move as PSM2 should be configured with accelerator support, hence the restricted version) if [[ "$EESSI_VERSION" == "2023.06" ]]; then - DEPS_TO_FILTER="${DEPS_TO_FILTER},psm2" + DEPS_TO_FILTER="${DEPS_TO_FILTER},PSM2" fi export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER From 3cdb7b2653336dd707b65197bf578f4012a6b0ff Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 15 May 2024 12:43:43 +0200 Subject: [PATCH 0784/1795] Add a rebuild of GROMACS --- ...240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml new file mode 100644 index 0000000000..e767c63f6e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml @@ -0,0 +1,11 @@ +# 2024.05.15 +# Originally shipped version forgot to bump the gmxapi version and source +# tarball, it was still using an older version from the 2023.3 tarball. Looking +# at https://gitlab.com/gromacs/gromacs/-/blob/v2024.1/python_packaging/gmxapi/src/gmxapi/version.py?ref_type=tags#L68, +# the 2024.1 release includes gmxapi 0.5.0. +# +# See https://github.com/easybuilders/easybuild-easyconfigs/pull/20522 +easyconfigs: + - GROMACS-2024.1-foss-2023b.eb: + options: + from-pr: 20522 From fb329a4bfe535cffc8d70958fc65fa0072180416 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 15 May 2024 12:57:36 +0200 Subject: [PATCH 0785/1795] Also add new dependency on scikit-build-core --- .../20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml index e767c63f6e..4eca76fd66 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml @@ -4,8 +4,13 @@ # at https://gitlab.com/gromacs/gromacs/-/blob/v2024.1/python_packaging/gmxapi/src/gmxapi/version.py?ref_type=tags#L68, # the 2024.1 release includes gmxapi 0.5.0. # +# This also introduced a new build dependency on scikit-build-core for GROMACS +# # See https://github.com/easybuilders/easybuild-easyconfigs/pull/20522 easyconfigs: + - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: + options: + from-pr: 20526 - GROMACS-2024.1-foss-2023b.eb: options: from-pr: 20522 From 792ea3ebdbf011665f3b673ecd974b93e961f8c1 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 15 May 2024 15:01:57 +0200 Subject: [PATCH 0786/1795] Use from-commit for merged PRs --- .../20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml index 4eca76fd66..93d87bbcf3 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml @@ -10,7 +10,7 @@ easyconfigs: - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: options: - from-pr: 20526 + from-commit: 61d07bff09afe63cfe1ae35dc58a0c8be01eed62 - GROMACS-2024.1-foss-2023b.eb: options: - from-pr: 20522 + from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 From 6cf63359d04a7662ae3aa021763765bf01d5f8c7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 15 May 2024 16:06:35 +0200 Subject: [PATCH 0787/1795] Allow EESSI-extend as a loaded module when using EasyBuild --- EESSI-extend-2023.06-easybuild.eb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 76e96766f9..42defbacc8 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -139,12 +139,17 @@ setenv ("EASYBUILD_SYSROOT", sysroot) setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py")) -setenv ("EASYBUILD_UMASK", "002") + +-- Make sure to use the general umask that allows a global read +setenv ("EASYBUILD_UMASK", "022") + +-- All this module to be loaded when running EasyBuild +setenv ("EASYBUILD_ALLOW_LOADED_MODULES", "EasyBuild,EESSI-extend") -- Set all related environment variables if we have project or user installations (including extending MODULEPATH) if (user_modulepath ~= nil) then -- Use a more restrictive umask for this case - setenv ("EASYBUILD_UMASK", "022") + setenv ("EASYBUILD_UMASK", "077") setenv ("EASYBUILD_STICKY_BIT", "1") -- configure MODULEPATH if (project_modulepath ~= nil) then @@ -155,6 +160,7 @@ elseif (project_modulepath ~= nil) then setenv ("EASYBUILD_SET_GID_BIT", "1") setenv ("EASYBUILD_GROUP_WRITABLE_INSTALLDIR", "1") setenv ("EASYBUILD_STICKY_BIT", "0") + setenv ("EASYBUILD_UMASK", "002") -- configure MODULEPATH prepend_path("MODULEPATH", project_modulepath) end From fbdb8461fca7dbb28add02b128877d5b59bb24fc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 15 May 2024 16:11:17 +0200 Subject: [PATCH 0788/1795] Rebuild the module --- .../20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml new file mode 100644 index 0000000000..5cf2bd6b9d --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml @@ -0,0 +1,5 @@ +# 2024.05.15 +# The module is an EasyBuil created module and therefore needs to be an allowed +# module when running EasyBuild +easyconfigs: + - EESSI-extend-2023.06-easybuild.eb From 375bf370137a33b748ed649625e532d836a962d5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 15 May 2024 16:17:53 +0200 Subject: [PATCH 0789/1795] Update 20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml Issues seen when trying to use `from-commit` in https://github.com/EESSI/software-layer/pull/577 --- .../20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml index 93d87bbcf3..eacfad7079 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml @@ -10,7 +10,9 @@ easyconfigs: - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: options: - from-commit: 61d07bff09afe63cfe1ae35dc58a0c8be01eed62 + # from-commit: 61d07bff09afe63cfe1ae35dc58a0c8be01eed62 + from-pr: 20526 - GROMACS-2024.1-foss-2023b.eb: options: - from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 + # from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 + from-pr: 20522 From 2b30e1f2cca0a60b1dbf9ee0680aa48ba3f260da Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 15 May 2024 17:09:18 +0200 Subject: [PATCH 0790/1795] Allow overriding the Lmod GPU driver check --- EESSI-install-software.sh | 3 +++ create_lmodsitepackage.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 8a5789c2b2..40f0ed8c3d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -243,6 +243,9 @@ fi # if not, an error is produced, and the bot flags the whole build as failed (even when not installing GPU software) # ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +# Don't run the Lmod GPU driver check when doing builds (may not have a GPU, and it's not relevant for vanilla builds anyway) +export EESSI_OVERRIDE_GPU_CHECK=1 + # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 47aa20e51e..816302f7fc 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -131,8 +131,9 @@ end -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the EESSI linker, -- otherwise, refuse to load the requested module and print error message - local haveGpu = mt:haveProperty(simpleName,"arch","gpu") - if haveGpu then + local checkGpu = mt:haveProperty(simpleName,"arch","gpu") + local overrideGpuCheck = os.getenv("EESSI_OVERRIDE_GPU_CHECK") + if checkGpu and (overrideGpuCheck == nil) then local arch = os.getenv("EESSI_CPU_FAMILY") or "" local cudaVersionFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" local cudaDriverFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" @@ -141,7 +142,9 @@ if not (cudaDriverExists or singularityCudaExists) then local advice = "which relies on the CUDA runtime environment and driver libraries. " advice = advice .. "In order to be able to use the module, you will need " - advice = advice .. "to make sure EESSI can find the GPU driver libraries on your host system.\\n" + advice = advice .. "to make sure EESSI can find the GPU driver libraries on your host system. You can " + advice = advice .. "override this check by setting the environment variable EESSI_OVERRIDE_GPU_CHECK but " + advice = advice .. "the loaded application will not be able to execute on your system.\\n" advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) else From 705030d6aca132df57e5432aa6ada663f04d682d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 16 May 2024 12:48:25 +0200 Subject: [PATCH 0791/1795] use directory of called script as base dir for other scripts --- bot/inspect.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index 9d1fa87e1f..f0589cd827 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -27,6 +27,8 @@ # stop as soon as something fails set -e +SCRIPT_DIR=$(dirname $(realpath $BASH_SOURCE)) + display_help() { echo "usage: $0 [OPTIONS]" echo " -h | --help - display this usage information" @@ -81,8 +83,8 @@ done set -- "${POSITIONAL_ARGS[@]}" # source utils.sh and cfg_files.sh -source scripts/utils.sh -source scripts/cfg_files.sh +source ${SCRIPT_DIR}/../scripts/utils.sh +source ${SCRIPT_DIR}/../scripts/cfg_files.sh if [[ -z ${resume_tgz} ]]; then echo_red "path to tarball for resuming build job is missing" @@ -432,14 +434,14 @@ echo "Executing command to start interactive session to inspect build job:" # These initializations are combined into a single script that is executed when # the shell in startprefix is started. We set the env variable BASH_ENV here. if [[ -z ${run_in_prefix} ]]; then - echo "./eessi_container.sh ${CMDLINE_ARGS[@]}" + echo "${SCRIPT_DIR}/../eessi_container.sh ${CMDLINE_ARGS[@]}" echo " -- ${EESSI_COMPAT_LAYER_DIR}/startprefix" - ./eessi_container.sh "${CMDLINE_ARGS[@]}" \ + ${SCRIPT_DIR}/../eessi_container.sh "${CMDLINE_ARGS[@]}" \ -- ${EESSI_COMPAT_LAYER_DIR}/startprefix else - echo "./eessi_container.sh ${CMDLINE_ARGS[@]}" + echo "${SCRIPT_DIR}/../eessi_container.sh ${CMDLINE_ARGS[@]}" echo " -- ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< ${run_in_prefix}" - ./eessi_container.sh "${CMDLINE_ARGS[@]}" \ + ${SCRIPT_DIR}/../eessi_container.sh "${CMDLINE_ARGS[@]}" \ -- ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< ${run_in_prefix} fi From 082722eb6cb9028567f2b5af3bbb1c12d9fc5c15 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 16 May 2024 17:17:01 +0200 Subject: [PATCH 0792/1795] use single variable for directory containing script; use lowercase var name --- bot/inspect.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index f0589cd827..533968bffc 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -27,7 +27,9 @@ # stop as soon as something fails set -e -SCRIPT_DIR=$(dirname $(realpath $BASH_SOURCE)) +# script_dir is the directory that contains THIS (inspect.sh) script, usually +# stored in the directory '.../bot' +script_dir=$(dirname $(realpath $BASH_SOURCE)) display_help() { echo "usage: $0 [OPTIONS]" @@ -83,8 +85,8 @@ done set -- "${POSITIONAL_ARGS[@]}" # source utils.sh and cfg_files.sh -source ${SCRIPT_DIR}/../scripts/utils.sh -source ${SCRIPT_DIR}/../scripts/cfg_files.sh +source ${script_dir}/../scripts/utils.sh +source ${script_dir}/../scripts/cfg_files.sh if [[ -z ${resume_tgz} ]]; then echo_red "path to tarball for resuming build job is missing" @@ -257,10 +259,8 @@ CMDLINE_ARGS+=("--storage" "${JOB_STORAGE}") # make sure some environment settings are available inside the shell started via # startprefix -base_dir=$(dirname $(realpath $0)) -# base_dir of inspect.sh script is '.../bot', 'init' dir is at the same level # TODO better use script from tarball??? -source ${base_dir}/../init/eessi_defaults +source ${script_dir}/../init/eessi_defaults if [ -z $EESSI_VERSION ]; then echo "ERROR: \$EESSI_VERSION must be set!" >&2 @@ -434,14 +434,14 @@ echo "Executing command to start interactive session to inspect build job:" # These initializations are combined into a single script that is executed when # the shell in startprefix is started. We set the env variable BASH_ENV here. if [[ -z ${run_in_prefix} ]]; then - echo "${SCRIPT_DIR}/../eessi_container.sh ${CMDLINE_ARGS[@]}" + echo "${script_dir}/../eessi_container.sh ${CMDLINE_ARGS[@]}" echo " -- ${EESSI_COMPAT_LAYER_DIR}/startprefix" - ${SCRIPT_DIR}/../eessi_container.sh "${CMDLINE_ARGS[@]}" \ + ${script_dir}/../eessi_container.sh "${CMDLINE_ARGS[@]}" \ -- ${EESSI_COMPAT_LAYER_DIR}/startprefix else - echo "${SCRIPT_DIR}/../eessi_container.sh ${CMDLINE_ARGS[@]}" + echo "${script_dir}/../eessi_container.sh ${CMDLINE_ARGS[@]}" echo " -- ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< ${run_in_prefix}" - ${SCRIPT_DIR}/../eessi_container.sh "${CMDLINE_ARGS[@]}" \ + ${script_dir}/../eessi_container.sh "${CMDLINE_ARGS[@]}" \ -- ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< ${run_in_prefix} fi From 9fd84f95e19477a75d781cbf1590fd0068ee7587 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 17 May 2024 20:15:44 +0200 Subject: [PATCH 0793/1795] Fix syntax errors --- create_lmodsitepackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 61b40c4252..4817f14181 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -155,12 +155,12 @@ local mt = frameStk:mt() local simpleName = string.match(t.modFullName, "(.-)/") local version = string.match(t.modFullName, "%d.%d.%d") - if simpleName == 'ESPResSo' and version == '4.2.1' + if simpleName == 'ESPResSo' and version == '4.2.1' then local advice = 'Prefer versions >= 4.2.2 which include important bugfixes.\\n' advice = advice .. 'For details see https://github.com/espressomd/espresso/issues/4856\\n' advice = advice .. 'Use version 4.2.1 at your own risk!\\nn' LmodWarning("\\nESPReSso v4.2.1 has known issues and has been deprecated. ", advice) - end + end end -- Combine both functions into a single one, as we can only register one function as load hook in lmod From 9d8dbb9e105b8c325a267cb3e7f1504dccc82d94 Mon Sep 17 00:00:00 2001 From: Bob Droge Date: Tue, 21 May 2024 14:09:04 +0000 Subject: [PATCH 0794/1795] add (empty) RISC-V spec file --- init/arch_specs/eessi_arch_riscv.spec | 1 + 1 file changed, 1 insertion(+) create mode 100644 init/arch_specs/eessi_arch_riscv.spec diff --git a/init/arch_specs/eessi_arch_riscv.spec b/init/arch_specs/eessi_arch_riscv.spec new file mode 100644 index 0000000000..430dd2e72d --- /dev/null +++ b/init/arch_specs/eessi_arch_riscv.spec @@ -0,0 +1 @@ +# Software path in EESSI | Vendor ID | List of defining CPU features From 19d14eeb913fedd61604e1fc44675dbcc2a11799 Mon Sep 17 00:00:00 2001 From: Bob Droge Date: Tue, 21 May 2024 14:09:35 +0000 Subject: [PATCH 0795/1795] add initial riscv64 support --- init/eessi_archdetect.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 58b79b0f2a..e89bc75e13 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -85,6 +85,7 @@ cpupath(){ "x86_64") local spec_file="eessi_arch_x86.spec";; "aarch64") local spec_file="eessi_arch_arm.spec";; "ppc64le") local spec_file="eessi_arch_ppc.spec";; + "riscv64") local spec_file="eessi_arch_riscv.spec";; *) log "ERROR" "cpupath: Unsupported CPU architecture $machine_type" esac # spec files are located in a subfolder with this script From 02683fb5325fc7c599e4bd847a927a4447894e0c Mon Sep 17 00:00:00 2001 From: Bob Droge Date: Tue, 21 May 2024 14:09:56 +0000 Subject: [PATCH 0796/1795] use different defaults for RISC-V --- init/eessi_defaults | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/init/eessi_defaults b/init/eessi_defaults index d1779a36ae..c1ce82d1ce 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -8,8 +8,20 @@ # license: GPLv2 # -export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}" -export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}" +# use different defaults for RISC-V, as we want to redirect to the riscv.eessi.io repo +if [[ $(uname -m) == "riscv64" ]]; then + export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/riscv.eessi.io}" + export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=20240402}" + if [[ ! -v EESSI_SILENT ]]; then + echo "RISC-V architecture detected, but there is no RISC-V support yet in the production repository." + echo "Automatically switching to version ${EESSI_VERSION} of the RISC-V development repository ${EESSI_CVMFS_REPO}." + echo "For more details about this repository, see https://www.eessi.io/docs/repositories/riscv.eessi.io/." + echo "" + fi +else + export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}" + export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}" +fi # use archdetect by default, unless otherwise specified export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=1}" export EESSI_USE_ARCHSPEC="${EESSI_USE_ARCHSPEC:=0}" From 243620e734d1514d9104c85d8c0b835007c9dfd7 Mon Sep 17 00:00:00 2001 From: Bob Droge Date: Tue, 21 May 2024 14:10:23 +0000 Subject: [PATCH 0797/1795] update comment about CPU families: add RISC-V, remove POWER --- init/minimal_eessi_env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5b3562068d..03b337ccf7 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -16,7 +16,7 @@ else export EESSI_OS_TYPE='macos' fi -# aarch64 (Arm 64-bit), ppc64le (POWER 64-bit), x86_64 (x86 64-bit) +# aarch64 (Arm 64-bit), riscv64 (RISC-V 64-bit), x86_64 (x86 64-bit) export EESSI_CPU_FAMILY=$(uname -m) # set $EPREFIX since that is basically a standard in Gentoo Prefix From 84036fa5a262712ccde2bd0bdb9ee5b701e87e20 Mon Sep 17 00:00:00 2001 From: Bob Droge Date: Tue, 21 May 2024 21:06:19 +0000 Subject: [PATCH 0798/1795] also install RISC-V spec file --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index 17f0b81008..ab06e47997 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -92,7 +92,7 @@ copy_files_by_list ${TOPDIR}/init ${INSTALL_PREFIX}/init "${init_files[@]}" # Copy for the init/arch_specs directory arch_specs_files=( - eessi_arch_arm.spec eessi_arch_ppc.spec eessi_arch_x86.spec + eessi_arch_arm.spec eessi_arch_ppc.spec eessi_arch_riscv.spec eessi_arch_x86.spec ) copy_files_by_list ${TOPDIR}/init/arch_specs ${INSTALL_PREFIX}/init/arch_specs "${arch_specs_files[@]}" From 9a185ea42e8d96282f43fc9a90652347da9afb28 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 22 May 2024 17:51:15 +0200 Subject: [PATCH 0799/1795] Add ESPResSo v.4.2.2 to foss/2023a 4.9.1 stack --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index e65747e4a5..2fdc40e284 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -4,3 +4,6 @@ easyconfigs: - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: options: from-pr: 20379 + - ESPResSo-4.2.2-foss-2023a.eb: + options: + from-pr: 20595 From 20ccd475f61f664fd55cc17b857c750f0821ad19 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Thu, 23 May 2024 14:14:18 +0200 Subject: [PATCH 0800/1795] Advice refers to release notes, not single issue ESPResSo Lmod warning <= v4.2.1 points users to v4.2.2 release notes instead of only one of the bugs fixed --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index e0d7b31f9e..84b18d499b 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -179,7 +179,7 @@ local version = string.match(t.modFullName, "%d.%d.%d") if simpleName == 'ESPResSo' and version == '4.2.1' then local advice = 'Prefer versions >= 4.2.2 which include important bugfixes.\\n' - advice = advice .. 'For details see https://github.com/espressomd/espresso/issues/4856\\n' + advice = advice .. 'For details see https://github.com/espressomd/espresso/releases/tag/4.2.2\\n' advice = advice .. 'Use version 4.2.1 at your own risk!\\nn' LmodWarning("\\nESPReSso v4.2.1 has known issues and has been deprecated. ", advice) end From 7f328675deec19f88969eee91945f44617f68ab6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 23 May 2024 15:51:36 +0200 Subject: [PATCH 0801/1795] Use EESSI_OVERRIDE_GPU_CHECK everywhere --- eessi_container.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 7d00d1400c..a95a2c87c9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -464,10 +464,9 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" if [[ "${NVIDIA_MODE}" == "install" ]] ; then - # We need to "trick" our LMOD_RC file to allow us to load CUDA modules even without a CUDA driver - # (this works because we build within a container and the LMOD_RC recognises that) - touch ${EESSI_TMPDIR}/libcuda.so - export SINGULARITY_CONTAINLIBS="${EESSI_TMPDIR}/libcuda.so" + # No GPU so we need to "trick" Lmod to allow us to load CUDA modules even without a CUDA driver + # (this variable means EESSI_OVERRIDE_GPU_CHECK=1 will be set inside the container) + export SINGULARITYENV_EESSI_OVERRIDE_GPU_CHECK=1 fi fi fi From df4ac260e209e9ccc2578562845ceb2a328dbbb0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 23 May 2024 16:05:27 +0200 Subject: [PATCH 0802/1795] Restrict overridding the GPU driver check to when we know it is valid --- EESSI-install-software.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 40f0ed8c3d..8a5789c2b2 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -243,9 +243,6 @@ fi # if not, an error is produced, and the bot flags the whole build as failed (even when not installing GPU software) # ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh -# Don't run the Lmod GPU driver check when doing builds (may not have a GPU, and it's not relevant for vanilla builds anyway) -export EESSI_OVERRIDE_GPU_CHECK=1 - # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then From bee4b001322a0d22c79642b5b1427f2c397a9f3b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 24 May 2024 10:47:45 +0200 Subject: [PATCH 0803/1795] Make sure EESSI_OVERRIDE_GPU_CHECK is still set when in prefix shell --- run_in_compat_layer_env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index f57c4d0749..b8e9cf979b 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -26,6 +26,9 @@ fi if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_OVERRIDE_GPU_CHECK} ]; then + INPUT="export EESSI_OVERRIDE_GPU_CHECK=${EESSI_OVERRIDE_GPU_CHECK}; ${INPUT}" +fi if [ ! -z ${http_proxy} ]; then INPUT="export http_proxy=${http_proxy}; ${INPUT}" fi From f788ca3ab94ab384ee2e4a98e5b76e2a9317102f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 24 May 2024 11:18:04 +0200 Subject: [PATCH 0804/1795] Only install NVIDIA drivers if nvidia-smi command exists --- EESSI-install-software.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 8a5789c2b2..7d358e205a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -17,6 +17,11 @@ display_help() { echo " --skip-cuda-install - disable installing a full CUDA SDK in the host_injections prefix (e.g. in CI)" } +# Function to check if a command exists +function command_exists() { + command -v "$1" >/dev/null 2>&1 +} + function copy_build_log() { # copy specified build log to specified directory, with some context added build_log=${1} @@ -238,10 +243,11 @@ else echo "Skipping installation of CUDA SDK in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found" fi -# Install drivers in host_injections -# TODO: this is commented out for now, because the script assumes that nvidia-smi is available and works; -# if not, an error is produced, and the bot flags the whole build as failed (even when not installing GPU software) -# ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +# Install NVIDIA drivers in host_injections (if they exist) +if command_exists "nvidia-smi"; then + echo "Command 'nvidia-smi' found. Installing NVIDIA drivers for use in prefix shell..." + ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +fi # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') From f974c34a85b3fb322906b974b7fcf084fa6398df Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Mon, 27 May 2024 09:49:10 +0200 Subject: [PATCH 0805/1795] Fix typo in create_lmodsitepackage.py ESPReSso warning --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 84b18d499b..31c040718d 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -180,7 +180,7 @@ if simpleName == 'ESPResSo' and version == '4.2.1' then local advice = 'Prefer versions >= 4.2.2 which include important bugfixes.\\n' advice = advice .. 'For details see https://github.com/espressomd/espresso/releases/tag/4.2.2\\n' - advice = advice .. 'Use version 4.2.1 at your own risk!\\nn' + advice = advice .. 'Use version 4.2.1 at your own risk!\\n' LmodWarning("\\nESPReSso v4.2.1 has known issues and has been deprecated. ", advice) end end From 39c8abac220d428dc60f0fcf016dbcf6e0a2d049 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Mon, 27 May 2024 09:59:09 +0200 Subject: [PATCH 0806/1795] ESPReSso -> ESPResSo --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 31c040718d..bca2970e39 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -181,7 +181,7 @@ local advice = 'Prefer versions >= 4.2.2 which include important bugfixes.\\n' advice = advice .. 'For details see https://github.com/espressomd/espresso/releases/tag/4.2.2\\n' advice = advice .. 'Use version 4.2.1 at your own risk!\\n' - LmodWarning("\\nESPReSso v4.2.1 has known issues and has been deprecated. ", advice) + LmodWarning("\\nESPResSo v4.2.1 has known issues and has been deprecated. ", advice) end end From b15fc3d58c270ad3e28727f30e92c78744cbeee9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 27 May 2024 10:54:00 +0200 Subject: [PATCH 0807/1795] Update bot build script to support whether GPU is available or not --- bot/build.sh | 10 +++++++++- scripts/utils.sh | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index dcc61c19d4..c9a362fdca 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -223,7 +223,15 @@ mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") # add options required to handle NVIDIA support -BUILD_STEP_ARGS+=("--nvidia" "all") +if command_exists "nvidia-smi"; then + echo "Command 'nvidia-smi' found, using available GPU" + BUILD_STEP_ARGS+=("--nvidia" "all") +else + echo "No 'nvidia-smi' found, no available GPU but allowing overriding this check" + BUILD_STEP_ARGS+=("--nvidia" "install") +fi +# Retain location for host injections so we don't reinstall CUDA +# (Always need to run the driver installation as available driver may change) if [[ ! -z ${SHARED_FS_PATH} ]]; then BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi diff --git a/scripts/utils.sh b/scripts/utils.sh index b2be3f6221..962decd20e 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -78,6 +78,11 @@ function create_directory_structure() { return $return_code } +# Function to check if a command exists +function command_exists() { + command -v "$1" >/dev/null 2>&1 +} + function get_path_for_tool { tool_name=$1 tool_envvar_name=$2 From bb4b3c1b17e9eb6ca588a64690445a505353c1c9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 28 May 2024 20:31:11 +0200 Subject: [PATCH 0808/1795] {2023.06}[gfbf/2023a] ipympl v0.9.3 --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 10 +++++++--- .../2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 8 ++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 8f5608d881..4b58cb6106 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -48,9 +48,13 @@ easyconfigs: - PyOpenGL-3.1.7-GCCcore-12.3.0.eb: options: from-pr: 20007 - - ipympl-0.9.3-foss-2023a.eb: - options: - from-pr: 20126 + # removed by https://github.com/easybuilders/easybuild-easyconfigs/pull/20586 + # adding ipympl-0.9.3-gfbf-2023a.eb as a replacement in + # easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml + # comment the below out here or CI will fail + # - ipympl-0.9.3-foss-2023a.eb: + # options: + # from-pr: 20126 - OpenJPEG-2.5.0-GCCcore-12.3.0.eb - OpenFOAM-10-foss-2023a.eb - Highway-1.0.4-GCCcore-12.3.0.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index e65747e4a5..32b388854d 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -4,3 +4,11 @@ easyconfigs: - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: options: from-pr: 20379 + # replacement for ipympl-0.9.3-foss-2023a.eb which has been built via + # easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml and was + # later removed in https://github.com/easybuilders/easybuild-easyconfigs/pull/20586 + # below we use the replacement ec file which is provided by + # https://github.com/easybuilders/easybuild-easyconfigs/pull/18852 + - ipympl-0.9.3-gfbf-2023a.eb: + options: + from-pr: 18852 From 9a036d8c9059c4eda163279322dda915e601b7e3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 29 May 2024 06:34:05 +0200 Subject: [PATCH 0809/1795] {2023.06}[foss/2023a] GATK v4.5.0.0 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 8238b379f9..3e5c09090e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -15,3 +15,4 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb: options: from-pr: 20595 + - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb From 7c48321761b291bb68046722f628360837a134e0 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 31 May 2024 11:52:11 +0200 Subject: [PATCH 0810/1795] Print message, not warning On ESPResSo version 4.2.1 message, don't use `LmodWarning` but use `LmodMessage` instead --- create_lmodsitepackage.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index ed98d1f0f0..f159f186eb 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -175,16 +175,18 @@ end end -local function eessi_espresso_deprecated_warning(t) +local function eessi_espresso_deprecated_message(t) local frameStk = require("FrameStk"):singleton() local mt = frameStk:mt() local simpleName = string.match(t.modFullName, "(.-)/") local version = string.match(t.modFullName, "%d.%d.%d") if simpleName == 'ESPResSo' and version == '4.2.1' then + -- Print a message on loading ESPreSso v <= 4.2.1 recommending using v 4.2.2 and above. + -- A message and not a warning as the exit code would break CI runs otherwise. local advice = 'Prefer versions >= 4.2.2 which include important bugfixes.\\n' advice = advice .. 'For details see https://github.com/espressomd/espresso/releases/tag/4.2.2\\n' advice = advice .. 'Use version 4.2.1 at your own risk!\\n' - LmodWarning("\\nESPResSo v4.2.1 has known issues and has been deprecated. ", advice) + LmodMessage("\\nESPResSo v4.2.1 has known issues and has been deprecated. ", advice) end end From 76babe873669777a79a30e6abc33517873fa31a3 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 31 May 2024 13:57:20 +0200 Subject: [PATCH 0811/1795] Correctly use `eessi_espresso_deprecated_message()` --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index f159f186eb..7e55bce2a5 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -193,7 +193,7 @@ -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) - eessi_espresso_deprecated_warning(t) + eessi_espresso_deprecated_message(t) -- Only apply CUDA hooks if the loaded module is in the EESSI prefix -- This avoids getting an Lmod Error when trying to load a CUDA module from a local software stack if from_eessi_prefix(t) then From 22578c43328a6e6257b5023e0cac52d21c5f637b Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 3 Jun 2024 09:43:04 +0000 Subject: [PATCH 0812/1795] {2023.06}[foss/2022b] ParaView v5.11.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml index e547e45f2b..d023fbbaf1 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml @@ -2,3 +2,4 @@ easyconfigs: - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: options: from-pr: 20379 + - ParaView-5.11.1-foss-2022b.eb From c77c2676ce98260b88fc9b23130f82cf693168f3 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 4 Jun 2024 06:37:24 +0000 Subject: [PATCH 0813/1795] {2023.06}[foss/2023a] WhatsHap v2.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 3e5c09090e..49a7191121 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -16,3 +16,4 @@ easyconfigs: options: from-pr: 20595 - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb + - WhatsHap-2.2-foss-2023a.eb From 3d3019db4be7ecda681ae862ab33962cc1d61e37 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 4 Jun 2024 10:48:27 +0200 Subject: [PATCH 0814/1795] Update scorecards to latest version --- .github/workflows/scorecards.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 7eff557094..5b482763f2 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -4,24 +4,20 @@ name: Scorecards supply-chain security on: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - cron: '25 15 * * 3' push: - branches: [ "main" ] - pull_request: - branches: - - main + branches: + - '2023.06-software.eessi.io' # Declare default permissions as read only. permissions: read-all jobs: analysis: + if: github.repository_owner == 'EESSI' # Prevent running on forks name: Scorecards analysis runs-on: ubuntu-latest permissions: @@ -35,12 +31,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3 with: results_file: results.sarif results_format: sarif @@ -62,7 +58,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: SARIF file path: results.sarif @@ -70,6 +66,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27 + uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 with: sarif_file: results.sarif From a111b6db76a816e030b40d9d4e60e93686f80709 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Jun 2024 07:53:55 +0200 Subject: [PATCH 0815/1795] {2023.06,2023b,zen4} SciPy-bundle v2023.11 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml index 75dc4e6602..bc736c4056 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023b.yml @@ -1,2 +1,3 @@ easyconfigs: - foss-2023b.eb + - SciPy-bundle-2023.11-gfbf-2023b.eb From fe809a5228a05c699169fd1280de89ce15a753b8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Jun 2024 07:54:28 +0200 Subject: [PATCH 0816/1795] {2023.06,2023b,zen4} SciPy-bundle v2023.07 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml index c821df1afb..b3afe2b368 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,2 +1,3 @@ easyconfigs: - foss-2023a.eb + - SciPy-bundle-2023.07-gfbf-2023a.eb From fa2fd59c37432395ac45dceffdd99c8a4ea08948 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 6 Jun 2024 06:36:30 +0000 Subject: [PATCH 0817/1795] {2023.06}[gfbf/2022b] ASE v3.22.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml index d023fbbaf1..385f33b5cb 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml @@ -3,3 +3,4 @@ easyconfigs: options: from-pr: 20379 - ParaView-5.11.1-foss-2022b.eb + - ASE-3.22.1-gfbf-2022b.eb From 08e09ca1655348b7e08e11a4cca710522509f291 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 7 Jun 2024 06:14:03 +0000 Subject: [PATCH 0818/1795] {2023.06}[foss/2023b] NLTK v3.8.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml index 1392fa2f5a..b06cb2788e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml @@ -2,3 +2,4 @@ easyconfigs: - GROMACS-2024.1-foss-2023b.eb: options: from-pr: 20439 + - NLTK-3.8.1-foss-2023b.eb From 9b3f02da7e8c1279cf20cb9a37356b7c430e32f7 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 7 Jun 2024 08:07:14 +0000 Subject: [PATCH 0819/1795] {2023.06}[foss/2022b] SEPP v4.5.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml index 385f33b5cb..57c985b020 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml @@ -4,3 +4,4 @@ easyconfigs: from-pr: 20379 - ParaView-5.11.1-foss-2022b.eb - ASE-3.22.1-gfbf-2022b.eb + - SEPP-4.5.1-foss-2022b.eb From 46d9018f40db4ca174f60746102335ba87ad53b5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 7 Jun 2024 12:03:50 +0200 Subject: [PATCH 0820/1795] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- EESSI-extend-2023.06-easybuild.eb | 2 +- .../rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 42defbacc8..b525ee462d 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -143,7 +143,7 @@ setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuil -- Make sure to use the general umask that allows a global read setenv ("EASYBUILD_UMASK", "022") --- All this module to be loaded when running EasyBuild +-- Allow this module to be loaded when running EasyBuild setenv ("EASYBUILD_ALLOW_LOADED_MODULES", "EasyBuild,EESSI-extend") -- Set all related environment variables if we have project or user installations (including extending MODULEPATH) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml index 5cf2bd6b9d..93c4950fc8 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml @@ -1,5 +1,5 @@ # 2024.05.15 -# The module is an EasyBuil created module and therefore needs to be an allowed +# The module is an EasyBuild created module and therefore needs to be an allowed # module when running EasyBuild easyconfigs: - EESSI-extend-2023.06-easybuild.eb From 51118efbea0730b820493ff27dab231f256fb53f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 7 Jun 2024 12:08:09 +0200 Subject: [PATCH 0821/1795] remove double hyphen in filename --- ...loaded.yml => 20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/rebuilds/{20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml => 20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240506--eb-4.9.1-EESSI-extend-allow-loaded.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml From 408417394078072338bb13bce6d226cf08ebec7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 7 Jun 2024 14:19:29 +0200 Subject: [PATCH 0822/1795] use PRs with fixes for easyconfig+easyblock Co-authored-by: ocaisa --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 674cc49a39..5ca2d7d3ec 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -5,3 +5,6 @@ easyconfigs: options: from-pr: 20379 - OpenFOAM-v2312-foss-2023a.eb + options: + include-easyblocks-from-pr: 3328 + from-pr: 20517 From 958e78ac5f03af4faccf59caa5f18c2544f20da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 7 Jun 2024 14:27:38 +0200 Subject: [PATCH 0823/1795] add missing colon --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index cb7ee4ab27..c9334c2acf 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -17,7 +17,7 @@ easyconfigs: from-pr: 20595 - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - WhatsHap-2.2-foss-2023a.eb - - OpenFOAM-v2312-foss-2023a.eb + - OpenFOAM-v2312-foss-2023a.eb: options: include-easyblocks-from-pr: 3328 from-pr: 20517 From 3bdfff7a7fe360b066a971a0c29f67414470c12f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 8 Jun 2024 14:33:46 +0200 Subject: [PATCH 0824/1795] {2023.06,zen4}[foss/2023a] ESPResSo 4.2.1 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml index b3afe2b368..a529746676 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,3 +1,4 @@ easyconfigs: - foss-2023a.eb - SciPy-bundle-2023.07-gfbf-2023a.eb + - ESPResSo-4.2.1-foss-2023a.eb From 0b715d73f2fb7c157a28953de233cf71703b6136 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 10 Jun 2024 05:52:26 +0000 Subject: [PATCH 0825/1795] {2023.06}[gompi/2023a] BLAST+ v2.14.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 49a7191121..f53ad9235c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -17,3 +17,6 @@ easyconfigs: from-pr: 20595 - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - WhatsHap-2.2-foss-2023a.eb + - BLAST+-2.14.1-gompi-2023a.eb: + options: + from-pr: 20751 From c7a2e6e348e99a2ac7ca170c4c2d7b1eddc46a94 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 10 Jun 2024 11:06:39 +0200 Subject: [PATCH 0826/1795] Change smile for failed test runs --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index f045b9500a..3b16e5c415 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -101,7 +101,7 @@ elif [[ ${ERROR} -eq 1 ]]; then reason="EESSI test suite was not run, test step itself failed to execute." status="FAILURE" else - summary=":grin: FAILURE" + summary=":cry: FAILURE" reason="Failed for unknown reason" status="FAILURE" fi From ad17f4384c1e9500c9c34aa71ae6b0d51e3c446e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 10 Jun 2024 11:43:55 +0200 Subject: [PATCH 0827/1795] Fix emoticon --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index f045b9500a..3b16e5c415 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -101,7 +101,7 @@ elif [[ ${ERROR} -eq 1 ]]; then reason="EESSI test suite was not run, test step itself failed to execute." status="FAILURE" else - summary=":grin: FAILURE" + summary=":cry: FAILURE" reason="Failed for unknown reason" status="FAILURE" fi From 596c70299d3f73098f0e6b711e5b246ffd54766e Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:24:13 +0200 Subject: [PATCH 0828/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index f53ad9235c..be3b9c1bbf 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -19,4 +19,4 @@ easyconfigs: - WhatsHap-2.2-foss-2023a.eb - BLAST+-2.14.1-gompi-2023a.eb: options: - from-pr: 20751 + from-pr: 20784 From f288436f8193c70e87485bce1e3866915525720a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 10 Jun 2024 20:42:07 +0200 Subject: [PATCH 0829/1795] add Valgrind --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml | 1 + .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml index 57c985b020..1805c581c3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml @@ -5,3 +5,4 @@ easyconfigs: - ParaView-5.11.1-foss-2022b.eb - ASE-3.22.1-gfbf-2022b.eb - SEPP-4.5.1-foss-2022b.eb + - Valgrind-3.21.0-gompi-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 49a7191121..db9a6d7e36 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -17,3 +17,4 @@ easyconfigs: from-pr: 20595 - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - WhatsHap-2.2-foss-2023a.eb + - Valgrind-3.21.0-gompi-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml index b06cb2788e..c071f220cf 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml @@ -3,3 +3,4 @@ easyconfigs: options: from-pr: 20439 - NLTK-3.8.1-foss-2023b.eb + - Valgrind-3.23.0-gompi-2023b.eb From c9287ed954484034e94fed5a8b6292d8c135bb1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 10 Jun 2024 20:44:35 +0200 Subject: [PATCH 0830/1795] add --from-pr to Valgrind --- .../2023.06/eessi-2023.06-eb-4.9.1-2023b.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml index c071f220cf..888bddace3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml @@ -3,4 +3,6 @@ easyconfigs: options: from-pr: 20439 - NLTK-3.8.1-foss-2023b.eb - - Valgrind-3.23.0-gompi-2023b.eb + - Valgrind-3.23.0-gompi-2023b.eb: + options: + from-pr: 20792 From c4af3895e61781cba2dbb3d5982e024d95a15d77 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 11 Jun 2024 09:19:04 +0000 Subject: [PATCH 0831/1795] {2023.06}[foss/2023a] OrthoFinder v2.5.5 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index be3b9c1bbf..1d74452e45 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -20,3 +20,4 @@ easyconfigs: - BLAST+-2.14.1-gompi-2023a.eb: options: from-pr: 20784 + - OrthoFinder-2.5.5-foss-2023a.eb From 525b23e9af30453137617ce27f456279151bc1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 10:26:49 +0200 Subject: [PATCH 0832/1795] add easystack for EB 4.9.2 and install it --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml new file mode 100644 index 0000000000..cd87626494 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -0,0 +1,5 @@ +easyconfigs: + - EasyBuild-4.9.2.eb: + options: + from-pr: 20801 + include-easyblocks-from-pr: 3358 From 460da691db015a88e00a0df8548f3f0721b0c6ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 13:03:30 +0200 Subject: [PATCH 0833/1795] add EB 4.9.2 --- .../2023.06/eessi-2023.06-eb-4.9.1-001-system.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml index 46ac979719..8f96cf9450 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -3,3 +3,7 @@ easyconfigs: options: from-pr: 20299 - EESSI-extend-2023.06-easybuild.eb + - EasyBuild-4.9.2.eb: + options: + from-pr: 20801 + include-easyblocks-from-pr: 3358 From 911737d2bea6cea92c09262bcfa605e18ba553eb Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Jun 2024 14:05:46 +0200 Subject: [PATCH 0834/1795] Add template field for mem per node --- reframe_config_bot.py.tmpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl index 0cc3e9f530..607373767a 100644 --- a/reframe_config_bot.py.tmpl +++ b/reframe_config_bot.py.tmpl @@ -34,6 +34,11 @@ site_configuration = { 'options': ['--mem={size}'], } ], + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': __MEM_PER_NODE__, + }, 'max_jobs': 1 } ] From e10b227d3299a0945edd5ecede13b9784c4fd748 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Jun 2024 14:14:36 +0200 Subject: [PATCH 0835/1795] Get memory limit for cgroup of current job / UID and put this into the ReFrame config file --- test_suite.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 95eb9daa2a..5c157f7a47 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -135,7 +135,7 @@ export RFM_PREFIX=$PWD/reframe_runs echo "Configured reframe with the following environment variables:" env | grep "RFM_" -# Inject correct CPU properties into the ReFrame config file +# Inject correct CPU/memory properties into the ReFrame config file cpuinfo=$(lscpu) if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then cpu_count=${BASH_REMATCH[1]} @@ -157,11 +157,19 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi +cgroup_mem_bytes=$(cat /sys/fs/cgroup/memory/slurm/uid_${UID}/job_${SLURM_JOB_ID}/memory.limit_in_bytes) +if [[ $? -eq 0 ]] + # Convert to MiB + cgroup_mem_mib=$((cgroup_mem_bytes/(1024*1024))) +else + fatal_error "Failed to get the memory limit in bytes from the current cgroup" +fi cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES +sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib/g" $RFM_CONFIG_FILES # Workaround for https://github.com/EESSI/software-layer/pull/467#issuecomment-1973341966 export PSM3_DEVICES='self,shm' # this is enough, since we only run single node for now From 644eddcb66a46871cdfe93ef54ca70954169f548 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Jun 2024 14:16:02 +0200 Subject: [PATCH 0836/1795] Missing bracket --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 5c157f7a47..2d072d3d95 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -169,7 +169,7 @@ sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES -sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib/g" $RFM_CONFIG_FILES +sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES # Workaround for https://github.com/EESSI/software-layer/pull/467#issuecomment-1973341966 export PSM3_DEVICES='self,shm' # this is enough, since we only run single node for now From bcba10e3553806f4f1dda6d27412f4756464f13c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Jun 2024 14:16:48 +0200 Subject: [PATCH 0837/1795] Fix if-else syntax --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 2d072d3d95..46b43ee78a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -158,7 +158,7 @@ else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi cgroup_mem_bytes=$(cat /sys/fs/cgroup/memory/slurm/uid_${UID}/job_${SLURM_JOB_ID}/memory.limit_in_bytes) -if [[ $? -eq 0 ]] +if [[ $? -eq 0 ]]; then # Convert to MiB cgroup_mem_mib=$((cgroup_mem_bytes/(1024*1024))) else From 70c1b146789d3d0c4d92f0a368ab078c724b898a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Jun 2024 15:18:17 +0200 Subject: [PATCH 0838/1795] Make sure we bind-mount /sys/fs/cgroup into the container, so that we can use it to determine available memory for dynamically configuring ReFrame in the test step --- bot/test.sh | 3 ++ eessi_container.sh | 78 ++++++++++++++++++++++++++-------------------- test_suite.sh | 2 +- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index 4984340e6e..b3f6acf0ea 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -204,6 +204,9 @@ if [[ -z ${RESUME_DIR} ]]; then else TEST_STEP_ARGS+=("--resume" "${RESUME_DIR}") fi +# Bind mount /sys/fs/cgroup so that we can determine the amount of memory available in our cgroup for +# Reframe configuration +TEST_STEP_ARGS+=("--extra-bind-paths /sys/fs/cgroup:/hostsys/fs/cgroup:ro") # prepare arguments to test_suite.sh (specific to test step) declare -a TEST_SUITE_ARGS=() diff --git a/eessi_container.sh b/eessi_container.sh index a95a2c87c9..eac8e74c83 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -70,40 +70,43 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" display_help() { echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" - echo " -c | --container IMG - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian11]" - echo " -f | --fakeroot - run the container with --fakeroot [default: false]" - echo " -g | --storage DIR - directory space on host machine (used for" - echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" - echo " -h | --help - display this usage information [default: false]" - echo " -i | --host-injections - directory to link to for host_injections " - echo " [default: /..storage../opt-eessi]" - echo " -l | --list-repos - list available repository identifiers [default: false]" - echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script or command) [default: shell]" - echo " -n | --nvidia MODE - configure the container to work with NVIDIA GPUs," - echo " MODE==install for a CUDA installation, MODE==run to" - echo " attach a GPU, MODE==all for both [default: false]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI via" - echo " default container, see --container]" - echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," - echo " where DIR points to a previously used tmp directory" - echo " (check for output 'Using DIR as tmp ...' of a previous" - echo " run) and TGZ is the path to a tarball which is" - echo " unpacked the tmp dir stored on the local storage space" - echo " (see option --storage above) [default: not set]" - echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" - echo " directory DIR or provided with the fixed full path TGZ" - echo " when a directory is provided, the format of the" - echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" - echo " [default: not set]" - echo " -v | --verbose - display more information [default: false]" - echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" - echo " [default: not set]; uses env var \$http_proxy if set" - echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" - echo " [default: not set]; uses env var \$https_proxy if set" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMG - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian11]" + echo " -f | --fakeroot - run the container with --fakeroot [default: false]" + echo " -g | --storage DIR - directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -h | --help - display this usage information [default: false]" + echo " -i | --host-injections - directory to link to for host_injections " + echo " [default: /..storage../opt-eessi]" + echo " -l | --list-repos - list available repository identifiers [default: false]" + echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" + echo " MODE==run (run a script or command) [default: shell]" + echo " -n | --nvidia MODE - configure the container to work with NVIDIA GPUs," + echo " MODE==install for a CUDA installation, MODE==run to" + echo " attach a GPU, MODE==all for both [default: false]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI via" + echo " default container, see --container]" + echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," + echo " where DIR points to a previously used tmp directory" + echo " (check for output 'Using DIR as tmp ...' of a previous" + echo " run) and TGZ is the path to a tarball which is" + echo " unpacked the tmp dir stored on the local storage space" + echo " (see option --storage above) [default: not set]" + echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" + echo " directory DIR or provided with the fixed full path TGZ" + echo " when a directory is provided, the format of the" + echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" + echo " [default: not set]" + echo " -v | --verbose - display more information [default: false]" + echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + echo " [default: not set]; uses env var \$http_proxy if set" + echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + echo " [default: not set]; uses env var \$https_proxy if set" + echo " -b | --extra-bind-paths - specify extra paths to be bound into the container." + echo " To specify multiple bind paths, seperate by comma." + echo " Example: '/src:/dest:ro,/src2:/dest2:rw'" echo echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If" echo " arguments to the script/command start with '-' or '--', use the flag terminator" @@ -197,6 +200,10 @@ while [[ $# -gt 0 ]]; do export https_proxy=${HTTPS_PROXY} shift 2 ;; + -b|--extra-bind-paths) + EXTRA_BIND_PATHS="$2" + shift 2 + ;; --) shift POSITIONAL_ARGS+=("$@") # save positional args @@ -439,6 +446,9 @@ fi BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" # provide a '/tmp' inside the container BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" +if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then + BIND_PATHS="${BIND_PATHS},${EXTRA_BIND_PATHS}" +fi [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" diff --git a/test_suite.sh b/test_suite.sh index 46b43ee78a..6e73fbd87c 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -157,7 +157,7 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi -cgroup_mem_bytes=$(cat /sys/fs/cgroup/memory/slurm/uid_${UID}/job_${SLURM_JOB_ID}/memory.limit_in_bytes) +cgroup_mem_bytes=$(cat /hostsys/fs/cgroup/memory/slurm/uid_${UID}/job_${SLURM_JOB_ID}/memory.limit_in_bytes) if [[ $? -eq 0 ]]; then # Convert to MiB cgroup_mem_mib=$((cgroup_mem_bytes/(1024*1024))) From 5bfd7c1c0898ee268530295f794438cd1fd29836 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:45:32 +0200 Subject: [PATCH 0839/1795] Update eessi_container.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- eessi_container.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index eac8e74c83..55b0aa4c63 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -71,6 +71,9 @@ display_help() { echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -b | --extra-bind-paths - specify extra paths to be bound into the container." + echo " To specify multiple bind paths, seperate by comma." + echo " Example: '/src:/dest:ro,/src2:/dest2:rw'" echo " -c | --container IMG - image file or URL defining the container to use" echo " [default: docker://ghcr.io/eessi/build-node:debian11]" echo " -f | --fakeroot - run the container with --fakeroot [default: false]" @@ -104,9 +107,6 @@ display_help() { echo " [default: not set]; uses env var \$http_proxy if set" echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" echo " [default: not set]; uses env var \$https_proxy if set" - echo " -b | --extra-bind-paths - specify extra paths to be bound into the container." - echo " To specify multiple bind paths, seperate by comma." - echo " Example: '/src:/dest:ro,/src2:/dest2:rw'" echo echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If" echo " arguments to the script/command start with '-' or '--', use the flag terminator" From a7fca5155a4e9c145d73c416c85e4c1e6f2c2c12 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Jun 2024 16:46:33 +0200 Subject: [PATCH 0840/1795] Sort alphabetically --- eessi_container.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index eac8e74c83..aa5df50af6 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -137,6 +137,10 @@ while [[ $# -gt 0 ]]; do ACCESS="$2" shift 2 ;; + -b|--extra-bind-paths) + EXTRA_BIND_PATHS="$2" + shift 2 + ;; -c|--container) CONTAINER="$2" shift 2 @@ -200,10 +204,6 @@ while [[ $# -gt 0 ]]; do export https_proxy=${HTTPS_PROXY} shift 2 ;; - -b|--extra-bind-paths) - EXTRA_BIND_PATHS="$2" - shift 2 - ;; --) shift POSITIONAL_ARGS+=("$@") # save positional args From 13366e7c3f3c1aa59192f70e8a26e451611a15d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 16:48:42 +0200 Subject: [PATCH 0841/1795] fix typo --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 891b793f9d..fb14e2118f 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -72,7 +72,7 @@ display_help() { echo " OPTIONS:" echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -b | --extra-bind-paths - specify extra paths to be bound into the container." - echo " To specify multiple bind paths, seperate by comma." + echo " To specify multiple bind paths, separate by comma." echo " Example: '/src:/dest:ro,/src2:/dest2:rw'" echo " -c | --container IMG - image file or URL defining the container to use" echo " [default: docker://ghcr.io/eessi/build-node:debian11]" From 63c1abfb8d03fbceb23914d3143abef68dd23bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 16:55:06 +0200 Subject: [PATCH 0842/1795] remove EB 4.9.2 from 4.9.1 easystack --- .../2023.06/eessi-2023.06-eb-4.9.1-001-system.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml index 8f96cf9450..46ac979719 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -3,7 +3,3 @@ easyconfigs: options: from-pr: 20299 - EESSI-extend-2023.06-easybuild.eb - - EasyBuild-4.9.2.eb: - options: - from-pr: 20801 - include-easyblocks-from-pr: 3358 From 7f6cb8d740ef9761cabd65eb221d9cdc52a84f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 17:04:18 +0200 Subject: [PATCH 0843/1795] remove include-easyblocks-from-pr for EB 4.9.2 --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml index cd87626494..a90f565412 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -2,4 +2,3 @@ easyconfigs: - EasyBuild-4.9.2.eb: options: from-pr: 20801 - include-easyblocks-from-pr: 3358 From 1d56c7239aec4a4521f1114c482b95b7b1f8f2c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 17:21:47 +0200 Subject: [PATCH 0844/1795] use separate quotes for option and argument --- bot/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/test.sh b/bot/test.sh index b3f6acf0ea..04bff346cd 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -206,7 +206,7 @@ else fi # Bind mount /sys/fs/cgroup so that we can determine the amount of memory available in our cgroup for # Reframe configuration -TEST_STEP_ARGS+=("--extra-bind-paths /sys/fs/cgroup:/hostsys/fs/cgroup:ro") +TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro") # prepare arguments to test_suite.sh (specific to test step) declare -a TEST_SUITE_ARGS=() From 158f0eb96d2027a51bb8d5ad8ae347405790c8d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 20:17:44 +0200 Subject: [PATCH 0845/1795] add EB 4.9.2 --- .../2023.06/eessi-2023.06-eb-4.9.1-001-system.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml index 46ac979719..9e6f4ddb1a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -3,3 +3,7 @@ easyconfigs: options: from-pr: 20299 - EESSI-extend-2023.06-easybuild.eb +easyconfigs: + - EasyBuild-4.9.2.eb: + options: + from-pr: 20818 From 4c5cc45627041da2526ad9d9f8ca11b8be10580e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 20:19:45 +0200 Subject: [PATCH 0846/1795] remove 4.9.2 easystack --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml deleted file mode 100644 index a90f565412..0000000000 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml +++ /dev/null @@ -1,4 +0,0 @@ -easyconfigs: - - EasyBuild-4.9.2.eb: - options: - from-pr: 20801 From cb1672be6a54cc8ebcc76f38ebb8b18e72a368ec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 13 Jun 2024 20:59:15 +0200 Subject: [PATCH 0847/1795] add support for detecting A64FX to archdetect --- .github/workflows/tests_archdetect.yml | 1 + init/arch_specs/eessi_arch_arm.spec | 1 + tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.cpuinfo | 8 ++++++++ tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.output | 1 + 4 files changed, 11 insertions(+) create mode 100644 tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.cpuinfo create mode 100644 tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.output diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index bee348995d..f057251050 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -15,6 +15,7 @@ jobs: - x86_64/amd/zen3/Azure-CentOS7-7V73X - x86_64/amd/zen4/Azure-Alma8-9V33X - x86_64/amd/zen4/Shinx-RHEL8-9654 + - aarch64/a64fx/Deucalion-Rocky85 - aarch64/neoverse_n1/Azure-Ubuntu20-Altra - aarch64/neoverse_n1/AWS-awslinux-graviton2 - aarch64/neoverse_v1/AWS-awslinux-graviton3 diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index 8c1bc34d20..476d3d6119 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -1,5 +1,6 @@ # ARM CPU architecture specifications (see https://gpages.juszkiewicz.com.pl/arm-socs-table/arm-socs.html for guidance) # Software path in EESSI | Vendor ID | List of defining CPU features +"aarch64/a64fx" "" "asimdhp sve" # Fujitsu A64FX "aarch64/neoverse_n1" "ARM" "asimddp" # Ampere Altra "aarch64/neoverse_n1" "" "asimddp" # AWS Graviton2 "aarch64/neoverse_v1" "ARM" "asimddp svei8mm" diff --git a/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.cpuinfo b/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.cpuinfo new file mode 100644 index 0000000000..2484dbe3e7 --- /dev/null +++ b/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.cpuinfo @@ -0,0 +1,8 @@ +processor : 0 +BogoMIPS : 200.00 +Features : fp asimd evtstrm sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm fcma dcpop sve +CPU implementer : 0x46 +CPU architecture: 8 +CPU variant : 0x1 +CPU part : 0x001 +CPU revision : 0 diff --git a/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.output b/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.output new file mode 100644 index 0000000000..13b6c575c1 --- /dev/null +++ b/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.output @@ -0,0 +1 @@ +aarch64/a64fx From 8581d3f53f5dde36c46152faecb7de2d88521236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 13 Jun 2024 22:27:29 +0200 Subject: [PATCH 0848/1795] Remove line --- .../2023.06/eessi-2023.06-eb-4.9.1-001-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml index 9e6f4ddb1a..866766a36b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -3,7 +3,6 @@ easyconfigs: options: from-pr: 20299 - EESSI-extend-2023.06-easybuild.eb -easyconfigs: - EasyBuild-4.9.2.eb: options: from-pr: 20818 From 39cf7a0395e67e341808f96dbea812576bef5eb1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jun 2024 13:24:55 +0200 Subject: [PATCH 0849/1795] {2023.06,zen4}[foss/2023a] ESPResSo 4.2.2 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml new file mode 100644 index 0000000000..f148a7fdf9 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ESPResSo-4.2.2-foss-2023a.eb From eb363eddaccdde1d70a622d77123d480e6b226c9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jun 2024 15:49:49 +0200 Subject: [PATCH 0850/1795] {2023.06,zen4}[foss/2023a] TensorFlow 2.13.0 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index f148a7fdf9..f2f4c9d1f5 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,2 +1,3 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb + - TensorFlow-2.13.0-foss-2023a.eb From c810b4e3cf420b4c9f0285211da7f83a5a8e8f7b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jun 2024 21:47:31 +0200 Subject: [PATCH 0851/1795] {2023.06,zen4}[foss/2023a] R 4.3.2 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index f2f4c9d1f5..09f525364f 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,3 +1,4 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb - TensorFlow-2.13.0-foss-2023a.eb + - R-4.3.2-gfbf-2023a.eb From c92f941875f91696e305232fdca1e4d13bd2b091 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Jun 2024 23:22:14 +0200 Subject: [PATCH 0852/1795] {2023.06,zen4}[foss/2023b] GROMACS 2024.1 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml new file mode 100644 index 0000000000..af2c3bfca6 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GROMACS-2024.1-foss-2023b.eb From 1d4462095458803c6563178b5db8212a71138510 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 Jun 2024 09:25:55 +0200 Subject: [PATCH 0853/1795] {2023.06,zen4}[foss/2023b] R-bundle-Bioconductor 3.18 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 09f525364f..d39af75783 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -2,3 +2,4 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb - TensorFlow-2.13.0-foss-2023a.eb - R-4.3.2-gfbf-2023a.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb From ce370806de21ad95fd6825dcad24262bc8d28ec5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 15 Jun 2024 09:35:29 +0200 Subject: [PATCH 0854/1795] {2023.06,zen4} add missing installations with foss/2023b --- .../zen4/eessi-2023.06-eb-4.9.2-2023b.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml index af2c3bfca6..30a6745ccc 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,2 +1,18 @@ easyconfigs: - GROMACS-2024.1-foss-2023b.eb + - netCDF-4.9.2-gompi-2023b.eb + - matplotlib-3.8.2-gfbf-2023b.eb + - DP3-6.0-foss-2023b.eb + - WSClean-3.4-foss-2023b.eb + - CDO-2.2.2-gompi-2023b.eb + - python-casacore-3.5.2-foss-2023b.eb + - libspatialindex-1.9.3-GCCcore-13.2.0.eb + - LittleCMS-2.15-GCCcore-13.2.0.eb + - giflib-5.2.1-GCCcore-13.2.0.eb + - OpenJPEG-2.5.0-GCCcore-13.2.0.eb + - libwebp-1.3.2-GCCcore-13.2.0.eb + - Wayland-1.22.0-GCCcore-13.2.0.eb + - Qt5-5.15.13-GCCcore-13.2.0.eb + - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb + - NLTK-3.8.1-foss-2023b.eb + - Valgrind-3.23.0-gompi-2023b.eb From 1183ae9cc174af34792363048f18cd7086c4666c Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 17 Jun 2024 11:33:11 +0000 Subject: [PATCH 0855/1795] {2023.06}[GCCcore/13.2.0] IPython v8.17.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml new file mode 100644 index 0000000000..c23b5b001d --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - IPython-8.17.2-GCCcore-13.2.0.eb From e8c2d0636e087d4e565d5d5993d2894fa21da6b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Mon, 17 Jun 2024 15:10:28 +0200 Subject: [PATCH 0856/1795] first steps to support multiple CVMFS repositories --- eessi_container.sh | 99 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 22 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index fb14e2118f..9b1ab69a3b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -48,7 +48,11 @@ HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12)) +# we use an associative array for storing sets of settings per CVMFS repository +declare -A cvmfs_repo_settings + # CernVM-FS settings +# TODO may need to put them into repository specific map CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" @@ -89,8 +93,9 @@ display_help() { echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI via" - echo " default container, see --container]" + echo " repository to use; can be given multiple times" + echo " [default: software.eessi.io via CVMFS config available" + echo " via default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" echo " (check for output 'Using DIR as tmp ...' of a previous" @@ -123,7 +128,7 @@ STORAGE= LIST_REPOS=0 MODE="shell" SETUP_NVIDIA=0 -REPOSITORY="EESSI" +REPOSITORIES=() RESUME= SAVE= HTTP_PROXY=${http_proxy:-} @@ -179,7 +184,7 @@ while [[ $# -gt 0 ]]; do shift 2 ;; -r|--repository) - REPOSITORY="$2" + REPOSITORIES+=("$2") shift 2 ;; -s|--save) @@ -221,22 +226,50 @@ done set -- "${POSITIONAL_ARGS[@]}" +# define a list of CVMFS repositories that are accessible via the +# CVMFS config repository which is always mounted +declare -A eessi_cvmfs_repos=(["dev.eessi.io"]="extra", ["riscv.eessi.io"]="extra", ["software.eessi.io"]="default") +eessi_default_cvmfs_repo="software.eessi.io" + +# if REPOSITORIES is empty add default repository given above +if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then + REPOSITORIES+=(${eessi_default_cvmfs_repo}) +fi + +# define a list of CVMFS repositories that are accessible via the +# configuration file provided via $EESSI_REPOS_CFG_FILE +declare -A cfg_cvmfs_repos=() +if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + cfg_cvmfs_repos[${repo_id}]=${EESSI_REPOS_CFG_FILE} + done <<< "${sections}" +fi + if [[ ${LIST_REPOS} -eq 1 ]]; then - echo "Listing available repositories with format 'name [source]':" - echo " EESSI [default]" - if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then - cfg_load ${EESSI_REPOS_CFG_FILE} - sections=$(cfg_sections) - while IFS= read -r repo_id - do - echo " ${repo_id} [${EESSI_REPOS_CFG_FILE}]" - done <<< "${sections}" - fi + echo "Listing available repositories with format 'name [source[, 'default']]'." + echo "Note, without argument '--repository' the one labeled 'default' will be mounted." + for cvmfs_repo in "${!eessi_cvmfs_repos[@]}" + do + if [[ ${eessi_cvmfs_repos[${cvmfs_repo}]} == "default" ]] ; then + default_label=", default" + else + default_label="" + fi + echo " ${cvmfs_repo} [CVMFS config repo${default_label}]" + done + for cfg_repo in "${!cfg_cvmfs_repos[@]}" + do + echo " ${cfg_repo} [${cfg_cvmfs_repos[$cfg_repo]}]" + done exit 0 fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported +# TODO use the value as global setting, suffix to --repository can specify an access mode per repository if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi @@ -260,10 +293,16 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then fi fi -# TODO (arg -r|--repository) check if repository is known +# TODO (arg -r|--repository) check if all explicitly listed repositories are known # REPOSITORY_ERROR_EXITCODE -if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then - fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" +if [[ ${#REPOSITORIES[@]} -ne 0 ]] ; then + # iterate over entries in REPOSITORIES and check if they are known + for cvmfs_repo in "${REPOSITORIES[@]}" + do + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo}]}" && ! -n ${cfg_cvmfs_repos[${cvmfs_repo}]} ]]; then + fatal_error "The repository '${cvmfs_repo}' is not an EESSI CVMFS repository or it is not known how to mount it (could be due to a typo or missing configuration). Run '$0 -l' to obtain a list of available repositories." "${REPOSITORY_ERROR_EXITCODE}" + fi + done fi # TODO (arg -u|--resume) check if it exists, if user has read permission, @@ -337,22 +376,36 @@ if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then fi # 3. set up common vars and directories +# TODO change to be able to support multiple CVMFS repositories # directory structure should be: # ${EESSI_HOST_STORAGE} # |-singularity_cache -# |-${CVMFS_VAR_LIB} -# |-${CVMFS_VAR_RUN} -# |-overlay-upper -# |-overlay-work # |-home # |-repos_cfg -# |-opt-eessi (unless otherwise specificed for host_injections) +# |-CVMFS_REPO_1 +# | |-repo_settings (name, access_mode, host_injections) +# | |-${CVMFS_VAR_LIB} +# | |-${CVMFS_VAR_RUN} +# | |-overlay-upper +# | |-overlay-work +# | |-opt-eessi (unless otherwise specificed for host_injections) +# |-CVMFS_REPO_n +# |-repo_settings (name, access_mode, host_injections) +# |-${CVMFS_VAR_LIB} +# |-${CVMFS_VAR_RUN} +# |-overlay-upper +# |-overlay-work +# |-opt-eessi (unless otherwise specificed for host_injections) # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +# TODO make this specific to repository +# TODO move this code to when we already know which repositories we want to access +# actually we should know this already here, but we should rather move this to +# where repository args are being processed # Set host_injections directory and ensure it is a writable directory (if user provided) if [ -z ${USER_HOST_INJECTIONS+x} ]; then # Not set, so use our default @@ -486,6 +539,8 @@ if [[ ${FAKEROOT} -eq 1 ]]; then ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") fi +exit 0; # CONTINUE HERE +# TODO iterate over repositories in array REPOSITORIES # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg From 59ad3f572148b5e862ee415df37463cb9a91e646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 17 Jun 2024 22:40:47 +0200 Subject: [PATCH 0857/1795] use new PRs for OpenFOAM --- .../2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index f62267a0a3..4c3a0a5bd1 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -24,5 +24,5 @@ easyconfigs: - OrthoFinder-2.5.5-foss-2023a.eb - OpenFOAM-v2312-foss-2023a.eb: options: - include-easyblocks-from-pr: 3328 - from-pr: 20517 + include-easyblocks-from-pr: 3366 + from-pr: 20841 From 3af05e91a6289502978b53222abba74ce6a88b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 18 Jun 2024 10:26:19 +0200 Subject: [PATCH 0858/1795] move OpenFOAM to EB 4.9.2 easystack --- .../2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 4 ---- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 4c3a0a5bd1..27c18a487e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -22,7 +22,3 @@ easyconfigs: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb - - OpenFOAM-v2312-foss-2023a.eb: - options: - include-easyblocks-from-pr: 3366 - from-pr: 20841 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml new file mode 100644 index 0000000000..eb1169918f --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - OpenFOAM-v2312-foss-2023a.eb: + options: + include-easyblocks-from-pr: 3366 + from-pr: 20841 + From 4057496eabede14e228fe08d2fec670870411a1b Mon Sep 17 00:00:00 2001 From: julianmorillo Date: Tue, 18 Jun 2024 12:00:19 +0200 Subject: [PATCH 0859/1795] Update eessi-2023.06-eb-4.9.2-2023b.yml including DLB --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index c23b5b001d..38bfa81142 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,2 +1,3 @@ easyconfigs: - IPython-8.17.2-GCCcore-13.2.0.eb + - dlb-3.4-gompi-2023b.eb From c7a1ca2fbbb736b3eea2242c7143aaa3be063fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Tue, 18 Jun 2024 15:05:17 +0200 Subject: [PATCH 0860/1795] further changes to support multiple repositories --- eessi_container.sh | 248 +++++++++++++++++++++++++++------------------ 1 file changed, 148 insertions(+), 100 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9b1ab69a3b..d561d37792 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -228,16 +228,14 @@ set -- "${POSITIONAL_ARGS[@]}" # define a list of CVMFS repositories that are accessible via the # CVMFS config repository which is always mounted +# TODO instead of hard-coding the 'extra' and 'default' repositories here one +# could have another script in the GitHub and/or CVMFS repository which +# provides this "configuration" declare -A eessi_cvmfs_repos=(["dev.eessi.io"]="extra", ["riscv.eessi.io"]="extra", ["software.eessi.io"]="default") -eessi_default_cvmfs_repo="software.eessi.io" - -# if REPOSITORIES is empty add default repository given above -if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then - REPOSITORIES+=(${eessi_default_cvmfs_repo}) -fi +eessi_default_cvmfs_repo="software.eessi.io,access=${ACCESS}" # define a list of CVMFS repositories that are accessible via the -# configuration file provided via $EESSI_REPOS_CFG_FILE +# configuration file provided via $EESSI_REPOS_CFG_FILE declare -A cfg_cvmfs_repos=() if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then cfg_load ${EESSI_REPOS_CFG_FILE} @@ -267,6 +265,11 @@ if [[ ${LIST_REPOS} -eq 1 ]]; then exit 0 fi +# if REPOSITORIES is empty add default repository given above +if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then + REPOSITORIES+=(${eessi_default_cvmfs_repo}) +fi + # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported # TODO use the value as global setting, suffix to --repository can specify an access mode per repository @@ -295,15 +298,45 @@ fi # TODO (arg -r|--repository) check if all explicitly listed repositories are known # REPOSITORY_ERROR_EXITCODE -if [[ ${#REPOSITORIES[@]} -ne 0 ]] ; then - # iterate over entries in REPOSITORIES and check if they are known - for cvmfs_repo in "${REPOSITORIES[@]}" - do - if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo}]}" && ! -n ${cfg_cvmfs_repos[${cvmfs_repo}]} ]]; then - fatal_error "The repository '${cvmfs_repo}' is not an EESSI CVMFS repository or it is not known how to mount it (could be due to a typo or missing configuration). Run '$0 -l' to obtain a list of available repositories." "${REPOSITORY_ERROR_EXITCODE}" +# iterate over entries in REPOSITORIES and check if they are known +for cvmfs_repo in "${REPOSITORIES[@]}" +do + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + else + cvmfs_repo_name="${cvmfs_repo}" + fi + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" && ! -n ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then + fatal_error "The repository '${cvmfs_repo_name}' is not an EESSI CVMFS repository or it is not known how to mount it (could be due to a typo or missing configuration). Run '$0 -l' to obtain a list of available repositories." "${REPOSITORY_ERROR_EXITCODE}" + fi +done + +# make sure each repository is only listed once +declare -A listed_repos=() +for cvmfs_repo in "${REPOSITORIES[@]}" +do + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode + echo "checking for duplicates: '${cvmfs_repo}' and '${cvmfs_repo_name}'" + # if cvmfs_repo_name is not in eessi_cvmfs_repos, assume it's in cfg_cvmfs_repos + # and obtain actual repo_name from config + cfg_repo_id='' + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then + [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." + # cvmfs_repo_name is actually a repository ID, use that to obtain + # the actual name from the EESSI_REPOS_CFG_FILE + cfg_repo_id=${cvmfs_repo_name} + cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + fi + if [[ -n "${listed_repos[${cvmfs_repo_name}]}" ]] ; then + via_cfg="" + if [[ -n "${cfg_repo_id}" ]] ; then + via_cfg=" (via repository ID '${cfg_repo_id}')" fi - done -fi + fatal_error "CVMFS repository '${cvmfs_repo_name}'${via_cfg} listed multiple times" + fi + listed_repos+=([${cvmfs_repo_name}]=true) +done # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run @@ -382,17 +415,15 @@ fi # |-singularity_cache # |-home # |-repos_cfg +# |-${CVMFS_VAR_LIB} +# |-${CVMFS_VAR_RUN} # |-CVMFS_REPO_1 # | |-repo_settings (name, access_mode, host_injections) -# | |-${CVMFS_VAR_LIB} -# | |-${CVMFS_VAR_RUN} # | |-overlay-upper # | |-overlay-work # | |-opt-eessi (unless otherwise specificed for host_injections) # |-CVMFS_REPO_n # |-repo_settings (name, access_mode, host_injections) -# |-${CVMFS_VAR_LIB} -# |-${CVMFS_VAR_RUN} # |-overlay-upper # |-overlay-work # |-opt-eessi (unless otherwise specificed for host_injections) @@ -402,7 +433,7 @@ EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" -# TODO make this specific to repository +# TODO make this specific to repository? # TODO move this code to when we already know which repositories we want to access # actually we should know this already here, but we should rather move this to # where repository args are being processed @@ -481,6 +512,7 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}" # set env vars and create directories for CernVM-FS +# TODO need to use separate values for separate repos? EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} @@ -539,92 +571,106 @@ if [[ ${FAKEROOT} -eq 1 ]]; then ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") fi -exit 0; # CONTINUE HERE # TODO iterate over repositories in array REPOSITORIES # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg -if [[ "${REPOSITORY}" == "EESSI" ]]; then - # need to source defaults as late as possible (see other sourcing below) - source ${TOPDIR}/init/eessi_defaults - - # strip "/cvmfs/" from default setting - repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} -else - # TODO implement more flexible specification of repo cfgs - # REPOSITORY => repo-id OR repo-cfg-file (with a single section) OR - # repo-cfg-file:repo-id (repo-id defined in repo-cfg-file) - # - # for now, assuming repo-id is defined in config file pointed to - # EESSI_REPOS_CFG_FILE, which is to be copied into the working directory - # (could also become part of the software layer to define multiple - # standard EESSI repositories) - cfg_load ${EESSI_REPOS_CFG_FILE} - - # copy repos.cfg to job directory --> makes it easier to inspect the job - cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. - - # cfg file should include: repo_name, repo_version, config_bundle, - # map { local_filepath -> container_filepath } - # - # repo_name_domain is the domain part of the repo_name, e.g., - # eessi.io for software.eessi.io - # - # where config bundle includes the files (-> target location in container) - # - default.local -> /etc/cvmfs/default.local - # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... - # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf - # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), - # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... - # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} - # a directory that contains the public key to access the repository, key - # itself then doesn't need to be BIND mounted - # - ${repo_name_domain}/${repo_name}.pub - # (-> /etc/cvmfs/keys/${repo_name_domain}/${repo_name}.pub - # the public key to access the repository, key itself is BIND mounted - # via directory ${repo_name_domain} - repo_name=$(cfg_get_value ${REPOSITORY} "repo_name") - # derive domain part from repo_name (everything after first '.') - repo_name_domain=${repo_name#*.} - repo_version=$(cfg_get_value ${REPOSITORY} "repo_version") - config_bundle=$(cfg_get_value ${REPOSITORY} "config_bundle") - config_map=$(cfg_get_value ${REPOSITORY} "config_map") - - # convert config_map into associative array cfg_file_map - cfg_init_file_map "${config_map}" - [[ ${VERBOSE} -eq 1 ]] && cfg_print_map - - # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg - # if config_bundle is relative path (no '/' at start) prepend it with - # EESSI_REPOS_CFG_DIR - config_bundle_path= - if [[ ! "${config_bundle}" =~ ^/ ]]; then - config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} - else - config_bundle_path=${config_bundle} - fi +[[ ${VERBOSE} -eq 1 ]] && echo +[[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS before processing REPOSITORIES\n BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo +for cvmfs_repo in "${REPOSITORIES[@]}" +do + echo "process CVMFS repo spec '${cvmfs_repo}'" + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + cvmfs_repo_access=${cvmfs_repo/*,access=/} # remove repo name part + else + cvmfs_repo_name="${cvmfs_repo}" + cvmfs_repo_access="${ACCESS}" # use globally defined access mode + fi + # if cvmfs_repo_name is in cfg_cvmfs_repos, it is a "repository ID" and was + # derived from information in EESSI_REPOS_CFG_FILE, namely the section + # names in that .ini-type file + # in the if-block below, we'll use cfg_repo_id to refer to that ID + # we need to process/provide the config from EESSI_REPOS_CFG_FILE, such + # that the necessary information for accessing a CVMFS repository is made + # available inside the container + if [[ -n "${cfg_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then + cfg_repo_id=${cvmfs_repo_name} + + # obtain CVMFS repository name from section for the given ID + cfg_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + # derive domain part from (cfg_)repo_name (everything after first '.') + repo_name_domain=${repo_name#*.} + + # cfg_cvmfs_repos is populated through reading the file pointed to by + # EESSI_REPOS_CFG_FILE. We need to copy that file and data it needs + # into the job's working directory. + + # copy repos.cfg to job directory --> makes it easier to inspect the job + cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + + # cfg file should include sections (one per CVMFS repository to be mounted) + # with each section containing the settings: + # - repo_name, + # - repo_version, + # - config_bundle, and + # - a map { filepath_in_bundle -> container_filepath } + # + # The config_bundle includes the files which are mapped ('->') to a target + # location in container: + # - default.local -> /etc/cvmfs/default.local + # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... + # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf + # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), + # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... + # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} + # a directory that contains the public key to access the repository, key + # itself then doesn't need to be BIND mounted + # - ${repo_name_domain}/${cfg_repo_name}.pub + # (-> /etc/cvmfs/keys/${repo_name_domain}/${cfg_repo_name}.pub + # the public key to access the repository, key itself is BIND mounted + # via directory ${repo_name_domain} + cfg_repo_version=$(cfg_get_value ${cfg_repo_id} "repo_version") + cfg_config_bundle=$(cfg_get_value ${cfg_repo_id} "config_bundle") + cfg_config_map=$(cfg_get_value ${cfg_repo_id} "config_map") + + # convert cfg_config_map into associative array cfg_file_map + cfg_init_file_map "${cfg_config_map}" + [[ ${VERBOSE} -eq 1 ]] && cfg_print_map + + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg and define + # BIND mounts + # check if config_bundle exists, if so, unpack it into + # ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${cfg_config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${cfg_config_bundle} + else + config_bundle_path=${cfg_config_bundle} + fi - if [[ ! -r ${config_bundle_path} ]]; then - fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} - fi + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + fi - # only unpack config_bundle if we're not resuming from a previous run - if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg - fi + # only unpack cfg_config_bundle if we're not resuming from a previous run + if [[ -z ${RESUME} ]]; then + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg + fi - for src in "${!cfg_file_map[@]}" - do - target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" - done - export EESSI_VERSION_OVERRIDE=${repo_version} - export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" - # need to source defaults as late as possible (after *_OVERRIDEs) - source ${TOPDIR}/init/eessi_defaults -fi + for src in "${!cfg_file_map[@]}" + do + target=${cfg_file_map[${src}]} + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" + done + fi + [[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS after processing '${cvmfs_repo}'\n BIND_PATHS=${BIND_PATHS}" + [[ ${VERBOSE} -eq 1 ]] && echo +done # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to @@ -650,14 +696,16 @@ if [[ ! -z ${http_proxy} ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi +exit 0; # CONTINUE HERE # 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() -# always mount cvmfs-config repo (to get access to software.eessi.io) +# always mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") +# TODO iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) if [[ "${ACCESS}" == "ro" ]]; then export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" From 76c963f63ad8100fd87424e15be926bcdcb91d42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 19 Jun 2024 12:02:00 +0200 Subject: [PATCH 0861/1795] use from-commit instead of from-pr --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index eb1169918f..74e1f4610b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,6 +1,7 @@ easyconfigs: - OpenFOAM-v2312-foss-2023a.eb: options: - include-easyblocks-from-pr: 3366 - from-pr: 20841 - + # https://github.com/easybuilders/easybuild-easyblocks/pull/3366 + include-easyblocks-from-commit: d1f2231b2de4f8548c75c7367456873a8c33b5f9 + # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841 + from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 From 934059a731f55d5a11cc10b6e3719abf65b1d478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 19 Jun 2024 15:21:27 +0200 Subject: [PATCH 0862/1795] configure multiple fusemounts --- eessi_container.sh | 123 +++++++++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index d561d37792..c549d04d8c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -52,7 +52,6 @@ NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12)) declare -A cvmfs_repo_settings # CernVM-FS settings -# TODO may need to put them into repository specific map CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" @@ -272,7 +271,7 @@ fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported -# TODO use the value as global setting, suffix to --repository can specify an access mode per repository +# use the value as global setting, suffix to --repository can specify an access mode per repository if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi @@ -317,7 +316,7 @@ declare -A listed_repos=() for cvmfs_repo in "${REPOSITORIES[@]}" do cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode - echo "checking for duplicates: '${cvmfs_repo}' and '${cvmfs_repo_name}'" + [[ ${VERBOSE} -eq 1 ]] && echo "checking for duplicates: '${cvmfs_repo}' and '${cvmfs_repo_name}'" # if cvmfs_repo_name is not in eessi_cvmfs_repos, assume it's in cfg_cvmfs_repos # and obtain actual repo_name from config cfg_repo_id='' @@ -409,7 +408,6 @@ if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then fi # 3. set up common vars and directories -# TODO change to be able to support multiple CVMFS repositories # directory structure should be: # ${EESSI_HOST_STORAGE} # |-singularity_cache @@ -528,14 +526,18 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) -BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" +if [[ -z ${SINGULARITY_BIND} ]] ; then + SINGULARITY_BIND="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" +else + SINGULARITY_BIND="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi,${SINGULARITY_BIND}" +fi # provide a '/tmp' inside the container -BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" +SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then - BIND_PATHS="${BIND_PATHS},${EXTRA_BIND_PATHS}" + SINGULARITY_BIND="${SINGULARITY_BIND},${EXTRA_BIND_PATHS}" fi -[[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" declare -a ADDITIONAL_CONTAINER_OPTIONS=() @@ -556,8 +558,8 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then EESSI_USR_LOCAL_CUDA=${EESSI_TMPDIR}/usr-local-cuda mkdir -p ${EESSI_VAR_LOG} mkdir -p ${EESSI_USR_LOCAL_CUDA} - BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" - [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" + SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" + [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" if [[ "${NVIDIA_MODE}" == "install" ]] ; then # No GPU so we need to "trick" Lmod to allow us to load CUDA modules even without a CUDA driver # (this variable means EESSI_OVERRIDE_GPU_CHECK=1 will be set inside the container) @@ -571,16 +573,16 @@ if [[ ${FAKEROOT} -eq 1 ]]; then ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") fi -# TODO iterate over repositories in array REPOSITORIES # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg [[ ${VERBOSE} -eq 1 ]] && echo -[[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS before processing REPOSITORIES\n BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo -e "SINGULARITY_BIND before processing REPOSITORIES\n SINGULARITY_BIND=${SINGULARITY_BIND}" [[ ${VERBOSE} -eq 1 ]] && echo +# iterate over repositories in array REPOSITORIES for cvmfs_repo in "${REPOSITORIES[@]}" do - echo "process CVMFS repo spec '${cvmfs_repo}'" + [[ ${VERBOSE} -eq 1 ]] && echo "process CVMFS repo spec '${cvmfs_repo}'" # split into name and access mode if ',access=' in $cvmfs_repo if [[ ${cvmfs_repo} == *",access="* ]] ; then cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification @@ -665,17 +667,21 @@ do for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" + # if target is alreay BIND mounted, exit with an error + if [[ ${SINGULARITY_BIND} =~ "${target}" ]]; then + fatal_error "target '${target}' is already listed in paths to bind mount into the container ('${SINGULARITY_BIND}')" ${REPOSITORY_ERROR_EXITCODE} + fi + export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done fi - [[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS after processing '${cvmfs_repo}'\n BIND_PATHS=${BIND_PATHS}" + [[ ${VERBOSE} -eq 1 ]] && echo -e "SINGULARITY_BIND after processing '${cvmfs_repo}'\n SINGULARITY_BIND=${SINGULARITY_BIND}" [[ ${VERBOSE} -eq 1 ]] && echo done # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to -# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND -# MOUNT if it was not yet in BIND_PATHS) +# ${EESSI_TMPDIR}/repos_cfg/default.local on the host (and possibly add a BIND +# MOUNT if it was not yet in SINGULARITY_BIND) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is # http://SOME_HOSTNAME:SOME_PORT/ @@ -691,12 +697,14 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local" [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local - # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then - export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + # if default.local is not BIND mounted into container, add it to SINGULARITY_BIND + src=${EESSI_TMPDIR}/repos_cfg/default.local + target=/etc/cvmfs/default.local + if [[ ${SINGULARITY_BIND} =~ "${target}" ]]; then + fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${SINGULARITY_BIND}')" ${REPOSITORY_ERROR_EXITCODE} fi + export SINGULARITY_BIND="${SINGULARITY_BIND},${src}:${target}" fi -exit 0; # CONTINUE HERE # 4. set up vars and dirs specific to a scenario @@ -705,42 +713,59 @@ declare -a EESSI_FUSE_MOUNTS=() # always mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") -# TODO iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) -if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" - - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - export EESSI_FUSE_MOUNTS -fi +# iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) +for cvmfs_repo in "${REPOSITORIES[@]}" +do + [[ ${VERBOSE} -eq 1 ]] && echo "add fusemount options for CVMFS repo '${cvmfs_repo}'" + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + cvmfs_repo_access=${cvmfs_repo/*,access=/} # remove repo name part + else + cvmfs_repo_name="${cvmfs_repo}" + cvmfs_repo_access="${ACCESS}" # use globally defined access mode + fi + # obtain cvmfs_repo_name from EESSI_REPOS_CFG_FILE if cvmfs_repo is in cfg_cvmfs_repos + if [[ ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then + [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." + # cvmfs_repo_name is actually a repository ID, use that to obtain + # the actual name from the EESSI_REPOS_CFG_FILE + cfg_repo_id=${cvmfs_repo_name} + cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + fi -if [[ "${ACCESS}" == "rw" ]]; then - mkdir -p ${EESSI_TMPDIR}/overlay-upper - mkdir -p ${EESSI_TMPDIR}/overlay-work + # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) + if [[ ${cvmfs_repo_access} == "ro" ]] ; then + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" - # set environment variables for fuse mounts in Singularity container - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + export EESSI_FUSE_MOUNTS + elif [[ ${cvmfs_repo_access} == "rw" ]] ; then + # use repo-specific overlay directories + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + # set environment variables for fuse mounts in Singularity container + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" - EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" - EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" - EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" - EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") - export EESSI_FUSE_MOUNTS -fi + EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + export EESSI_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") + export EESSI_FUSE_MOUNTS + else + echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n '${cvmfs_repo_name}' is not known" + exit ${REPOSITORY_ERROR_EXITCODE} + fi +done # 5. run container -# final settings -if [[ -z ${SINGULARITY_BIND} ]]; then - export SINGULARITY_BIND="${BIND_PATHS}" -else - export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" -fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" # pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) From 32f413328cd3b8f1da7ea7521f7458c20cd7232b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 19 Jun 2024 16:02:40 +0200 Subject: [PATCH 0863/1795] revert back changes related to BIND_PATHS and SINGULARITY_BIND + some polishing --- eessi_container.sh | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index c549d04d8c..163792f622 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -416,12 +416,10 @@ fi # |-${CVMFS_VAR_LIB} # |-${CVMFS_VAR_RUN} # |-CVMFS_REPO_1 -# | |-repo_settings (name, access_mode, host_injections) # | |-overlay-upper # | |-overlay-work # | |-opt-eessi (unless otherwise specificed for host_injections) # |-CVMFS_REPO_n -# |-repo_settings (name, access_mode, host_injections) # |-overlay-upper # |-overlay-work # |-opt-eessi (unless otherwise specificed for host_injections) @@ -526,18 +524,15 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) -if [[ -z ${SINGULARITY_BIND} ]] ; then - SINGULARITY_BIND="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" -else - SINGULARITY_BIND="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi,${SINGULARITY_BIND}" -fi +BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" + # provide a '/tmp' inside the container -SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then - SINGULARITY_BIND="${SINGULARITY_BIND},${EXTRA_BIND_PATHS}" + BIND_PATHS="${BIND_PATHS},${EXTRA_BIND_PATHS}" fi -[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" +[[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" declare -a ADDITIONAL_CONTAINER_OPTIONS=() @@ -558,8 +553,8 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then EESSI_USR_LOCAL_CUDA=${EESSI_TMPDIR}/usr-local-cuda mkdir -p ${EESSI_VAR_LOG} mkdir -p ${EESSI_USR_LOCAL_CUDA} - SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" - [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" + BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" + [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" if [[ "${NVIDIA_MODE}" == "install" ]] ; then # No GPU so we need to "trick" Lmod to allow us to load CUDA modules even without a CUDA driver # (this variable means EESSI_OVERRIDE_GPU_CHECK=1 will be set inside the container) @@ -577,7 +572,7 @@ fi # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg [[ ${VERBOSE} -eq 1 ]] && echo -[[ ${VERBOSE} -eq 1 ]] && echo -e "SINGULARITY_BIND before processing REPOSITORIES\n SINGULARITY_BIND=${SINGULARITY_BIND}" +[[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS before processing REPOSITORIES\n BIND_PATHS=${BIND_PATHS}" [[ ${VERBOSE} -eq 1 ]] && echo # iterate over repositories in array REPOSITORIES for cvmfs_repo in "${REPOSITORIES[@]}" @@ -668,20 +663,20 @@ do do target=${cfg_file_map[${src}]} # if target is alreay BIND mounted, exit with an error - if [[ ${SINGULARITY_BIND} =~ "${target}" ]]; then - fatal_error "target '${target}' is already listed in paths to bind mount into the container ('${SINGULARITY_BIND}')" ${REPOSITORY_ERROR_EXITCODE} + if [[ ${BIND_PATHS} =~ "${target}" ]]; then + fatal_error "target '${target}' is already listed in paths to bind mount into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE} fi - export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done fi - [[ ${VERBOSE} -eq 1 ]] && echo -e "SINGULARITY_BIND after processing '${cvmfs_repo}'\n SINGULARITY_BIND=${SINGULARITY_BIND}" + [[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS after processing '${cvmfs_repo}'\n BIND_PATHS=${BIND_PATHS}" [[ ${VERBOSE} -eq 1 ]] && echo done # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to # ${EESSI_TMPDIR}/repos_cfg/default.local on the host (and possibly add a BIND -# MOUNT if it was not yet in SINGULARITY_BIND) +# MOUNT if it was not yet in BIND_PATHS) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is # http://SOME_HOSTNAME:SOME_PORT/ @@ -697,13 +692,13 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local" [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local - # if default.local is not BIND mounted into container, add it to SINGULARITY_BIND + # if default.local is not BIND mounted into container, add it to BIND_PATHS src=${EESSI_TMPDIR}/repos_cfg/default.local target=/etc/cvmfs/default.local - if [[ ${SINGULARITY_BIND} =~ "${target}" ]]; then - fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${SINGULARITY_BIND}')" ${REPOSITORY_ERROR_EXITCODE} + if [[ ${BIND_PATHS} =~ "${target}" ]]; then + fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE} fi - export SINGULARITY_BIND="${SINGULARITY_BIND},${src}:${target}" + BIND_PATHS="${BIND_PATHS},${src}:${target}" fi # 4. set up vars and dirs specific to a scenario @@ -744,6 +739,7 @@ do # use repo-specific overlay directories mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" @@ -766,6 +762,12 @@ do done # 5. run container +# final settings +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BIND_PATHS}" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" +fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" # pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) From 07de7008e60586d146cd4986de9d0a4f7e505731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Thu, 20 Jun 2024 09:07:24 +0200 Subject: [PATCH 0864/1795] store repository settings under tmp --- eessi_container.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index 163792f622..eaa4f59af9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -416,10 +416,12 @@ fi # |-${CVMFS_VAR_LIB} # |-${CVMFS_VAR_RUN} # |-CVMFS_REPO_1 +# | |-repo_settings.sh (name, id, access, host_injections) # | |-overlay-upper # | |-overlay-work # | |-opt-eessi (unless otherwise specificed for host_injections) # |-CVMFS_REPO_n +# |-repo_settings.sh (name, id, access, host_injections) # |-overlay-upper # |-overlay-work # |-opt-eessi (unless otherwise specificed for host_injections) @@ -711,6 +713,7 @@ EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/ # iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) for cvmfs_repo in "${REPOSITORIES[@]}" do + unset cfg_repo_id [[ ${VERBOSE} -eq 1 ]] && echo "add fusemount options for CVMFS repo '${cvmfs_repo}'" # split into name and access mode if ',access=' in $cvmfs_repo if [[ ${cvmfs_repo} == *",access="* ]] ; then @@ -729,6 +732,9 @@ do cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi + # always create a directory for the repository (e.g., to store settings, ...) + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} + # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) if [[ ${cvmfs_repo_access} == "ro" ]] ; then export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" @@ -759,6 +765,20 @@ do echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n '${cvmfs_repo_name}' is not known" exit ${REPOSITORY_ERROR_EXITCODE} fi + # create repo_settings.sh file in ${EESSI_TMPDIR}/${cvmfs_repo_name} to store + # (intention is that the file could be just sourced to obtain the settings) + # repo_name = ${cvmfs_repo_name} + # repo_id = ${cfg_repo_id} # empty if not an EESSI repo + # repo_access = ${cvmfs_repo_access} + # repo_host_injections = [ {"src_path":"target_path"}... ] # TODO + settings= + #[[ -n ${cfg_repo_id} ]] && settings="[${cvmfs_repo_name}]\n" || settings="[${cfg_repo_id}]\n" + settings="${settings}repo_name = ${cvmfs_repo_name}\n" + settings="${settings}repo_id = ${cfg_repo_id}\n" + settings="${settings}repo_access = ${cvmfs_repo_access}\n" + # TODO iterate over host_injections (first need means to define them (globally and/or per repository) + # settings="${settings}repo_host_injections = ${host_injections}\n" + echo -e "${settings}" > ${EESSI_TMPDIR}/${cvmfs_repo_name}/repo_settings.sh done # 5. run container From 3da63162f66480d4c4430e6725800fc0769e90f1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 24 Jun 2024 18:18:49 +0200 Subject: [PATCH 0865/1795] {2023.06,a64fx}[2023a] OpenMPI 4.1.5 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml new file mode 100644 index 0000000000..1475ad0866 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OpenMPI-4.1.5-GCC-12.3.0.eb From 871586fbb055a07ad7250a48a3a06c3ffec1d8b2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 24 Jun 2024 18:49:22 +0200 Subject: [PATCH 0866/1795] use latest Ubuntu in CI workflows --- .github/workflows/test-software.eessi.io.yml | 2 +- .github/workflows/test_eessi_container_script.yml | 2 +- .github/workflows/test_licenses.yml | 2 +- .github/workflows/tests.yml | 2 +- .github/workflows/tests_archdetect.yml | 2 +- .github/workflows/tests_init.yml | 2 +- .github/workflows/tests_readme.yml | 2 +- .github/workflows/tests_scripts.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 8cfb023bc6..06c02c8834 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -5,7 +5,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: check_missing: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 32120d0087..d31feb23fe 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -5,7 +5,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: eessi_container_script: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/test_licenses.yml b/.github/workflows/test_licenses.yml index 3b9675d523..6b6387d47d 100644 --- a/.github/workflows/test_licenses.yml +++ b/.github/workflows/test_licenses.yml @@ -5,7 +5,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index faa7eb82ff..6af6a83e96 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: python: [3.6, 3.7, 3.8, 3.9, '3.10'] diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index bee348995d..4f5d3d174d 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -5,7 +5,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: proc_cpuinfo: diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 053acb9730..aabe37de44 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -5,7 +5,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: python: [3.6, 3.7, 3.8, 3.9, '3.10'] diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index efdb796e5e..76f46a8abe 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -14,7 +14,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 76d19d29fe..6413fcf86f 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -28,7 +28,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 From 70878f66586c49b0a2412754a7937482f7910803 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 24 Jun 2024 18:58:42 +0200 Subject: [PATCH 0867/1795] only run CI workflows with Python >= 3.7 Co-authored-by: ocaisa --- .github/workflows/tests.yml | 2 +- .github/workflows/tests_init.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6af6a83e96..8249e50d9b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9, '3.10'] + python: [3.7, 3.8, 3.9, '3.10'] fail-fast: false steps: - name: checkout diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index aabe37de44..6c356a9184 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9, '3.10'] + python: [3.7, 3.8, 3.9, '3.10'] fail-fast: false steps: - name: checkout From 0cb1d2af5ad9ad8f2ccdb7e9da9d2614c61781d0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 Jun 2024 09:29:21 +0200 Subject: [PATCH 0868/1795] implement hook to inject -DCACHE_SECTOR_SIZE_READONLY in $CFLAGS when building BLIS 0.9.0 on A64FX --- eb_hooks.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8b0a11b0ed..fd3e0830ee 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -23,6 +23,7 @@ CPU_TARGET_NEOVERSE_N1 = 'aarch64/neoverse_n1' CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' +CPU_TARGET_A64FX = 'aarch64/a64fx' EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -335,6 +336,19 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) +def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): + """ + Pre-configure hook for BLIS when building for A64FX: + - add -DCACHE_SECTOR_SIZE_READONLY to $CFLAGS for BLIS 0.9.0, cfr. https://github.com/flame/blis/issues/800 + """ + if self.name == 'BLIS': + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.version == '0.9.0' and cpu_target == CPU_TARGET_A64FX: + self.cfg.update('configopts', 'CFLAGS="$CFLAGS -DCACHE_SECTOR_SIZE_READONLY"') + else: + raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") + + def pre_configure_hook_gromacs(self, *args, **kwargs): """ Pre-configure hook for GROMACS: @@ -680,12 +694,13 @@ def inject_gpu_property(ec): } PRE_CONFIGURE_HOOKS = { + 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, + 'BLIS': pre_configure_hook_BLIS_a64fx, 'GROMACS': pre_configure_hook_gromacs, 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, - 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, } PRE_TEST_HOOKS = { From 3aac7ea854e8f4613ad89fc8e15e2ca26df0af6f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 Jun 2024 09:29:38 +0200 Subject: [PATCH 0869/1795] {2023.06,a64fx} foss/2023a --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index 1475ad0866..3a060e30a7 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,2 +1,3 @@ easyconfigs: - OpenMPI-4.1.5-GCC-12.3.0.eb + - foss-2023a.eb From d9c1fb1104f19e741b8ac2a5a652eb22e07bf241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Tue, 25 Jun 2024 10:17:51 +0200 Subject: [PATCH 0870/1795] removed unnecessary variable --- eessi_container.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index eaa4f59af9..4b7b4eb719 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -48,9 +48,6 @@ HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12)) -# we use an associative array for storing sets of settings per CVMFS repository -declare -A cvmfs_repo_settings - # CernVM-FS settings CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" From e25cd1e720a3a0e787409093d561d5892929d10a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Tue, 25 Jun 2024 10:23:18 +0200 Subject: [PATCH 0871/1795] add more information about repository-specific access modes --- eessi_container.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 4b7b4eb719..9d2ce6c25f 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -70,7 +70,8 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" display_help() { echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -a | --access {ro,rw} - sets access globally for all CVMFS repositories:" + echo " ro (read-only), rw (read & write) [default: ro]" echo " -b | --extra-bind-paths - specify extra paths to be bound into the container." echo " To specify multiple bind paths, separate by comma." echo " Example: '/src:/dest:ro,/src2:/dest2:rw'" @@ -89,7 +90,9 @@ display_help() { echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use; can be given multiple times" + echo " repository to use; can be given multiple times;" + echo " CFG may include a suffix ';access={ro,rw}' to" + echo " overwrite the global access mode for this repository" echo " [default: software.eessi.io via CVMFS config available" echo " via default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," From da1b6710152dcf912db34147e88c739914e0290d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 25 Jun 2024 18:16:24 +0200 Subject: [PATCH 0872/1795] fix hook to tweak BLIS configure command when building for A64FX --- eb_hooks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index fd3e0830ee..b14c799eae 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -344,7 +344,12 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): if self.name == 'BLIS': cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.version == '0.9.0' and cpu_target == CPU_TARGET_A64FX: - self.cfg.update('configopts', 'CFLAGS="$CFLAGS -DCACHE_SECTOR_SIZE_READONLY"') + # last argument of BLIS' configure command is configuration target (usually 'auto' for auto-detect), + # specifying of variables should be done before that + config_opts = self.cfg['configopts'].split(' ') + cflags_var = 'CFLAGS="$CFLAGS -DCACHE_SECTOR_SIZE_READONLY"' + config_target = config_opts[-1] + self.cfg['configopts'] = ' '.join(config_opts[:-1] + [cflags_var, config_target]) else: raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") From 80d8d6e5789a972342fd57532e84d8df3a39bb88 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 26 Jun 2024 11:46:11 +0200 Subject: [PATCH 0873/1795] ignore scipy test failures when installing SciPy-bundle for A64FX --- eb_hooks.py | 22 ++++++++++++++++------ eessi-2023.06-known-issues.yml | 7 +++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index b14c799eae..e6f0cf0dd2 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -488,20 +488,30 @@ def pre_test_hook_ignore_failing_tests_FFTWMPI(self, *args, **kwargs): def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): """ Pre-test hook for SciPy-bundle: skip failing tests for selected SciPy-bundle versions - In version 2021.10, 2 failing tests in scipy 1.6.3: + In version 2021.10 on neoverse_v1, 2 failing tests in scipy 1.6.3: FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A... FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 = 2 failed, 30554 passed, 2064 skipped, 10992 deselected, 76 xfailed, 7 xpassed, 40 warnings in 380.27s (0:06:20) = - In versions 2023.02, 2023.07, and 2023.11, 2 failing tests in scipy (versions 1.10.1, 1.11.1, 1.11.4): + In versions 2023.02 + 2023.07 + 2023.11 on neoverse_v1, 2 failing tests in scipy (versions 1.10.1, 1.11.1, 1.11.4): FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32 = 2 failed, 54409 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 892.04s (0:14:52) = - In previous versions we were not as strict yet on the numpy/SciPy tests + In version 2023.07 on a64fx, 4 failing tests in scipy 1.11.1: + FAILED scipy/optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 + FAILED scipy/optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 + FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris + FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32 + = 4 failed, 54407 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 6068.43s (1:41:08) = + (in previous versions we were not as strict yet on the numpy/SciPy tests) """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - scipy_bundle_versions = ('2021.10', '2023.02', '2023.07', '2023.11') - if self.name == 'SciPy-bundle' and self.version in scipy_bundle_versions and cpu_target == CPU_TARGET_NEOVERSE_V1: - self.cfg['testopts'] = "|| echo ignoring failing tests" + scipy_bundle_versions_nv1 = ('2021.10', '2023.02', '2023.07', '2023.11') + scipy_bundle_versions_a64fx = ('2023.07', '2023.11') + if self.name == 'SciPy-bundle': + if cpu_target == CPU_TARGET_NEOVERSE_V1 and self.version in scipy_bundle_versions_nv1: + self.cfg['testopts'] = "|| echo ignoring failing tests" + elif cpu_target == CPU_TARGET_A64FX and self.version in scipy_bundle_versions_a64fx: + self.cfg['testopts'] = "|| echo ignoring failing tests" def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): """ diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 2d1256354f..c5cdc68941 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -1,3 +1,10 @@ +- aarch64/a64x: + - SciPy-bundle-2023.07-gfbf-2023a: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "4 failing tests (vs 54407 passed) in scipy test suite" + - SciPy-bundle-2023.11-gfbf-2023b: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "3 failing tests (vs 54875 passed) in scipy test suite" - aarch64/generic: - PyTorch-2.1.2-foss-2023a: - issue: https://github.com/EESSI/software-layer/issues/461 From f07fd8567f1154cb8b4f0ab338979caff4dbfcce Mon Sep 17 00:00:00 2001 From: torri Date: Wed, 26 Jun 2024 14:03:51 +0200 Subject: [PATCH 0874/1795] fix for broken curl installation --- init/eessi_environment_variables | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 78851a9c95..d6e5b75dce 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -15,6 +15,20 @@ function show_msg { fi } +function check_rhel_ca { + + os=$(grep -oP '(?<=^ID_LIKE=).+' /etc/os-release | tr -d '"') + + version_id=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"') + version_major=${version_id%\.*} + + if [[ $os =~ "rhel" ]] && [ $version_major -gt 7 ] + then + export CURL_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt + fi +} + + # set up minimal environment: $EESSI_PREFIX, $EESSI_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX source $EESSI_INIT_DIR_PATH/minimal_eessi_env @@ -106,6 +120,8 @@ if [ -d $EESSI_PREFIX ]; then false fi + # Fix wrong path for RHEL >=8 libcurl + check_rhel_ca else error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!" From 8ed871b53b4bcca8a55f9365b7aaaf0288dc36c9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 26 Jun 2024 15:43:38 +0200 Subject: [PATCH 0875/1795] {2023.06,a64fx}[foss/2023a] SciPy-bundle 2023.07 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index 3a060e30a7..5326f6c493 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,3 +1,4 @@ easyconfigs: - OpenMPI-4.1.5-GCC-12.3.0.eb - foss-2023a.eb + - SciPy-bundle-2023.07-gfbf-2023a.eb From 69c059b758c29400ac7039cf1ee03af5eba2515c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 26 Jun 2024 22:29:52 +0200 Subject: [PATCH 0876/1795] {2023.06,a64fx}[foss/2023a] ESPResSo 4.2.2 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index 5326f6c493..3fe1e43d3f 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -2,3 +2,4 @@ easyconfigs: - OpenMPI-4.1.5-GCC-12.3.0.eb - foss-2023a.eb - SciPy-bundle-2023.07-gfbf-2023a.eb + - ESPResSo-4.2.2-foss-2023a.eb From 69a812962486940fe553facc459939af8f6549fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 27 Jun 2024 13:27:48 +0200 Subject: [PATCH 0877/1795] {2023.06,a64fx} foss/2023b --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023b.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023b.yml new file mode 100644 index 0000000000..c7411ada0a --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023b.yml @@ -0,0 +1,6 @@ +easyconfigs: + - OpenBLAS-0.3.24-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20820 + from-commit: 1fc885b35dacdeb2feef4af207a2daa2502bae08 + - foss-2023b.eb From f31f09142b1749ffbe2aa12ba20d2974b43a82e4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 29 Jun 2024 11:39:23 +0200 Subject: [PATCH 0878/1795] {2023.06,2023a,zen4} Highway v1.0.4 with fix for failing test --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 09f525364f..082204087e 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -2,3 +2,7 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb - TensorFlow-2.13.0-foss-2023a.eb - R-4.3.2-gfbf-2023a.eb + - Highway-1.0.4-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20942 + from-pr: 20942 From fa2914e1c31f69a83119d4e0829941ead1fff7ca Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 29 Jun 2024 11:56:58 +0200 Subject: [PATCH 0879/1795] double-check without the patch --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 082204087e..664555aeab 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -2,7 +2,8 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb - TensorFlow-2.13.0-foss-2023a.eb - R-4.3.2-gfbf-2023a.eb - - Highway-1.0.4-GCCcore-12.3.0.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20942 - from-pr: 20942 + - Highway-1.0.4-GCCcore-12.3.0.eb + #- Highway-1.0.4-GCCcore-12.3.0.eb: + # options: + # # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20942 + # from-pr: 20942 From 36dea35585cbc86b7044ec454593a00307c130f9 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 1 Jul 2024 08:45:07 +0000 Subject: [PATCH 0880/1795] {2023.06}[GCC/12.3.0] BCFtools v1.18 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml new file mode 100644 index 0000000000..ade226c217 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - BCFtools-1.18-GCC-12.3.0.eb From 9f841a2cbcf1c27ea6d7517bbb4edba9cb3744b8 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 1 Jul 2024 15:03:18 +0200 Subject: [PATCH 0881/1795] {2023.06}[foss/2023a] CP2K 2023.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index ade226c217..bd412f1579 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,2 +1,3 @@ easyconfigs: - BCFtools-1.18-GCC-12.3.0.eb + - CP2K-2023.1-foss-2023a.eb From 9747ee497161f23dd999094e78bae6c863c5a9fb Mon Sep 17 00:00:00 2001 From: torri Date: Mon, 1 Jul 2024 16:31:37 +0200 Subject: [PATCH 0882/1795] simplified the implementation --- init/eessi_environment_variables | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index d6e5b75dce..c28ddb45d9 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -15,20 +15,6 @@ function show_msg { fi } -function check_rhel_ca { - - os=$(grep -oP '(?<=^ID_LIKE=).+' /etc/os-release | tr -d '"') - - version_id=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"') - version_major=${version_id%\.*} - - if [[ $os =~ "rhel" ]] && [ $version_major -gt 7 ] - then - export CURL_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt - fi -} - - # set up minimal environment: $EESSI_PREFIX, $EESSI_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX source $EESSI_INIT_DIR_PATH/minimal_eessi_env @@ -121,7 +107,11 @@ if [ -d $EESSI_PREFIX ]; then fi # Fix wrong path for RHEL >=8 libcurl - check_rhel_ca + rhel_libcurl_file="/etc/pki/tls/certs/ca-bundle.crt" + if [ -f $rhel_libcurl_file ]; then + show_msg "Found libcurl CAs file at RHEL location, setting CURL_CA_BUNDLE" + export CURL_CA_BUNDLE=$rhel_libcurl_file + fi else error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!" From ba8de248ef1de35d2eecb359dd33ebcdb3981da0 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 2 Jul 2024 12:22:46 +0200 Subject: [PATCH 0883/1795] add from-pr 20951 for ARM support --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index bd412f1579..5d4aa9a772 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,3 +1,6 @@ easyconfigs: - BCFtools-1.18-GCC-12.3.0.eb - - CP2K-2023.1-foss-2023a.eb + - CP2K-2023.1-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 + options: + from-pr: 20951 From 51c6b1c998a69d8dd4a75f4f83b101b5ba81f1b4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jul 2024 10:51:41 +0200 Subject: [PATCH 0884/1795] improve comment that explains setting of `$CURL_CA_BUNDLE` on RHEL-based systems Co-authored-by: ocaisa --- init/eessi_environment_variables | 3 +++ 1 file changed, 3 insertions(+) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index c28ddb45d9..f5bb387955 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -107,6 +107,9 @@ if [ -d $EESSI_PREFIX ]; then fi # Fix wrong path for RHEL >=8 libcurl + # This is required here because we ship curl in our compat layer. If we only provided + # curl as a module file we could instead do this via a `modluafooter` in an EasyBuild + # hook (or via an Lmod hook) rhel_libcurl_file="/etc/pki/tls/certs/ca-bundle.crt" if [ -f $rhel_libcurl_file ]; then show_msg "Found libcurl CAs file at RHEL location, setting CURL_CA_BUNDLE" From d56cd31a81ae2a88adf8ca6b0d15e11df5624d7a Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 3 Jul 2024 12:07:22 +0200 Subject: [PATCH 0885/1795] add hook for missing installation check on aarch64 --- eb_hooks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index e6f0cf0dd2..b4862cb572 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -294,6 +294,22 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") +def parse_hook_CP2K_remove_deps_for_aarch64(ec, *args, **kwargs): + """ + Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64 + """ + if ec.name == 'CP2K' and ec.version in ('2023.1',): + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': + # LIBXSMM is not supported on ARM with GCC 12.2.0 and 12.3.0 + # See https://www.cp2k.org/dev:compiler_support + # See https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 + # we need this hook because we check for missing installations for all CPU targets + # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) + ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('libxsmm',)] + else: + raise EasyBuildError("CP2K-specific hook triggered for non-CP2K easyconfig?!")] + + def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): """ Solve issues with compiling or running the tests on both @@ -693,6 +709,7 @@ def inject_gpu_property(ec): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, + 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, From 95dd8a02239cbff967a3a5e51e3186398693bfed Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 3 Jul 2024 12:56:06 +0200 Subject: [PATCH 0886/1795] fix CP2K hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b4862cb572..4dc39c2fe8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -307,7 +307,7 @@ def parse_hook_CP2K_remove_deps_for_aarch64(ec, *args, **kwargs): # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('libxsmm',)] else: - raise EasyBuildError("CP2K-specific hook triggered for non-CP2K easyconfig?!")] + raise EasyBuildError("CP2K-specific hook triggered for non-CP2K easyconfig?!") def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): From 84de38a958417daf880c347dba0e9ae728d0e3f0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 3 Jul 2024 21:37:43 +0200 Subject: [PATCH 0887/1795] {2023.06,zen4}[foss/2023a] ParaView 5.11.2 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 09f525364f..d4c70c562d 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -2,3 +2,4 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb - TensorFlow-2.13.0-foss-2023a.eb - R-4.3.2-gfbf-2023a.eb + - ParaView-5.11.2-foss-2023a.eb From c895bf16668c588d7eec827092849d883ce10bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Jul 2024 13:24:12 +0200 Subject: [PATCH 0888/1795] remove TODO for using separate cache dirs for each repo --- eessi_container.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9d2ce6c25f..94de82df0f 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -510,7 +510,6 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}" # set env vars and create directories for CernVM-FS -# TODO need to use separate values for separate repos? EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} From 417f4dd9b82795520e5923a9c1532ed04f384747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Jul 2024 13:24:44 +0200 Subject: [PATCH 0889/1795] fix usage line for access permissions --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 94de82df0f..e404b7ee18 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -91,7 +91,7 @@ display_help() { echo " attach a GPU, MODE==all for both [default: false]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use; can be given multiple times;" - echo " CFG may include a suffix ';access={ro,rw}' to" + echo " CFG may include a suffix ',access={ro,rw}' to" echo " overwrite the global access mode for this repository" echo " [default: software.eessi.io via CVMFS config available" echo " via default container, see --container]" From d8cf6a60540b36b5c3e64c687af5d1482df021a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Jul 2024 13:41:40 +0200 Subject: [PATCH 0890/1795] check for software.eessi.io in the default --list-repos output --- .github/workflows/test_eessi_container_script.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index d31feb23fe..c0346774cb 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -45,7 +45,8 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then outfile=out_listrepos.txt ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI" ${outfile} + # make sure that the default EESSI software repository is available + grep "software.eessi.io" ${outfile} # test use of --list-repos with custom repos.cfg elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then From aa9895e44ac1afd5a44524d004161f3804449a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Jul 2024 15:27:27 +0200 Subject: [PATCH 0891/1795] fix regex to find the custom EESSI version in the output of --list-repos --- .github/workflows/test_eessi_container_script.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index c0346774cb..d44788927f 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -58,11 +58,12 @@ jobs: echo "[EESSI/20HT.TP]" >> cfg/repos.cfg echo "repo_version = 20HT.TP" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI" ${outfile} + # make sure that the default EESSI software repository is available + grep "software.eessi.io" ${outfile} export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile2} - grep "[EESSI/2023.02]" ${outfile2} + grep "EESSI/20AB.CD" ${outfile2} # test use of --mode run elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then From cbedafef9f104540151ca9093bc2e95b178036b3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jul 2024 21:07:39 +0200 Subject: [PATCH 0892/1795] {2023.06,a64fx}[foss/2023a] ParaView 5.11.2 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index 3fe1e43d3f..5521b92398 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -3,3 +3,4 @@ easyconfigs: - foss-2023a.eb - SciPy-bundle-2023.07-gfbf-2023a.eb - ESPResSo-4.2.2-foss-2023a.eb + - ParaView-5.11.2-foss-2023a.eb From c97dd7ed8e9d7cc23436927f4dfff00fae93d623 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jul 2024 21:58:10 +0200 Subject: [PATCH 0893/1795] add missing /cvmfs/ in $EESSI_CVMFS_REPO_OVERRIDE defined by bot/build.sh and bot/test.sh --- bot/build.sh | 2 +- bot/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index c9a362fdca..145be740d3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -138,7 +138,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} diff --git a/bot/test.sh b/bot/test.sh index 04bff346cd..d3f3630ea8 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -160,7 +160,7 @@ echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE} # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} From 3bae44ca2c8ee6cf0b53f753b74f47314d532222 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jul 2024 21:58:10 +0200 Subject: [PATCH 0894/1795] add missing /cvmfs/ in $EESSI_CVMFS_REPO_OVERRIDE defined by bot/build.sh and bot/test.sh --- bot/build.sh | 2 +- bot/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index c9a362fdca..145be740d3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -138,7 +138,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} diff --git a/bot/test.sh b/bot/test.sh index 04bff346cd..d3f3630ea8 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -160,7 +160,7 @@ echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE} # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} From 3fc6cdc1558e21005470e815adeb3f8620ec53e8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jul 2024 08:18:27 +0200 Subject: [PATCH 0895/1795] reduce parallellism when building Qt5 on A64FX systems, due to scarce memory --- eb_hooks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index e6f0cf0dd2..40071ea067 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -81,8 +81,11 @@ def post_ready_hook(self, *args, **kwargs): """ # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores. # here we reduce parallellism to only use half of that for selected software, - # to avoid failing builds/tests due to out-of-memory problems - if self.name in ['TensorFlow', 'libxc']: + # to avoid failing builds/tests due to out-of-memory problems; + memory_hungry_build = self.name in ['libxc', 'TensorFlow'] + # on A64FX systems, (HBM) memory is typically scarce, so we need to use fewer cores for some builds + memory_hungry_build_a64fx = self.name in ['Qt5'] + if memory_hungry_build or memory_hungry_build_a64fx: parallel = self.cfg['parallel'] if parallel > 1: self.cfg['parallel'] = parallel // 2 From a74669585a742ca83e9b976d7f6854a20570460f Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 5 Jul 2024 12:24:01 +0000 Subject: [PATCH 0896/1795] {2023.06}[GCCcore/12.3.0-GCC/12.3.0-gompi/2023a] bio-packages --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index ade226c217..7fc42e2e34 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,2 +1,17 @@ easyconfigs: - BCFtools-1.18-GCC-12.3.0.eb + - BWA-0.7.18-GCCcore-12.3.0.eb + - CapnProto-1.0.1-GCCcore-12.3.0.eb + - DendroPy-4.6.1-GCCcore-12.3.0.eb + - DIAMOND-2.1.8-GCC-12.3.0.eb + - FastME-2.1.6.3-GCC-12.3.0.eb + - fastp-0.23.4-GCC-12.3.0.eb + - HMMER-3.4-gompi-2023a.eb + - IQ-TREE-2.3.5-gompi-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20955 + from-commit: 185f88b9a03d65a7fb74edc7acb4221e87e90784 + - KronaTools-2.8.1-GCCcore-12.3.0.eb + - LSD2-2.4.1-GCCcore-12.3.0.eb + - MAFFT-7.520-GCC-12.3.0-with-extensions.eb + - ncbi-vdb-3.0.10-gompi-2023a.e From 4d9390ef015f03abe54ca3f61cd84ca6b1164c07 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 5 Jul 2024 14:30:32 +0200 Subject: [PATCH 0897/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 7fc42e2e34..34ec3257e8 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -14,4 +14,4 @@ easyconfigs: - KronaTools-2.8.1-GCCcore-12.3.0.eb - LSD2-2.4.1-GCCcore-12.3.0.eb - MAFFT-7.520-GCC-12.3.0-with-extensions.eb - - ncbi-vdb-3.0.10-gompi-2023a.e + - ncbi-vdb-3.0.10-gompi-2023a.eb From 868a8a8c94e932762299aabc6c6ed4eca749ef42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Jul 2024 16:07:05 +0200 Subject: [PATCH 0898/1795] /tmp/overlay-upper should now be /tmp//overlay-upper --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 0a7669f73f..ed999cde89 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -26,7 +26,7 @@ if [ ! -d ${software_dir} ]; then exit 2 fi -overlay_upper_dir="${eessi_tmpdir}/overlay-upper" +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo}/overlay-upper" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then From d5ecd74837cde2feb834275632e327beb182c00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Jul 2024 21:58:23 +0200 Subject: [PATCH 0899/1795] remove /cvmfs/ from cvmfs_repo variable --- create_tarball.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index ed999cde89..2dee665060 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -26,7 +26,8 @@ if [ ! -d ${software_dir} ]; then exit 2 fi -overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo}/overlay-upper" +cvmfs_repo_name=${cvmfs_repo#/cvmfs/} +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then From 49cbcc8de452e9cc51a3969b860932caf109a9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Jul 2024 22:02:31 +0200 Subject: [PATCH 0900/1795] implement changes from https://github.com/EESSI/software-layer/pull/635 and https://github.com/EESSI/software-layer/pull/636 --- create_tarball.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 0a7669f73f..2dee665060 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -26,7 +26,8 @@ if [ ! -d ${software_dir} ]; then exit 2 fi -overlay_upper_dir="${eessi_tmpdir}/overlay-upper" +cvmfs_repo_name=${cvmfs_repo#/cvmfs/} +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then From 88eebb9ee7245c56e4ad623c4055b8f4c365b59a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Jul 2024 11:26:40 +0200 Subject: [PATCH 0901/1795] rebuild OpenFOAM-10-foss-2023a.eb and OpenFOAM-11-foss-2023a.eb without -ftree-vectorize --- ...20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml new file mode 100644 index 0000000000..c12d244790 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml @@ -0,0 +1,12 @@ +# 2024.07.06 +# OpenFOAM 10 and 11 built with GCC 11.3.0 or 12.3.0 and -ftree-vectorize yields incorrect results, +# see https://github.com/easybuilders/easybuild-easyconfigs/issues/20927 +easyconfigs: + - OpenFOAM-10-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - OpenFOAM-11-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f From f2507201e02ea6da362f784fad1bbb5b27caa4a7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 6 Jul 2024 11:29:01 +0200 Subject: [PATCH 0902/1795] {2023.06,zen4}[foss/2023a] OpenFOAM 10 + 11 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index d4c70c562d..65a12695e4 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -3,3 +3,11 @@ easyconfigs: - TensorFlow-2.13.0-foss-2023a.eb - R-4.3.2-gfbf-2023a.eb - ParaView-5.11.2-foss-2023a.eb + - OpenFOAM-10-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - OpenFOAM-11-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f From 085e32a603842363607eeb10743f1f20107afd27 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Sat, 6 Jul 2024 12:10:46 +0200 Subject: [PATCH 0903/1795] {2023.06}[foss/2023a] Extrae 4.2.0 Alternative to #554 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 38bfa81142..fd26743499 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,3 +1,7 @@ easyconfigs: - IPython-8.17.2-GCCcore-13.2.0.eb - dlb-3.4-gompi-2023b.eb + - Extrae-4.2.0-gompi-2023b.eb: + options: + from-pr: 20690 + include-easyblocks-from-pr: 3339 From 7cb667929694c8a330a5e3f4921ea97e258f933a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sun, 7 Jul 2024 12:56:29 +0200 Subject: [PATCH 0904/1795] Re-add hook from #554 but keep running tests --- eb_hooks.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index e6f0cf0dd2..42b230c006 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -1,5 +1,6 @@ # Hooks to customize how EasyBuild installs software in EESSI # see https://docs.easybuild.io/en/latest/Hooks.html +import glob import os import re @@ -353,6 +354,31 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): else: raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") +def pre_configure_hook_extrae(self, *args, **kwargs): + """ + Pre-configure hook for Extrae + - avoid use of 'which' in configure script + - specify correct path to binutils (in compat layer) + """ + if self.name == 'Extrae': + + # determine path to Prefix installation in compat layer via $EPREFIX + eprefix = get_eessi_envvar('EPREFIX') + + binutils_lib_path_glob_pattern = os.path.join(eprefix, 'usr', 'lib*', 'binutils', '*-pc-linux-gnu', '2.*') + binutils_lib_path = glob.glob(binutils_lib_path_glob_pattern) + if len(binutils_lib_path) == 1: + self.cfg.update('configopts', '--with-binutils=' + binutils_lib_path[0]) + else: + raise EasyBuildError("Failed to isolate path for binutils libraries using %s, got %s", + binutils_lib_path_glob_pattern, binutils_lib_path) + + # replace use of 'which' with 'command -v', since 'which' is broken in EESSI build container; + # this must be done *after* running configure script, because initial configuration re-writes configure script, + # and problem due to use of which only pops up when running make ?! + self.cfg.update('prebuildopts', "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && ") + else: + raise EasyBuildError("Extrae-specific hook triggered for non-Extrae easyconfig?!") def pre_configure_hook_gromacs(self, *args, **kwargs): """ @@ -711,6 +737,7 @@ def inject_gpu_property(ec): PRE_CONFIGURE_HOOKS = { 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, 'BLIS': pre_configure_hook_BLIS_a64fx, + 'Extrae': pre_configure_hook_extrae, 'GROMACS': pre_configure_hook_gromacs, 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, From b07445e75cb874fb87003df1d92a5ba879deac4c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 7 Jul 2024 21:56:20 +0200 Subject: [PATCH 0905/1795] only use half the cores for Qt5 when building for A64FX --- eb_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 40071ea067..3b381e5963 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -84,7 +84,8 @@ def post_ready_hook(self, *args, **kwargs): # to avoid failing builds/tests due to out-of-memory problems; memory_hungry_build = self.name in ['libxc', 'TensorFlow'] # on A64FX systems, (HBM) memory is typically scarce, so we need to use fewer cores for some builds - memory_hungry_build_a64fx = self.name in ['Qt5'] + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + memory_hungry_build_a64fx = cpu_target == CPU_TARGET_A64FX and self.name in ['Qt5'] if memory_hungry_build or memory_hungry_build_a64fx: parallel = self.cfg['parallel'] if parallel > 1: From d6d5a2bb5c2eead7cd931ea307ea4cd3b44bf9db Mon Sep 17 00:00:00 2001 From: Helena Vela Beltran <47674829+hvelab@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:18:02 +0200 Subject: [PATCH 0906/1795] Update init/eessi_environment_variables Add bigger comment Co-authored-by: ocaisa --- init/eessi_environment_variables | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index f5bb387955..9417d5b2a2 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -106,15 +106,15 @@ if [ -d $EESSI_PREFIX ]; then false fi - # Fix wrong path for RHEL >=8 libcurl - # This is required here because we ship curl in our compat layer. If we only provided - # curl as a module file we could instead do this via a `modluafooter` in an EasyBuild - # hook (or via an Lmod hook) - rhel_libcurl_file="/etc/pki/tls/certs/ca-bundle.crt" - if [ -f $rhel_libcurl_file ]; then - show_msg "Found libcurl CAs file at RHEL location, setting CURL_CA_BUNDLE" - export CURL_CA_BUNDLE=$rhel_libcurl_file - fi + # Fix wrong path for RHEL >=8 libcurl + # This is required here because we ship curl in our compat layer. If we only provided + # curl as a module file we could instead do this via a `modluafooter` in an EasyBuild + # hook (or via an Lmod hook) + rhel_libcurl_file="/etc/pki/tls/certs/ca-bundle.crt" + if [ -f $rhel_libcurl_file ]; then + show_msg "Found libcurl CAs file at RHEL location, setting CURL_CA_BUNDLE" + export CURL_CA_BUNDLE=$rhel_libcurl_file + fi else error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!" From 0fef63a6ce712956e04b9cb007c21565d624f1fa Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 9 Jul 2024 00:15:40 +0200 Subject: [PATCH 0907/1795] Update eb_hooks.py --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 42b230c006..f43b695266 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -365,7 +365,7 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # determine path to Prefix installation in compat layer via $EPREFIX eprefix = get_eessi_envvar('EPREFIX') - binutils_lib_path_glob_pattern = os.path.join(eprefix, 'usr', 'lib*', 'binutils', '*-pc-linux-gnu', '2.*') + binutils_lib_path_glob_pattern = os.path.join(eprefix, 'usr', 'lib*', 'binutils', '*-linux-gnu', '2.*') binutils_lib_path = glob.glob(binutils_lib_path_glob_pattern) if len(binutils_lib_path) == 1: self.cfg.update('configopts', '--with-binutils=' + binutils_lib_path[0]) From 391d5eda743660471b7bcb6aa4df6df0229052e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 9 Jul 2024 15:47:14 +0200 Subject: [PATCH 0908/1795] use easyblock from PR 3388 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 74e1f4610b..05e86a8565 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,7 +1,7 @@ easyconfigs: - OpenFOAM-v2312-foss-2023a.eb: options: - # https://github.com/easybuilders/easybuild-easyblocks/pull/3366 - include-easyblocks-from-commit: d1f2231b2de4f8548c75c7367456873a8c33b5f9 + # https://github.com/easybuilders/easybuild-easyblocks/pull/3388 + include-easyblocks-from-commit: c8256a36e7062bc09f5ce30552a9de9827054c9e # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841 from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 From 28eabaaefcde5b112b60e8c94f46494ab7b0944e Mon Sep 17 00:00:00 2001 From: Xin An Date: Fri, 12 Jul 2024 12:11:19 +0200 Subject: [PATCH 0909/1795] {2023.06}[gfbf/2023b] Pystencils 1.3.4 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 15c02951d7..3963f79ecd 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -58,3 +58,4 @@ easyconfigs: options: from-pr: 20201 - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb + - pystencils-1.3.4-gfbf-2023b.eb From 2ca3684b845165425af638daaea27897fd1cbe7e Mon Sep 17 00:00:00 2001 From: Xin An <34663977+xinan1911@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:29:19 +0200 Subject: [PATCH 0910/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 3963f79ecd..8d9779c59c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -58,4 +58,7 @@ easyconfigs: options: from-pr: 20201 - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb - - pystencils-1.3.4-gfbf-2023b.eb + - pystencils-1.3.4-gfbf-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 + from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 From 641288af0c2bc1d1e35d411815465abdeb8a1ddc Mon Sep 17 00:00:00 2001 From: Xin An <34663977+xinan1911@users.noreply.github.com> Date: Fri, 12 Jul 2024 16:08:14 +0200 Subject: [PATCH 0911/1795] Update eessi-2023.06-eb-4.9.0-2023b.yml Remove pystencils --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 8d9779c59c..15c02951d7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -58,7 +58,3 @@ easyconfigs: options: from-pr: 20201 - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb - - pystencils-1.3.4-gfbf-2023b.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 - from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 From bb8b4aa1442c032b88d6cacc94b2d8f42cce3721 Mon Sep 17 00:00:00 2001 From: Xin An <34663977+xinan1911@users.noreply.github.com> Date: Fri, 12 Jul 2024 16:10:02 +0200 Subject: [PATCH 0912/1795] add pystencils update eessi-2023.06-eb-4.9.2-2023b --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 38bfa81142..e0057f18ab 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,3 +1,7 @@ easyconfigs: - IPython-8.17.2-GCCcore-13.2.0.eb - dlb-3.4-gompi-2023b.eb + - pystencils-1.3.4-gfbf-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 + from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 From f204afa5d804dee715476ac3d2a96e8e69fa6842 Mon Sep 17 00:00:00 2001 From: crivella Date: Mon, 15 Jul 2024 10:47:34 +0200 Subject: [PATCH 0913/1795] Add MetalWalls to easystack file --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 4b58cb6106..0cdbd4b5b3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -74,3 +74,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 + - MetalWalls-21.06.1-foss-2023a.eb From 58fb56f2a8f2fb11d5df2b8332388ace82eda993 Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 16 Jul 2024 14:41:39 +0200 Subject: [PATCH 0914/1795] Switch target to EB 4.9.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 - .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 0cdbd4b5b3..4b58cb6106 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -74,4 +74,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 - - MetalWalls-21.06.1-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 34ec3257e8..e7730d1eb6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -15,3 +15,4 @@ easyconfigs: - LSD2-2.4.1-GCCcore-12.3.0.eb - MAFFT-7.520-GCC-12.3.0-with-extensions.eb - ncbi-vdb-3.0.10-gompi-2023a.eb + - MetalWalls-21.06.1-foss-2023a.eb From a7a895408c9b194cd8aeff27dd5f17f54a5fc514 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Jul 2024 13:45:39 +0200 Subject: [PATCH 0915/1795] Move changes to latest eb release --- .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 6 ------ .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 428c1ac77f..4b58cb6106 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -74,9 +74,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 - - QuantumESPRESSO-7.3.1-foss-2023a.eb: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138 - # see https://github.com/easybuilders/easybuild-easyblocks/pull/3257 - options: - from-pr: 20138 - include-easyblocks-from-pr: 3257 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e7730d1eb6..af69b288e8 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -16,3 +16,9 @@ easyconfigs: - MAFFT-7.520-GCC-12.3.0-with-extensions.eb - ncbi-vdb-3.0.10-gompi-2023a.eb - MetalWalls-21.06.1-foss-2023a.eb + - QuantumESPRESSO-7.3.1-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3338 + options: + from-pr: 20138 + include-easyblocks-from-pr: 3338 From 0ab0440e4ee990bbe76d7e8b7d0912701b21e8fa Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 17 Jul 2024 14:00:00 +0200 Subject: [PATCH 0916/1795] Allow initialisation scripts to be used with bash < 4.2 --- init/bash | 2 +- init/eessi_defaults | 2 +- init/eessi_environment_variables | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/init/bash b/init/bash index 93471e075e..4ad09f6a1b 100644 --- a/init/bash +++ b/init/bash @@ -1,7 +1,7 @@ function show_msg { # only echo msg if EESSI_SILENT is unset msg=$1 - if [[ ! -v EESSI_SILENT ]]; then + if [[ -z ${EESSI_SILENT+x} ]]; then echo "$msg" fi } diff --git a/init/eessi_defaults b/init/eessi_defaults index c1ce82d1ce..654a829425 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -12,7 +12,7 @@ if [[ $(uname -m) == "riscv64" ]]; then export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/riscv.eessi.io}" export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=20240402}" - if [[ ! -v EESSI_SILENT ]]; then + if [[ -z ${EESSI_SILENT+x} ]]; then echo "RISC-V architecture detected, but there is no RISC-V support yet in the production repository." echo "Automatically switching to version ${EESSI_VERSION} of the RISC-V development repository ${EESSI_CVMFS_REPO}." echo "For more details about this repository, see https://www.eessi.io/docs/repositories/riscv.eessi.io/." diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 9417d5b2a2..3e35f845a1 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -10,7 +10,7 @@ function error() { function show_msg { # only echo msg if EESSI_SILENT is unset msg=$1 - if [[ ! -v EESSI_SILENT ]]; then + if [[ -z ${EESSI_SILENT+x} ]]; then echo "$msg" fi } From 5ff9d1660c736ddf88052035fa7a10a6ac7a33c1 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 18 Jul 2024 16:02:55 +0200 Subject: [PATCH 0917/1795] Fixes for ARM --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 4 ++-- eb_hooks.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index bc7bee302a..3f18d02db5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -7,5 +7,5 @@ easyconfigs: from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 - Extrae-4.2.0-gompi-2023b.eb: options: - from-pr: 20690 - include-easyblocks-from-pr: 3339 + from-pr: 21017 + include-easyblocks-from-pr: 3393 diff --git a/eb_hooks.py b/eb_hooks.py index ec08e96847..73adce90f1 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -362,7 +362,7 @@ def pre_configure_hook_extrae(self, *args, **kwargs): """ Pre-configure hook for Extrae - avoid use of 'which' in configure script - - specify correct path to binutils (in compat layer) + - specify correct path to binutils/zlib (in compat layer) """ if self.name == 'Extrae': @@ -377,6 +377,9 @@ def pre_configure_hook_extrae(self, *args, **kwargs): raise EasyBuildError("Failed to isolate path for binutils libraries using %s, got %s", binutils_lib_path_glob_pattern, binutils_lib_path) + # zlib is a filtered dependency, so we need to manually specify it's location to avoid the host version + self.cfg.update('configopts', '--with-libz=' + eprefix) + # replace use of 'which' with 'command -v', since 'which' is broken in EESSI build container; # this must be done *after* running configure script, because initial configuration re-writes configure script, # and problem due to use of which only pops up when running make ?! From 060e9815f905b28dd963fda12ba9bdfd4163b94e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 18 Jul 2024 16:05:55 +0200 Subject: [PATCH 0918/1795] add R 4.4.1 and R-bundle-CRAN 2024.06 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index e0057f18ab..c371fb16bc 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -5,3 +5,10 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 + - R-4.4.1-gfbf-2023b.eb + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 + from-commit: 37e673217efbecc359d6008dcf19f8dbf72e777c + - R-bundle-CRAN-2024.06-foss-2023b.eb + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 + from-commit: 37e673217efbecc359d6008dcf19f8dbf72e777c From de84033c321129be15d986a0665821dc1210967e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 18 Jul 2024 16:21:55 +0200 Subject: [PATCH 0919/1795] add missing colons --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index c371fb16bc..7a0b70867d 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -5,10 +5,10 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 - - R-4.4.1-gfbf-2023b.eb + - R-4.4.1-gfbf-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 from-commit: 37e673217efbecc359d6008dcf19f8dbf72e777c - - R-bundle-CRAN-2024.06-foss-2023b.eb + - R-bundle-CRAN-2024.06-foss-2023b.eb: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 from-commit: 37e673217efbecc359d6008dcf19f8dbf72e777c From a3b713479fa42f2e720bbe491713953b53b2b6e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 18 Jul 2024 16:42:11 +0200 Subject: [PATCH 0920/1795] add missing options key --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 7a0b70867d..9779adfc4d 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -10,5 +10,6 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 from-commit: 37e673217efbecc359d6008dcf19f8dbf72e777c - R-bundle-CRAN-2024.06-foss-2023b.eb: + options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 from-commit: 37e673217efbecc359d6008dcf19f8dbf72e777c From 192e494b56603dcead6b30c19e00064efe7b2769 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 18 Jul 2024 16:59:47 +0200 Subject: [PATCH 0921/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 3f18d02db5..f118e94adb 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -7,5 +7,7 @@ easyconfigs: from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 - Extrae-4.2.0-gompi-2023b.eb: options: - from-pr: 21017 - include-easyblocks-from-pr: 3393 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21017 + from-commit: 120f4d56efebd2bc61382db4c84a664a339c66cf + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3393 + include-easyblocks-from-commit: c4951c78d62fa5cf8e9f6fe0ead212d2a4d7cb9c From 8b350fd435924a18705d1869c0bc87be15d76465 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 18 Jul 2024 18:02:08 +0200 Subject: [PATCH 0922/1795] Update eessi-2023.06-eb-4.9.2-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index ed9896191b..f40d7d85e1 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -19,4 +19,4 @@ easyconfigs: - CP2K-2023.1-foss-2023a.eb: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 options: - from-pr: 20951 \ No newline at end of file + from-pr: 20951 From c0bec39b3c0528f9bde6f91c853f08b75cee1796 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 19 Jul 2024 09:28:03 +0200 Subject: [PATCH 0923/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index f40d7d85e1..bd6c767260 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -17,6 +17,6 @@ easyconfigs: - ncbi-vdb-3.0.10-gompi-2023a.eb - MetalWalls-21.06.1-foss-2023a.eb - CP2K-2023.1-foss-2023a.eb: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 options: - from-pr: 20951 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 + from-commit: a92667fe32396bbd4106243658625f7ff2adcd68 From a61925fb430a0e1b1adaf22f8e7e7c81a782dadf Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 19 Jul 2024 09:34:19 +0200 Subject: [PATCH 0924/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index af69b288e8..67cc4bc155 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -17,8 +17,8 @@ easyconfigs: - ncbi-vdb-3.0.10-gompi-2023a.eb - MetalWalls-21.06.1-foss-2023a.eb - QuantumESPRESSO-7.3.1-foss-2023a.eb: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138 - # see https://github.com/easybuilders/easybuild-easyblocks/pull/3338 options: - from-pr: 20138 - include-easyblocks-from-pr: 3338 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138 + from-commit: dbdaacc0739fdee91baa9123864ea4428cf21273 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3338 + include-easyblocks-from-commit: 32e45bd1f2d916732ca5852d55d17fa4d99e388b From b3faff32e0dbdb55f6a86ad8fc45ceaaf01adf49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Jul 2024 09:36:05 +0200 Subject: [PATCH 0925/1795] add hook that fixes Pango+GObject-Introspection issue --- eb_hooks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 3b381e5963..5c8d19f052 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -439,6 +439,21 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") +def pre_configure_hook_pango_filter_ld_library_path(self, *args, **kwargs): + """ + pre-configure hook for Pango: + - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH + when EasyBuild is configured to filter it, see: + https://github.com/EESSI/software-layer/issues/196 + """ + if self.name == 'Pango' and self.version in ['1.51.0']: + if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'): + sed_cmd = 'sed -i "s/gir_args = \[/gir_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/pango/meson.build && ' + self.cfg.update('preconfigopts', sed_cmd) + else: + raise EasyBuildError("Pango-specific hook triggered for non-Pango easyconfig?!") + + def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): """ pre-configure hook for at-spi2-core: @@ -719,6 +734,7 @@ def inject_gpu_property(ec): 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, + 'Pango': pre_configure_hook_pango_filter_ld_library_path, 'WRF': pre_configure_hook_wrf_aarch64, } From 1660e048811192f8d072815963e960fdd9620828 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 19 Jul 2024 09:49:39 +0200 Subject: [PATCH 0926/1795] Remove duplicate testing of PRs --- .github/workflows/test-software.eessi.io.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 06c02c8834..d4d980901f 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -1,6 +1,10 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Check for missing software installations in software.eessi.io -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) jobs: From 4d93a2d87b7289b12ab65daac9ddcc16e69a47ee Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 19 Jul 2024 09:57:54 +0200 Subject: [PATCH 0927/1795] Restrict tests for all CI --- .github/workflows/scorecards.yml | 2 +- .github/workflows/test_eessi_container_script.yml | 6 +++++- .github/workflows/test_licenses.yml | 5 ++++- .github/workflows/tests.yml | 5 ++++- .github/workflows/tests_archdetect.yml | 5 ++++- .github/workflows/tests_init.yml | 5 ++++- 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 5b482763f2..91c2576b7d 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -10,7 +10,7 @@ on: - cron: '25 15 * * 3' push: branches: - - '2023.06-software.eessi.io' + - '*-software.eessi.io' # Declare default permissions as read only. permissions: read-all diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index d44788927f..fb587ec489 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -1,6 +1,10 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for eessi_container.sh script -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/.github/workflows/test_licenses.yml b/.github/workflows/test_licenses.yml index 6b6387d47d..bf6965c620 100644 --- a/.github/workflows/test_licenses.yml +++ b/.github/workflows/test_licenses.yml @@ -1,6 +1,9 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Test software licenses -on: [push, pull_request] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8249e50d9b..940614b6bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,9 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests -on: [push, pull_request] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 4f5d3d174d..f5b42b3d55 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -1,6 +1,9 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for eessi_archdetect.sh -on: [push, pull_request] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 6c356a9184..8581681d63 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -1,6 +1,9 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for init scripts -on: [push, pull_request] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: permissions: contents: read # to fetch code (actions/checkout) jobs: From 8a2eca3cf3c949cff5fb1f61e1a3ef4e6d2cdfd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Jul 2024 12:06:20 +0200 Subject: [PATCH 0928/1795] add a hook for GObject-Introspection that prevents environment vars from being set if EB filters them --- eb_hooks.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 73adce90f1..deea409cf4 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -387,6 +387,24 @@ def pre_configure_hook_extrae(self, *args, **kwargs): else: raise EasyBuildError("Extrae-specific hook triggered for non-Extrae easyconfig?!") + +def pre_configure_hook_gobject_introspection(self, *args, **kwargs): + """ + pre-configure hook for GObject-Introspection: + - prevent GObject-Introspection from setting $LD_LIBRARY_PATH if Easybuild is configured to filter it + that are configured to be filtered by EasyBuild, see: + https://github.com/EESSI/software-layer/issues/196 + """ + if self.name == 'GObject-Introspection': + # inject a line that removes all items from runtime_path_envvar that are in $EASYBUILD_FILTER_ENVVARS + sed_cmd = r'sed -i "s@\(^\s*runtime_path_envvar = \)\(.*\)@' + sed_cmd += r'\1\2\n\1 [x for x in runtime_path_envvar if not x in os.environ.get(\'EASYBUILD_FILTER_ENV_VARS\', \'\').split(\',\')]@g"' + sed_cmd += ' %(start_dir)s/giscanner/ccompiler.py && ' + self.cfg.update('preconfigopts', sed_cmd) + else: + raise EasyBuildError("GObject-Introspection-specific hook triggered for non-GObject-Introspection easyconfig?!") + + def pre_configure_hook_gromacs(self, *args, **kwargs): """ Pre-configure hook for GROMACS: @@ -744,6 +762,7 @@ def inject_gpu_property(ec): PRE_CONFIGURE_HOOKS = { 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, 'BLIS': pre_configure_hook_BLIS_a64fx, + 'GObject-Introspection': pre_configure_hook_gobject_introspection, 'Extrae': pre_configure_hook_extrae, 'GROMACS': pre_configure_hook_gromacs, 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, From 62c1f7d000473bd53a60fa7d814dcccf11978524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Jul 2024 13:45:42 +0200 Subject: [PATCH 0929/1795] rebuild all GObject-Introspection versions to apply new hook --- ...eb-4.9.2-GObject-Introspection-filter-envvars.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml new file mode 100644 index 0000000000..676a2fe548 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml @@ -0,0 +1,12 @@ +# 2024.07.19 +# GObject-Introspection sets $LD_LIBRARY_PATH (to many different paths, including $EPREFIX/lib) +# when calling gcc, and this causes a lot of issues for, especially, scripts using /bin/bash. +# +# This rebuild ensures (by using a new EasyBuild hook) that GObject-Introspection will not set +# environment variables that are configured to be filtered by EasyBuild. +# +# See https://github.com/EESSI/software-layer/issues/196 +easyconfigs: + - GObject-Introspection-1.74.0-GCCcore-12.2.0.eb + - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb + - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb From 6277bc382003651012a8196bc7667ae1a3371704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Jul 2024 13:58:53 +0200 Subject: [PATCH 0930/1795] fix comment Co-authored-by: ocaisa --- eb_hooks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index deea409cf4..aa9bde383a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -391,8 +391,7 @@ def pre_configure_hook_extrae(self, *args, **kwargs): def pre_configure_hook_gobject_introspection(self, *args, **kwargs): """ pre-configure hook for GObject-Introspection: - - prevent GObject-Introspection from setting $LD_LIBRARY_PATH if Easybuild is configured to filter it - that are configured to be filtered by EasyBuild, see: + - prevent GObject-Introspection from setting $LD_LIBRARY_PATH if EasyBuild is configured to filter it, see: https://github.com/EESSI/software-layer/issues/196 """ if self.name == 'GObject-Introspection': From 5593999aa6acfaef2d235abc987a908643109cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Jul 2024 14:39:10 +0200 Subject: [PATCH 0931/1795] remove the at-spi2-core hook --- eb_hooks.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index deea409cf4..00d26df37b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -486,21 +486,6 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") -def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): - """ - pre-configure hook for at-spi2-core: - - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH - when EasyBuild is configured to filter it, see: - https://github.com/EESSI/software-layer/issues/196 - """ - if self.name == 'at-spi2-core': - if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'): - sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ' - self.cfg.update('preconfigopts', sed_cmd) - else: - raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!") - - def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: @@ -760,7 +745,6 @@ def inject_gpu_property(ec): } PRE_CONFIGURE_HOOKS = { - 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, 'BLIS': pre_configure_hook_BLIS_a64fx, 'GObject-Introspection': pre_configure_hook_gobject_introspection, 'Extrae': pre_configure_hook_extrae, From 46a9c85149e1a9ff29bc8da9c322eda639c7f923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Jul 2024 14:39:34 +0200 Subject: [PATCH 0932/1795] rebuild at-spi2-core with the rebuilt GObject-Introspection --- .../20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml index 676a2fe548..a61cd9705b 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml @@ -10,3 +10,5 @@ easyconfigs: - GObject-Introspection-1.74.0-GCCcore-12.2.0.eb - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb + - at-spi2-core-2.46.0-GCCcore-12.2.0.eb + - at-spi2-core-2.49.91-GCCcore-12.3.0.eb From bd328a2c357a4f1fe998d4b922363abe4461b904 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 19 Jul 2024 15:03:34 +0200 Subject: [PATCH 0933/1795] Use instead of (CentOS6) --- init/eessi_archdetect.sh | 2 +- init/eessi_environment_variables | 2 +- init/minimal_eessi_env | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index e89bc75e13..ddfed1bfae 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -89,7 +89,7 @@ cpupath(){ *) log "ERROR" "cpupath: Unsupported CPU architecture $machine_type" esac # spec files are located in a subfolder with this script - local base_dir=$(dirname $(realpath $0)) + local base_dir=$(dirname $(readlink -f $0)) update_arch_specs "$base_dir/arch_specs/${spec_file}" # Identify the host CPU vendor diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 3e35f845a1..8c10b1fca8 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -1,6 +1,6 @@ # this script is *sourced*, not executed, so can't rely on $0 to determine path to self # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 -EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) +EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) function error() { echo -e "\e[31mERROR: $1\e[0m" >&2 diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 03b337ccf7..5273f27862 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -2,7 +2,7 @@ # # this script is *sourced*, not executed, so can't rely on $0 to determine path to self # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 -EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) +EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # set up defaults: EESSI_CVMFS_REPO, EESSI_VERSION # script takes *_OVERRIDEs into account From 20eafc70aec97f46f311b27fe9b44b845964f723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Jul 2024 15:30:35 +0200 Subject: [PATCH 0934/1795] rebuild GObject-Introspection for zen4 --- ....9.2-GObject-Introspection-filter-envvars-zen4.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-zen4.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-zen4.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-zen4.yml new file mode 100644 index 0000000000..2c9b411736 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-zen4.yml @@ -0,0 +1,11 @@ +# 2024.07.19 +# GObject-Introspection sets $LD_LIBRARY_PATH (to many different paths, including $EPREFIX/lib) +# when calling gcc, and this causes a lot of issues for, especially, scripts using /bin/bash. +# +# This rebuild ensures (by using a new EasyBuild hook) that GObject-Introspection will not set +# environment variables that are configured to be filtered by EasyBuild. +# +# See https://github.com/EESSI/software-layer/issues/196 +easyconfigs: + - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb + - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb From ba755b26f79dfdc8b3702b2c2e3e5ceff3fcb23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 19 Jul 2024 17:07:56 +0200 Subject: [PATCH 0935/1795] remove Pango hook again, fixed in GObject-Introspection --- eb_hooks.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 5c8d19f052..3b381e5963 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -439,21 +439,6 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") -def pre_configure_hook_pango_filter_ld_library_path(self, *args, **kwargs): - """ - pre-configure hook for Pango: - - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH - when EasyBuild is configured to filter it, see: - https://github.com/EESSI/software-layer/issues/196 - """ - if self.name == 'Pango' and self.version in ['1.51.0']: - if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'): - sed_cmd = 'sed -i "s/gir_args = \[/gir_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/pango/meson.build && ' - self.cfg.update('preconfigopts', sed_cmd) - else: - raise EasyBuildError("Pango-specific hook triggered for non-Pango easyconfig?!") - - def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): """ pre-configure hook for at-spi2-core: @@ -734,7 +719,6 @@ def inject_gpu_property(ec): 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, - 'Pango': pre_configure_hook_pango_filter_ld_library_path, 'WRF': pre_configure_hook_wrf_aarch64, } From f2f40ceac1d4af66d304e155d55c98369b165eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 20 Jul 2024 13:48:38 +0200 Subject: [PATCH 0936/1795] disable Highway tests for now on neoverse v1 --- eb_hooks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 73adce90f1..e497109986 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -318,6 +318,9 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg if cpu_target == CPU_TARGET_NEOVERSE_N1: self.orig_optarch = build_option('optarch') update_build_option('optarch', OPTARCH_GENERIC) + if self.version in ['1.0.7'] and tcname == 'GCCcore' and tcversion == '13.2.0': + if cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') else: raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") From e8dab521b86fb9b78e1ef4b44abb16da38236b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 20 Jul 2024 16:53:45 +0200 Subject: [PATCH 0937/1795] Disable Highway 1.0.7 tests for all Arm targets --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index e497109986..4a6068f7fa 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -319,7 +319,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg self.orig_optarch = build_option('optarch') update_build_option('optarch', OPTARCH_GENERIC) if self.version in ['1.0.7'] and tcname == 'GCCcore' and tcversion == '13.2.0': - if cpu_target == CPU_TARGET_NEOVERSE_V1: + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') else: raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") From 55a32613d3f6c4a295186cd93e33234e88146e48 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 22 Jul 2024 13:54:55 +0000 Subject: [PATCH 0938/1795] {2023.06}[gompi/2022b] BLAST+ V2.14.0 --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml new file mode 100644 index 0000000000..ce28312e8c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -0,0 +1,4 @@ +easyconfigs: + - BLAST+-2.14.0-gompi-2022b.eb: + options: + from-commit: 1775912a1d36101d560e1355967732b07e82cd24 From 42c41e98d5283022ddc897d3c908f77090fa31d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 22 Jul 2024 21:54:59 +0200 Subject: [PATCH 0939/1795] Use newer commit from R PR --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 25fcc3cf75..25a41413c9 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -14,8 +14,8 @@ easyconfigs: - R-4.4.1-gfbf-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 - from-commit: 37e673217efbecc359d6008dcf19f8dbf72e777c + from-commit: 12ef0f174250928ca9a5ccf47e4f7ca0ce56811f - R-bundle-CRAN-2024.06-foss-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 - from-commit: 37e673217efbecc359d6008dcf19f8dbf72e777c \ No newline at end of file + from-commit: 12ef0f174250928ca9a5ccf47e4f7ca0ce56811f \ No newline at end of file From 808a9032f942d1a88108a0b7d7067498b7e6e2d5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 23 Jul 2024 10:09:09 +0200 Subject: [PATCH 0940/1795] {2023.06}[gompi/2023a] amdahl 0.3.1 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 7e9f4ff05a..d405d17193 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -26,3 +26,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 from-commit: a92667fe32396bbd4106243658625f7ff2adcd68 + - amdahl-0.3.1-gompi-2023a.eb From 7d06f13fa83c7774d74ca9a09499ee34f7f1b17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 23 Jul 2024 15:56:21 +0200 Subject: [PATCH 0941/1795] add pyMBE 0.8.0 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index f118e94adb..883483f5d2 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -11,3 +11,7 @@ easyconfigs: from-commit: 120f4d56efebd2bc61382db4c84a664a339c66cf # see https://github.com/easybuilders/easybuild-easyblocks/pull/3393 include-easyblocks-from-commit: c4951c78d62fa5cf8e9f6fe0ead212d2a4d7cb9c + - pyMBE-0.8.0-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 + from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 From c0c784efbcb98389d6c61c43b2442dd07a981703 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 26 Jul 2024 18:25:45 +0200 Subject: [PATCH 0942/1795] Update eessi-2023.06-eb-4.9.2-2022b.yml --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index ce28312e8c..7a04eaa9d3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -1,4 +1,2 @@ easyconfigs: - - BLAST+-2.14.0-gompi-2022b.eb: - options: - from-commit: 1775912a1d36101d560e1355967732b07e82cd24 + - BLAST+-2.14.0-gompi-2022b.eb From 05164ad4bc76f565088dd4f743646a8571528dd8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jul 2024 11:24:34 +0200 Subject: [PATCH 0943/1795] Add a check for the bash shell to archdetect --- init/eessi_archdetect.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index ddfed1bfae..afa6c0b387 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -1,4 +1,26 @@ #!/usr/bin/env bash + +# Confirm the current shell is Bash >= 4 +check_bash() { + if [ -n "$BASH_VERSION" ]; then + # Extract the major version numbers + bash_version=$(echo "$BASH_VERSION" | grep -oP '^\d+\.\d+') + major_version=$(echo "$bash_version" | cut -d. -f1) + + # Check if the major version is 4 or higher + if [ "$major_version" -ge 4 ]; then + return 0 + else + echo "Error: This script must be run with Bash > 4, you have $BASH_VERSION." >&2 + exit 1 + fi + else + echo "Error: This script must be run with Bash." >&2 + exit 1 + fi +} +check_bash + VERSION="1.1.0" # default log level: only emit warnings or errors From 6f43f1349daecd474805828af255abec4d83f2a7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jul 2024 11:35:22 +0200 Subject: [PATCH 0944/1795] List compatibility --- init/eessi_archdetect.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index afa6c0b387..ed6a1f8c51 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Confirm the current shell is Bash >= 4 +# (works for sh, bash, dash, zsh, ksh, but not fish, tcsh, elvish) check_bash() { if [ -n "$BASH_VERSION" ]; then # Extract the major version numbers From 47c0a34173e8431f22794905e8c6addd81a3210c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jul 2024 11:38:33 +0200 Subject: [PATCH 0945/1795] Don't declare a function for bash check --- init/eessi_archdetect.sh | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index ed6a1f8c51..0e83a45d2f 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -2,25 +2,22 @@ # Confirm the current shell is Bash >= 4 # (works for sh, bash, dash, zsh, ksh, but not fish, tcsh, elvish) -check_bash() { - if [ -n "$BASH_VERSION" ]; then - # Extract the major version numbers - bash_version=$(echo "$BASH_VERSION" | grep -oP '^\d+\.\d+') - major_version=$(echo "$bash_version" | cut -d. -f1) - - # Check if the major version is 4 or higher - if [ "$major_version" -ge 4 ]; then - return 0 - else - echo "Error: This script must be run with Bash > 4, you have $BASH_VERSION." >&2 - exit 1 - fi +if [ -n "$BASH_VERSION" ]; then + # Extract the major version numbers + bash_version=$(echo "$BASH_VERSION" | grep -oP '^\d+\.\d+') + major_version=$(echo "$bash_version" | cut -d. -f1) + + # Check if the major version is 4 or higher + if [ "$major_version" -ge 4 ]; then + return 0 else - echo "Error: This script must be run with Bash." >&2 + echo "Error: This script must be run with Bash > 4, you have $BASH_VERSION." >&2 exit 1 fi -} -check_bash +else + echo "Error: This script must be run with Bash." >&2 + exit 1 +fi VERSION="1.1.0" From 0750f5f4f0753dcb08362a4ed9dae90faf4b157c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jul 2024 11:41:34 +0200 Subject: [PATCH 0946/1795] Don't use return values --- init/eessi_archdetect.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 0e83a45d2f..001683f50a 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -8,9 +8,7 @@ if [ -n "$BASH_VERSION" ]; then major_version=$(echo "$bash_version" | cut -d. -f1) # Check if the major version is 4 or higher - if [ "$major_version" -ge 4 ]; then - return 0 - else + if [ "$major_version" -lt 4 ]; then echo "Error: This script must be run with Bash > 4, you have $BASH_VERSION." >&2 exit 1 fi From a4e838beb3b59d91ce4b93506947cb0d89586d6a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jul 2024 13:43:19 +0200 Subject: [PATCH 0947/1795] Add Lmod wrapper for eessi_archdetect.sh --- init/lmod_eessi_archdetect_wrapper.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 init/lmod_eessi_archdetect_wrapper.sh diff --git a/init/lmod_eessi_archdetect_wrapper.sh b/init/lmod_eessi_archdetect_wrapper.sh new file mode 100644 index 0000000000..a90ca28285 --- /dev/null +++ b/init/lmod_eessi_archdetect_wrapper.sh @@ -0,0 +1 @@ +export EESSI_ARCHDETECT_OPTIONS=$($(dirname $(readlink -f $BASH_SOURCE))/eessi_archdetect.sh -a cpupath) From c9046186b992d5bf6d11e5d3854f1135662db1fd Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jul 2024 13:52:12 +0200 Subject: [PATCH 0948/1795] Add comment --- init/lmod_eessi_archdetect_wrapper.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/init/lmod_eessi_archdetect_wrapper.sh b/init/lmod_eessi_archdetect_wrapper.sh index a90ca28285..c12452c71d 100644 --- a/init/lmod_eessi_archdetect_wrapper.sh +++ b/init/lmod_eessi_archdetect_wrapper.sh @@ -1 +1,2 @@ +# This can be leveraged by the source_sh() feature of Lmod export EESSI_ARCHDETECT_OPTIONS=$($(dirname $(readlink -f $BASH_SOURCE))/eessi_archdetect.sh -a cpupath) From ba8ee7a0ee2b1668eccf95665acf40886dce0868 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 29 Jul 2024 14:47:31 +0200 Subject: [PATCH 0949/1795] Rebuild Python with ctypes fix --- .../20240729-eb-4.9.2-Python-ctypes.yml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml new file mode 100644 index 0000000000..95bd27852a --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml @@ -0,0 +1,21 @@ +# 2024.07.29 +# Python ctypes relies on LD_LIBRARY_PATH and doesn't respect rpath linking. There is a workaround +# for the EasyBuild context in https://github.com/easybuilders/easybuild-easyblocks/pull/3352. +# +# This rebuild ensures this fix is available for all Python versions shipped with EESSI. +# +# See https://gitlab.com/eessi/support/-/issues/77 +easyconfigs: + - Python/3.10.8-GCCcore-12.2.0-bare + options: + include-easyblocks-from-pr: 3352 + - Python/3.10.8-GCCcore-12.2.0 + options: + include-easyblocks-from-pr: 3352 + - Python/3.11.3-GCCcore-12.3.0 + options: + include-easyblocks-from-pr: 3352 + - Python/3.11.5-GCCcore-13.2.0 + options: + include-easyblocks-from-pr: 3352 + - librosa-0.10.1-foss-2023a.eb From bfd12d220e9487939a5b6c1372d074a7290c579e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 29 Jul 2024 15:44:23 +0200 Subject: [PATCH 0950/1795] Finish with colon if we want to pass options --- .../2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml index 95bd27852a..6fa1b34992 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml @@ -6,16 +6,16 @@ # # See https://gitlab.com/eessi/support/-/issues/77 easyconfigs: - - Python/3.10.8-GCCcore-12.2.0-bare + - Python/3.10.8-GCCcore-12.2.0-bare: options: include-easyblocks-from-pr: 3352 - - Python/3.10.8-GCCcore-12.2.0 + - Python/3.10.8-GCCcore-12.2.0: options: include-easyblocks-from-pr: 3352 - - Python/3.11.3-GCCcore-12.3.0 + - Python/3.11.3-GCCcore-12.3.0: options: include-easyblocks-from-pr: 3352 - - Python/3.11.5-GCCcore-13.2.0 + - Python/3.11.5-GCCcore-13.2.0: options: include-easyblocks-from-pr: 3352 - librosa-0.10.1-foss-2023a.eb From 0f816f5a9e4081b3abccae0851f3a22cd9c187fd Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 29 Jul 2024 15:55:25 +0200 Subject: [PATCH 0951/1795] Move librosa to a regular EasyStack file, instead of a rebuilds --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + .../2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index d405d17193..8db39671ee 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -27,3 +27,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 from-commit: a92667fe32396bbd4106243658625f7ff2adcd68 - amdahl-0.3.1-gompi-2023a.eb + - librosa-0.10.1-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml index 6fa1b34992..09dde357cd 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml @@ -18,4 +18,3 @@ easyconfigs: - Python/3.11.5-GCCcore-13.2.0: options: include-easyblocks-from-pr: 3352 - - librosa-0.10.1-foss-2023a.eb From 66093b7a154bdbcd22bb5ce4dcbb6397951b4369 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 29 Jul 2024 16:24:01 +0200 Subject: [PATCH 0952/1795] Replace slash with dash to make this a proper easyconfig name, instead of a module name --- .../2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml index 09dde357cd..c73d67fcd2 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml @@ -6,15 +6,15 @@ # # See https://gitlab.com/eessi/support/-/issues/77 easyconfigs: - - Python/3.10.8-GCCcore-12.2.0-bare: + - Python-3.10.8-GCCcore-12.2.0-bare: options: include-easyblocks-from-pr: 3352 - - Python/3.10.8-GCCcore-12.2.0: + - Python-3.10.8-GCCcore-12.2.0: options: include-easyblocks-from-pr: 3352 - - Python/3.11.3-GCCcore-12.3.0: + - Python-3.11.3-GCCcore-12.3.0: options: include-easyblocks-from-pr: 3352 - - Python/3.11.5-GCCcore-13.2.0: + - Python-3.11.5-GCCcore-13.2.0: options: include-easyblocks-from-pr: 3352 From a2041a00cdf4847ea71db941139bb4d83250f4e4 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 30 Jul 2024 09:10:49 +0200 Subject: [PATCH 0953/1795] Update 20240729-eb-4.9.2-Python-ctypes.yml --- .../rebuilds/20240729-eb-4.9.2-Python-ctypes.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml index c73d67fcd2..7554289c3b 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240729-eb-4.9.2-Python-ctypes.yml @@ -8,13 +8,17 @@ easyconfigs: - Python-3.10.8-GCCcore-12.2.0-bare: options: - include-easyblocks-from-pr: 3352 + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f - Python-3.10.8-GCCcore-12.2.0: options: - include-easyblocks-from-pr: 3352 + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f - Python-3.11.3-GCCcore-12.3.0: options: - include-easyblocks-from-pr: 3352 + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f - Python-3.11.5-GCCcore-13.2.0: options: - include-easyblocks-from-pr: 3352 + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f From bbe101b0dda05658d2f0692f8d65132fa7ab6900 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 30 Jul 2024 13:16:15 +0200 Subject: [PATCH 0954/1795] Update eessi_archdetect.sh --- init/eessi_archdetect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 001683f50a..ad6dce6f9a 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -9,7 +9,7 @@ if [ -n "$BASH_VERSION" ]; then # Check if the major version is 4 or higher if [ "$major_version" -lt 4 ]; then - echo "Error: This script must be run with Bash > 4, you have $BASH_VERSION." >&2 + echo "Error: This script must be run with Bash >= 4, you have $BASH_VERSION." >&2 exit 1 fi else From 9a2e0a1532154e6654ff504fc0887aee8f1ae99f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 31 Jul 2024 10:50:16 +0200 Subject: [PATCH 0955/1795] Add SciTools-iris and xarray --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 2 ++ .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index ade226c217..2536c4bcc5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,2 +1,4 @@ easyconfigs: - BCFtools-1.18-GCC-12.3.0.eb + - xarray-2023.9.0-gfbf-2023a.eb + - SciTools-Iris-3.9.0-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 38bfa81142..39294717b6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,3 +1,4 @@ easyconfigs: - IPython-8.17.2-GCCcore-13.2.0.eb - dlb-3.4-gompi-2023b.eb + - xarray-2024.5.0-gfbf-2023b.eb From 1ba5d41de2cbfce6cc51fb6b14ebdcd28b99538f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 31 Jul 2024 16:08:41 +0200 Subject: [PATCH 0956/1795] Remove xarray for 2023b. There's no rush, and this one is not in eb 4.9.2 yet. --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 39294717b6..38bfa81142 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,4 +1,3 @@ easyconfigs: - IPython-8.17.2-GCCcore-13.2.0.eb - dlb-3.4-gompi-2023b.eb - - xarray-2024.5.0-gfbf-2023b.eb From f8ebe2036dc9f5baca1205ffb82d47e078aec506 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 1 Aug 2024 11:15:41 +0200 Subject: [PATCH 0957/1795] Update easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index c56b8e05fd..fdd1a80630 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -15,4 +15,4 @@ easyconfigs: - Highway-1.0.4-GCCcore-12.3.0.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20942 - from-pr: 20942 + from-commit: 524da37b903585cea5a9eeb4156d1c8d57636bd8 From 544b9beddc89f2d4b547822a883cac4a7383aa5d Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 1 Aug 2024 14:00:31 +0200 Subject: [PATCH 0958/1795] Update easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index fdd1a80630..5773087db3 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -11,7 +11,6 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f - #- Highway-1.0.4-GCCcore-12.3.0.eb - Highway-1.0.4-GCCcore-12.3.0.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20942 From 3931e33132179ff505f2b126a507fe14f09db125 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 1 Aug 2024 20:57:09 +0200 Subject: [PATCH 0959/1795] {2023.06,2023a+2023b,zen4} Rebuild Python + add librosa 0.10.1 --- .../20240801-eb-4.9.2-Python-ctypes-zen4.yml | 17 +++++++++++++++++ .../zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 2 files changed, 18 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240801-eb-4.9.2-Python-ctypes-zen4.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240801-eb-4.9.2-Python-ctypes-zen4.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240801-eb-4.9.2-Python-ctypes-zen4.yml new file mode 100644 index 0000000000..2104b4d836 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240801-eb-4.9.2-Python-ctypes-zen4.yml @@ -0,0 +1,17 @@ +# 2024.08.01 +# Python ctypes relies on LD_LIBRARY_PATH and doesn't respect rpath linking. There is a workaround +# for the EasyBuild context in https://github.com/easybuilders/easybuild-easyblocks/pull/3352. +# +# This rebuild ensures this fix is available for all Python versions shipped for +# zen4 with EESSI. +# +# See https://gitlab.com/eessi/support/-/issues/77 +easyconfigs: + - Python-3.11.3-GCCcore-12.3.0: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + - Python-3.11.5-GCCcore-13.2.0: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 5773087db3..825b1f5082 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -15,3 +15,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20942 from-commit: 524da37b903585cea5a9eeb4156d1c8d57636bd8 + - librosa-0.10.1-foss-2023a.eb From 448f0ab05440c4904a64cf5b83d9f340f349f54c Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 5 Aug 2024 06:52:05 +0000 Subject: [PATCH 0960/1795] {2023.06}[GCCcore/12.2.0][GCCcore/12.3.0] BioPerl v1.7.8 --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 1 + .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + eb_hooks.py | 11 +++++++++++ 3 files changed, 13 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 7a04eaa9d3..27a79d8c80 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -1,2 +1,3 @@ easyconfigs: - BLAST+-2.14.0-gompi-2022b.eb + - BioPerl-1.7.8-GCCcore-12.2.0.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 90dda5236b..e60cd83909 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -30,3 +30,4 @@ easyconfigs: - librosa-0.10.1-foss-2023a.eb - xarray-2023.9.0-gfbf-2023a.eb - SciTools-Iris-3.9.0-foss-2023a.eb + - BioPerl-1.7.8-GCCcore-12.3.0.eb diff --git a/eb_hooks.py b/eb_hooks.py index 7f7e4e57bd..eafcb5b417 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -647,6 +647,16 @@ def post_sanitycheck_hook(self, *args, **kwargs): POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) +def post_sanitycheck_bioperl(self, *args, **kwargs): + """ + Allow the copy of easyblock perlmodule.py to /tmp/eb... after sanity check by changing directory permissions + """ + if self.name == 'BioPerl': + print_msg("Adding user write permission for EasyBuild directories under /tmp/eb*...") + bioperlcmd = "chmod u+w -R /tmp/eb*" + run_cmd(bioperlcmd) + + def post_sanitycheck_cuda(self, *args, **kwargs): """ Remove files from CUDA installation that we are not allowed to ship, @@ -791,5 +801,6 @@ def inject_gpu_property(ec): } POST_SANITYCHECK_HOOKS = { + 'BioPerl': post_sanitycheck_bioperl, 'CUDA': post_sanitycheck_cuda, } From 4b37ea27c969fea88211761141238e4f65b90872 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 5 Aug 2024 12:42:33 +0000 Subject: [PATCH 0961/1795] removed BioPerl hook --- eb_hooks.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index eafcb5b417..7f7e4e57bd 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -647,16 +647,6 @@ def post_sanitycheck_hook(self, *args, **kwargs): POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) -def post_sanitycheck_bioperl(self, *args, **kwargs): - """ - Allow the copy of easyblock perlmodule.py to /tmp/eb... after sanity check by changing directory permissions - """ - if self.name == 'BioPerl': - print_msg("Adding user write permission for EasyBuild directories under /tmp/eb*...") - bioperlcmd = "chmod u+w -R /tmp/eb*" - run_cmd(bioperlcmd) - - def post_sanitycheck_cuda(self, *args, **kwargs): """ Remove files from CUDA installation that we are not allowed to ship, @@ -801,6 +791,5 @@ def inject_gpu_property(ec): } POST_SANITYCHECK_HOOKS = { - 'BioPerl': post_sanitycheck_bioperl, 'CUDA': post_sanitycheck_cuda, } From 9bf2ce3824d34d5eff7f80000f1916ddc6054fba Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 5 Aug 2024 18:02:12 +0200 Subject: [PATCH 0962/1795] Add modflow for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 09f525364f..959d4fe92d 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -2,3 +2,4 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb - TensorFlow-2.13.0-foss-2023a.eb - R-4.3.2-gfbf-2023a.eb + - MODFLOW-6.4.4-foss-2023a.eb From b2341ff68e5bdfeb3189c92e342fec3ef2653e28 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 6 Aug 2024 11:55:15 +0200 Subject: [PATCH 0963/1795] {2023.06}[foss/2023a] grpcio v1.57.0 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + eb_hooks.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 90dda5236b..a8a38ec80c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -30,3 +30,4 @@ easyconfigs: - librosa-0.10.1-foss-2023a.eb - xarray-2023.9.0-gfbf-2023a.eb - SciTools-Iris-3.9.0-foss-2023a.eb + - grpcio-1.57.0-GCCcore-12.3.0.eb diff --git a/eb_hooks.py b/eb_hooks.py index 7f7e4e57bd..13b619344a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -220,6 +220,19 @@ def parse_hook_fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") +def parse_hook_grpcio_zlib(ec, ecprefix): + """Adjust preinstallopts to use ZLIB from compat layer.""" + if ec.name == 'grpcio' and ec.version in ['1.57.0']: + exts_list = ec['exts_list'] + original_preinstallopts = (exts_list[0][2])['preinstallopts'] + original_option = "GRPC_PYTHON_BUILD_SYSTEM_ZLIB=True" + new_option = "GRPC_PYTHON_BUILD_SYSTEM_ZLIB=False" + (exts_list[0][2])['preinstallopts'] = original_preinstallopts.replace(original_option, new_option, 1) + print_msg("Modified the easyconfig to use compat ZLIB with GRPC_PYTHON_BUILD_SYSTEM_ZLIB=False") + else: + raise EasyBuildError("grpcio-specific hook triggered for a non-grpcio easyconfig?!") + + def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix): """Relax number of failing numerical LAPACK tests for aarch64/neoverse_v1 CPU target for OpenBLAS < 0.3.23""" cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') From e194d2a88a2b38879ac816a392ff77c145480583 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 6 Aug 2024 11:28:00 +0000 Subject: [PATCH 0964/1795] Added --include-easyblock and an issue reference to EasyBuild framework --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 5 ++++- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 27a79d8c80..1c4ea68b3f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -1,3 +1,6 @@ easyconfigs: - BLAST+-2.14.0-gompi-2022b.eb - - BioPerl-1.7.8-GCCcore-12.2.0.eb + - BioPerl-1.7.8-GCCcore-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-framework/issues/4602 + include-easyblocks-from-commit: 89ea2e4737d792055de828e18f38bccee26b76cf diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e60cd83909..a5b425dc09 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -30,4 +30,7 @@ easyconfigs: - librosa-0.10.1-foss-2023a.eb - xarray-2023.9.0-gfbf-2023a.eb - SciTools-Iris-3.9.0-foss-2023a.eb - - BioPerl-1.7.8-GCCcore-12.3.0.eb + - BioPerl-1.7.8-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-framework/issues/4602 + include-easyblocks-from-commit: 89ea2e4737d792055de828e18f38bccee26b76cf From 475653cd5a84f8ac8608dbc1fc471404ed68f4cd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 6 Aug 2024 13:38:22 +0200 Subject: [PATCH 0965/1795] enable parse hook for grpcio --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index 13b619344a..73e1ff7df6 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -756,6 +756,7 @@ def inject_gpu_property(ec): 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, + 'grpcio': parse_hook_grpcio_zlib, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, From 448e6b30c47dbea249c0ed6f89880ed229f3e659 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 6 Aug 2024 16:16:59 +0200 Subject: [PATCH 0966/1795] Apply suggestions from code review --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 4 ++-- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 1c4ea68b3f..140876f43b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -2,5 +2,5 @@ easyconfigs: - BLAST+-2.14.0-gompi-2022b.eb - BioPerl-1.7.8-GCCcore-12.2.0.eb: options: - # see https://github.com/easybuilders/easybuild-framework/issues/4602 - include-easyblocks-from-commit: 89ea2e4737d792055de828e18f38bccee26b76cf + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 14dc2a17bc..e07ed409a4 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -38,5 +38,5 @@ easyconfigs: from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 - BioPerl-1.7.8-GCCcore-12.3.0.eb: options: - # see https://github.com/easybuilders/easybuild-framework/issues/4602 - include-easyblocks-from-commit: 89ea2e4737d792055de828e18f38bccee26b76cf + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc From dc0d5f8fc258257fe24c3dbaffee13a21c8cc48f Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 7 Aug 2024 07:29:00 +0000 Subject: [PATCH 0967/1795] {2023.06}[GCCcore/12.2.0] gnuplot v5.4.6 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 140876f43b..d0f9f66b06 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -3,4 +3,5 @@ easyconfigs: - BioPerl-1.7.8-GCCcore-12.2.0.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 - from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + - gnuplot-5.4.6-GCCcore-12.2.0.eb From 7f2bd7cc87de25c1c2d90d8ae9dd76b6153e2201 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 7 Aug 2024 07:45:45 +0000 Subject: [PATCH 0968/1795] {2023.06,zen4}[2023a] BioPerl 1.7.8 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 79d6136af9..871958f363 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -17,3 +17,7 @@ easyconfigs: from-commit: 524da37b903585cea5a9eeb4156d1c8d57636bd8 - librosa-0.10.1-foss-2023a.eb - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb + - BioPerl-1.7.8-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc From 92767a4421203df209084671931ce54c64718896 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 8 Aug 2024 08:08:40 +0000 Subject: [PATCH 0969/1795] {2023.06}[foss/2022b] h5py v3.8.0 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index d0f9f66b06..4e236656f5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -4,4 +4,5 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - - gnuplot-5.4.6-GCCcore-12.2.0.eb + - gnuplot-5.4.6-GCCcore-12.2.0.eb + - h5py-3.8.0-foss-2022b.eb From f9bbb59439fc811da28c690548f22046c99f1778 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 8 Aug 2024 10:25:15 +0200 Subject: [PATCH 0970/1795] Make sure the new script is shipped --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index ab06e47997..6f01818840 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -86,7 +86,7 @@ TOPDIR=$(dirname $(realpath $0)) # Copy for init directory init_files=( bash eessi_archdetect.sh eessi_defaults eessi_environment_variables eessi_software_subdir_for_host.py - minimal_eessi_env README.md test.py + minimal_eessi_env README.md test.py lmod_eessi_archdetect_wrapper.sh ) copy_files_by_list ${TOPDIR}/init ${INSTALL_PREFIX}/init "${init_files[@]}" From 3cf558b03b83b8485fa28d42abbe7def64d3fdee Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 9 Aug 2024 08:02:41 +0000 Subject: [PATCH 0971/1795] {2023.06}[foss/2022b] MDAnalysis v2.4.2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index d0f9f66b06..bd928e3608 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -4,4 +4,5 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - - gnuplot-5.4.6-GCCcore-12.2.0.eb + - gnuplot-5.4.6-GCCcore-12.2.0.eb + - MDAnalysis-2.4.2-foss-2022b.eb From a10f8198703d84b912c4ac5859c4be6e9c486f8a Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 9 Aug 2024 09:32:34 +0000 Subject: [PATCH 0972/1795] {2023.06}[GCCcore/12.3.0] orjson v3.9.15 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 0c08646cc8..e8fe8ab120 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -41,3 +41,7 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - grpcio-1.57.0-GCCcore-12.3.0.eb + - orjson-3.9.15-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 + from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 From 19e1bd201b1a718d899a0407db9118e1874247d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 9 Aug 2024 15:05:22 +0200 Subject: [PATCH 0973/1795] {2023.06,a64fx}[foss/2023a] OpenFOAM 10, 11, v2312 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index 5521b92398..f38c06ea8b 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -4,3 +4,17 @@ easyconfigs: - SciPy-bundle-2023.07-gfbf-2023a.eb - ESPResSo-4.2.2-foss-2023a.eb - ParaView-5.11.2-foss-2023a.eb + - OpenFOAM-10-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - OpenFOAM-11-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - OpenFOAM-v2312-foss-2023a.eb: + options: + # https://github.com/easybuilders/easybuild-easyblocks/pull/3388 + include-easyblocks-from-commit: c8256a36e7062bc09f5ce30552a9de9827054c9e + # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841 + from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 From 7ae01febe12ee76c334af831ccdbebb91bf8266d Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 12 Aug 2024 07:20:27 +0000 Subject: [PATCH 0974/1795] EESSI bash initialization to module file --- init/modules/EESSI/2023.06.lua | 55 ++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 init/modules/EESSI/2023.06.lua diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua new file mode 100644 index 0000000000..fcc4249ca1 --- /dev/null +++ b/init/modules/EESSI/2023.06.lua @@ -0,0 +1,55 @@ +help([[ +Description +=========== +The European Environment for Scientific Software Installations (EESSI, pronounced as easy) is a collaboration between different European partners in HPC community.The goal of this project is to build a common stack of scientific software installations for HPC systems and beyond, including laptops, personal workstations and cloud infrastructure. + +More information +================ + - URL: https://www.eessi.io/docs/ +]]) +whatis("Description: The European Environment for Scientific Software Installations (EESSI, pronounced as easy) is a collaboration between different European partners in HPC community. The goal of this project is to build a common stack of scientific software installations for HPC systems and beyond, including laptops, personal workstations and cloud infrastructure.") +whatis("URL: https://www.eessi.io/docs/:") + +local eessi_version = myModuleVersion() +local eessi_repo = "/cvmfs/software.eessi.io" +local eessi_prefix = pathJoin(eessi_repo, "versions", eessi_version) +local eessi_os_type = "linux" +setenv("EESSI_VERSION", eessi_version) +setenv("EESSI_CVMFS_REPO", eessi_repo) +setenv("EESSI_OS_TYPE", eessi_os_type) +function archdetect_cpu() + local script = pathJoin(eessi_repo, 'versions', eessi_version, 'init', 'lmod_eessi_archdetect_wrapper.sh') + if not os.getenv("EESSI_ARCHDETECT_OPTIONS") then + if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then + LmodMessage("Loading this modulefile requires using Lmod version > 8.6, but you can export EESSI_ARCHDETECT_OPTIONS to the available cpu architecture in the form of: x86_64/intel/haswell or aarch64/neoverse_v1") + os.exit(1) + end + source_sh("bash", script) + end + for archdetect_filter_cpu in string.gmatch(os.getenv("EESSI_ARCHDETECT_OPTIONS"), "([^" .. ":" .. "]+)") do + if isDir(pathJoin(string.gsub(script, "init/lmod_eessi_archdetect_wrapper.sh", "software/" .. eessi_os_type), archdetect_filter_cpu)) then + return archdetect_filter_cpu + end + end + LmodError("Software directory check for the detected architecture failed") +end +local archdetect = archdetect_cpu() +local eessi_cpu_family = archdetect:match("([^/]+)") +local eessi_software_subdir = os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE") or archdetect +local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family) +local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) +local eessi_module_path = pathJoin(eessi_software_path, "modules/all") +setenv("EESSI_SITE_MODULEPATH", string.gsub(eessi_module_path, eessi_repo, "host_injections")) +setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir) +setenv("EESSI_PREFIX", eessi_prefix) +setenv("EESSI_EPREFIX", eessi_eprefix) +prepend_path("PATH", pathJoin(eessi_eprefix, "bin")) +prepend_path("PATH", pathJoin(eessi_eprefix, "usr/bin")) +setenv("EESSI_SOFTWARE_PATH", eessi_software_path) +setenv("EESSI_MODULEPATH", eessi_module_path) +prepend_path("MODULEPATH", os.getenv("EESSI_SITE_MODULEPATH") .. ":" .. os.getenv("EESSI_MODULEPATH")) +setenv("LMOD_CONFIG_DIR", pathJoin(eessi_software_path, ".lmod")) +setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) +if mode() == "load" then + LmodMessage("EESSI/" .. eessi_version .. " loaded successfully") +end From b099220c3a68d1643d872a4dd15f7adb9a62e84e Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:54:09 +0200 Subject: [PATCH 0975/1795] Update init/modules/EESSI/2023.06.lua Co-authored-by: ocaisa --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index fcc4249ca1..59c941efe2 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -18,7 +18,7 @@ setenv("EESSI_VERSION", eessi_version) setenv("EESSI_CVMFS_REPO", eessi_repo) setenv("EESSI_OS_TYPE", eessi_os_type) function archdetect_cpu() - local script = pathJoin(eessi_repo, 'versions', eessi_version, 'init', 'lmod_eessi_archdetect_wrapper.sh') + local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper.sh') if not os.getenv("EESSI_ARCHDETECT_OPTIONS") then if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then LmodMessage("Loading this modulefile requires using Lmod version > 8.6, but you can export EESSI_ARCHDETECT_OPTIONS to the available cpu architecture in the form of: x86_64/intel/haswell or aarch64/neoverse_v1") From 2bfb7afbcd13e7915ab33e10eb68e5fa9d92353f Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:54:50 +0200 Subject: [PATCH 0976/1795] Update init/modules/EESSI/2023.06.lua Co-authored-by: ocaisa --- init/modules/EESSI/2023.06.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 59c941efe2..5180235b06 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -21,8 +21,7 @@ function archdetect_cpu() local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper.sh') if not os.getenv("EESSI_ARCHDETECT_OPTIONS") then if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then - LmodMessage("Loading this modulefile requires using Lmod version > 8.6, but you can export EESSI_ARCHDETECT_OPTIONS to the available cpu architecture in the form of: x86_64/intel/haswell or aarch64/neoverse_v1") - os.exit(1) + LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ARCHDETECT_OPTIONS to the available cpu architecture in the form of: x86_64/intel/haswell:x86_64/generic or aarch64/neoverse_v1:aarch64/generic") end source_sh("bash", script) end From e302e66c94110e3b039f6a6f3bf26ee6653999e0 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:56:04 +0200 Subject: [PATCH 0977/1795] Update init/modules/EESSI/2023.06.lua Co-authored-by: ocaisa --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 5180235b06..1a2c3a0d64 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -26,7 +26,7 @@ function archdetect_cpu() source_sh("bash", script) end for archdetect_filter_cpu in string.gmatch(os.getenv("EESSI_ARCHDETECT_OPTIONS"), "([^" .. ":" .. "]+)") do - if isDir(pathJoin(string.gsub(script, "init/lmod_eessi_archdetect_wrapper.sh", "software/" .. eessi_os_type), archdetect_filter_cpu)) then + if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then return archdetect_filter_cpu end end From 34d4d42043cc44124d0fe93dd7de2ff875402308 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:56:28 +0200 Subject: [PATCH 0978/1795] Update init/modules/EESSI/2023.06.lua Co-authored-by: ocaisa --- init/modules/EESSI/2023.06.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 1a2c3a0d64..d0d7e7bc5f 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -37,7 +37,8 @@ local eessi_cpu_family = archdetect:match("([^/]+)") local eessi_software_subdir = os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE") or archdetect local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family) local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) -local eessi_module_path = pathJoin(eessi_software_path, "modules/all") +local eessi_module_path = pathJoin(eessi_software_path, "modules", "all") +local eessi_site_module_path = string.gsub(eessi_module_path, "versions", "host_injections") setenv("EESSI_SITE_MODULEPATH", string.gsub(eessi_module_path, eessi_repo, "host_injections")) setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir) setenv("EESSI_PREFIX", eessi_prefix) From 6c23ec1a9358c701481606a4cfe6c9f583365569 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:57:26 +0200 Subject: [PATCH 0979/1795] Update init/modules/EESSI/2023.06.lua Co-authored-by: ocaisa --- init/modules/EESSI/2023.06.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index d0d7e7bc5f..5f74b969c6 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -47,7 +47,8 @@ prepend_path("PATH", pathJoin(eessi_eprefix, "bin")) prepend_path("PATH", pathJoin(eessi_eprefix, "usr/bin")) setenv("EESSI_SOFTWARE_PATH", eessi_software_path) setenv("EESSI_MODULEPATH", eessi_module_path) -prepend_path("MODULEPATH", os.getenv("EESSI_SITE_MODULEPATH") .. ":" .. os.getenv("EESSI_MODULEPATH")) +prepend_path("MODULEPATH", eessi_module_path) +prepend_path("MODULEPATH", eessi_site_module_path) setenv("LMOD_CONFIG_DIR", pathJoin(eessi_software_path, ".lmod")) setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) if mode() == "load" then From f3b29a841467f947acc14f7086031a1cd9894d5e Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:57:48 +0200 Subject: [PATCH 0980/1795] Update init/modules/EESSI/2023.06.lua Co-authored-by: ocaisa --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 5f74b969c6..989734f361 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -39,7 +39,7 @@ local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_ local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) local eessi_module_path = pathJoin(eessi_software_path, "modules", "all") local eessi_site_module_path = string.gsub(eessi_module_path, "versions", "host_injections") -setenv("EESSI_SITE_MODULEPATH", string.gsub(eessi_module_path, eessi_repo, "host_injections")) +setenv("EESSI_SITE_MODULEPATH", eessi_site_module_path) setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir) setenv("EESSI_PREFIX", eessi_prefix) setenv("EESSI_EPREFIX", eessi_eprefix) From d51267543584dfd8b8533ab5791092817241ee5d Mon Sep 17 00:00:00 2001 From: Matti Kaupenjohann Date: Mon, 12 Aug 2024 13:37:05 +0200 Subject: [PATCH 0981/1795] Add init scripts - zsh - fish --- init/fish | 9 +++++++++ init/zsh | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 init/fish create mode 100644 init/zsh diff --git a/init/fish b/init/fish new file mode 100644 index 0000000000..de51c41a4a --- /dev/null +++ b/init/fish @@ -0,0 +1,9 @@ +# Choose an EESSI version +set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") +# Automatically load the module for EESSI version +set -x LMOD_SYSTEM_DEFAULT_MODULES (set -q LMOD_SYSTEM_DEFAULT_MODULE; and echo "$LMOD_SYSTEM_DEFAULT_MODULE"; or echo "EESSI/${EESSI_VERSION}") +# Path to top-level module tree +set -x MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules +. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh + +module load $LMOD_SYSTEM_DEFAULT_MODULES diff --git a/init/zsh b/init/zsh new file mode 100644 index 0000000000..b11c08459b --- /dev/null +++ b/init/zsh @@ -0,0 +1,9 @@ +# Choose an EESSI version +EESSI_VERSION="${EESSI_VERSION:-2023.06}" +# Automatically load the module for EESSI version +export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI_VERSION}}" +# Path to top-level module tree +export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules +. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh + +module load "$LMOD_SYSTEM_DEFAULT_MODULES" From 7f71c28e03c71c4d93b13c6dac6ed8adf9d1bc12 Mon Sep 17 00:00:00 2001 From: makanu Date: Mon, 12 Aug 2024 12:34:14 +0000 Subject: [PATCH 0982/1795] Fix fish init script variable setting and calling --- init/fish | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/init/fish b/init/fish index de51c41a4a..f391dc366b 100644 --- a/init/fish +++ b/init/fish @@ -1,9 +1,9 @@ # Choose an EESSI version set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") # Automatically load the module for EESSI version -set -x LMOD_SYSTEM_DEFAULT_MODULES (set -q LMOD_SYSTEM_DEFAULT_MODULE; and echo "$LMOD_SYSTEM_DEFAULT_MODULE"; or echo "EESSI/${EESSI_VERSION}") +set -x LMOD_SYSTEM_DEFAULT_MODULES (set -q LMOD_SYSTEM_DEFAULT_MODULE; and echo "$LMOD_SYSTEM_DEFAULT_MODULE"; or echo "EESSI/$EESSI_VERSION") # Path to top-level module tree -set -x MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules -. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh +set -x MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules +. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/fish module load $LMOD_SYSTEM_DEFAULT_MODULES From 1e2f154bb45c13688066863a5d5c3f75a4529164 Mon Sep 17 00:00:00 2001 From: makanu Date: Mon, 12 Aug 2024 12:35:26 +0000 Subject: [PATCH 0983/1795] Add init script - ksh --- init/ksh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 init/ksh diff --git a/init/ksh b/init/ksh new file mode 100644 index 0000000000..d6cefe85c4 --- /dev/null +++ b/init/ksh @@ -0,0 +1,9 @@ +# Choose an EESSI version +EESSI_VERSION="${EESSI_VERSION:-2023.06}" +# Automatically load the module for EESSI version +export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI_VERSION}}" +# Path to top-level module tree +export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules +. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/ksh + +module load "$LMOD_SYSTEM_DEFAULT_MODULES" From a0c4377e90989b0190b629447cbfcd853480aa9d Mon Sep 17 00:00:00 2001 From: makanu Date: Mon, 12 Aug 2024 12:35:44 +0000 Subject: [PATCH 0984/1795] convert bash init script --- init/bash | 50 +++++++++----------------------------------------- 1 file changed, 9 insertions(+), 41 deletions(-) diff --git a/init/bash b/init/bash index 4ad09f6a1b..468237a36f 100644 --- a/init/bash +++ b/init/bash @@ -1,41 +1,9 @@ -function show_msg { - # only echo msg if EESSI_SILENT is unset - msg=$1 - if [[ -z ${EESSI_SILENT+x} ]]; then - echo "$msg" - fi -} - -# The following method should be safe, but might break if file is a symlink -# (could switch to $(dirname "$(readlink -f "$BASH_SOURCE")") in that case) -source $(dirname "$BASH_SOURCE")/eessi_environment_variables - -# only continue if setting EESSI environment variables worked fine -if [ $? -eq 0 ]; then - - export PS1="{EESSI $EESSI_VERSION} $PS1" - - # add location of commands provided by compat layer to $PATH; - # see https://github.com/EESSI/software-layer/issues/52 - export PATH=$EPREFIX/usr/bin:$EPREFIX/bin:$PATH - - # init Lmod - show_msg "Initializing Lmod..." - source $EESSI_EPREFIX/usr/share/Lmod/init/bash - - # prepend location of modules for EESSI software stack to $MODULEPATH - show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." - module use $EESSI_MODULEPATH - show_msg "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." - module use $EESSI_SITE_MODULEPATH - - #show_msg "" - #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***" - #show_msg "" - #show_msg "1) ..." - #show_msg "" - #show_msg "" - - echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!" - -fi +# Choose an EESSI version +EESSI_VERSION="${EESSI_VERSION:-2023.06}" +# Automatically load the module for EESSI version +export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI_VERSION}}" +# Path to top-level module tree +export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules +. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/bash + +module load "$LMOD_SYSTEM_DEFAULT_MODULES" From 68aea9fe6ae192ccfa15299ed97833579dcae659 Mon Sep 17 00:00:00 2001 From: makanu Date: Mon, 12 Aug 2024 12:46:34 +0000 Subject: [PATCH 0985/1795] Add init script - csh --- init/csh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 init/csh diff --git a/init/csh b/init/csh new file mode 100644 index 0000000000..c43a50c412 --- /dev/null +++ b/init/csh @@ -0,0 +1,10 @@ +# Choose an EESSI version +if (! $?EESSI_VERSION) then; set EESSI_VERSION = "2023.06"; endif +# Automatically load the module for EESSI version +if (! $?LMOD_SYSTEM_DEFAULT_MODULES) then; setenv LMOD_SYSTEM_DEFAULT_MODULES "EESSI/$EESSI_VERSION"; endif +# Path to top-level module tree +setenv MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules +source /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/`uname -m`/usr/share/Lmod/init/csh + +module load "$LMOD_SYSTEM_DEFAULT_MODULES" + From 8ead05fbe52b59a33645c2a986c29824f10f9358 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:12:00 +0200 Subject: [PATCH 0986/1795] Update 2023.06.lua Added zen4 redirection --- init/modules/EESSI/2023.06.lua | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 989734f361..c6091c5be6 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -27,6 +27,15 @@ function archdetect_cpu() end for archdetect_filter_cpu in string.gmatch(os.getenv("EESSI_ARCHDETECT_OPTIONS"), "([^" .. ":" .. "]+)") do if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then + -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, + -- since optimized software installations for Zen4 are a work-in-progress, + -- see https://gitlab.com/eessi/support/-/issues/37 + if archdetect_filter_cpu == "x86_64/amd/zen4" then + archdetect_filter_cpu = "x86_64/amd/zen3" + if mode() == "load" then + LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.") + end + end return archdetect_filter_cpu end end From 788375d16ce0e34a89461966011445dce9a8f2a3 Mon Sep 17 00:00:00 2001 From: makanu Date: Mon, 12 Aug 2024 14:06:46 +0000 Subject: [PATCH 0987/1795] Export ${debian_chroot:+($debian_chroot)}\u@\h:\w\$ for all init scripts --- init/bash | 1 + init/csh | 1 + init/fish | 1 + init/ksh | 1 + init/zsh | 1 + 5 files changed, 5 insertions(+) diff --git a/init/bash b/init/bash index 468237a36f..4cd3fbe701 100644 --- a/init/bash +++ b/init/bash @@ -6,4 +6,5 @@ export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/bash +export PS1=$PS1 module load "$LMOD_SYSTEM_DEFAULT_MODULES" diff --git a/init/csh b/init/csh index c43a50c412..e71f4a9e81 100644 --- a/init/csh +++ b/init/csh @@ -6,5 +6,6 @@ if (! $?LMOD_SYSTEM_DEFAULT_MODULES) then; setenv LMOD_SYSTEM_DEFAULT_MODULES "E setenv MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules source /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/`uname -m`/usr/share/Lmod/init/csh +setenv PS1 $PS1 module load "$LMOD_SYSTEM_DEFAULT_MODULES" diff --git a/init/fish b/init/fish index f391dc366b..f5659edcf4 100644 --- a/init/fish +++ b/init/fish @@ -6,4 +6,5 @@ set -x LMOD_SYSTEM_DEFAULT_MODULES (set -q LMOD_SYSTEM_DEFAULT_MODULE; and echo set -x MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/fish +set -x PS1 $PS1 module load $LMOD_SYSTEM_DEFAULT_MODULES diff --git a/init/ksh b/init/ksh index d6cefe85c4..a849f1b0d9 100644 --- a/init/ksh +++ b/init/ksh @@ -6,4 +6,5 @@ export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/ksh +export PS1=$PS1 module load "$LMOD_SYSTEM_DEFAULT_MODULES" diff --git a/init/zsh b/init/zsh index b11c08459b..e3efbd87e2 100644 --- a/init/zsh +++ b/init/zsh @@ -6,4 +6,5 @@ export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh +export PS1=$PS1 module load "$LMOD_SYSTEM_DEFAULT_MODULES" From 824ffafcc7e3a77089b484ca935be4a294af1975 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:58:34 +0200 Subject: [PATCH 0988/1795] Update 2023.06.lua Added EPREFIX + EESSI_CPU_FAMILY env vars --- init/modules/EESSI/2023.06.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index c6091c5be6..d39fa20a4b 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -48,6 +48,8 @@ local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_ local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) local eessi_module_path = pathJoin(eessi_software_path, "modules", "all") local eessi_site_module_path = string.gsub(eessi_module_path, "versions", "host_injections") +setenv("EPREFIX", eessi_eprefix) +setenv("EESSI_CPU_FAMILY", eessi_cpu_family) setenv("EESSI_SITE_MODULEPATH", eessi_site_module_path) setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir) setenv("EESSI_PREFIX", eessi_prefix) From b14cef843a13e51c2871fd9a4d8e4d7e74cb349a Mon Sep 17 00:00:00 2001 From: makanu Date: Tue, 13 Aug 2024 21:17:39 +0000 Subject: [PATCH 0989/1795] Implement module load following doc --- init/bash | 15 +++++++++++---- init/csh | 15 +++++++++++---- init/fish | 14 ++++++++++---- init/ksh | 15 +++++++++++---- init/zsh | 15 +++++++++++---- 5 files changed, 54 insertions(+), 20 deletions(-) diff --git a/init/bash b/init/bash index 4cd3fbe701..0451601679 100644 --- a/init/bash +++ b/init/bash @@ -1,10 +1,17 @@ # Choose an EESSI version EESSI_VERSION="${EESSI_VERSION:-2023.06}" -# Automatically load the module for EESSI version -export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI_VERSION}}" # Path to top-level module tree export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/bash -export PS1=$PS1 -module load "$LMOD_SYSTEM_DEFAULT_MODULES" +export PS1 +if [ -z "$__Init_Default_Modules" ]; then + export __Init_Default_Modules=1; + + ## ability to predefine elsewhere the default list + LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"StdEnv"} + export LMOD_SYSTEM_DEFAULT_MODULES + module --initial_load --no_redirect restore +else + module refresh +fi diff --git a/init/csh b/init/csh index e71f4a9e81..920581fccb 100644 --- a/init/csh +++ b/init/csh @@ -1,11 +1,18 @@ # Choose an EESSI version if (! $?EESSI_VERSION) then; set EESSI_VERSION = "2023.06"; endif -# Automatically load the module for EESSI version -if (! $?LMOD_SYSTEM_DEFAULT_MODULES) then; setenv LMOD_SYSTEM_DEFAULT_MODULES "EESSI/$EESSI_VERSION"; endif # Path to top-level module tree setenv MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules source /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/`uname -m`/usr/share/Lmod/init/csh -setenv PS1 $PS1 -module load "$LMOD_SYSTEM_DEFAULT_MODULES" +setenv PS1 $prompt +if (! $?__Init_Default_Modules ) then + setenv __Init_Default_Modules 1; + + ## ability to predefine elsewhere the default list + if (! $?LMOD_SYSTEM_DEFAULT_MODULES) then; setenv LMOD_SYSTEM_DEFAULT_MODULES "StdEnv"; endif + module --initial_load --no_redirect restore +else + module refresh +endif +#module load "$LMOD_SYSTEM_DEFAULT_MODULES" diff --git a/init/fish b/init/fish index f5659edcf4..9ae68b39dc 100644 --- a/init/fish +++ b/init/fish @@ -1,10 +1,16 @@ # Choose an EESSI version set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") -# Automatically load the module for EESSI version -set -x LMOD_SYSTEM_DEFAULT_MODULES (set -q LMOD_SYSTEM_DEFAULT_MODULE; and echo "$LMOD_SYSTEM_DEFAULT_MODULE"; or echo "EESSI/$EESSI_VERSION") # Path to top-level module tree set -x MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/fish -set -x PS1 $PS1 -module load $LMOD_SYSTEM_DEFAULT_MODULES +export PS1 +if test -z "$__Init_Default_Modules" + export __Init_Default_Modules=1; + + ## ability to predefine elsewhere the default list + set -x LMOD_SYSTEM_DEFAULT_MODULES (set -q LMOD_SYSTEM_DEFAULT_MODULE; and echo "$LMOD_SYSTEM_DEFAULT_MODULE"; or echo "StdEnv") + module --initial_load --no_redirect restore +else + module refresh +end diff --git a/init/ksh b/init/ksh index a849f1b0d9..e4eaafb2e6 100644 --- a/init/ksh +++ b/init/ksh @@ -1,10 +1,17 @@ # Choose an EESSI version EESSI_VERSION="${EESSI_VERSION:-2023.06}" -# Automatically load the module for EESSI version -export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI_VERSION}}" # Path to top-level module tree export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/ksh -export PS1=$PS1 -module load "$LMOD_SYSTEM_DEFAULT_MODULES" +export PS1 +if [ -z "$__Init_Default_Modules" ]; then + export __Init_Default_Modules=1; + + ## ability to predefine elsewhere the default list + LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"StdEnv"} + export LMOD_SYSTEM_DEFAULT_MODULES + module --initial_load --no_redirect restore +else + module refresh +fi diff --git a/init/zsh b/init/zsh index e3efbd87e2..90ffeff4ef 100644 --- a/init/zsh +++ b/init/zsh @@ -1,10 +1,17 @@ # Choose an EESSI version EESSI_VERSION="${EESSI_VERSION:-2023.06}" -# Automatically load the module for EESSI version -export LMOD_SYSTEM_DEFAULT_MODULES="${LMOD_SYSTEM_DEFAULT_MODULES:-EESSI/${EESSI_VERSION}}" # Path to top-level module tree export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh -export PS1=$PS1 -module load "$LMOD_SYSTEM_DEFAULT_MODULES" +export PS1 +if [ -z "$__Init_Default_Modules" ]; then + export __Init_Default_Modules=1; + + ## ability to predefine elsewhere the default list + LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"StdEnv"} + export LMOD_SYSTEM_DEFAULT_MODULES + module --initial_load --no_redirect restore +else + module refresh +fi From b85e37b988ac2e7a9ae67fd2506638b48bd7b2ae Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 14 Aug 2024 11:58:58 +0000 Subject: [PATCH 0990/1795] Added EESSI/2023.06 to install_scripts.sh --- install_scripts.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/install_scripts.sh b/install_scripts.sh index 6f01818840..0cfa5310c2 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -102,6 +102,12 @@ mc_files=( ) copy_files_by_list ${TOPDIR}/init/Magic_Castle ${INSTALL_PREFIX}/init/Magic_Castle "${mc_files[@]}" +# Copy for init/modules/EESSI directory +mc_files=( + 2023.06.lua +) +copy_files_by_list ${TOPDIR}/init/modules/EESSI ${INSTALL_PREFIX}/init/modules/EESSI "${mc_files[@]}" + # Copy for the scripts directory script_files=( utils.sh From 4fd3d47b36a6d714344c52b80bcf1205afa958ab Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 14 Aug 2024 17:15:45 +0200 Subject: [PATCH 0991/1795] Rebuild hatchling-1.18.0-GCCcore-12.3.0 --- ...0814-eb-4.9.2-hatchling-1.18.0-updated-easyconfig.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240814-eb-4.9.2-hatchling-1.18.0-updated-easyconfig.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240814-eb-4.9.2-hatchling-1.18.0-updated-easyconfig.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240814-eb-4.9.2-hatchling-1.18.0-updated-easyconfig.yml new file mode 100644 index 0000000000..7ab02420ca --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240814-eb-4.9.2-hatchling-1.18.0-updated-easyconfig.yml @@ -0,0 +1,9 @@ +# 2024.08.14 +# hatchling-1.18.0 rebuild to account for easyconfig changed upstream +# see https://gitlab.com/eessi/support/-/issues/85 and +# https://github.com/easybuilders/easybuild-easyconfigs/pull/20389 +easyconfigs: + - hatchling-1.18.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20389 + from-commit: 9580c0d67d6dd97b160b768a839bfcba6d5b21b9 From 53d5e26d84f57ecc425f2c6a441a793df83930a7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 14 Aug 2024 17:32:29 +0200 Subject: [PATCH 0992/1795] Fix test_suite not being able to retrieve memory on systems that use cgroups v2 --- test_suite.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 6e73fbd87c..cfaec82026 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -157,7 +157,13 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi -cgroup_mem_bytes=$(cat /hostsys/fs/cgroup/memory/slurm/uid_${UID}/job_${SLURM_JOB_ID}/memory.limit_in_bytes) +cgroup_v1_mem_limit="/sys/fs/cgroup/memory/$( Date: Wed, 14 Aug 2024 17:36:40 +0200 Subject: [PATCH 0993/1795] Add something in order to prove that this fixes the test --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 9d13d5eb3a..695c4a06d9 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -22,3 +22,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - MODFLOW-6.4.4-foss-2023a.eb + - gnuplot-5.4.8-GCCcore-12.3.0.eb From 4e52f59e1e90988a3721e8bf768a1f8d60182c6b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 14 Aug 2024 17:41:19 +0200 Subject: [PATCH 0994/1795] Fix if-else syntax --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index cfaec82026..5749e5c995 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -161,7 +161,7 @@ cgroup_v1_mem_limit="/sys/fs/cgroup/memory/$( Date: Thu, 15 Aug 2024 11:46:03 +0200 Subject: [PATCH 0995/1795] If max is specified in the cgroupsv2 file, just get the system memory --- test_suite.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 5749e5c995..02b22361db 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -163,10 +163,19 @@ if [ -f "$cgroup_v1_mem_limit" ]; then cgroup_mem_bytes=$(cat "$cgroup_v1_mem_limit") else cgroup_mem_bytes=$(cat "$cgroup_v2_mem_limit") + if [ "$cgroup_mem_bytes" = 'max' ]; then + # In cgroupsv2, the memory.max file may contain 'max', meaning the group can use the full system memory + # Here, we get the system memory from /proc/meminfo. Units are supposedly always in kb, but lets match them too + cgroup_mem_kilobytes=$(grep -oP 'MemTotal:\s+\K\d+(?=\s+kB)' /proc/meminfo) + if [[ $? -eq 0 ]] || [[ -z "$cgroup_mem_kilobytes" ]]; then + fatal_error "Failed to get memory limit from /proc/meminfo" + fi + cgroup_mem_bytes=$(("$cgroup_mem_kilobytes"*1024)) + fi fi if [[ $? -eq 0 ]]; then # Convert to MiB - cgroup_mem_mib=$((cgroup_mem_bytes/(1024*1024))) + cgroup_mem_mib=$(("$cgroup_mem_bytes"/(1024*1024))) else fatal_error "Failed to get the memory limit in bytes from the current cgroup" fi From e69642c1e450bc4a6803c83fedc2a15843a398e2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 13:05:00 +0200 Subject: [PATCH 0996/1795] Correct bash test --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 02b22361db..2986e37f7b 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -167,7 +167,7 @@ else # In cgroupsv2, the memory.max file may contain 'max', meaning the group can use the full system memory # Here, we get the system memory from /proc/meminfo. Units are supposedly always in kb, but lets match them too cgroup_mem_kilobytes=$(grep -oP 'MemTotal:\s+\K\d+(?=\s+kB)' /proc/meminfo) - if [[ $? -eq 0 ]] || [[ -z "$cgroup_mem_kilobytes" ]]; then + if [[ $? -ne 0 ]]; then || [[ -z "$cgroup_mem_kilobytes" ]]; then fatal_error "Failed to get memory limit from /proc/meminfo" fi cgroup_mem_bytes=$(("$cgroup_mem_kilobytes"*1024)) From b1902b34161455ef0f77880346ef14aa033bb6f9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 13:07:50 +0200 Subject: [PATCH 0997/1795] Fix syntax error --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 2986e37f7b..847f5ea18e 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -167,7 +167,7 @@ else # In cgroupsv2, the memory.max file may contain 'max', meaning the group can use the full system memory # Here, we get the system memory from /proc/meminfo. Units are supposedly always in kb, but lets match them too cgroup_mem_kilobytes=$(grep -oP 'MemTotal:\s+\K\d+(?=\s+kB)' /proc/meminfo) - if [[ $? -ne 0 ]]; then || [[ -z "$cgroup_mem_kilobytes" ]]; then + if [[ $? -ne 0 ]] || [[ -z "$cgroup_mem_kilobytes" ]]; then fatal_error "Failed to get memory limit from /proc/meminfo" fi cgroup_mem_bytes=$(("$cgroup_mem_kilobytes"*1024)) From 8f39d6f9814f3e3ad30c46277b62a4cf17168f2f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 14:42:39 +0200 Subject: [PATCH 0998/1795] Exclude TensorFlow test for now, it seems to get stuck. First lets make sure we run the test suite on the zen4 prefix... --- test_suite.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 847f5ea18e..7598d9fad0 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -209,7 +209,9 @@ fi # Run all tests echo "Running tests: reframe ${REFRAME_ARGS} --run" -reframe ${REFRAME_ARGS} --run +# Exclude TensorFlow, which got stuck on https://github.com/EESSI/software-layer/pull/670#issuecomment-2291084266 +# Let's first make sure the test suite runs on the zen4 prefix... +reframe ${REFRAME_ARGS} --run --exclude TensorFlow reframe_exit_code=$? if [[ ${reframe_exit_code} -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." From aa35e754dec13eadec328752457adaf0b20ac81c Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 15 Aug 2024 12:57:16 +0000 Subject: [PATCH 0999/1795] Added a draft CI tests for EESSI module --- .github/workflows/tests_eessi_module.yml | 86 ++++++++++++++++++++++++ init/modules/EESSI/2023.06.lua | 2 +- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests_eessi_module.yml diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml new file mode 100644 index 0000000000..ef72ffeb0e --- /dev/null +++ b/.github/workflows/tests_eessi_module.yml @@ -0,0 +1,86 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Check for EESSI module functionality in software.eessi.io +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: + - aarch64/generic + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + - x86_64/generic + fail-fast: false + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + + - name: Test for archdetect_cpu functionality with only one valid path + run: | + export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH + CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) + export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 + if module load EESSI/${{matrix.EESSI_VERSION}}; then + if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then + echo "Test for picking up on \$archdetect_cpu PASSED" + else + echo "Test for picking up on \$archdetdect_cpu FAILED" >&2 + exit 1 + fi + module unload EESSI/${{matrix.EESSI_VERSION}} + fi + unset EESSI_ARCHDETECT_OPTIONS + + - name: Test for archdetect_cpu functionality with invalid path + run: | + export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH + export EESSI_ARCHDETECT_OPTIONS=dummy/cpu + outfile=outfile.txt + module load EESSI/${{matrix.EESSI_VERSION}} > $outfile 2>&1 + result=$(grep "Software" $outfile) + if [[ "$result" == *"Software directory check for the detected architecture failed"* ]]; then + echo "Test for picking up invalid path on \$archdetect_cpu PASSED" + else + echo "Test for picking up invalid path on \$archdetect_cpu FAILED" >&2 + exit 1 + fi + rm $outfile + unset EESSI_ARCHDETECT_OPTIONS + + - name: Test for expected variables after loading EESSI module + run: | + export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH + moduleoutfile=moduleout.txt + sourceoutfile=sourceout.txt + module load EESSI/${{matrix.EESSI_VERSION}} + env | grep -E '^(EESSI_S|EESSI_C)' | sort | > ${moduleoutfile} + module unload EESSI/${{matrix.EESSI_VERSION}} + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + env | grep -E '^(EESSI_S|EESSI_C)' | sort | > ${sourceoutfile} + if (diff ${moduleoutfile} ${sourceoutfile} > /dev/null); then + echo "Test for checking env variables PASSED" + else + echo "Test for checking env variables FAILED" >&2 + exit 1 + fi + rm $moduleoutfile $sourceoutfile diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index d39fa20a4b..3cccffd2bb 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -43,7 +43,7 @@ function archdetect_cpu() end local archdetect = archdetect_cpu() local eessi_cpu_family = archdetect:match("([^/]+)") -local eessi_software_subdir = os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE") or archdetect +local eessi_software_subdir = archdetect local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family) local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) local eessi_module_path = pathJoin(eessi_software_path, "modules", "all") From 932d9a09e17b8798948522d0d3a1e0dfc4c0bc13 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 15:06:05 +0200 Subject: [PATCH 1000/1795] Add some debugging echo's --- test_suite.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 7598d9fad0..69002bdbb5 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -74,8 +74,13 @@ fi TMPDIR=$(mktemp -d) echo ">> Setting up environment..." -module --force purge +# No module command, we are outside of the compat layer +# module --force purge +echo "Current EESSI env:" +env | grep "EESSI" +echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE before calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) +echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE after calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" source $TOPDIR/init/bash From aa2a1d61df8d12b390e74ef30456a5a034f3a4b7 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:29:10 +0200 Subject: [PATCH 1001/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index ef72ffeb0e..03d81cc3d6 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -13,16 +13,7 @@ jobs: strategy: matrix: EESSI_VERSION: - - 2023.06 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: - - aarch64/generic - - aarch64/neoverse_n1 - - aarch64/neoverse_v1 - - x86_64/amd/zen2 - - x86_64/amd/zen3 - - x86_64/intel/haswell - - x86_64/intel/skylake_avx512 - - x86_64/generic + - 2023.06 fail-fast: false steps: - name: Check out software-layer repository From 8eb07102bf23d9f3dee1146cafb947823b4144f5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 15:31:10 +0200 Subject: [PATCH 1002/1795] Make sure test suite only lists. Also make sure python script that uses archspec is called in a shell that has EESSI initialized, so that archspec is available --- test_suite.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 69002bdbb5..6845b230ef 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -79,7 +79,9 @@ echo ">> Setting up environment..." echo "Current EESSI env:" env | grep "EESSI" echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE before calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" -export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) +# For this call to be succesful, it needs to be able to import archspec (which is part of EESSI) +# Thus, we execute it in a subshell where EESSI is already initialized (a bit like a bootstrap) +export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(source $TOPDIR/init/bash; python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE after calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" source $TOPDIR/init/bash @@ -216,7 +218,9 @@ fi echo "Running tests: reframe ${REFRAME_ARGS} --run" # Exclude TensorFlow, which got stuck on https://github.com/EESSI/software-layer/pull/670#issuecomment-2291084266 # Let's first make sure the test suite runs on the zen4 prefix... -reframe ${REFRAME_ARGS} --run --exclude TensorFlow +#reframe ${REFRAME_ARGS} --run --exclude TensorFlow +# Limit to listing again, until we are sure we run with zen4... +reframe ${REFRAME_ARGS} --list reframe_exit_code=$? if [[ ${reframe_exit_code} -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." From 890c38b152d9979cee43006689d5b8b7c206824f Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:43:38 +0200 Subject: [PATCH 1003/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 03d81cc3d6..5c467942b6 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -11,10 +11,10 @@ jobs: build: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: EESSI_VERSION: - 2023.06 - fail-fast: false steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 From 584c637f2ae87f39bf0d50a0ae8c639d7c1a370b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 16:14:57 +0200 Subject: [PATCH 1004/1795] Redirect the output from sourcing the init script, as that is not what we want to capture in the environment variable --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 6845b230ef..86a347931b 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -81,7 +81,7 @@ env | grep "EESSI" echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE before calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" # For this call to be succesful, it needs to be able to import archspec (which is part of EESSI) # Thus, we execute it in a subshell where EESSI is already initialized (a bit like a bootstrap) -export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(source $TOPDIR/init/bash; python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) +export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(source $TOPDIR/init/bash > /dev/null 2>&1; python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE after calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" source $TOPDIR/init/bash From faddc7515ea0cf28ab8bd65da15980eb61c6786c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 17:13:17 +0200 Subject: [PATCH 1005/1795] Remove debugging echo's --- test_suite.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 86a347931b..f9f6d0113a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -74,10 +74,6 @@ fi TMPDIR=$(mktemp -d) echo ">> Setting up environment..." -# No module command, we are outside of the compat layer -# module --force purge -echo "Current EESSI env:" -env | grep "EESSI" echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE before calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" # For this call to be succesful, it needs to be able to import archspec (which is part of EESSI) # Thus, we execute it in a subshell where EESSI is already initialized (a bit like a bootstrap) From bd24ca8e613aa9f511c64f2689aff037e06cc2b6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 17:24:59 +0200 Subject: [PATCH 1006/1795] Remove gnuplot and add ReFrame --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml | 2 ++ .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml new file mode 100644 index 0000000000..f1fde247d0 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ReFrame-4.3.3.eb diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 695c4a06d9..9d13d5eb3a 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -22,4 +22,3 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - MODFLOW-6.4.4-foss-2023a.eb - - gnuplot-5.4.8-GCCcore-12.3.0.eb From 19af8cfe7f6de9c2fb91c254a608d8500829fe0d Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 15 Aug 2024 16:00:14 +0000 Subject: [PATCH 1007/1795] Added a draft CI tests for EESSI module --- .github/workflows/tests_eessi_module.yml | 118 +++++++++++------------ 1 file changed, 54 insertions(+), 64 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 5c467942b6..028730cded 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -1,12 +1,3 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Check for EESSI module functionality in software.eessi.io -on: - push: - branches: [ "*-software.eessi.io" ] - pull_request: - workflow_dispatch: -permissions: - contents: read # to fetch code (actions/checkout) jobs: build: runs-on: ubuntu-latest @@ -14,64 +5,63 @@ jobs: fail-fast: false matrix: EESSI_VERSION: - - 2023.06 + - 2023.06 steps: - - name: Check out software-layer repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 - with: - cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb - cvmfs_http_proxy: DIRECT - cvmfs_repositories: software.eessi.io + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io - - name: Test for archdetect_cpu functionality with only one valid path - run: | - export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH - CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) - export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 - if module load EESSI/${{matrix.EESSI_VERSION}}; then - if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then - echo "Test for picking up on \$archdetect_cpu PASSED" - else - echo "Test for picking up on \$archdetdect_cpu FAILED" >&2 - exit 1 - fi - module unload EESSI/${{matrix.EESSI_VERSION}} - fi - unset EESSI_ARCHDETECT_OPTIONS - - - name: Test for archdetect_cpu functionality with invalid path - run: | - export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH - export EESSI_ARCHDETECT_OPTIONS=dummy/cpu - outfile=outfile.txt - module load EESSI/${{matrix.EESSI_VERSION}} > $outfile 2>&1 - result=$(grep "Software" $outfile) - if [[ "$result" == *"Software directory check for the detected architecture failed"* ]]; then - echo "Test for picking up invalid path on \$archdetect_cpu PASSED" + - name: Test for archdetect_cpu functionality with only one valid path + run: | + export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH + CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) + export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 + if module load EESSI/${{matrix.EESSI_VERSION}}; then + if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then + echo "Test for picking up on \$archdetect_cpu PASSED" else - echo "Test for picking up invalid path on \$archdetect_cpu FAILED" >&2 - exit 1 + echo "Test for picking up on \$archdetect_cpu FAILED" >&2 + exit 1 fi - rm $outfile - unset EESSI_ARCHDETECT_OPTIONS + module unload EESSI/${{matrix.EESSI_VERSION}} + fi + unset EESSI_ARCHDETECT_OPTIONS - - name: Test for expected variables after loading EESSI module - run: | - export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH - moduleoutfile=moduleout.txt - sourceoutfile=sourceout.txt - module load EESSI/${{matrix.EESSI_VERSION}} - env | grep -E '^(EESSI_S|EESSI_C)' | sort | > ${moduleoutfile} - module unload EESSI/${{matrix.EESSI_VERSION}} - source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash - env | grep -E '^(EESSI_S|EESSI_C)' | sort | > ${sourceoutfile} - if (diff ${moduleoutfile} ${sourceoutfile} > /dev/null); then - echo "Test for checking env variables PASSED" - else - echo "Test for checking env variables FAILED" >&2 - exit 1 - fi - rm $moduleoutfile $sourceoutfile + - name: Test for archdetect_cpu functionality with invalid path + run: | + export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH + export EESSI_ARCHDETECT_OPTIONS=dummy/cpu + outfile=outfile.txt + module load EESSI/${{matrix.EESSI_VERSION}} > $outfile 2>&1 + result=$(grep "Software" $outfile) + if [[ "$result" == *"Software directory check for the detected architecture failed"* ]]; then + echo "Test for picking up invalid path on \$archdetect_cpu PASSED" + else + echo "Test for picking up invalid path on \$archdetect_cpu FAILED" >&2 + exit 1 + fi + unset EESSI_ARCHDETECT_OPTIONS + + - name: Test for expected variables after loading EESSI module + run: | + export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH + moduleoutfile=moduleout.txt + sourceoutfile=sourceout.txt + module load EESSI/${{matrix.EESSI_VERSION}} + env | grep -E '^(EESSI_S|EESSI_C)' | sort | > ${moduleoutfile} + module unload EESSI/${{matrix.EESSI_VERSION}} + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + env | grep -E '^(EESSI_S|EESSI_C)' | sort | > ${sourceoutfile} + if (diff ${moduleoutfile} ${sourceoutfile} > /dev/null); then + echo "Test for checking env variables PASSED" + else + echo "Test for checking env variables FAILED" >&2 + exit 1 + fi + From 5cb1f3647d0a11c6c07edeb403946a336fc7e844 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 15 Aug 2024 16:10:44 +0000 Subject: [PATCH 1008/1795] Added missing required yml file format --- .github/workflows/tests_eessi_module.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 028730cded..6cec64380a 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -1,3 +1,11 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for eessi_module_functionality in software.eessi.io +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: +permissions: + contents: read # to fetch code (actions/checkout) jobs: build: runs-on: ubuntu-latest From 9845452b29054d5739d2e89abafaa5e1ea4410c2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 15 Aug 2024 18:32:40 +0200 Subject: [PATCH 1009/1795] Debugging issue with loading ReFrame --- test_suite.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test_suite.sh b/test_suite.sh index f9f6d0113a..fe08f7bf83 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -82,6 +82,14 @@ echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE after calling eessi_software_subdir.py: $EE source $TOPDIR/init/bash +# DEBUGGING +echo "Checking if ReFrame module exists:" +ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen4/modules/all/ReFrame/4.3.3.lua +echo "Checking available ReFrame modules" +module av ReFrame +echo "Checking if we can load ReFrame when ignoring the cache" +module load --ignore_cache ReFrame + # Load the ReFrame module # Currently, we load the default version. Maybe we should somehow make this configurable in the future? module load ReFrame From 64d483acade68942dd3408972c15ce0a9d0326c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jurij=20Pe=C4=8Dar?= Date: Fri, 16 Aug 2024 10:25:23 +0200 Subject: [PATCH 1010/1795] {2023.06}[GCC/13.2.0] STAR 2.7.11b --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 883483f5d2..dcac6cfc72 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -15,3 +15,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 + - STAR-2.7.11b-GCC-13.2.0.eb From 56b9be9ad48d41a2555fdeb3c072297ebcd29dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jurij=20Pe=C4=8Dar?= Date: Fri, 16 Aug 2024 11:12:42 +0200 Subject: [PATCH 1011/1795] {2023.06}[GCC/13.2.0] STAR 2.7.11b try 2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index dcac6cfc72..5011c9d49c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -15,4 +15,6 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 + - xxd-9.1.0307-GCCcore-13.2.0.eb + - HTSlib-1.19.1-GCC-13.2.0.eb - STAR-2.7.11b-GCC-13.2.0.eb From d189d21dd4b3e3f052a4850814e0b6602d67236f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 16 Aug 2024 12:52:49 +0200 Subject: [PATCH 1012/1795] Remove redundant easyconfigs easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml Co-authored-by: Kenneth Hoste --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 5011c9d49c..dcac6cfc72 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -15,6 +15,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 - - xxd-9.1.0307-GCCcore-13.2.0.eb - - HTSlib-1.19.1-GCC-13.2.0.eb - STAR-2.7.11b-GCC-13.2.0.eb From 1aa8cb80d64dd9f352fd8623166b281c8ae499a6 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 16 Aug 2024 13:58:00 +0000 Subject: [PATCH 1013/1795] Add init scripts test - add test script - add github actions --- .../workflows/scripts/test_init_scripts.sh | 58 +++++++++++++++++++ .github/workflows/tests_init_module.yml | 44 ++++++++++++++ 2 files changed, 102 insertions(+) create mode 100755 .github/workflows/scripts/test_init_scripts.sh create mode 100644 .github/workflows/tests_init_module.yml diff --git a/.github/workflows/scripts/test_init_scripts.sh b/.github/workflows/scripts/test_init_scripts.sh new file mode 100755 index 0000000000..3a486eb0e2 --- /dev/null +++ b/.github/workflows/scripts/test_init_scripts.sh @@ -0,0 +1,58 @@ +#!/bin/bash +EESSI_VERSION="2023.06" +export LMOD_PAGER=cat + +# initialize assert framework +if [ ! -d assert.sh ]; then + echo "assert.sh not cloned." + echo "" + echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`" + exit 1 +fi +. assert.sh/assert.sh + +SHELLS=$@ + +for shell in ${SHELLS[@]}; do + echo = | awk 'NF += (OFS = $_) + 100' + echo RUNNING TESTS FOR SHELL: $shell + echo = | awk 'NF += (OFS = $_) + 100' + + # TEST 1: Source Script and check Module Output + assert "$shell -c 'source init/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" + # TEST 2: Check PS1 Prompt (Previous exported) + PROMPT="${shell^^}_PROMPT" + assert "echo ${!PROMPT}" "{EESSI $EESSI_VERSION}" + # Test 3: Check module overview + MODULE_SECTIONS=($($shell -c "source init/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) + assert "echo ${MODULE_SECTIONS[1]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/modules/all" + assert "echo ${MODULE_SECTIONS[4]}" "/home/eessi/EESSI_testing/modules" + # Test 4: Load Python module and check version and path + command="$shell -c 'source init/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'" + expected="Python 3.10.8" + assert "$command" "$expected" + command="$shell -c 'source init/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python'" + expected="/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/software/Python/3.10.8-GCCcore-12.2.0/bin/python" + assert "$command" "$expected" +done + +#ZSH TESTS +#assert "zsh -c 'source init/zsh' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" +#assert "echo $ZSH_PROMPT" "{EESSI $EESSI_VERSION}" + +#KSH TESTS +#assert "ksh -c 'source init/ksh' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" +#assert "echo $KSH_PROMPT" "{EESSI $EESSI_VERSION}" + +#FISH TESTS +#assert "fish -c 'source init/fish' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" +#assert "echo $FISH_PROMPT" "{EESSI $EESSI_VERSION}" + +#CSH TESTS +#assert "csh -c 'source init/csh' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" +#assert "echo $CSH_PROMPT" "{EESSI $EESSI_VERSION}" + +assert_end source_eessi + +# RESET PAGER +export LMOD_PAGER= diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml new file mode 100644 index 0000000000..952a58b7e1 --- /dev/null +++ b/.github/workflows/tests_init_module.yml @@ -0,0 +1,44 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Check for EESSI init shell scripts to load eessi software module in software.eessi.io +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: + - x86_64/generic + fail-fast: false + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + + - name: Clone assert.sh script + run: git clone https://github.com/lehmannro/assert.sh.git + + - name: Prepare PROMPT variables + run: | + export BASH_PROMPT=$(bash -c 'source init/bash; echo "$PS1"') + export ZSH_PROMPT=$(zsh -c 'source init/zsh; echo "$PS1"') + export KSH_PROMPT=$(ksh -c 'source init/ksh; echo "$PS1"') + export FISH_PROMPT=$(fish -c 'source init/fish; echo "$PS1"') + + - name: Run tests for available shells + run: | + .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" + From 0b6bc542727b9b2e2263cf497adc4b6bfdc29ec9 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 16 Aug 2024 14:02:57 +0000 Subject: [PATCH 1014/1795] Cleanup script and FIX init module path --- .github/workflows/scripts/test_init_scripts.sh | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/scripts/test_init_scripts.sh b/.github/workflows/scripts/test_init_scripts.sh index 3a486eb0e2..99fefecdf8 100755 --- a/.github/workflows/scripts/test_init_scripts.sh +++ b/.github/workflows/scripts/test_init_scripts.sh @@ -26,7 +26,7 @@ for shell in ${SHELLS[@]}; do # Test 3: Check module overview MODULE_SECTIONS=($($shell -c "source init/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) assert "echo ${MODULE_SECTIONS[1]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/modules/all" - assert "echo ${MODULE_SECTIONS[4]}" "/home/eessi/EESSI_testing/modules" + assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules" # Test 4: Load Python module and check version and path command="$shell -c 'source init/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'" expected="Python 3.10.8" @@ -36,22 +36,6 @@ for shell in ${SHELLS[@]}; do assert "$command" "$expected" done -#ZSH TESTS -#assert "zsh -c 'source init/zsh' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" -#assert "echo $ZSH_PROMPT" "{EESSI $EESSI_VERSION}" - -#KSH TESTS -#assert "ksh -c 'source init/ksh' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" -#assert "echo $KSH_PROMPT" "{EESSI $EESSI_VERSION}" - -#FISH TESTS -#assert "fish -c 'source init/fish' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" -#assert "echo $FISH_PROMPT" "{EESSI $EESSI_VERSION}" - -#CSH TESTS -#assert "csh -c 'source init/csh' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" -#assert "echo $CSH_PROMPT" "{EESSI $EESSI_VERSION}" - assert_end source_eessi # RESET PAGER From 4aa7d00682cd92161aaffff79bc2bd01f8e17ca1 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 16 Aug 2024 14:05:35 +0000 Subject: [PATCH 1015/1795] Use haswell for testing not sure what the output for generic would be. But different arches should be already tested on module --- .github/workflows/tests_init_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index 952a58b7e1..f9d99e3e4f 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -15,7 +15,7 @@ jobs: EESSI_VERSION: - 2023.06 EESSI_SOFTWARE_SUBDIR_OVERRIDE: - - x86_64/generic + - x86_64/haswell fail-fast: false steps: - name: Check out software-layer repository From 71543e312745552409ebd137b1a3c6ee60b5e9a0 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 16 Aug 2024 14:45:24 +0000 Subject: [PATCH 1016/1795] Fix Pipeline - Fix Format --- .github/workflows/tests_init_module.yml | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index f9d99e3e4f..6edfbb41e9 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -11,34 +11,34 @@ jobs: build: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: EESSI_VERSION: - - 2023.06 + - 2023.06 EESSI_SOFTWARE_SUBDIR_OVERRIDE: - - x86_64/haswell - fail-fast: false + - x86_64/haswell steps: - - name: Check out software-layer repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 - with: - cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb - cvmfs_http_proxy: DIRECT - cvmfs_repositories: software.eessi.io + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io - - name: Clone assert.sh script - run: git clone https://github.com/lehmannro/assert.sh.git + - name: Clone assert.sh script + run: git clone https://github.com/lehmannro/assert.sh.git - - name: Prepare PROMPT variables - run: | - export BASH_PROMPT=$(bash -c 'source init/bash; echo "$PS1"') - export ZSH_PROMPT=$(zsh -c 'source init/zsh; echo "$PS1"') - export KSH_PROMPT=$(ksh -c 'source init/ksh; echo "$PS1"') - export FISH_PROMPT=$(fish -c 'source init/fish; echo "$PS1"') + - name: Prepare PROMPT variables + run: | + export BASH_PROMPT=$(bash -c 'source init/bash; echo "$PS1"') + export ZSH_PROMPT=$(zsh -c 'source init/zsh; echo "$PS1"') + export KSH_PROMPT=$(ksh -c 'source init/ksh; echo "$PS1"') + export FISH_PROMPT=$(fish -c 'source init/fish; echo "$PS1"') - - name: Run tests for available shells - run: | - .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" + - name: Run tests for available shells + run: | + .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" From ea32ffe4160ec60b180235061e84127a323aa78e Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 16 Aug 2024 14:50:34 +0000 Subject: [PATCH 1017/1795] Add StdEnv.lua module --- init/modules/StdEnv.lua | 1 + 1 file changed, 1 insertion(+) create mode 100644 init/modules/StdEnv.lua diff --git a/init/modules/StdEnv.lua b/init/modules/StdEnv.lua new file mode 100644 index 0000000000..ae34402e9a --- /dev/null +++ b/init/modules/StdEnv.lua @@ -0,0 +1 @@ +load("EESSI/2023.06") From bb3c8bd589dde2bbea78a38866e5084651e3c148 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 16 Aug 2024 14:54:18 +0000 Subject: [PATCH 1018/1795] Install Shells on github actions --- .github/workflows/tests_init_module.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index 6edfbb41e9..c2cbf8f8e5 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -31,6 +31,11 @@ jobs: - name: Clone assert.sh script run: git clone https://github.com/lehmannro/assert.sh.git + - name: Install missing shells + run: | + apt install zsh ksh fish + echo "# INIT ZSH" > ~/.zshrc + - name: Prepare PROMPT variables run: | export BASH_PROMPT=$(bash -c 'source init/bash; echo "$PS1"') From 385aee024808a1beaa2e483448ebf82526bf40b1 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 16 Aug 2024 14:57:10 +0000 Subject: [PATCH 1019/1795] Fix requirements install on github action --- .github/workflows/tests_init_module.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index c2cbf8f8e5..58c48c5b68 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -33,7 +33,8 @@ jobs: - name: Install missing shells run: | - apt install zsh ksh fish + sudo apt update + sudo apt install zsh ksh fish echo "# INIT ZSH" > ~/.zshrc - name: Prepare PROMPT variables From 3ce0c951b2308b66f04989aeab64ddd69415d313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20Pe=C4=8Dar?= Date: Fri, 16 Aug 2024 22:47:35 +0200 Subject: [PATCH 1020/1795] Update eessi-2023.06-eb-4.9.2-2023b.yml fetch updated .eb file to remove hardcoded -mavx2 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index dcac6cfc72..8479061de7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -16,3 +16,6 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 - STAR-2.7.11b-GCC-13.2.0.eb + options: + #see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 + from-commit: cd8b1ce595195f132125d9b25a98a9efe702c72a From 95f9d6d7dc33375ff961030aa340d1c8d7a8d13c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Mon, 19 Aug 2024 09:20:38 +0200 Subject: [PATCH 1021/1795] Add missing ':' at end of line --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 8479061de7..edfd491cce 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -15,7 +15,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 - - STAR-2.7.11b-GCC-13.2.0.eb + - STAR-2.7.11b-GCC-13.2.0.eb: options: #see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 from-commit: cd8b1ce595195f132125d9b25a98a9efe702c72a From 56251cdfccbcefc977abd9633138d6d05f4f24cf Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 09:58:53 +0200 Subject: [PATCH 1022/1795] Add ReFrame for Zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml new file mode 100644 index 0000000000..f1fde247d0 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ReFrame-4.3.3.eb From ff3e006123c18685f4a5d1d220fefd0717df30f7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 10:07:44 +0200 Subject: [PATCH 1023/1795] Add gnuplot, just so we can show how test selection depending on the software that is build works --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e8fe8ab120..e13b1935fd 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -45,3 +45,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 + - gnuplot-5.4.8-GCCcore-12.3.0.eb From 461c941b130a238a88e4e07025b6f406573e0e9c Mon Sep 17 00:00:00 2001 From: makanu Date: Mon, 19 Aug 2024 08:34:18 +0000 Subject: [PATCH 1024/1795] Fix Fish command call issue --- init/fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/fish b/init/fish index 9ae68b39dc..33952769e1 100644 --- a/init/fish +++ b/init/fish @@ -2,7 +2,7 @@ set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") # Path to top-level module tree set -x MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules -. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/fish +. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/(uname -m)/usr/share/Lmod/init/fish export PS1 if test -z "$__Init_Default_Modules" From ab203e2e125857e824d9e9e7c38a6db350276cac Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 11:35:21 +0200 Subject: [PATCH 1025/1795] Add gnuplot back in as test build --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + .../map_software_to_test.py | 57 +++++++++++++++++++ .../eessi_test_mapping/software_to_tests.yml | 7 +++ tests/eessi_test_mapping/test_mapping.yml | 5 ++ 4 files changed, 70 insertions(+) create mode 100644 tests/eessi_test_mapping/map_software_to_test.py create mode 100644 tests/eessi_test_mapping/software_to_tests.yml create mode 100644 tests/eessi_test_mapping/test_mapping.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e8fe8ab120..e13b1935fd 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -45,3 +45,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 + - gnuplot-5.4.8-GCCcore-12.3.0.eb diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py new file mode 100644 index 0000000000..464921b11b --- /dev/null +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -0,0 +1,57 @@ +import yaml +import re +import os + +def load_mappings(file_path): + """Load the YAML mappings from a file.""" + with open(file_path, 'r') as file: + config = yaml.safe_load(file) + return config['mappings'] + +def get_tests_for_software(software_name, mappings): + """Get the list of tests for a given software name based on the first matching regex pattern.""" + + # Iterate over patterns in the order they appear in the YAML file + for pattern, tests in mappings.items(): + if re.match(pattern, software_name): + return tests + + # If no matches are found, return the default tests if they exist + if 'default_tests' in mappings: + return mappings['default_tests'] + + return [] + +def read_software_names(file_path): + """Read software names from the module_files.list.txt file.""" + with open(file_path, 'r') as file: + software_names = [line.strip() for line in file if line.strip()] + return software_names + +if __name__ == "__main__": + mappings = load_mappings("software_to_tests.yml") + + # Check if the file module_files.list.txt exists + module_file_path = "module_files.list.txt" + if not os.path.exists(module_file_path): + print(f"Error: {module_file_path} does not exist.") + else: + software_names = read_software_names(module_file_path) + tests_to_run = [] + for software_name in software_names: + additional_tests = get_tests_for_software(software_name, mappings) + for test in additional_tests: + if test not in tests_to_run: + tests_to_run.append(test) + + if additional_tests: + print(f"Software: {software_name} -> Tests: {additional_tests}") + else: + print(f"Software: {software_name} -> No tests found") + + if tests_to_run: + arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) + print(f"Full list of tests to run: {tests_to_run}") + print(f"Argument string: {arg_string}") + else: + print(f"Full list of tests to run: No tests found") diff --git a/tests/eessi_test_mapping/software_to_tests.yml b/tests/eessi_test_mapping/software_to_tests.yml new file mode 100644 index 0000000000..2c6d504fae --- /dev/null +++ b/tests/eessi_test_mapping/software_to_tests.yml @@ -0,0 +1,7 @@ +mappings: + gnuplot/*: + - EESSI_ESPRESSO + Perl/*: + - EESSI_GROMACS + default_tests: + - EESSI_OSU diff --git a/tests/eessi_test_mapping/test_mapping.yml b/tests/eessi_test_mapping/test_mapping.yml new file mode 100644 index 0000000000..603e765a64 --- /dev/null +++ b/tests/eessi_test_mapping/test_mapping.yml @@ -0,0 +1,5 @@ +mappings: + - gnuplot/*: + - EESSI_ESPRESSO + - Perl/*: + - EESSI_GROMACS From cd0c6c6dedfebd884a6df96f0b5f796dbf029789 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 11:36:10 +0200 Subject: [PATCH 1026/1795] Remove ReFrame, as that was now build and deployed in a separate PR --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml deleted file mode 100644 index f1fde247d0..0000000000 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - ReFrame-4.3.3.eb From 9ee97147506766d464430f7c1a285ea8560f0d80 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 11:38:05 +0200 Subject: [PATCH 1027/1795] Add test mapping so that we can run selected tests depending on which software was built --- tests/eessi_test_mapping/module_files.list.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/eessi_test_mapping/module_files.list.txt diff --git a/tests/eessi_test_mapping/module_files.list.txt b/tests/eessi_test_mapping/module_files.list.txt new file mode 100644 index 0000000000..0d6c821fc6 --- /dev/null +++ b/tests/eessi_test_mapping/module_files.list.txt @@ -0,0 +1,5 @@ +Lua/5.4.4-GCCcore-12.2.0 +Perl/5.36.0-GCCcore-12.2.0-minimal +gnuplot/5.4.6-GCCcore-12.2.0 +libcerf/2.3-GCCcore-12.2.0 +libgd/2.3.3-GCCcore-12.2.0 From 5f2ce9b759124f26d1fa97bed36a223a994efaee Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 11:39:32 +0200 Subject: [PATCH 1028/1795] This file was there for testing, and added by accident --- tests/eessi_test_mapping/module_files.list.txt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 tests/eessi_test_mapping/module_files.list.txt diff --git a/tests/eessi_test_mapping/module_files.list.txt b/tests/eessi_test_mapping/module_files.list.txt deleted file mode 100644 index 0d6c821fc6..0000000000 --- a/tests/eessi_test_mapping/module_files.list.txt +++ /dev/null @@ -1,5 +0,0 @@ -Lua/5.4.4-GCCcore-12.2.0 -Perl/5.36.0-GCCcore-12.2.0-minimal -gnuplot/5.4.6-GCCcore-12.2.0 -libcerf/2.3-GCCcore-12.2.0 -libgd/2.3.3-GCCcore-12.2.0 From bc02ddb4f2be79dc9b3580d4c05842d4674949f4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 11:41:13 +0200 Subject: [PATCH 1029/1795] Add mapping for software to test names --- .../map_software_to_test.py | 57 +++++++++++++++++++ .../eessi_test_mapping/software_to_tests.yml | 7 +++ tests/eessi_test_mapping/test_mapping.yml | 5 ++ 3 files changed, 69 insertions(+) create mode 100644 tests/eessi_test_mapping/map_software_to_test.py create mode 100644 tests/eessi_test_mapping/software_to_tests.yml create mode 100644 tests/eessi_test_mapping/test_mapping.yml diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py new file mode 100644 index 0000000000..464921b11b --- /dev/null +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -0,0 +1,57 @@ +import yaml +import re +import os + +def load_mappings(file_path): + """Load the YAML mappings from a file.""" + with open(file_path, 'r') as file: + config = yaml.safe_load(file) + return config['mappings'] + +def get_tests_for_software(software_name, mappings): + """Get the list of tests for a given software name based on the first matching regex pattern.""" + + # Iterate over patterns in the order they appear in the YAML file + for pattern, tests in mappings.items(): + if re.match(pattern, software_name): + return tests + + # If no matches are found, return the default tests if they exist + if 'default_tests' in mappings: + return mappings['default_tests'] + + return [] + +def read_software_names(file_path): + """Read software names from the module_files.list.txt file.""" + with open(file_path, 'r') as file: + software_names = [line.strip() for line in file if line.strip()] + return software_names + +if __name__ == "__main__": + mappings = load_mappings("software_to_tests.yml") + + # Check if the file module_files.list.txt exists + module_file_path = "module_files.list.txt" + if not os.path.exists(module_file_path): + print(f"Error: {module_file_path} does not exist.") + else: + software_names = read_software_names(module_file_path) + tests_to_run = [] + for software_name in software_names: + additional_tests = get_tests_for_software(software_name, mappings) + for test in additional_tests: + if test not in tests_to_run: + tests_to_run.append(test) + + if additional_tests: + print(f"Software: {software_name} -> Tests: {additional_tests}") + else: + print(f"Software: {software_name} -> No tests found") + + if tests_to_run: + arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) + print(f"Full list of tests to run: {tests_to_run}") + print(f"Argument string: {arg_string}") + else: + print(f"Full list of tests to run: No tests found") diff --git a/tests/eessi_test_mapping/software_to_tests.yml b/tests/eessi_test_mapping/software_to_tests.yml new file mode 100644 index 0000000000..2c6d504fae --- /dev/null +++ b/tests/eessi_test_mapping/software_to_tests.yml @@ -0,0 +1,7 @@ +mappings: + gnuplot/*: + - EESSI_ESPRESSO + Perl/*: + - EESSI_GROMACS + default_tests: + - EESSI_OSU diff --git a/tests/eessi_test_mapping/test_mapping.yml b/tests/eessi_test_mapping/test_mapping.yml new file mode 100644 index 0000000000..603e765a64 --- /dev/null +++ b/tests/eessi_test_mapping/test_mapping.yml @@ -0,0 +1,5 @@ +mappings: + - gnuplot/*: + - EESSI_ESPRESSO + - Perl/*: + - EESSI_GROMACS From 6f63aea97a24571d87f1017d1af2b5decad6203a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 11:48:17 +0200 Subject: [PATCH 1030/1795] These files belonged to another PR... --- .../map_software_to_test.py | 57 ------------------- .../eessi_test_mapping/software_to_tests.yml | 7 --- tests/eessi_test_mapping/test_mapping.yml | 5 -- 3 files changed, 69 deletions(-) delete mode 100644 tests/eessi_test_mapping/map_software_to_test.py delete mode 100644 tests/eessi_test_mapping/software_to_tests.yml delete mode 100644 tests/eessi_test_mapping/test_mapping.yml diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py deleted file mode 100644 index 464921b11b..0000000000 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ /dev/null @@ -1,57 +0,0 @@ -import yaml -import re -import os - -def load_mappings(file_path): - """Load the YAML mappings from a file.""" - with open(file_path, 'r') as file: - config = yaml.safe_load(file) - return config['mappings'] - -def get_tests_for_software(software_name, mappings): - """Get the list of tests for a given software name based on the first matching regex pattern.""" - - # Iterate over patterns in the order they appear in the YAML file - for pattern, tests in mappings.items(): - if re.match(pattern, software_name): - return tests - - # If no matches are found, return the default tests if they exist - if 'default_tests' in mappings: - return mappings['default_tests'] - - return [] - -def read_software_names(file_path): - """Read software names from the module_files.list.txt file.""" - with open(file_path, 'r') as file: - software_names = [line.strip() for line in file if line.strip()] - return software_names - -if __name__ == "__main__": - mappings = load_mappings("software_to_tests.yml") - - # Check if the file module_files.list.txt exists - module_file_path = "module_files.list.txt" - if not os.path.exists(module_file_path): - print(f"Error: {module_file_path} does not exist.") - else: - software_names = read_software_names(module_file_path) - tests_to_run = [] - for software_name in software_names: - additional_tests = get_tests_for_software(software_name, mappings) - for test in additional_tests: - if test not in tests_to_run: - tests_to_run.append(test) - - if additional_tests: - print(f"Software: {software_name} -> Tests: {additional_tests}") - else: - print(f"Software: {software_name} -> No tests found") - - if tests_to_run: - arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) - print(f"Full list of tests to run: {tests_to_run}") - print(f"Argument string: {arg_string}") - else: - print(f"Full list of tests to run: No tests found") diff --git a/tests/eessi_test_mapping/software_to_tests.yml b/tests/eessi_test_mapping/software_to_tests.yml deleted file mode 100644 index 2c6d504fae..0000000000 --- a/tests/eessi_test_mapping/software_to_tests.yml +++ /dev/null @@ -1,7 +0,0 @@ -mappings: - gnuplot/*: - - EESSI_ESPRESSO - Perl/*: - - EESSI_GROMACS - default_tests: - - EESSI_OSU diff --git a/tests/eessi_test_mapping/test_mapping.yml b/tests/eessi_test_mapping/test_mapping.yml deleted file mode 100644 index 603e765a64..0000000000 --- a/tests/eessi_test_mapping/test_mapping.yml +++ /dev/null @@ -1,5 +0,0 @@ -mappings: - - gnuplot/*: - - EESSI_ESPRESSO - - Perl/*: - - EESSI_GROMACS From 4fef822203aed8a523771d14afa7b7d51f628c72 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 12:05:34 +0200 Subject: [PATCH 1031/1795] Remove gnuplot again, it was already present... --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 - test_suite.sh | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e13b1935fd..e8fe8ab120 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -45,4 +45,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 - - gnuplot-5.4.8-GCCcore-12.3.0.eb diff --git a/test_suite.sh b/test_suite.sh index fe08f7bf83..fffd4a7888 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -220,11 +220,7 @@ fi # Run all tests echo "Running tests: reframe ${REFRAME_ARGS} --run" -# Exclude TensorFlow, which got stuck on https://github.com/EESSI/software-layer/pull/670#issuecomment-2291084266 -# Let's first make sure the test suite runs on the zen4 prefix... -#reframe ${REFRAME_ARGS} --run --exclude TensorFlow -# Limit to listing again, until we are sure we run with zen4... -reframe ${REFRAME_ARGS} --list +reframe ${REFRAME_ARGS} --run reframe_exit_code=$? if [[ ${reframe_exit_code} -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." From 6c4dfc8d2a2d97b89dc95581caefe2d8a43a5a86 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 12:11:54 +0200 Subject: [PATCH 1032/1795] Make sure module files list is available in the test step --- create_tarball.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/create_tarball.sh b/create_tarball.sh index 2dee665060..0c1fe302db 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -88,6 +88,9 @@ echo "wrote file list to ${files_list}" echo "wrote module file list to ${module_files_list}" [ -r ${module_files_list} ] && cat ${module_files_list} +# Copy the module files list to current workindg dir for later use in the test step +cp ${module_files_list} module_files.list.txt + topdir=${cvmfs_repo}/versions/ echo ">> Creating tarball ${target_tgz} from ${topdir}..." From 3388003133c67948ca6685e1897024b3fff17d07 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 12:13:31 +0200 Subject: [PATCH 1033/1795] This was just an example, removing --- tests/eessi_test_mapping/test_mapping.yml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 tests/eessi_test_mapping/test_mapping.yml diff --git a/tests/eessi_test_mapping/test_mapping.yml b/tests/eessi_test_mapping/test_mapping.yml deleted file mode 100644 index 603e765a64..0000000000 --- a/tests/eessi_test_mapping/test_mapping.yml +++ /dev/null @@ -1,5 +0,0 @@ -mappings: - - gnuplot/*: - - EESSI_ESPRESSO - - Perl/*: - - EESSI_GROMACS From 6c04860f0afd958b0365962bf6e9df422373e379 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 12:32:58 +0200 Subject: [PATCH 1034/1795] Integrate selection of tests into test step --- test_suite.sh | 9 ++- .../map_software_to_test.py | 80 ++++++++++++------- 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 6e73fbd87c..37a1874b80 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -182,8 +182,15 @@ else fatal_error "Failed to run 'reframe --version'" fi +# Get the subset of test names based on the test mapping and tags (e.g. CI, 1_node) +module_list="module_file.list.txt" +mapping_config="tests/eessi_test_mapping/software_to_tests.yml" +# Run with --debug for easier debugging in case there are issues: +python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug +REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}") +export REFRAME_ARGS="--tag CI --tag 1_node --nocolor ${REFRAME_NAME_ARGS}" + # List the tests we want to run -export REFRAME_ARGS='--tag CI --tag 1_node --nocolor' echo "Listing tests: reframe ${REFRAME_ARGS} --list" reframe ${REFRAME_ARGS} --list if [[ $? -eq 0 ]]; then diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py index 464921b11b..6e37189a2f 100644 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -1,13 +1,24 @@ import yaml import re import os +import argparse def load_mappings(file_path): """Load the YAML mappings from a file.""" + if not os.path.exists(file_path): + raise FileNotFoundError(f"Error: {file_path} does not exist.") with open(file_path, 'r') as file: config = yaml.safe_load(file) return config['mappings'] +def read_software_names(file_path): + """Read software names from the module_files.list.txt file.""" + if not os.path.exists(file_path): + raise FileNotFoundError(f"Error: {file_path} does not exist.") + with open(file_path, 'r') as file: + software_names = [line.strip() for line in file if line.strip()] + return software_names + def get_tests_for_software(software_name, mappings): """Get the list of tests for a given software name based on the first matching regex pattern.""" @@ -22,36 +33,45 @@ def get_tests_for_software(software_name, mappings): return [] -def read_software_names(file_path): - """Read software names from the module_files.list.txt file.""" - with open(file_path, 'r') as file: - software_names = [line.strip() for line in file if line.strip()] - return software_names +def main(yaml_file, module_file, debug): + """Main function to process software names and their tests.""" + mappings = load_mappings(yaml_file) + if debug: + print(f"Loaded mappings from '{yaml_file}'") + + software_names = read_software_names(module_file) + if debug: + print(f"Debug: Read software names from '{module_file}'") + + tests_to_run = [] + arg_string = "" + for software_name in software_names: + additional_tests = get_tests_for_software(software_name, mappings) + for test in additional_tests: + if test not in tests_to_run: + tests_to_run.append(test) + + if additional_tests and debug: + print(f"Software: {software_name} -> Tests: {additional_tests}") + elif debug: + print(f"Software: {software_name} -> No tests found") + + if tests_to_run and debug: + arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) + print(f"Full list of tests to run: {tests_to_run}") + print(f"Argument string: {arg_string}") + elif debug: + print(f"Full list of tests to run: No tests found") + + # This is the main thing this script should return + print(f"{arg_string}") if __name__ == "__main__": - mappings = load_mappings("software_to_tests.yml") + parser = argparse.ArgumentParser(description="Map software names to their tests based on a YAML configuration.") + parser.add_argument('yaml_file', type=str, help='Path to the YAML file containing the test mappings.') + parser.add_argument('module_file', type=str, help='Path to the file containing the list of software names.') + parser.add_argument('--debug', action='store_true', help='Enable debug output.') + + args = parser.parse_args() - # Check if the file module_files.list.txt exists - module_file_path = "module_files.list.txt" - if not os.path.exists(module_file_path): - print(f"Error: {module_file_path} does not exist.") - else: - software_names = read_software_names(module_file_path) - tests_to_run = [] - for software_name in software_names: - additional_tests = get_tests_for_software(software_name, mappings) - for test in additional_tests: - if test not in tests_to_run: - tests_to_run.append(test) - - if additional_tests: - print(f"Software: {software_name} -> Tests: {additional_tests}") - else: - print(f"Software: {software_name} -> No tests found") - - if tests_to_run: - arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) - print(f"Full list of tests to run: {tests_to_run}") - print(f"Argument string: {arg_string}") - else: - print(f"Full list of tests to run: No tests found") + main(args.yaml_file, args.module_file, args.debug) From 63deb003f7f991772b3e8c13dd93842a7514d860 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 12:37:05 +0200 Subject: [PATCH 1035/1795] Fix arguments --- tests/eessi_test_mapping/map_software_to_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py index 6e37189a2f..9e82d6b947 100644 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -68,10 +68,10 @@ def main(yaml_file, module_file, debug): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Map software names to their tests based on a YAML configuration.") - parser.add_argument('yaml_file', type=str, help='Path to the YAML file containing the test mappings.') - parser.add_argument('module_file', type=str, help='Path to the file containing the list of software names.') + parser.add_argument('--mapping-file', type=str, help='Path to the YAML file containing the test mappings.') + parser.add_argument('--module-list', type=str, help='Path to the file containing the list of software names.') parser.add_argument('--debug', action='store_true', help='Enable debug output.') args = parser.parse_args() - main(args.yaml_file, args.module_file, args.debug) + main(args.mapping_file, args.module_list, args.debug) From 2921f99faf1e6e5c284bf1a6de1d6bd8de6b8712 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 12:38:39 +0200 Subject: [PATCH 1036/1795] Fix small bug that it wasn't generating the argument list when --debug wasn't passed' --- tests/eessi_test_mapping/map_software_to_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py index 9e82d6b947..f86fe2fcc3 100644 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -56,10 +56,11 @@ def main(yaml_file, module_file, debug): elif debug: print(f"Software: {software_name} -> No tests found") - if tests_to_run and debug: + if tests_to_run: arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) - print(f"Full list of tests to run: {tests_to_run}") - print(f"Argument string: {arg_string}") + if debug: + print(f"Full list of tests to run: {tests_to_run}") + print(f"Argument string: {arg_string}") elif debug: print(f"Full list of tests to run: No tests found") From a22ee425509b016fbaad5e0437087e8a2acd8696 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 19 Aug 2024 14:36:02 +0200 Subject: [PATCH 1037/1795] use upper directory of former 'rw' session in 'ro' mode --- eessi_container.sh | 71 +++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index e404b7ee18..7f0c64ef27 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -254,7 +254,7 @@ if [[ ${LIST_REPOS} -eq 1 ]]; then default_label=", default" else default_label="" - fi + fi echo " ${cvmfs_repo} [CVMFS config repo${default_label}]" done for cfg_repo in "${!cfg_cvmfs_repos[@]}" @@ -323,7 +323,7 @@ do if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." # cvmfs_repo_name is actually a repository ID, use that to obtain - # the actual name from the EESSI_REPOS_CFG_FILE + # the actual name from the EESSI_REPOS_CFG_FILE cfg_repo_id=${cvmfs_repo_name} cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi @@ -595,11 +595,11 @@ do # that the necessary information for accessing a CVMFS repository is made # available inside the container if [[ -n "${cfg_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then - cfg_repo_id=${cvmfs_repo_name} + cfg_repo_id=${cvmfs_repo_name} - # obtain CVMFS repository name from section for the given ID + # obtain CVMFS repository name from section for the given ID cfg_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") - # derive domain part from (cfg_)repo_name (everything after first '.') + # derive domain part from (cfg_)repo_name (everything after first '.') repo_name_domain=${repo_name#*.} # cfg_cvmfs_repos is populated through reading the file pointed to by @@ -609,15 +609,15 @@ do # copy repos.cfg to job directory --> makes it easier to inspect the job cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. - # cfg file should include sections (one per CVMFS repository to be mounted) - # with each section containing the settings: - # - repo_name, - # - repo_version, - # - config_bundle, and - # - a map { filepath_in_bundle -> container_filepath } + # cfg file should include sections (one per CVMFS repository to be mounted) + # with each section containing the settings: + # - repo_name, + # - repo_version, + # - config_bundle, and + # - a map { filepath_in_bundle -> container_filepath } # - # The config_bundle includes the files which are mapped ('->') to a target - # location in container: + # The config_bundle includes the files which are mapped ('->') to a target + # location in container: # - default.local -> /etc/cvmfs/default.local # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf @@ -641,7 +641,7 @@ do # use information to set up dir ${EESSI_TMPDIR}/repos_cfg and define # BIND mounts # check if config_bundle exists, if so, unpack it into - # ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error + # ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error # if config_bundle is relative path (no '/' at start) prepend it with # EESSI_REPOS_CFG_DIR config_bundle_path= @@ -726,7 +726,7 @@ do if [[ ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." # cvmfs_repo_name is actually a repository ID, use that to obtain - # the actual name from the EESSI_REPOS_CFG_FILE + # the actual name from the EESSI_REPOS_CFG_FILE cfg_repo_id=${cvmfs_repo_name} cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi @@ -736,15 +736,46 @@ do # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) if [[ ${cvmfs_repo_access} == "ro" ]] ; then - export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" + # need to distinguish between basic "ro" access and "ro" after a "rw" session + if [[ -d ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ]]; then + # the overlay-upper directory is only created in a read-write-session, thus + # we are resuming from such a session here (otherwise there shouldn't be such + # directory yet as it is only created for read-write-sessions a bit further + # below); the overlay-upper directory can only exist because it is part of + # the ${RESUME} directory or tarball + # to be able to see the contents of the read-write session we have to mount + # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository + + # make the target CernVM-FS repository available under /cvmfs_ro + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" + + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + + # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs + EESSI_READONLY_OVERLAY="container:fuse-overlayfs" + # The contents of the previous session are available under + # ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}. + # Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper + # the left-most directory given for the lowerdir argument is put on top, + # and with no upperdir=... the whole overlayfs is made available read-only + EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}" + EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + export EESSI_READONLY_OVERLAY + + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY_OVERLAY}") + export EESSI_FUSE_MOUNTS + else + # basic "ro" access that doesn't require any fuseoverlay-fs + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - export EESSI_FUSE_MOUNTS + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + export EESSI_FUSE_MOUNTS + fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then # use repo-specific overlay directories mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work - [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" @@ -762,7 +793,7 @@ do export EESSI_FUSE_MOUNTS else echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n '${cvmfs_repo_name}' is not known" - exit ${REPOSITORY_ERROR_EXITCODE} + exit ${REPOSITORY_ERROR_EXITCODE} fi # create repo_settings.sh file in ${EESSI_TMPDIR}/${cvmfs_repo_name} to store # (intention is that the file could be just sourced to obtain the settings) From 634abb077d9f0283f6452c4bcb35e1e3f7b30fd4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 14:39:21 +0200 Subject: [PATCH 1038/1795] Be a bit more verbose, for easier debugging of future issues. Remove statements that were inserted just to debug this particular issue --- test_suite.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index fffd4a7888..464670b653 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -74,22 +74,13 @@ fi TMPDIR=$(mktemp -d) echo ">> Setting up environment..." -echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE before calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" # For this call to be succesful, it needs to be able to import archspec (which is part of EESSI) # Thus, we execute it in a subshell where EESSI is already initialized (a bit like a bootstrap) export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(source $TOPDIR/init/bash > /dev/null 2>&1; python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) -echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE after calling eessi_software_subdir.py: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" +echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" source $TOPDIR/init/bash -# DEBUGGING -echo "Checking if ReFrame module exists:" -ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen4/modules/all/ReFrame/4.3.3.lua -echo "Checking available ReFrame modules" -module av ReFrame -echo "Checking if we can load ReFrame when ignoring the cache" -module load --ignore_cache ReFrame - # Load the ReFrame module # Currently, we load the default version. Maybe we should somehow make this configurable in the future? module load ReFrame @@ -147,24 +138,29 @@ echo "Configured reframe with the following environment variables:" env | grep "RFM_" # Inject correct CPU/memory properties into the ReFrame config file +echo "Collecting system-specific input for the ReFrame configuration file" cpuinfo=$(lscpu) if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then cpu_count=${BASH_REMATCH[1]} + echo "Detected CPU count: ${cpu_count}" else fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu." fi if [[ "${cpuinfo}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then socket_count=${BASH_REMATCH[1]} + echo "Detected socket count: ${socket_count}" else fatal_error "Failed to get the number of sockets for the current test hardware with lscpu." fi if [[ "${cpuinfo}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then threads_per_core=${BASH_REMATCH[2]} + echo "Detected threads per core: ${threads_per_core}" else fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu." fi if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then cores_per_socket=${BASH_REMATCH[2]} + echo "Detected cores per socket: ${cores_per_socket}" else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi @@ -190,12 +186,18 @@ if [[ $? -eq 0 ]]; then else fatal_error "Failed to get the memory limit in bytes from the current cgroup" fi +echo "Detected available memory: ${cgroup_mem_mib} MiB" + +echo "Replacing detected system information in template ReFrame config file..." cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES +# Make debugging easier by printing the final config file: +echo "Final config file (after replacements):" +cat "${RFM_CONFIG_FILES}" # Workaround for https://github.com/EESSI/software-layer/pull/467#issuecomment-1973341966 export PSM3_DEVICES='self,shm' # this is enough, since we only run single node for now From e1721002287b364cb9015a0ae9878dacd4aded9f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 15:19:07 +0200 Subject: [PATCH 1039/1795] Remove gnuplot, since it is already present --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e13b1935fd..e8fe8ab120 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -45,4 +45,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 - - gnuplot-5.4.8-GCCcore-12.3.0.eb From dd1f31562359f94968f350613be13eb9f9558287 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 15:20:48 +0200 Subject: [PATCH 1040/1795] Add some fast building but irrelevant piece of software to demonstrate the new test step --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index a22b78718f..6996e6f712 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -6,4 +6,5 @@ easyconfigs: from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - gnuplot-5.4.6-GCCcore-12.2.0.eb - h5py-3.8.0-foss-2022b.eb - - MDAnalysis-2.4.2-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - patchelf-0.17.2-GCCcore-12.2.0.eb From 4a41bad6aec7511faf5d2425a3808be6d7bf14cf Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 15:32:39 +0200 Subject: [PATCH 1041/1795] Make sure the module_file.list.txt is copied to the original working dir --- create_tarball.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 0c1fe302db..e59ec421c3 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -35,6 +35,7 @@ if [ ! -d ${software_dir_overlay} ]; then exit 3 fi +current_workdir=${PWD} cd ${overlay_upper_dir}/versions/ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..." @@ -89,7 +90,7 @@ echo "wrote module file list to ${module_files_list}" [ -r ${module_files_list} ] && cat ${module_files_list} # Copy the module files list to current workindg dir for later use in the test step -cp ${module_files_list} module_files.list.txt +cp ${module_files_list} ${current_workdir}/module_files.list.txt topdir=${cvmfs_repo}/versions/ From d2facfd58b2fd14ac747bc939efbd87ae71134bc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 15:40:03 +0200 Subject: [PATCH 1042/1795] Make sure we throw a fatal_error when the test selection isn't retreived correctly. Also, end the test step --- test_suite.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 37a1874b80..dc00dd454a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -188,6 +188,13 @@ mapping_config="tests/eessi_test_mapping/software_to_tests.yml" # Run with --debug for easier debugging in case there are issues: python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}") +test_selection_exit_code=$? +if [[ ${test_selection_exit_code} -eq 0 ]]; then + echo_green "Succesfully extracted names of tests to run: ${REFRAME_NAME_ARGS}" +else + fatal_error "Failed to extract names of tests to run: ${REFRAME_NAME_ARGS}" + exit ${test_selection_exit_code} +fi export REFRAME_ARGS="--tag CI --tag 1_node --nocolor ${REFRAME_NAME_ARGS}" # List the tests we want to run @@ -201,7 +208,9 @@ fi # Run all tests echo "Running tests: reframe ${REFRAME_ARGS} --run" -reframe ${REFRAME_ARGS} --run +#reframe ${REFRAME_ARGS} --run +# First, lets make sure test selection works before we run... +reframe ${REFRAME_ARGS} --list reframe_exit_code=$? if [[ ${reframe_exit_code} -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." From 8cd2944eeb217752236c5396b2d4b14474c1bb72 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 15:54:18 +0200 Subject: [PATCH 1043/1795] Corrected typo --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index dc00dd454a..963725cbbd 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -183,7 +183,7 @@ else fi # Get the subset of test names based on the test mapping and tags (e.g. CI, 1_node) -module_list="module_file.list.txt" +module_list="module_files.list.txt" mapping_config="tests/eessi_test_mapping/software_to_tests.yml" # Run with --debug for easier debugging in case there are issues: python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug From 825ec945957f8b24b565962d5b0dd01d471712fb Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 16:20:22 +0200 Subject: [PATCH 1044/1795] Re-enable actual test runs, this PR now seems to do what it should be doing --- test_suite.sh | 4 +--- tests/eessi_test_mapping/map_software_to_test.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 963725cbbd..7a30a162c1 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -208,9 +208,7 @@ fi # Run all tests echo "Running tests: reframe ${REFRAME_ARGS} --run" -#reframe ${REFRAME_ARGS} --run -# First, lets make sure test selection works before we run... -reframe ${REFRAME_ARGS} --list +reframe ${REFRAME_ARGS} --run reframe_exit_code=$? if [[ ${reframe_exit_code} -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py index f86fe2fcc3..5b39e47dcd 100644 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -41,7 +41,7 @@ def main(yaml_file, module_file, debug): software_names = read_software_names(module_file) if debug: - print(f"Debug: Read software names from '{module_file}'") + print(f"Read software names from '{module_file}'") tests_to_run = [] arg_string = "" From bf6764be240b8862784ab8273f3d59ac69a4a81f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 16:32:20 +0200 Subject: [PATCH 1045/1795] Add comments to mapping file. Tweak mapping script so that default tests are always added at the end, even if all test had other matches --- .../map_software_to_test.py | 26 ++++++++++++++----- .../eessi_test_mapping/software_to_tests.yml | 11 ++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py index 5b39e47dcd..09cb2d2987 100644 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -45,6 +45,7 @@ def main(yaml_file, module_file, debug): tests_to_run = [] arg_string = "" + # For each module name, get the relevant set of tests for software_name in software_names: additional_tests = get_tests_for_software(software_name, mappings) for test in additional_tests: @@ -56,15 +57,26 @@ def main(yaml_file, module_file, debug): elif debug: print(f"Software: {software_name} -> No tests found") - if tests_to_run: - arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) - if debug: - print(f"Full list of tests to run: {tests_to_run}") - print(f"Argument string: {arg_string}") + # Always add the default set of tests, if default_tests is specified + if 'default_tests' in mappings: + additional_tests = mappings['default_tests'] + for test in additional_tests: + if test not in tests_to_run: + tests_to_run.append(test) + + if additional_tests and debug: + print(f"Adding default set of tests: {additional_tests}") + + # Create argument string out of the list of tests to run + if tests_to_run: + arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) + if debug: + print(f"Full list of tests to run: {tests_to_run}") + print(f"Argument string: {arg_string}") elif debug: - print(f"Full list of tests to run: No tests found") + print(f"Full list of tests to run: No tests found") - # This is the main thing this script should return + # This is the only thing this script should return, unless run with --debug print(f"{arg_string}") if __name__ == "__main__": diff --git a/tests/eessi_test_mapping/software_to_tests.yml b/tests/eessi_test_mapping/software_to_tests.yml index 2c6d504fae..fdb51e741a 100644 --- a/tests/eessi_test_mapping/software_to_tests.yml +++ b/tests/eessi_test_mapping/software_to_tests.yml @@ -1,3 +1,14 @@ +# This file creates a mapping between (regular expressions for) module names and test names from the EESSI test suite +# If a module name matches one of the regular expressions, the listed set of tests will be run in the test step +# For a given module name, the test list for the first matching regular expression is returned +# E.g. for +# mappings: +# foo-v1: +# - bar +# foo-* +# - bar2 +# only the bar test will be run for foo-v1 (even though it also matches the pattern (foo-*) +# If a module name does not match anything, the default_tests will be run mappings: gnuplot/*: - EESSI_ESPRESSO From 24d4e850c9bcad9c03884bf25c1c66cde91535e5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 16:43:43 +0200 Subject: [PATCH 1046/1795] For the sake of argument, let's show that now, this will run both the EESSI_ESPRESSO and EESS_OSU tests --- tests/eessi_test_mapping/software_to_tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/eessi_test_mapping/software_to_tests.yml b/tests/eessi_test_mapping/software_to_tests.yml index fdb51e741a..d26f7367a7 100644 --- a/tests/eessi_test_mapping/software_to_tests.yml +++ b/tests/eessi_test_mapping/software_to_tests.yml @@ -10,9 +10,7 @@ # only the bar test will be run for foo-v1 (even though it also matches the pattern (foo-*) # If a module name does not match anything, the default_tests will be run mappings: - gnuplot/*: + patchelf/*: - EESSI_ESPRESSO - Perl/*: - - EESSI_GROMACS default_tests: - EESSI_OSU From fa3a29979282ee9b4ffcb783f8a136787336b854 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 16:59:11 +0200 Subject: [PATCH 1047/1795] Final polishing --- tests/eessi_test_mapping/map_software_to_test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py index 09cb2d2987..24cf246ef1 100644 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -70,14 +70,14 @@ def main(yaml_file, module_file, debug): # Create argument string out of the list of tests to run if tests_to_run: arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run]) - if debug: - print(f"Full list of tests to run: {tests_to_run}") - print(f"Argument string: {arg_string}") - elif debug: - print(f"Full list of tests to run: No tests found") - # This is the only thing this script should return, unless run with --debug - print(f"{arg_string}") + # Print final lists & argument string + if debug: + print(f"Full list of tests to run: {tests_to_run}") + print(f"Argument string: {arg_string}") + else: + # This is the only thing this script should print, unless run with --debug + print(f"{arg_string}") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Map software names to their tests based on a YAML configuration.") From 3bccfa11b51688d1669b28657df3d77351b1e3f9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 17:47:13 +0200 Subject: [PATCH 1048/1795] Make reasonable initial mapping file --- .../eessi_test_mapping/software_to_tests.yml | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/eessi_test_mapping/software_to_tests.yml b/tests/eessi_test_mapping/software_to_tests.yml index d26f7367a7..626477781f 100644 --- a/tests/eessi_test_mapping/software_to_tests.yml +++ b/tests/eessi_test_mapping/software_to_tests.yml @@ -9,8 +9,27 @@ # - bar2 # only the bar test will be run for foo-v1 (even though it also matches the pattern (foo-*) # If a module name does not match anything, the default_tests will be run +# Note that to list all available tests by name, one can do execute +# reframe -R -c /path/to/eessi/test-suite/ --list | grep -Po "\bEESSI_\S+?(?=[\s'])" | uniq +# Note that this regular expression is a bit sensitive to changes in the structure of ReFrame's output, +# but is confirmed to work for ReFrame version 4.6.1 mappings: - patchelf/*: + PyTorch-Bundle/*: + - EESSI_PyTorch_torchvision + QuantumESPRESSO/*: + - EESSI_QuantumESPRESSO + CP2K/*: + - EESSI_CP2K + ESPResSo/*: - EESSI_ESPRESSO + LAMMPS/*: + - EESSI_LAMMPS + OSU-Micro-Benchmarks/*: + - EESSI_OSU_Micro_Benchmarks + GROMACS/*: + - EESSI_GROMACS default_tests: - - EESSI_OSU + # Low level tests + - EESSI_OSU_Micro_Benchmarks + # A very quick-to-run high level application test + - EESSI_LAMMPS From 05c0b038868e2e72510570896f46c31d74710d32 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 11:02:37 +0200 Subject: [PATCH 1049/1795] Make things more verbose, so we see where the memory limit comes from --- test_suite.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 464670b653..be11fba565 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -167,8 +167,10 @@ fi cgroup_v1_mem_limit="/sys/fs/cgroup/memory/$( Date: Tue, 20 Aug 2024 11:50:31 +0200 Subject: [PATCH 1050/1795] More verobse --- test_suite.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_suite.sh b/test_suite.sh index be11fba565..73ef5d966f 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -166,6 +166,9 @@ else fi cgroup_v1_mem_limit="/sys/fs/cgroup/memory/$( Date: Tue, 20 Aug 2024 11:50:49 +0200 Subject: [PATCH 1051/1795] add messages to help understanding which mount options were used --- eessi_container.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index 7f0c64ef27..34fa833d99 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -746,6 +746,10 @@ do # to be able to see the contents of the read-write session we have to mount # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository + echo "While processing '${cvmfs_repo_name}' to be mounted 'read-only' we detected an overlay-upper" + echo " directory (${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper) likely from a previous" + echo " session. Will use it as left-most directory in 'lowerdir' argument for fuse-overlayfs." + # make the target CernVM-FS repository available under /cvmfs_ro export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" @@ -766,6 +770,8 @@ do export EESSI_FUSE_MOUNTS else # basic "ro" access that doesn't require any fuseoverlay-fs + echo "Mounting '${cvmfs_repo_name}' 'read-only' without fuse-overlayfs." + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") From b24c30d8c68bb024dc84b8bbab6aabc2e0c9eda2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 11:56:25 +0200 Subject: [PATCH 1052/1795] Of course, we should use the mounted dir /hostsys. It is entirely possible that inside the container, the /sys directory is different --- test_suite.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 73ef5d966f..7c3b7bfb0f 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -164,11 +164,11 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi -cgroup_v1_mem_limit="/sys/fs/cgroup/memory/$( Date: Tue, 20 Aug 2024 11:59:47 +0200 Subject: [PATCH 1053/1795] get cpuset file from host as well --- bot/test.sh | 2 +- test_suite.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index d3f3630ea8..4a917af91e 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -206,7 +206,7 @@ else fi # Bind mount /sys/fs/cgroup so that we can determine the amount of memory available in our cgroup for # Reframe configuration -TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro") +TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro,/proc/self:/hostproc/self:ro"") # prepare arguments to test_suite.sh (specific to test step) declare -a TEST_SUITE_ARGS=() diff --git a/test_suite.sh b/test_suite.sh index 7c3b7bfb0f..6f42547df0 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -164,8 +164,8 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi -cgroup_v1_mem_limit="/hostsys/fs/cgroup/memory/$( Date: Tue, 20 Aug 2024 12:02:31 +0200 Subject: [PATCH 1054/1795] Removed duplicate quote --- bot/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/test.sh b/bot/test.sh index 4a917af91e..fe40a7c06c 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -206,7 +206,7 @@ else fi # Bind mount /sys/fs/cgroup so that we can determine the amount of memory available in our cgroup for # Reframe configuration -TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro,/proc/self:/hostproc/self:ro"") +TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro,/proc/self:/hostproc/self:ro") # prepare arguments to test_suite.sh (specific to test step) declare -a TEST_SUITE_ARGS=() From 8ae3eed71860ccf7293248e3808e2673796bd69c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Aug 2024 12:05:20 +0200 Subject: [PATCH 1055/1795] use more recent commit for installing STAR 2.7.11b w/ GCC/13.2.0 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index edfd491cce..a1e45f0baa 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -17,5 +17,5 @@ easyconfigs: from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 - STAR-2.7.11b-GCC-13.2.0.eb: options: - #see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 - from-commit: cd8b1ce595195f132125d9b25a98a9efe702c72a + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 + from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00 From 6d194632ae7c3b73d80e569e46e123cfb91d120d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 12:07:21 +0200 Subject: [PATCH 1056/1795] Apparently, bind-mounting the hosts /proc gives garbage, but reading it from the containers' /proc is fine. So let's do that --- test_suite.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 6f42547df0..7c3b7bfb0f 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -164,8 +164,8 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi -cgroup_v1_mem_limit="/hostsys/fs/cgroup/memory/$( Date: Tue, 20 Aug 2024 12:16:27 +0200 Subject: [PATCH 1057/1795] Remove the bind-mind for /proc, it's not needed anymore --- bot/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/test.sh b/bot/test.sh index fe40a7c06c..d3f3630ea8 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -206,7 +206,7 @@ else fi # Bind mount /sys/fs/cgroup so that we can determine the amount of memory available in our cgroup for # Reframe configuration -TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro,/proc/self:/hostproc/self:ro") +TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro") # prepare arguments to test_suite.sh (specific to test step) declare -a TEST_SUITE_ARGS=() From a8602734a213f331ccd77718b96970196b4748fa Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 12:20:28 +0200 Subject: [PATCH 1058/1795] Cleanup debugging output --- test_suite.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 7c3b7bfb0f..5dae1c1bf3 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -166,9 +166,6 @@ else fi cgroup_v1_mem_limit="/hostsys/fs/cgroup/memory/$( Date: Tue, 20 Aug 2024 13:35:58 +0200 Subject: [PATCH 1059/1795] Show effect of disableing LMOD cache --- test_suite.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test_suite.sh b/test_suite.sh index 464670b653..2a54956c50 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -81,6 +81,18 @@ echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" source $TOPDIR/init/bash +# Demonstrate the effect of disabling the cache +echo "Trying to load modules while LMOD cache is used" +# export LMOD_IGNORE_CACHE=1 +module av patchelf +module load patchelf/0.17.2-GCCcore-12.2.0.eb +command -v patchelf +echo "Trying to load modules while LMOD cache is not used" +export LMOD_IGNORE_CACHE=1 +module av patchelf +module load patchelf/0.17.2-GCCcore-12.2.0.eb +command -v patchelf + # Load the ReFrame module # Currently, we load the default version. Maybe we should somehow make this configurable in the future? module load ReFrame From 14a89cb2c74c8c8adafc5d53ecde9223130f8b01 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 13:37:13 +0200 Subject: [PATCH 1060/1795] Add patchelf to demonstrate effect of cache in test step --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index a22b78718f..6996e6f712 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -6,4 +6,5 @@ easyconfigs: from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - gnuplot-5.4.6-GCCcore-12.2.0.eb - h5py-3.8.0-foss-2022b.eb - - MDAnalysis-2.4.2-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - patchelf-0.17.2-GCCcore-12.2.0.eb From e6cceb5d8c6f014f1788419698607d3f7f6ac9ed Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 16:06:28 +0200 Subject: [PATCH 1061/1795] Fix module name --- test_suite.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 2a54956c50..bb1913e086 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -85,12 +85,12 @@ source $TOPDIR/init/bash echo "Trying to load modules while LMOD cache is used" # export LMOD_IGNORE_CACHE=1 module av patchelf -module load patchelf/0.17.2-GCCcore-12.2.0.eb +module load patchelf/0.17.2-GCCcore-12.2.0 command -v patchelf echo "Trying to load modules while LMOD cache is not used" export LMOD_IGNORE_CACHE=1 module av patchelf -module load patchelf/0.17.2-GCCcore-12.2.0.eb +module load patchelf/0.17.2-GCCcore-12.2.0 command -v patchelf # Load the ReFrame module From 129973eb71263f9ef1189d91eb7baaf83f982eae Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 20:05:47 +0200 Subject: [PATCH 1062/1795] Remove debugging output, add clarifying comments --- test_suite.sh | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index bb1913e086..41af69643a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -81,17 +81,9 @@ echo "EESSI_SOFTWARE_SUBDIR_OVERRIDE: $EESSI_SOFTWARE_SUBDIR_OVERRIDE" source $TOPDIR/init/bash -# Demonstrate the effect of disabling the cache -echo "Trying to load modules while LMOD cache is used" -# export LMOD_IGNORE_CACHE=1 -module av patchelf -module load patchelf/0.17.2-GCCcore-12.2.0 -command -v patchelf -echo "Trying to load modules while LMOD cache is not used" +# We have to ignore the LMOD cache, otherwise the software that is built in the build step cannot be found/loaded +# Reason is that the LMOD cache is normally only updated on the Stratum 0, once everything is ingested export LMOD_IGNORE_CACHE=1 -module av patchelf -module load patchelf/0.17.2-GCCcore-12.2.0 -command -v patchelf # Load the ReFrame module # Currently, we load the default version. Maybe we should somehow make this configurable in the future? From af8d9dc69f1af69b0f94749db79c92b2d1706465 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 20:06:38 +0200 Subject: [PATCH 1063/1795] Don't add patchelf, this PR wasn't really about that - we just used it to prove that disabling the LMOD cache was needed --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 6996e6f712..a22b78718f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -6,5 +6,4 @@ easyconfigs: from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - gnuplot-5.4.6-GCCcore-12.2.0.eb - h5py-3.8.0-foss-2022b.eb - - MDAnalysis-2.4.2-foss-2022b.eb - - patchelf-0.17.2-GCCcore-12.2.0.eb + - MDAnalysis-2.4.2-foss-2022b.eb From b82e18322870c36781966637daa06d17dc53f453 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 21 Aug 2024 12:03:51 +0200 Subject: [PATCH 1064/1795] Add clear description for why we use /hostsys --- test_suite.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test_suite.sh b/test_suite.sh index 5dae1c1bf3..584c9dccfc 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -164,6 +164,11 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then else fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." fi + +# The /sys inside the container is not the same as the /sys of the host +# We want to extract the memory limit from the cgroup on the host (which is typically set by SLURM). +# Thus, bot/test.sh bind-mounts the host's /sys/fs/cgroup into /hostsys/fs/cgroup +# and that's the prefix we use to extract the memory limit from cgroup_v1_mem_limit="/hostsys/fs/cgroup/memory/$( Date: Wed, 21 Aug 2024 18:50:46 +0200 Subject: [PATCH 1065/1795] {2023.06,a64fx}[foss/2023a] R v4.3.2 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index f38c06ea8b..2f884f1838 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -18,3 +18,4 @@ easyconfigs: include-easyblocks-from-commit: c8256a36e7062bc09f5ce30552a9de9827054c9e # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841 from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 + - R-4.3.2-gfbf-2023a.eb From bbba8a9ddb5cab343f8c5cc5b4ed16ae7a00780e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 16:28:27 +0200 Subject: [PATCH 1066/1795] Try to change strategy: don't gather processor info ourselves, but make sure it gets autodetected _for every job we run_ --- reframe_config_bot.py.tmpl | 15 ++++++++------- test_suite.sh | 16 ++++++++++------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl index 607373767a..dc94d97063 100644 --- a/reframe_config_bot.py.tmpl +++ b/reframe_config_bot.py.tmpl @@ -15,19 +15,20 @@ site_configuration = { 'modules_system': 'lmod', 'partitions': [ { - 'name': 'default', + 'name': '__JOBID__', 'scheduler': 'local', 'launcher': 'mpirun', 'environs': ['default'], 'features': [ FEATURES[CPU] ] + list(SCALES.keys()), - 'processor': { - 'num_cpus': __NUM_CPUS__, - 'num_sockets': __NUM_SOCKETS__, - 'num_cpus_per_core': __NUM_CPUS_PER_CORE__, - 'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__, - }, + # 'processor': { + # 'num_cpus': __NUM_CPUS__, + # 'num_sockets': __NUM_SOCKETS__, + # 'num_cpus_per_core': __NUM_CPUS_PER_CORE__, + # 'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__, + # 'num_cores_per_numa_node': __NUM_CORES_PER_NUMA_NODE__, + # }, 'resources': [ { 'name': 'memory', diff --git a/test_suite.sh b/test_suite.sh index c5a75c2d25..1901f4af43 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -201,13 +201,17 @@ else fi echo "Detected available memory: ${cgroup_mem_mib} MiB" -echo "Replacing detected system information in template ReFrame config file..." +# echo "Replacing detected system information in template ReFrame config file..." +# cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} +# sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES +# sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES +# sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES +# sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES +# sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES +echo "Replacing partition name in the template ReFrame config file, to trigger CPU autodetection for this job" cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} -sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES -sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES -sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES -sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES -sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES +sed -i "s/__JOBID__/${SLURM_JOB_ID}/g" $RFM_CONFIG_FILES + # Make debugging easier by printing the final config file: echo "Final config file (after replacements):" cat "${RFM_CONFIG_FILES}" From 784cfa4e5dfd0029d856888c6aa7a94fd37b8f94 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 16:34:57 +0200 Subject: [PATCH 1067/1795] Make sure CPU topology file gets cleaned up --- reframe_config_bot.py.tmpl | 2 +- test_suite.sh | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl index dc94d97063..ff9d627c24 100644 --- a/reframe_config_bot.py.tmpl +++ b/reframe_config_bot.py.tmpl @@ -15,7 +15,7 @@ site_configuration = { 'modules_system': 'lmod', 'partitions': [ { - 'name': '__JOBID__', + 'name': '__RFM_PARTITION__', 'scheduler': 'local', 'launcher': 'mpirun', 'environs': ['default'], diff --git a/test_suite.sh b/test_suite.sh index 1901f4af43..7c8438d30e 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -210,7 +210,8 @@ echo "Detected available memory: ${cgroup_mem_mib} MiB" # sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES echo "Replacing partition name in the template ReFrame config file, to trigger CPU autodetection for this job" cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} -sed -i "s/__JOBID__/${SLURM_JOB_ID}/g" $RFM_CONFIG_FILES +RFM_PARTITION="$SLURM_JOB_ID" +sed -i "s/__RFM_PARTITION__/${RFM_PARTITION}/g" $RFM_CONFIG_FILES # Make debugging easier by printing the final config file: echo "Final config file (after replacements):" @@ -250,4 +251,8 @@ fi echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} +RFM_SYSTEM=$(python3 -c "import ${RFM_CONFIG_FILES}; site_configuration['systems'][0]['name']") +RFM_TOPOLOGY_FILE="${HOME}/.reframe/topology/${RFM_SYSTEM}-${RFM_PARTITION}/processor.json" +echo ">> Cleaning up ReFrame CPU topology file" + exit ${reframe_exit_code} From ba21ef5324628b9350ad64ed5159722d46008ed8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 16:40:01 +0200 Subject: [PATCH 1068/1795] Fix actual cleanup... --- test_suite.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 7c8438d30e..f0ba64eda2 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -252,7 +252,8 @@ echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} RFM_SYSTEM=$(python3 -c "import ${RFM_CONFIG_FILES}; site_configuration['systems'][0]['name']") -RFM_TOPOLOGY_FILE="${HOME}/.reframe/topology/${RFM_SYSTEM}-${RFM_PARTITION}/processor.json" +RFM_TOPOLOGY_DIR="${HOME}/.reframe/topology/${RFM_SYSTEM}-${RFM_PARTITION}" echo ">> Cleaning up ReFrame CPU topology file" +rm -rf ${RFM_TOPOLOGY_DIR} exit ${reframe_exit_code} From 011391a29c5a0e4a6b9aa40e742243d3bae2d247 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 16:50:04 +0200 Subject: [PATCH 1069/1795] Still needed to replace memory limits, doing that now --- test_suite.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index f0ba64eda2..ffbfa1887a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -202,14 +202,14 @@ fi echo "Detected available memory: ${cgroup_mem_mib} MiB" # echo "Replacing detected system information in template ReFrame config file..." -# cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} +cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} # sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES # sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES # sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES # sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES -# sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES +sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES echo "Replacing partition name in the template ReFrame config file, to trigger CPU autodetection for this job" -cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} +# cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} RFM_PARTITION="$SLURM_JOB_ID" sed -i "s/__RFM_PARTITION__/${RFM_PARTITION}/g" $RFM_CONFIG_FILES From 4463d73b49e450ce687063043983e253cba12b48 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 17:03:56 +0200 Subject: [PATCH 1070/1795] ReFrame forces me to start the partition name with an a-z character --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index ffbfa1887a..1e7d0ba9a9 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -210,7 +210,7 @@ cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES echo "Replacing partition name in the template ReFrame config file, to trigger CPU autodetection for this job" # cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} -RFM_PARTITION="$SLURM_JOB_ID" +RFM_PARTITION="job-$SLURM_JOB_ID" sed -i "s/__RFM_PARTITION__/${RFM_PARTITION}/g" $RFM_CONFIG_FILES # Make debugging easier by printing the final config file: From 2f11592d1bb00aa21bd0b58f5aef306201ea2736 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 17:08:30 +0200 Subject: [PATCH 1071/1795] Get the partition name from the environment. This way, we only need to auto-detect once per partition. Then, the file is cached and available to be used in the next run! --- test_suite.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 1e7d0ba9a9..6a30a4318e 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -210,7 +210,7 @@ cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES echo "Replacing partition name in the template ReFrame config file, to trigger CPU autodetection for this job" # cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} -RFM_PARTITION="job-$SLURM_JOB_ID" +RFM_PARTITION="${SLURM_JOB_PARTITION}" sed -i "s/__RFM_PARTITION__/${RFM_PARTITION}/g" $RFM_CONFIG_FILES # Make debugging easier by printing the final config file: @@ -251,9 +251,4 @@ fi echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} -RFM_SYSTEM=$(python3 -c "import ${RFM_CONFIG_FILES}; site_configuration['systems'][0]['name']") -RFM_TOPOLOGY_DIR="${HOME}/.reframe/topology/${RFM_SYSTEM}-${RFM_PARTITION}" -echo ">> Cleaning up ReFrame CPU topology file" -rm -rf ${RFM_TOPOLOGY_DIR} - exit ${reframe_exit_code} From 7eacbf494c5e1e4ea6c9108d0e726fadfde3a8c6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 17:10:33 +0200 Subject: [PATCH 1072/1795] Enable remote detect. I'm not sure if this is correct, since we are using local spawner. But right now, it doesn't seem to detect anything --- reframe_config_bot.py.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl index ff9d627c24..8f3c74134b 100644 --- a/reframe_config_bot.py.tmpl +++ b/reframe_config_bot.py.tmpl @@ -57,8 +57,7 @@ site_configuration = { { 'purge_environment': True, 'resolve_module_conflicts': False, # avoid loading the module before submitting the job - # disable automatic detection of CPU architecture (since we're using local scheduler) - 'remote_detect': False, + 'remote_detect': True, } ], 'logging': common_logging_config(), From c6056abf2c207d1c7d8543218a97a0718299f42e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 17:17:11 +0200 Subject: [PATCH 1073/1795] Cleanup of code that is no longer needed --- test_suite.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 6a30a4318e..de1c64d977 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -203,14 +203,10 @@ echo "Detected available memory: ${cgroup_mem_mib} MiB" # echo "Replacing detected system information in template ReFrame config file..." cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} -# sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES -# sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES -# sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES -# sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES +echo "Replacing memory limit in the ReFrame config file with the detected CGROUP memory limit: ${cgroup_mem_mib} MiB" sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES -echo "Replacing partition name in the template ReFrame config file, to trigger CPU autodetection for this job" -# cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} RFM_PARTITION="${SLURM_JOB_PARTITION}" +echo "Replacing partition name in the template ReFrame config file: ${RFM_PARTITION}" sed -i "s/__RFM_PARTITION__/${RFM_PARTITION}/g" $RFM_CONFIG_FILES # Make debugging easier by printing the final config file: From bbc3566bb6427b2f04f1e4917d01317e92fcb3d4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 22 Aug 2024 17:19:32 +0200 Subject: [PATCH 1074/1795] Cleanup comments --- reframe_config_bot.py.tmpl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl index 8f3c74134b..323aafd5ec 100644 --- a/reframe_config_bot.py.tmpl +++ b/reframe_config_bot.py.tmpl @@ -22,13 +22,6 @@ site_configuration = { 'features': [ FEATURES[CPU] ] + list(SCALES.keys()), - # 'processor': { - # 'num_cpus': __NUM_CPUS__, - # 'num_sockets': __NUM_SOCKETS__, - # 'num_cpus_per_core': __NUM_CPUS_PER_CORE__, - # 'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__, - # 'num_cores_per_numa_node': __NUM_CORES_PER_NUMA_NODE__, - # }, 'resources': [ { 'name': 'memory', From 2a339c3bb4f07af6698cdc65c75a90f1b8754665 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:56:41 +0200 Subject: [PATCH 1075/1795] Create 20240719-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml --- ...2-GObject-Introspection-filter-envvars-a64fx.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml new file mode 100644 index 0000000000..782db66e78 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml @@ -0,0 +1,13 @@ +# 2024.08.23 +# GObject-Introspection sets $LD_LIBRARY_PATH (to many different paths, including $EPREFIX/lib) +# when calling gcc, and this causes a lot of issues for, especially, scripts using /bin/bash. +# +# This rebuild ensures (by using a new EasyBuild hook) that GObject-Introspection will not set +# environment variables that are configured to be filtered by EasyBuild. +# This rebuild was not done initially for A64FX. This file is meant to do the same as the +# previous rebuild of GObject-Introspection-1.76.1-GCCcore-12.3.0 in other architectures, +# but for A64FX. +# +# See https://github.com/EESSI/software-layer/issues/196 +easyconfigs: + - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb From 73cf81abc14185c42440bad5660bf023e44d8146 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 23 Aug 2024 10:01:04 +0200 Subject: [PATCH 1076/1795] Renamed to correct date --- ...40823-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/rebuilds/{20240719-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml => 20240823-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240823-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240823-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml From b3b106b34d310f644193f25ed1ec5693712b0a9f Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 26 Aug 2024 06:00:05 +0000 Subject: [PATCH 1077/1795] {2023.06}[foss/2023a] wradlib v2.0.3 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e8fe8ab120..1d19379346 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -45,3 +45,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 + - wradlib-2.0.3-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21094 + from-commit: 9807cf9f13a9e969b447f136e9e8ad9e5c4548a2 From 8f75d56010281cbe363c016c792c57441c64b869 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 08:12:56 +0200 Subject: [PATCH 1078/1795] Update init/modules/EESSI/2023.06.lua MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Röblitz --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 3cccffd2bb..20c58218be 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -8,7 +8,7 @@ More information - URL: https://www.eessi.io/docs/ ]]) whatis("Description: The European Environment for Scientific Software Installations (EESSI, pronounced as easy) is a collaboration between different European partners in HPC community. The goal of this project is to build a common stack of scientific software installations for HPC systems and beyond, including laptops, personal workstations and cloud infrastructure.") -whatis("URL: https://www.eessi.io/docs/:") +whatis("URL: https://www.eessi.io/docs/") local eessi_version = myModuleVersion() local eessi_repo = "/cvmfs/software.eessi.io" From 05820184e4046fc3856f35fc4739d47a6361a7aa Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 09:51:51 +0200 Subject: [PATCH 1079/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 6cec64380a..b14a04289e 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -30,6 +30,7 @@ jobs: export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 + . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash if module load EESSI/${{matrix.EESSI_VERSION}}; then if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then echo "Test for picking up on \$archdetect_cpu PASSED" @@ -42,8 +43,7 @@ jobs: unset EESSI_ARCHDETECT_OPTIONS - name: Test for archdetect_cpu functionality with invalid path - run: | - export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH + run: | export EESSI_ARCHDETECT_OPTIONS=dummy/cpu outfile=outfile.txt module load EESSI/${{matrix.EESSI_VERSION}} > $outfile 2>&1 @@ -57,8 +57,7 @@ jobs: unset EESSI_ARCHDETECT_OPTIONS - name: Test for expected variables after loading EESSI module - run: | - export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH + run: | moduleoutfile=moduleout.txt sourceoutfile=sourceout.txt module load EESSI/${{matrix.EESSI_VERSION}} From a7d7aff8928f1aae1260f0d5f0f291d48e4dc282 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:01:28 +0200 Subject: [PATCH 1080/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index b14a04289e..31e790c90a 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -31,7 +31,7 @@ jobs: CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash - if module load EESSI/${{matrix.EESSI_VERSION}}; then + if module --ignore_cache load EESSI/${{matrix.EESSI_VERSION}}; then if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then echo "Test for picking up on \$archdetect_cpu PASSED" else From 53bc4a0145aba4e2d064e44fd60546cb9effdb3e Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:42:23 +0200 Subject: [PATCH 1081/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 31e790c90a..e668a3f2bc 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -31,6 +31,7 @@ jobs: CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash + module avail EESSI if module --ignore_cache load EESSI/${{matrix.EESSI_VERSION}}; then if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then echo "Test for picking up on \$archdetect_cpu PASSED" From f4ee932d23db0ed432f69ca6f5fc77c3191c6504 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:46:29 +0200 Subject: [PATCH 1082/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index e668a3f2bc..2e3b824ccc 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -27,10 +27,10 @@ jobs: - name: Test for archdetect_cpu functionality with only one valid path run: | - export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules/:$MODULEPATH CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash + export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules module avail EESSI if module --ignore_cache load EESSI/${{matrix.EESSI_VERSION}}; then if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then From 51700885e575117521edf426d7eadd528adc79d8 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:13:31 +0200 Subject: [PATCH 1083/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 2e3b824ccc..4b0fbe881f 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -31,6 +31,8 @@ jobs: export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules + echo $MODULEPATH + env | grep LMOD module avail EESSI if module --ignore_cache load EESSI/${{matrix.EESSI_VERSION}}; then if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then From 3cea84681ad2bb312c7d83aa2df6f8d584f37bea Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:28:47 +0200 Subject: [PATCH 1084/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 4b0fbe881f..b6bd865e4c 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -31,7 +31,7 @@ jobs: export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules - echo $MODULEPATH + ls $MODULEPATH/EESSI env | grep LMOD module avail EESSI if module --ignore_cache load EESSI/${{matrix.EESSI_VERSION}}; then From 5484b921fc1289bb71cc0a28a9714d0991bca2a3 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:18:10 +0200 Subject: [PATCH 1085/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 43 +++++++++--------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index b6bd865e4c..fdbd2fac96 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -25,32 +25,17 @@ jobs: cvmfs_http_proxy: DIRECT cvmfs_repositories: software.eessi.io - - name: Test for archdetect_cpu functionality with only one valid path + - name: Initialise Lmod run: | - CPU_ARCH=$(./init/eessi_archdetect.sh cpupath) - export EESSI_ARCHDETECT_OPTIONS=dummy/cpu:$CPU_ARCH:dummy1/cpu1 + export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash - export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules - ls $MODULEPATH/EESSI - env | grep LMOD - module avail EESSI - if module --ignore_cache load EESSI/${{matrix.EESSI_VERSION}}; then - if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR}" ]]; then - echo "Test for picking up on \$archdetect_cpu PASSED" - else - echo "Test for picking up on \$archdetect_cpu FAILED" >&2 - exit 1 - fi - module unload EESSI/${{matrix.EESSI_VERSION}} - fi - unset EESSI_ARCHDETECT_OPTIONS - + - name: Test for archdetect_cpu functionality with invalid path run: | - export EESSI_ARCHDETECT_OPTIONS=dummy/cpu - outfile=outfile.txt - module load EESSI/${{matrix.EESSI_VERSION}} > $outfile 2>&1 - result=$(grep "Software" $outfile) + export EESSI_ARCHDETECT_OPTIONS="dummy/cpu" + outfile="outfile.txt" + module load EESSI/${{matrix.EESSI_VERSION}} > "$outfile" 2>&1 + result=$(grep "Software" "$outfile") if [[ "$result" == *"Software directory check for the detected architecture failed"* ]]; then echo "Test for picking up invalid path on \$archdetect_cpu PASSED" else @@ -60,15 +45,17 @@ jobs: unset EESSI_ARCHDETECT_OPTIONS - name: Test for expected variables after loading EESSI module - run: | - moduleoutfile=moduleout.txt - sourceoutfile=sourceout.txt + run: | + CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) + export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:$CPU_ARCH:dummy1/cpu1" + moduleoutfile="moduleout.txt" + sourceoutfile="sourceout.txt" module load EESSI/${{matrix.EESSI_VERSION}} - env | grep -E '^(EESSI_S|EESSI_C)' | sort | > ${moduleoutfile} + env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "$moduleoutfile" module unload EESSI/${{matrix.EESSI_VERSION}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash - env | grep -E '^(EESSI_S|EESSI_C)' | sort | > ${sourceoutfile} - if (diff ${moduleoutfile} ${sourceoutfile} > /dev/null); then + env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "$sourceoutfile" + if (diff "$moduleoutfile" "$sourceoutfile" > /dev/null); then echo "Test for checking env variables PASSED" else echo "Test for checking env variables FAILED" >&2 From 63002d5f898f67a008f271ddfd434e71e9cfbeb4 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:47:05 +0200 Subject: [PATCH 1086/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index fdbd2fac96..b2294fe171 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -31,31 +31,33 @@ jobs: . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash - name: Test for archdetect_cpu functionality with invalid path - run: | + run: | + set +e # Do not exit immediately if a command exits with a non-zero status export EESSI_ARCHDETECT_OPTIONS="dummy/cpu" outfile="outfile.txt" - module load EESSI/${{matrix.EESSI_VERSION}} > "$outfile" 2>&1 - result=$(grep "Software" "$outfile") - if [[ "$result" == *"Software directory check for the detected architecture failed"* ]]; then - echo "Test for picking up invalid path on \$archdetect_cpu PASSED" + module load EESSI/${{matrix.EESSI_VERSION}} > "${outfile}" 2>&1 + result=$(grep "Software" "${outfile}") + if [[ "${result}" == *"Software directory check for the detected architecture failed"* ]]; then + echo "Test for picking up invalid path on \${archdetect_cpu} PASSED" else - echo "Test for picking up invalid path on \$archdetect_cpu FAILED" >&2 + echo "Test for picking up invalid path on \${archdetect_cpu} FAILED" >&2 exit 1 fi unset EESSI_ARCHDETECT_OPTIONS + set -e # Re-enable exit on non-zero status - - name: Test for expected variables after loading EESSI module + - name: Test for expected variables while adding dummy cpu archs and loading EESSI module run: | CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) - export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:$CPU_ARCH:dummy1/cpu1" + export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" moduleoutfile="moduleout.txt" sourceoutfile="sourceout.txt" module load EESSI/${{matrix.EESSI_VERSION}} - env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "$moduleoutfile" + env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash - env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "$sourceoutfile" - if (diff "$moduleoutfile" "$sourceoutfile" > /dev/null); then + env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "${sourceoutfile}" + if (diff "${moduleoutfile}" "${sourceoutfile}" > /dev/null); then echo "Test for checking env variables PASSED" else echo "Test for checking env variables FAILED" >&2 From f8240b2a16c9f9e8b56b4288fc48c543f7998270 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 26 Aug 2024 12:43:59 +0000 Subject: [PATCH 1087/1795] {2023.06}[foss/2023a] MBX v1.1.0 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e8fe8ab120..c42edfbc5a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -45,3 +45,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 + - MBX-1.1.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21155 + from-commit: 6929a67401f2a2ec58f91fb306332a77497d73ff From 50782e4324967dd27e4a818704ca1d24fbef24dc Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 08:17:42 +0200 Subject: [PATCH 1088/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index b2294fe171..2e521f5809 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -27,9 +27,9 @@ jobs: - name: Initialise Lmod run: | - export MODULEPATH=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/modules . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash - + module use init/modules + - name: Test for archdetect_cpu functionality with invalid path run: | set +e # Do not exit immediately if a command exits with a non-zero status From 477476b05a3b6fc23f0112d2eb6aaaf445be5f89 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 08:33:09 +0200 Subject: [PATCH 1089/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 2e521f5809..335b8d9a59 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -36,6 +36,7 @@ jobs: export EESSI_ARCHDETECT_OPTIONS="dummy/cpu" outfile="outfile.txt" module load EESSI/${{matrix.EESSI_VERSION}} > "${outfile}" 2>&1 + cat "${outfile}" result=$(grep "Software" "${outfile}") if [[ "${result}" == *"Software directory check for the detected architecture failed"* ]]; then echo "Test for picking up invalid path on \${archdetect_cpu} PASSED" From b651030abaeb072af74843407c5a6fda16f20f9e Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:15:16 +0200 Subject: [PATCH 1090/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 335b8d9a59..9b2e8aeab5 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -28,6 +28,7 @@ jobs: - name: Initialise Lmod run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash + module --version module use init/modules - name: Test for archdetect_cpu functionality with invalid path From 543f7601d8dd0c5059a0bc79e18e1104d51afa27 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:19:10 +0200 Subject: [PATCH 1091/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 9b2e8aeab5..f3f26208ad 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -25,14 +25,9 @@ jobs: cvmfs_http_proxy: DIRECT cvmfs_repositories: software.eessi.io - - name: Initialise Lmod - run: | - . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash - module --version - module use init/modules - - name: Test for archdetect_cpu functionality with invalid path run: | + . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod set +e # Do not exit immediately if a command exits with a non-zero status export EESSI_ARCHDETECT_OPTIONS="dummy/cpu" outfile="outfile.txt" From 7acc85ed5e147fc6e083cb97dc472ffa51dcf79b Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:25:06 +0200 Subject: [PATCH 1092/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index f3f26208ad..8b7911e0c4 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -28,6 +28,7 @@ jobs: - name: Test for archdetect_cpu functionality with invalid path run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod + export MODULEPATH=init/modules set +e # Do not exit immediately if a command exits with a non-zero status export EESSI_ARCHDETECT_OPTIONS="dummy/cpu" outfile="outfile.txt" @@ -45,6 +46,8 @@ jobs: - name: Test for expected variables while adding dummy cpu archs and loading EESSI module run: | + . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod + export MODULEPATH=init/modules CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" moduleoutfile="moduleout.txt" From 7791be4c9f3ff14b0c224237f80de30dd67f3a65 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:34:37 +0200 Subject: [PATCH 1093/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 8b7911e0c4..3a7c217719 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -34,7 +34,7 @@ jobs: outfile="outfile.txt" module load EESSI/${{matrix.EESSI_VERSION}} > "${outfile}" 2>&1 cat "${outfile}" - result=$(grep "Software" "${outfile}") + result=$(cat "${outfile}") if [[ "${result}" == *"Software directory check for the detected architecture failed"* ]]; then echo "Test for picking up invalid path on \${archdetect_cpu} PASSED" else From 3fab812be5fce883a63bc0bf0aaa6c619dda3a34 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:48:30 +0200 Subject: [PATCH 1094/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 3a7c217719..ab7fec4953 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -34,8 +34,7 @@ jobs: outfile="outfile.txt" module load EESSI/${{matrix.EESSI_VERSION}} > "${outfile}" 2>&1 cat "${outfile}" - result=$(cat "${outfile}") - if [[ "${result}" == *"Software directory check for the detected architecture failed"* ]]; then + if grep -q "Software directory check" "${outfile}"; then echo "Test for picking up invalid path on \${archdetect_cpu} PASSED" else echo "Test for picking up invalid path on \${archdetect_cpu} FAILED" >&2 From 623fd67d4d376c0accb3606e522d344bfb1a01a1 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:56:32 +0200 Subject: [PATCH 1095/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index ab7fec4953..e40f1e9903 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -56,6 +56,8 @@ jobs: module unload EESSI/${{matrix.EESSI_VERSION}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "${sourceoutfile}" + cat "${moduleoutfile}" + cat "${sourceoutfile}" if (diff "${moduleoutfile}" "${sourceoutfile}" > /dev/null); then echo "Test for checking env variables PASSED" else From 2e4d5d6ac8027f3516d3683da0087720f1334c4d Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:03:31 +0200 Subject: [PATCH 1096/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index e40f1e9903..8a26638438 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -52,10 +52,10 @@ jobs: moduleoutfile="moduleout.txt" sourceoutfile="sourceout.txt" module load EESSI/${{matrix.EESSI_VERSION}} - env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "${moduleoutfile}" + env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash - env | grep -E '^(EESSI_S|EESSI_C)' | sort | > "${sourceoutfile}" + env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${sourceoutfile}" cat "${moduleoutfile}" cat "${sourceoutfile}" if (diff "${moduleoutfile}" "${sourceoutfile}" > /dev/null); then From 090133cb7f7c998bb2e8f1160638691f9a511b5a Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:26:50 +0200 Subject: [PATCH 1097/1795] Update install_scripts.sh --- install_scripts.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index 0cfa5310c2..d880679fbc 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -106,7 +106,11 @@ copy_files_by_list ${TOPDIR}/init/Magic_Castle ${INSTALL_PREFIX}/init/Magic_Cast mc_files=( 2023.06.lua ) -copy_files_by_list ${TOPDIR}/init/modules/EESSI ${INSTALL_PREFIX}/init/modules/EESSI "${mc_files[@]}" +if [ -n "$EESSI_CVMFS_REPO" ]; then + copy_files_by_list ${TOPDIR}/init/modules/EESSI ${EESSI_CVMFS_REPO}/modules/EESSI "${mc_files[@]}" +else + echo "Error: EESSI_CVMFS_REPO is required for copying the module files." >&2 +fi # Copy for the scripts directory script_files=( From 53e51d950c8d56bd1eeccc8e4a18ccdeaf326e77 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 27 Aug 2024 13:11:57 +0000 Subject: [PATCH 1098/1795] Added MBX to memory-hungry-list --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 73e1ff7df6..6cac9b8609 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -83,7 +83,7 @@ def post_ready_hook(self, *args, **kwargs): # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores. # here we reduce parallellism to only use half of that for selected software, # to avoid failing builds/tests due to out-of-memory problems; - memory_hungry_build = self.name in ['libxc', 'TensorFlow'] + memory_hungry_build = self.name in ['libxc', 'MBX', 'TensorFlow'] # on A64FX systems, (HBM) memory is typically scarce, so we need to use fewer cores for some builds cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') memory_hungry_build_a64fx = cpu_target == CPU_TARGET_A64FX and self.name in ['Qt5'] From 3ee468c238a189d5756575d6f639f6ebb9272e51 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 30 Aug 2024 07:11:02 +0000 Subject: [PATCH 1099/1795] {2023.06}[GCC/12.3.0] Transrate v1.0.3 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e8fe8ab120..d0d8058e6e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -45,3 +45,9 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 + - Transrate-1.0.3-GCC-12.3.0.eb: + options: + # https://github.com/easybuilders/easybuild-easyblocks/pull/3381 + include-easyblocks-from-commit: bb86f05d4917b29e022023f152efdf0ca5c14ded + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20964 + from-commit: 7d539a9e599d8bc5ac2bda6ee9587ef62351ee03 From 8cea03bb7a149da2a86066d0f31f453f41cf6933 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:29:38 +0200 Subject: [PATCH 1100/1795] Update 2023.06.lua --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 20c58218be..915756173e 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -9,7 +9,7 @@ More information ]]) whatis("Description: The European Environment for Scientific Software Installations (EESSI, pronounced as easy) is a collaboration between different European partners in HPC community. The goal of this project is to build a common stack of scientific software installations for HPC systems and beyond, including laptops, personal workstations and cloud infrastructure.") whatis("URL: https://www.eessi.io/docs/") - +conflict("EESSI") local eessi_version = myModuleVersion() local eessi_repo = "/cvmfs/software.eessi.io" local eessi_prefix = pathJoin(eessi_repo, "versions", eessi_version) From a5d78121ed056166267dfab8b1890026102f66f5 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:34:05 +0200 Subject: [PATCH 1101/1795] Update install_scripts.sh --- install_scripts.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/install_scripts.sh b/install_scripts.sh index d880679fbc..0cfa5310c2 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -106,11 +106,7 @@ copy_files_by_list ${TOPDIR}/init/Magic_Castle ${INSTALL_PREFIX}/init/Magic_Cast mc_files=( 2023.06.lua ) -if [ -n "$EESSI_CVMFS_REPO" ]; then - copy_files_by_list ${TOPDIR}/init/modules/EESSI ${EESSI_CVMFS_REPO}/modules/EESSI "${mc_files[@]}" -else - echo "Error: EESSI_CVMFS_REPO is required for copying the module files." >&2 -fi +copy_files_by_list ${TOPDIR}/init/modules/EESSI ${INSTALL_PREFIX}/init/modules/EESSI "${mc_files[@]}" # Copy for the scripts directory script_files=( From 6a1d61526653eb91a39654316f9133fe29092ae5 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:41:34 +0200 Subject: [PATCH 1102/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 8a26638438..9149f336c2 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -24,7 +24,18 @@ jobs: cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb cvmfs_http_proxy: DIRECT cvmfs_repositories: software.eessi.io - + + - name: Test for making sure spider cache is being used and not being rebuilt + run: | + . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod + export MODULEPATH=init/modules + if timeout 10s bash -c "LMOD_PAGER=none module --terse avail"; then + echo "EESSI spider cache is being used" + else + echo "EESSI spider cache is being rebuilt" >&2 + exit 1 + fi + - name: Test for archdetect_cpu functionality with invalid path run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod From 69a7e660b47de43e544aa1600c9e76caa3830f28 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:57:27 +0200 Subject: [PATCH 1103/1795] Update .github/workflows/tests_eessi_module.yml Co-authored-by: ocaisa --- .github/workflows/tests_eessi_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 9149f336c2..4f261b85d1 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -29,6 +29,7 @@ jobs: run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod export MODULEPATH=init/modules + module load EESSI/${{matrix.EESSI_VERSION}} if timeout 10s bash -c "LMOD_PAGER=none module --terse avail"; then echo "EESSI spider cache is being used" else From c7dce8cff91f8e97eb7241bc11ebf5f19a1639c4 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:57:39 +0200 Subject: [PATCH 1104/1795] Update .github/workflows/tests_eessi_module.yml Co-authored-by: ocaisa --- .github/workflows/tests_eessi_module.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 4f261b85d1..9d668e1aec 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -36,6 +36,8 @@ jobs: echo "EESSI spider cache is being rebuilt" >&2 exit 1 fi + module purge + unset MODULEPATH - name: Test for archdetect_cpu functionality with invalid path run: | From cc3b877c6432e322e9e0b35868a48f98a6ca93e1 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 15:54:50 +0200 Subject: [PATCH 1105/1795] Update 2023.06.lua --- init/modules/EESSI/2023.06.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 915756173e..7770fe756c 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -58,8 +58,10 @@ prepend_path("PATH", pathJoin(eessi_eprefix, "bin")) prepend_path("PATH", pathJoin(eessi_eprefix, "usr/bin")) setenv("EESSI_SOFTWARE_PATH", eessi_software_path) setenv("EESSI_MODULEPATH", eessi_module_path) -prepend_path("MODULEPATH", eessi_module_path) -prepend_path("MODULEPATH", eessi_site_module_path) +if ( mode() ~= "spider" ) then + prepend_path("MODULEPATH", eessi_module_path) + prepend_path("MODULEPATH", eessi_site_module_path) +end setenv("LMOD_CONFIG_DIR", pathJoin(eessi_software_path, ".lmod")) setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) if mode() == "load" then From fc64dcf829f29a496f06805942119aa47882fde1 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 15:56:09 +0200 Subject: [PATCH 1106/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 9d668e1aec..27896b9b50 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -36,6 +36,7 @@ jobs: echo "EESSI spider cache is being rebuilt" >&2 exit 1 fi + env | grep LMOD module purge unset MODULEPATH From c8141a91a11c827c1809089902a49d048e58b7f6 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 18:16:21 +0200 Subject: [PATCH 1107/1795] Update 2023.06.lua --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 7770fe756c..61a35d20a1 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -62,7 +62,7 @@ if ( mode() ~= "spider" ) then prepend_path("MODULEPATH", eessi_module_path) prepend_path("MODULEPATH", eessi_site_module_path) end -setenv("LMOD_CONFIG_DIR", pathJoin(eessi_software_path, ".lmod")) +prepend_path("LMOD_RC", pathJoin(eessi_software_path, "/.lmod/lmodrc.lua")) setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) if mode() == "load" then LmodMessage("EESSI/" .. eessi_version .. " loaded successfully") From e688d65e73e79f15f7bd33ca5da98909c8891b52 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 19:05:39 +0200 Subject: [PATCH 1108/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 27896b9b50..f0f7fa09a3 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -29,8 +29,11 @@ jobs: run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod export MODULEPATH=init/modules + configfile="configfile.txt" module load EESSI/${{matrix.EESSI_VERSION}} - if timeout 10s bash -c "LMOD_PAGER=none module --terse avail"; then + module --config > "${configfile.txt}" 2>&1 + grep cache "${configfile}" | grep software | grep -v compat + if timeout 10s bash -c "LMOD_PAGER=none module --terse avail" && grep cache "${configfile}" | grep software | grep -v compat; then echo "EESSI spider cache is being used" else echo "EESSI spider cache is being rebuilt" >&2 From 17b512d7e7223f58e5cba054898ee31bf1c88d72 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Fri, 30 Aug 2024 19:08:30 +0200 Subject: [PATCH 1109/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index f0f7fa09a3..caf2715f93 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -31,7 +31,7 @@ jobs: export MODULEPATH=init/modules configfile="configfile.txt" module load EESSI/${{matrix.EESSI_VERSION}} - module --config > "${configfile.txt}" 2>&1 + module --config > "${configfile}" 2>&1 grep cache "${configfile}" | grep software | grep -v compat if timeout 10s bash -c "LMOD_PAGER=none module --terse avail" && grep cache "${configfile}" | grep software | grep -v compat; then echo "EESSI spider cache is being used" From de91ea392529a225f0ebb69aff7d8a2a4ce4ff90 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 2 Sep 2024 08:16:26 +0200 Subject: [PATCH 1110/1795] Update 2023.06.lua To prevent error bad argument #1 to 'gmatch' when module show is used and EESSI_ARCHDETECT_OPTIONS is not set --- init/modules/EESSI/2023.06.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 61a35d20a1..9ad85f42d8 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -25,7 +25,8 @@ function archdetect_cpu() end source_sh("bash", script) end - for archdetect_filter_cpu in string.gmatch(os.getenv("EESSI_ARCHDETECT_OPTIONS"), "([^" .. ":" .. "]+)") do +local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or "" + for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, -- since optimized software installations for Zen4 are a work-in-progress, From 74dc6e9346a62888d6283318c2d46f368e3f79ef Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 2 Sep 2024 11:59:37 +0000 Subject: [PATCH 1111/1795] {2023.06}[foss/2023a] Critic2 v1.2 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e8fe8ab120..c624316089 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -45,3 +45,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 + - Critic2-1.2-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20833 + from-commit: 78426c2383fc7e4b9b9e77d7a77f336e1bee3843 From dff4b1313d85d7045c85dd6d0f9b305e711e8e91 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Mon, 2 Sep 2024 18:43:12 +0200 Subject: [PATCH 1112/1795] Update tests_eessi_module.yml --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index caf2715f93..d2e3cd1338 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -30,7 +30,7 @@ jobs: . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod export MODULEPATH=init/modules configfile="configfile.txt" - module load EESSI/${{matrix.EESSI_VERSION}} + module -T load EESSI/${{matrix.EESSI_VERSION}} module --config > "${configfile}" 2>&1 grep cache "${configfile}" | grep software | grep -v compat if timeout 10s bash -c "LMOD_PAGER=none module --terse avail" && grep cache "${configfile}" | grep software | grep -v compat; then From 0a615423859d35955ca3e7e1125d20ad7cb6da56 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 3 Sep 2024 08:04:56 +0200 Subject: [PATCH 1113/1795] Update init/modules/EESSI/2023.06.lua Co-authored-by: ocaisa --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 9ad85f42d8..ddc0fef561 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -25,7 +25,7 @@ function archdetect_cpu() end source_sh("bash", script) end -local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or "" + local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or "" for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, From 47959c5df773bb9e57240a527e92d16612f20275 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:28:57 +0200 Subject: [PATCH 1114/1795] Update 2023.06.lua --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index ddc0fef561..32aaf6c07f 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -61,9 +61,9 @@ setenv("EESSI_SOFTWARE_PATH", eessi_software_path) setenv("EESSI_MODULEPATH", eessi_module_path) if ( mode() ~= "spider" ) then prepend_path("MODULEPATH", eessi_module_path) - prepend_path("MODULEPATH", eessi_site_module_path) end prepend_path("LMOD_RC", pathJoin(eessi_software_path, "/.lmod/lmodrc.lua")) +prepend_path("MODULEPATH", eessi_site_module_path) setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) if mode() == "load" then LmodMessage("EESSI/" .. eessi_version .. " loaded successfully") From 4b0445fd9e3e1fc33c173052f679cba61a20dc2d Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Tue, 3 Sep 2024 15:25:35 +0200 Subject: [PATCH 1115/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 1d19379346..597ef0152b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -48,4 +48,4 @@ easyconfigs: - wradlib-2.0.3-foss-2023a.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21094 - from-commit: 9807cf9f13a9e969b447f136e9e8ad9e5c4548a2 + from-commit: 3a2e0b8e6ee45277d01fb7e2eb93027a28c9461a From 238b4074cdb3c8fe6f28681e900f217b25ef0bfe Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 3 Sep 2024 18:10:00 +0200 Subject: [PATCH 1116/1795] Cleanup of code that is now no longer needed since we use ReFrame's CPU autodetection --- test_suite.sh | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index de1c64d977..f42c164872 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -141,34 +141,6 @@ export RFM_PREFIX=$PWD/reframe_runs echo "Configured reframe with the following environment variables:" env | grep "RFM_" -# Inject correct CPU/memory properties into the ReFrame config file -echo "Collecting system-specific input for the ReFrame configuration file" -cpuinfo=$(lscpu) -if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then - cpu_count=${BASH_REMATCH[1]} - echo "Detected CPU count: ${cpu_count}" -else - fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu." -fi -if [[ "${cpuinfo}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then - socket_count=${BASH_REMATCH[1]} - echo "Detected socket count: ${socket_count}" -else - fatal_error "Failed to get the number of sockets for the current test hardware with lscpu." -fi -if [[ "${cpuinfo}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then - threads_per_core=${BASH_REMATCH[2]} - echo "Detected threads per core: ${threads_per_core}" -else - fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu." -fi -if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then - cores_per_socket=${BASH_REMATCH[2]} - echo "Detected cores per socket: ${cores_per_socket}" -else - fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu." -fi - # The /sys inside the container is not the same as the /sys of the host # We want to extract the memory limit from the cgroup on the host (which is typically set by SLURM). # Thus, bot/test.sh bind-mounts the host's /sys/fs/cgroup into /hostsys/fs/cgroup From 76b83b7075e321213033247140771eaab5212d6c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 3 Sep 2024 18:38:27 +0200 Subject: [PATCH 1117/1795] Add hpl for 2023b --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index a1e45f0baa..8b99e5c506 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -19,3 +19,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00 + - HPL-2.3-foss-2023b.eb From 21eb5d34ce2596397b76c281cc5c3f5a0f572a6c Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Wed, 4 Sep 2024 08:59:28 +0200 Subject: [PATCH 1118/1795] Update eessi-2023.06-eb-4.9.2-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 74eb86d9ba..fc8e4c4d9b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -59,7 +59,7 @@ easyconfigs: include-easyblocks-from-commit: bb86f05d4917b29e022023f152efdf0ca5c14ded # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20964 from-commit: 7d539a9e599d8bc5ac2bda6ee9587ef62351ee03 - - Critic2-1.2-foss-2023a.eb: + - Critic2-1.2-foss-2023a.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20833 from-commit: 78426c2383fc7e4b9b9e77d7a77f336e1bee3843 From c6e0cc29a7badc4d32bc9eca45e9976bd7fe55fc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 4 Sep 2024 10:52:17 +0200 Subject: [PATCH 1119/1795] Remove old comment --- test_suite.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index f42c164872..31f85f60fd 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -173,7 +173,6 @@ else fi echo "Detected available memory: ${cgroup_mem_mib} MiB" -# echo "Replacing detected system information in template ReFrame config file..." cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES} echo "Replacing memory limit in the ReFrame config file with the detected CGROUP memory limit: ${cgroup_mem_mib} MiB" sed -i "s/__MEM_PER_NODE__/${cgroup_mem_mib}/g" $RFM_CONFIG_FILES From 0cd0c38c9bdcea8d0f092a3774fd3554ed1a9023 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 5 Sep 2024 11:44:08 +0000 Subject: [PATCH 1120/1795] {2023.06}[foss/2023a] LRBinner v0.1 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index fc8e4c4d9b..3a0ea14ae8 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -63,3 +63,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20833 from-commit: 78426c2383fc7e4b9b9e77d7a77f336e1bee3843 + - LRBinner-0.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21310 + from-commit: 799d9101df2cf81aabe252f00cc82a7246363f53 From fcf493758852e9842812a314e5e690f739ee9e9d Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 06:25:55 +0000 Subject: [PATCH 1121/1795] move init scripts into init/lmod - move new scripts - copy bash --- init/lmod/bash | 17 +++++++++++++++++ init/{ => lmod}/csh | 0 init/{ => lmod}/fish | 0 init/{ => lmod}/ksh | 0 init/{ => lmod}/zsh | 0 5 files changed, 17 insertions(+) create mode 100644 init/lmod/bash rename init/{ => lmod}/csh (100%) rename init/{ => lmod}/fish (100%) rename init/{ => lmod}/ksh (100%) rename init/{ => lmod}/zsh (100%) diff --git a/init/lmod/bash b/init/lmod/bash new file mode 100644 index 0000000000..0451601679 --- /dev/null +++ b/init/lmod/bash @@ -0,0 +1,17 @@ +# Choose an EESSI version +EESSI_VERSION="${EESSI_VERSION:-2023.06}" +# Path to top-level module tree +export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules +. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/bash + +export PS1 +if [ -z "$__Init_Default_Modules" ]; then + export __Init_Default_Modules=1; + + ## ability to predefine elsewhere the default list + LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"StdEnv"} + export LMOD_SYSTEM_DEFAULT_MODULES + module --initial_load --no_redirect restore +else + module refresh +fi diff --git a/init/csh b/init/lmod/csh similarity index 100% rename from init/csh rename to init/lmod/csh diff --git a/init/fish b/init/lmod/fish similarity index 100% rename from init/fish rename to init/lmod/fish diff --git a/init/ksh b/init/lmod/ksh similarity index 100% rename from init/ksh rename to init/lmod/ksh diff --git a/init/zsh b/init/lmod/zsh similarity index 100% rename from init/zsh rename to init/lmod/zsh From 032cdae88f319ab247f513ab91cb0ead3a28e873 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 06:32:03 +0000 Subject: [PATCH 1122/1795] Revert a0c4377e restore original content of bash init --- init/bash | 56 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/init/bash b/init/bash index 0451601679..4ad09f6a1b 100644 --- a/init/bash +++ b/init/bash @@ -1,17 +1,41 @@ -# Choose an EESSI version -EESSI_VERSION="${EESSI_VERSION:-2023.06}" -# Path to top-level module tree -export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules -. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/bash - -export PS1 -if [ -z "$__Init_Default_Modules" ]; then - export __Init_Default_Modules=1; - - ## ability to predefine elsewhere the default list - LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"StdEnv"} - export LMOD_SYSTEM_DEFAULT_MODULES - module --initial_load --no_redirect restore -else - module refresh +function show_msg { + # only echo msg if EESSI_SILENT is unset + msg=$1 + if [[ -z ${EESSI_SILENT+x} ]]; then + echo "$msg" + fi +} + +# The following method should be safe, but might break if file is a symlink +# (could switch to $(dirname "$(readlink -f "$BASH_SOURCE")") in that case) +source $(dirname "$BASH_SOURCE")/eessi_environment_variables + +# only continue if setting EESSI environment variables worked fine +if [ $? -eq 0 ]; then + + export PS1="{EESSI $EESSI_VERSION} $PS1" + + # add location of commands provided by compat layer to $PATH; + # see https://github.com/EESSI/software-layer/issues/52 + export PATH=$EPREFIX/usr/bin:$EPREFIX/bin:$PATH + + # init Lmod + show_msg "Initializing Lmod..." + source $EESSI_EPREFIX/usr/share/Lmod/init/bash + + # prepend location of modules for EESSI software stack to $MODULEPATH + show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." + module use $EESSI_MODULEPATH + show_msg "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." + module use $EESSI_SITE_MODULEPATH + + #show_msg "" + #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***" + #show_msg "" + #show_msg "1) ..." + #show_msg "" + #show_msg "" + + echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!" + fi From 33b6b9267fef8b5cef81a553ea55583315a479b5 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 07:21:26 +0000 Subject: [PATCH 1123/1795] Use EESSI as LMOD SYSTEM DEFAULT makes StdEnv.lua unnecessary --- init/lmod/bash | 2 +- init/lmod/csh | 2 +- init/lmod/fish | 2 +- init/lmod/ksh | 2 +- init/lmod/zsh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/init/lmod/bash b/init/lmod/bash index 0451601679..563187ecd1 100644 --- a/init/lmod/bash +++ b/init/lmod/bash @@ -9,7 +9,7 @@ if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; ## ability to predefine elsewhere the default list - LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"StdEnv"} + LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"EESSI/$EESSI_VERSION"} export LMOD_SYSTEM_DEFAULT_MODULES module --initial_load --no_redirect restore else diff --git a/init/lmod/csh b/init/lmod/csh index 920581fccb..40fe099fc4 100644 --- a/init/lmod/csh +++ b/init/lmod/csh @@ -9,7 +9,7 @@ if (! $?__Init_Default_Modules ) then setenv __Init_Default_Modules 1; ## ability to predefine elsewhere the default list - if (! $?LMOD_SYSTEM_DEFAULT_MODULES) then; setenv LMOD_SYSTEM_DEFAULT_MODULES "StdEnv"; endif + if (! $?LMOD_SYSTEM_DEFAULT_MODULES) then; setenv LMOD_SYSTEM_DEFAULT_MODULES "EESSI/$EESSI_VERSION"; endif module --initial_load --no_redirect restore else module refresh diff --git a/init/lmod/fish b/init/lmod/fish index 33952769e1..17dfee9859 100644 --- a/init/lmod/fish +++ b/init/lmod/fish @@ -9,7 +9,7 @@ if test -z "$__Init_Default_Modules" export __Init_Default_Modules=1; ## ability to predefine elsewhere the default list - set -x LMOD_SYSTEM_DEFAULT_MODULES (set -q LMOD_SYSTEM_DEFAULT_MODULE; and echo "$LMOD_SYSTEM_DEFAULT_MODULE"; or echo "StdEnv") + set -x LMOD_SYSTEM_DEFAULT_MODULES (set -q LMOD_SYSTEM_DEFAULT_MODULE; and echo "$LMOD_SYSTEM_DEFAULT_MODULE"; or echo "EESSI/$EESSI_VERSION") module --initial_load --no_redirect restore else module refresh diff --git a/init/lmod/ksh b/init/lmod/ksh index e4eaafb2e6..ca7336f9ed 100644 --- a/init/lmod/ksh +++ b/init/lmod/ksh @@ -9,7 +9,7 @@ if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; ## ability to predefine elsewhere the default list - LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"StdEnv"} + LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"EESSI/$EESSI_VERSION"} export LMOD_SYSTEM_DEFAULT_MODULES module --initial_load --no_redirect restore else diff --git a/init/lmod/zsh b/init/lmod/zsh index 90ffeff4ef..33b29cad3f 100644 --- a/init/lmod/zsh +++ b/init/lmod/zsh @@ -9,7 +9,7 @@ if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; ## ability to predefine elsewhere the default list - LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"StdEnv"} + LMOD_SYSTEM_DEFAULT_MODULES=${LMOD_SYSTEM_DEFAULT_MODULES:-"EESSI/$EESSI_VERSION"} export LMOD_SYSTEM_DEFAULT_MODULES module --initial_load --no_redirect restore else From d4bc3bc54907906f49691ec0e640589de5d3280c Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 07:21:46 +0000 Subject: [PATCH 1124/1795] Revert "Add StdEnv.lua module" This reverts commit ea32ffe4160ec60b180235061e84127a323aa78e. --- init/modules/StdEnv.lua | 1 - 1 file changed, 1 deletion(-) delete mode 100644 init/modules/StdEnv.lua diff --git a/init/modules/StdEnv.lua b/init/modules/StdEnv.lua deleted file mode 100644 index ae34402e9a..0000000000 --- a/init/modules/StdEnv.lua +++ /dev/null @@ -1 +0,0 @@ -load("EESSI/2023.06") From 63482f5c3c759260eda5f75c2f868a8d30efa956 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 14:54:50 +0000 Subject: [PATCH 1125/1795] Remove PS1 export handling PS1 variable is moved to later PR --- init/lmod/bash | 1 - init/lmod/csh | 1 - init/lmod/fish | 1 - init/lmod/ksh | 1 - init/lmod/zsh | 1 - 5 files changed, 5 deletions(-) diff --git a/init/lmod/bash b/init/lmod/bash index 563187ecd1..b4941d6766 100644 --- a/init/lmod/bash +++ b/init/lmod/bash @@ -4,7 +4,6 @@ EESSI_VERSION="${EESSI_VERSION:-2023.06}" export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/bash -export PS1 if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; diff --git a/init/lmod/csh b/init/lmod/csh index 40fe099fc4..5ae13fcc50 100644 --- a/init/lmod/csh +++ b/init/lmod/csh @@ -4,7 +4,6 @@ if (! $?EESSI_VERSION) then; set EESSI_VERSION = "2023.06"; endif setenv MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules source /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/`uname -m`/usr/share/Lmod/init/csh -setenv PS1 $prompt if (! $?__Init_Default_Modules ) then setenv __Init_Default_Modules 1; diff --git a/init/lmod/fish b/init/lmod/fish index 17dfee9859..d4252ef32a 100644 --- a/init/lmod/fish +++ b/init/lmod/fish @@ -4,7 +4,6 @@ set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "202 set -x MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/(uname -m)/usr/share/Lmod/init/fish -export PS1 if test -z "$__Init_Default_Modules" export __Init_Default_Modules=1; diff --git a/init/lmod/ksh b/init/lmod/ksh index ca7336f9ed..71dc29542f 100644 --- a/init/lmod/ksh +++ b/init/lmod/ksh @@ -4,7 +4,6 @@ EESSI_VERSION="${EESSI_VERSION:-2023.06}" export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/ksh -export PS1 if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; diff --git a/init/lmod/zsh b/init/lmod/zsh index 33b29cad3f..5f605579c8 100644 --- a/init/lmod/zsh +++ b/init/lmod/zsh @@ -4,7 +4,6 @@ EESSI_VERSION="${EESSI_VERSION:-2023.06}" export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules . /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh -export PS1 if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; From 7f95d5e3478f6aa16cef41628ad718d069b83b7f Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 14:58:26 +0000 Subject: [PATCH 1126/1795] Update init scripts path in test script from `/init/$shell` to `/init/lmod/$shell` --- .github/workflows/scripts/test_init_scripts.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/test_init_scripts.sh b/.github/workflows/scripts/test_init_scripts.sh index 99fefecdf8..06b658d988 100755 --- a/.github/workflows/scripts/test_init_scripts.sh +++ b/.github/workflows/scripts/test_init_scripts.sh @@ -19,19 +19,19 @@ for shell in ${SHELLS[@]}; do echo = | awk 'NF += (OFS = $_) + 100' # TEST 1: Source Script and check Module Output - assert "$shell -c 'source init/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" + assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" # TEST 2: Check PS1 Prompt (Previous exported) PROMPT="${shell^^}_PROMPT" assert "echo ${!PROMPT}" "{EESSI $EESSI_VERSION}" # Test 3: Check module overview - MODULE_SECTIONS=($($shell -c "source init/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) + MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) assert "echo ${MODULE_SECTIONS[1]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/modules/all" assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules" # Test 4: Load Python module and check version and path - command="$shell -c 'source init/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'" + command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'" expected="Python 3.10.8" assert "$command" "$expected" - command="$shell -c 'source init/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python'" + command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python'" expected="/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/software/Python/3.10.8-GCCcore-12.2.0/bin/python" assert "$command" "$expected" done From a2ce628df900072a4040b3950af53ba1a639afc6 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 15:00:23 +0000 Subject: [PATCH 1127/1795] Remove Prompt test from unittest since the delayed implementation of PS1 handling this test is removed --- .github/workflows/scripts/test_init_scripts.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/scripts/test_init_scripts.sh b/.github/workflows/scripts/test_init_scripts.sh index 06b658d988..6e4f66af45 100755 --- a/.github/workflows/scripts/test_init_scripts.sh +++ b/.github/workflows/scripts/test_init_scripts.sh @@ -20,14 +20,11 @@ for shell in ${SHELLS[@]}; do # TEST 1: Source Script and check Module Output assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" - # TEST 2: Check PS1 Prompt (Previous exported) - PROMPT="${shell^^}_PROMPT" - assert "echo ${!PROMPT}" "{EESSI $EESSI_VERSION}" - # Test 3: Check module overview + # Test 2: Check module overview MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) assert "echo ${MODULE_SECTIONS[1]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/modules/all" assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules" - # Test 4: Load Python module and check version and path + # Test 3: Load Python module and check version and path command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'" expected="Python 3.10.8" assert "$command" "$expected" From 2ab75c3daf7271ff6473638b2e78e994b74ad4b7 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 16:34:25 +0000 Subject: [PATCH 1128/1795] Rename Test Numbers - Classify each assert as a test - Give each test an number comment (increment) - Move `assert_end` into for loop to create seperate testsuite This results that failing test presents the matching number to the comment. --- .github/workflows/scripts/test_init_scripts.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scripts/test_init_scripts.sh b/.github/workflows/scripts/test_init_scripts.sh index 6e4f66af45..24a59a90a3 100755 --- a/.github/workflows/scripts/test_init_scripts.sh +++ b/.github/workflows/scripts/test_init_scripts.sh @@ -20,20 +20,24 @@ for shell in ${SHELLS[@]}; do # TEST 1: Source Script and check Module Output assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" - # Test 2: Check module overview + # TEST 2: Check if module overviews first section is the loaded EESSI module MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) assert "echo ${MODULE_SECTIONS[1]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/modules/all" + # TEST 3: Check if module overviews second section is the EESSI init module assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules" - # Test 3: Load Python module and check version and path + # Test 4: Load Python module and check version command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'" expected="Python 3.10.8" assert "$command" "$expected" + # Test 5: Load Python module and check path command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python'" expected="/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/software/Python/3.10.8-GCCcore-12.2.0/bin/python" assert "$command" "$expected" + + #End Test Suite + assert_end "source_eessi_$shell" done -assert_end source_eessi # RESET PAGER export LMOD_PAGER= From 7c798d4fc48e5ccfe5aa81e18d633fe6bfb8ca37 Mon Sep 17 00:00:00 2001 From: makanu Date: Fri, 6 Sep 2024 16:53:14 +0000 Subject: [PATCH 1129/1795] Remove PROMPT variable from action since delayed feature for PS1 manipulation this is no longer needed --- .github/workflows/tests_init_module.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index 58c48c5b68..f9283b68cd 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -37,13 +37,6 @@ jobs: sudo apt install zsh ksh fish echo "# INIT ZSH" > ~/.zshrc - - name: Prepare PROMPT variables - run: | - export BASH_PROMPT=$(bash -c 'source init/bash; echo "$PS1"') - export ZSH_PROMPT=$(zsh -c 'source init/zsh; echo "$PS1"') - export KSH_PROMPT=$(ksh -c 'source init/ksh; echo "$PS1"') - export FISH_PROMPT=$(fish -c 'source init/fish; echo "$PS1"') - - name: Run tests for available shells run: | .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" From 8d74aea96e184530e93ea2fc75560dc13b16dc52 Mon Sep 17 00:00:00 2001 From: MaKaNu <32844273+MaKaNu@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:15:26 +0200 Subject: [PATCH 1130/1795] Set correct arch override Co-authored-by: ocaisa --- .github/workflows/tests_init_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index f9283b68cd..e09afb8fb7 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -16,7 +16,7 @@ jobs: EESSI_VERSION: - 2023.06 EESSI_SOFTWARE_SUBDIR_OVERRIDE: - - x86_64/haswell + - x86_64/intel/haswell steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 From 45b28f042345c9093366a76e4e1ebe3e25e4d3b0 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 10 Sep 2024 06:40:24 +0000 Subject: [PATCH 1131/1795] {2023.06}[GCC/12.3.0] Redland v1.0.17 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 3a0ea14ae8..e9011a0664 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -67,3 +67,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21310 from-commit: 799d9101df2cf81aabe252f00cc82a7246363f53 + - Redland-1.0.17-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21227 + from-commit: 4c5e3455dec31e68e8383c7fd86d1f80c434676d From 74a37138d3729d571d9161b4e0595f6cb9ecc592 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 10 Sep 2024 13:05:18 +0200 Subject: [PATCH 1132/1795] Fix typo in `create_lmodsitepackage.py` Current typo results in the following problem ```bash [ritop7377@login-2.BETZY ~]$ ml Currently Loaded Modules: 1) StdEnv (S) 2) EESSI/2023.06 Where: S: Module is Sticky, requires --force to unload or purge [ritop@login ~]$ module purge /usr/bin/lua: /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/.lmod/SitePackage.lua:21: variable 'False' is not declared stack traceback: [C]: in function 'error' /cluster/installations/lmod/lmod/libexec/../tools/strict.lua:36: in function /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/.lmod/SitePackage.lua:21: in function 'from_eessi_prefix' /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/.lmod/SitePackage.lua:189: in function (tail call): ? /cluster/installations/lmod/lmod/libexec/Master.lua:381: in function 'load' /cluster/installations/lmod/lmod/libexec/MasterControl.lua:1009: in function 'load' /cluster/installations/lmod/lmod/libexec/Master.lua:773: in function 'reload_sticky' /cluster/installations/lmod/lmod/libexec/MasterControl.lua:1120: in function 'unload_usr' /cluster/installations/lmod/lmod/libexec/cmdfuncs.lua:496: in function 'Purge' /cluster/installations/lmod/lmod/libexec/cmdfuncs.lua:473: in function 'cmd' /cluster/installations/lmod/lmod/libexec/lmod:512: in function 'main' /cluster/installations/lmod/lmod/libexec/lmod:570: in main chunk [C]: ? ``` --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 7e55bce2a5..62f073c9a6 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -28,7 +28,7 @@ -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set if eessi_prefix == nil then - return False + return false else -- NOTE: exact paths for site so may need to be updated later. -- See https://github.com/EESSI/software-layer/pull/371 From e0401ac14ea7aef9a4de3a275c646527baff8536 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 10 Sep 2024 20:02:17 +0200 Subject: [PATCH 1133/1795] {2023.06,a64fx}[foss/2023a] Highway v1.0.4 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + eb_hooks.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index 2f884f1838..58907f0ba0 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -19,3 +19,4 @@ easyconfigs: # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841 from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 - R-4.3.2-gfbf-2023a.eb + - Highway-1.0.4-GCCcore-12.3.0.eb diff --git a/eb_hooks.py b/eb_hooks.py index 6cac9b8609..3e095129f9 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -333,7 +333,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg Solve issues with compiling or running the tests on both neoverse_n1 and neoverse_v1 with Highway 1.0.4 and GCC 12.3.0: - for neoverse_n1 we set optarch to GENERIC - - for neoverse_v1 we completely disable the tests + - for neoverse_v1 and a64fx we completely disable the tests cfr. https://github.com/EESSI/software-layer/issues/469 """ if self.name == 'Highway': @@ -342,7 +342,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg # note: keep condition in sync with the one used in # post_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': - if cpu_target == CPU_TARGET_NEOVERSE_V1: + if cpu_target in [CPU_TARGET_A64FX, CPU_TARGET_NEOVERSE_V1]: self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') if cpu_target == CPU_TARGET_NEOVERSE_N1: self.orig_optarch = build_option('optarch') From 0ba9c3f1abfbdcdb7d29bd7fafa67793df9094c2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 10 Sep 2024 20:22:06 +0200 Subject: [PATCH 1134/1795] {2023.06,a64fx}[foss/2023a] Brunsli v0.1 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index 2f884f1838..1fd5fb6d14 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -19,3 +19,7 @@ easyconfigs: # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841 from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 - R-4.3.2-gfbf-2023a.eb + - Brunsli-0.1-GCCcore-12.3.0.eb: + options: + # https://github.com/easybuilders/easybuild-easyconfigs/pull/21366 + from-commit: 1736a123b1685836452587a5c51793257570bb2d From 32ede2008efd3517f275af1ec4dbf8e7b46e90f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Sep 2024 20:27:05 +0200 Subject: [PATCH 1135/1795] remove hook for Highway 1.0.7 again --- eb_hooks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4a6068f7fa..73adce90f1 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -318,9 +318,6 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg if cpu_target == CPU_TARGET_NEOVERSE_N1: self.orig_optarch = build_option('optarch') update_build_option('optarch', OPTARCH_GENERIC) - if self.version in ['1.0.7'] and tcname == 'GCCcore' and tcversion == '13.2.0': - if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') else: raise EasyBuildError("Highway-specific hook triggered for non-Highway easyconfig?!") From ea9d4691131c3e133717c6ebf00a984df976538a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Sep 2024 22:31:56 +0200 Subject: [PATCH 1136/1795] Toolchain for Brunsli was wrong --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 401410fc1b..863fe5e2e0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -20,7 +20,7 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00 - HPL-2.3-foss-2023b.eb - - Brunsli-0.1-GCCcore-12.3.0.eb: + - Brunsli-0.1-GCCcore-13.2.0.eb: options: # https://github.com/easybuilders/easybuild-easyconfigs/pull/21366 from-commit: 1736a123b1685836452587a5c51793257570bb2d From a6999fa80881584a4b4dee634c458eda0f1f90f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 11 Sep 2024 16:14:48 +0200 Subject: [PATCH 1137/1795] remove explicitly listed R, use newer commit for R-bundle-CRAN --- .../2023.06/eessi-2023.06-eb-4.9.2-2023b.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 863fe5e2e0..6398f014dc 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -24,11 +24,7 @@ easyconfigs: options: # https://github.com/easybuilders/easybuild-easyconfigs/pull/21366 from-commit: 1736a123b1685836452587a5c51793257570bb2d - - R-4.4.1-gfbf-2023b.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 - from-commit: 12ef0f174250928ca9a5ccf47e4f7ca0ce56811f - R-bundle-CRAN-2024.06-foss-2023b.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20898 - from-commit: 12ef0f174250928ca9a5ccf47e4f7ca0ce56811f + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21285 + from-commit: 41a2cd83f9fb017b76f0693f6a264d8acb548317 From 596f60542d6d34f0c8a8e21c7408dff21e6146be Mon Sep 17 00:00:00 2001 From: makanu Date: Wed, 11 Sep 2024 17:26:35 +0000 Subject: [PATCH 1138/1795] Fix arch differences Local Test system runs intel/haswell CICD System runs amd/zen3 No matter what I override the Path I am testing against is always the real arch Fixed by using regex detection, might fail if CICD System changes --- .github/workflows/scripts/test_init_scripts.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/scripts/test_init_scripts.sh b/.github/workflows/scripts/test_init_scripts.sh index 24a59a90a3..938bdfb505 100755 --- a/.github/workflows/scripts/test_init_scripts.sh +++ b/.github/workflows/scripts/test_init_scripts.sh @@ -22,7 +22,8 @@ for shell in ${SHELLS[@]}; do assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" # TEST 2: Check if module overviews first section is the loaded EESSI module MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) - assert "echo ${MODULE_SECTIONS[1]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/modules/all" + PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all" + assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"' # TEST 3: Check if module overviews second section is the EESSI init module assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules" # Test 4: Load Python module and check version @@ -30,10 +31,11 @@ for shell in ${SHELLS[@]}; do expected="Python 3.10.8" assert "$command" "$expected" # Test 5: Load Python module and check path - command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python'" - expected="/cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/linux/x86_64/intel/haswell/software/Python/3.10.8-GCCcore-12.2.0/bin/python" - assert "$command" "$expected" - + PYTHON_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python") + PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/Python/3\.10\.8-GCCcore-12\.2\.0/bin/python" + echo "$PYTHON_PATH" | grep -E "$PATTERN" + assert_raises 'echo "$PYTHON_PATH" | grep -E "$PATTERN"' + #End Test Suite assert_end "source_eessi_$shell" done From e5c261203b0419f26b53568f907c19ae9f6814a8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Sep 2024 13:03:19 +0200 Subject: [PATCH 1139/1795] install_cuda_host_injections: Don't fail if temp dir exists or parent is missing Use `mkdir -p` such that `--temp-dir /tmp/$USER/EESSI` works out of the box. It also avoids failures if the user got "ERROR: Could not create directory /tmp/root/EESSI/temp" and did `mkdir -p /tmp/root/EESSI/temp` manually. --- scripts/gpu_support/nvidia/install_cuda_host_injections.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh index a9310d817a..0d363fde5e 100755 --- a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh +++ b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh @@ -123,7 +123,7 @@ else tmpdir=$(mktemp -d) else tmpdir="${CUDA_TEMP_DIR}"/temp - if ! mkdir "$tmpdir" ; then + if ! mkdir -p "$tmpdir" ; then fatal_error "Could not create directory ${tmpdir}" fi fi From 007712c32da14d139e2b12ed232f1cf0b1776b6d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Sep 2024 13:39:46 +0200 Subject: [PATCH 1140/1795] install_cuda_host_injections: Fix CUDA-EC-not-found error message The search query is a regexp not a glob, so add the missing dot. Also add the missing "of CUDA" in the message --- scripts/gpu_support/nvidia/install_cuda_host_injections.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh index 0d363fde5e..3842aff307 100755 --- a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh +++ b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh @@ -175,13 +175,13 @@ else # Check the exit code if [ $? -ne 0 ]; then eb_version=$(eb --version) - available_cuda_easyconfigs=$(eb --search ^CUDA-*.eb|grep CUDA) + available_cuda_easyconfigs=$(eb --search "^CUDA-.*.eb"|grep CUDA) error="The easyconfig ${cuda_easyconfig} was not found in EasyBuild version:\n" error="${error} ${eb_version}\n" error="${error}You either need to give a different version of CUDA to install _or_ \n" error="${error}use a different version of EasyBuild for the installation.\n" - error="${error}\nThe versions of available with the current eb command are:\n" + error="${error}\nThe versions of CUDA available with the current eb command are:\n" error="${error}${available_cuda_easyconfigs}" fatal_error "${error}" fi From 462f18019d3a820e979a004dfa12018a39ce67d3 Mon Sep 17 00:00:00 2001 From: makanu Date: Thu, 12 Sep 2024 13:59:30 +0000 Subject: [PATCH 1141/1795] Add Exception for CSH csh is not as feature rich as other shells I have implemented. Maybe someday I learn how to approach it! --- .github/workflows/scripts/test_init_scripts.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/scripts/test_init_scripts.sh b/.github/workflows/scripts/test_init_scripts.sh index 938bdfb505..2bcbbc8d86 100755 --- a/.github/workflows/scripts/test_init_scripts.sh +++ b/.github/workflows/scripts/test_init_scripts.sh @@ -40,6 +40,9 @@ for shell in ${SHELLS[@]}; do assert_end "source_eessi_$shell" done +### EXCEPTION FOR CSH ### +echo -e "\033[33mWe don't now how to test csh correctly, PRs are Welcome.\033[0m" + # RESET PAGER export LMOD_PAGER= From 68ad3affa38707af85e0be221ff0c4a042fdca3d Mon Sep 17 00:00:00 2001 From: makanu Date: Thu, 12 Sep 2024 15:47:09 +0000 Subject: [PATCH 1142/1795] Add Check for not testable shells before we run any tests we check if the shell provided is actually testable. If a shell is not tested yet only means we don't know how to test the shell or the shell is not added to the TEST_SHELLS Array. --- .../workflows/scripts/test_init_scripts.sh | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/.github/workflows/scripts/test_init_scripts.sh b/.github/workflows/scripts/test_init_scripts.sh index 2bcbbc8d86..048fba81f5 100755 --- a/.github/workflows/scripts/test_init_scripts.sh +++ b/.github/workflows/scripts/test_init_scripts.sh @@ -11,38 +11,40 @@ if [ ! -d assert.sh ]; then fi . assert.sh/assert.sh +TEST_SHELLS=("bash" "zsh" "fish" "ksh") SHELLS=$@ for shell in ${SHELLS[@]}; do echo = | awk 'NF += (OFS = $_) + 100' echo RUNNING TESTS FOR SHELL: $shell echo = | awk 'NF += (OFS = $_) + 100' - - # TEST 1: Source Script and check Module Output - assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" - # TEST 2: Check if module overviews first section is the loaded EESSI module - MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) - PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all" - assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"' - # TEST 3: Check if module overviews second section is the EESSI init module - assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules" - # Test 4: Load Python module and check version - command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'" - expected="Python 3.10.8" - assert "$command" "$expected" - # Test 5: Load Python module and check path - PYTHON_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python") - PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/Python/3\.10\.8-GCCcore-12\.2\.0/bin/python" - echo "$PYTHON_PATH" | grep -E "$PATTERN" - assert_raises 'echo "$PYTHON_PATH" | grep -E "$PATTERN"' - - #End Test Suite - assert_end "source_eessi_$shell" + if [[ ! " ${TEST_SHELLS[*]} " =~ [[:space:]]${shell}[[:space:]] ]]; then + ### EXCEPTION FOR CSH ### + echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m" + else + # TEST 1: Source Script and check Module Output + assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully" + # TEST 2: Check if module overviews first section is the loaded EESSI module + MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'")) + PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all" + assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"' + # TEST 3: Check if module overviews second section is the EESSI init module + assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules" + # Test 4: Load Python module and check version + command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'" + expected="Python 3.10.8" + assert "$command" "$expected" + # Test 5: Load Python module and check path + PYTHON_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python") + PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/Python/3\.10\.8-GCCcore-12\.2\.0/bin/python" + echo "$PYTHON_PATH" | grep -E "$PATTERN" + assert_raises 'echo "$PYTHON_PATH" | grep -E "$PATTERN"' + + #End Test Suite + assert_end "source_eessi_$shell" + fi done -### EXCEPTION FOR CSH ### -echo -e "\033[33mWe don't now how to test csh correctly, PRs are Welcome.\033[0m" - # RESET PAGER export LMOD_PAGER= From bccd55a2edfc0a0e366fc74241ad885c28e096cc Mon Sep 17 00:00:00 2001 From: makanu Date: Thu, 12 Sep 2024 15:47:56 +0000 Subject: [PATCH 1143/1795] Add csh to test We actually don't know how to csh for now. But if we figure out we don't need to update here. --- .github/workflows/tests_init_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index e09afb8fb7..cfc4ae7b3d 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -39,5 +39,5 @@ jobs: - name: Run tests for available shells run: | - .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" + .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" "csh" From 7ad306ac462bdea86edac1f3780e9a5b82454863 Mon Sep 17 00:00:00 2001 From: MaKaNu <32844273+MaKaNu@users.noreply.github.com> Date: Thu, 12 Sep 2024 18:29:29 +0200 Subject: [PATCH 1144/1795] Update init/lmod/csh remove leftover comment Co-authored-by: ocaisa --- init/lmod/csh | 1 - 1 file changed, 1 deletion(-) diff --git a/init/lmod/csh b/init/lmod/csh index 5ae13fcc50..8e50d5e5c8 100644 --- a/init/lmod/csh +++ b/init/lmod/csh @@ -13,5 +13,4 @@ if (! $?__Init_Default_Modules ) then else module refresh endif -#module load "$LMOD_SYSTEM_DEFAULT_MODULES" From 1b38cecc6a132c79a5e74b97e8a7577d5f9d069f Mon Sep 17 00:00:00 2001 From: MaKaNu <32844273+MaKaNu@users.noreply.github.com> Date: Thu, 12 Sep 2024 22:41:14 +0200 Subject: [PATCH 1145/1795] Update install_scripts.sh Add copy of init lmod scripts --- install_scripts.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/install_scripts.sh b/install_scripts.sh index 0cfa5310c2..11c7fc2a9f 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -108,6 +108,12 @@ mc_files=( ) copy_files_by_list ${TOPDIR}/init/modules/EESSI ${INSTALL_PREFIX}/init/modules/EESSI "${mc_files[@]}" +# Copy for init/lmod directory +init_script_files=( + bash zsh ksh fish csh +) +copy_files_by_list ${TOPDIR}/init/lmod ${INSTALL_PREFIX}/init/lmod "${init_script_files[@]}" + # Copy for the scripts directory script_files=( utils.sh From 79000d16a8e1eb4f4c4d1f01f019f9e2a336d270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 16 Sep 2024 12:04:43 +0200 Subject: [PATCH 1146/1795] build EB 4.9.3 with EB 4.9.2 --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml new file mode 100644 index 0000000000..82e72580a3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -0,0 +1,6 @@ +easyconfigs: + - EasyBuild-4.9.3.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21412 + from-commit: https://github.com/easybuilders/easybuild-easyconfigs/pull/21412 + From ae1bf81c9dcb6e6107c21fd04a190613f5d22882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 16 Sep 2024 12:05:36 +0200 Subject: [PATCH 1147/1795] remove empty line --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml index 82e72580a3..9eaa97f8b0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -3,4 +3,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21412 from-commit: https://github.com/easybuilders/easybuild-easyconfigs/pull/21412 - From 344d67e6b651b0f779eec69e3967c2d4c2837d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 16 Sep 2024 12:14:47 +0200 Subject: [PATCH 1148/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml Co-authored-by: ocaisa --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml index 9eaa97f8b0..97c6031c79 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -2,4 +2,4 @@ easyconfigs: - EasyBuild-4.9.3.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21412 - from-commit: https://github.com/easybuilders/easybuild-easyconfigs/pull/21412 + from-commit: 1cdd81524c974a29825e37bcf8ef3ccc291f5227 From ccfb1c12f201e5ab4f3501385a437dbd5ee37dae Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 17 Sep 2024 11:03:04 +0200 Subject: [PATCH 1149/1795] {2023.06}[2023a] ALL 0.9.2 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 9d13d5eb3a..316754a6d1 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -22,3 +22,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - MODFLOW-6.4.4-foss-2023a.eb + - ALL-0.9.2-foss-2023a.eb From b81151ad8c03adbc4de023dbb908cacb87e2ff6f Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 17 Sep 2024 13:40:06 +0200 Subject: [PATCH 1150/1795] {2023.06}[2023a,zen4] LAMMPS 2Aug2023 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..df3d0dedaa --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb From 700f37abe0fc335f4b06fafe069862caa143ab0d Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 17 Sep 2024 12:07:52 +0000 Subject: [PATCH 1151/1795] {2023.06}[GCCcore/12.2.0-GCC/12.2.0-gompi/2022b-foss/2022b] bio-packages --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index a22b78718f..02b00e6173 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -6,4 +6,25 @@ easyconfigs: from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - gnuplot-5.4.6-GCCcore-12.2.0.eb - h5py-3.8.0-foss-2022b.eb - - MDAnalysis-2.4.2-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - ncbi-vdb-3.0.5-gompi-2022b.eb + - Bio-DB-HTS-3.01-GCC-12.2.0.eb + - MAFFT-7.505-GCC-12.2.0-with-extensions.eb + - MetaEuk-6-GCC-12.2.0.eb + - BamTools-2.5.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - Mash-2.3-GCC-12.2.0.eb + - CapnProto-0.10.3-GCCcore-12.2.0.eb + - WhatsHap-2.1-foss-2022b.eb + - SAMtools-1.17-GCC-12.2.0.eb + - Bowtie2-2.5.1-GCC-12.2.0.eb + - CD-HIT-4.8.1-GCC-12.2.0.eb + - VCFtools-0.1.16-GCC-12.2.0.eb + - GenomeTools-1.6.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - parallel-20230722-GCCcore-12.2.0.eb + - BCFtools-1.17-GCC-12.2.0.eb + - lpsolve-5.5.2.11-GCC-12.2.0.eb + - fastp-0.23.4-GCC-12.2.0.eb + - KronaTools-2.8.1-GCCcore-12.2.0.eb + - MultiQC-1.14-foss-2022b.eb From 9cced0ea972ef5570d47520006458def319ffc05 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 17 Sep 2024 14:58:33 +0200 Subject: [PATCH 1152/1795] {2023.6}[2023a,a64fx] LAMMPS 2 Aug2023 for a64fx --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..df3d0dedaa --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb From 93b699fecbb164437dd39b095413ea8575b17efe Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 17 Sep 2024 15:02:39 +0200 Subject: [PATCH 1153/1795] Undo adding patchelf --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 6996e6f712..0e58e2a553 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -7,4 +7,3 @@ easyconfigs: - gnuplot-5.4.6-GCCcore-12.2.0.eb - h5py-3.8.0-foss-2022b.eb - MDAnalysis-2.4.2-foss-2022b.eb - - patchelf-0.17.2-GCCcore-12.2.0.eb From 48c01077a960623489c82167e0fe88edbec56daa Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 17 Sep 2024 15:34:16 +0200 Subject: [PATCH 1154/1795] Add ReFrame 4.6.2 --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml index 97c6031c79..66169b735d 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -3,3 +3,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21412 from-commit: 1cdd81524c974a29825e37bcf8ef3ccc291f5227 + - ReFrame-4.6.2.eb: + options: + from-commit: 0c4bd5c5a80f571a8932fbc38880d72455406816 + include-easyblocks-from-commit: efddeb02abe1a679324ac01ef19601dedbe79cc0 From e6760dbfef4b23d4fd5d1e97c1cd07bbd9cf6a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 17 Sep 2024 15:49:39 +0200 Subject: [PATCH 1155/1795] add GROMACS 2024.3 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml new file mode 100644 index 0000000000..8e19788425 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -0,0 +1,5 @@ +easyconfigs: + - GROMACS-2024.3-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 + from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b From 42df639223495e4d256ffd764b59cae8e8357857 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:21:06 +0200 Subject: [PATCH 1156/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml index 66169b735d..1b2343ec1f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -5,5 +5,7 @@ easyconfigs: from-commit: 1cdd81524c974a29825e37bcf8ef3ccc291f5227 - ReFrame-4.6.2.eb: options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21307 from-commit: 0c4bd5c5a80f571a8932fbc38880d72455406816 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3431 include-easyblocks-from-commit: efddeb02abe1a679324ac01ef19601dedbe79cc0 From 8d0085ceaa5ea92fe75a8f3cc0140b5b5a6c4266 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Sep 2024 15:28:25 +0200 Subject: [PATCH 1157/1795] take into account accelerator target when configuring EasyBuild --- EESSI-install-software.sh | 12 ++++++++++++ bot/build.sh | 6 +++++- configure_easybuild | 19 +++++++++++++++++++ run_in_compat_layer_env.sh | 3 +++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7d358e205a..27c96eba9f 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -207,7 +207,19 @@ echo ">> Setting up \$MODULEPATH..." module --force purge # ignore current $MODULEPATH entirely module unuse $MODULEPATH + +# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all + if [ -d ${CPU_ONLY_MODULES_PATH} ]; then + module use ${CPU_ONLY_MODULES_PATH} + else + fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}" + fi +fi + module use $EASYBUILD_INSTALLPATH/modules/all + if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" else diff --git a/bot/build.sh b/bot/build.sh index 145be740d3..7a07d09c4c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -141,7 +141,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" -# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# determine CPU architecture to be used from entry .architecture in ${JOB_CFG_FILE} # fallbacks: # - ${CPU_TARGET} handed over from bot # - left empty to let downstream script(s) determine subdir to be used @@ -150,6 +150,10 @@ EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" +# determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} +export EESSI_ACCELERATOR_TARGET=$(cfg_get_value "architecture" "accelerator") +echo "bot/build.sh: EESSI_ACCELERATOR_TARGET='${EESSI_ACCELERATOR_TARGET}'" + # get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} diff --git a/configure_easybuild b/configure_easybuild index ed3e651a4c..3b6d40cd96 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -1,7 +1,26 @@ +# if $WORKDIR is not defined, use a local temporary directory +if [ -z ${WORKDIR} ]; then + WORKDIR=$(mktemp -d) +fi + export EASYBUILD_PREFIX=${WORKDIR}/easybuild export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} export EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH} +# take into account accelerator target (if specified via $EESSI_ACCELERATOR_TARGET) +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + if [[ "${EESSI_ACCELERATOR_TARGET}" =~ ^nvidia/cc[0-9][0-9]$ ]]; then + # tweak path to installation directories used by EasyBuild + export EASYBUILD_INSTALLPATH=${EASYBUILD_INSTALLPATH}/accel/${EESSI_ACCELERATOR_TARGET} + # nvidia/cc80 should result in setting $EASYBUILD_CUDA_COMPUTE_CAPABILITIES to '8.0' + export EASYBUILD_CUDA_COMPUTE_CAPABILITIES=$(echo ${EESSI_ACCELERATOR_TARGET} | cut -f2 -d/ | sed 's/^cc\([0-9]\)\([0-9]\)/\1.\2/g') + else + fatal_error "Incorrect value for \$EESSI_ACCELERATOR_TARGET: ${EESSI_ACCELERATOR_TARGET}" + fi +else + echo_yellow "(configure_easybuild) \$EESSI_ACCELERATOR_TARGET not defined" +fi + # just ignore OS dependencies for now, see https://github.com/easybuilders/easybuild-framework/issues/3430 export EASYBUILD_IGNORE_OSDEPS=1 diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index b8e9cf979b..744e208ae0 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -20,6 +20,9 @@ fi if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + INPUT="export EESSI_ACCELERATOR_TARGET=${EESSI_ACCELERATOR_TARGET}; ${INPUT}" +fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" fi From e8af7d829f961e0d0cb3e5529e8ecb72ba1b3c37 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Sep 2024 18:42:57 +0200 Subject: [PATCH 1158/1795] update create_tarball.sh script to make it aware of installations in accel/ subdirectory --- EESSI-install-software.sh | 2 -- bot/build.sh | 4 +-- create_tarball.sh | 59 +++++++++++++++++++++------------------ 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 27c96eba9f..f9dd971a0d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -309,8 +309,6 @@ else done fi -### add packages here - echo ">> Creating/updating Lmod RC file..." export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" diff --git a/bot/build.sh b/bot/build.sh index 7a07d09c4c..1c4032ecbc 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -282,8 +282,8 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/create_tarball.sh b/create_tarball.sh index 2dee665060..7b38da3b07 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -4,14 +4,15 @@ set -e base_dir=$(dirname $(realpath $0)) -if [ $# -ne 4 ]; then - echo "ERROR: Usage: $0 " >&2 +if [ $# -ne 5 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi eessi_tmpdir=$1 eessi_version=$2 cpu_arch_subdir=$3 -target_tgz=$4 +accel_subdir=$4 +target_tgz=$5 tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" @@ -57,30 +58,34 @@ if [ -d ${eessi_version}/init ]; then find ${eessi_version}/init -type f | grep -v '/\.wh\.' >> ${files_list} fi -if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then - # module files - find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} - # module symlinks - find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} - # module files and symlinks - find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ - | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ - >> ${module_files_list} -fi - -if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then - # installation directories but only those for which module files were created - # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' - # using EasyBuild's standard module naming scheme) match the name of the - # software installation directory (expected to be 'PACKAGE_NAME/VERSION/'). - # If either side changes (module naming scheme or naming of software - # installation directories), the procedure will likely not work. - for package_version in $(cat ${module_files_list}); do - echo "handling ${package_version}" - ls -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ - | grep -v '/\.wh\.' >> ${files_list} - done -fi +# consider both CPU-only and accelerator subdirectories +for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do + + if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then + # module files + find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + # module symlinks + find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + # module files and symlinks + find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \ + | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ + >> ${module_files_list} + fi + + if [ -d ${eessi_version}/software/${os}/${subdir}/software -a -r ${module_files_list} ]; then + # installation directories but only those for which module files were created + # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' + # using EasyBuild's standard module naming scheme) match the name of the + # software installation directory (expected to be 'PACKAGE_NAME/VERSION/'). + # If either side changes (module naming scheme or naming of software + # installation directories), the procedure will likely not work. + for package_version in $(cat ${module_files_list}); do + echo "handling ${package_version}" + ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \ + | grep -v '/\.wh\.' >> ${files_list} + done + fi +done # add a bit debug output echo "wrote file list to ${files_list}" From 0eee4960d05d25f025b510984b572416e421a5d3 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 18 Sep 2024 09:57:57 +0200 Subject: [PATCH 1159/1795] add hook for zen4 builds to set --- eb_hooks.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 3e095129f9..0709389ffb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -26,6 +26,8 @@ CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' CPU_TARGET_A64FX = 'aarch64/a64fx' +CPU_TARGET_ZEN4 = 'x86_64/amd/zen4' + EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] @@ -514,6 +516,23 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") +def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): + """ + pre-configure hook for LAMMPS: + - set kokkos_arch on x86_64/amd/zen4 + """ + + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'LAMMPS': + if self.version == '2Aug2023': + if get_cpu_architecture() == X86_64: + if cpu_target == CPU_TARGET_ZEN4: + # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 + self.cfg['kokkos'] = 'ZEN3' + else: + raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") + + def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: @@ -783,6 +802,7 @@ def inject_gpu_property(ec): 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, + 'LAMMPS': pre_configure_hook_LAMMPS_zen4, } PRE_TEST_HOOKS = { From eba92ca6c2b292c64fd47e6e5df6cfd97c5e8fdc Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:20:36 +0200 Subject: [PATCH 1160/1795] Set correct kokkos variable --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0709389ffb..740f5e6cd8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -528,7 +528,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 - self.cfg['kokkos'] = 'ZEN3' + self.cfg['kokkos_arch'] = 'ZEN3' else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") From 1fca3170505a5ba32ccfcb97121405acd4bed5cc Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 18 Sep 2024 12:45:55 +0200 Subject: [PATCH 1161/1795] {2023.06}[2023a,GPU] LAMMPS 2Aug2023 CUDA 12.1.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml new file mode 100644 index 0000000000..91c4b1f472 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb From 54faa34196fe387fe9231677a07d83fc0021af36 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:55:27 +0200 Subject: [PATCH 1162/1795] fix lammps version in hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 740f5e6cd8..b3e457cfe3 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -524,7 +524,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'LAMMPS': - if self.version == '2Aug2023': + if self.version == '2Aug2023_update2': if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 From 4794e09643bd122dbbfa2ae72cf219629aa6f77c Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 18 Sep 2024 14:59:02 +0200 Subject: [PATCH 1163/1795] {2023.06}[2023a,zen4] JupyterNotebook v7.0.2 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index df3d0dedaa..0818701bbb 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,2 +1,3 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb + - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb From 028e57ea6110ce0347d8e33c4b6dd843fbb0f7ff Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 18 Sep 2024 15:23:26 +0200 Subject: [PATCH 1164/1795] use fixed prefix for tmp tarball and compress it --- eessi_container.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 34fa833d99..fc97f9877c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -846,14 +846,14 @@ if [[ ! -z ${SAVE} ]]; then # of these aspects to where the script is used if [[ -d ${SAVE} ]]; then # assume SAVE is name of a directory to which tarball shall be written to - # name format: {REPO_ID}-{TIMESTAMP}.tgz + # name format: tmp_storage-{TIMESTAMP}.tgz ts=$(date +%s) - TGZ=${SAVE}/${REPOSITORY}-${ts}.tgz + TGZ=${SAVE}/tmp_storage-${ts}.tgz else # assume SAVE is the full path to a tarball's name TGZ=${SAVE} fi - tar cf ${TGZ} -C ${EESSI_TMPDIR} . + tar czf ${TGZ} -C ${EESSI_TMPDIR} . echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')" fi From 03e0a7a2db36be52862a2a0db32898c1dc374157 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:05:17 +0200 Subject: [PATCH 1165/1795] Add CUDA-Samples for the accelerator prefix --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..01a47bbc99 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb From f66e1608fbfbfcc6a2be668e6ede8a66a318b6f1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:10:44 +0200 Subject: [PATCH 1166/1795] Add OSU Microbenchmarks 7.2 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..cccbfa6808 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb From b201c533c2271f6b4815661850570c26576e120c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:26:17 +0200 Subject: [PATCH 1167/1795] Added CUDA-12.1.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml new file mode 100644 index 0000000000..630bf6aa25 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - CUDA-12.1.1.eb From 5c9fca2fd5ea3160ec4e45f0f47cb73a0cdc264c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:29:31 +0200 Subject: [PATCH 1168/1795] UCX cuda 1.14.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..bfe4a4dc52 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb From e8f92f62f83c5c939467edb863c995e829a808d1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:54:24 +0200 Subject: [PATCH 1169/1795] Rebuild CUDA in accelerator prefix --- .../2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml | 2 -- .../20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml deleted file mode 100644 index 630bf6aa25..0000000000 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - CUDA-12.1.1.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml new file mode 100644 index 0000000000..755bea096e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml @@ -0,0 +1,7 @@ +# 2024.09.18 +# We need to reinstall CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/720 +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA From 78fbc1e86412379e382e473f36bf7ae98b1613e2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 22:17:47 +0200 Subject: [PATCH 1170/1795] make sure removal script also takes software prefix into account --- EESSI-remove-software.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 446a156cb8..e435dd8088 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -79,6 +79,17 @@ echo ">> Setting up \$MODULEPATH..." module --force purge # ignore current $MODULEPATH entirely module unuse $MODULEPATH + +# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all + if [ -d ${CPU_ONLY_MODULES_PATH} ]; then + module use ${CPU_ONLY_MODULES_PATH} + else + fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}" + fi +fi + module use $EASYBUILD_INSTALLPATH/modules/all if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" From 20b65e67e633f68f49c9beb5129d03043aee6ad6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 23:01:49 +0200 Subject: [PATCH 1171/1795] Make sure we remove from where the module is really located - and not just assume that it's in EASYBUILD_INSTALLPATH --- EESSI-remove-software.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index e435dd8088..25593b5a79 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -120,8 +120,12 @@ if [ $EUID -eq 0 ]; then # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') for app in ${rebuild_apps}; do - app_dir=${EASYBUILD_INSTALLPATH}/software/${app} - app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua + # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all: + app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://') + # Two dirname invocations, so returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2 + app_installprefix=$(dirname $(dirname app_modulpath)) + app_dir=${app_installprefix}/software/${app} + app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${app_dir} rm -rf ${app_module} From 4b3612dc807edc3988eb2d1727a812afcd751d7e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 23:11:35 +0200 Subject: [PATCH 1172/1795] Fix typo --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 25593b5a79..05572257a5 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -123,7 +123,7 @@ if [ $EUID -eq 0 ]; then # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all: app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://') # Two dirname invocations, so returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2 - app_installprefix=$(dirname $(dirname app_modulpath)) + app_installprefix=$(dirname $(dirname ${app_modulepath})) app_dir=${app_installprefix}/software/${app} app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." From 4f732776fc96e1a779ffb053bd0937137ef6c020 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 07:05:37 +0200 Subject: [PATCH 1173/1795] Limit this PR to only the EESSI-remove-software.sh change --- .../20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml deleted file mode 100644 index 755bea096e..0000000000 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml +++ /dev/null @@ -1,7 +0,0 @@ -# 2024.09.18 -# We need to reinstall CUDA in the accelerator prefixes -# See https://github.com/EESSI/software-layer/pull/720 -easyconfigs: - - CUDA-12.1.1.eb: - options: - accept-eula-for: CUDA From 53661bf07c9d240da40a12ae81bb02703c6bd4df Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 08:05:49 +0200 Subject: [PATCH 1174/1795] Fix issue with grep not returning anything on the CPU prefix --- create_tarball.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index e70a9b18d6..9c212681a5 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -64,9 +64,9 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then # module files - find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module symlinks - find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module files and symlinks find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ @@ -83,7 +83,7 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \ - | grep -v '/\.wh\.' >> ${files_list} + | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match done fi done From ce09733386a5a9b99695c548731c4989bd0041c4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 08:41:25 +0200 Subject: [PATCH 1175/1795] Readd cuda reinstall --- .../20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml new file mode 100644 index 0000000000..755bea096e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml @@ -0,0 +1,7 @@ +# 2024.09.18 +# We need to reinstall CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/720 +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA From 6d8ff958be445f3e46f74526614cc0c3c61981ee Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 08:43:42 +0200 Subject: [PATCH 1176/1795] Fix creation of tarball, since grep doesn't match (and thus returns non-zero exit code) for the CPU prefix if nothing got build there --- create_tarball.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index e70a9b18d6..9c212681a5 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -64,9 +64,9 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then # module files - find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module symlinks - find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module files and symlinks find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ @@ -83,7 +83,7 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \ - | grep -v '/\.wh\.' >> ${files_list} + | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match done fi done From 67fb8131f0aeb9af3d59205531ed440f4352d742 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 09:24:30 +0200 Subject: [PATCH 1177/1795] {2023.06}[foss/2023a,zen4] PyQt5 v5.15.10 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 0818701bbb..ddf0ec4507 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,3 +1,4 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb + - PyQt5-5.15.10-GCCcore-12.3.0.eb From 3d43781830578a34384ee54541662e8b698a9590 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 09:31:14 +0200 Subject: [PATCH 1178/1795] {2023} --- .../2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml new file mode 100644 index 0000000000..382241b98e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -0,0 +1,5 @@ +easyconfigs: + - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 + options: + from-pr: 21436 From 8adaaafbe93a89a60702e97c08afa17903c9452d Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 09:49:44 +0200 Subject: [PATCH 1179/1795] update hook for v29Aug2024 --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b3e457cfe3..f25aacfc9c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -301,7 +301,7 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI to pass on aarch64 """ - if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2',): + if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2', '29Aug2024'): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets, # so remove them as dependencies for LAMMPS (they're optional); From 914d5c13572d78defa82e3bd13aa01c28b5f022a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:31:39 +0200 Subject: [PATCH 1180/1795] Move to rebuilds --- .../20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml From 24879a80c16eec3da1b19ea409fc20331b19b407 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:34:39 +0200 Subject: [PATCH 1181/1795] Move to rebuilds --- .../20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml From aef0999a54787988fd844ec7b07643efba74c28c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:44:42 +0200 Subject: [PATCH 1182/1795] Move into rebuilds --- .../20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml From 5dcce7688a3af265da6b966eaceb13ff2b3de599 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:45:44 +0200 Subject: [PATCH 1183/1795] Add comment --- .../20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml index cccbfa6808..23801e0250 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall OSU-Micro-Benchmarks in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/716 easyconfigs: - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb From 4805c09a06977c77177f700fa135954cde6a3859 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:49:20 +0200 Subject: [PATCH 1184/1795] Add comment --- .../20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml index 01a47bbc99..da2c06ae1e 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall CUDA-Samples in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/715 easyconfigs: - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb From 9232ac91aa8b3c5ee734b3e72397aa90e16831e7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:49:58 +0200 Subject: [PATCH 1185/1795] Add comment --- .../rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml index bfe4a4dc52..d347af335a 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall UCX-CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/719 easyconfigs: - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb From 09bfb97a4d29f62b901316c9dd08dc6a422bc519 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 11:09:34 +0200 Subject: [PATCH 1186/1795] update missing installations hook --- eb_hooks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index f25aacfc9c..d999c735c3 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -297,9 +297,9 @@ def parse_hook_ucx_eprefix(ec, eprefix): raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!") -def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): +def parse_hook_lammps_remove_deps_for_aarch64(ec, *args, **kwargs): """ - Remove x86_64 specific dependencies for the CI to pass on aarch64 + Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64 """ if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2', '29Aug2024'): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': @@ -309,7 +309,7 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000; # we need this hook because we check for missing installations for all CPU targets # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) - ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb')] + ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb',)] else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") @@ -776,7 +776,7 @@ def inject_gpu_property(ec): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'grpcio': parse_hook_grpcio_zlib, - 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, + 'LAMMPS': parse_hook_lammps_remove_deps_for_aarch64, 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, From db91170bf73112ff98e92c818a857cfa4b9a6c93 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 12:01:44 +0200 Subject: [PATCH 1187/1795] use --from-commit --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml index 382241b98e..b3c903161a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -2,4 +2,4 @@ easyconfigs: - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 options: - from-pr: 21436 + from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 From 05be1a2a9ea800796e40c551e42f4639088916a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 19 Sep 2024 16:54:37 +0200 Subject: [PATCH 1188/1795] adjust the prefix variable for accelerator builds --- bot/check-build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/check-build.sh b/bot/check-build.sh index d8246c67be..f185b18dda 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -457,8 +457,14 @@ if [[ ! -z ${TARBALL} ]]; then repo_version=$(cfg_get_value "repository" "repo_version") os_type=$(cfg_get_value "architecture" "os_type") software_subdir=$(cfg_get_value "architecture" "software_subdir") + accelerator=$(cfg_get_value "architecture" "accelerator") prefix="${repo_version}/software/${os_type}/${software_subdir}" + # if we build for an accelerator, the prefix is different + if [[ ! -z ${accelerator} ]]; then + prefix="${prefix}/accel/${accelerator}" + fi + # extract directories/entries from tarball content modules_entries=$(grep "${prefix}/modules" ${tmpfile}) software_entries=$(grep "${prefix}/software" ${tmpfile}) From 07572887b9ae6683ffe6eb700eb8a51aa0e9325a Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 20 Sep 2024 08:23:46 +0200 Subject: [PATCH 1189/1795] {2023.06}[foss/2023a] NCCL v2.18.3 w/ CUDA 12.1.1 --- .../20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml new file mode 100644 index 0000000000..6ff3b1bfbf --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml @@ -0,0 +1,5 @@ +# 2024.09.19 +# We need to reinstall NCCL in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/487 +easyconfigs: + - NCCL-2.18.3-GCCcore-12.3.0-CUDA-12.1.1.eb From 7475143a08d4d0da33304debd8a849d7a165a3bb Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 20 Sep 2024 08:31:57 +0200 Subject: [PATCH 1190/1795] Update eessi-2023.06-eb-4.9.3-2023a.yml --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index ddf0ec4507..c51a0b273a 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -2,3 +2,4 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb - PyQt5-5.15.10-GCCcore-12.3.0.eb + - OrthoFinder-2.5.5-foss-2023a.eb From ac3b4dd80a1802c42b6e4684e9105c095a226b94 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:14:43 +0200 Subject: [PATCH 1191/1795] {2023.06}[foss/2023a,zen4] snakemake v8.4.2 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index c51a0b273a..f5d5f6c16c 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -3,3 +3,4 @@ easyconfigs: - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb - PyQt5-5.15.10-GCCcore-12.3.0.eb - OrthoFinder-2.5.5-foss-2023a.eb + - snakemake-8.4.2-foss-2023a.eb From b963a4889d1a4c80c57d9bc72f5061c68520c200 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:39:11 +0200 Subject: [PATCH 1192/1795] {2023.06}[2023a,zen4] Rivet 3.1.9 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index f5d5f6c16c..cc35cb7af5 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -4,3 +4,4 @@ easyconfigs: - PyQt5-5.15.10-GCCcore-12.3.0.eb - OrthoFinder-2.5.5-foss-2023a.eb - snakemake-8.4.2-foss-2023a.eb + - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb From b27229f0980c15dc61f2685ab350a975750f9184 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:35:03 +0200 Subject: [PATCH 1193/1795] {2023.06}[foss/2023a,zen4] GATK v4.5.0.0 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index cc35cb7af5..6864fcf632 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -5,3 +5,4 @@ easyconfigs: - OrthoFinder-2.5.5-foss-2023a.eb - snakemake-8.4.2-foss-2023a.eb - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb + - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb From 193948cb4c906be2ef494e8b4fb82611b8664a29 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 23 Sep 2024 09:19:55 +0200 Subject: [PATCH 1194/1795] {2023.06}[gfbf/2023a,zen4] ipympl v0.9.3 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 6864fcf632..ded94d4cbe 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -6,3 +6,5 @@ easyconfigs: - snakemake-8.4.2-foss-2023a.eb - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb + - ipympl-0.9.3-foss-2023a.eb + - ipympl-0.9.3-gfbf-2023a.eb From 4ad4da28aab74e5076d4adfaba65d8cfc0980d62 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 23 Sep 2024 11:26:23 +0200 Subject: [PATCH 1195/1795] remove ipympl-0.9.3-foss-2023a.eb --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index ded94d4cbe..cbacef6501 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -6,5 +6,4 @@ easyconfigs: - snakemake-8.4.2-foss-2023a.eb - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - - ipympl-0.9.3-foss-2023a.eb - ipympl-0.9.3-gfbf-2023a.eb From d31d052eaaf607d5f9bc9798cf7eb435403d80c3 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 23 Sep 2024 11:39:39 +0000 Subject: [PATCH 1196/1795] {2023.06}[foss/2022b] math-packages --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 02b00e6173..5f63a7bc14 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -27,4 +27,8 @@ easyconfigs: - lpsolve-5.5.2.11-GCC-12.2.0.eb - fastp-0.23.4-GCC-12.2.0.eb - KronaTools-2.8.1-GCCcore-12.2.0.eb - - MultiQC-1.14-foss-2022b.eb + - MultiQC-1.14-foss-2022b.eb + - CGAL-5.5.2-GCCcore-12.2.0.eb + - KaHIP-3.14-gompi-2022b.eb + - MPC-1.3.1-GCCcore-12.2.0.eb + - MUMPS-5.6.1-foss-2022b-metis.eb From c14690cad1395fc2a12e08af6a0fc848edd904ca Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 24 Sep 2024 11:42:46 +0200 Subject: [PATCH 1197/1795] {2023.06}[foss/2023a,zen4] LHAPDF v6.5.4 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index cbacef6501..29ab1609da 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -7,3 +7,4 @@ easyconfigs: - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - ipympl-0.9.3-gfbf-2023a.eb + - LHAPDF-6.5.4-GCC-12.3.0.eb From 8fa60259ea3dda1bef61de33dad61f9c966595f5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 12:51:22 +0200 Subject: [PATCH 1198/1795] Limit CUDA hook to EESSI installs only, and remove duplication when creating symlinks --- eb_hooks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b3e457cfe3..e3c6c4faeb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -684,7 +684,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs): Remove files from CUDA installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. """ - if self.name == 'CUDA': + # Make sure we only do this for CUDA and only if we are doing a CVMFS installation + if self.name == 'CUDA' and self.installdir.startswith('/cvmfs/software.eessi.io/versions'): print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped @@ -733,6 +734,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): basename, full_path) # if it is not in the allowlist, delete the file and create a symlink to host_injections host_inj_path = full_path.replace('versions', 'host_injections') + # CUDA itself doesn't care about compute capability so remove this duplication from + # under host_injections + host_inj_path = re.sub(r"accel/nvidia/cc\d+/", '', host_inj_path) # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From c1764335999be619d5b2ae8629fd9552a1b87abc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Sep 2024 15:23:06 +0200 Subject: [PATCH 1199/1795] Add gmsh --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..f2d8a9380f --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - gmsh-4.12.2-foss-2023a.eb From 087b7d7d42436b07c035b3433d03c9318fd9ec4c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 15:46:08 +0200 Subject: [PATCH 1200/1795] Use EESSI_ACCELERATOR_TARGET rather than regex --- eb_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index e3c6c4faeb..0a42a91d49 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -736,7 +736,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): host_inj_path = full_path.replace('versions', 'host_injections') # CUDA itself doesn't care about compute capability so remove this duplication from # under host_injections - host_inj_path = re.sub(r"accel/nvidia/cc\d+/", '', host_inj_path) + accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") + if accel_subdir: + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From 8594bb2abbbc01ec5c23e87fb95a6e8198b43284 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Sep 2024 15:59:45 +0200 Subject: [PATCH 1201/1795] Also add basemap and geopandas --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml index f2d8a9380f..a34e5d768e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,2 +1,4 @@ easyconfigs: - gmsh-4.12.2-foss-2023a.eb + - basemap-1.3.9-foss-2023a.eb + - geopandas-0.14.2-foss-2023a.eb From b8555d1ee64330a0a43df838f4d1ec58a916a4b5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 16:00:32 +0200 Subject: [PATCH 1202/1795] Ensure we are making an EESSI install when using the CUDA hook --- eb_hooks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0a42a91d49..4d94357e8c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -685,7 +685,12 @@ def post_sanitycheck_cuda(self, *args, **kwargs): and replace them with a symlink to a corresponding installation under host_injections. """ # Make sure we only do this for CUDA and only if we are doing a CVMFS installation - if self.name == 'CUDA' and self.installdir.startswith('/cvmfs/software.eessi.io/versions'): + is_eessi_install = ( + self.installdir.startswith("/cvmfs/software.eessi.io/versions") + and not build_option("sanity_check_only") + and not build_option("module_only") + ) + if self.name == 'CUDA' and is_eessi_install: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped From fb2c0858a6a5c42d6056b019c62e043b8172ad92 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 24 Sep 2024 16:07:16 +0200 Subject: [PATCH 1203/1795] Sloppy copy/paste --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4d94357e8c..96fa48e129 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -743,7 +743,7 @@ def post_sanitycheck_cuda(self, *args, **kwargs): # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From cf63589f1c5a0f0f08a4eb44384ee0a920754848 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 24 Sep 2024 16:08:37 +0200 Subject: [PATCH 1204/1795] Update eb_hooks.py --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 96fa48e129..ed006bae1c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -743,7 +743,7 @@ def post_sanitycheck_cuda(self, *args, **kwargs): # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') + host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From 4478fa89b26472c24824c053d67d35f040fc1bc2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Sep 2024 16:12:22 +0200 Subject: [PATCH 1205/1795] Add various tools --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..d3927ea3f1 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - ccache-4.9-GCCcore-12.3.0.eb + - GDB-13.2-GCCcore-12.3.0.eb + - mold-1.11.0-GCCcore-12.3.0.eb + - tmux-3.3a-GCCcore-12.3.0.eb + - Vim-9.1.0004-GCCcore-12.3.0.eb From 2ca35fc51326cee97b21fe381c55421c047f87ae Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 09:32:10 +0200 Subject: [PATCH 1206/1795] Add fPIC through hooks for FreeImage on ARM, see if that resolves https://github.com/EESSI/software-layer/pull/736#issuecomment-2373261889 --- eb_hooks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index b3e457cfe3..0987d6d683 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -297,6 +297,18 @@ def parse_hook_ucx_eprefix(ec, eprefix): raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!") +def parse_hook_freeimage_aarch64(ec, *args, **kwargs): + """ + Make sure to build with -fPIC on ARM to avoid + https://github.com/EESSI/software-layer/pull/736#issuecomment-2373261889 + """ + if ec.name == 'FreeImage' and ec.version in ('3.18.0',): + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': + if not hasattr(ec, 'toolchainopts'): + ec['toolchainopts'] = {} + ec['toolchainopts']['fPIC'] = True + print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) + def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI to pass on aarch64 From 3b9fbdc9bf4b1396388efd01647468ec7a36a2de Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 25 Sep 2024 09:32:47 +0200 Subject: [PATCH 1207/1795] Update eessi-2023.06-eb-4.9.3-2023a.yml --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 29ab1609da..1d1b2caa09 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -8,3 +8,4 @@ easyconfigs: - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - ipympl-0.9.3-gfbf-2023a.eb - LHAPDF-6.5.4-GCC-12.3.0.eb + - LoopTools-2.15-GCC-12.3.0.eb From e61617ce19307a16132c5154fbc9729abee9d434 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 09:51:34 +0200 Subject: [PATCH 1208/1795] {2023.06}[system] EasyBuild v4.9.4 --- .../2023.06/eessi-2023.06-eb-4.9.3-001-system.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml new file mode 100644 index 0000000000..d9c6075561 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml @@ -0,0 +1,5 @@ +easyconfigs: + - EasyBuild-4.9.4.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 From 07c5f0aab33b591c30101c01fc08f904a21d7335 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 10:05:21 +0200 Subject: [PATCH 1209/1795] {2023.06}[foss/2023a] NCCL 2.18.3 w/ CUDA 12.1.1 --- ...ix.yml => 20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename easystacks/software.eessi.io/2023.06/rebuilds/{20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml => 20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml} (92%) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml similarity index 92% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml index 6ff3b1bfbf..d6667af9a1 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml @@ -1,4 +1,4 @@ -# 2024.09.19 +# 2024.09.25 # We need to reinstall NCCL in the accelerator prefixes # See https://github.com/EESSI/software-layer/pull/487 easyconfigs: From ae62bd31a55cac6dc789ebacae464a9b308c5b5d Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:32:49 +0200 Subject: [PATCH 1210/1795] {2023.06}[2023a,zen4] ncdu 1.18 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 1d1b2caa09..7ced0d9218 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -9,3 +9,4 @@ easyconfigs: - ipympl-0.9.3-gfbf-2023a.eb - LHAPDF-6.5.4-GCC-12.3.0.eb - LoopTools-2.15-GCC-12.3.0.eb + - ncdu-1.18-GCC-12.3.0.eb From b4f6b60c2f5c582360a1148ceb5052077d7287fd Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 10:41:07 +0200 Subject: [PATCH 1211/1795] Actually use the hook... --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index 0987d6d683..15e51e688c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -787,6 +787,7 @@ def inject_gpu_property(ec): 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, + 'FreeImage': parse_hook_freeimage_aarch64, 'grpcio': parse_hook_grpcio_zlib, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, From d37a958d008ba679af7872b4bb9038df74338842 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 10:54:02 +0200 Subject: [PATCH 1212/1795] Move CUDA hook to post-install, allow hook to trigger for any EESSI distributed repo (but always make symlinks to software.eessi.io) --- eb_hooks.py | 76 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4d94357e8c..788a0419ba 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -131,7 +131,8 @@ def pre_prepare_hook(self, *args, **kwargs): def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ Post-configure hook for GCCcore: - - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' + - copy RPATH wrapper script for linker commands to also have a wrapper in + place with system type prefix like 'x86_64-pc-linux-gnu' """ if self.name == 'GCCcore': config_guess = obtain_config_guess() @@ -279,10 +280,10 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. """ if ec.name == 'Qt5': - # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, - # see also https://github.com/easybuilders/easybuild-framework/pull/4290 - ec['check_qtwebengine'] = False - print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") + # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, + # see also https://github.com/easybuilders/easybuild-framework/pull/4290 + ec['check_qtwebengine'] = False + print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") else: raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") @@ -341,7 +342,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - # note: keep condition in sync with the one used in + # note: keep condition in sync with the one used in # post_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target in [CPU_TARGET_A64FX, CPU_TARGET_NEOVERSE_V1]: @@ -360,12 +361,13 @@ def post_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwar if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - # note: keep condition in sync with the one used in + # note: keep condition in sync with the one used in # pre_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target == CPU_TARGET_NEOVERSE_N1: update_build_option('optarch', self.orig_optarch) + def pre_configure_hook(self, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" if self.name in PRE_CONFIGURE_HOOKS: @@ -389,6 +391,7 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): else: raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") + def pre_configure_hook_extrae(self, *args, **kwargs): """ Pre-configure hook for Extrae @@ -414,7 +417,11 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # replace use of 'which' with 'command -v', since 'which' is broken in EESSI build container; # this must be done *after* running configure script, because initial configuration re-writes configure script, # and problem due to use of which only pops up when running make ?! - self.cfg.update('prebuildopts', "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && ") + self.cfg.update( + 'prebuildopts', + "cp config/mpi-macros.m4 config/mpi-macros.m4.orig &&" + "sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && " + ) else: raise EasyBuildError("Extrae-specific hook triggered for non-Extrae easyconfig?!") @@ -445,7 +452,10 @@ def pre_configure_hook_gromacs(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg.update('configopts', '-DGMX_SIMD=ARM_NEON_ASIMD') - print_msg("Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", self.version) + print_msg( + "Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", + self.version + ) else: raise EasyBuildError("GROMACS-specific hook triggered for non-GROMACS easyconfig?!") @@ -506,12 +516,12 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): pattern = "Linux x86_64 ppc64le, gfortran" repl = "Linux x86_64 aarch64 ppc64le, gfortran" if LooseVersion(self.version) <= LooseVersion('3.9.0'): - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'): - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") @@ -533,7 +543,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") -def pre_test_hook(self,*args, **kwargs): +def pre_test_hook(self, *args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: PRE_TEST_HOOKS[self.name](self, *args, **kwargs) @@ -596,6 +606,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): elif cpu_target == CPU_TARGET_A64FX and self.version in scipy_bundle_versions_a64fx: self.cfg['testopts'] = "|| echo ignoring failing tests" + def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): """ Pre-test hook for netCDF: skip failing tests for selected netCDF versions on neoverse_v1 @@ -609,6 +620,7 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" + def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): """ Pre-test hook for PyTorch: increase max failing tests for ARM for PyTorch 2.1.2 @@ -673,24 +685,24 @@ def pre_single_extension_testthat(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " -def post_sanitycheck_hook(self, *args, **kwargs): - """Main post-sanity-check hook: trigger custom functions based on software name.""" - if self.name in POST_SANITYCHECK_HOOKS: - POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) +def post_postproc_hook(self, *args, **kwargs): + """Main post-postprocessing hook: trigger custom functions based on software name.""" + if self.name in POST_POSTPROC_HOOKS: + POST_POSTPROC_HOOKS[self.name](self, *args, **kwargs) -def post_sanitycheck_cuda(self, *args, **kwargs): +def post_postproc_cuda(self, *args, **kwargs): """ Remove files from CUDA installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. """ - # Make sure we only do this for CUDA and only if we are doing a CVMFS installation - is_eessi_install = ( - self.installdir.startswith("/cvmfs/software.eessi.io/versions") - and not build_option("sanity_check_only") - and not build_option("module_only") - ) - if self.name == 'CUDA' and is_eessi_install: + + # We need to check if we are doing an EESSI-distributed installation + eessi_pattern = r"^/cvmfs/[^/]*.eessi.io/versions/" + host_injections_location = "/cvmfs/software.eessi.io/host_injections/" + eessi_installation = bool(re.search(eessi_pattern, self.installdir)) + + if self.name == 'CUDA' and eessi_installation: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped @@ -738,12 +750,14 @@ def post_sanitycheck_cuda(self, *args, **kwargs): self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", basename, full_path) # if it is not in the allowlist, delete the file and create a symlink to host_injections - host_inj_path = full_path.replace('versions', 'host_injections') + + # the host_injections path is under a fixed repo/location for CUDA + host_inj_path = re.sub(eessi_pattern, host_injections_location, full_path) # CUDA itself doesn't care about compute capability so remove this duplication from # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " @@ -775,7 +789,7 @@ def inject_gpu_property(ec): ec_dict['builddependencies'].append(dep) value = '\n'.join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) if key in ec_dict: - if not value in ec_dict[key]: + if value not in ec_dict[key]: ec[key] = '\n'.join([ec_dict[key], value]) else: ec[key] = value @@ -835,6 +849,6 @@ def inject_gpu_property(ec): 'numpy': post_single_extension_numpy, } -POST_SANITYCHECK_HOOKS = { - 'CUDA': post_sanitycheck_cuda, +POST_POSTPROC_HOOKS = { + 'CUDA': post_postproc_cuda, } From 1d446dcddcc68b09efa85429198435f7a3cdfe05 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 11:05:38 +0200 Subject: [PATCH 1213/1795] The eb toolchainopts is called pic, not fPIC --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 15e51e688c..2b503f4f9d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -306,7 +306,7 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': if not hasattr(ec, 'toolchainopts'): ec['toolchainopts'] = {} - ec['toolchainopts']['fPIC'] = True + ec['toolchainopts']['pic'] = True print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): From 4fe6194a76ba39f808b5b1cf65b46ca5701594e5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 11:10:51 +0200 Subject: [PATCH 1214/1795] Remove mold, it's a complicated case since it is a linker, and it's not urgent now --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml index d3927ea3f1..0a4a4e5554 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,6 +1,5 @@ easyconfigs: - ccache-4.9-GCCcore-12.3.0.eb - GDB-13.2-GCCcore-12.3.0.eb - - mold-1.11.0-GCCcore-12.3.0.eb - tmux-3.3a-GCCcore-12.3.0.eb - Vim-9.1.0004-GCCcore-12.3.0.eb From c58033861c94c4b01dca443ef7a672b63da274fa Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 11:14:29 +0200 Subject: [PATCH 1215/1795] Address review comments --- eb_hooks.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index f9ecefe1f6..9b0e9c8dcb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -32,6 +32,9 @@ SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] +EESSI_INSTALLATION_REGEX = r"^/cvmfs/[^/]*.eessi.io/versions/" +HOST_INJECTIONS_LOCATION = "/cvmfs/software.eessi.io/host_injections/" + def get_eessi_envvar(eessi_envvar): """Get an EESSI environment variable from the environment""" @@ -419,7 +422,7 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # and problem due to use of which only pops up when running make ?! self.cfg.update( 'prebuildopts', - "cp config/mpi-macros.m4 config/mpi-macros.m4.orig &&" + "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && " "sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && " ) else: @@ -698,9 +701,7 @@ def post_postproc_cuda(self, *args, **kwargs): """ # We need to check if we are doing an EESSI-distributed installation - eessi_pattern = r"^/cvmfs/[^/]*.eessi.io/versions/" - host_injections_location = "/cvmfs/software.eessi.io/host_injections/" - eessi_installation = bool(re.search(eessi_pattern, self.installdir)) + eessi_installation = bool(re.search(EESSI_INSTALLATION_REGEX, self.installdir)) if self.name == 'CUDA' and eessi_installation: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") @@ -752,12 +753,12 @@ def post_postproc_cuda(self, *args, **kwargs): # if it is not in the allowlist, delete the file and create a symlink to host_injections # the host_injections path is under a fixed repo/location for CUDA - host_inj_path = re.sub(eessi_pattern, host_injections_location, full_path) + host_inj_path = re.sub(EESSI_INSTALLATION_REGEX, HOST_INJECTIONS_LOCATION, full_path) # CUDA itself doesn't care about compute capability so remove this duplication from - # under host_injections + # under host_injections (symlink to a single CUDA installation for all compute + # capabilities) accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: From e59156577044ee348de0ec415122c8a840453284 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 12:09:36 +0200 Subject: [PATCH 1216/1795] {2023.06} Nextflow/23.10.0 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml new file mode 100644 index 0000000000..1e30631e57 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - Nextflow-23.10.0.eb From b235c7cbc7ecad2cf2f1709ece38db8cf64ddff8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 12:20:52 +0200 Subject: [PATCH 1217/1795] Make sure Lmod site package and RC file do not appear in accelerator subdir --- create_lmodrc.py | 6 ++++++ create_lmodsitepackage.py | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index 28ad2a1915..e06f8ccfa4 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -33,6 +33,12 @@ def error(msg): error("Prefix directory %s does not exist!" % prefix) lmodrc_path = os.path.join(prefix, DOT_LMOD, 'lmodrc.lua') +# Lmod itself doesn't care about compute capability so remove this duplication from +# the install path (if it exists) +accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") +if accel_subdir: + lmodrc_path = lmodrc_path.replace("/accel/%s" % accel_subdir, '') + lmodrc_txt = TEMPLATE_LMOD_RC % { 'dot_lmod': DOT_LMOD, 'prefix': prefix, diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 62f073c9a6..11ca614be5 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -8,7 +8,7 @@ DOT_LMOD = '.lmod' -hook_txt ="""require("strict") +hook_txt = """require("strict") local hook = require("Hook") local open = io.open @@ -36,7 +36,7 @@ -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules) -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06 local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections') - + -- Check if the full modulepath starts with the eessi_prefix_* return string.find(t.fn, "^" .. eessi_prefix) ~= nil or string.find(t.fn, "^" .. eessi_prefix_host_injections) ~= nil end @@ -103,7 +103,7 @@ if isFile(archSitePackage) then dofile(archSitePackage) end - + end @@ -111,7 +111,7 @@ local frameStk = require("FrameStk"):singleton() local mt = frameStk:mt() local simpleName = string.match(t.modFullName, "(.-)/") - -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. + -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" @@ -207,6 +207,7 @@ load_site_specific_hooks() """ + def error(msg): sys.stderr.write("ERROR: %s\n" % msg) sys.exit(1) @@ -221,12 +222,18 @@ def error(msg): error("Prefix directory %s does not exist!" % prefix) sitepackage_path = os.path.join(prefix, DOT_LMOD, 'SitePackage.lua') + +# Lmod itself doesn't care about compute capability so remove this duplication from +# the install path (if it exists) +accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") +if accel_subdir: + sitepackage_path = sitepackage_path.replace("/accel/%s" % accel_subdir, '') try: os.makedirs(os.path.dirname(sitepackage_path), exist_ok=True) with open(sitepackage_path, 'w') as fp: fp.write(hook_txt) # Make sure that the created Lmod file has "read/write" for the user/group and "read" permissions for others - os.chmod(sitepackage_path, S_IREAD|S_IWRITE|S_IRGRP|S_IWGRP|S_IROTH) + os.chmod(sitepackage_path, S_IREAD | S_IWRITE | S_IRGRP | S_IWGRP | S_IROTH) except (IOError, OSError) as err: error("Failed to create %s: %s" % (sitepackage_path, err)) From 93dae0daa8b44150aa4bdd9f606eafd3a7742c1a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 12:23:33 +0200 Subject: [PATCH 1218/1795] Tweak comment --- create_lmodrc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index e06f8ccfa4..1720b762f0 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -33,8 +33,8 @@ def error(msg): error("Prefix directory %s does not exist!" % prefix) lmodrc_path = os.path.join(prefix, DOT_LMOD, 'lmodrc.lua') -# Lmod itself doesn't care about compute capability so remove this duplication from -# the install path (if it exists) +# Lmod itself doesn't care about the accelerator subdir so remove this duplication from +# the target path (if it exists) accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: lmodrc_path = lmodrc_path.replace("/accel/%s" % accel_subdir, '') From 74c382447bb5bccd3a467263a910fc5bad15bb00 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:31:58 +0200 Subject: [PATCH 1219/1795] {2023.06}[foss/2023a,zen4] WhatsHap v2.2 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 7ced0d9218..4bae944d45 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -10,3 +10,4 @@ easyconfigs: - LHAPDF-6.5.4-GCC-12.3.0.eb - LoopTools-2.15-GCC-12.3.0.eb - ncdu-1.18-GCC-12.3.0.eb + - WhatsHap-2.2-foss-2023a.eb From 973ee2d18079683dd683daae4e7ea069a3c29c02 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 15:14:04 +0200 Subject: [PATCH 1220/1795] Allow EESSI-extend to work even when LMOD_EXACT_MATCH is set --- EESSI-extend-2023.06-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index b525ee462d..ba8629c02e 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -166,7 +166,7 @@ elseif (project_modulepath ~= nil) then end -- Make sure EasyBuild itself is loaded if not ( isloaded("EasyBuild") ) then - load("EasyBuild") + load(latest("EasyBuild")) end """ From 9f1971c52800881157098bde58e3bac82649c99c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 15:15:27 +0200 Subject: [PATCH 1221/1795] Dont use hasattr on ec, since it's a dict, not an object --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 2b503f4f9d..4f9cd88bd3 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -304,10 +304,10 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): """ if ec.name == 'FreeImage' and ec.version in ('3.18.0',): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - if not hasattr(ec, 'toolchainopts'): + if toolchainopts not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True - print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) + print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ From eebd184890922cd205ae0c8c2e66892701e1491f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 15:25:31 +0200 Subject: [PATCH 1222/1795] Should be a string of course --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4f9cd88bd3..9cff30f939 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -304,7 +304,7 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): """ if ec.name == 'FreeImage' and ec.version in ('3.18.0',): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - if toolchainopts not in ec: + if 'toolchainopts' not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) From fc735871760159595a43ec5cbf12bf0679316db2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 15:30:10 +0200 Subject: [PATCH 1223/1795] Also add the rebuild yml --- .../2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml new file mode 100644 index 0000000000..9cd1b451cd --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml @@ -0,0 +1,6 @@ +# 2024.09.25 +# EESSI-extend did not support LMOD_EXACT_MATCH +# (see https://github.com/EESSI/software-layer/pull/747) +easyconfigs: + - EESSI-extend-2023.06-easybuild.eb + From 150188043d361fa165e30cafe76eebb6122a709f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 25 Sep 2024 17:05:41 +0200 Subject: [PATCH 1224/1795] Add ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..d9429ad8a8 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,5 @@ +easyconfigs: + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From 00157593ecc1795d58b82462b59a84f31ef00194 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 22:04:54 +0200 Subject: [PATCH 1225/1795] UCC CUDA rebuild now that we have an accel prefix --- .../rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml new file mode 100644 index 0000000000..a418086c44 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml @@ -0,0 +1,4 @@ +# 2024.09.19 +# We need to reinstall UCC-CUDA in the accelerator prefixes +easyconfigs: + - UCC-CUDA-1.2.0-GCCcore-12.3.0-CUDA-12.1.1.eb From 61c61237028ba23b3643fc2478a2e1071c86554f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 22:58:45 +0200 Subject: [PATCH 1226/1795] Fix issue with missing symbol --- eb_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9cff30f939..0538650595 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -186,7 +186,7 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): ): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if cpu_target == CPU_TARGET_NEOVERSE_V1: - if not hasattr(ec, 'toolchainopts'): + if 'toolchainopts' not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['vectorize'] = False print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) @@ -307,8 +307,10 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): if 'toolchainopts' not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True + ec['toolchainopts']['extracflags'] = '-DPNG_ARM_NEON_OPT=0' print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) + def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI to pass on aarch64 From 4273dab9cafffa7f28c3d67e6d999caa5d08ce99 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 23:10:33 +0200 Subject: [PATCH 1227/1795] Use correct keyword --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0538650595..a33b71a812 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -307,7 +307,7 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): if 'toolchainopts' not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True - ec['toolchainopts']['extracflags'] = '-DPNG_ARM_NEON_OPT=0' + ec['toolchainopts']['extra_cflags'] = '-DPNG_ARM_NEON_OPT=0' print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) From d5870f5816fcfd6addb36219ad4e1e7d448c790f Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:02:44 +0200 Subject: [PATCH 1228/1795] Update eessi-2023.06-eb-4.9.3-2023a.yml --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 4bae944d45..4f7f88f938 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -11,3 +11,4 @@ easyconfigs: - LoopTools-2.15-GCC-12.3.0.eb - ncdu-1.18-GCC-12.3.0.eb - WhatsHap-2.2-foss-2023a.eb + - PyOpenGL-3.1.7-GCCcore-12.3.0.eb From 44377065d88f00aa5d6512d84744303dcd542b6e Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:10:58 +0200 Subject: [PATCH 1229/1795] Relax grep for finding easystack files --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 1c4032ecbc..ea6459ea92 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -188,7 +188,7 @@ fi pr_diff=$(ls [0-9]*.diff | head -1) # the true at the end of the next command is important: grep will expectedly return 1 if there is no easystack file being added under rebuilds, # but due to "set -e" the entire script would otherwise fail -changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | (grep "/rebuilds/" || true)) +changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | (grep "/rebuilds/" || true)) if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else From 22be85241b18bc2837bc3dabdc953f3cd46967fe Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 26 Sep 2024 14:27:46 +0200 Subject: [PATCH 1230/1795] Move CUDA builds to accel/ subdir --- .../eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/{2023.06/eessi-2023.06-eb-4.9.3-2023a.yml => accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml rename to easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From d933c8c440197b61d94f7b60ed8e6af99197aba1 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 26 Sep 2024 14:43:44 +0200 Subject: [PATCH 1231/1795] Fix EasyBuild PR link --- .../accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml index d9429ad8a8..f41bc986ec 100644 --- a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -1,5 +1,5 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From e20f1dde2c72d85a5beee41afe7a5a9a4191295b Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 26 Sep 2024 15:10:52 +0200 Subject: [PATCH 1232/1795] use subdir accel for CUDA builds --- .../eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023-CUDA.yml => accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml rename to easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From 5915390cafe46e26f3d03a33691c53113b6ac752 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 26 Sep 2024 15:47:36 +0200 Subject: [PATCH 1233/1795] create nvidia directory --- .../accel/{ => nvidia}/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/accel/{ => nvidia}/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml (100%) diff --git a/easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From 7368d39d3ebbe9370f59755eb4fb304ac76e3811 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Sep 2024 20:40:04 +0200 Subject: [PATCH 1234/1795] use easystack file in accel/nvidia for ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb --- .../accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 4 ++++ .../accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml index 91c4b1f472..8935a3f3c3 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -1,2 +1,6 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c diff --git a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml deleted file mode 100644 index f41bc986ec..0000000000 --- a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ /dev/null @@ -1,5 +0,0 @@ -easyconfigs: - - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 - from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From 0b5de15b937e525a465fbb23c3f1c91af532be1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 10:28:57 +0200 Subject: [PATCH 1235/1795] add mapping for cpu->accelerators and check for missing accelerator builds --- .github/workflows/test-software.eessi.io.yml | 36 +++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index d4d980901f..c4e3286f6c 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -2,11 +2,25 @@ name: Check for missing software installations in software.eessi.io on: push: - branches: [ "*-software.eessi.io" ] +# branches: [ "*-software.eessi.io" ] + branches: "*" pull_request: workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) +env: + EESSI_ACCELERATOR_TARGETS: | + aarch64/generic: + aarch64/neoverse_n1: + aarch64/neoverse_v1: + x86_64/generic: + x86_64/amd/zen2: + - nvidia/cc80 + x86_64/amd/zen3: + - nvidia/cc80 + x86_64/amd/zen4: + x86_64/intel/haswell: + x86_64/intel/skylake_avx512: jobs: check_missing: runs-on: ubuntu-latest @@ -48,6 +62,8 @@ jobs: export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux env | grep ^EESSI | sort + + # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." @@ -56,6 +72,24 @@ jobs: if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi done + # now check the accelerator builds for this CPU target + accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]") + if [ -z ${accelerators} ]; then + echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" + else + for accel in ${accelerators}; do + module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/nvidia/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "check missing installations for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + done + module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + done + fi + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} From 4a5808c5f343b34190b0fd050023bd892ce23928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:01:11 +0200 Subject: [PATCH 1236/1795] remove entries without an accelerator --- .github/workflows/test-software.eessi.io.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index c4e3286f6c..bad392aef8 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -10,17 +10,10 @@ permissions: contents: read # to fetch code (actions/checkout) env: EESSI_ACCELERATOR_TARGETS: | - aarch64/generic: - aarch64/neoverse_n1: - aarch64/neoverse_v1: - x86_64/generic: x86_64/amd/zen2: - nvidia/cc80 x86_64/amd/zen3: - nvidia/cc80 - x86_64/amd/zen4: - x86_64/intel/haswell: - x86_64/intel/skylake_avx512: jobs: check_missing: runs-on: ubuntu-latest From eaf656a9d38e1af007484589f4fee615c99d5778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:07:35 +0200 Subject: [PATCH 1237/1795] replace hardcoded nvidia in easystack dir by "dirname $accel" --- .github/workflows/test-software.eessi.io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index bad392aef8..48df5f9ca3 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -73,7 +73,7 @@ jobs: for accel in ${accelerators}; do module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" - for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/nvidia/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/$(dirname ${accel})/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 3c5520a81832fa10071413ceb10a5b5c595b4e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:27:08 +0200 Subject: [PATCH 1238/1795] only run on *-software.eessi.io branches again --- .github/workflows/test-software.eessi.io.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 48df5f9ca3..e247c96676 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -2,8 +2,7 @@ name: Check for missing software installations in software.eessi.io on: push: -# branches: [ "*-software.eessi.io" ] - branches: "*" + branches: [ "*-software.eessi.io" ] pull_request: workflow_dispatch: permissions: From 72fc6e242311902d1e4887bc3ab5b8d6c9f8a961 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 27 Sep 2024 13:47:35 +0200 Subject: [PATCH 1239/1795] Allow Nvidia driver script to set LD_PRELOAD --- .../nvidia/link_nvidia_host_libraries.sh | 317 +++++++++++++----- 1 file changed, 235 insertions(+), 82 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index e8d7f0d0a7..718229b1d7 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -1,74 +1,212 @@ #!/bin/bash # This script links host libraries related to GPU drivers to a location where -# they can be found by the EESSI linker +# they can be found by the EESSI linker (or sets LD_PRELOAD as an +# alternative.) # Initialise our bash functions -TOPDIR=$(dirname $(realpath $BASH_SOURCE)) +TOPDIR=$(dirname "$(realpath "$BASH_SOURCE")") source "$TOPDIR"/../../utils.sh -# We rely on ldconfig to give us the location of the libraries on the host -command_name="ldconfig" -# We cannot use a version of ldconfig that's being shipped under CVMFS -exclude_prefix="/cvmfs" +# Define a function to find the host ld_config +get_host_ldconfig() { + local command_name="ldconfig" # Set command to find + local exclude_prefix="/cvmfs" # Set excluded prefix (paths to ignore) + local found_paths=() # Initialize an array to store found paths -found_paths=() -# Always attempt to use /sbin/ldconfig -if [ -x "/sbin/$command_name" ]; then - found_paths+=("/sbin/$command_name") -fi -IFS=':' read -ra path_dirs <<< "$PATH" -for dir in "${path_dirs[@]}"; do - if [ "$dir" = "/sbin" ]; then - continue # we've already checked for $command_name in /sbin, don't need to do it twice - fi - if [[ ! "$dir" =~ ^$exclude_prefix ]]; then - if [ -x "$dir/$command_name" ]; then - found_paths+=("$dir/$command_name") - fi - fi -done + # Always attempt to use /sbin/ldconfig + if [ -x "/sbin/$command_name" ]; then + found_paths+=("/sbin/$command_name") + fi -if [ ${#found_paths[@]} -gt 0 ]; then - echo "Found $command_name in the following locations:" - printf -- "- %s\n" "${found_paths[@]}" - echo "Using first version" - host_ldconfig=${found_paths[0]} -else - error="$command_name not found in PATH or only found in paths starting with $exclude_prefix." - fatal_error "$error" -fi + # Split the $PATH and iterate over each directory + IFS=':' read -ra path_dirs <<< "$PATH" + for dir in "${path_dirs[@]}"; do + if [ "$dir" = "/sbin" ]; then + continue # Skip /sbin since it's already checked + fi + + # Check if directory does not start with the exclude prefix + if [[ ! "$dir" =~ ^$exclude_prefix ]]; then + if [ -x "$dir/$command_name" ]; then + found_paths+=("$dir/$command_name") + fi + fi + done + + # Check if any paths were found + if [ ${#found_paths[@]} -gt 0 ]; then + # echo the first version we found and return success + echo "${found_paths[0]}" + return 0 + else + fatal_error "$command_name not found in PATH or only found in paths starting with $exclude_prefix." + fi +} -# Make sure EESSI is initialised (doesn't matter what version) -check_eessi_initialised +get_nvlib_list() { + local nvliblist_url="https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf" + local default_nvlib_list=( + "libcuda.so" + "libcudadebugger.so" + "libEGL_installertest.so" + "libEGL_nvidia.so" + "libEGL.so" + "libGLdispatch.so" + "libGLESv1_CM_nvidia.so" + "libGLESv1_CM.so" + "libGLESv2_nvidia.so" + "libGLESv2.so" + "libGL.so" + "libGLX_installertest.so" + "libGLX_nvidia.so" + "libglx.so" + "libGLX.so" + "libnvcuvid.so" + "libnvidia-cbl.so" + "libnvidia-cfg.so" + "libnvidia-compiler.so" + "libnvidia-eglcore.so" + "libnvidia-egl-wayland.so" + "libnvidia-encode.so" + "libnvidia-fatbinaryloader.so" + "libnvidia-fbc.so" + "libnvidia-glcore.so" + "libnvidia-glsi.so" + "libnvidia-glvkspirv.so" + "libnvidia-gpucomp.so" + "libnvidia-gtk2.so" + "libnvidia-gtk3.so" + "libnvidia-ifr.so" + "libnvidia-ml.so" + "libnvidia-nvvm.so" + "libnvidia-opencl.so" + "libnvidia-opticalflow.so" + "libnvidia-ptxjitcompiler.so" + "libnvidia-rtcore.so" + "libnvidia-tls.so" + "libnvidia-wfb.so" + "libnvoptix.so.1" + "libOpenCL.so" + "libOpenGL.so" + "libvdpau_nvidia.so" + "nvidia_drv.so" + "tls_test_.so" + ) -# Find the CUDA version of the host CUDA drivers -# (making sure that this can still work inside prefix environment inside a container) + # Try to download the nvliblist.conf file with curl + echo_yellow "Downloading latest version of nvliblist.conf from Apptainer" + nvliblist_content=$(curl --silent "$nvliblist_url") + + # Check if curl failed (i.e., the content is empty) + if [ -z "$nvliblist_content" ]; then + # Failed to download nvliblist.conf, using default list instead + printf "%s\n" "${default_nvlib_list[@]}" + return 1 + fi + + # If curl succeeded, filter and return the libraries from the downloaded content + echo "$nvliblist_content" | grep '.so$' +} + + +# Check for required commands +command -v nvidia-smi >/dev/null 2>&1 || echo_yellow "nvidia-smi not found, this script won't do anything useful"; exit + +# Variables +LD_PRELOAD_MODE=0 + +# Parse command-line options +while [[ "$#" -gt 0 ]]; do + case $1 in + --ld-preload) LD_PRELOAD_MODE=1 ;; # Enable LD_PRELOAD mode + *) fatal_error "Unknown option: $1";; + esac + shift +done + +# Gather information about NVIDIA drivers (even if we are inside a Gentoo Prefix in a container) export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH -nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader" -if $nvidia_smi_command > /dev/null; then - host_driver_version=$($nvidia_smi_command | tail -n1) + +# Command to give to get the CUDA driver version +nvidia_smi_driver_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader" +if $nvidia_smi_driver_command > /dev/null 2>&1; then + host_driver_version=$($nvidia_smi_driver_command | tail -n1) echo_green "Found NVIDIA GPU driver version ${host_driver_version}" + # If the first worked, this should work too - host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}') + host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}') echo_green "Found host CUDA version ${host_cuda_version}" else - error="Failed to successfully execute\n $nvidia_smi_command\n" - fatal_error "$error" + fatal_error "Failed to execute $nvidia_smi_driver_command" fi -# Let's make sure the driver libraries are not already in place +# Gather any CUDA related driver libraries from the host +# - First let's see what driver libraries are there +# - then extract the ones we need for CUDA + +# Find the host ldconfig +host_ldconfig=$(get_host_ldconfig) +# Gather libraries on the host (_must_ be host ldconfig) +host_libraries=$($host_ldconfig -p | awk '{print $NF}') +singularity_libs=$(ls /.singularity.d/libs/* 2>/dev/null) + +# Now gather the list of possible CUDA libraries +cuda_candidate_libraries=$(get_nvlib_list) +# Check if the function returned an error (e.g., curl failed) +if [ $? -ne 0 ]; then + echo "Using default list of libraries" +else + echo "Using downloaded list of libraries" +fi + +# Filter the host libraries to find the CUDA libaries locations +# Initialize an array to hold the matched libraries +matched_libraries=() + +# Process each library and check for matches in libs.txt +echo "$nvlib_list" | while read -r library; do + # Search for the library in libs.txt and add it to the matched_libraries array + matched=$(echo "$ldconfig_output $singularity_libs" | grep "$library") + if [ -n "$matched" ]; then + matched_libraries+=("$matched") # Add matched library to the array + fi +done + +# Output the matched libraries +echo "Matched CUDA Libraries:" +printf "%s\n" "${matched_libraries[@]}" + +# LD_PRELOAD Mode +if [ "$LD_PRELOAD_MODE" -eq 1 ]; then + # Set LD_PRELOAD with the matched libraries + if [ ${#matched_libraries[@]} -gt 0 ]; then + LD_PRELOAD=$(printf "%s\n" "${matched_libraries[@]}" | tr '\n' ':') + # Remove the trailing colon from LD_PRELOAD if it exists + LD_PRELOAD=${LD_PRELOAD%:} + export LD_PRELOAD + echo "LD_PRELOAD set to: $LD_PRELOAD" + export EESSI_OVERRIDE_GPU_CHECK=1 + echo "Allowing overriding GPU checks in EESSI via EESSI_OVERRIDE_GPU_CHECK" + else + echo "No libraries matched, LD_PRELOAD not set." + exit 0 +fi + +# If we haven't already exited, we may need to create the symlinks + +# First let's make sure the driver libraries are not already in place link_drivers=1 -# first make sure that target of host_injections variant symlink is an existing directory -host_injections_target=$(realpath -m ${EESSI_CVMFS_REPO}/host_injections) -if [ ! -d ${host_injections_target} ]; then - create_directory_structure ${host_injections_target} +# Make sure that target of host_injections variant symlink is an existing directory +host_injections_target=$(realpath -m "${EESSI_CVMFS_REPO}/host_injections") +if [ ! -d "$host_injections_target" ]; then + create_directory_structure "$host_injections_target" fi host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}" host_injection_driver_dir="${host_injections_nvidia_dir}/host" -host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt" +host_injection_driver_version_file="${host_injection_driver_dir}/driver_version.txt" if [ -e "$host_injection_driver_version_file" ]; then if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then echo_green "The host GPU driver libraries (v${host_driver_version}) have already been linked! (based on ${host_injection_driver_version_file})" @@ -76,11 +214,7 @@ if [ -e "$host_injection_driver_version_file" ]; then else # There's something there but it is out of date echo_yellow "Cleaning out outdated symlinks" - rm $host_injection_driver_dir/* - if [ $? -ne 0 ]; then - error="Unable to remove files under '$host_injection_driver_dir'." - fatal_error "$error" - fi + rm "${host_injection_driver_dir}"/* || fatal_error "Unable to remove files under '${host_injection_driver_dir}'." fi fi @@ -89,56 +223,75 @@ if [ "$link_drivers" -eq 1 ]; then if ! create_directory_structure "${host_injection_driver_dir}" ; then fatal_error "No write permissions to directory ${host_injection_driver_dir}" fi - cd ${host_injection_driver_dir} - # Need a small temporary space to hold a couple of files - temp_dir=$(mktemp -d) - - # Gather libraries on the host (_must_ be host ldconfig) - $host_ldconfig -p | awk '{print $NF}' > "$temp_dir"/libs.txt - # Allow for the fact that we may be in a container so the CUDA libs might be in there - ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null - - # Leverage singularity to find the full list of libraries we should be linking to - echo_yellow "Downloading latest version of nvliblist.conf from Apptainer to ${temp_dir}/nvliblist.conf" - curl --silent --output "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf + cd "${host_injection_driver_dir}" || fatal_error "Failed to cd to ${host_injection_driver_dir}" # Make symlinks to all the interesting libraries - grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i ln -s {} + # Loop over each matched library + for library in "${matched_libraries[@]}"; do + # Check if the library file exists + if [ -e "$library" ]; then + # Create a symlink in the current directory + ln -s "$library" . + # Check if the symlink was created successfully + if [ $? -eq 0 ]; then + echo "Successfully created symlink for library $library in $PWD" + else + fatal_error "Error: Failed to create symlink for library $library in $PWD" + fi + else + echo "Warning: Library not found: $library" + fi + done - # Inject driver and CUDA versions into dir - echo $host_driver_version > driver_version.txt - echo $host_cuda_version > cuda_version.txt + # Inject driver and CUDA versions into the directory + echo "$host_driver_version" > driver_version.txt + echo "$host_cuda_version" > cuda_version.txt drivers_linked=1 - - # Remove the temporary directory when done - rm -r "$temp_dir" fi # Make latest symlink for NVIDIA drivers -cd $host_injections_nvidia_dir +cd "$host_injections_nvidia_dir" || fatal_error "Failed to cd to $host_injections_nvidia_dir" symlink="latest" if [ -L "$symlink" ]; then - # Unless the drivers have been installed, leave the symlink alone if [ "$drivers_linked" -eq 1 ]; then - ln -sf host latest + ln -sf host "$symlink" + if [ $? -eq 0 ]; then + echo "Successfully created symlink between $symlink and host in $PWD" + else + fatal_error "Failed to create symlink between $symlink and host in $PWD" + fi fi else - # No link exists yet - ln -s host latest + ln -s host "$symlink" + if [ $? -eq 0 ]; then + echo "Successfully created symlink between $symlink and host in $PWD" + else + fatal_error "Failed to create symlink between $symlink and host in $PWD" + fi fi # Make sure the libraries can be found by the EESSI linker host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections} if [ -L "$host_injection_linker_dir/lib" ]; then target_path=$(readlink -f "$host_injection_linker_dir/lib") - if [ "$target_path" != "$$host_injections_nvidia_dir/latest" ]; then - cd $host_injection_linker_dir - ln -sf $host_injections_nvidia_dir/latest lib + if [ "$target_path" != "$host_injections_nvidia_dir/latest" ]; then + cd "$host_injection_linker_dir" || fatal_error "Failed to cd to $host_injection_linker_dir" + ln -sf "$host_injections_nvidia_dir/latest" lib + if [ $? -eq 0 ]; then + echo "Successfully created symlink between $host_injections_nvidia_dir/latest and lib in $PWD" + else + fatal_error "Failed to create symlink between $host_injections_nvidia_dir/latest and lib in $PWD" + fi fi else - create_directory_structure $host_injection_linker_dir - cd $host_injection_linker_dir - ln -s $host_injections_nvidia_dir/latest lib + create_directory_structure "$host_injection_linker_dir" + cd "$host_injection_linker_dir" || fatal_error "Failed to cd to $host_injection_linker_dir" + ln -s "$host_injections_nvidia_dir/latest" lib + if [ $? -eq 0 ]; then + echo "Successfully created symlink between $host_injections_nvidia_dir/latest and lib in $PWD" + else + fatal_error "Failed to create symlink between $host_injections_nvidia_dir/latest and lib in $PWD" + fi fi echo_green "Host NVIDIA GPU drivers linked successfully for EESSI" From 7229764b75a29836a96509a62476456bdd0258e6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 27 Sep 2024 14:08:10 +0200 Subject: [PATCH 1240/1795] Make an option to not download anything --- .../gpu_support/nvidia/link_nvidia_host_libraries.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 718229b1d7..71872665bd 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -94,8 +94,13 @@ get_nvlib_list() { "tls_test_.so" ) + # Check if the function was called with the "default" argument + if [[ "$1" == "default" ]]; then + printf "%s\n" "${default_nvlib_list[@]}" + return 0 + fi + # Try to download the nvliblist.conf file with curl - echo_yellow "Downloading latest version of nvliblist.conf from Apptainer" nvliblist_content=$(curl --silent "$nvliblist_url") # Check if curl failed (i.e., the content is empty) @@ -115,11 +120,13 @@ command -v nvidia-smi >/dev/null 2>&1 || echo_yellow "nvidia-smi not found, this # Variables LD_PRELOAD_MODE=0 +DOWNLOAD="" # Parse command-line options while [[ "$#" -gt 0 ]]; do case $1 in --ld-preload) LD_PRELOAD_MODE=1 ;; # Enable LD_PRELOAD mode + --no-download) DOWNLOAD="default" ;; # Download latest list of CUDA libraries *) fatal_error "Unknown option: $1";; esac shift @@ -152,7 +159,7 @@ host_libraries=$($host_ldconfig -p | awk '{print $NF}') singularity_libs=$(ls /.singularity.d/libs/* 2>/dev/null) # Now gather the list of possible CUDA libraries -cuda_candidate_libraries=$(get_nvlib_list) +cuda_candidate_libraries=$(get_nvlib_list "${DOWNLOAD}") # Check if the function returned an error (e.g., curl failed) if [ $? -ne 0 ]; then echo "Using default list of libraries" From 513cbb7957dea10447ee1dafecad72ae02fd896d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 27 Sep 2024 14:16:42 +0200 Subject: [PATCH 1241/1795] Make sure the umask allows for global reading --- .../nvidia/link_nvidia_host_libraries.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 71872665bd..b5ec314096 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -114,6 +114,21 @@ get_nvlib_list() { echo "$nvliblist_content" | grep '.so$' } +# Function to check if umask allows global read +check_global_read() { + # Get the current umask value + local current_umask=$(umask) + + # Convert umask to decimal to analyze + local umask_decimal=$((8#$current_umask)) + + # Check if umask allows global read + if [[ $umask_decimal -eq 0 || $umask_decimal -eq 22 ]]; then + echo "The current umask ($current_umask) allows global read permissions." + else + fatal_error "The current umask ($current_umask) does not allow global read permissions." + fi +} # Check for required commands command -v nvidia-smi >/dev/null 2>&1 || echo_yellow "nvidia-smi not found, this script won't do anything useful"; exit @@ -208,6 +223,7 @@ link_drivers=1 # Make sure that target of host_injections variant symlink is an existing directory host_injections_target=$(realpath -m "${EESSI_CVMFS_REPO}/host_injections") if [ ! -d "$host_injections_target" ]; then + check_global_read create_directory_structure "$host_injections_target" fi @@ -227,6 +243,7 @@ fi drivers_linked=0 if [ "$link_drivers" -eq 1 ]; then + check_global_read if ! create_directory_structure "${host_injection_driver_dir}" ; then fatal_error "No write permissions to directory ${host_injection_driver_dir}" fi @@ -291,6 +308,7 @@ if [ -L "$host_injection_linker_dir/lib" ]; then fi fi else + check_global_read create_directory_structure "$host_injection_linker_dir" cd "$host_injection_linker_dir" || fatal_error "Failed to cd to $host_injection_linker_dir" ln -s "$host_injections_nvidia_dir/latest" lib From 3b80a1b3843997b09fb178245c462d8fe3672ad2 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 14:38:00 +0200 Subject: [PATCH 1242/1795] Careful on the exit --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index b5ec314096..22dfb138bb 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -131,7 +131,7 @@ check_global_read() { } # Check for required commands -command -v nvidia-smi >/dev/null 2>&1 || echo_yellow "nvidia-smi not found, this script won't do anything useful"; exit +command -v nvidia-smi >/dev/null 2>&1 || { echo_yellow "nvidia-smi not found, this script won't do anything useful"; exit 1; } # Variables LD_PRELOAD_MODE=0 From 45c59946f060bbbe7c9e4661dedaab99ceb11886 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 27 Sep 2024 12:39:00 +0000 Subject: [PATCH 1243/1795] {2023.06}[foss/2022b] vis-packages --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 5f63a7bc14..0f866b3aa0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -32,3 +32,6 @@ easyconfigs: - KaHIP-3.14-gompi-2022b.eb - MPC-1.3.1-GCCcore-12.2.0.eb - MUMPS-5.6.1-foss-2022b-metis.eb + - GL2PS-1.4.2-GCCcore-12.2.0.eb + - GST-plugins-base-1.22.1-GCC-12.2.0.eb + - wxWidgets-3.2.2.1-GCC-12.2.0.eb From 2475e4d5ca4a9100aaaba04d055a919d64e94989 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 14:45:09 +0200 Subject: [PATCH 1244/1795] Referenced wrong variable --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 22dfb138bb..51babbba3d 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -187,7 +187,7 @@ fi matched_libraries=() # Process each library and check for matches in libs.txt -echo "$nvlib_list" | while read -r library; do +for library in "${cuda_candidate_libraries[@]}"; do # Search for the library in libs.txt and add it to the matched_libraries array matched=$(echo "$ldconfig_output $singularity_libs" | grep "$library") if [ -n "$matched" ]; then From bd6041ebc70eb37507270ec7b3442ba68c698835 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 14:56:31 +0200 Subject: [PATCH 1245/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 51babbba3d..a1ff156f50 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -135,13 +135,13 @@ command -v nvidia-smi >/dev/null 2>&1 || { echo_yellow "nvidia-smi not found, th # Variables LD_PRELOAD_MODE=0 -DOWNLOAD="" +LIBS_LIST="" # Parse command-line options while [[ "$#" -gt 0 ]]; do case $1 in --ld-preload) LD_PRELOAD_MODE=1 ;; # Enable LD_PRELOAD mode - --no-download) DOWNLOAD="default" ;; # Download latest list of CUDA libraries + --no-download) LIBS_LIST="default" ;; # Download latest list of CUDA libraries *) fatal_error "Unknown option: $1";; esac shift @@ -174,7 +174,7 @@ host_libraries=$($host_ldconfig -p | awk '{print $NF}') singularity_libs=$(ls /.singularity.d/libs/* 2>/dev/null) # Now gather the list of possible CUDA libraries -cuda_candidate_libraries=$(get_nvlib_list "${DOWNLOAD}") +cuda_candidate_libraries=$(get_nvlib_list "${LIBS_LIST}") # Check if the function returned an error (e.g., curl failed) if [ $? -ne 0 ]; then echo "Using default list of libraries" From 9d721025e46721f63e8a7a1d96940ff424c05c6a Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 14:58:45 +0200 Subject: [PATCH 1246/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index a1ff156f50..0afc7b6b86 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -157,7 +157,7 @@ if $nvidia_smi_driver_command > /dev/null 2>&1; then echo_green "Found NVIDIA GPU driver version ${host_driver_version}" # If the first worked, this should work too - host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}') + host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk '{NF>1; print $NF}') echo_green "Found host CUDA version ${host_cuda_version}" else fatal_error "Failed to execute $nvidia_smi_driver_command" From 225accc6dba397b07f1b8642864f8e1993ac050c Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 09:51:34 +0200 Subject: [PATCH 1247/1795] {2023.06}[system] EasyBuild v4.9.4 --- .../2023.06/eessi-2023.06-eb-4.9.3-001-system.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml new file mode 100644 index 0000000000..d9c6075561 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml @@ -0,0 +1,5 @@ +easyconfigs: + - EasyBuild-4.9.4.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 From f3820a1d43121d6aa8d9bdeb6bead0ff9c5bdbdc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 12:51:22 +0200 Subject: [PATCH 1248/1795] Limit CUDA hook to EESSI installs only, and remove duplication when creating symlinks --- eb_hooks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b3e457cfe3..e3c6c4faeb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -684,7 +684,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs): Remove files from CUDA installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. """ - if self.name == 'CUDA': + # Make sure we only do this for CUDA and only if we are doing a CVMFS installation + if self.name == 'CUDA' and self.installdir.startswith('/cvmfs/software.eessi.io/versions'): print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped @@ -733,6 +734,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): basename, full_path) # if it is not in the allowlist, delete the file and create a symlink to host_injections host_inj_path = full_path.replace('versions', 'host_injections') + # CUDA itself doesn't care about compute capability so remove this duplication from + # under host_injections + host_inj_path = re.sub(r"accel/nvidia/cc\d+/", '', host_inj_path) # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From a857bf11ccfdc9205f5f225574081a816c3d3800 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 08:41:25 +0200 Subject: [PATCH 1249/1795] Readd cuda reinstall --- .../20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml new file mode 100644 index 0000000000..755bea096e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml @@ -0,0 +1,7 @@ +# 2024.09.18 +# We need to reinstall CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/720 +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA From 1a72a4c2ca36c62d494516c1a8d7ec28341cdcc6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 15:46:08 +0200 Subject: [PATCH 1250/1795] Use EESSI_ACCELERATOR_TARGET rather than regex --- eb_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index e3c6c4faeb..0a42a91d49 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -736,7 +736,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): host_inj_path = full_path.replace('versions', 'host_injections') # CUDA itself doesn't care about compute capability so remove this duplication from # under host_injections - host_inj_path = re.sub(r"accel/nvidia/cc\d+/", '', host_inj_path) + accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") + if accel_subdir: + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From 5d66e4ce96d9e673d286dfd781f137c5e9037a15 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 16:00:32 +0200 Subject: [PATCH 1251/1795] Ensure we are making an EESSI install when using the CUDA hook --- eb_hooks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0a42a91d49..4d94357e8c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -685,7 +685,12 @@ def post_sanitycheck_cuda(self, *args, **kwargs): and replace them with a symlink to a corresponding installation under host_injections. """ # Make sure we only do this for CUDA and only if we are doing a CVMFS installation - if self.name == 'CUDA' and self.installdir.startswith('/cvmfs/software.eessi.io/versions'): + is_eessi_install = ( + self.installdir.startswith("/cvmfs/software.eessi.io/versions") + and not build_option("sanity_check_only") + and not build_option("module_only") + ) + if self.name == 'CUDA' and is_eessi_install: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped From ae55491b75e77b1f445654a77d06394040a370aa Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 10:54:02 +0200 Subject: [PATCH 1252/1795] Move CUDA hook to post-install, allow hook to trigger for any EESSI distributed repo (but always make symlinks to software.eessi.io) --- eb_hooks.py | 76 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4d94357e8c..788a0419ba 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -131,7 +131,8 @@ def pre_prepare_hook(self, *args, **kwargs): def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ Post-configure hook for GCCcore: - - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' + - copy RPATH wrapper script for linker commands to also have a wrapper in + place with system type prefix like 'x86_64-pc-linux-gnu' """ if self.name == 'GCCcore': config_guess = obtain_config_guess() @@ -279,10 +280,10 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. """ if ec.name == 'Qt5': - # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, - # see also https://github.com/easybuilders/easybuild-framework/pull/4290 - ec['check_qtwebengine'] = False - print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") + # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, + # see also https://github.com/easybuilders/easybuild-framework/pull/4290 + ec['check_qtwebengine'] = False + print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") else: raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") @@ -341,7 +342,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - # note: keep condition in sync with the one used in + # note: keep condition in sync with the one used in # post_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target in [CPU_TARGET_A64FX, CPU_TARGET_NEOVERSE_V1]: @@ -360,12 +361,13 @@ def post_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwar if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - # note: keep condition in sync with the one used in + # note: keep condition in sync with the one used in # pre_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target == CPU_TARGET_NEOVERSE_N1: update_build_option('optarch', self.orig_optarch) + def pre_configure_hook(self, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" if self.name in PRE_CONFIGURE_HOOKS: @@ -389,6 +391,7 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): else: raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") + def pre_configure_hook_extrae(self, *args, **kwargs): """ Pre-configure hook for Extrae @@ -414,7 +417,11 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # replace use of 'which' with 'command -v', since 'which' is broken in EESSI build container; # this must be done *after* running configure script, because initial configuration re-writes configure script, # and problem due to use of which only pops up when running make ?! - self.cfg.update('prebuildopts', "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && ") + self.cfg.update( + 'prebuildopts', + "cp config/mpi-macros.m4 config/mpi-macros.m4.orig &&" + "sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && " + ) else: raise EasyBuildError("Extrae-specific hook triggered for non-Extrae easyconfig?!") @@ -445,7 +452,10 @@ def pre_configure_hook_gromacs(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg.update('configopts', '-DGMX_SIMD=ARM_NEON_ASIMD') - print_msg("Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", self.version) + print_msg( + "Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", + self.version + ) else: raise EasyBuildError("GROMACS-specific hook triggered for non-GROMACS easyconfig?!") @@ -506,12 +516,12 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): pattern = "Linux x86_64 ppc64le, gfortran" repl = "Linux x86_64 aarch64 ppc64le, gfortran" if LooseVersion(self.version) <= LooseVersion('3.9.0'): - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'): - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") @@ -533,7 +543,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") -def pre_test_hook(self,*args, **kwargs): +def pre_test_hook(self, *args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: PRE_TEST_HOOKS[self.name](self, *args, **kwargs) @@ -596,6 +606,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): elif cpu_target == CPU_TARGET_A64FX and self.version in scipy_bundle_versions_a64fx: self.cfg['testopts'] = "|| echo ignoring failing tests" + def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): """ Pre-test hook for netCDF: skip failing tests for selected netCDF versions on neoverse_v1 @@ -609,6 +620,7 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" + def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): """ Pre-test hook for PyTorch: increase max failing tests for ARM for PyTorch 2.1.2 @@ -673,24 +685,24 @@ def pre_single_extension_testthat(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " -def post_sanitycheck_hook(self, *args, **kwargs): - """Main post-sanity-check hook: trigger custom functions based on software name.""" - if self.name in POST_SANITYCHECK_HOOKS: - POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) +def post_postproc_hook(self, *args, **kwargs): + """Main post-postprocessing hook: trigger custom functions based on software name.""" + if self.name in POST_POSTPROC_HOOKS: + POST_POSTPROC_HOOKS[self.name](self, *args, **kwargs) -def post_sanitycheck_cuda(self, *args, **kwargs): +def post_postproc_cuda(self, *args, **kwargs): """ Remove files from CUDA installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. """ - # Make sure we only do this for CUDA and only if we are doing a CVMFS installation - is_eessi_install = ( - self.installdir.startswith("/cvmfs/software.eessi.io/versions") - and not build_option("sanity_check_only") - and not build_option("module_only") - ) - if self.name == 'CUDA' and is_eessi_install: + + # We need to check if we are doing an EESSI-distributed installation + eessi_pattern = r"^/cvmfs/[^/]*.eessi.io/versions/" + host_injections_location = "/cvmfs/software.eessi.io/host_injections/" + eessi_installation = bool(re.search(eessi_pattern, self.installdir)) + + if self.name == 'CUDA' and eessi_installation: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped @@ -738,12 +750,14 @@ def post_sanitycheck_cuda(self, *args, **kwargs): self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", basename, full_path) # if it is not in the allowlist, delete the file and create a symlink to host_injections - host_inj_path = full_path.replace('versions', 'host_injections') + + # the host_injections path is under a fixed repo/location for CUDA + host_inj_path = re.sub(eessi_pattern, host_injections_location, full_path) # CUDA itself doesn't care about compute capability so remove this duplication from # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " @@ -775,7 +789,7 @@ def inject_gpu_property(ec): ec_dict['builddependencies'].append(dep) value = '\n'.join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) if key in ec_dict: - if not value in ec_dict[key]: + if value not in ec_dict[key]: ec[key] = '\n'.join([ec_dict[key], value]) else: ec[key] = value @@ -835,6 +849,6 @@ def inject_gpu_property(ec): 'numpy': post_single_extension_numpy, } -POST_SANITYCHECK_HOOKS = { - 'CUDA': post_sanitycheck_cuda, +POST_POSTPROC_HOOKS = { + 'CUDA': post_postproc_cuda, } From 7a95dd2b980b4e6796b3e7e1295b44417aa36d79 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 24 Sep 2024 16:08:37 +0200 Subject: [PATCH 1253/1795] Update eb_hooks.py --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 788a0419ba..a8182df971 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -757,7 +757,7 @@ def post_postproc_cuda(self, *args, **kwargs): # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') + host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From 64a0f378fb7bdf12537d99b9b45d0a9623f18575 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 11:14:29 +0200 Subject: [PATCH 1254/1795] Address review comments --- eb_hooks.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index a8182df971..9b0e9c8dcb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -32,6 +32,9 @@ SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] +EESSI_INSTALLATION_REGEX = r"^/cvmfs/[^/]*.eessi.io/versions/" +HOST_INJECTIONS_LOCATION = "/cvmfs/software.eessi.io/host_injections/" + def get_eessi_envvar(eessi_envvar): """Get an EESSI environment variable from the environment""" @@ -419,7 +422,7 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # and problem due to use of which only pops up when running make ?! self.cfg.update( 'prebuildopts', - "cp config/mpi-macros.m4 config/mpi-macros.m4.orig &&" + "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && " "sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && " ) else: @@ -698,9 +701,7 @@ def post_postproc_cuda(self, *args, **kwargs): """ # We need to check if we are doing an EESSI-distributed installation - eessi_pattern = r"^/cvmfs/[^/]*.eessi.io/versions/" - host_injections_location = "/cvmfs/software.eessi.io/host_injections/" - eessi_installation = bool(re.search(eessi_pattern, self.installdir)) + eessi_installation = bool(re.search(EESSI_INSTALLATION_REGEX, self.installdir)) if self.name == 'CUDA' and eessi_installation: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") @@ -752,9 +753,10 @@ def post_postproc_cuda(self, *args, **kwargs): # if it is not in the allowlist, delete the file and create a symlink to host_injections # the host_injections path is under a fixed repo/location for CUDA - host_inj_path = re.sub(eessi_pattern, host_injections_location, full_path) + host_inj_path = re.sub(EESSI_INSTALLATION_REGEX, HOST_INJECTIONS_LOCATION, full_path) # CUDA itself doesn't care about compute capability so remove this duplication from - # under host_injections + # under host_injections (symlink to a single CUDA installation for all compute + # capabilities) accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') From a5ca128e550c4afaac678fabcc20c843e31a59f3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 25 Sep 2024 17:05:41 +0200 Subject: [PATCH 1255/1795] Add ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..d9429ad8a8 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,5 @@ +easyconfigs: + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From 730869a1a336514246c9c4e73c075247356a108b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 26 Sep 2024 14:27:46 +0200 Subject: [PATCH 1256/1795] Move CUDA builds to accel/ subdir --- .../eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/{2023.06/eessi-2023.06-eb-4.9.3-2023a.yml => accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml rename to easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From d0259dcdcff0c451c61683323b8e75feb9ed4ff0 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 26 Sep 2024 14:43:44 +0200 Subject: [PATCH 1257/1795] Fix EasyBuild PR link --- .../accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml index d9429ad8a8..f41bc986ec 100644 --- a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -1,5 +1,5 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From c2cdf3b7867ff9af0ab77fae7de39c58336fd8c2 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 18 Sep 2024 12:45:55 +0200 Subject: [PATCH 1258/1795] {2023.06}[2023a,GPU] LAMMPS 2Aug2023 CUDA 12.1.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml new file mode 100644 index 0000000000..91c4b1f472 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb From ea6fdec621655d8a689e3940d4eb98e5160e8973 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:29:31 +0200 Subject: [PATCH 1259/1795] UCX cuda 1.14.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..bfe4a4dc52 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb From 1e19767896d3b58d5d4f36b73ea990c54b50446e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:34:39 +0200 Subject: [PATCH 1260/1795] Move to rebuilds --- .../20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml From 8b3a3cde4a2ee51f044a7c7cd4999b80b871d5ac Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:49:58 +0200 Subject: [PATCH 1261/1795] Add comment --- .../rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml index bfe4a4dc52..d347af335a 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall UCX-CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/719 easyconfigs: - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb From 25475f789c3d40b25f389f58cfcfd46761094b15 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:05:17 +0200 Subject: [PATCH 1262/1795] Add CUDA-Samples for the accelerator prefix --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..01a47bbc99 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb From f7f2e26fa0f05f7bd3fe9bca137474456fd41433 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:31:39 +0200 Subject: [PATCH 1263/1795] Move to rebuilds --- .../20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml From c30842df015a8f6217ab694f982778dce1a0d03e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:49:20 +0200 Subject: [PATCH 1264/1795] Add comment --- .../20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml index 01a47bbc99..da2c06ae1e 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall CUDA-Samples in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/715 easyconfigs: - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb From 1d91f4836b848e97416cd6ba0bd878501e8062e1 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 20 Sep 2024 08:23:46 +0200 Subject: [PATCH 1265/1795] {2023.06}[foss/2023a] NCCL v2.18.3 w/ CUDA 12.1.1 --- .../20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml new file mode 100644 index 0000000000..6ff3b1bfbf --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml @@ -0,0 +1,5 @@ +# 2024.09.19 +# We need to reinstall NCCL in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/487 +easyconfigs: + - NCCL-2.18.3-GCCcore-12.3.0-CUDA-12.1.1.eb From b09b90fb8b53b452e27945c34322328aa8f3776a Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 10:05:21 +0200 Subject: [PATCH 1266/1795] {2023.06}[foss/2023a] NCCL 2.18.3 w/ CUDA 12.1.1 --- ...ix.yml => 20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename easystacks/software.eessi.io/2023.06/rebuilds/{20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml => 20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml} (92%) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml similarity index 92% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml index 6ff3b1bfbf..d6667af9a1 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml @@ -1,4 +1,4 @@ -# 2024.09.19 +# 2024.09.25 # We need to reinstall NCCL in the accelerator prefixes # See https://github.com/EESSI/software-layer/pull/487 easyconfigs: From 6c63de6b8913d76b9f0039df772d77da1c6b1bee Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 22:04:54 +0200 Subject: [PATCH 1267/1795] UCC CUDA rebuild now that we have an accel prefix --- .../rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml new file mode 100644 index 0000000000..a418086c44 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml @@ -0,0 +1,4 @@ +# 2024.09.19 +# We need to reinstall UCC-CUDA in the accelerator prefixes +easyconfigs: + - UCC-CUDA-1.2.0-GCCcore-12.3.0-CUDA-12.1.1.eb From 9182b21d95d67a1f5604378d2d50d3bfba990057 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Sep 2024 16:12:22 +0200 Subject: [PATCH 1268/1795] Add various tools --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..d3927ea3f1 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - ccache-4.9-GCCcore-12.3.0.eb + - GDB-13.2-GCCcore-12.3.0.eb + - mold-1.11.0-GCCcore-12.3.0.eb + - tmux-3.3a-GCCcore-12.3.0.eb + - Vim-9.1.0004-GCCcore-12.3.0.eb From e181237f32b9c11a0cd5081703c8e31bb8f7f4ca Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 11:10:51 +0200 Subject: [PATCH 1269/1795] Remove mold, it's a complicated case since it is a linker, and it's not urgent now --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml index d3927ea3f1..0a4a4e5554 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,6 +1,5 @@ easyconfigs: - ccache-4.9-GCCcore-12.3.0.eb - GDB-13.2-GCCcore-12.3.0.eb - - mold-1.11.0-GCCcore-12.3.0.eb - tmux-3.3a-GCCcore-12.3.0.eb - Vim-9.1.0004-GCCcore-12.3.0.eb From c906e0c1e2e97970fb207ec7d17d598aa0c22a9e Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 26 Sep 2024 15:10:52 +0200 Subject: [PATCH 1270/1795] use subdir accel for CUDA builds --- .../eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023-CUDA.yml => accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml rename to easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From 7d59905789554b13f464ba452edba5c330ac5bf0 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 26 Sep 2024 15:47:36 +0200 Subject: [PATCH 1271/1795] create nvidia directory --- .../accel/{ => nvidia}/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/accel/{ => nvidia}/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml (100%) diff --git a/easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From 31b37d609eaa50914868cd0bd29abb5def416b22 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:10:44 +0200 Subject: [PATCH 1272/1795] Add OSU Microbenchmarks 7.2 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..cccbfa6808 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb From 0f159b4bfdca7bfd43584a059fb321aca02e3613 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:44:42 +0200 Subject: [PATCH 1273/1795] Move into rebuilds --- .../20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml From 5d85411d63199b3de4a21bfa57879294404cdcec Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:45:44 +0200 Subject: [PATCH 1274/1795] Add comment --- .../20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml index cccbfa6808..23801e0250 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall OSU-Micro-Benchmarks in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/716 easyconfigs: - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb From 925b48e7e5e84b2b39258659d050ceb6776a0b75 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Sep 2024 20:40:04 +0200 Subject: [PATCH 1275/1795] use easystack file in accel/nvidia for ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb --- .../accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 4 ++++ .../accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml index 91c4b1f472..8935a3f3c3 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -1,2 +1,6 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c diff --git a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml deleted file mode 100644 index f41bc986ec..0000000000 --- a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ /dev/null @@ -1,5 +0,0 @@ -easyconfigs: - - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 - from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From 46ee666d435d4e2e50d6ecbf5e463a13bdca6342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 10:28:57 +0200 Subject: [PATCH 1276/1795] add mapping for cpu->accelerators and check for missing accelerator builds --- .github/workflows/test-software.eessi.io.yml | 36 +++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index d4d980901f..c4e3286f6c 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -2,11 +2,25 @@ name: Check for missing software installations in software.eessi.io on: push: - branches: [ "*-software.eessi.io" ] +# branches: [ "*-software.eessi.io" ] + branches: "*" pull_request: workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) +env: + EESSI_ACCELERATOR_TARGETS: | + aarch64/generic: + aarch64/neoverse_n1: + aarch64/neoverse_v1: + x86_64/generic: + x86_64/amd/zen2: + - nvidia/cc80 + x86_64/amd/zen3: + - nvidia/cc80 + x86_64/amd/zen4: + x86_64/intel/haswell: + x86_64/intel/skylake_avx512: jobs: check_missing: runs-on: ubuntu-latest @@ -48,6 +62,8 @@ jobs: export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux env | grep ^EESSI | sort + + # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." @@ -56,6 +72,24 @@ jobs: if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi done + # now check the accelerator builds for this CPU target + accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]") + if [ -z ${accelerators} ]; then + echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" + else + for accel in ${accelerators}; do + module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/nvidia/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "check missing installations for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + done + module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + done + fi + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} From bd0edd18d33ed4e1ea21f9c09fe7cb07546e30af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:01:11 +0200 Subject: [PATCH 1277/1795] remove entries without an accelerator --- .github/workflows/test-software.eessi.io.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index c4e3286f6c..bad392aef8 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -10,17 +10,10 @@ permissions: contents: read # to fetch code (actions/checkout) env: EESSI_ACCELERATOR_TARGETS: | - aarch64/generic: - aarch64/neoverse_n1: - aarch64/neoverse_v1: - x86_64/generic: x86_64/amd/zen2: - nvidia/cc80 x86_64/amd/zen3: - nvidia/cc80 - x86_64/amd/zen4: - x86_64/intel/haswell: - x86_64/intel/skylake_avx512: jobs: check_missing: runs-on: ubuntu-latest From af7dfc31b8f950df11d35d2ec964e8c68f1e393d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:07:35 +0200 Subject: [PATCH 1278/1795] replace hardcoded nvidia in easystack dir by "dirname $accel" --- .github/workflows/test-software.eessi.io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index bad392aef8..48df5f9ca3 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -73,7 +73,7 @@ jobs: for accel in ${accelerators}; do module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" - for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/nvidia/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/$(dirname ${accel})/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 645ea271a4f4781ed9b2781964fb1a3b8d6b08b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:27:08 +0200 Subject: [PATCH 1279/1795] only run on *-software.eessi.io branches again --- .github/workflows/test-software.eessi.io.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 48df5f9ca3..e247c96676 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -2,8 +2,7 @@ name: Check for missing software installations in software.eessi.io on: push: -# branches: [ "*-software.eessi.io" ] - branches: "*" + branches: [ "*-software.eessi.io" ] pull_request: workflow_dispatch: permissions: From c81857074772b906890b9bee76b25f478b241a6c Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 15:08:15 +0200 Subject: [PATCH 1280/1795] {2023.06}[foss/2023a,zen4] SAMtools v1.18 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 4bae944d45..21b1886555 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -11,3 +11,4 @@ easyconfigs: - LoopTools-2.15-GCC-12.3.0.eb - ncdu-1.18-GCC-12.3.0.eb - WhatsHap-2.2-foss-2023a.eb + - SAMtools-1.18-GCC-12.3.0.eb From f04d976048b00126d23933dbd3bd1d32203b4d25 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 15:12:40 +0200 Subject: [PATCH 1281/1795] Use same return code in any scenario where we use default list of CUDA libraries --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 0afc7b6b86..8ecd76e780 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -97,7 +97,7 @@ get_nvlib_list() { # Check if the function was called with the "default" argument if [[ "$1" == "default" ]]; then printf "%s\n" "${default_nvlib_list[@]}" - return 0 + return 1 fi # Try to download the nvliblist.conf file with curl @@ -112,6 +112,8 @@ get_nvlib_list() { # If curl succeeded, filter and return the libraries from the downloaded content echo "$nvliblist_content" | grep '.so$' + + return 0 } # Function to check if umask allows global read From 9c2865f9be7c002dde888006675cf7db5dd246b9 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 15:23:37 +0200 Subject: [PATCH 1282/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 8ecd76e780..6a42bcdd3c 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -86,7 +86,7 @@ get_nvlib_list() { "libnvidia-rtcore.so" "libnvidia-tls.so" "libnvidia-wfb.so" - "libnvoptix.so.1" + "libnvoptix.so" "libOpenCL.so" "libOpenGL.so" "libvdpau_nvidia.so" From 0e19946274fbedce25bd7b05d6ee627daccff410 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 15:30:20 +0200 Subject: [PATCH 1283/1795] update eb_hook so that EasyBuildError is not not trigered for unlisted LAMMPS versions --- eb_hooks.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index d999c735c3..f7d19235c4 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -301,15 +301,16 @@ def parse_hook_lammps_remove_deps_for_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64 """ - if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2', '29Aug2024'): - if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets, - # so remove them as dependencies for LAMMPS (they're optional); - # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/19164 + - # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000; - # we need this hook because we check for missing installations for all CPU targets - # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) - ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb',)] + if ec.name == 'LAMMPS': + if ec.version in ('2Aug2023_update2', '29Aug2024'): + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': + # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets, + # so remove them as dependencies for LAMMPS (they're optional); + # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/19164 + + # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000; + # we need this hook because we check for missing installations for all CPU targets + # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) + ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb',)] else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") From a263d6cb714aafd0ef38959fcc1ef63ee50d8db6 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 15:40:19 +0200 Subject: [PATCH 1284/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 6a42bcdd3c..ee42c81b65 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -175,8 +175,8 @@ host_ldconfig=$(get_host_ldconfig) host_libraries=$($host_ldconfig -p | awk '{print $NF}') singularity_libs=$(ls /.singularity.d/libs/* 2>/dev/null) -# Now gather the list of possible CUDA libraries -cuda_candidate_libraries=$(get_nvlib_list "${LIBS_LIST}") +# Now gather the list of possible CUDA libraries and make them into an array +cuda_candidate_libraries=($(get_nvlib_list "${LIBS_LIST}")) # Check if the function returned an error (e.g., curl failed) if [ $? -ne 0 ]; then echo "Using default list of libraries" @@ -191,7 +191,7 @@ matched_libraries=() # Process each library and check for matches in libs.txt for library in "${cuda_candidate_libraries[@]}"; do # Search for the library in libs.txt and add it to the matched_libraries array - matched=$(echo "$ldconfig_output $singularity_libs" | grep "$library") + matched=$(echo "$host_libraries $singularity_libs" | grep "$library") if [ -n "$matched" ]; then matched_libraries+=("$matched") # Add matched library to the array fi From 9f946424fd7e32cd390e2d7d9b4645a82ceef2ef Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 15:42:37 +0200 Subject: [PATCH 1285/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index ee42c81b65..4e7343d351 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -214,6 +214,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then echo "Allowing overriding GPU checks in EESSI via EESSI_OVERRIDE_GPU_CHECK" else echo "No libraries matched, LD_PRELOAD not set." + fi exit 0 fi From 71e8898554d937fcc52d285d5cc371e9283df2a7 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 15:44:58 +0200 Subject: [PATCH 1286/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 4e7343d351..c18e655110 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -197,9 +197,8 @@ for library in "${cuda_candidate_libraries[@]}"; do fi done -# Output the matched libraries -echo "Matched CUDA Libraries:" -printf "%s\n" "${matched_libraries[@]}" +# Output the number of matched libraries +echo "Matched ${#matched_libraries[@]} CUDA Libraries" # LD_PRELOAD Mode if [ "$LD_PRELOAD_MODE" -eq 1 ]; then From ed9e868e865c4e43b058465fc101df02f16a10c0 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 15:49:00 +0200 Subject: [PATCH 1287/1795] This script might be `source`d, don't use exit --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index c18e655110..ce3b37cbc5 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -133,7 +133,7 @@ check_global_read() { } # Check for required commands -command -v nvidia-smi >/dev/null 2>&1 || { echo_yellow "nvidia-smi not found, this script won't do anything useful"; exit 1; } +command -v nvidia-smi >/dev/null 2>&1 || { echo_yellow "nvidia-smi not found, this script won't do anything useful"; return 1; } # Variables LD_PRELOAD_MODE=0 @@ -214,7 +214,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then else echo "No libraries matched, LD_PRELOAD not set." fi - exit 0 + return 0 fi # If we haven't already exited, we may need to create the symlinks From d8d335e72643a13d782931b2c6d7cd9a08a809ec Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 16:06:20 +0200 Subject: [PATCH 1288/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index ce3b37cbc5..ec911c6e9a 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -214,7 +214,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then else echo "No libraries matched, LD_PRELOAD not set." fi - return 0 + [[ "${BASH_SOURCE[0]}" != "${0}" ]] && return 1 || exit 1 fi # If we haven't already exited, we may need to create the symlinks From aa675ab20690704a69750a1ac18216f7921dffd6 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 16:09:00 +0200 Subject: [PATCH 1289/1795] add missing check for zen4 --- .github/workflows/test-software.eessi.io.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index e247c96676..e774180d99 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -27,6 +27,7 @@ jobs: - aarch64/neoverse_v1 - x86_64/amd/zen2 - x86_64/amd/zen3 + - x86_64/amd/zen4 - x86_64/intel/haswell - x86_64/intel/skylake_avx512 - x86_64/generic @@ -58,6 +59,9 @@ jobs: # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ] && grep -q 2022b <<<"${easystack_file}"; then + continue + fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 2d5c96b8f80408052937ba8ff90835f5ba61ef9e Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 17:26:45 +0200 Subject: [PATCH 1290/1795] Exclude anything graphics related from an LD_PRELOAD approach --- .../nvidia/link_nvidia_host_libraries.sh | 69 ++++++++++++++++--- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index ec911c6e9a..8e5a7ddc32 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -202,17 +202,68 @@ echo "Matched ${#matched_libraries[@]} CUDA Libraries" # LD_PRELOAD Mode if [ "$LD_PRELOAD_MODE" -eq 1 ]; then - # Set LD_PRELOAD with the matched libraries - if [ ${#matched_libraries[@]} -gt 0 ]; then - LD_PRELOAD=$(printf "%s\n" "${matched_libraries[@]}" | tr '\n' ':') + echo + echo_yellow "When attempting to use LD_PRELOAD we exclude anything related to graphics" + + # Filter out all libraries that have missing library dependencies under EESSI + filtered_libraries=() + for library in "${matched_libraries[@]}"; do + # Run ldd on the given binary and filter for "not found" libraries + NOT_FOUND_LIBS=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') + # Check if it is missing an so dep under EESSI + if [[ -z "$NOT_FOUND_LIBS" ]]; then + # Anything graphics is out, as is libnvidia-fbc* + if [[ "$library" != *"GL"* ]]; then + if [[ "$library" != *"libnvidia-fbc"* ]]; then + filtered_libraries+=("$library") + fi + fi + else + # Iterate over "not found" libraries and check if they are in the array + all_found=true + for lib in $NOT_FOUND_LIBS; do + found=false + for listed_lib in "${matched_libraries[@]}"; do + if [[ "$lib" == "$listed_lib" ]]; then + found=true + break + fi + done + + if [[ "$found" == false ]]; then + echo "$lib is NOT in the provided preload list, filtering $library." + all_found=false + break + fi + done + + # If we find all the missing libs in our list include it + if [[ "$all_found" == true ]]; then + # Anything graphics is out, as is libnvidia-fbc* + if [[ "$library" != *"GL"* ]]; then + if [[ "$library" != *"libnvidia-fbc"* ]]; then + filtered_libraries+=("$library") + fi + fi + fi + fi + done + + # Set EESSI_GPU_LD_PRELOAD with the matched libraries + if [ ${#filtered_libraries[@]} -gt 0 ]; then + echo + echo_yellow "The recommended way to use LD_PRELOAD is to only use it when you need to:" + echo + EESSI_GPU_LD_PRELOAD=$(printf "%s\n" "${filtered_libraries[@]}" | tr '\n' ':') # Remove the trailing colon from LD_PRELOAD if it exists - LD_PRELOAD=${LD_PRELOAD%:} - export LD_PRELOAD - echo "LD_PRELOAD set to: $LD_PRELOAD" + EESSI_GPU_LD_PRELOAD=${EESSI_GPU_LD_PRELOAD%:} + export EESSI_GPU_LD_PRELOAD + echo_green "export EESSI_GPU_LD_PRELOAD=\"$EESSI_GPU_LD_PRELOAD\"" export EESSI_OVERRIDE_GPU_CHECK=1 - echo "Allowing overriding GPU checks in EESSI via EESSI_OVERRIDE_GPU_CHECK" - else - echo "No libraries matched, LD_PRELOAD not set." + echo_green "export EESSI_OVERRIDE_GPU_CHECK=\"$EESSI_OVERRIDE_GPU_CHECK\"" + echo + echo_yellow "Then you can set LD_PRELOAD only when you want to run a GPU application, e.g.," + echo_yellow " LD_PRELOAD=\"\$EESSI_GPU_LD_PRELOAD\" device_query" fi [[ "${BASH_SOURCE[0]}" != "${0}" ]] && return 1 || exit 1 fi From 1a325f2823fdb6bfc5208b7dee5194b428f35f99 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 17:31:22 +0200 Subject: [PATCH 1291/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 8e5a7ddc32..a356e0e3a5 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -264,6 +264,8 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then echo echo_yellow "Then you can set LD_PRELOAD only when you want to run a GPU application, e.g.," echo_yellow " LD_PRELOAD=\"\$EESSI_GPU_LD_PRELOAD\" device_query" + else + echo "No libraries matched, LD_PRELOAD not set." fi [[ "${BASH_SOURCE[0]}" != "${0}" ]] && return 1 || exit 1 fi From 000e6c1cc26002aacea88e12de9d7bcda6c93cb0 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 27 Sep 2024 17:59:33 +0200 Subject: [PATCH 1292/1795] Update link_nvidia_host_libraries.sh --- .../nvidia/link_nvidia_host_libraries.sh | 31 +++++++------------ 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index a356e0e3a5..3de39e33a4 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -120,15 +120,13 @@ get_nvlib_list() { check_global_read() { # Get the current umask value local current_umask=$(umask) - + # Convert umask to decimal to analyze - local umask_decimal=$((8#$current_umask)) + local umask_octal=$(printf '%03o\n' "$current_umask") # Check if umask allows global read - if [[ $umask_decimal -eq 0 || $umask_decimal -eq 22 ]]; then - echo "The current umask ($current_umask) allows global read permissions." - else - fatal_error "The current umask ($current_umask) does not allow global read permissions." + if [ "$umask_octal" -gt 022 ]; then + fatal_error "The current umask ($current_umask) does not allow global read permissions, you'll want everyone to be able to read the created directory." fi } @@ -157,7 +155,7 @@ nvidia_smi_driver_command="nvidia-smi --query-gpu=driver_version --format=csv,no if $nvidia_smi_driver_command > /dev/null 2>&1; then host_driver_version=$($nvidia_smi_driver_command | tail -n1) echo_green "Found NVIDIA GPU driver version ${host_driver_version}" - + # If the first worked, this should work too host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk '{NF>1; print $NF}') echo_green "Found host CUDA version ${host_cuda_version}" @@ -193,7 +191,7 @@ for library in "${cuda_candidate_libraries[@]}"; do # Search for the library in libs.txt and add it to the matched_libraries array matched=$(echo "$host_libraries $singularity_libs" | grep "$library") if [ -n "$matched" ]; then - matched_libraries+=("$matched") # Add matched library to the array + matched_libraries+=( $matched ) # Add matched library to the array fi done @@ -307,18 +305,11 @@ if [ "$link_drivers" -eq 1 ]; then # Make symlinks to all the interesting libraries # Loop over each matched library for library in "${matched_libraries[@]}"; do - # Check if the library file exists - if [ -e "$library" ]; then - # Create a symlink in the current directory - ln -s "$library" . - # Check if the symlink was created successfully - if [ $? -eq 0 ]; then - echo "Successfully created symlink for library $library in $PWD" - else - fatal_error "Error: Failed to create symlink for library $library in $PWD" - fi - else - echo "Warning: Library not found: $library" + # Create a symlink in the current directory + ln -s "$library" . + # Check if the symlink was created successfully + if [ $? -ne 0 ]; then + fatal_error "Error: Failed to create symlink for library $library in $PWD" fi done From c30dca94917e5d6c064373cbec44fc510def06c6 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 27 Sep 2024 18:01:14 +0200 Subject: [PATCH 1293/1795] Use $EESSI_COMPAT_LAYER_DIR_OVERRIDE --- run_in_compat_layer_env.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 744e208ae0..13956f1bf7 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -7,7 +7,13 @@ if [ -z $EESSI_VERSION ]; then echo "ERROR: \$EESSI_VERSION must be set!" >&2 exit 1 fi -EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" + +if [ -z ${EESSI_COMPAT_LAYER_DIR_OVERRIDE} ]; then + EESSI_COMPAT_LAYER_DIR=${EESSI_COMPAT_LAYER_DIR_OVERRIDE} +else + EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" +fi + if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2 exit 1 From cf676dedbaec568a9bf09d278d92beb17822cf1f Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 27 Sep 2024 18:21:35 +0200 Subject: [PATCH 1294/1795] Add log message for compat layer override --- run_in_compat_layer_env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 13956f1bf7..ecc0610dff 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -9,6 +9,7 @@ if [ -z $EESSI_VERSION ]; then fi if [ -z ${EESSI_COMPAT_LAYER_DIR_OVERRIDE} ]; then + echo "EESSI_COMPAT_LAYER_DIR_OVERRIDE found. Setting EESSI_COMPAT_LAYER_DIR to ${EESSI_COMPAT_LAYER_DIR_OVERRIDE}" EESSI_COMPAT_LAYER_DIR=${EESSI_COMPAT_LAYER_DIR_OVERRIDE} else EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)" From aa5fade441cf052a65fe197f56d4da0b72279668 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 27 Sep 2024 18:24:56 +0200 Subject: [PATCH 1295/1795] set kokkos hoop for zen4 Zen4 nodes in Azure are not detected as `zen4` by archspec --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index f7d19235c4..8a357a8cde 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -525,7 +525,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'LAMMPS': - if self.version == '2Aug2023_update2': + if ec.version in ('2Aug2023_update2', '29Aug2024'): if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 From abbbf35db63271508a60dd003c2cfe94b2f691ce Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 27 Sep 2024 19:32:17 +0200 Subject: [PATCH 1296/1795] fix hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8a357a8cde..4d13523e40 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -525,7 +525,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'LAMMPS': - if ec.version in ('2Aug2023_update2', '29Aug2024'): + if self.version in ('2Aug2023_update2', '29Aug2024'): if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 From 748300c7cdb9df39bf75b01d4265b388f8a92246 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 19:43:54 +0200 Subject: [PATCH 1297/1795] {2023.06}[system,zen4] EasyBuild v4.9.0 and v4.8.2 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml index 1e30631e57..25337649ce 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml @@ -1,2 +1,4 @@ easyconfigs: - Nextflow-23.10.0.eb + - EasyBuild-4.8.2.eb + - EasyBuild-4.9.0.eb From 2fe88f178a6b5da3d48ece850f17499e541c6a57 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 27 Sep 2024 20:11:46 +0200 Subject: [PATCH 1298/1795] Add debug output --- run_in_compat_layer_env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index ecc0610dff..7907204abf 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -8,6 +8,8 @@ if [ -z $EESSI_VERSION ]; then exit 1 fi +echo "EESSI_COMPAT_LAYER_DIR_OVERRIDE: ${EESSI_COMPAT_LAYER_DIR_OVERRIDE}" + if [ -z ${EESSI_COMPAT_LAYER_DIR_OVERRIDE} ]; then echo "EESSI_COMPAT_LAYER_DIR_OVERRIDE found. Setting EESSI_COMPAT_LAYER_DIR to ${EESSI_COMPAT_LAYER_DIR_OVERRIDE}" EESSI_COMPAT_LAYER_DIR=${EESSI_COMPAT_LAYER_DIR_OVERRIDE} From 6a3e5492d2824af35baea53b9ecae90daeb410fe Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 27 Sep 2024 21:00:37 +0200 Subject: [PATCH 1299/1795] Fix check if compat layer override is set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I was checking that it is *not* set 🤦 --- run_in_compat_layer_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 7907204abf..ba9d84bee4 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -10,7 +10,7 @@ fi echo "EESSI_COMPAT_LAYER_DIR_OVERRIDE: ${EESSI_COMPAT_LAYER_DIR_OVERRIDE}" -if [ -z ${EESSI_COMPAT_LAYER_DIR_OVERRIDE} ]; then +if [ ! -z ${EESSI_COMPAT_LAYER_DIR_OVERRIDE} ]; then echo "EESSI_COMPAT_LAYER_DIR_OVERRIDE found. Setting EESSI_COMPAT_LAYER_DIR to ${EESSI_COMPAT_LAYER_DIR_OVERRIDE}" EESSI_COMPAT_LAYER_DIR=${EESSI_COMPAT_LAYER_DIR_OVERRIDE} else From 64b7958071fe672499e0b4f571be0805cac80ea7 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 27 Sep 2024 21:26:03 +0200 Subject: [PATCH 1300/1795] exclude CUDA builds in CPU path --- .github/workflows/test-software.eessi.io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index e774180d99..39972ef18f 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -59,7 +59,7 @@ jobs: # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do - if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ] && grep -q 2022b <<<"${easystack_file}"; then + if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ] && grep -q 2022b <<<"${easystack_file}" || grep -q CUDA <<<"${easystack_file}"; then continue fi echo "check missing installations for ${easystack_file}..." From ede35425aa97cc898997c939b1f0036cc0b8797d Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 27 Sep 2024 21:43:46 +0200 Subject: [PATCH 1301/1795] fix conditional for CUDA --- .github/workflows/test-software.eessi.io.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 39972ef18f..29af8f57f1 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -59,8 +59,12 @@ jobs: # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do - if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ] && grep -q 2022b <<<"${easystack_file}" || grep -q CUDA <<<"${easystack_file}"; then - continue + if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then + if grep -q 2022b <<<"${easystack_file}"; then + continue + elif grep -q CUDA <<<"${easystack_file}"; then + continue + fi fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} From 33f5f4cb6112771c96a3220ad18fa2b66e9e59e0 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 21:50:54 +0200 Subject: [PATCH 1302/1795] {2023.06}[2023a,zen4] OSU-Micro-Benchmarks v7.1-1 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 48ac47cbfb..4153b4a5e4 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -13,3 +13,4 @@ easyconfigs: - WhatsHap-2.2-foss-2023a.eb - PyOpenGL-3.1.7-GCCcore-12.3.0.eb - SAMtools-1.18-GCC-12.3.0.eb + - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb From 60f0d10f0931e0dfe0ac4595f586c76614d4a71b Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 21:56:40 +0200 Subject: [PATCH 1303/1795] {2023.06}[gompi/2023a] CDO 2.2.2 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 48ac47cbfb..d787241397 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -13,3 +13,4 @@ easyconfigs: - WhatsHap-2.2-foss-2023a.eb - PyOpenGL-3.1.7-GCCcore-12.3.0.eb - SAMtools-1.18-GCC-12.3.0.eb + - CDO-2.2.2-gompi-2023a.eb From 3e5fefb1b33af1574f09b9146f71c7bae57d7c61 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 23:22:22 +0200 Subject: [PATCH 1304/1795] {2023.06}{GCCcore/12.3.0,zen4} BWA 0.7.17.20220923 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 7b93c754c3..730594a22b 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -15,3 +15,4 @@ easyconfigs: - SAMtools-1.18-GCC-12.3.0.eb - CDO-2.2.2-gompi-2023a.eb - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb + - BWA-0.7.17-20220923-GCCcore-12.3.0.eb From a0712df5ba08bc84f285871e22b9f71dc730d31d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Sep 2024 09:01:02 +0200 Subject: [PATCH 1305/1795] move CUDA installations in CPU-only paths to dedicated easyconfigs --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml | 9 +++++++++ .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 8 -------- .../2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml | 2 ++ .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 - 4 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml new file mode 100644 index 0000000000..f8bde420de --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml @@ -0,0 +1,9 @@ +easyconfigs: + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb: + # use easyconfig that only install subset of CUDA samples, + # to circumvent problem with nvcc linking to glibc of host OS, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189; + # and where additional samples are excluded because they fail to build on aarch64, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19451; + options: + from-pr: 19451 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 7244219dc3..43b081b122 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -35,14 +35,6 @@ easyconfigs: - Boost-1.82.0-GCC-12.3.0.eb - netCDF-4.9.2-gompi-2023a.eb - FFmpeg-6.0-GCCcore-12.3.0.eb - - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb: - # use easyconfig that only install subset of CUDA samples, - # to circumvent problem with nvcc linking to glibc of host OS, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189; - # and where additional samples are excluded because they fail to build on aarch64, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19451; - options: - from-pr: 19451 - ALL-0.9.2-foss-2023a.eb: options: from-pr: 19455 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml new file mode 100644 index 0000000000..cccbfa6808 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 4b58cb6106..3f6590c3cd 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -34,7 +34,6 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996 from-pr: 19996 - dask-2023.9.2-foss-2023a.eb - - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb: options: From 7234585d806171868e7f7304f034599d952978c4 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 30 Sep 2024 07:10:44 +0000 Subject: [PATCH 1306/1795] {2023.06}[GCCcore/12.2.0] libs-tools --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 0f866b3aa0..ca35e95bd4 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -35,3 +35,9 @@ easyconfigs: - GL2PS-1.4.2-GCCcore-12.2.0.eb - GST-plugins-base-1.22.1-GCC-12.2.0.eb - wxWidgets-3.2.2.1-GCC-12.2.0.eb + - Archive-Zip-1.68-GCCcore-12.2.0.eb + - jemalloc-5.3.0-GCCcore-12.2.0.eb + - Judy-1.0.5-GCCcore-12.2.0.eb + - libaio-0.3.113-GCCcore-12.2.0.eb + - Z3-4.12.2-GCCcore-12.2.0.eb + - tbb-2021.10.0-GCCcore-12.2.0.eb From a9a9c3925a36443254ea3b82c069b74ca68c495a Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 30 Sep 2024 10:22:27 +0200 Subject: [PATCH 1307/1795] {2023.06}[2023a,2023b,zen4] Valgrind v3.21.0 and v3.23.0 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 730594a22b..519d7701dc 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -16,3 +16,4 @@ easyconfigs: - CDO-2.2.2-gompi-2023a.eb - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb - BWA-0.7.17-20220923-GCCcore-12.3.0.eb + - Valgrind-3.21.0-gompi-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml new file mode 100644 index 0000000000..c33ed99240 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - Valgrind-3.23.0-gompi-2023b.eb From ecd8853f0f5a3ab5138c4211f606b1922fadf28c Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 30 Sep 2024 10:58:34 +0200 Subject: [PATCH 1308/1795] remove because Valgrind on 2023b is already build --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml deleted file mode 100644 index c33ed99240..0000000000 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - Valgrind-3.23.0-gompi-2023b.eb From e3682e61f136215486016389b73dfd0570621c07 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:53:59 +0200 Subject: [PATCH 1309/1795] add comments to CI --- .github/workflows/test-software.eessi.io.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 29af8f57f1..6f592cf4c4 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -61,8 +61,10 @@ jobs: for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then if grep -q 2022b <<<"${easystack_file}"; then + # skip the check of installed software on zen4 for foss/2022b builds continue elif grep -q CUDA <<<"${easystack_file}"; then + # skip the check of install CUDA software in the CPU path for zen4 continue fi fi From 8b2582929103996c8567828c5e3aa5fdd8a2b817 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:16:34 +0200 Subject: [PATCH 1310/1795] Also use software.eessi.io in the container We need it for the compat layer --- bot/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index ea6459ea92..aee70d1812 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -169,6 +169,12 @@ COMMON_ARGS+=("--mode" "run") [[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") [[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") +# Also expose software.eessi.io when configured for dev.eessi.io +# Need software.eessi.io for the compat layer +if [[ "${REPOSITORY}" == dev.eessi.io ]] then + COMMON_ARGS+=("--repository" "software.eessi.io,access=ro") +fi + # make sure to use the same parent dir for storing tarballs of tmp PREVIOUS_TMP_DIR=${PWD}/previous_tmp From 1b7497530377b65e686ce691aa98dc3d50061624 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:28:57 +0200 Subject: [PATCH 1311/1795] Fix syntax --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index aee70d1812..237d5574f2 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -171,7 +171,7 @@ COMMON_ARGS+=("--mode" "run") # Also expose software.eessi.io when configured for dev.eessi.io # Need software.eessi.io for the compat layer -if [[ "${REPOSITORY}" == dev.eessi.io ]] then +if [[ "${REPOSITORY}" == dev.eessi.io ]]; then COMMON_ARGS+=("--repository" "software.eessi.io,access=ro") fi From 6f726328424fce66d65ec025d4191aa4de6afa5d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Sep 2024 16:54:19 +0200 Subject: [PATCH 1312/1795] disable using x86_64/amd/zen3 installations when x86_64/amd/zen4 is detected --- init/eessi_environment_variables | 10 ---------- init/modules/EESSI/2023.06.lua | 9 --------- 2 files changed, 19 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 8c10b1fca8..1f16204b36 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -48,16 +48,6 @@ if [ -d $EESSI_PREFIX ]; then fi if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then - # use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, - # since optimized software installations for Zen4 are a work-in-progress, - # see https://gitlab.com/eessi/support/-/issues/37 - if [[ "${EESSI_SOFTWARE_SUBDIR}" == "x86_64/amd/zen4" ]]; then - if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR="x86_64/amd/zen3" - echo -e "\e[33mSticking to ${EESSI_SOFTWARE_SUBDIR} for now, since optimized installations for AMD Genoa (Zen4) are a work in progress, see https://gitlab.com/eessi/support/-/issues/37 for more information\e[0m" - fi - fi - show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 32aaf6c07f..e45eb6fa5f 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -28,15 +28,6 @@ function archdetect_cpu() local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or "" for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then - -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, - -- since optimized software installations for Zen4 are a work-in-progress, - -- see https://gitlab.com/eessi/support/-/issues/37 - if archdetect_filter_cpu == "x86_64/amd/zen4" then - archdetect_filter_cpu = "x86_64/amd/zen3" - if mode() == "load" then - LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.") - end - end return archdetect_filter_cpu end end From 67bed257d099da38a303a175024c44cbba3a7650 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Sep 2024 18:28:16 +0200 Subject: [PATCH 1313/1795] enhance archdetect to support detection of NVIDIA GPUs + using that in EESSI init script --- init/bash | 5 +++++ init/eessi_archdetect.sh | 22 ++++++++++++++++++++-- init/eessi_environment_variables | 14 ++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/init/bash b/init/bash index 4ad09f6a1b..928ac6efdf 100644 --- a/init/bash +++ b/init/bash @@ -29,6 +29,11 @@ if [ $? -eq 0 ]; then show_msg "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." module use $EESSI_SITE_MODULEPATH + if [ ! -z ${EESSI_MODULEPATH_ACCEL} ]; then + show_msg "Prepending $EESSI_MODULEPATH_ACCEL to \$MODULEPATH..." + module use $EESSI_MODULEPATH_ACCEL + fi + #show_msg "" #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***" #show_msg "" diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index ad6dce6f9a..76de7cace9 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -150,8 +150,25 @@ cpupath(){ fi } +accelpath() { + # If EESSI_ACCELERATOR_TARGET_OVERRIDE is set, use it + log "DEBUG" "accelpath: Override variable set as '$EESSI_ACCELERATOR_TARGET_OVERRIDE' " + [ $EESSI_ACCELERATOR_TARGET_OVERRIDE ] && echo ${EESSI_ACCELERATOR_TARGET_OVERRIDE} && exit + + # check for NVIDIA GPUs via nvidia-smi command + nvidia_smi=$(command -v nvidia-smi) + if [[ $? -eq 0 ]]; then + log "DEBUG" "accelpath: nvidia-smi command found @ ${nvidia_smi}" + gpu_info=$(nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader | head -1) + cuda_cc=$(echo $gpu_info | sed 's/, /,/g' | cut -f4 -d, | sed 's/\.//g') + echo "accel/nvidia/cc${cuda_cc}" + else + log "DEBUG" "accelpath: nvidia-smi command not found" + fi +} + # Parse command line arguments -USAGE="Usage: eessi_archdetect.sh [-h][-d][-a] " +USAGE="Usage: eessi_archdetect.sh [-h][-d][-a] " while getopts 'hdva' OPTION; do case "$OPTION" in @@ -168,5 +185,6 @@ ARGUMENT=${1:-none} case "$ARGUMENT" in "cpupath") cpupath; exit;; - *) echo "$USAGE"; log "ERROR" "Missing argument (possible actions: 'cpupath')";; + "accelpath") accelpath; exit;; + *) echo "$USAGE"; log "ERROR" "Missing argument (possible actions: 'cpupath', 'accelpath')";; esac diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 8c10b1fca8..aad5fe5003 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -38,6 +38,14 @@ if [ -d $EESSI_PREFIX ]; then break fi done + export EESSI_ACCEL_PATH=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh accelpath) + if [ -z ${EESSI_ACCEL_PATH} ]; then + show_msg "archdetect could not find any accelerators" + else + if [ -d ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR}/${EESSI_ACCEL_PATH} ]; then + show_msg "archdetect found accelerator: ${EESSI_ACCEL_PATH}" + fi + fi elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined! export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3 @@ -106,6 +114,12 @@ if [ -d $EESSI_PREFIX ]; then false fi + EESSI_MODULEPATH_ACCEL=${EESSI_SOFTWARE_PATH}/${EESSI_ACCEL_PATH}/${EESSI_MODULE_SUBDIR} + if [ -d ${EESSI_MODULEPATH_ACCEL} ]; then + export EESSI_MODULEPATH_ACCEL=${EESSI_MODULEPATH_ACCEL} + show_msg "Using ${EESSI_MODULEPATH_ACCEL} as additional directory (for accelerators) to be added to MODULEPATH." + fi + # Fix wrong path for RHEL >=8 libcurl # This is required here because we ship curl in our compat layer. If we only provided # curl as a module file we could instead do this via a `modluafooter` in an EasyBuild From 7b103b30bedb94097c09f4b5174445849df63419 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 1 Oct 2024 12:01:00 +0000 Subject: [PATCH 1314/1795] {2023.06}[foss/2022b] data-lang --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index ca35e95bd4..016ddc6c0b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -41,3 +41,6 @@ easyconfigs: - libaio-0.3.113-GCCcore-12.2.0.eb - Z3-4.12.2-GCCcore-12.2.0.eb - tbb-2021.10.0-GCCcore-12.2.0.eb + - dask-2023.7.1-foss-2022b.eb + - netcdf4-python-1.6.3-foss-2022b.eb + - Ruby-3.2.2-GCCcore-12.2.0.eb From 114a8eb850cc40da18070f0f7ab990dd54634ec6 Mon Sep 17 00:00:00 2001 From: parosen Date: Tue, 1 Oct 2024 14:17:47 +0200 Subject: [PATCH 1315/1795] {2023.06}[foss/2023a] ROOT 6.30.06 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 27c18a487e..6823ea5cca 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -22,3 +22,4 @@ easyconfigs: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb + - ROOT-6.30.06-foss-2023a.eb From 1c82faa752e2144a01525c122926768a98f622b5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 1 Oct 2024 14:46:33 +0200 Subject: [PATCH 1316/1795] Install ROOT with EasyBuild 4.9.4 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml new file mode 100644 index 0000000000..992e4c301c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ROOT-6.30.06-foss-2023a.eb From 2b2e57d092230f755c635cf9abe031b01bb968dc Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 1 Oct 2024 14:46:58 +0200 Subject: [PATCH 1317/1795] Update eessi-2023.06-eb-4.9.1-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 6823ea5cca..27c18a487e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -22,4 +22,3 @@ easyconfigs: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb - - ROOT-6.30.06-foss-2023a.eb From 0080c9c9b971532482aaba5cb7cc37dad34a9d49 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Oct 2024 20:36:33 +0200 Subject: [PATCH 1318/1795] add gmsh & co easyconfig at bottom of easystack file --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml index 9e5582485c..0c863f0025 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,8 +1,8 @@ easyconfigs: - - gmsh-4.12.2-foss-2023a.eb - - basemap-1.3.9-foss-2023a.eb - - geopandas-0.14.2-foss-2023a.eb - ccache-4.9-GCCcore-12.3.0.eb - GDB-13.2-GCCcore-12.3.0.eb - tmux-3.3a-GCCcore-12.3.0.eb - - Vim-9.1.0004-GCCcore-12.3.0.eb \ No newline at end of file + - Vim-9.1.0004-GCCcore-12.3.0.eb + - gmsh-4.12.2-foss-2023a.eb + - basemap-1.3.9-foss-2023a.eb + - geopandas-0.14.2-foss-2023a.eb \ No newline at end of file From f0892b19d03ae5f51c1c83b6b3930d9767bb25fc Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 2 Oct 2024 10:13:56 +0200 Subject: [PATCH 1319/1795] Update link_nvidia_host_libraries.sh --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 3de39e33a4..07c57360fa 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -229,7 +229,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then done if [[ "$found" == false ]]; then - echo "$lib is NOT in the provided preload list, filtering $library." + echo "$lib is NOT in the provided preload list, filtering $library" all_found=false break fi From cde0cc8db464e69baefb9280c3e187b254e5974e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 10:50:52 +0200 Subject: [PATCH 1320/1795] More safely add to the toolchainopts --- eb_hooks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 3754564c5c..79bdeeee0d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -190,7 +190,9 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): ): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if cpu_target == CPU_TARGET_NEOVERSE_V1: - if 'toolchainopts' not in ec: + # Make sure the toolchainopts key exists, and the value is a dict, + # before we add the option to disable vectorization + if 'toolchainopts' not in ec or ec['toolchainopts'] is None: ec['toolchainopts'] = {} ec['toolchainopts']['vectorize'] = False print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) @@ -308,7 +310,9 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): """ if ec.name == 'FreeImage' and ec.version in ('3.18.0',): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - if 'toolchainopts' not in ec: + # Make sure the toolchainopts key exists, and the value is a dict, + # before we add the option to enable PIC and disable PNG_ARM_NEON_OPT + if 'toolchainopts' not in ec or ec['toolchainopts'] is None: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True ec['toolchainopts']['extra_cflags'] = '-DPNG_ARM_NEON_OPT=0' From c9613e79ef0585443a9e36a9773683f2904ca1ef Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 2 Oct 2024 13:22:05 +0200 Subject: [PATCH 1321/1795] only copy module_files.list.txt if it exists in create_tarball.sh script --- create_tarball.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 9c212681a5..01f498e1ac 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -89,13 +89,17 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do done # add a bit debug output -echo "wrote file list to ${files_list}" -[ -r ${files_list} ] && cat ${files_list} -echo "wrote module file list to ${module_files_list}" -[ -r ${module_files_list} ] && cat ${module_files_list} +if [ -r ${files_list} ]; then + echo "wrote file list to ${files_list}" + cat ${files_list} +fi +if [ -r ${module_files_list} ]; then + echo "wrote module file list to ${module_files_list}" + cat ${module_files_list} -# Copy the module files list to current workindg dir for later use in the test step -cp ${module_files_list} ${current_workdir}/module_files.list.txt + # Copy the module files list to current workindg dir for later use in the test step + cp ${module_files_list} ${current_workdir}/module_files.list.txt +fi topdir=${cvmfs_repo}/versions/ From a636f63df20a9367c9e4eb772856052697cdcf3a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 13:46:23 +0200 Subject: [PATCH 1322/1795] For PRs that only deploy scripts, such as our second attempt at https://github.com/EESSI/software-layer/pull/736, the test step should be skipped. --- bot/check-test.sh | 14 ++++++++++++++ test_suite.sh | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 3b16e5c415..9ff383a13b 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -23,6 +23,16 @@ else [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi +# First, account for the scenario where we skipped the ReFrame test suite +SKIPPED=-1 +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + GP_skipped='Skipping EESSI test site run' + grep_reframe_skipped=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_skipped}") + [[ $? -eq 0 ]] && SKIPPED=1 || SKIPPED=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_skipped}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_skipped}" +fi # ReFrame prints e.g. #[----------] start processing checks @@ -90,6 +100,10 @@ elif [[ ${SUCCESS} -eq 1 ]]; then summary=":grin: SUCCESS" reason="" status="SUCCESS" +elif [[ ${SKIPPED} -eq 1]]; then + summary="SKIPPED" + reason="The EESSI test suite was skipped. If this PR only deploys scripts, that's expected behavior." + status="SUCCESS" # Should come before general errors: if FAILED==1, it indicates the test suite ran # otherwise the pattern wouldn't have been there elif [[ ${FAILED} -eq 1 ]]; then diff --git a/test_suite.sh b/test_suite.sh index e7151e00e7..ad756f9c76 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -198,6 +198,10 @@ fi # Get the subset of test names based on the test mapping and tags (e.g. CI, 1_node) module_list="module_files.list.txt" mapping_config="tests/eessi_test_mapping/software_to_tests.yml" +if [[ ! -f "$module_list" || ! -f "$mapping_config" ]]; then + fatal_error "No new module files were found. Skipping EESSI test site run." +fi + # Run with --debug for easier debugging in case there are issues: python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}") From b34de3674051d6fc78daf5d99036845cfc492ecb Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:56:17 +0200 Subject: [PATCH 1323/1795] Allow for overriden compat layer in Gentoo Prefix checks --- EESSI-install-software.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f9dd971a0d..d1f7608fc7 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -123,7 +123,9 @@ else fi # make sure we're in Prefix environment by checking $SHELL -if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then +if [[ ${SHELL} = ${EESSI_COMPAT_LAYER_OVERRIDE}/bin/bash ]]; then + echo_green ">> It looks like we're in an overriden Gentoo Prefix environment, good!" +elif [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" else fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" From 30dd71727a50b8dffd14de57f47cf396c634da1a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 14:00:33 +0200 Subject: [PATCH 1324/1795] Only run default tests if the mapping file isn't found --- bot/check-test.sh | 15 ------- test_suite.sh | 19 +++++---- .../map_software_to_test.py | 39 +++++++++++-------- 3 files changed, 34 insertions(+), 39 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 9ff383a13b..ed81198885 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -23,17 +23,6 @@ else [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi -# First, account for the scenario where we skipped the ReFrame test suite -SKIPPED=-1 -if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then - GP_skipped='Skipping EESSI test site run' - grep_reframe_skipped=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_skipped}") - [[ $? -eq 0 ]] && SKIPPED=1 || SKIPPED=0 - # have to be careful to not add searched for pattern into slurm out file - [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_skipped}"'" - [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_skipped}" -fi - # ReFrame prints e.g. #[----------] start processing checks #[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default @@ -100,10 +89,6 @@ elif [[ ${SUCCESS} -eq 1 ]]; then summary=":grin: SUCCESS" reason="" status="SUCCESS" -elif [[ ${SKIPPED} -eq 1]]; then - summary="SKIPPED" - reason="The EESSI test suite was skipped. If this PR only deploys scripts, that's expected behavior." - status="SUCCESS" # Should come before general errors: if FAILED==1, it indicates the test suite ran # otherwise the pattern wouldn't have been there elif [[ ${FAILED} -eq 1 ]]; then diff --git a/test_suite.sh b/test_suite.sh index ad756f9c76..1f0b91c477 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -198,14 +198,19 @@ fi # Get the subset of test names based on the test mapping and tags (e.g. CI, 1_node) module_list="module_files.list.txt" mapping_config="tests/eessi_test_mapping/software_to_tests.yml" -if [[ ! -f "$module_list" || ! -f "$mapping_config" ]]; then - fatal_error "No new module files were found. Skipping EESSI test site run." +if [[ ! -f "$module_list" ]]; then + echo_green "File ${module_list} not found, so only running the default set of tests from ${mapping_config}" + # Run with --debug for easier debugging in case there are issues: + python3 tests/eessi_test_mapping/map_software_to_test.py --mapping-file "${mapping_config}" --debug --defaults-only + REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --mapping-file "${mapping_config}" --defaults-only) + test_selection_exit_code=$? +else + # Run with --debug for easier debugging in case there are issues: + python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug + REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}") + test_selection_exit_code=$? fi - -# Run with --debug for easier debugging in case there are issues: -python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug -REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}") -test_selection_exit_code=$? +# Check exit status if [[ ${test_selection_exit_code} -eq 0 ]]; then echo_green "Succesfully extracted names of tests to run: ${REFRAME_NAME_ARGS}" else diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py index 24cf246ef1..a0da6258c8 100644 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -33,29 +33,32 @@ def get_tests_for_software(software_name, mappings): return [] -def main(yaml_file, module_file, debug): +def main(yaml_file, module_file, debug, defaults_only): """Main function to process software names and their tests.""" mappings = load_mappings(yaml_file) if debug: print(f"Loaded mappings from '{yaml_file}'") - software_names = read_software_names(module_file) - if debug: - print(f"Read software names from '{module_file}'") + if not defaults_only: + software_names = read_software_names(module_file) + if debug: + print(f"Read software names from '{module_file}'") tests_to_run = [] arg_string = "" - # For each module name, get the relevant set of tests - for software_name in software_names: - additional_tests = get_tests_for_software(software_name, mappings) - for test in additional_tests: - if test not in tests_to_run: - tests_to_run.append(test) - - if additional_tests and debug: - print(f"Software: {software_name} -> Tests: {additional_tests}") - elif debug: - print(f"Software: {software_name} -> No tests found") + + if not defaults_only: + # For each module name, get the relevant set of tests + for software_name in software_names: + additional_tests = get_tests_for_software(software_name, mappings) + for test in additional_tests: + if test not in tests_to_run: + tests_to_run.append(test) + + if additional_tests and debug: + print(f"Software: {software_name} -> Tests: {additional_tests}") + elif debug: + print(f"Software: {software_name} -> No tests found") # Always add the default set of tests, if default_tests is specified if 'default_tests' in mappings: @@ -83,8 +86,10 @@ def main(yaml_file, module_file, debug): parser = argparse.ArgumentParser(description="Map software names to their tests based on a YAML configuration.") parser.add_argument('--mapping-file', type=str, help='Path to the YAML file containing the test mappings.') parser.add_argument('--module-list', type=str, help='Path to the file containing the list of software names.') - parser.add_argument('--debug', action='store_true', help='Enable debug output.') + defaults_help = "Don't consider the module-list file, only return the default tests from the mapping file" + parser.add_argument('--defaults-only', action='store_true', default=False, help=defaults_help) + parser.add_argument('--debug', action='store_true', default=False, help='Enable debug output.') args = parser.parse_args() - main(args.mapping_file, args.module_list, args.debug) + main(args.mapping_file, args.module_list, args.debug, args.defaults_only) From fba72eb561d0e404bf8cf321fc7eb5be12c93c15 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 14:55:07 +0200 Subject: [PATCH 1325/1795] More extensive reporting of the test results --- bot/check-test.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index ed81198885..aae1948d33 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -65,7 +65,7 @@ SUCCESS=-1 # Grep for the success pattern, so we can report the amount of tests run if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_success='\[\s*PASSED\s*\].*Ran .* test case' - grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") + grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" @@ -75,7 +75,10 @@ fi if [[ ! -z ${grep_reframe_failed} ]]; then grep_reframe_result=${grep_reframe_failed} else - grep_reframe_result=${grep_reframe_success} + # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report + GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' + grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") + grep_reframe_result=${grep_reframe_success_full} fi echo "[TEST]" > ${job_test_result_file} From e06e303c3a0840d1ef07f136d56c691e4b9c5cf5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 14:56:27 +0200 Subject: [PATCH 1326/1795] This grep can be a simple one --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index aae1948d33..8ea6eddcb0 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -65,7 +65,7 @@ SUCCESS=-1 # Grep for the success pattern, so we can report the amount of tests run if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_success='\[\s*PASSED\s*\].*Ran .* test case' - grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") + grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" From 7b9cc97fdf6f919fc08b0ea00268db8f21218f82 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:06:51 +0200 Subject: [PATCH 1327/1795] Make EPREFIX=$EESSI_COMPAT_LAYER_DIR To be able to also pick up on overriden compat layers --- init/minimal_eessi_env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5273f27862..4f2d52a0fb 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -20,4 +20,4 @@ fi export EESSI_CPU_FAMILY=$(uname -m) # set $EPREFIX since that is basically a standard in Gentoo Prefix -export EPREFIX=$EESSI_PREFIX/compat/$EESSI_OS_TYPE/$EESSI_CPU_FAMILY +export EPREFIX=$EESSI_COMPAT_LAYER_DIR From 125b8499ff9745478c191f0993d4bd1fdfc540de Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:08:32 +0200 Subject: [PATCH 1328/1795] Revert override in compat layer for EPREFIX Moved it to minimal_eessi_env --- EESSI-install-software.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index d1f7608fc7..f9dd971a0d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -123,9 +123,7 @@ else fi # make sure we're in Prefix environment by checking $SHELL -if [[ ${SHELL} = ${EESSI_COMPAT_LAYER_OVERRIDE}/bin/bash ]]; then - echo_green ">> It looks like we're in an overriden Gentoo Prefix environment, good!" -elif [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then +if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" else fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" From 0691dc8c831bf3573c0317446d6df484409f8e4f Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:21:48 +0200 Subject: [PATCH 1329/1795] Pass $EESSI_COMPAT_LAYER_DIR to container --- run_in_compat_layer_env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index ba9d84bee4..fa5dad4a57 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -38,6 +38,9 @@ fi if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_COMPAT_LAYER_DIR} ]; then + INPUT="export EESSI_COMPAT_LAYER_DIR=${EESSI_COMPAT_LAYER_DIR}; ${INPUT}" +fi if [ ! -z ${EESSI_OVERRIDE_GPU_CHECK} ]; then INPUT="export EESSI_OVERRIDE_GPU_CHECK=${EESSI_OVERRIDE_GPU_CHECK}; ${INPUT}" fi From 7dff938197b6167287e33d7396c8ff38ef7c0365 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:36:11 +0200 Subject: [PATCH 1330/1795] Make software.eessi.io modules available when building for dev.eessi.io --- EESSI-install-software.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f9dd971a0d..104fe20c7b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -218,6 +218,11 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi +# If in dev.eessi.io, allow building on top of softw +if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then + module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all +fi + module use $EASYBUILD_INSTALLPATH/modules/all if [[ -z ${MODULEPATH} ]]; then From 139f422fda241768ec9e033fa1cdc032ba7c328d Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 2 Oct 2024 16:56:45 +0200 Subject: [PATCH 1331/1795] Update eessi-2023.06-eb-4.9.4-2023a.yml --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 992e4c301c..3ca4d5f4c8 100644 --- a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -1,2 +1,5 @@ easyconfigs: - - ROOT-6.30.06-foss-2023a.eb + - ROOT-6.30.06-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From e709e6ffa9a467aaca8665fefbe34084230ade9c Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:07:48 +0200 Subject: [PATCH 1332/1795] Use $base_dir to be able to run from another working dir --- install_software_layer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_software_layer.sh b/install_software_layer.sh index 82ca70b73f..8b88e75713 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,4 +1,4 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@" +$base_dir/run_in_compat_layer_env.sh $base_dir/EESSI-install-software.sh "$@" From 4f01309f2da83376e30e4672705731466be15c12 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:10:06 +0200 Subject: [PATCH 1333/1795] Relax regular expression searching for easystacks --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 05572257a5..98576efcb0 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -102,7 +102,7 @@ pr_diff=$(ls [0-9]*.diff | head -1) # if this script is run as root, use PR patch file to determine if software needs to be removed first if [ $EUID -eq 0 ]; then - changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") + changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") if [ -z ${changed_easystacks_rebuilds} ]; then echo "No software needs to be removed." else From bded640d6adace8e5221f6f1bc2cca5757daeac9 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:12:59 +0200 Subject: [PATCH 1334/1795] Use $topdir to run bot/build.sh from another directory --- bot/build.sh | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 237d5574f2..cfae5c9219 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -21,12 +21,14 @@ # stop as soon as something fails set -e +# Make sure we are referring to software-layer as working directory +topdir=$(dirname $(realpath $0)) # source utils.sh and cfg_files.sh -source scripts/utils.sh -source scripts/cfg_files.sh +source $topdir/scripts/utils.sh +source $topdir/scripts/cfg_files.sh # defaults -export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=$topdir/cfg/job.cfg}" HOST_ARCH=$(uname -m) # check if ${JOB_CFG_FILE} exists @@ -214,10 +216,10 @@ else removal_outerr=$(mktemp remove.outerr.XXXX) echo "Executing command to remove software:" - echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" - echo " -- ./EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" - ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ - -- ./EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} + echo "$topdir/eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" + echo " -- $topdir/EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" + $topdir/eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ + -- $topdir/EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} # make sure that the build step resumes from the same temporary directory # this is important, as otherwise the removed software will still be there @@ -250,10 +252,10 @@ fi build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" -echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" -echo " -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" -./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ - -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} +echo "$topdir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" +echo " -- $topdir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" +$topdir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ + -- $topdir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} # prepare directory to store tarball of tmp for tarball step TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step @@ -278,7 +280,7 @@ fi timestamp=$(date +%s) # to set EESSI_VERSION we need to source init/eessi_defaults now -source init/eessi_defaults +source $topdir/init/eessi_defaults export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be @@ -287,9 +289,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI # /tmp as default? TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" -echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" -./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} +echo "$topdir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" +echo " -- $topdir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +$topdir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ + -- $topdir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 From 0fad00c8315f4175d1b3bbde913d14a45af8a461 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:13:59 +0200 Subject: [PATCH 1335/1795] Relax regex for looking for easystacks --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 104fe20c7b..c79a8a3b03 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -267,7 +267,7 @@ if command_exists "nvidia-smi"; then fi # use PR patch file to determine in which easystack files stuff was added -changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') +changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here else From 4f8f4fb537cb123dc108c3240bd23bc2c81f0146 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:19:01 +0200 Subject: [PATCH 1336/1795] Don't run install_scripts.sh in dev.eessi.io --- EESSI-install-software.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index c79a8a3b03..9a63a6a36b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -237,7 +237,11 @@ pr_diff=$(ls [0-9]*.diff | head -1) # install any additional required scripts # order is important: these are needed to install a full CUDA SDK in host_injections # for now, this just reinstalls all scripts. Note the most elegant, but works -${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} + +# Only run install_scripts.sh if not dev.eessi.io for security +if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then + ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} +fi # Install full CUDA SDK in host_injections # Hardcode this for now, see if it works From 0c136fc85c2814b1edc32ab05f4f57d32f1225ca Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:22:17 +0200 Subject: [PATCH 1337/1795] Working directory is one level up --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index cfae5c9219..77bdac00e6 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -22,7 +22,7 @@ set -e # Make sure we are referring to software-layer as working directory -topdir=$(dirname $(realpath $0)) +topdir=$(dirname $(dirname $(realpath $0))) # source utils.sh and cfg_files.sh source $topdir/scripts/utils.sh source $topdir/scripts/cfg_files.sh From ceaf892fd8050a602e740890ef70113f2767bd23 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:26:56 +0200 Subject: [PATCH 1338/1795] Revert topdir in job_cfg_file override We're already in the right path! --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 77bdac00e6..743c19120c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -28,7 +28,7 @@ source $topdir/scripts/utils.sh source $topdir/scripts/cfg_files.sh # defaults -export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=$topdir/cfg/job.cfg}" +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=/cfg/job.cfg}" HOST_ARCH=$(uname -m) # check if ${JOB_CFG_FILE} exists From f46653431834c1c5931aab2fcc28b6768da5654e Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:28:10 +0200 Subject: [PATCH 1339/1795] Fix silly path error --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 743c19120c..be0f34e92e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -28,7 +28,7 @@ source $topdir/scripts/utils.sh source $topdir/scripts/cfg_files.sh # defaults -export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=/cfg/job.cfg}" +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=cfg/job.cfg}" HOST_ARCH=$(uname -m) # check if ${JOB_CFG_FILE} exists From d3b8d5d668ec7102dc4d632f71a75089b99d406d Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:38:54 +0200 Subject: [PATCH 1340/1795] When using dev.eessi.io easystack files and removing installations are not in $TOPDIR --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9a63a6a36b..44f524ebbc 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -297,7 +297,7 @@ else if [ -f ${easystack_file} ]; then echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - ${EB} --easystack ${TOPDIR}/${easystack_file} --robot + ${EB} --easystack ${easystack_file} --robot ec=$? # copy EasyBuild log file if EasyBuild exited with an error @@ -310,7 +310,7 @@ else copy_build_log "${eb_last_log}" "${build_logs_dir}" fi - $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} ${TOPDIR}/${pr_diff} + $TOPDIR/check_missing_installations.sh ${easystack_file} ${pr_diff} else fatal_error "Easystack file ${easystack_file} not found!" fi From 247a23d247feda8b0d239294609760d33b599ce4 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:54:02 +0200 Subject: [PATCH 1341/1795] Pass down change in robot paths to compat layer --- run_in_compat_layer_env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index fa5dad4a57..b4299c7a0d 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -50,6 +50,9 @@ fi if [ ! -z ${https_proxy} ]; then INPUT="export https_proxy=${https_proxy}; ${INPUT}" fi +if [ ! -z ${EASYBUILD_ROBOT_PATHS} ]; then + INPUT="export EASYBUILD_ROBOT_PATHS=${EASYBUILD_ROBOT_PATHS}; ${INPUT}" +fi echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From 4c5c5a36e165f61bf72401fa6083bba13cc9c897 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 2 Oct 2024 22:33:34 +0200 Subject: [PATCH 1342/1795] {2023.06}[foss/2023a] cuDNN 8.9.2.26 w/ CUDA 12.1.1 --- .../eessi-2023.06-eb-4.9.4-2023a-CUDA.yml | 2 + eb_hooks.py | 174 +++++++++++++----- 2 files changed, 128 insertions(+), 48 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml new file mode 100644 index 0000000000..d54780804b --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - cuDNN-8.9.2.26-CUDA-12.1.1.eb diff --git a/eb_hooks.py b/eb_hooks.py index 79bdeeee0d..ce99ed1dfe 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -756,64 +756,141 @@ def post_postproc_cuda(self, *args, **kwargs): if 'libcudart' not in allowlist: raise EasyBuildError("Did not find 'libcudart' in allowlist: %s" % allowlist) - # iterate over all files in the CUDA installation directory - for dir_path, _, files in os.walk(self.installdir): - for filename in files: - full_path = os.path.join(dir_path, filename) - # we only really care about real files, i.e. not symlinks - if not os.path.islink(full_path): - # check if the current file name stub is part of the allowlist - basename = filename.split('.')[0] - if basename in allowlist: - self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) - else: - self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", - basename, full_path) - # if it is not in the allowlist, delete the file and create a symlink to host_injections - - # the host_injections path is under a fixed repo/location for CUDA - host_inj_path = re.sub(EESSI_INSTALLATION_REGEX, HOST_INJECTIONS_LOCATION, full_path) - # CUDA itself doesn't care about compute capability so remove this duplication from - # under host_injections (symlink to a single CUDA installation for all compute - # capabilities) - accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") - if accel_subdir: - host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') - # make sure source and target of symlink are not the same - if full_path == host_inj_path: - raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " - "are using this hook for an EESSI installation?", - full_path, host_inj_path) - remove_file(full_path) - symlink(host_inj_path, full_path) + # replace files that are not distributable with symlinks into + # host_injections + replace_non_distributable_files_with_symlinks(self.log, self.installdir, self.name, allowlist) else: raise EasyBuildError("CUDA-specific hook triggered for non-CUDA easyconfig?!") +def post_postproc_cudnn(self, *args, **kwargs): + """ + Remove files from cuDNN installation that we are not allowed to ship, + and replace them with a symlink to a corresponding installation under host_injections. + """ + + # We need to check if we are doing an EESSI-distributed installation + eessi_installation = bool(re.search(EESSI_INSTALLATION_REGEX, self.installdir)) + + if self.name == 'cuDNN' and eessi_installation: + print_msg("Replacing files in cuDNN installation that we can not ship with symlinks to host_injections...") + + allowlist = ['LICENSE'] + + # read cuDNN LICENSE, construct allowlist based on section "2. Distribution" that specifies list of files that can be shipped + license_path = os.path.join(self.installdir, 'LICENSE') + search_string = "2. Distribution. The following portions of the SDK are distributable under the Agreement:" + with open(license_path) as infile: + for line in infile: + if line.strip().startswith(search_string): + # remove search string, split into words, remove trailing + # dots '.' and only retain words starting with a dot '.' + distributable = line[len(search_string):] + for word in distributable.split(): + if word[0] == '.': + allowlist.append(word.rstrip('.')) + + allowlist = sorted(set(allowlist)) + self.log.info("Allowlist for files in cuDNN installation that can be redistributed: " + ', '.join(allowlist)) + + # replace files that are not distributable with symlinks into + # host_injections + replace_non_distributable_files_with_symlinks(self.log, self.installdir, self.name, allowlist) + else: + raise EasyBuildError("cuDNN-specific hook triggered for non-cuDNN easyconfig?!") + + +def replace_non_distributable_files_with_symlinks(log, install_dir, package, allowlist): + """ + Replace files that cannot be distributed with symlinks into host_injections + """ + extension_based = { "CUDA": False, "cuDNN": True } + if not package in extension_based: + raise EasyBuildError("Don't know how to strip non-distributable files from package %s.", package) + + # iterate over all files in the package installation directory + for dir_path, _, files in os.walk(install_dir): + for filename in files: + full_path = os.path.join(dir_path, filename) + # we only really care about real files, i.e. not symlinks + if not os.path.islink(full_path): + # check if the current file name stub is part of the allowlist + basename = filename.split('.')[0] + if extension_based[package]: + if '.' in filename: + extension = '.' + filename.split('.')[1] + if basename in allowlist: + log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) + elif extension_based[package] and '.' in filename and extension in allowlist: + log.debug("%s is found in allowlist, so keeping it: %s", extension, full_path) + else: + if extension_based[package]: + print_name = filename + else: + print_name = basename + log.debug("%s is not found in allowlist, so replacing it with symlink: %s", + print_name, full_path) + # the host_injections path is under a fixed repo/location for CUDA or cuDNN + host_inj_path = re.sub(EESSI_INSTALLATION_REGEX, HOST_INJECTIONS_LOCATION, full_path) + # CUDA and cuDNN itself don't care about compute capability so remove this duplication from + # under host_injections (symlink to a single CUDA or cuDNN installation for all compute + # capabilities) + accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") + if accel_subdir: + host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') + # make sure source and target of symlink are not the same + if full_path == host_inj_path: + raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " + "are using this hook for an EESSI installation?", + full_path, host_inj_path) + remove_file(full_path) + symlink(host_inj_path, full_path) + + def inject_gpu_property(ec): """ - Add 'gpu' property, via modluafooter easyconfig parameter + Add 'gpu' property EESSIVERSION envvars and drop dependencies to + build dependencies, via modluafooter easyconfig parameter """ ec_dict = ec.asdict() - # Check if CUDA is in the dependencies, if so add the 'gpu' Lmod property - if ('CUDA' in [dep[0] for dep in iter(ec_dict['dependencies'])]): - ec.log.info("Injecting gpu as Lmod arch property and envvar with CUDA version") + # Check if CUDA, cuDNN, you-name-it is in the dependencies, if so + # - drop dependency to build dependency + # - add 'gpu' Lmod property + # - add envvar with package version + packages_list = ( "CUDA", "cuDNN" ) + packages_version = { } + add_gpu_property = '' + + for package in packages_list: + # Check if package is in the dependencies, if so drop dependency to build + # dependency and set variable for later adding the 'gpu' Lmod property + if (package in [dep[0] for dep in iter(ec_dict['dependencies'])]): + add_gpu_property = 'add_property("arch","gpu")' + for dep in iter(ec_dict['dependencies']): + if package in dep[0]: + # make package a build dependency only (rpathing saves us from link errors) + ec.log.info("Dropping dependency on %s to build dependency" % package) + ec_dict['dependencies'].remove(dep) + if dep not in ec_dict['builddependencies']: + ec_dict['builddependencies'].append(dep) + # take note of version for creating the modluafooter + packages_version[package] = dep[1] + if add_gpu_property: + ec.log.info("Injecting gpu as Lmod arch property and envvars for dependencies with their version") key = 'modluafooter' - value = 'add_property("arch","gpu")' - cuda_version = 0 - for dep in iter(ec_dict['dependencies']): - # Make CUDA a build dependency only (rpathing saves us from link errors) - if 'CUDA' in dep[0]: - cuda_version = dep[1] - ec_dict['dependencies'].remove(dep) - if dep not in ec_dict['builddependencies']: - ec_dict['builddependencies'].append(dep) - value = '\n'.join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) - if key in ec_dict: - if value not in ec_dict[key]: - ec[key] = '\n'.join([ec_dict[key], value]) + values = [add_gpu_property] + for package, version in packages_version.items(): + envvar = "EESSI%sVERSION" % package.upper() + values.append('setenv("%s","%s")' % (envvar, version)) + if not key in ec_dict: + ec[key] = '\n'.join(values) else: - ec[key] = value + new_value = ec_dict[key] + for value in values: + if not value in new_value: + new_value = '\n'.join([new_value, value]) + ec[key] = new_value + return ec @@ -873,4 +950,5 @@ def inject_gpu_property(ec): POST_POSTPROC_HOOKS = { 'CUDA': post_postproc_cuda, + 'cuDNN': post_postproc_cudnn, } From 428489e5b85f8b80acec8607441e7c773a4a2e88 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 22:57:05 +0200 Subject: [PATCH 1343/1795] Debug why the full reframe report is not printed --- bot/check-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 8ea6eddcb0..3ed6c36a2b 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -80,6 +80,7 @@ else grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") grep_reframe_result=${grep_reframe_success_full} fi +echo "grep_reframe_result: ${grep_reframe_result}" echo "[TEST]" > ${job_test_result_file} if [[ ${SLURM_OUTPUT_FOUND} -eq 0 ]]; then From da7c1e4669f4b3947b059162f5e72de1d634d777 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 2 Oct 2024 23:27:02 +0200 Subject: [PATCH 1344/1795] use post sanity-check hook for cuDNN --- eb_hooks.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index ce99ed1dfe..25eefaf27a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -763,7 +763,13 @@ def post_postproc_cuda(self, *args, **kwargs): raise EasyBuildError("CUDA-specific hook triggered for non-CUDA easyconfig?!") -def post_postproc_cudnn(self, *args, **kwargs): +def post_sanitycheck_hook(self, *args, **kwargs): + """Main post-sanitycheck hook: trigger custom functions based on software name.""" + if self.name in POST_SANITYCHECK_HOOKS: + POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) + + +def post_sanitycheck_cudnn(self, *args, **kwargs): """ Remove files from cuDNN installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. @@ -950,5 +956,8 @@ def inject_gpu_property(ec): POST_POSTPROC_HOOKS = { 'CUDA': post_postproc_cuda, - 'cuDNN': post_postproc_cudnn, +} + +POST_SANITYCHECK_HOOKS = { + 'cuDNN': post_sanitycheck_cudnn, } From 860d9ac8220a3cc2a56ae30b97ea8df3763ccca4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 23:27:08 +0200 Subject: [PATCH 1345/1795] more debugging output --- bot/check-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 3ed6c36a2b..fff9a71491 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -79,6 +79,7 @@ else GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") grep_reframe_result=${grep_reframe_success_full} + echo "grep_reframe_success_full: ${grep_reframe_success_full}" fi echo "grep_reframe_result: ${grep_reframe_result}" From 65710351ec20e715c074c13132851169b4a1419f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 11:11:35 +0200 Subject: [PATCH 1346/1795] introduce $EESSI_GPU_SOFTWARE_PATH which can be overriden via $EESSI_GPU_SOFTWARE_SUBDIR_OVERRIDE Co-authored-by: ocaisa --- init/eessi_environment_variables | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index aad5fe5003..aa066b924c 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -42,7 +42,9 @@ if [ -d $EESSI_PREFIX ]; then if [ -z ${EESSI_ACCEL_PATH} ]; then show_msg "archdetect could not find any accelerators" else - if [ -d ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR}/${EESSI_ACCEL_PATH} ]; then + EESSI_GPU_SOFTWARE_SUBDIR=${$EESSI_GPU_SOFTWARE_SUBDIR_OVERRIDE:-${EESSI_SOFTWARE_SUBDIR}} + EESSI_GPU_SOFTWARE_PATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_GPU_SOFTWARE_SUBDIR} + if [ -d ${EESSI_GPU_SOFTWARE_PATH}/${EESSI_ACCEL_PATH} ]; then show_msg "archdetect found accelerator: ${EESSI_ACCEL_PATH}" fi fi @@ -114,7 +116,7 @@ if [ -d $EESSI_PREFIX ]; then false fi - EESSI_MODULEPATH_ACCEL=${EESSI_SOFTWARE_PATH}/${EESSI_ACCEL_PATH}/${EESSI_MODULE_SUBDIR} + EESSI_MODULEPATH_ACCEL=${EESSI_GPU_SOFTWARE_PATH}/${EESSI_ACCEL_PATH}/${EESSI_MODULE_SUBDIR} if [ -d ${EESSI_MODULEPATH_ACCEL} ]; then export EESSI_MODULEPATH_ACCEL=${EESSI_MODULEPATH_ACCEL} show_msg "Using ${EESSI_MODULEPATH_ACCEL} as additional directory (for accelerators) to be added to MODULEPATH." From 148e8efae37d2ad1e51756cd64574661fccb9ca5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 3 Oct 2024 14:17:36 +0200 Subject: [PATCH 1347/1795] Also use patched easyconfig --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 3ca4d5f4c8..2a8cb36994 100644 --- a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -1,5 +1,7 @@ easyconfigs: - ROOT-6.30.06-foss-2023a.eb: options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 0707601ccd7dd497a419ed39b1b5527ddd0450f4 # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From 454d2bb0f5076704cdfb7e161e9a187cf6241f3a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 14:57:55 +0200 Subject: [PATCH 1348/1795] install cuDNN under host_injections before installing it under /cvmfs --- EESSI-install-software.sh | 9 +- install_scripts.sh | 5 +- .../eessi-2023.06-cuda-and-libraries.yml | 3 + .../nvidia/install_cuda_and_libraries.sh | 205 ++++++++++++++++++ 4 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml create mode 100755 scripts/gpu_support/nvidia/install_cuda_and_libraries.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f9dd971a0d..d54da4a404 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -234,7 +234,7 @@ pr_diff=$(ls [0-9]*.diff | head -1) # for now, this just reinstalls all scripts. Note the most elegant, but works ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} -# Install full CUDA SDK in host_injections +# Install full CUDA SDK and cu* libraries in host_injections # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install # Allow skipping CUDA SDK install in e.g. CI environments @@ -250,9 +250,12 @@ else fi if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then - ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula + ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \ + -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml \ + -t /tmp/temp \ + --accept-cuda-eula else - echo "Skipping installation of CUDA SDK in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found" + echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found" fi # Install NVIDIA drivers in host_injections (if they exist) diff --git a/install_scripts.sh b/install_scripts.sh index 11c7fc2a9f..df9bda3ad3 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -122,7 +122,10 @@ copy_files_by_list ${TOPDIR}/scripts ${INSTALL_PREFIX}/scripts "${script_files[@ # Copy files for the scripts/gpu_support/nvidia directory nvidia_files=( - install_cuda_host_injections.sh link_nvidia_host_libraries.sh + eessi-2023.06-cuda-and-libraries.yml + install_cuda_and_libraries.sh + install_cuda_host_injections.sh + link_nvidia_host_libraries.sh ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" diff --git a/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml b/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml new file mode 100644 index 0000000000..e0e47bf2d8 --- /dev/null +++ b/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml @@ -0,0 +1,3 @@ +easyconfigs: + - CUDA-12.1.1.eb + - cuDNN-8.9.2.26-CUDA-12.1.1.eb diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh new file mode 100755 index 0000000000..2fea64d7a6 --- /dev/null +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash + +# This script can be used to install CUDA and other libraries by NVIDIA under +# the `.../host_injections` directory. +# +# This provides the parts of the CUDA installation and other libriaries that +# cannot be redistributed as part of EESSI due to license limitations. While +# GPU-based software from EESSI will _run_ without these, installation of +# additional software that builds upon CUDA or other libraries requires that +# these installation are present under `host_injections`. +# +# The `host_injections` directory is a variant symlink that by default points to +# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see +# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the +# installation to be successful, this directory needs to be writeable by the user +# executing this script. + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath $BASH_SOURCE)) +source "$TOPDIR"/../../utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " --accept-cuda-eula You _must_ accept the CUDA EULA to install" + echo " CUDA, see the EULA at" + echo " https://docs.nvidia.com/cuda/eula/index.html" + echo " -e, --easystack EASYSTACKFILE Path to easystack file that defines which" + echo " packages shall be installed" + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the installation of CUDA" + echo " and/or other libraries (must have" + echo " several GB available; depends on the number of installations)" +} + +# Initialize variables +eula_accepted=0 +EASYSTACKFILE= +TEMP_DIR= + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + --accept-cuda-eula) + eula_accepted=1 + shift 1 + ;; + -e|--easystack) + if [ -n "$2" ]; then + EASYSTACKFILE="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +if [[ -z "${EASYSTACKFILE}" ]]; then + fatal_error "Need the name/path to an easystack file. See command line options\n" +fi + +# Make sure EESSI is initialised +check_eessi_initialised + +# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` +# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup) +export EESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections} + +# we need a directory we can use for temporary storage +if [[ -z "${TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) +else + mkdir -p ${TEMP_DIR} + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} cuda_n_co.XXX) + if [[ ! -d "$tmpdir" ]] ; then + fatal_error "Could not create directory ${tmpdir}" + fi +fi +echo "Created temporary directory '${tmpdir}'" + +# workaround for EasyBuild not being found when loading "extend" module +module load EasyBuild/4.9.4 + +# load EESSI-extend/2023.06-easybuild module && verify that it is loaded +EESSI_EXTEND_MODULE="EESSI-extend/2023.06-easybuild" +module load ${EESSI_EXTEND_MODULE} +ret=$? +if [ "${ret}" -ne 0 ]; then + fatal_error "An error occured while trying to load ${EESSI_EXTEND_MODULE}\n" +fi + +# do a 'eb --dry-run-short' with the EASYSTACKFILE and determine list of packages +# to be installed +echo ">> Determining if packages specified in ${EASYSTACKFILE} are missing under ${EESSI_SITE_INSTALL}" +eb_dry_run_short_out=${tmpdir}/eb_dry_run_short.out +eb --dry-run-short --rebuild --easystack ${EASYSTACKFILE} 2>&1 | tee ${eb_dry_run_short_out} +ret=$? + +# Check if CUDA shall be installed +cuda_install_needed=0 +cat ${eb_dry_run_short_out} | grep "^ \* \[[xR]\]" | grep "module: CUDA/" +ret=$? +if [ "${ret}" -eq 0 ]; then + cuda_install_needed=1 +fi + +# Make sure the CUDA EULA is accepted if it shall be installed +if [ "${cuda_install_needed}" -eq 1 ] && [ "${eula_accepted}" -ne 1 ]; then + show_help + error="\nCUDA shall be installed. However, the CUDA EULA has not been accepted\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n" + fatal_error "${error}" +fi + +# determine the number of packages to be installed (assume 5 GB + num_packages * +# 3GB space needed) +number_of_packages=$(cat ${eb_dry_run_short_out} | grep "^ \* \[[xR]\]" | sed -e 's/^.*module: //' | uniq | wc -l) +echo "number of packages to be (re-)installed: '${number_of_packages}'" +base_storage_space=$((5000000 + ${number_of_packages} * 3000000)) + +required_space_in_tmpdir=${base_storage_space} +# Let's see if we have sources and build locations defined if not, we use the temporary space +if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then + export EASYBUILD_BUILDPATH=${tmpdir}/build + required_space_in_tmpdir=$((required_space_in_tmpdir + ${base_storage_space})) +fi +if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then + export EASYBUILD_SOURCEPATH=${tmpdir}/sources + required_space_in_tmpdir=$((required_space_in_tmpdir + ${base_storage_space})) +fi + +# The install is pretty fat, you need lots of space for download/unpack/install +# (~3*${base_storage_space}*1000 Bytes), +# need to do a space check before we proceed +avail_space=$(df --output=avail "${EESSI_SITE_INSTALL}"/ | tail -n 1 | awk '{print $1}') +min_disk_storage=$((3 * ${base_storage_space})) +if (( avail_space < ${min_disk_storage} )); then + fatal_error "Need at least $(echo "${min_disk_storage} / 1000000" | bc) GB disk space to install CUDA and other libraries under ${EESSI_SITE_INSTALL}, exiting now..." +fi +avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') +if (( avail_space < required_space_in_tmpdir )); then + error="Need at least $(echo "${required_space_in_tmpdir} / 1000000" | bc) temporary disk space under ${tmpdir}.\n" + error="${error}Set the environment variable TEMP_DIR to a location with adequate space to pass this check." + error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH" + error="${error}to reduce this requirement. Exiting now..." + fatal_error "${error}" +fi + +# Brief explanation of parameters: +# - prefix: using $tmpdir as default base directory for several EB settings +# - rebuild: we need the --rebuild option, as the CUDA module may or may not be on the +# `MODULEPATH` yet. Even if it is, we still want to redo this installation +# since it will provide the symlinked targets for the parts of the CUDA +# and/or other installation in the `.../versions/...` prefix +# - installpath-modules: We install the module in our `tmpdir` since we do not need the modulefile, +# we only care about providing the targets for the symlinks. +# - ${cuda_arg}: We only set the --accept-eula-for=CUDA option if CUDA will be installed and if +# this script was called with the argument --accept-cuda-eula. +# - hooks: We don't want hooks used in this install, we need vanilla +# installations of CUDA and/or other libraries +# - easystack: Path to easystack file that defines which packages shall be +# installed +cuda_arg= +if [[ ${eula_accepted} -eq 1 ]]; then + cuda_arg="--accept-eula-for=CUDA" +fi +touch "$tmpdir"/none.py +eb --prefix="$tmpdir" \ + --rebuild \ + --installpath-modules=${tmpdir} \ + "${cuda_arg}" \ + --hooks="$tmpdir"/none.py \ + --easystack ${EASYSTACKFILE} +ret=$? +if [ $ret -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + cp -a ${eb_last_log} . + fatal_error "some installation failed, please check EasyBuild logs $(basename ${eb_last_log})..." +else + echo_green "all installations at ${EESSI_SITE_INSTALL}/software/... succeeded!" +fi +# clean up tmpdir +rm -rf "${tmpdir}" From 6824d75253b3ced2b4ce00f5a6f1fe5548bcbe41 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 15:00:33 +0200 Subject: [PATCH 1349/1795] use post_postproc hook to convert some cuDNN files to symlinks --- eb_hooks.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 25eefaf27a..ce99ed1dfe 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -763,13 +763,7 @@ def post_postproc_cuda(self, *args, **kwargs): raise EasyBuildError("CUDA-specific hook triggered for non-CUDA easyconfig?!") -def post_sanitycheck_hook(self, *args, **kwargs): - """Main post-sanitycheck hook: trigger custom functions based on software name.""" - if self.name in POST_SANITYCHECK_HOOKS: - POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) - - -def post_sanitycheck_cudnn(self, *args, **kwargs): +def post_postproc_cudnn(self, *args, **kwargs): """ Remove files from cuDNN installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. @@ -956,8 +950,5 @@ def inject_gpu_property(ec): POST_POSTPROC_HOOKS = { 'CUDA': post_postproc_cuda, -} - -POST_SANITYCHECK_HOOKS = { - 'cuDNN': post_sanitycheck_cudnn, + 'cuDNN': post_postproc_cudnn, } From 719b5628ea93bdd321c96efc17e16d7c8d66f3ac Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 3 Oct 2024 15:29:10 +0200 Subject: [PATCH 1350/1795] Update eessi-2023.06-eb-4.9.4-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 2a8cb36994..19d9aedfae 100644 --- a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -2,6 +2,6 @@ easyconfigs: - ROOT-6.30.06-foss-2023a.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 - from-commit: 0707601ccd7dd497a419ed39b1b5527ddd0450f4 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From 06ddb081ffb3593a39824a28b3663cce2b010d91 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 3 Oct 2024 17:57:30 +0200 Subject: [PATCH 1351/1795] Fix name of pattern to pass to grep --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index fff9a71491..2d4e1231b4 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -77,7 +77,7 @@ if [[ ! -z ${grep_reframe_failed} ]]; then else # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' - grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") + grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success_full}") grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" fi From edc5bca0563fb3946ea2cfa9e0e098a96b5e05af Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 3 Oct 2024 17:59:10 +0200 Subject: [PATCH 1352/1795] Trim trailing null character which is the result of using the -z argument for grep --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 2d4e1231b4..1b973fcb3d 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -77,7 +77,7 @@ if [[ ! -z ${grep_reframe_failed} ]]; then else # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' - grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success_full}") + grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success_full} | tr -d '\0'") grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" fi From 228eaa29372ae5f46b9b7d56ab505a86a5a01025 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 19:29:54 +0200 Subject: [PATCH 1353/1795] use non-zero exit code in archdetect if nvidia-smi was not found or failed to run + take that into account in EESSI init script + allow overriding software subdirectory for accel/* via $EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE --- init/eessi_archdetect.sh | 19 +++++++++++++++---- init/eessi_environment_variables | 32 +++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 76de7cace9..be3154a62f 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -17,7 +17,7 @@ else exit 1 fi -VERSION="1.1.0" +VERSION="1.2.0" # default log level: only emit warnings or errors LOG_LEVEL="WARN" @@ -159,11 +159,22 @@ accelpath() { nvidia_smi=$(command -v nvidia-smi) if [[ $? -eq 0 ]]; then log "DEBUG" "accelpath: nvidia-smi command found @ ${nvidia_smi}" - gpu_info=$(nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader | head -1) - cuda_cc=$(echo $gpu_info | sed 's/, /,/g' | cut -f4 -d, | sed 's/\.//g') - echo "accel/nvidia/cc${cuda_cc}" + nvidia_smi_out=$(mktemp -p /tmp nvidia_smi_out.XXXXX) + nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader 2>&1 > $nvidia_smi_out + if [[ $? -eq 0 ]]; then + nvidia_smi_info=$(head -1 $nvidia_smi_out) + cuda_cc=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f4 -d, | sed 's/\.//g') + log "DEBUG" "accelpath: CUDA compute capability '${cuda_cc}' derived from nvidia-smi output '${nvidia_smi_info}'" + res="accel/nvidia/cc${cuda_cc}" + log "DEBUG" "accelpath: result: ${res}" + echo $res + rm -f $nvidia_smi_out + else + log "ERROR" "accelpath: nvidia-smi command failed, see output in $nvidia_smi_out" + fi else log "DEBUG" "accelpath: nvidia-smi command not found" + exit 2 fi } diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index aa066b924c..2003949369 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -38,15 +38,29 @@ if [ -d $EESSI_PREFIX ]; then break fi done - export EESSI_ACCEL_PATH=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh accelpath) - if [ -z ${EESSI_ACCEL_PATH} ]; then - show_msg "archdetect could not find any accelerators" - else - EESSI_GPU_SOFTWARE_SUBDIR=${$EESSI_GPU_SOFTWARE_SUBDIR_OVERRIDE:-${EESSI_SOFTWARE_SUBDIR}} - EESSI_GPU_SOFTWARE_PATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_GPU_SOFTWARE_SUBDIR} - if [ -d ${EESSI_GPU_SOFTWARE_PATH}/${EESSI_ACCEL_PATH} ]; then - show_msg "archdetect found accelerator: ${EESSI_ACCEL_PATH}" + # to be able to grab exit code of archdetect trying to detect accelerators, + # we can not run it via $(...), so we have to redirect the output to a temporary file + tmpout=$(mktemp) + ${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh accelpath 2>&1 > $tmpout + ec=$? + if [[ $ec -eq 0 ]]; then + export EESSI_ACCEL_SUBDIR=$(tail -1 $tmpout && rm -f $tmpout) + if [ -z ${EESSI_ACCEL_SUBDIR} ]; then + error "accelerator detection with archdetect worked, but no result was returned?!" + else + # allow specifying different parent directory for accel/* subdirectory via $EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE + EESSI_ACCEL_SOFTWARE_SUBDIR=${EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE:-$EESSI_SOFTWARE_SUBDIR} + # path to where accel/* subdirectory is located + EESSI_ACCEL_SOFTWARE_PATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_ACCEL_SOFTWARE_SUBDIR} + if [ -d $EESSI_ACCEL_SOFTWARE_PATH/${EESSI_ACCEL_SUBDIR} ]; then + show_msg "archdetect found supported accelerator for CPU target ${EESSI_ACCEL_SOFTWARE_SUBDIR}: ${EESSI_ACCEL_SUBDIR}" + else + show_msg "No matching path found in ${EESSI_ACCEL_SOFTWARE_SUBDIR} for accelerator detected by archdetect (${EESSI_ACCEL_SUBDIR})" + fi fi + else + show_msg "archdetect could not detect any accelerators" + rm -f $tmpout fi elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined! @@ -116,7 +130,7 @@ if [ -d $EESSI_PREFIX ]; then false fi - EESSI_MODULEPATH_ACCEL=${EESSI_GPU_SOFTWARE_PATH}/${EESSI_ACCEL_PATH}/${EESSI_MODULE_SUBDIR} + EESSI_MODULEPATH_ACCEL=${EESSI_ACCEL_SOFTWARE_PATH}/${EESSI_ACCEL_SUBDIR}/${EESSI_MODULE_SUBDIR} if [ -d ${EESSI_MODULEPATH_ACCEL} ]; then export EESSI_MODULEPATH_ACCEL=${EESSI_MODULEPATH_ACCEL} show_msg "Using ${EESSI_MODULEPATH_ACCEL} as additional directory (for accelerators) to be added to MODULEPATH." From f823375c315e389f9c4af00e1fe77ecaed2d5a8f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 19:35:58 +0200 Subject: [PATCH 1354/1795] sanity check value of $EESSI_ACCELERATOR_TARGET_OVERRIDE, if it is set, must be 'accel/nvidia/cc[0-9][0-9]' --- init/eessi_archdetect.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index be3154a62f..87d47364f7 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -153,7 +153,15 @@ cpupath(){ accelpath() { # If EESSI_ACCELERATOR_TARGET_OVERRIDE is set, use it log "DEBUG" "accelpath: Override variable set as '$EESSI_ACCELERATOR_TARGET_OVERRIDE' " - [ $EESSI_ACCELERATOR_TARGET_OVERRIDE ] && echo ${EESSI_ACCELERATOR_TARGET_OVERRIDE} && exit + if [ ! -z $EESSI_ACCELERATOR_TARGET_OVERRIDE ]; then + if [[ "$EESSI_ACCELERATOR_TARGET_OVERRIDE" =~ ^accel/nvidia/cc[0-9][0-9]$ ]]; then + echo ${EESSI_ACCELERATOR_TARGET_OVERRIDE} + return 0 + else + log "ERROR" "Value of \$EESSI_ACCELERATOR_TARGET_OVERRIDE should match 'accel/nvidia/cc[0-9[0-9]', but it does not: '$EESSI_ACCELERATOR_TARGET_OVERRIDE'" + fi + return 0 + fi # check for NVIDIA GPUs via nvidia-smi command nvidia_smi=$(command -v nvidia-smi) From b04cc298e0e593a85e838eb25696dc1ab8962bf9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 19:54:59 +0200 Subject: [PATCH 1355/1795] take into account that nvidia-smi command may be available, but fail with 'No devices were found' if no GPUs are available in Slurm job --- init/eessi_archdetect.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 87d47364f7..2b1534ce62 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -178,7 +178,8 @@ accelpath() { echo $res rm -f $nvidia_smi_out else - log "ERROR" "accelpath: nvidia-smi command failed, see output in $nvidia_smi_out" + log "DEBUG" "accelpath: nvidia-smi command failed, see output in $nvidia_smi_out" + exit 3 fi else log "DEBUG" "accelpath: nvidia-smi command not found" From d9bca16a1a0965b6ed2f25ae4b7d07f35168f614 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 3 Oct 2024 19:56:59 +0200 Subject: [PATCH 1356/1795] only set $EESSI_MODULEPATH_ACCEL if directory exists --- init/eessi_environment_variables | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 2003949369..fb1b731ee1 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -130,9 +130,8 @@ if [ -d $EESSI_PREFIX ]; then false fi - EESSI_MODULEPATH_ACCEL=${EESSI_ACCEL_SOFTWARE_PATH}/${EESSI_ACCEL_SUBDIR}/${EESSI_MODULE_SUBDIR} - if [ -d ${EESSI_MODULEPATH_ACCEL} ]; then - export EESSI_MODULEPATH_ACCEL=${EESSI_MODULEPATH_ACCEL} + if [ -d ${EESSI_ACCEL_SOFTWARE_PATH}/${EESSI_ACCEL_SUBDIR}/${EESSI_MODULE_SUBDIR} ]; then + export EESSI_MODULEPATH_ACCEL=${EESSI_ACCEL_SOFTWARE_PATH}/${EESSI_ACCEL_SUBDIR}/${EESSI_MODULE_SUBDIR} show_msg "Using ${EESSI_MODULEPATH_ACCEL} as additional directory (for accelerators) to be added to MODULEPATH." fi From 044e168572f110fc9420336ee868fe13df538bba Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 20:46:32 +0200 Subject: [PATCH 1357/1795] explain idea for extension_based and reformat its definition --- eb_hooks.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index ce99ed1dfe..2e4b41fe2a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -804,7 +804,19 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, package, all """ Replace files that cannot be distributed with symlinks into host_injections """ - extension_based = { "CUDA": False, "cuDNN": True } + # Different packages use different ways to specify which files or file + # 'types' may be redistributed. For CUDA, the 'EULA.txt' lists full file + # names. For cuDNN, the 'LICENSE' lists file endings/suffixes (e.g., '.so') + # that can be redistributed. + # The map 'extension_based' defines which of these two ways are employed. If + # full file names are used it maps a package name (key) to False (value). If + # endings/suffixes are used, it maps a package name to True. Later we can + # easily use this data structure to employ the correct method for + # postprocessing an installation. + extension_based = { + "CUDA": False, + "cuDNN": True, + } if not package in extension_based: raise EasyBuildError("Don't know how to strip non-distributable files from package %s.", package) From f983fed3d06514da5534952d5588786273e0339f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 20:57:54 +0200 Subject: [PATCH 1358/1795] explain why we need to obtain the extension and improve cond expr --- eb_hooks.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 2e4b41fe2a..6b3acaba29 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -828,9 +828,13 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, package, all if not os.path.islink(full_path): # check if the current file name stub is part of the allowlist basename = filename.split('.')[0] - if extension_based[package]: - if '.' in filename: - extension = '.' + filename.split('.')[1] + if extension_based[package] and '.' in filename: + # if the allowlist only contains extensions, we have to + # determine that from filename. we assume the extension is + # the second element when splitting the filename at dots + # (e.g., for 'libcudnn_adv_infer.so.8.9.2' the extension + # would be '.so') + extension = '.' + filename.split('.')[1] if basename in allowlist: log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) elif extension_based[package] and '.' in filename and extension in allowlist: From 6e95efdb246fc4cfa43b14675170304f2c3ff5ae Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 21:03:01 +0200 Subject: [PATCH 1359/1795] use local var for conditional expression + slightly reorder code --- eb_hooks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 6b3acaba29..144f5b1333 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -826,18 +826,19 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, package, all full_path = os.path.join(dir_path, filename) # we only really care about real files, i.e. not symlinks if not os.path.islink(full_path): - # check if the current file name stub is part of the allowlist - basename = filename.split('.')[0] - if extension_based[package] and '.' in filename: + check_by_extension = extension_based[package] and '.' in filename + if check_by_extension: # if the allowlist only contains extensions, we have to # determine that from filename. we assume the extension is # the second element when splitting the filename at dots # (e.g., for 'libcudnn_adv_infer.so.8.9.2' the extension # would be '.so') extension = '.' + filename.split('.')[1] + # check if the current file name stub or its extension is part of the allowlist + basename = filename.split('.')[0] if basename in allowlist: log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) - elif extension_based[package] and '.' in filename and extension in allowlist: + elif check_by_extension and extension in allowlist: log.debug("%s is found in allowlist, so keeping it: %s", extension, full_path) else: if extension_based[package]: From 21916fd4e12ff7f880483e6e0f2c107288c6f361 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 21:05:54 +0200 Subject: [PATCH 1360/1795] code golf --- eb_hooks.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 144f5b1333..61c96b921e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -841,10 +841,7 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, package, all elif check_by_extension and extension in allowlist: log.debug("%s is found in allowlist, so keeping it: %s", extension, full_path) else: - if extension_based[package]: - print_name = filename - else: - print_name = basename + print_name = filename if extension_based[package] else basename log.debug("%s is not found in allowlist, so replacing it with symlink: %s", print_name, full_path) # the host_injections path is under a fixed repo/location for CUDA or cuDNN From bf2684685070a199766f72fccfe0825a8ace573e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 21:16:41 +0200 Subject: [PATCH 1361/1795] improve comment (also anticipating additional cu* libraries in the future) --- eb_hooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 61c96b921e..952b518d6b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -846,9 +846,9 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, package, all print_name, full_path) # the host_injections path is under a fixed repo/location for CUDA or cuDNN host_inj_path = re.sub(EESSI_INSTALLATION_REGEX, HOST_INJECTIONS_LOCATION, full_path) - # CUDA and cuDNN itself don't care about compute capability so remove this duplication from - # under host_injections (symlink to a single CUDA or cuDNN installation for all compute - # capabilities) + # CUDA and cu* libraries themselves don't care about compute capability so remove this + # duplication from under host_injections (symlink to a single CUDA or cu* library + # installation for all compute capabilities) accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') From e1ba74ffa959316521461e448fe8370ed307c4c9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 21:21:20 +0200 Subject: [PATCH 1362/1795] improve parameter name --- eb_hooks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 952b518d6b..6a4fdfe54b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -800,7 +800,7 @@ def post_postproc_cudnn(self, *args, **kwargs): raise EasyBuildError("cuDNN-specific hook triggered for non-cuDNN easyconfig?!") -def replace_non_distributable_files_with_symlinks(log, install_dir, package, allowlist): +def replace_non_distributable_files_with_symlinks(log, install_dir, pkg_name, allowlist): """ Replace files that cannot be distributed with symlinks into host_injections """ @@ -817,8 +817,8 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, package, all "CUDA": False, "cuDNN": True, } - if not package in extension_based: - raise EasyBuildError("Don't know how to strip non-distributable files from package %s.", package) + if not pkg_name in extension_based: + raise EasyBuildError("Don't know how to strip non-distributable files from package %s.", pkg_name) # iterate over all files in the package installation directory for dir_path, _, files in os.walk(install_dir): @@ -826,7 +826,7 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, package, all full_path = os.path.join(dir_path, filename) # we only really care about real files, i.e. not symlinks if not os.path.islink(full_path): - check_by_extension = extension_based[package] and '.' in filename + check_by_extension = extension_based[pkg_name] and '.' in filename if check_by_extension: # if the allowlist only contains extensions, we have to # determine that from filename. we assume the extension is @@ -841,7 +841,7 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, package, all elif check_by_extension and extension in allowlist: log.debug("%s is found in allowlist, so keeping it: %s", extension, full_path) else: - print_name = filename if extension_based[package] else basename + print_name = filename if extension_based[pkg_name] else basename log.debug("%s is not found in allowlist, so replacing it with symlink: %s", print_name, full_path) # the host_injections path is under a fixed repo/location for CUDA or cuDNN From 1f1eada7ba6535c1eb37a1cac6c249222d2700c0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 21:30:42 +0200 Subject: [PATCH 1363/1795] explain use of rstrip --- eb_hooks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 6a4fdfe54b..0e9bbcfc90 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -786,8 +786,11 @@ def post_postproc_cudnn(self, *args, **kwargs): # remove search string, split into words, remove trailing # dots '.' and only retain words starting with a dot '.' distributable = line[len(search_string):] + # distributable looks like ' the runtime files .so and .dll.' + # note the '.' after '.dll' for word in distributable.split(): if word[0] == '.': + # rstrip is used to remove the '.' after '.dll' allowlist.append(word.rstrip('.')) allowlist = sorted(set(allowlist)) From fc00d0cf758e6f4b250cbbd3e33a2db53f046740 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 21:45:53 +0200 Subject: [PATCH 1364/1795] raise error if search string wasn't found --- eb_hooks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 0e9bbcfc90..0f3f1a593d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -780,9 +780,11 @@ def post_postproc_cudnn(self, *args, **kwargs): # read cuDNN LICENSE, construct allowlist based on section "2. Distribution" that specifies list of files that can be shipped license_path = os.path.join(self.installdir, 'LICENSE') search_string = "2. Distribution. The following portions of the SDK are distributable under the Agreement:" + found_search_string = False with open(license_path) as infile: for line in infile: if line.strip().startswith(search_string): + found_search_string = True # remove search string, split into words, remove trailing # dots '.' and only retain words starting with a dot '.' distributable = line[len(search_string):] @@ -792,6 +794,11 @@ def post_postproc_cudnn(self, *args, **kwargs): if word[0] == '.': # rstrip is used to remove the '.' after '.dll' allowlist.append(word.rstrip('.')) + if not found_search_string: + # search string wasn't found in LICENSE file + raise EasyBuildError("search string '%s' was not found in license file '%s';" + "hence installation may be replaced by symlinks only", + search_string, license_path) allowlist = sorted(set(allowlist)) self.log.info("Allowlist for files in cuDNN installation that can be redistributed: " + ', '.join(allowlist)) From e968608a8d9478468315fc90dc373dcdf875a28e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 3 Oct 2024 21:52:17 +0200 Subject: [PATCH 1365/1795] improved docstring --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0f3f1a593d..85cb739391 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -873,8 +873,8 @@ def replace_non_distributable_files_with_symlinks(log, install_dir, pkg_name, al def inject_gpu_property(ec): """ - Add 'gpu' property EESSIVERSION envvars and drop dependencies to - build dependencies, via modluafooter easyconfig parameter + Add 'gpu' property and EESSIVERSION envvars via modluafooter + easyconfig parameter, and drop dependencies to build dependencies """ ec_dict = ec.asdict() # Check if CUDA, cuDNN, you-name-it is in the dependencies, if so From bcaeec7737d6eb8679483f014c2b01c5a203c3c0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 4 Oct 2024 07:38:05 +0200 Subject: [PATCH 1366/1795] Move file to correct location --- .../2023.06 => }/eessi-2023.06-eb-4.9.4-2023a.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{easystacks/software.eessi.io/2023.06 => }/eessi-2023.06-eb-4.9.4-2023a.yml (100%) diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml rename to easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml From a34ecc5bbec7845d5acfef9bfd2eb60e8360599f Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 4 Oct 2024 06:34:47 +0000 Subject: [PATCH 1367/1795] {2023.06}[foss/2022b] ROOT v6.26.10 --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 016ddc6c0b..969b0d469b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -44,3 +44,9 @@ easyconfigs: - dask-2023.7.1-foss-2022b.eb - netcdf4-python-1.6.3-foss-2022b.eb - Ruby-3.2.2-GCCcore-12.2.0.eb + - ROOT-6.26.10-foss-2022b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From 1afc5995b7f53a11f8f196d1f0a7c764c348b755 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:01:02 +0200 Subject: [PATCH 1368/1795] Guard against `$EESSI_COMPAT_LAYER` being undefined Co-authored-by: Kenneth Hoste --- init/minimal_eessi_env | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 4f2d52a0fb..5e513c3c9f 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -20,4 +20,9 @@ fi export EESSI_CPU_FAMILY=$(uname -m) # set $EPREFIX since that is basically a standard in Gentoo Prefix -export EPREFIX=$EESSI_COMPAT_LAYER_DIR +# if $EESSI_COMPAT_LAYER_DIR is defined (for example by run_in_compat_layer_env.sh script), we use that value +if [ ! -z ${EESSI_COMPAT_LAYER_DIR} ]; then + export EPREFIX=$EESSI_COMPAT_LAYER_DIR +else + export EPREFIX=$EESSI_PREFIX/compat/$EESSI_OS_TYPE/$EESSI_CPU_FAMILY +fi From 211840f3b89e8b26b8180ba3a5012c76fbf2c527 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 4 Oct 2024 13:03:06 +0200 Subject: [PATCH 1369/1795] `$topdir` -> `$software_layer_dir` --- bot/build.sh | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index be0f34e92e..3fd343e96f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -22,10 +22,10 @@ set -e # Make sure we are referring to software-layer as working directory -topdir=$(dirname $(dirname $(realpath $0))) +software_layer_dir=$(dirname $(dirname $(realpath $0))) # source utils.sh and cfg_files.sh -source $topdir/scripts/utils.sh -source $topdir/scripts/cfg_files.sh +source $software_layer_dir/scripts/utils.sh +source $software_layer_dir/scripts/cfg_files.sh # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=cfg/job.cfg}" @@ -216,10 +216,10 @@ else removal_outerr=$(mktemp remove.outerr.XXXX) echo "Executing command to remove software:" - echo "$topdir/eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" - echo " -- $topdir/EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" - $topdir/eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ - -- $topdir/EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} + echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" + echo " -- $software_layer_dir/EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" + $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ + -- $software_layer_dir/EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} # make sure that the build step resumes from the same temporary directory # this is important, as otherwise the removed software will still be there @@ -252,10 +252,10 @@ fi build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" -echo "$topdir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" -echo " -- $topdir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" -$topdir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ - -- $topdir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} +echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" +echo " -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}" +$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ + -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr} # prepare directory to store tarball of tmp for tarball step TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step @@ -280,7 +280,7 @@ fi timestamp=$(date +%s) # to set EESSI_VERSION we need to source init/eessi_defaults now -source $topdir/init/eessi_defaults +source $software_layer_dir/init/eessi_defaults export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be @@ -289,9 +289,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI # /tmp as default? TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" -echo "$topdir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- $topdir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" -$topdir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $topdir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} +echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" +echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 From b7565c987a0663f323b3ede79f8dce526252e512 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 4 Oct 2024 18:02:19 +0200 Subject: [PATCH 1370/1795] {2023.6}[2023b,a64fx] SciPy-bundle 2023.11 + GROMACS 2024.1 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml new file mode 100644 index 0000000000..7fea9fe41e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml @@ -0,0 +1,3 @@ +easyconfigs: + - SciPy-bundle-2023.11-gfbf-2023b.eb + - GROMACS-2024.1-foss-2023b.eb From 97d5b67f0d0306f084039062d8ce17ea2127c3f4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 4 Oct 2024 20:24:53 +0200 Subject: [PATCH 1371/1795] use TMPDIR as base for temporary storage --- EESSI-install-software.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index d54da4a404..7821655c07 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -249,10 +249,12 @@ else export skip_cuda_install=True fi +temp_install_storage=${TMPDIR}/temp_install_storage +mkdir -p ${temp_install_storage} if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \ -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml \ - -t /tmp/temp \ + -t ${temp_install_storage} \ --accept-cuda-eula else echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found" From 8e7a0e80ab3cefa1f3c0c2b7fbce71f2206f6102 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 4 Oct 2024 20:47:02 +0200 Subject: [PATCH 1372/1795] attempt to use a single easystack file for CUDA/cu* packages --- EESSI-install-software.sh | 7 ++++++- .../nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml | 1 + install_scripts.sh | 12 +++++++++++- .../nvidia/eessi-2023.06-cuda-and-libraries.yml | 3 --- 4 files changed, 18 insertions(+), 5 deletions(-) delete mode 100644 scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7821655c07..4c85fdfc31 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -251,9 +251,14 @@ fi temp_install_storage=${TMPDIR}/temp_install_storage mkdir -p ${temp_install_storage} +# Note the eessi...CUDA.yml file(s) is(are) copied by 'install_scripts.sh' from +# the EESSI/software-layer easystacks/software.eessi.io/2023.06/accel/nvidia +# directory to /cvmfs to avoid keeping them in sync manually. If more than one +# such file is used (e.g., because different EasyBuild versions were used), the +# install script 'install_cuda_and_libraries.sh' has to be run multiple times. if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \ - -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml \ + -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml \ -t ${temp_install_storage} \ --accept-cuda-eula else diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml index d54780804b..873c19aa33 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml @@ -1,2 +1,3 @@ easyconfigs: + - CUDA-12.1.1.eb - cuDNN-8.9.2.26-CUDA-12.1.1.eb diff --git a/install_scripts.sh b/install_scripts.sh index df9bda3ad3..ad73e769dd 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -122,13 +122,23 @@ copy_files_by_list ${TOPDIR}/scripts ${INSTALL_PREFIX}/scripts "${script_files[@ # Copy files for the scripts/gpu_support/nvidia directory nvidia_files=( - eessi-2023.06-cuda-and-libraries.yml install_cuda_and_libraries.sh install_cuda_host_injections.sh link_nvidia_host_libraries.sh ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" +# special treatment for the easystack file(s) that lists CUDA and cu* libraries +# To be picked up by a build job they have to be stored under +# easystacks/software.eessi.io/2023.06/accel/nvidia/ on GitHub. +# To avoid keeping that file and the one that we distribute via CernVM-FS so +# users/sites can install the full CUDA SDK and cu* libraries under +# 'host_injections' we copy the above file to the right location under /cvmfs. +nvidia_host_injections_files=( + eessi-2023.06-eb-4.9.4-2023a-CUDA.yml +) +copy_files_by_list ${TOPDIR}/easystacks/software.eessi.io/2023.06/accel/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_host_injections_files[@]}" + # Copy over EasyBuild hooks file used for installations hook_files=( eb_hooks.py diff --git a/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml b/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml deleted file mode 100644 index e0e47bf2d8..0000000000 --- a/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml +++ /dev/null @@ -1,3 +0,0 @@ -easyconfigs: - - CUDA-12.1.1.eb - - cuDNN-8.9.2.26-CUDA-12.1.1.eb From 57f5a485796a859bada8751f70319319f4e27e75 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 4 Oct 2024 21:19:40 +0200 Subject: [PATCH 1373/1795] various improvements for inject_gpu_property --- eb_hooks.py | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 85cb739391..3179ac170f 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -881,39 +881,44 @@ def inject_gpu_property(ec): # - drop dependency to build dependency # - add 'gpu' Lmod property # - add envvar with package version - packages_list = ( "CUDA", "cuDNN" ) - packages_version = { } + pkg_names = ( "CUDA", "cuDNN" ) + pkg_versions = { } add_gpu_property = '' - for package in packages_list: - # Check if package is in the dependencies, if so drop dependency to build + for pkg_name in pkg_names: + # Check if pkg_name is in the dependencies, if so drop dependency to build # dependency and set variable for later adding the 'gpu' Lmod property - if (package in [dep[0] for dep in iter(ec_dict['dependencies'])]): + # to '.remove' dependencies from ec_dict['dependencies'] we make a copy, + # iterate over the copy and can then savely use '.remove' on the original + # ec_dict['dependencies']. + deps = ec_dict['dependencies'][:] + if (pkg_name in [dep[0] for dep in deps]): add_gpu_property = 'add_property("arch","gpu")' - for dep in iter(ec_dict['dependencies']): - if package in dep[0]: - # make package a build dependency only (rpathing saves us from link errors) - ec.log.info("Dropping dependency on %s to build dependency" % package) + for dep in deps: + if pkg_name == dep[0]: + # make pkg_name a build dependency only (rpathing saves us from link errors) + ec.log.info("Dropping dependency on %s to build dependency" % pkg_name) ec_dict['dependencies'].remove(dep) if dep not in ec_dict['builddependencies']: ec_dict['builddependencies'].append(dep) # take note of version for creating the modluafooter - packages_version[package] = dep[1] + pkg_versions[pkg_name] = dep[1] if add_gpu_property: ec.log.info("Injecting gpu as Lmod arch property and envvars for dependencies with their version") - key = 'modluafooter' - values = [add_gpu_property] - for package, version in packages_version.items(): - envvar = "EESSI%sVERSION" % package.upper() - values.append('setenv("%s","%s")' % (envvar, version)) - if not key in ec_dict: - ec[key] = '\n'.join(values) + modluafooter = 'modluafooter' + extra_mod_footer_lines = [add_gpu_property] + for pkg_name, version in pkg_versions.items(): + envvar = "EESSI%sVERSION" % pkg_name.upper() + extra_mod_footer_lines.append('setenv("%s","%s")' % (envvar, version)) + # take into account that modluafooter may already be set + if modluafooter in ec_dict: + value = ec_dict[modluafooter] + for line in extra_mod_footer_lines: + if not line in value: + value = '\n'.join([value, line]) + ec[modluafooter] = value else: - new_value = ec_dict[key] - for value in values: - if not value in new_value: - new_value = '\n'.join([new_value, value]) - ec[key] = new_value + ec[modluafooter] = '\n'.join(extra_mod_footer_lines) return ec From 7374ee719151c8eb7cfb4a5fc61593ace5745e6b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 4 Oct 2024 21:53:51 +0200 Subject: [PATCH 1374/1795] Fix quote location and change null characters back to newlines --- bot/check-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 1b973fcb3d..3e1209781f 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -77,7 +77,8 @@ if [[ ! -z ${grep_reframe_failed} ]]; then else # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' - grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success_full} | tr -d '\0'") + # tr '\0' '\n' places back the newline characters that the -z option of grep turned into null characters + grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success_full}" | tr '\0' '\n') grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" fi From 21ffc180068a967f28ad9cbd4b3d4e6f60b4407c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 4 Oct 2024 22:02:53 +0200 Subject: [PATCH 1375/1795] various improvements for install_cuda_and_libraries.sh --- .../nvidia/install_cuda_and_libraries.sh | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 2fea64d7a6..e13b9ad386 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -27,7 +27,7 @@ show_help() { echo " --accept-cuda-eula You _must_ accept the CUDA EULA to install" echo " CUDA, see the EULA at" echo " https://docs.nvidia.com/cuda/eula/index.html" - echo " -e, --easystack EASYSTACKFILE Path to easystack file that defines which" + echo " -e, --easystack EASYSTACK_FILE Path to easystack file that defines which" echo " packages shall be installed" echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" echo " storage during the installation of CUDA" @@ -37,7 +37,7 @@ show_help() { # Initialize variables eula_accepted=0 -EASYSTACKFILE= +EASYSTACK_FILE= TEMP_DIR= # Parse command-line options @@ -53,7 +53,7 @@ while [[ $# -gt 0 ]]; do ;; -e|--easystack) if [ -n "$2" ]; then - EASYSTACKFILE="$2" + EASYSTACK_FILE="$2" shift 2 else echo "Error: Argument required for $1" @@ -78,7 +78,7 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -z "${EASYSTACKFILE}" ]]; then +if [[ -z "${EASYSTACK_FILE}" ]]; then fatal_error "Need the name/path to an easystack file. See command line options\n" fi @@ -102,21 +102,24 @@ fi echo "Created temporary directory '${tmpdir}'" # workaround for EasyBuild not being found when loading "extend" module -module load EasyBuild/4.9.4 +# module load EasyBuild/4.9.4 # load EESSI-extend/2023.06-easybuild module && verify that it is loaded -EESSI_EXTEND_MODULE="EESSI-extend/2023.06-easybuild" +EESSI_EXTEND_MODULE="EESSI-extend/${EESSI_VERSION}-easybuild" module load ${EESSI_EXTEND_MODULE} ret=$? if [ "${ret}" -ne 0 ]; then fatal_error "An error occured while trying to load ${EESSI_EXTEND_MODULE}\n" fi -# do a 'eb --dry-run-short' with the EASYSTACKFILE and determine list of packages +# show EasyBuild configuration +eb --show-config + +# do a 'eb --dry-run-short' with the EASYSTACK_FILE and determine list of packages # to be installed -echo ">> Determining if packages specified in ${EASYSTACKFILE} are missing under ${EESSI_SITE_INSTALL}" +echo ">> Determining if packages specified in ${EASYSTACK_FILE} are missing under ${EESSI_SITE_INSTALL}" eb_dry_run_short_out=${tmpdir}/eb_dry_run_short.out -eb --dry-run-short --rebuild --easystack ${EASYSTACKFILE} 2>&1 | tee ${eb_dry_run_short_out} +eb --dry-run-short --rebuild --easystack ${EASYSTACK_FILE} 2>&1 | tee ${eb_dry_run_short_out} ret=$? # Check if CUDA shall be installed @@ -136,7 +139,7 @@ fi # determine the number of packages to be installed (assume 5 GB + num_packages * # 3GB space needed) -number_of_packages=$(cat ${eb_dry_run_short_out} | grep "^ \* \[[xR]\]" | sed -e 's/^.*module: //' | uniq | wc -l) +number_of_packages=$(cat ${eb_dry_run_short_out} | grep "^ \* \[[xR]\]" | sed -e 's/^.*module: //' | sort -u | wc -l) echo "number of packages to be (re-)installed: '${number_of_packages}'" base_storage_space=$((5000000 + ${number_of_packages} * 3000000)) @@ -176,28 +179,28 @@ fi # and/or other installation in the `.../versions/...` prefix # - installpath-modules: We install the module in our `tmpdir` since we do not need the modulefile, # we only care about providing the targets for the symlinks. -# - ${cuda_arg}: We only set the --accept-eula-for=CUDA option if CUDA will be installed and if +# - ${accept_eula_opt}: We only set the --accept-eula-for=CUDA option if CUDA will be installed and if # this script was called with the argument --accept-cuda-eula. # - hooks: We don't want hooks used in this install, we need vanilla # installations of CUDA and/or other libraries # - easystack: Path to easystack file that defines which packages shall be # installed -cuda_arg= +accept_eula_opt= if [[ ${eula_accepted} -eq 1 ]]; then - cuda_arg="--accept-eula-for=CUDA" + accept_eula_opt="--accept-eula-for=CUDA" fi touch "$tmpdir"/none.py eb --prefix="$tmpdir" \ --rebuild \ - --installpath-modules=${tmpdir} \ - "${cuda_arg}" \ + --installpath-modules=${tmpdir}/modules \ + "${accept_eula_opt}" \ --hooks="$tmpdir"/none.py \ - --easystack ${EASYSTACKFILE} + --easystack ${EASYSTACK_FILE} ret=$? if [ $ret -ne 0 ]; then eb_last_log=$(unset EB_VERBOSE; eb --last-log) cp -a ${eb_last_log} . - fatal_error "some installation failed, please check EasyBuild logs $(basename ${eb_last_log})..." + fatal_error "some installation failed, please check EasyBuild logs ${PWD}/$(basename ${eb_last_log})..." else echo_green "all installations at ${EESSI_SITE_INSTALL}/software/... succeeded!" fi From a3edc20e439f4916283da0421727efb470d729c0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 4 Oct 2024 22:28:00 +0200 Subject: [PATCH 1376/1795] show available *CUDA* modules for easier debugging --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index e13b9ad386..7a8d1d74ba 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -101,8 +101,8 @@ else fi echo "Created temporary directory '${tmpdir}'" -# workaround for EasyBuild not being found when loading "extend" module -# module load EasyBuild/4.9.4 +echo "List available *CUDA* modules before loading EESSI-extend/${EESSI_VERSION}-easybuild" +module avail CUDA # load EESSI-extend/2023.06-easybuild module && verify that it is loaded EESSI_EXTEND_MODULE="EESSI-extend/${EESSI_VERSION}-easybuild" @@ -112,7 +112,11 @@ if [ "${ret}" -ne 0 ]; then fatal_error "An error occured while trying to load ${EESSI_EXTEND_MODULE}\n" fi +echo "List available *CUDA* modules after loading EESSI-extend/${EESSI_VERSION}-easybuild" +module avail CUDA + # show EasyBuild configuration +echo "Show EasyBuild configuration" eb --show-config # do a 'eb --dry-run-short' with the EASYSTACK_FILE and determine list of packages From c764572b9cc68a0969efeba80fa35f0e89426a72 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 4 Oct 2024 22:29:21 +0200 Subject: [PATCH 1377/1795] Add newline char --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 3e1209781f..bd703d9ce7 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -78,7 +78,7 @@ else # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' # tr '\0' '\n' places back the newline characters that the -z option of grep turned into null characters - grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success_full}" | tr '\0' '\n') + grep_reframe_success_full=$(echo -e "$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success_full}" | tr '\0' '\n')\n") grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" fi From 7f601dc60a1ec221eed71a874e1a30d48bfb3cce Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 4 Oct 2024 22:50:01 +0200 Subject: [PATCH 1378/1795] print and adjust MODULEPATH --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 7a8d1d74ba..b36eeafd29 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -101,6 +101,7 @@ else fi echo "Created temporary directory '${tmpdir}'" +echo "MODULEPATH=${MODULEPATH}" echo "List available *CUDA* modules before loading EESSI-extend/${EESSI_VERSION}-easybuild" module avail CUDA @@ -112,9 +113,15 @@ if [ "${ret}" -ne 0 ]; then fatal_error "An error occured while trying to load ${EESSI_EXTEND_MODULE}\n" fi +echo "MODULEPATH=${MODULEPATH}" echo "List available *CUDA* modules after loading EESSI-extend/${EESSI_VERSION}-easybuild" module avail CUDA +# use install_path/modules/all as MODULEPATH +SAVE_MODULEPATH=${MODULEPATH} +MODULEPATH=${EASYBUILD_INSTALLPATH}/modules/all +echo "set MODULEPATH=${MODULEPATH}" + # show EasyBuild configuration echo "Show EasyBuild configuration" eb --show-config From e3101b0c82d554823b33d9cab2abee074e330222 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 4 Oct 2024 23:26:47 +0200 Subject: [PATCH 1379/1795] implement option 3 to install module files in hidden directory --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index b36eeafd29..40bff34bb7 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -119,7 +119,7 @@ module avail CUDA # use install_path/modules/all as MODULEPATH SAVE_MODULEPATH=${MODULEPATH} -MODULEPATH=${EASYBUILD_INSTALLPATH}/modules/all +MODULEPATH=${EASYBUILD_INSTALLPATH}/.modules/all echo "set MODULEPATH=${MODULEPATH}" # show EasyBuild configuration @@ -150,7 +150,7 @@ fi # determine the number of packages to be installed (assume 5 GB + num_packages * # 3GB space needed) -number_of_packages=$(cat ${eb_dry_run_short_out} | grep "^ \* \[[xR]\]" | sed -e 's/^.*module: //' | sort -u | wc -l) +number_of_packages=$(cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | sed -e 's/^.*module: //' | sort -u | wc -l) echo "number of packages to be (re-)installed: '${number_of_packages}'" base_storage_space=$((5000000 + ${number_of_packages} * 3000000)) @@ -203,7 +203,7 @@ fi touch "$tmpdir"/none.py eb --prefix="$tmpdir" \ --rebuild \ - --installpath-modules=${tmpdir}/modules \ + --installpath-modules=${EASYBUILD_INSTALLPATH}/.modules \ "${accept_eula_opt}" \ --hooks="$tmpdir"/none.py \ --easystack ${EASYSTACK_FILE} From 8e0560a6707841bdaba79e457e3646d85721e905 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 4 Oct 2024 23:31:48 +0200 Subject: [PATCH 1380/1795] Make sure all null and new-line characters are replaced by
    --- bot/check-test.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index bd703d9ce7..d439fc6380 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -78,7 +78,12 @@ else # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' # tr '\0' '\n' places back the newline characters that the -z option of grep turned into null characters - grep_reframe_success_full=$(echo -e "$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success_full}" | tr '\0' '\n')\n") + grep_reframe_success_full=$( \ + grep -v "^>> searching for " ${job_dir}/${job_out} | \ + grep -Pzo "${GP_success_full}" | \ + sed 's/\x00//g' | + sed ':a;N;$!ba;s/\n//g' \ + ) grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" fi From 5e74d567141c791e36014b225aa37b793b2c22ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 5 Oct 2024 09:33:36 +0200 Subject: [PATCH 1381/1795] remove GROMACS 2024.1 for now, since there's a mysterious build error --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml index 7fea9fe41e..11523e7f03 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml @@ -1,3 +1,2 @@ easyconfigs: - SciPy-bundle-2023.11-gfbf-2023b.eb - - GROMACS-2024.1-foss-2023b.eb From 24f0620e2b0df2af156df907fcd32a873ded5a35 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Oct 2024 13:45:58 +0200 Subject: [PATCH 1382/1795] add CI workflow for testing NVIDIA GPU accelerator detection --- .../workflows/tests_archdetect_nvidia_gpu.yml | 132 ++++++++++++++++++ tests/archdetect/nvidia-smi/1xa100.output | 1 + tests/archdetect/nvidia-smi/1xa100.sh | 5 + tests/archdetect/nvidia-smi/2xa100.output | 1 + tests/archdetect/nvidia-smi/2xa100.sh | 6 + tests/archdetect/nvidia-smi/4xa100.output | 1 + tests/archdetect/nvidia-smi/4xa100.sh | 8 ++ tests/archdetect/nvidia-smi/cc01.output | 1 + tests/archdetect/nvidia-smi/cc01.sh | 6 + tests/archdetect/nvidia-smi/no_devices.output | 1 + tests/archdetect/nvidia-smi/no_devices.sh | 3 + tests/archdetect/nvidia-smi/none.output | 1 + 12 files changed, 166 insertions(+) create mode 100644 .github/workflows/tests_archdetect_nvidia_gpu.yml create mode 100644 tests/archdetect/nvidia-smi/1xa100.output create mode 100755 tests/archdetect/nvidia-smi/1xa100.sh create mode 100644 tests/archdetect/nvidia-smi/2xa100.output create mode 100755 tests/archdetect/nvidia-smi/2xa100.sh create mode 100644 tests/archdetect/nvidia-smi/4xa100.output create mode 100755 tests/archdetect/nvidia-smi/4xa100.sh create mode 100644 tests/archdetect/nvidia-smi/cc01.output create mode 100755 tests/archdetect/nvidia-smi/cc01.sh create mode 100644 tests/archdetect/nvidia-smi/no_devices.output create mode 100755 tests/archdetect/nvidia-smi/no_devices.sh create mode 100644 tests/archdetect/nvidia-smi/none.output diff --git a/.github/workflows/tests_archdetect_nvidia_gpu.yml b/.github/workflows/tests_archdetect_nvidia_gpu.yml new file mode 100644 index 0000000000..d8b5adeb88 --- /dev/null +++ b/.github/workflows/tests_archdetect_nvidia_gpu.yml @@ -0,0 +1,132 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for accelerator detection (NVIDIA GPU) +on: + push: + pull_request: +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + fake_nvidia_smi_script: + - none # no nvidia-smi command + - no_devices # nvidia-smi command works, but no GPUs available + - 1xa100 # cc80, supported with (atleast) zen2 CPU + - 2xa100 # cc80, supported with (atleast) zen2 CPU + - 4xa100 # cc80, supported with (atleast) zen2 CPU + - cc01 # non-existing GPU + fail-fast: false + steps: + - name: checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + # we deliberately do not use the eessi/github-action-eessi action, + # because we want to control when the EESSI environment is initialized + - name: Mount EESSI CernVM-FS repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + + - name: test accelerator detection + run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/amd/zen2' + + # put fake nvidia-smi command in place (unless we don't want to) + if [[ "${{matrix.fake_nvidia_smi_script}}" != "none" ]]; then + tmpdir=$(mktemp -d) + ln -s $PWD/tests/archdetect/nvidia-smi/${{matrix.fake_nvidia_smi_script}}.sh $tmpdir/nvidia-smi + export PATH=$tmpdir:$PATH + fi + + # first run with debugging enabled, just to show the output + ./init/eessi_archdetect.sh -d accelpath || echo "non-zero exit code: $?" + + # verify output (or exit code if non-zero) + out=$(./init/eessi_archdetect.sh accelpath || echo "non-zero exit code: $?") + + if [[ $out == "$( cat ./tests/archdetect/nvidia-smi/${{matrix.fake_nvidia_smi_script}}.output )" ]]; then + + echo "Test for '${{matrix.fake_nvidia_smi_script}}' PASSED: '$out'" + + # by default the 'errexit' option is enabled (set -e), + # which causes trouble when 'eessi_archdetect.sh accelpath' + # fails to detect an accelerator and produces a non-zero exit code, + # so we have to unset it using 'set +e' + echo "set flags: $-" + set +e + echo "set flags after unsetting errexit option: $-" + + # run full EESSI init script, which pick up on the accelerator (if available) + echo + . init/bash 2>&1 | tee init.out + echo "-----------------------------------------------------------------------------" + + if [[ "${{matrix.fake_nvidia_smi_script}}" == "none" ]] || [[ "${{matrix.fake_nvidia_smi_script}}" == "no_devices" ]]; then + + pattern="archdetect could not detect any accelerators" + echo ">>> checking for pattern '${pattern}' in init output..." + grep "${pattern}" init.out || (echo "FAILED 1" || exit 1) + + pattern="archdetect found supported accelerator" + echo ">>> checking for lack of pattern '${pattern}' in init output..." + match=$(grep "${pattern}" init.out || true) + test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1) + + pattern="Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/.*/accel/.*/modules/all to \$MODULEPATH" + echo ">>> checking for lack of pattern '${pattern}' in init output..." + match=$(grep "${pattern}" init.out || true) + test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1) + + elif [[ "${{matrix.fake_nvidia_smi_script}}" == "cc01" ]]; then + + pattern="No matching path found in x86_64/amd/zen2 for accelerator detected by archdetect (accel/nvidia/cc01)" + echo ">>> checking for pattern '${pattern}' in init output..." + grep "${pattern}" init.out || (echo "FAILED 1" || exit 1) + + pattern="Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/.*/accel/.*/modules/all to \$MODULEPATH" + echo ">>> checking for lack of pattern '${pattern}' in init output..." + match=$(grep "${pattern}" init.out || true) + test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1) + + else + echo ">>> checking for 'accel/nvidia/cc80' in init output..." + grep "archdetect found supported accelerator for CPU target x86_64/amd/zen2: accel/nvidia/cc80" init.out || (echo "FAILED 2" && exit 1) + grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1) + fi + + echo ">>> checking last line of init output..." + tail -1 init.out | grep "Environment set up to use EESSI (2023.06), have fun!" || (echo "FAILED, full init utput:" && cat init.out && exit 1) + + echo "All checks on init output PASSED" + else + echo "Test for '${{matrix.fake_nvidia_smi_script}}' FAILED: '$out'" >&2 + exit 1 + fi + + - name: test accelerator detection under $EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE + $EESSI_ACCELERATOR_TARGET_OVERRIDE + run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/amd/zen2' + export EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE='x86_64/amd/zen3' + export EESSI_ACCELERATOR_TARGET_OVERRIDE='accel/nvidia/cc80' + + # first run with debugging enabled, just to show the output + ./init/eessi_archdetect.sh -d accelpath || echo "non-zero exit code: $?" + + # verify output (or exit code if non-zero) + out=$(./init/eessi_archdetect.sh accelpath || echo "non-zero exit code: $?") + + echo + . init/bash 2>&1 | tee init.out + echo "-----------------------------------------------------------------------------" + + echo ">>> checking for 'accel/nvidia/cc80' in init output..." + grep "archdetect found supported accelerator for CPU target x86_64/amd/zen3: accel/nvidia/cc80" init.out || (echo "FAILED 1" && exit 1) + grep "Using x86_64/amd/zen2 as software subdirectory" init.out || (echo "FAILED 2" && exit 1) + grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1) + grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 4" && exit 1) + + echo "All checks on init output PASSED" diff --git a/tests/archdetect/nvidia-smi/1xa100.output b/tests/archdetect/nvidia-smi/1xa100.output new file mode 100644 index 0000000000..5eb3aaff18 --- /dev/null +++ b/tests/archdetect/nvidia-smi/1xa100.output @@ -0,0 +1 @@ +accel/nvidia/cc80 diff --git a/tests/archdetect/nvidia-smi/1xa100.sh b/tests/archdetect/nvidia-smi/1xa100.sh new file mode 100755 index 0000000000..ead191418b --- /dev/null +++ b/tests/archdetect/nvidia-smi/1xa100.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# output from NVIDIA A100 system, +# produced by: nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader +echo "NVIDIA A100-SXM4-80GB, 1, 545.23.08, 8.0" +exit 0 diff --git a/tests/archdetect/nvidia-smi/2xa100.output b/tests/archdetect/nvidia-smi/2xa100.output new file mode 100644 index 0000000000..5eb3aaff18 --- /dev/null +++ b/tests/archdetect/nvidia-smi/2xa100.output @@ -0,0 +1 @@ +accel/nvidia/cc80 diff --git a/tests/archdetect/nvidia-smi/2xa100.sh b/tests/archdetect/nvidia-smi/2xa100.sh new file mode 100755 index 0000000000..5539607fbe --- /dev/null +++ b/tests/archdetect/nvidia-smi/2xa100.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# output from NVIDIA A100 system, +# produced by: nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader +echo "NVIDIA A100-SXM4-80GB, 2, 545.23.08, 8.0" +echo "NVIDIA A100-SXM4-80GB, 2, 545.23.08, 8.0" +exit 0 diff --git a/tests/archdetect/nvidia-smi/4xa100.output b/tests/archdetect/nvidia-smi/4xa100.output new file mode 100644 index 0000000000..5eb3aaff18 --- /dev/null +++ b/tests/archdetect/nvidia-smi/4xa100.output @@ -0,0 +1 @@ +accel/nvidia/cc80 diff --git a/tests/archdetect/nvidia-smi/4xa100.sh b/tests/archdetect/nvidia-smi/4xa100.sh new file mode 100755 index 0000000000..45458ea7bd --- /dev/null +++ b/tests/archdetect/nvidia-smi/4xa100.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# output from NVIDIA A100 system, +# produced by: nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader +echo "NVIDIA A100-SXM4-80GB, 4, 545.23.08, 8.0" +echo "NVIDIA A100-SXM4-80GB, 4, 545.23.08, 8.0" +echo "NVIDIA A100-SXM4-80GB, 4, 545.23.08, 8.0" +echo "NVIDIA A100-SXM4-80GB, 4, 545.23.08, 8.0" +exit 0 diff --git a/tests/archdetect/nvidia-smi/cc01.output b/tests/archdetect/nvidia-smi/cc01.output new file mode 100644 index 0000000000..9cbf66a131 --- /dev/null +++ b/tests/archdetect/nvidia-smi/cc01.output @@ -0,0 +1 @@ +accel/nvidia/cc01 diff --git a/tests/archdetect/nvidia-smi/cc01.sh b/tests/archdetect/nvidia-smi/cc01.sh new file mode 100755 index 0000000000..81011a1d16 --- /dev/null +++ b/tests/archdetect/nvidia-smi/cc01.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# output from non-existing NVIDIA GPU system, +# to test handling of unknown GPU model +# (supposedly) produced by: nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader +echo "NVIDIA does-not-exist, 1, 000.00.00, 0.1" +exit 0 diff --git a/tests/archdetect/nvidia-smi/no_devices.output b/tests/archdetect/nvidia-smi/no_devices.output new file mode 100644 index 0000000000..b251bfc837 --- /dev/null +++ b/tests/archdetect/nvidia-smi/no_devices.output @@ -0,0 +1 @@ +non-zero exit code: 3 diff --git a/tests/archdetect/nvidia-smi/no_devices.sh b/tests/archdetect/nvidia-smi/no_devices.sh new file mode 100755 index 0000000000..0bc26dcddc --- /dev/null +++ b/tests/archdetect/nvidia-smi/no_devices.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo "No devices were found" +exit 6 diff --git a/tests/archdetect/nvidia-smi/none.output b/tests/archdetect/nvidia-smi/none.output new file mode 100644 index 0000000000..e287574cc3 --- /dev/null +++ b/tests/archdetect/nvidia-smi/none.output @@ -0,0 +1 @@ +non-zero exit code: 2 From 3bf3b12e8536e04eaa4027ceaaf68782f41b3733 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Oct 2024 15:15:24 +0200 Subject: [PATCH 1383/1795] temporarily disable errexit shell option when calling out to archdetect to detect accelerator --- .../workflows/tests_archdetect_nvidia_gpu.yml | 10 +--------- init/eessi_environment_variables | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests_archdetect_nvidia_gpu.yml b/.github/workflows/tests_archdetect_nvidia_gpu.yml index d8b5adeb88..8ad5f4fb36 100644 --- a/.github/workflows/tests_archdetect_nvidia_gpu.yml +++ b/.github/workflows/tests_archdetect_nvidia_gpu.yml @@ -52,14 +52,6 @@ jobs: echo "Test for '${{matrix.fake_nvidia_smi_script}}' PASSED: '$out'" - # by default the 'errexit' option is enabled (set -e), - # which causes trouble when 'eessi_archdetect.sh accelpath' - # fails to detect an accelerator and produces a non-zero exit code, - # so we have to unset it using 'set +e' - echo "set flags: $-" - set +e - echo "set flags after unsetting errexit option: $-" - # run full EESSI init script, which pick up on the accelerator (if available) echo . init/bash 2>&1 | tee init.out @@ -95,7 +87,7 @@ jobs: else echo ">>> checking for 'accel/nvidia/cc80' in init output..." grep "archdetect found supported accelerator for CPU target x86_64/amd/zen2: accel/nvidia/cc80" init.out || (echo "FAILED 2" && exit 1) - grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1) + grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1) fi echo ">>> checking last line of init output..." diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index fb1b731ee1..ab4894fb16 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -38,12 +38,27 @@ if [ -d $EESSI_PREFIX ]; then break fi done + + # we need to make sure that errexit shell option (set -e) is not enabled, + # since archdetect will produce non-zero exit code if no accelerator was found + if [[ "$-" =~ e ]]; then + errexit_shell_option_set='yes' + set +e + else + errexit_shell_option_set='no' + fi + # to be able to grab exit code of archdetect trying to detect accelerators, # we can not run it via $(...), so we have to redirect the output to a temporary file tmpout=$(mktemp) ${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh accelpath 2>&1 > $tmpout - ec=$? - if [[ $ec -eq 0 ]]; then + accelpath_exit_code=$? + + if [[ "$errexit_shell_option_set" == "yes" ]]; then + set -e + fi + + if [[ $accelpath_exit_code -eq 0 ]]; then export EESSI_ACCEL_SUBDIR=$(tail -1 $tmpout && rm -f $tmpout) if [ -z ${EESSI_ACCEL_SUBDIR} ]; then error "accelerator detection with archdetect worked, but no result was returned?!" From 6b3ada5256f27ac4abc47330ee1927d015f54bb6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 7 Oct 2024 09:19:21 +0200 Subject: [PATCH 1384/1795] See if we can excape % sign with backslash character --- bot/check-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index d439fc6380..8e92532090 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -82,7 +82,8 @@ else grep -v "^>> searching for " ${job_dir}/${job_out} | \ grep -Pzo "${GP_success_full}" | \ sed 's/\x00//g' | - sed ':a;N;$!ba;s/\n//g' \ + sed ':a;N;$!ba;s/\n//g' | + sed 's/%/\%/g \ ) grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" From eb41886d46c90b0f7f0b8ad2f34d0fe69577d645 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 7 Oct 2024 14:31:50 +0200 Subject: [PATCH 1385/1795] Forgot closing quote --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 8e92532090..7b2e4b6a11 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -83,7 +83,7 @@ else grep -Pzo "${GP_success_full}" | \ sed 's/\x00//g' | sed ':a;N;$!ba;s/\n//g' | - sed 's/%/\%/g \ + sed 's/%/\%/g' \ ) grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" From 6cd99178accce6167018caa3c8c1016c3fc9db67 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Mon, 7 Oct 2024 17:30:32 +0200 Subject: [PATCH 1386/1795] Let ${EASYBUILD_ROBOT_PATHS} be defined earlier So that we can override it for dev.eessi.io (and other cases where we build to another repository but on top of `software.eessi.io`) --- check_missing_installations.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index d8135ea3cb..e6eb982d19 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -26,8 +26,11 @@ easystack=$1 LOCAL_TMPDIR=$(mktemp -d) # Clone the develop branch of EasyBuild and use that to search for easyconfigs -git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs -export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs + +if [[ -z ${EASYBUILD_ROBOT_PATHS} ]] + git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs + export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs +fi # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) From cf21c2536e433582cf0788086eb9472618992c6b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 7 Oct 2024 20:12:05 +0200 Subject: [PATCH 1387/1795] Escape the escape character --- bot/check-test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 7b2e4b6a11..e18ed64a84 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -81,9 +81,9 @@ else grep_reframe_success_full=$( \ grep -v "^>> searching for " ${job_dir}/${job_out} | \ grep -Pzo "${GP_success_full}" | \ - sed 's/\x00//g' | - sed ':a;N;$!ba;s/\n//g' | - sed 's/%/\%/g' \ + sed 's/\x00//g' | # Replace null character with
    + sed ':a;N;$!ba;s/\n//g' | # Replace new line characters with
    + sed 's/%/\\%/g' \ # Replace % with \%. Use \\% to interpret \ as character, instead of escape character ) grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" From ebb8dac4657aa7af9af891c75b4c08d4cf7ea1fc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 7 Oct 2024 22:02:56 +0200 Subject: [PATCH 1388/1795] Escape first percent sign once --- bot/check-test.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index e18ed64a84..cb426f7468 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -77,13 +77,15 @@ if [[ ! -z ${grep_reframe_failed} ]]; then else # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' - # tr '\0' '\n' places back the newline characters that the -z option of grep turned into null characters + # Replace null character with
    + # Replace new line characters with
    + # Replace % with \%. Use \\% to interpret \ as character, instead of escape character grep_reframe_success_full=$( \ grep -v "^>> searching for " ${job_dir}/${job_out} | \ grep -Pzo "${GP_success_full}" | \ - sed 's/\x00//g' | # Replace null character with
    - sed ':a;N;$!ba;s/\n//g' | # Replace new line characters with
    - sed 's/%/\\%/g' \ # Replace % with \%. Use \\% to interpret \ as character, instead of escape character + sed 's/\x00//g' | + sed ':a;N;$!ba;s/\n//g' | + sed 's/\%/\\%/g' \ ) grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" From e6697680f6618ac790e080665f10ba36ba20e1c3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 7 Oct 2024 23:36:18 +0200 Subject: [PATCH 1389/1795] Escaping % is done with %% --- bot/check-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index cb426f7468..1f66514bb2 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -79,13 +79,13 @@ else GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' # Replace null character with
    # Replace new line characters with
    - # Replace % with \%. Use \\% to interpret \ as character, instead of escape character + # Replace % with %%. Use \%\% to interpret both %% as (non-special) characters grep_reframe_success_full=$( \ grep -v "^>> searching for " ${job_dir}/${job_out} | \ grep -Pzo "${GP_success_full}" | \ sed 's/\x00//g' | sed ':a;N;$!ba;s/\n//g' | - sed 's/\%/\\%/g' \ + sed 's/\%/\%\%/g' \ ) grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" From 47889d36668c735b0de9f523384222819261bf3c Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:29:26 +0200 Subject: [PATCH 1390/1795] Fix syntax error --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index e6eb982d19..280de294af 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -27,7 +27,7 @@ LOCAL_TMPDIR=$(mktemp -d) # Clone the develop branch of EasyBuild and use that to search for easyconfigs -if [[ -z ${EASYBUILD_ROBOT_PATHS} ]] +if [[ -z ${EASYBUILD_ROBOT_PATHS} ]]; then git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs fi From 8404bd1cb06340b411ed11a38e4c4875b473a998 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Oct 2024 15:23:41 +0200 Subject: [PATCH 1391/1795] Rebuild EESSI-extend to use EESSI_SITE_INSTALLPATH --- .../2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml | 5 +++++ init/eessi_environment_variables | 2 ++ init/modules/EESSI/2023.06.lua | 2 ++ 3 files changed, 9 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml new file mode 100644 index 0000000000..5491ef8427 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml @@ -0,0 +1,5 @@ +# 2024.10.08 +# EESSI-extend should use EESSI_SITE_INSTALLPATH, instead of recalculating this +easyconfigs: + - EESSI-extend-2023.06-easybuild.eb + diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 8c10b1fca8..307616ac35 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -81,6 +81,8 @@ if [ -d $EESSI_PREFIX ]; then if [ ! -z $EESSI_BASIC_ENV ]; then show_msg "Only setting up basic environment, so we're done" elif [ -d $EESSI_SOFTWARE_PATH ]; then + EESSI_SITE_INSTALLPATH=${EESSI_SOFTWARE_PATH/versions/host_injections} + show_msg "Using ${EESSI_SITE_INSTALLPATH} as the site extension directory for installations." # Allow for the use of a custom MNS if [ -z ${EESSI_CUSTOM_MODULEPATH+x} ]; then # EESSI_CUSTOM_MODULEPATH not set so we use our defaults diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 32aaf6c07f..ecc145c8e4 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -48,9 +48,11 @@ local eessi_software_subdir = archdetect local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family) local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) local eessi_module_path = pathJoin(eessi_software_path, "modules", "all") +local eessi_site_install_path = string.gsub(eessi_software_path, "versions", "host_injections") local eessi_site_module_path = string.gsub(eessi_module_path, "versions", "host_injections") setenv("EPREFIX", eessi_eprefix) setenv("EESSI_CPU_FAMILY", eessi_cpu_family) +setenv("EESSI_SITE_INSTALLPATH", eessi_site_install_path) setenv("EESSI_SITE_MODULEPATH", eessi_site_module_path) setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir) setenv("EESSI_PREFIX", eessi_prefix) From 227581646d495bdbca676fa7fdc43297cd2967d0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:13:20 +0200 Subject: [PATCH 1392/1795] Apply suggestions from code review Co-authored-by: ocaisa --- init/eessi_environment_variables | 2 +- init/modules/EESSI/2023.06.lua | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 307616ac35..c4afd9ab5e 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -81,7 +81,7 @@ if [ -d $EESSI_PREFIX ]; then if [ ! -z $EESSI_BASIC_ENV ]; then show_msg "Only setting up basic environment, so we're done" elif [ -d $EESSI_SOFTWARE_PATH ]; then - EESSI_SITE_INSTALLPATH=${EESSI_SOFTWARE_PATH/versions/host_injections} + EESSI_SITE_SOFTWARE_PATH=${EESSI_SOFTWARE_PATH/versions/host_injections} show_msg "Using ${EESSI_SITE_INSTALLPATH} as the site extension directory for installations." # Allow for the use of a custom MNS if [ -z ${EESSI_CUSTOM_MODULEPATH+x} ]; then diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index ecc145c8e4..463706ce6c 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -48,11 +48,11 @@ local eessi_software_subdir = archdetect local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family) local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) local eessi_module_path = pathJoin(eessi_software_path, "modules", "all") -local eessi_site_install_path = string.gsub(eessi_software_path, "versions", "host_injections") -local eessi_site_module_path = string.gsub(eessi_module_path, "versions", "host_injections") +local eessi_site_software_path = string.gsub(eessi_software_path, "versions", "host_injections") +local eessi_site_module_path = pathJoin(eessi_site_software_path, "modules", "all") setenv("EPREFIX", eessi_eprefix) setenv("EESSI_CPU_FAMILY", eessi_cpu_family) -setenv("EESSI_SITE_INSTALLPATH", eessi_site_install_path) +setenv("EESSI_SITE_SOFTWARE_PATH", eessi_site_software_path) setenv("EESSI_SITE_MODULEPATH", eessi_site_module_path) setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir) setenv("EESSI_PREFIX", eessi_prefix) From 30c1bb362ef9197e234984065f0b5193b8b0845c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Oct 2024 16:19:32 +0200 Subject: [PATCH 1393/1795] Be consistent --- init/eessi_environment_variables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index c4afd9ab5e..90b295e56d 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -82,7 +82,7 @@ if [ -d $EESSI_PREFIX ]; then show_msg "Only setting up basic environment, so we're done" elif [ -d $EESSI_SOFTWARE_PATH ]; then EESSI_SITE_SOFTWARE_PATH=${EESSI_SOFTWARE_PATH/versions/host_injections} - show_msg "Using ${EESSI_SITE_INSTALLPATH} as the site extension directory for installations." + show_msg "Using ${EESSI_SITE_SOFTWARE_PATH} as the site extension directory for installations." # Allow for the use of a custom MNS if [ -z ${EESSI_CUSTOM_MODULEPATH+x} ]; then # EESSI_CUSTOM_MODULEPATH not set so we use our defaults From d209b272837b60bed6dd221e74eb323fa7b8009f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Oct 2024 16:23:58 +0200 Subject: [PATCH 1394/1795] Make sure EESSI_MODULE_SUBDIR is always set. Then, use that to define the EESSI_SITE_MODULEPATH --- init/eessi_environment_variables | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 90b295e56d..1ff03ac198 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -83,15 +83,15 @@ if [ -d $EESSI_PREFIX ]; then elif [ -d $EESSI_SOFTWARE_PATH ]; then EESSI_SITE_SOFTWARE_PATH=${EESSI_SOFTWARE_PATH/versions/host_injections} show_msg "Using ${EESSI_SITE_SOFTWARE_PATH} as the site extension directory for installations." + # Allow for use of alternative module tree shipped with EESSI + if [ -z ${EESSI_MODULE_SUBDIR+x} ]; then + # EESSI_MODULE_SUBDIR not set + EESSI_MODULE_SUBDIR="modules/all" + fi # Allow for the use of a custom MNS if [ -z ${EESSI_CUSTOM_MODULEPATH+x} ]; then # EESSI_CUSTOM_MODULEPATH not set so we use our defaults - # Allow for use of alternative module tree shipped with EESSI - if [ -z ${EESSI_MODULE_SUBDIR+x} ]; then - # EESSI_MODULE_SUBDIR not set - EESSI_MODULE_SUBDIR="modules/all" - fi EESSI_MODULEPATH=$EESSI_SOFTWARE_PATH/$EESSI_MODULE_SUBDIR else show_msg "Using defined environment variable \$EESSI_CUSTOM_MODULEPATH to set EESSI_MODULEPATH." @@ -101,7 +101,7 @@ if [ -d $EESSI_PREFIX ]; then if [ -d $EESSI_MODULEPATH ]; then export EESSI_MODULEPATH=$EESSI_MODULEPATH show_msg "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." - export EESSI_SITE_MODULEPATH=${EESSI_MODULEPATH/versions/host_injections} + export EESSI_SITE_MODULEPATH=$EESSI_SITE_SOFTWARE_PATH/$EESSI_MODULE_SUBDIR show_msg "Using ${EESSI_SITE_MODULEPATH} as the site extension directory to be added to MODULEPATH." else error "EESSI module path at $EESSI_MODULEPATH not found!" From 022be33385fec3dd425174d8de7c632b4c2a6c31 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Oct 2024 16:27:30 +0200 Subject: [PATCH 1395/1795] This was what it was all about... have EESSI extend use the EESSI_SITE_INSTALLPATH --- EESSI-extend-2023.06-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index ba8629c02e..d7bdbe562d 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -92,7 +92,7 @@ elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables") end - easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') + easybuild_installpath = os.getenv("EESSI_SITE_INSTALLPATH") else -- Deal with user and project installs project_install = os.getenv("EESSI_PROJECT_INSTALL") From e9ede03e72a46b4ea79ad3ca37c4a5e81af2d376 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Oct 2024 16:52:35 +0200 Subject: [PATCH 1396/1795] Shorten the ReFrame output, only get the relevant lines from it --- bot/check-test.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 1f66514bb2..2e58535c4a 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -77,16 +77,25 @@ if [[ ! -z ${grep_reframe_failed} ]]; then else # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' - # Replace null character with
    - # Replace new line characters with
    - # Replace % with %%. Use \%\% to interpret both %% as (non-special) characters + # Grab the full ReFrame report, than cut the irrelevant parts + # Note that the character limit for messages in github is around 65k, so cutting is important grep_reframe_success_full=$( \ grep -v "^>> searching for " ${job_dir}/${job_out} | \ - grep -Pzo "${GP_success_full}" | \ - sed 's/\x00//g' | - sed ':a;N;$!ba;s/\n//g' | - sed 's/\%/\%\%/g' \ + grep -Pzo "${GP_success_full}" | \ # Use -z + sed 's/\x00/\n/g' | \ # Replace null character with newline, to undo the -z option + grep -v -P '\[\s*RUN\s*]' | \ # Remove the [ RUN ] lines from reframe, they are not very informative + grep -v '\[-*\]' | \ # Remove the line '[----------] all spawned checks have finished' + grep -v '\[=*\]' | \ # Remove the line '[==========] Finished on Mon Oct 7 21' + grep -v '^$' | \ # Remove blank line(s) from the report + sed ':a;N;$!ba;s/\n//g' | \ # Replace all newline characters with
    + sed 's/\%/\%\%/g' \ # Replace % with %%. Use \%\% to interpret both %% as (non-special) characters ) + # TODO (optional): we could impose a character limit here, and truncate if too long + # (though we should do that before inserting the
    statements). + # If we do, we should probably re-append the final summary, e.g. + # [ PASSED ] Ran 10/10 test case(s) from 10 check(s) (0 failure(s), 0 skipped, 0 aborted) + # so that that is always displayed + # However, that's not implemented yet - let's see if this ever even becomes an issue grep_reframe_result=${grep_reframe_success_full} echo "grep_reframe_success_full: ${grep_reframe_success_full}" fi From 3ff1b30e9f56b9ea9eff6f4d78b110189b4d6c35 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Oct 2024 16:56:25 +0200 Subject: [PATCH 1397/1795] Strip color coding characters --- bot/check-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 2e58535c4a..a8190e5e0d 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -87,6 +87,7 @@ else grep -v '\[-*\]' | \ # Remove the line '[----------] all spawned checks have finished' grep -v '\[=*\]' | \ # Remove the line '[==========] Finished on Mon Oct 7 21' grep -v '^$' | \ # Remove blank line(s) from the report + sed 's/\x1B\[[0-9;]*m//g' | \ # Strip color coding characters sed ':a;N;$!ba;s/\n//g' | \ # Replace all newline characters with
    sed 's/\%/\%\%/g' \ # Replace % with %%. Use \%\% to interpret both %% as (non-special) characters ) From fe64129fd9312a68a1910a2ee68f646eac3307f9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 8 Oct 2024 22:56:52 +0200 Subject: [PATCH 1398/1795] Don't have comments after line breaks --- bot/check-test.sh | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index a8190e5e0d..625e88d8eb 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -81,15 +81,24 @@ else # Note that the character limit for messages in github is around 65k, so cutting is important grep_reframe_success_full=$( \ grep -v "^>> searching for " ${job_dir}/${job_out} | \ - grep -Pzo "${GP_success_full}" | \ # Use -z - sed 's/\x00/\n/g' | \ # Replace null character with newline, to undo the -z option - grep -v -P '\[\s*RUN\s*]' | \ # Remove the [ RUN ] lines from reframe, they are not very informative - grep -v '\[-*\]' | \ # Remove the line '[----------] all spawned checks have finished' - grep -v '\[=*\]' | \ # Remove the line '[==========] Finished on Mon Oct 7 21' - grep -v '^$' | \ # Remove blank line(s) from the report - sed 's/\x1B\[[0-9;]*m//g' | \ # Strip color coding characters - sed ':a;N;$!ba;s/\n//g' | \ # Replace all newline characters with
    - sed 's/\%/\%\%/g' \ # Replace % with %%. Use \%\% to interpret both %% as (non-special) characters + # Use -z + grep -Pzo "${GP_success_full}" | \ + # Replace null character with newline, to undo the -z option + sed 's/\x00/\n/g' | \ + # Remove the [ RUN ] lines from reframe, they are not very informative + grep -v -P '\[\s*RUN\s*]' | \ + # Remove the line '[----------] all spawned checks have finished' + grep -v '\[-*\]' | \ + # Remove the line '[==========] Finished on Mon Oct 7 21' + grep -v '\[=*\]' | \ + # Remove blank line(s) from the report + grep -v '^$' | \ + # Strip color coding characters + sed 's/\x1B\[[0-9;]*m//g' | \ + # Replace all newline characters with
    + sed ':a;N;$!ba;s/\n//g' | \ + # Replace % with %%. Use \%\% to interpret both %% as (non-special) characters + sed 's/\%/\%\%/g' \ ) # TODO (optional): we could impose a character limit here, and truncate if too long # (though we should do that before inserting the
    statements). From a0a948d67f50bf7f2126c3612c2ec154982089c1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 10:58:12 +0200 Subject: [PATCH 1399/1795] Remove echo that was only there for debugging --- bot/check-test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 625e88d8eb..79ae0be61f 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -107,7 +107,6 @@ else # so that that is always displayed # However, that's not implemented yet - let's see if this ever even becomes an issue grep_reframe_result=${grep_reframe_success_full} - echo "grep_reframe_success_full: ${grep_reframe_success_full}" fi echo "grep_reframe_result: ${grep_reframe_result}" From 6f5419a7c7757b1fa79f2fa2ea4898b75863f499 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 11:00:35 +0200 Subject: [PATCH 1400/1795] Renamed --- EESSI-extend-2023.06-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index d7bdbe562d..8e328c3ece 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -92,7 +92,7 @@ elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables") end - easybuild_installpath = os.getenv("EESSI_SITE_INSTALLPATH") + easybuild_installpath = os.getenv("EESSI_SITE_SOFTWARE_PATH") else -- Deal with user and project installs project_install = os.getenv("EESSI_PROJECT_INSTALL") From 4b61bc0a6aa88f95fa52a71a240aaa2c17b7eac9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 11:01:18 +0200 Subject: [PATCH 1401/1795] Export EESSI_SITE_SOFTWARE_PATH --- init/eessi_environment_variables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 1ff03ac198..43a95a7889 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -81,7 +81,7 @@ if [ -d $EESSI_PREFIX ]; then if [ ! -z $EESSI_BASIC_ENV ]; then show_msg "Only setting up basic environment, so we're done" elif [ -d $EESSI_SOFTWARE_PATH ]; then - EESSI_SITE_SOFTWARE_PATH=${EESSI_SOFTWARE_PATH/versions/host_injections} + export EESSI_SITE_SOFTWARE_PATH=${EESSI_SOFTWARE_PATH/versions/host_injections} show_msg "Using ${EESSI_SITE_SOFTWARE_PATH} as the site extension directory for installations." # Allow for use of alternative module tree shipped with EESSI if [ -z ${EESSI_MODULE_SUBDIR+x} ]; then From 79332f34eb7a73f1d2eeb53e7e1b4a9a94054167 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 9 Oct 2024 11:43:44 +0200 Subject: [PATCH 1402/1795] Get the script to explain the differences between the Lmod and init scripts --- .github/workflows/tests_eessi_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index d2e3cd1338..8d6bd6edb0 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -80,6 +80,7 @@ jobs: echo "Test for checking env variables PASSED" else echo "Test for checking env variables FAILED" >&2 + diff "${moduleoutfile}" "${sourceoutfile}" exit 1 fi From 9f7982d73caeef98315ce54aca8b38360906bb23 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 9 Oct 2024 11:51:39 +0200 Subject: [PATCH 1403/1795] Use the `init/bash` in the repo, not the one shipped by EESSI --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 8d6bd6edb0..cbcffe6385 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -72,7 +72,7 @@ jobs: module load EESSI/${{matrix.EESSI_VERSION}} env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} - source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + source ./init/bash env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${sourceoutfile}" cat "${moduleoutfile}" cat "${sourceoutfile}" From 8ae0be5757ce8598de7fdfa0f7eda5c1d83213b7 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 9 Oct 2024 13:18:10 +0200 Subject: [PATCH 1404/1795] Add waLBerla-6.1-foss-2023a --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 19d9aedfae..170a639064 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -5,3 +5,7 @@ easyconfigs: from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 + - waLBerla-6.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 + from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 From e9c2472b75a6527a57862ba9da27402a3abcbe2c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 15:06:44 +0200 Subject: [PATCH 1405/1795] Remove warning as well for the local spawner not having memory request support - we know that --- bot/check-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 79ae0be61f..2731e75464 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -93,6 +93,8 @@ else grep -v '\[=*\]' | \ # Remove blank line(s) from the report grep -v '^$' | \ + # Remove warnings about the local spawner not supporting memory requests + grep -v 'WARNING\: hooks\.req_memory_per_node does not support the scheduler you configured .local.*$' | \ # Strip color coding characters sed 's/\x1B\[[0-9;]*m//g' | \ # Replace all newline characters with
    From e9018fb784d41983235768212efd7df88765bfc2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 16:34:38 +0200 Subject: [PATCH 1406/1795] Move to gpu_support/nvidia subdir --- .../eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml diff --git a/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml b/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml new file mode 100644 index 0000000000..24be8ddbc3 --- /dev/null +++ b/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml @@ -0,0 +1,5 @@ +# This EasyStack provides a list of all the EasyConfigs that should be installed in host_injections +# because they cannot (fully) be shipped as part of EESSI due to license constraints +easyconfigs: + - CUDA-12.1.1.eb + - cuDNN-8.9.2.26-CUDA-12.1.1.eb From 54753c33bb79605a5c13e68726e795de4ad6b23f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 16:40:51 +0200 Subject: [PATCH 1407/1795] Make comment more explicit that this is only about nvidia GPU support --- .../eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml b/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml index 24be8ddbc3..2a6f9bd6f9 100644 --- a/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml +++ b/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml @@ -1,5 +1,5 @@ -# This EasyStack provides a list of all the EasyConfigs that should be installed in host_injections -# because they cannot (fully) be shipped as part of EESSI due to license constraints +# This EasyStack provides a list of all the EasyConfigs that should be installed in host_injections +# for nvidia GPU support, because they cannot (fully) be shipped as part of EESSI due to license constraints easyconfigs: - CUDA-12.1.1.eb - cuDNN-8.9.2.26-CUDA-12.1.1.eb From 02b48afb928baad72f3307ed689413eb83877cd3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 9 Oct 2024 16:59:06 +0200 Subject: [PATCH 1408/1795] Switch to nvidia-smi commands that are spoofed in CI --- .../nvidia/link_nvidia_host_libraries.sh | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 07c57360fa..2ecb7c06c0 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -150,17 +150,24 @@ done # Gather information about NVIDIA drivers (even if we are inside a Gentoo Prefix in a container) export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH -# Command to give to get the CUDA driver version -nvidia_smi_driver_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader" -if $nvidia_smi_driver_command > /dev/null 2>&1; then - host_driver_version=$($nvidia_smi_driver_command | tail -n1) - echo_green "Found NVIDIA GPU driver version ${host_driver_version}" - - # If the first worked, this should work too - host_cuda_version=$(nvidia-smi -q --display=COMPUTE | grep CUDA | awk '{NF>1; print $NF}') - echo_green "Found host CUDA version ${host_cuda_version}" +# Check for NVIDIA GPUs via nvidia-smi command +nvidia_smi=$(command -v nvidia-smi) +if [[ $? -eq 0 ]]; then + nvidia_smi_out=$(mktemp -p /tmp nvidia_smi_out.XXXXX) + nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader 2>&1 > $nvidia_smi_out + if [[ $? -eq 0 ]]; then + nvidia_smi_info=$(head -1 $nvidia_smi_out) + host_cuda_version=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f4 -d, | sed 's/\.//g') + host_driver_version=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f3 -d,) + echo_green "Found host CUDA version ${host_cuda_version}" + echo_green "Found NVIDIA GPU driver version ${host_driver_version}" + rm -f $nvidia_smi_out + else + fatal_error "nvidia-smi command failed, see output in $nvidia_smi_out" + fi else - fatal_error "Failed to execute $nvidia_smi_driver_command" + fatal_error "nvidia-smi command not found" + exit 2 fi # Gather any CUDA related driver libraries from the host From 2b76a54c0b1c4f1256679d6d1825c5590939d90b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 17:12:41 +0200 Subject: [PATCH 1409/1795] Moved easystack file --- .../eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/gpu_support/nvidia/{ => easystacks}/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml (100%) diff --git a/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml similarity index 100% rename from scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml rename to scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml From 086ba5a570e70426d4bd52119247e094f1f9cabc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 17:14:37 +0200 Subject: [PATCH 1410/1795] Change how EESSI_SITE_INSTALL is used --- .../nvidia/install_cuda_and_libraries.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 40bff34bb7..7cbf253256 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -85,9 +85,8 @@ fi # Make sure EESSI is initialised check_eessi_initialised -# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` -# (CUDA is a binary installation so no need to worry too much about the EasyBuild setup) -export EESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections} +# Make sure that `EESSI-extend` will install in the site installation path EESSI_SITE_SOFTWARE_PATH +export EESSI_SITE_INSTALL=1 # we need a directory we can use for temporary storage if [[ -z "${TEMP_DIR}" ]]; then @@ -128,7 +127,7 @@ eb --show-config # do a 'eb --dry-run-short' with the EASYSTACK_FILE and determine list of packages # to be installed -echo ">> Determining if packages specified in ${EASYSTACK_FILE} are missing under ${EESSI_SITE_INSTALL}" +echo ">> Determining if packages specified in ${EASYSTACK_FILE} are missing under ${EESSI_SITE_SOFTWARE_PATH}" eb_dry_run_short_out=${tmpdir}/eb_dry_run_short.out eb --dry-run-short --rebuild --easystack ${EASYSTACK_FILE} 2>&1 | tee ${eb_dry_run_short_out} ret=$? @@ -168,10 +167,10 @@ fi # The install is pretty fat, you need lots of space for download/unpack/install # (~3*${base_storage_space}*1000 Bytes), # need to do a space check before we proceed -avail_space=$(df --output=avail "${EESSI_SITE_INSTALL}"/ | tail -n 1 | awk '{print $1}') +avail_space=$(df --output=avail "${EESSI_SITE_SOFTWARE_PATH}"/ | tail -n 1 | awk '{print $1}') min_disk_storage=$((3 * ${base_storage_space})) if (( avail_space < ${min_disk_storage} )); then - fatal_error "Need at least $(echo "${min_disk_storage} / 1000000" | bc) GB disk space to install CUDA and other libraries under ${EESSI_SITE_INSTALL}, exiting now..." + fatal_error "Need at least $(echo "${min_disk_storage} / 1000000" | bc) GB disk space to install CUDA and other libraries under ${EESSI_SITE_SOFTWARE_PATH}, exiting now..." fi avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') if (( avail_space < required_space_in_tmpdir )); then @@ -213,7 +212,7 @@ if [ $ret -ne 0 ]; then cp -a ${eb_last_log} . fatal_error "some installation failed, please check EasyBuild logs ${PWD}/$(basename ${eb_last_log})..." else - echo_green "all installations at ${EESSI_SITE_INSTALL}/software/... succeeded!" + echo_green "all installations at ${EESSI_SITE_SOFTWARE_PATH}/software/... succeeded!" fi # clean up tmpdir rm -rf "${tmpdir}" From 6410c940d03b98d0d0dd22fee462750eaa061797 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 9 Oct 2024 17:22:41 +0200 Subject: [PATCH 1411/1795] Filter out symlinks in LD_PRELOAD mode --- .../nvidia/link_nvidia_host_libraries.sh | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 2ecb7c06c0..7408eabf27 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -210,44 +210,47 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then echo echo_yellow "When attempting to use LD_PRELOAD we exclude anything related to graphics" - # Filter out all libraries that have missing library dependencies under EESSI + # Filter out all symlinks and libraries that have missing library dependencies under EESSI filtered_libraries=() for library in "${matched_libraries[@]}"; do - # Run ldd on the given binary and filter for "not found" libraries - NOT_FOUND_LIBS=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') - # Check if it is missing an so dep under EESSI - if [[ -z "$NOT_FOUND_LIBS" ]]; then - # Anything graphics is out, as is libnvidia-fbc* - if [[ "$library" != *"GL"* ]]; then - if [[ "$library" != *"libnvidia-fbc"* ]]; then - filtered_libraries+=("$library") + if [ ! -L "$library" ]; then + # $library is not a symlink + # Run ldd on the given binary and filter for "not found" libraries + NOT_FOUND_LIBS=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') + # Check if it is missing an so dep under EESSI + if [[ -z "$NOT_FOUND_LIBS" ]]; then + # Anything graphics is out, as is libnvidia-fbc* + if [[ "$library" != *"GL"* ]]; then + if [[ "$library" != *"libnvidia-fbc"* ]]; then + filtered_libraries+=("$library") + fi fi - fi - else - # Iterate over "not found" libraries and check if they are in the array - all_found=true - for lib in $NOT_FOUND_LIBS; do - found=false - for listed_lib in "${matched_libraries[@]}"; do - if [[ "$lib" == "$listed_lib" ]]; then - found=true + else + # Iterate over "not found" libraries and check if they are in the array + all_found=true + for lib in $NOT_FOUND_LIBS; do + found=false + for listed_lib in "${matched_libraries[@]}"; do + if [[ "$lib" == "$listed_lib" ]]; then + found=true + break + fi + done + + if [[ "$found" == false ]]; then + echo "$lib is NOT in the provided preload list, filtering $library" + all_found=false break fi done - if [[ "$found" == false ]]; then - echo "$lib is NOT in the provided preload list, filtering $library" - all_found=false - break - fi - done - - # If we find all the missing libs in our list include it - if [[ "$all_found" == true ]]; then - # Anything graphics is out, as is libnvidia-fbc* - if [[ "$library" != *"GL"* ]]; then - if [[ "$library" != *"libnvidia-fbc"* ]]; then - filtered_libraries+=("$library") + # If we find all the missing libs in our list include it + if [[ "$all_found" == true ]]; then + # Anything graphics is out, as is libnvidia-fbc* + if [[ "$library" != *"GL"* ]]; then + if [[ "$library" != *"libnvidia-fbc"* ]]; then + filtered_libraries+=("$library") + fi fi fi fi From b8257ca61e41de7e5f5c6706380adfec8299898b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 9 Oct 2024 17:30:41 +0200 Subject: [PATCH 1412/1795] Retain CUDA compute capability with . --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 7408eabf27..b08a4b50b1 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -157,7 +157,7 @@ if [[ $? -eq 0 ]]; then nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader 2>&1 > $nvidia_smi_out if [[ $? -eq 0 ]]; then nvidia_smi_info=$(head -1 $nvidia_smi_out) - host_cuda_version=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f4 -d, | sed 's/\.//g') + host_cuda_version=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f4 -d,) host_driver_version=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f3 -d,) echo_green "Found host CUDA version ${host_cuda_version}" echo_green "Found NVIDIA GPU driver version ${host_driver_version}" From ecd30ca4bdb9982ea42d0294b84cc60ff2165118 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 17:50:35 +0200 Subject: [PATCH 1413/1795] First attempt at making this loop over EasyStack files, loading the correct easybuild version before loading eessi-extend --- .../nvidia/install_cuda_and_libraries.sh | 265 +++++++++--------- 1 file changed, 139 insertions(+), 126 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 7cbf253256..60f3bb30be 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -27,8 +27,9 @@ show_help() { echo " --accept-cuda-eula You _must_ accept the CUDA EULA to install" echo " CUDA, see the EULA at" echo " https://docs.nvidia.com/cuda/eula/index.html" - echo " -e, --easystack EASYSTACK_FILE Path to easystack file that defines which" - echo " packages shall be installed" + echo " --accept-cudnn-eula You _must_ accept the cuDNN EULA to install" + echo " cuDNN, see the EULA at" + echo " https://docs.nvidia.com/deeplearning/cudnn/latest/reference/eula.html" echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" echo " storage during the installation of CUDA" echo " and/or other libraries (must have" @@ -36,7 +37,7 @@ show_help() { } # Initialize variables -eula_accepted=0 +cuda_eula_accepted=0 EASYSTACK_FILE= TEMP_DIR= @@ -48,18 +49,12 @@ while [[ $# -gt 0 ]]; do exit 0 ;; --accept-cuda-eula) - eula_accepted=1 + cuda_eula_accepted=1 shift 1 ;; - -e|--easystack) - if [ -n "$2" ]; then - EASYSTACK_FILE="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi + --accept-cudnn-eula) + cudnn_eula_accepted=1 + shift 1 ;; -t|--temp-dir) if [ -n "$2" ]; then @@ -78,10 +73,6 @@ while [[ $# -gt 0 ]]; do esac done -if [[ -z "${EASYSTACK_FILE}" ]]; then - fatal_error "Need the name/path to an easystack file. See command line options\n" -fi - # Make sure EESSI is initialised check_eessi_initialised @@ -100,119 +91,141 @@ else fi echo "Created temporary directory '${tmpdir}'" -echo "MODULEPATH=${MODULEPATH}" -echo "List available *CUDA* modules before loading EESSI-extend/${EESSI_VERSION}-easybuild" -module avail CUDA +# use install_path/modules/all as MODULEPATH +SAVE_MODULEPATH=${MODULEPATH} -# load EESSI-extend/2023.06-easybuild module && verify that it is loaded -EESSI_EXTEND_MODULE="EESSI-extend/${EESSI_VERSION}-easybuild" -module load ${EESSI_EXTEND_MODULE} -ret=$? -if [ "${ret}" -ne 0 ]; then - fatal_error "An error occured while trying to load ${EESSI_EXTEND_MODULE}\n" -fi +for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*.yml; do + echo -e "Processing easystack file ${easystack_file}...\n\n" + + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${EASYSTACK_FILE} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + + # Load EasyBuild version for this easystack file _before_ loading EESSI-extend + module load EasyBuild/${eb_version} + module load EESSI-extend/${EESSI_VERSION}-easybuild + + # Install modules in hidden .modules dir to keep track of what was installed before + MODULEPATH=${EASYBUILD_INSTALLPATH}/.modules/all + echo "set MODULEPATH=${MODULEPATH}" + + # show EasyBuild configuration + echo "Show EasyBuild configuration" + eb --show-config + + # do a 'eb --dry-run-short' with the EASYSTACK_FILE and determine list of packages + # to be installed + echo ">> Determining if packages specified in ${EASYSTACK_FILE} are missing under ${EESSI_SITE_SOFTWARE_PATH}" + eb_dry_run_short_out=${tmpdir}/eb_dry_run_short.out + eb --dry-run-short --rebuild --easystack ${EASYSTACK_FILE} 2>&1 | tee ${eb_dry_run_short_out} + ret=$? + + # Check if CUDA shall be installed + cuda_install_needed=0 + cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | grep "module: CUDA/" + ret=$? + if [ "${ret}" -eq 0 ]; then + cuda_install_needed=1 + fi -echo "MODULEPATH=${MODULEPATH}" -echo "List available *CUDA* modules after loading EESSI-extend/${EESSI_VERSION}-easybuild" -module avail CUDA + # Make sure the CUDA EULA is accepted if it shall be installed + if [ "${cuda_install_needed}" -eq 1 ] && [ "${cuda_eula_accepted}" -ne 1 ]; then + show_help + error="\nCUDA shall be installed. However, the CUDA EULA has not been accepted\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n" + fatal_error "${error}" + fi -# use install_path/modules/all as MODULEPATH -SAVE_MODULEPATH=${MODULEPATH} -MODULEPATH=${EASYBUILD_INSTALLPATH}/.modules/all -echo "set MODULEPATH=${MODULEPATH}" - -# show EasyBuild configuration -echo "Show EasyBuild configuration" -eb --show-config - -# do a 'eb --dry-run-short' with the EASYSTACK_FILE and determine list of packages -# to be installed -echo ">> Determining if packages specified in ${EASYSTACK_FILE} are missing under ${EESSI_SITE_SOFTWARE_PATH}" -eb_dry_run_short_out=${tmpdir}/eb_dry_run_short.out -eb --dry-run-short --rebuild --easystack ${EASYSTACK_FILE} 2>&1 | tee ${eb_dry_run_short_out} -ret=$? - -# Check if CUDA shall be installed -cuda_install_needed=0 -cat ${eb_dry_run_short_out} | grep "^ \* \[[xR]\]" | grep "module: CUDA/" -ret=$? -if [ "${ret}" -eq 0 ]; then - cuda_install_needed=1 -fi + # Check if cdDNN shall be installed + cudnn_install_needed=0 + cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | grep "module: cuDNN/" + ret=$? + if [ "${ret}" -eq 0 ]; then + cudnn_install_needed=1 + fi -# Make sure the CUDA EULA is accepted if it shall be installed -if [ "${cuda_install_needed}" -eq 1 ] && [ "${eula_accepted}" -ne 1 ]; then - show_help - error="\nCUDA shall be installed. However, the CUDA EULA has not been accepted\nYou _must_ accept the CUDA EULA via the appropriate command line option.\n" - fatal_error "${error}" -fi + # Make sure the cuDNN EULA is accepted if it shall be installed + if [ "${cudnn_install_needed}" -eq 1 ] && [ "${cudnn_eula_accepted}" -ne 1 ]; then + show_help + error="\ncuDNN shall be installed. However, the cuDNNDA EULA has not been accepted\nYou _must_ accept the cuDNN EULA via the appropriate command line option.\n" + fatal_error "${error}" + fi -# determine the number of packages to be installed (assume 5 GB + num_packages * -# 3GB space needed) -number_of_packages=$(cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | sed -e 's/^.*module: //' | sort -u | wc -l) -echo "number of packages to be (re-)installed: '${number_of_packages}'" -base_storage_space=$((5000000 + ${number_of_packages} * 3000000)) - -required_space_in_tmpdir=${base_storage_space} -# Let's see if we have sources and build locations defined if not, we use the temporary space -if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then - export EASYBUILD_BUILDPATH=${tmpdir}/build - required_space_in_tmpdir=$((required_space_in_tmpdir + ${base_storage_space})) -fi -if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then - export EASYBUILD_SOURCEPATH=${tmpdir}/sources - required_space_in_tmpdir=$((required_space_in_tmpdir + ${base_storage_space})) -fi + # determine the number of packages to be installed (assume 5 GB + num_packages * + # 3GB space needed). Both CUDA and cuDNN are about this size + number_of_packages=$(cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | sed -e 's/^.*module: //' | sort -u | wc -l) + echo "number of packages to be (re-)installed: '${number_of_packages}'" + base_storage_space=$((5000000 + ${number_of_packages} * 3000000)) + + required_space_in_tmpdir=${base_storage_space} + # Let's see if we have sources and build locations defined if not, we use the temporary space + if [[ -z "${EASYBUILD_BUILDPATH}" ]]; then + export EASYBUILD_BUILDPATH=${tmpdir}/build + required_space_in_tmpdir=$((required_space_in_tmpdir + ${base_storage_space})) + fi + if [[ -z "${EASYBUILD_SOURCEPATH}" ]]; then + export EASYBUILD_SOURCEPATH=${tmpdir}/sources + required_space_in_tmpdir=$((required_space_in_tmpdir + ${base_storage_space})) + fi + + # The install is pretty fat, you need lots of space for download/unpack/install + # (~3*${base_storage_space}*1000 Bytes), + # need to do a space check before we proceed + avail_space=$(df --output=avail "${EESSI_SITE_SOFTWARE_PATH}"/ | tail -n 1 | awk '{print $1}') + min_disk_storage=$((3 * ${base_storage_space})) + if (( avail_space < ${min_disk_storage} )); then + fatal_error "Need at least $(echo "${min_disk_storage} / 1000000" | bc) GB disk space to install CUDA and other libraries under ${EESSI_SITE_SOFTWARE_PATH}, exiting now..." + fi + avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') + if (( avail_space < required_space_in_tmpdir )); then + error="Need at least $(echo "${required_space_in_tmpdir} / 1000000" | bc) temporary disk space under ${tmpdir}.\n" + error="${error}Set the environment variable TEMP_DIR to a location with adequate space to pass this check." + error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH" + error="${error}to reduce this requirement. Exiting now..." + fatal_error "${error}" + fi -# The install is pretty fat, you need lots of space for download/unpack/install -# (~3*${base_storage_space}*1000 Bytes), -# need to do a space check before we proceed -avail_space=$(df --output=avail "${EESSI_SITE_SOFTWARE_PATH}"/ | tail -n 1 | awk '{print $1}') -min_disk_storage=$((3 * ${base_storage_space})) -if (( avail_space < ${min_disk_storage} )); then - fatal_error "Need at least $(echo "${min_disk_storage} / 1000000" | bc) GB disk space to install CUDA and other libraries under ${EESSI_SITE_SOFTWARE_PATH}, exiting now..." -fi -avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') -if (( avail_space < required_space_in_tmpdir )); then - error="Need at least $(echo "${required_space_in_tmpdir} / 1000000" | bc) temporary disk space under ${tmpdir}.\n" - error="${error}Set the environment variable TEMP_DIR to a location with adequate space to pass this check." - error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH" - error="${error}to reduce this requirement. Exiting now..." - fatal_error "${error}" -fi + # Brief explanation of parameters: + # - prefix: using $tmpdir as default base directory for several EB settings + # - rebuild: we need the --rebuild option, as the CUDA module may or may not be on the + # `MODULEPATH` yet. Even if it is, we still want to redo this installation + # since it will provide the symlinked targets for the parts of the CUDA + # and/or other installation in the `.../versions/...` prefix + # - installpath-modules: We install the module in our `tmpdir` since we do not need the modulefile, + # we only care about providing the targets for the symlinks. + # - ${accept_eula_opt}: We only set the --accept-eula-for=CUDA option if CUDA will be installed and if + # this script was called with the argument --accept-cuda-eula. + # - hooks: We don't want hooks used in this install, we need vanilla + # installations of CUDA and/or other libraries + # - easystack: Path to easystack file that defines which packages shall be + # installed + accept_eula_opt= + if [[ ${cuda_eula_accepted} -eq 1 ]]; then + accept_eula_opt="--accept-eula-for=CUDA" + fi + if [[ ${cudnn_eula_accepted} -eq 1 ]]; then + if [[ -z ${accept_eula_opt} ]]; then + accept_eula_opt="--accept-eula-for=cuDNN" + else + accept_eula_opt="${accept_eula_opt},cuDNN" + fi + fi + touch "$tmpdir"/none.py + eb --prefix="$tmpdir" \ + --installpath-modules=${EASYBUILD_INSTALLPATH}/.modules \ + "${accept_cuda_eula_opt}" \ + --hooks="$tmpdir"/none.py \ + --easystack ${EASYSTACK_FILE} + ret=$? + if [ $ret -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + cp -a ${eb_last_log} . + fatal_error "some installation failed, please check EasyBuild logs ${PWD}/$(basename ${eb_last_log})..." + else + echo_green "all installations at ${EESSI_SITE_SOFTWARE_PATH}/software/... succeeded!" + fi -# Brief explanation of parameters: -# - prefix: using $tmpdir as default base directory for several EB settings -# - rebuild: we need the --rebuild option, as the CUDA module may or may not be on the -# `MODULEPATH` yet. Even if it is, we still want to redo this installation -# since it will provide the symlinked targets for the parts of the CUDA -# and/or other installation in the `.../versions/...` prefix -# - installpath-modules: We install the module in our `tmpdir` since we do not need the modulefile, -# we only care about providing the targets for the symlinks. -# - ${accept_eula_opt}: We only set the --accept-eula-for=CUDA option if CUDA will be installed and if -# this script was called with the argument --accept-cuda-eula. -# - hooks: We don't want hooks used in this install, we need vanilla -# installations of CUDA and/or other libraries -# - easystack: Path to easystack file that defines which packages shall be -# installed -accept_eula_opt= -if [[ ${eula_accepted} -eq 1 ]]; then - accept_eula_opt="--accept-eula-for=CUDA" -fi -touch "$tmpdir"/none.py -eb --prefix="$tmpdir" \ - --rebuild \ - --installpath-modules=${EASYBUILD_INSTALLPATH}/.modules \ - "${accept_eula_opt}" \ - --hooks="$tmpdir"/none.py \ - --easystack ${EASYSTACK_FILE} -ret=$? -if [ $ret -ne 0 ]; then - eb_last_log=$(unset EB_VERBOSE; eb --last-log) - cp -a ${eb_last_log} . - fatal_error "some installation failed, please check EasyBuild logs ${PWD}/$(basename ${eb_last_log})..." -else - echo_green "all installations at ${EESSI_SITE_SOFTWARE_PATH}/software/... succeeded!" -fi -# clean up tmpdir -rm -rf "${tmpdir}" + # clean up tmpdir + rm -rf "${tmpdir}" + + # Restore MODULEPATH for next loop iteration + MODUELPATH=${SAVE_MODULEPATH} +done From 18cdaa252b827744a7629d22d3dc150234292aca Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 18:04:22 +0200 Subject: [PATCH 1414/1795] not sure why this is not working, see if this solves it --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 60f3bb30be..1092472fc2 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -121,7 +121,7 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*.yml; do # Check if CUDA shall be installed cuda_install_needed=0 - cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | grep "module: CUDA/" + cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | grep "module: CUDA/" > /dev/null ret=$? if [ "${ret}" -eq 0 ]; then cuda_install_needed=1 @@ -136,7 +136,7 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*.yml; do # Check if cdDNN shall be installed cudnn_install_needed=0 - cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | grep "module: cuDNN/" + cat ${eb_dry_run_short_out} | grep "^ \* \[[ ]\]" | grep "module: cuDNN/" > /dev/null ret=$? if [ "${ret}" -eq 0 ]; then cudnn_install_needed=1 @@ -205,7 +205,7 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*.yml; do if [[ -z ${accept_eula_opt} ]]; then accept_eula_opt="--accept-eula-for=cuDNN" else - accept_eula_opt="${accept_eula_opt},cuDNN" + accept_eula_opt="$accept_eula_opt,cuDNN" fi fi touch "$tmpdir"/none.py From 57164242647969dfcf4bcfdf7dbe50f197602064 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 18:15:00 +0200 Subject: [PATCH 1415/1795] This was the only way in which I got this to work. Otherwise, it doesn't seem like the option is correctly accepted by EasyBuild - something related to bash variable expansion --- .../nvidia/install_cuda_and_libraries.sh | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 1092472fc2..73a4a7f6a9 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -199,21 +199,25 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*.yml; do # installed accept_eula_opt= if [[ ${cuda_eula_accepted} -eq 1 ]]; then - accept_eula_opt="--accept-eula-for=CUDA" + accept_eula_opt="CUDA" fi if [[ ${cudnn_eula_accepted} -eq 1 ]]; then if [[ -z ${accept_eula_opt} ]]; then - accept_eula_opt="--accept-eula-for=cuDNN" + accept_eula_opt="cuDNN" else - accept_eula_opt="$accept_eula_opt,cuDNN" + accept_eula_opt="${accept_eula_opt},cuDNN" fi fi touch "$tmpdir"/none.py - eb --prefix="$tmpdir" \ - --installpath-modules=${EASYBUILD_INSTALLPATH}/.modules \ - "${accept_cuda_eula_opt}" \ - --hooks="$tmpdir"/none.py \ - --easystack ${EASYSTACK_FILE} + eb_args="--prefix=$tmpdir" + eb_args="$eb_args --installpath-modules=${EASYBUILD_INSTALLPATH}/.modules" + eb_args="$eb_args --hooks="$tmpdir"/none.py" + eb_args="$eb_args --easystack ${EASYSTACK_FILE}" + if [[ ! -z ${accept_eula_opt} ]]; then + eb_args="$eb_args --accept-eula-for=$accept_eula_opt" + fi + echo "Running eb $eb_args" + eb $eb_args ret=$? if [ $ret -ne 0 ]; then eb_last_log=$(unset EB_VERBOSE; eb --last-log) From 9070096978e5dc77185e82db2c7f621ebacfb777 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 9 Oct 2024 20:19:16 +0200 Subject: [PATCH 1416/1795] Resolve all symlinks in LD_PRELOAD mode --- .../nvidia/link_nvidia_host_libraries.sh | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index b08a4b50b1..b9e6877ce9 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -213,42 +213,44 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Filter out all symlinks and libraries that have missing library dependencies under EESSI filtered_libraries=() for library in "${matched_libraries[@]}"; do - if [ ! -L "$library" ]; then - # $library is not a symlink - # Run ldd on the given binary and filter for "not found" libraries - NOT_FOUND_LIBS=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') - # Check if it is missing an so dep under EESSI - if [[ -z "$NOT_FOUND_LIBS" ]]; then - # Anything graphics is out, as is libnvidia-fbc* - if [[ "$library" != *"GL"* ]]; then - if [[ "$library" != *"libnvidia-fbc"* ]]; then + library=$(realpath "$library") + # Run ldd on the given binary and filter for "not found" libraries + NOT_FOUND_LIBS=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') + # Check if it is missing an so dep under EESSI + if [[ -z "$NOT_FOUND_LIBS" ]]; then + # Anything graphics is out, as is libnvidia-fbc* + if [[ "$library" != *"GL"* ]]; then + if [[ "$library" != *"libnvidia-fbc"* ]]; then + if [[ ! " ${filtered_libraries[@]} " =~ " $library " ]]; then filtered_libraries+=("$library") fi fi - else - # Iterate over "not found" libraries and check if they are in the array - all_found=true - for lib in $NOT_FOUND_LIBS; do - found=false - for listed_lib in "${matched_libraries[@]}"; do - if [[ "$lib" == "$listed_lib" ]]; then - found=true - break - fi - done - - if [[ "$found" == false ]]; then - echo "$lib is NOT in the provided preload list, filtering $library" - all_found=false + fi + else + # Iterate over "not found" libraries and check if they are in the array + all_found=true + for lib in $NOT_FOUND_LIBS; do + found=false + for listed_lib in "${matched_libraries[@]}"; do + if [[ "$lib" == "$listed_lib" ]]; then + found=true break fi done - # If we find all the missing libs in our list include it - if [[ "$all_found" == true ]]; then - # Anything graphics is out, as is libnvidia-fbc* - if [[ "$library" != *"GL"* ]]; then - if [[ "$library" != *"libnvidia-fbc"* ]]; then + if [[ "$found" == false ]]; then + echo "$lib is NOT in the provided preload list, filtering $library" + all_found=false + break + fi + done + + # If we find all the missing libs in our list include it + if [[ "$all_found" == true ]]; then + # Anything graphics is out, as is libnvidia-fbc* + if [[ "$library" != *"GL"* ]]; then + if [[ "$library" != *"libnvidia-fbc"* ]]; then + if [[ ! " ${filtered_libraries[@]} " =~ " $library " ]]; then filtered_libraries+=("$library") fi fi From add2574606646105a72975ee7916824319084ac9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 9 Oct 2024 20:26:51 +0200 Subject: [PATCH 1417/1795] Don't resolve symlinks until we store them --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index b9e6877ce9..97948a5e70 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -213,7 +213,6 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Filter out all symlinks and libraries that have missing library dependencies under EESSI filtered_libraries=() for library in "${matched_libraries[@]}"; do - library=$(realpath "$library") # Run ldd on the given binary and filter for "not found" libraries NOT_FOUND_LIBS=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') # Check if it is missing an so dep under EESSI @@ -221,6 +220,8 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Anything graphics is out, as is libnvidia-fbc* if [[ "$library" != *"GL"* ]]; then if [[ "$library" != *"libnvidia-fbc"* ]]; then + # Resolve any symlink + library=$(realpath "$library") if [[ ! " ${filtered_libraries[@]} " =~ " $library " ]]; then filtered_libraries+=("$library") fi @@ -250,6 +251,8 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Anything graphics is out, as is libnvidia-fbc* if [[ "$library" != *"GL"* ]]; then if [[ "$library" != *"libnvidia-fbc"* ]]; then + # Resolve any symlink + library=$(realpath "$library") if [[ ! " ${filtered_libraries[@]} " =~ " $library " ]]; then filtered_libraries+=("$library") fi From 9fcf33c8c76fd1b41543393a18fc380e04fc9dc3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 9 Oct 2024 20:40:23 +0200 Subject: [PATCH 1418/1795] Allow partial maps for dependent .so files --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 97948a5e70..a35023ebdb 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -233,7 +233,8 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then for lib in $NOT_FOUND_LIBS; do found=false for listed_lib in "${matched_libraries[@]}"; do - if [[ "$lib" == "$listed_lib" ]]; then + # Matching to the .so is enough + if [[ "$lib" == "$listed_lib"* ]]; then found=true break fi From 26a91f07876e31d55f901c4ec14737f0e5ecadb7 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 9 Oct 2024 20:57:58 +0200 Subject: [PATCH 1419/1795] Make sure EESSI is initialised --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index a35023ebdb..1b450c6a37 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -130,6 +130,9 @@ check_global_read() { fi } +# Make sure EESSI is initialised (doesn't matter what version) +check_eessi_initialised + # Check for required commands command -v nvidia-smi >/dev/null 2>&1 || { echo_yellow "nvidia-smi not found, this script won't do anything useful"; return 1; } From 9f3853ca08d76def4a8b818cb0212693565d9be6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 21:17:04 +0200 Subject: [PATCH 1420/1795] Added include-easyblocks-from-commit --- .../eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml index 2a6f9bd6f9..4e3fffacca 100644 --- a/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml +++ b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml @@ -2,4 +2,7 @@ # for nvidia GPU support, because they cannot (fully) be shipped as part of EESSI due to license constraints easyconfigs: - CUDA-12.1.1.eb - - cuDNN-8.9.2.26-CUDA-12.1.1.eb + - cuDNN-8.9.2.26-CUDA-12.1.1.eb: + options: + # Needed for support for --accept-uela-for option + include-easyblocks-from-commit: 11afb88ec55e0ca431cbe823696aa43e2a9bfca8 From b9017d446053807152cfb84681d10c896f9c6f81 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 21:25:43 +0200 Subject: [PATCH 1421/1795] Easystack no longer passed as option. Comment is outdated, since we now _do_ have a separate EasyStack file in scripts/gpu_support/nvidia/easystacks --- EESSI-install-software.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 4c85fdfc31..7c4c1036d1 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -251,16 +251,11 @@ fi temp_install_storage=${TMPDIR}/temp_install_storage mkdir -p ${temp_install_storage} -# Note the eessi...CUDA.yml file(s) is(are) copied by 'install_scripts.sh' from -# the EESSI/software-layer easystacks/software.eessi.io/2023.06/accel/nvidia -# directory to /cvmfs to avoid keeping them in sync manually. If more than one -# such file is used (e.g., because different EasyBuild versions were used), the -# install script 'install_cuda_and_libraries.sh' has to be run multiple times. if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \ - -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml \ -t ${temp_install_storage} \ - --accept-cuda-eula + --accept-cuda-eula \ + --accept-cudnn-eula else echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found" fi From eea2879a7d5afee4d3b5f6cbdabe7b925305c56f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 21:37:59 +0200 Subject: [PATCH 1422/1795] Make sure easystack file for host_injections is shipped --- install_scripts.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/install_scripts.sh b/install_scripts.sh index ad73e769dd..b6b5ac92b0 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -128,16 +128,12 @@ nvidia_files=( ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" -# special treatment for the easystack file(s) that lists CUDA and cu* libraries -# To be picked up by a build job they have to be stored under -# easystacks/software.eessi.io/2023.06/accel/nvidia/ on GitHub. -# To avoid keeping that file and the one that we distribute via CernVM-FS so -# users/sites can install the full CUDA SDK and cu* libraries under -# 'host_injections' we copy the above file to the right location under /cvmfs. -nvidia_host_injections_files=( - eessi-2023.06-eb-4.9.4-2023a-CUDA.yml +# Easystacks to be used to install software in host injections +host_injections_easystacks=( + eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml ) -copy_files_by_list ${TOPDIR}/easystacks/software.eessi.io/2023.06/accel/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_host_injections_files[@]}" +copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia/easystacks \ +${INSTALL_PREFIX}/scripts/gpu_support/nvidia/easystacks "${host_injections_easystacks[@]}" # Copy over EasyBuild hooks file used for installations hook_files=( From 33199d78a34f8ead4d834af3225608ba253ccf21 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 22:04:23 +0200 Subject: [PATCH 1423/1795] Remove rebuild, change comments that were out of date --- .../gpu_support/nvidia/install_cuda_and_libraries.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 73a4a7f6a9..b3da2a13da 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -116,7 +116,7 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*.yml; do # to be installed echo ">> Determining if packages specified in ${EASYSTACK_FILE} are missing under ${EESSI_SITE_SOFTWARE_PATH}" eb_dry_run_short_out=${tmpdir}/eb_dry_run_short.out - eb --dry-run-short --rebuild --easystack ${EASYSTACK_FILE} 2>&1 | tee ${eb_dry_run_short_out} + eb --dry-run-short --easystack ${EASYSTACK_FILE} 2>&1 | tee ${eb_dry_run_short_out} ret=$? # Check if CUDA shall be installed @@ -185,12 +185,8 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*.yml; do # Brief explanation of parameters: # - prefix: using $tmpdir as default base directory for several EB settings - # - rebuild: we need the --rebuild option, as the CUDA module may or may not be on the - # `MODULEPATH` yet. Even if it is, we still want to redo this installation - # since it will provide the symlinked targets for the parts of the CUDA - # and/or other installation in the `.../versions/...` prefix - # - installpath-modules: We install the module in our `tmpdir` since we do not need the modulefile, - # we only care about providing the targets for the symlinks. + # - installpath-modules: We install the module in a hidden .modules, so that next time this script + # is run, it is not reinstalled. # - ${accept_eula_opt}: We only set the --accept-eula-for=CUDA option if CUDA will be installed and if # this script was called with the argument --accept-cuda-eula. # - hooks: We don't want hooks used in this install, we need vanilla From 06cd2eae5f169d3f7eb1951acef2175b0d8d3719 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 9 Oct 2024 22:07:25 +0200 Subject: [PATCH 1424/1795] Only loop over the easystacks with CUDA in the name --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index b3da2a13da..3d273f0fbd 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -94,7 +94,7 @@ echo "Created temporary directory '${tmpdir}'" # use install_path/modules/all as MODULEPATH SAVE_MODULEPATH=${MODULEPATH} -for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*.yml; do +for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do echo -e "Processing easystack file ${easystack_file}...\n\n" # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file From 5b3fb27acdaa4a6ab92325b4ffa4abaabea6e930 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 10 Oct 2024 11:57:37 +0200 Subject: [PATCH 1425/1795] Try to improve things --- .../nvidia/link_nvidia_host_libraries.sh | 79 +++++++++++++------ 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index a35023ebdb..2a2306d2d8 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -209,32 +209,45 @@ echo "Matched ${#matched_libraries[@]} CUDA Libraries" if [ "$LD_PRELOAD_MODE" -eq 1 ]; then echo echo_yellow "When attempting to use LD_PRELOAD we exclude anything related to graphics" - + local cuda_compat_nvlib_list=( + "libcuda.so" + "libcudadebugger.so" + "libnvidia-nvvm.so" + "libnvidia-ptxjitcompiler.so" + ) # Filter out all symlinks and libraries that have missing library dependencies under EESSI filtered_libraries=() + compat_filtered_libraries=() for library in "${matched_libraries[@]}"; do # Run ldd on the given binary and filter for "not found" libraries - NOT_FOUND_LIBS=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') + not_found_libs=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') # Check if it is missing an so dep under EESSI - if [[ -z "$NOT_FOUND_LIBS" ]]; then - # Anything graphics is out, as is libnvidia-fbc* - if [[ "$library" != *"GL"* ]]; then - if [[ "$library" != *"libnvidia-fbc"* ]]; then - # Resolve any symlink - library=$(realpath "$library") - if [[ ! " ${filtered_libraries[@]} " =~ " $library " ]]; then - filtered_libraries+=("$library") + if [[ -z "$not_found_libs" ]]; then + # Resolve any symlink + realpath_library=$(realpath "$library") + if [[ ! " ${filtered_libraries[@]} " =~ " $realpath_library " ]]; then + filtered_libraries+=("$realpath_library") + # Also prepare compat only libraries for the short list + for item in "${cuda_compat_nvlib_list[@]}"; do + # Check if the current item is a substring of $library + if [[ "$realpath_library" == "$item"* ]]; then + echo "Match found for $item for CUDA compat libraries" + if [[ ! " ${compat_filtered_libraries[@]} " =~ " $realpath_library " ]]; then + compat_filtered_libraries+=("$realpath_library") + fi + break fi - fi - fi + done + fi else # Iterate over "not found" libraries and check if they are in the array all_found=true - for lib in $NOT_FOUND_LIBS; do + for lib in $not_found_libs; do found=false for listed_lib in "${matched_libraries[@]}"; do - # Matching to the .so is enough - if [[ "$lib" == "$listed_lib"* ]]; then + # Matching to the .so or a symlink target is enough + realpath_lib=$(realpath "$listed_lib") + if [[ "$lib" == "$listed_lib"* || "$lib" == "$realpath_lib" ]]; then found=true break fi @@ -249,15 +262,21 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # If we find all the missing libs in our list include it if [[ "$all_found" == true ]]; then - # Anything graphics is out, as is libnvidia-fbc* - if [[ "$library" != *"GL"* ]]; then - if [[ "$library" != *"libnvidia-fbc"* ]]; then - # Resolve any symlink - library=$(realpath "$library") - if [[ ! " ${filtered_libraries[@]} " =~ " $library " ]]; then - filtered_libraries+=("$library") + # Resolve any symlink + realpath_library=$(realpath "$library") + if [[ ! " ${filtered_libraries[@]} " =~ " $realpath_library " ]]; then + filtered_libraries+=("$realpath_library") + # Also prepare compat only libraries for the short list + for item in "${cuda_compat_nvlib_list[@]}"; do + # Check if the current item is a substring of $library + if [[ "$realpath_library" == "$item"* ]]; then + echo "Match found for $item for CUDA compat libraries" + if [[ ! " ${compat_filtered_libraries[@]} " =~ " $realpath_library " ]]; then + compat_filtered_libraries+=("$realpath_library") + fi + break fi - fi + done fi fi fi @@ -266,18 +285,28 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Set EESSI_GPU_LD_PRELOAD with the matched libraries if [ ${#filtered_libraries[@]} -gt 0 ]; then echo - echo_yellow "The recommended way to use LD_PRELOAD is to only use it when you need to:" + echo_yellow "The recommended way to use LD_PRELOAD is to only use it when you need to." echo + EESSI_GPU_COMPAT_LD_PRELOAD=$(printf "%s\n" "${compat_filtered_libraries[@]}" | tr '\n' ':') + # Remove the trailing colon from LD_PRELOAD if it exists + EESSI_GPU_COMPAT_LD_PRELOAD=${EESSI_GPU_COMPAT_LD_PRELOAD%:} + export EESSI_GPU_COMPAT_LD_PRELOAD + echo_yellow "A minimal preload which should work in most cases:" + echo_green "export EESSI_GPU_COMPAT_LD_PRELOAD=\"$EESSI_GPU_LD_PRELOAD\"" + echo + EESSI_GPU_LD_PRELOAD=$(printf "%s\n" "${filtered_libraries[@]}" | tr '\n' ':') # Remove the trailing colon from LD_PRELOAD if it exists EESSI_GPU_LD_PRELOAD=${EESSI_GPU_LD_PRELOAD%:} export EESSI_GPU_LD_PRELOAD + echo_yellow "A corner-case full preload (which is hard on memory) for exceptional use:" + echo_green "export EESSI_GPU_LD_PRELOAD=\"$EESSI_GPU_LD_PRELOAD\"" export EESSI_OVERRIDE_GPU_CHECK=1 echo_green "export EESSI_OVERRIDE_GPU_CHECK=\"$EESSI_OVERRIDE_GPU_CHECK\"" echo echo_yellow "Then you can set LD_PRELOAD only when you want to run a GPU application, e.g.," - echo_yellow " LD_PRELOAD=\"\$EESSI_GPU_LD_PRELOAD\" device_query" + echo_yellow " LD_PRELOAD=\"\$EESSI_GPU_COMPAT_LD_PRELOAD\" device_query" else echo "No libraries matched, LD_PRELOAD not set." fi From 6534b99d2c5a78528a872f6787b991ba7a69abc6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 10 Oct 2024 12:03:44 +0200 Subject: [PATCH 1426/1795] Try to improve things --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 5fb3bfc2ba..ce3d75ab4a 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -233,7 +233,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Also prepare compat only libraries for the short list for item in "${cuda_compat_nvlib_list[@]}"; do # Check if the current item is a substring of $library - if [[ "$realpath_library" == "$item"* ]]; then + if [[ "$library" == "$item"* ]]; then echo "Match found for $item for CUDA compat libraries" if [[ ! " ${compat_filtered_libraries[@]} " =~ " $realpath_library " ]]; then compat_filtered_libraries+=("$realpath_library") @@ -272,7 +272,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Also prepare compat only libraries for the short list for item in "${cuda_compat_nvlib_list[@]}"; do # Check if the current item is a substring of $library - if [[ "$realpath_library" == "$item"* ]]; then + if [[ "$library" == "$item"* ]]; then echo "Match found for $item for CUDA compat libraries" if [[ ! " ${compat_filtered_libraries[@]} " =~ " $realpath_library " ]]; then compat_filtered_libraries+=("$realpath_library") From d0b02ed4c2e456ed6612d4605f9950245506e366 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 10 Oct 2024 12:04:24 +0200 Subject: [PATCH 1427/1795] Try to improve things --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index ce3d75ab4a..7083905446 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -212,7 +212,7 @@ echo "Matched ${#matched_libraries[@]} CUDA Libraries" if [ "$LD_PRELOAD_MODE" -eq 1 ]; then echo echo_yellow "When attempting to use LD_PRELOAD we exclude anything related to graphics" - local cuda_compat_nvlib_list=( + cuda_compat_nvlib_list=( "libcuda.so" "libcudadebugger.so" "libnvidia-nvvm.so" From e9a1d293cc7c2bfeb2854bae155e74437d76a555 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 10 Oct 2024 12:07:54 +0200 Subject: [PATCH 1428/1795] Try to improve things --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 7083905446..d716f5b2e9 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -233,7 +233,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Also prepare compat only libraries for the short list for item in "${cuda_compat_nvlib_list[@]}"; do # Check if the current item is a substring of $library - if [[ "$library" == "$item"* ]]; then + if [[ "$realpath_library" == *"$item"* ]]; then echo "Match found for $item for CUDA compat libraries" if [[ ! " ${compat_filtered_libraries[@]} " =~ " $realpath_library " ]]; then compat_filtered_libraries+=("$realpath_library") @@ -272,7 +272,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Also prepare compat only libraries for the short list for item in "${cuda_compat_nvlib_list[@]}"; do # Check if the current item is a substring of $library - if [[ "$library" == "$item"* ]]; then + if [[ "$realpath_library" == *"$item"* ]]; then echo "Match found for $item for CUDA compat libraries" if [[ ! " ${compat_filtered_libraries[@]} " =~ " $realpath_library " ]]; then compat_filtered_libraries+=("$realpath_library") From ac224a09f450af97dd82c41a55fd17123b918cd8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 10 Oct 2024 12:11:07 +0200 Subject: [PATCH 1429/1795] Try to improve things --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index d716f5b2e9..1b02c14efa 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -295,7 +295,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then EESSI_GPU_COMPAT_LD_PRELOAD=${EESSI_GPU_COMPAT_LD_PRELOAD%:} export EESSI_GPU_COMPAT_LD_PRELOAD echo_yellow "A minimal preload which should work in most cases:" - echo_green "export EESSI_GPU_COMPAT_LD_PRELOAD=\"$EESSI_GPU_LD_PRELOAD\"" + echo_green "export EESSI_GPU_COMPAT_LD_PRELOAD=\"$EESSI_GPU_COMPAT_LD_PRELOAD\"" echo EESSI_GPU_LD_PRELOAD=$(printf "%s\n" "${filtered_libraries[@]}" | tr '\n' ':') From 465dd47e7aa7f9a873925339fc63790a1ae839ea Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 10 Oct 2024 12:14:56 +0200 Subject: [PATCH 1430/1795] Try to improve things --- scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 1b02c14efa..967e7d3589 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -250,7 +250,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then for listed_lib in "${matched_libraries[@]}"; do # Matching to the .so or a symlink target is enough realpath_lib=$(realpath "$listed_lib") - if [[ "$lib" == "$listed_lib"* || "$lib" == "$realpath_lib" ]]; then + if [[ "$lib" == "$listed_lib"* || "$realpath_lib" == *"$lib" ]]; then found=true break fi From 887bc8a84f1ea07bec56cca5b675309938d049e5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 11 Oct 2024 17:43:35 +0200 Subject: [PATCH 1431/1795] Add accelerator detection to Lmod version of EESSI initialisation --- init/lmod_eessi_archdetect_wrapper_accel.sh | 2 ++ init/modules/EESSI/2023.06.lua | 35 +++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 init/lmod_eessi_archdetect_wrapper_accel.sh diff --git a/init/lmod_eessi_archdetect_wrapper_accel.sh b/init/lmod_eessi_archdetect_wrapper_accel.sh new file mode 100644 index 0000000000..d4a0038cb5 --- /dev/null +++ b/init/lmod_eessi_archdetect_wrapper_accel.sh @@ -0,0 +1,2 @@ +# This can be leveraged by the source_sh() feature of Lmod +export EESSI_ACCEL_SUBDIR=$($(dirname $(readlink -f $BASH_SOURCE))/eessi_archdetect.sh accelpath) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 463706ce6c..d3ecf58813 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -42,14 +42,27 @@ function archdetect_cpu() end LmodError("Software directory check for the detected architecture failed") end +function archdetect_accel() + local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper_accel.sh') + if not os.getenv("EESSI_ACCEL_SUBDIR") then + if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then + LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ACCEL_SUBDIR to the available accelerator architecture in the form of: accel/nvidia/cc80") + end + source_sh("bash", script) + end + local archdetect_accel = os.getenv("EESSI_ACCEL_SUBDIR") or "" + return archdetect_accel +end local archdetect = archdetect_cpu() +local archdetect_accel = archdetect_accel() local eessi_cpu_family = archdetect:match("([^/]+)") local eessi_software_subdir = archdetect local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family) local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) -local eessi_module_path = pathJoin(eessi_software_path, "modules", "all") +local eessi_modules_subdir = pathJoin("modules", "all") +local eessi_module_path = pathJoin(eessi_software_path, eessi_modules_subdir) local eessi_site_software_path = string.gsub(eessi_software_path, "versions", "host_injections") -local eessi_site_module_path = pathJoin(eessi_site_software_path, "modules", "all") +local eessi_site_module_path = pathJoin(eessi_site_software_path, eessi_modules_subdir) setenv("EPREFIX", eessi_eprefix) setenv("EESSI_CPU_FAMILY", eessi_cpu_family) setenv("EESSI_SITE_SOFTWARE_PATH", eessi_site_software_path) @@ -65,8 +78,24 @@ if ( mode() ~= "spider" ) then prepend_path("MODULEPATH", eessi_module_path) end prepend_path("LMOD_RC", pathJoin(eessi_software_path, "/.lmod/lmodrc.lua")) -prepend_path("MODULEPATH", eessi_site_module_path) setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) + +-- the accelerator may have an empty value and we need to give some flexibility +-- * construct the path we expect to find +-- * then check it exists +-- * then update the modulepath +if not (archdetect_accel == nil or archdetect_accel == '') then + eessi_accel_software_subdir = os.getenv("EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE") or eessi_software_subdir + eessi_accel_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_accel_software_subdir) + eessi_module_path_accel = pathJoin(eessi_accel_software_path, eessi_accel_software_subdir, eessi_modules_subdir) + if isDir(eessi_modulepath_accel) then + setenv("EESSI_MODULEPATH_ACCEL", eessi_module_path_accel) + prepend_path("MODULEPATH", eessi_module_path_accel) + end +end + +-- prepend the site module path last so it has priority +prepend_path("MODULEPATH", eessi_site_module_path) if mode() == "load" then LmodMessage("EESSI/" .. eessi_version .. " loaded successfully") end From acc2f083c312890b077ac93a13057b0330c70906 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Oct 2024 17:55:09 +0200 Subject: [PATCH 1432/1795] {2023.6}[2023b,a64fx] ESPResSo 4.2.2 + pyMBE 0.8.0 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml index 11523e7f03..a60f9bec6a 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023b.yml @@ -1,2 +1,4 @@ easyconfigs: - SciPy-bundle-2023.11-gfbf-2023b.eb + - ESPResSo-4.2.2-foss-2023b.eb + - pyMBE-0.8.0-foss-2023b.eb From 2c7553bb4a34f5c09e5b9bc4e83d8d0801bdc2a9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 11 Oct 2024 18:37:10 +0200 Subject: [PATCH 1433/1795] Make a separate PR for the lmod wrapper --- init/lmod_eessi_archdetect_wrapper_accel.sh | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 init/lmod_eessi_archdetect_wrapper_accel.sh diff --git a/init/lmod_eessi_archdetect_wrapper_accel.sh b/init/lmod_eessi_archdetect_wrapper_accel.sh deleted file mode 100644 index d4a0038cb5..0000000000 --- a/init/lmod_eessi_archdetect_wrapper_accel.sh +++ /dev/null @@ -1,2 +0,0 @@ -# This can be leveraged by the source_sh() feature of Lmod -export EESSI_ACCEL_SUBDIR=$($(dirname $(readlink -f $BASH_SOURCE))/eessi_archdetect.sh accelpath) From 275edb404b8c9a63a445fa6ad181de8d69d34e4f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 11 Oct 2024 18:38:51 +0200 Subject: [PATCH 1434/1795] Put a Lmod-relevant wrapper in place for archdetect accelerator detection --- init/lmod_eessi_archdetect_wrapper_accel.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 init/lmod_eessi_archdetect_wrapper_accel.sh diff --git a/init/lmod_eessi_archdetect_wrapper_accel.sh b/init/lmod_eessi_archdetect_wrapper_accel.sh new file mode 100644 index 0000000000..d4a0038cb5 --- /dev/null +++ b/init/lmod_eessi_archdetect_wrapper_accel.sh @@ -0,0 +1,2 @@ +# This can be leveraged by the source_sh() feature of Lmod +export EESSI_ACCEL_SUBDIR=$($(dirname $(readlink -f $BASH_SOURCE))/eessi_archdetect.sh accelpath) From 90a80dace2a4cc953f082b71d0deb4577e05a3ec Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 11 Oct 2024 18:42:46 +0200 Subject: [PATCH 1435/1795] Make sure the wrapper is shipped --- install_scripts.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index 11c7fc2a9f..09a4c5847f 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -86,7 +86,8 @@ TOPDIR=$(dirname $(realpath $0)) # Copy for init directory init_files=( bash eessi_archdetect.sh eessi_defaults eessi_environment_variables eessi_software_subdir_for_host.py - minimal_eessi_env README.md test.py lmod_eessi_archdetect_wrapper.sh + minimal_eessi_env README.md test.py lmod_eessi_archdetect_wrapper.sh lmod_eessi_archdetect_wrapper_accel.sh + ) copy_files_by_list ${TOPDIR}/init ${INSTALL_PREFIX}/init "${init_files[@]}" From 6c16231bce0b75e499bdfa7bed513eadbec324ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Oct 2024 18:58:19 +0200 Subject: [PATCH 1436/1795] {2023.06}[2023a,a64fx] waLBerla 6.1 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml new file mode 100644 index 0000000000..55ecb2a3a9 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml @@ -0,0 +1,5 @@ +easyconfigs: + - waLBerla-6.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 + from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 From 2409a25b25cd7a15444265c4704722ab8f02f6ef Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Oct 2024 18:32:47 +0200 Subject: [PATCH 1437/1795] {2023.06}[2023a,a64fx] ROOT 6.30.06 --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml | 7 +++++++ eb_hooks.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml new file mode 100644 index 0000000000..19d9aedfae --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-2023a.yml @@ -0,0 +1,7 @@ +easyconfigs: + - ROOT-6.30.06-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 diff --git a/eb_hooks.py b/eb_hooks.py index 79bdeeee0d..78a2fa9c57 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -91,7 +91,7 @@ def post_ready_hook(self, *args, **kwargs): memory_hungry_build = self.name in ['libxc', 'MBX', 'TensorFlow'] # on A64FX systems, (HBM) memory is typically scarce, so we need to use fewer cores for some builds cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - memory_hungry_build_a64fx = cpu_target == CPU_TARGET_A64FX and self.name in ['Qt5'] + memory_hungry_build_a64fx = cpu_target == CPU_TARGET_A64FX and self.name in ['Qt5', 'ROOT'] if memory_hungry_build or memory_hungry_build_a64fx: parallel = self.cfg['parallel'] if parallel > 1: From 0e7c9d8a4f5150c6734f35d0d5ebc13ba2990150 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 09:54:14 +0200 Subject: [PATCH 1438/1795] add a bit more debug output, use *SITE_SOFTWARE_PATH and minor tweaks --- EESSI-install-software.sh | 1 + scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f20b1b6c64..235e29477e 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -249,6 +249,7 @@ fi # Allow skipping CUDA SDK install in e.g. CI environments # The install_cuda... script uses EasyBuild. So, we need to check if we have EB # or skip this step. +echo "Going to install full CUDA SDK and cu* libraries under host_injections if necessary" module_avail_out=$TMPDIR/ml.out module avail 2>&1 | grep EasyBuild &> ${module_avail_out} if [[ $? -eq 0 ]]; then diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 3d273f0fbd..5250348b40 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -38,6 +38,7 @@ show_help() { # Initialize variables cuda_eula_accepted=0 +cudnn_eula_accepted=0 EASYSTACK_FILE= TEMP_DIR= @@ -91,7 +92,7 @@ else fi echo "Created temporary directory '${tmpdir}'" -# use install_path/modules/all as MODULEPATH +# use EESSI_SITE_SOFTWARE_PATH/.modules/all as MODULEPATH SAVE_MODULEPATH=${MODULEPATH} for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do @@ -101,11 +102,12 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do eb_version=$(echo ${EASYSTACK_FILE} | sed 's/.*eb-\([0-9.]*\).*/\1/g') # Load EasyBuild version for this easystack file _before_ loading EESSI-extend + module avail EasyBuild module load EasyBuild/${eb_version} module load EESSI-extend/${EESSI_VERSION}-easybuild # Install modules in hidden .modules dir to keep track of what was installed before - MODULEPATH=${EASYBUILD_INSTALLPATH}/.modules/all + MODULEPATH=${EESSI_SITE_SOFTWARE_PATH}/.modules/all echo "set MODULEPATH=${MODULEPATH}" # show EasyBuild configuration @@ -145,7 +147,7 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do # Make sure the cuDNN EULA is accepted if it shall be installed if [ "${cudnn_install_needed}" -eq 1 ] && [ "${cudnn_eula_accepted}" -ne 1 ]; then show_help - error="\ncuDNN shall be installed. However, the cuDNNDA EULA has not been accepted\nYou _must_ accept the cuDNN EULA via the appropriate command line option.\n" + error="\ncuDNN shall be installed. However, the cuDNN EULA has not been accepted\nYou _must_ accept the cuDNN EULA via the appropriate command line option.\n" fatal_error "${error}" fi From d57f8d8caa7dd17dd3e8e82a174f586998db08b0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 10:33:51 +0200 Subject: [PATCH 1439/1795] replace TAB with WHITESPACEs --- init/eessi_environment_variables | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index d2daf40ace..60d69cc198 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -153,10 +153,10 @@ if [ -d $EESSI_PREFIX ]; then fi # Fix wrong path for RHEL >=8 libcurl - # This is required here because we ship curl in our compat layer. If we only provided - # curl as a module file we could instead do this via a `modluafooter` in an EasyBuild - # hook (or via an Lmod hook) - rhel_libcurl_file="/etc/pki/tls/certs/ca-bundle.crt" + # This is required here because we ship curl in our compat layer. If we only provided + # curl as a module file we could instead do this via a `modluafooter` in an EasyBuild + # hook (or via an Lmod hook) + rhel_libcurl_file="/etc/pki/tls/certs/ca-bundle.crt" if [ -f $rhel_libcurl_file ]; then show_msg "Found libcurl CAs file at RHEL location, setting CURL_CA_BUNDLE" export CURL_CA_BUNDLE=$rhel_libcurl_file From 02d3e1eaf6c62b6811f4920c8a1eaf77afb4ae08 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 10:42:20 +0200 Subject: [PATCH 1440/1795] show more msgs when building and init full environment --- EESSI-install-software.sh | 2 +- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 235e29477e..31c3adeccb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -173,7 +173,7 @@ fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages # $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables +EESSI_SILENT=0 EESSI_BASIC_ENV=0 source $TOPDIR/init/eessi_environment_variables if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then fatal_error "Failed to determine software subdirectory?!" diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 5250348b40..7d30c65def 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -79,6 +79,7 @@ check_eessi_initialised # Make sure that `EESSI-extend` will install in the site installation path EESSI_SITE_SOFTWARE_PATH export EESSI_SITE_INSTALL=1 +echo "EESSI_SITE_SOFTWARE_PATH=${EESSI_SITE_SOFTWARE_PATH}" # we need a directory we can use for temporary storage if [[ -z "${TEMP_DIR}" ]]; then From c1d439483d0dae552b71a57c851108b83dd1cbae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 15 Oct 2024 10:45:27 +0200 Subject: [PATCH 1441/1795] rebuild LAMMPS for generic CPU targets --- .../20241015-eb-4.9.4-LAMMPS-generic-builds.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml new file mode 100644 index 0000000000..36e887644d --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml @@ -0,0 +1,16 @@ +# 2024.10.15 +# Generic build of LAMMPS have optimizations for the build host CPU, +# this is fixed in https://github.com/easybuilders/easybuild-easyblocks/pull/3484 +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb: + options: + # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3484/commits/64060fa9b6a4d444406cc403ae00fbbc3f36c416 + include-easyblocks-from-commit: 64060fa9b6a4d444406cc403ae00fbbc3f36c416 + - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 + from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 + # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3484/commits/64060fa9b6a4d444406cc403ae00fbbc3f36c416 + include-easyblocks-from-commit: 64060fa9b6a4d444406cc403ae00fbbc3f36c416 + + From b28017ad61c7bcc049edc522506b93b05d36f73b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 10:59:48 +0200 Subject: [PATCH 1442/1795] use zero length env vars --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 31c3adeccb..84c96ad369 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -173,7 +173,7 @@ fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages # $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT=0 EESSI_BASIC_ENV=0 source $TOPDIR/init/eessi_environment_variables +EESSI_SILENT= EESSI_BASIC_ENV= source $TOPDIR/init/eessi_environment_variables if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then fatal_error "Failed to determine software subdirectory?!" From 20aacdca6acf7013bea66a283086c63e4cc9761b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 11:48:20 +0200 Subject: [PATCH 1443/1795] fix syntax issue --- .../eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml index 4e3fffacca..677627eed3 100644 --- a/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml +++ b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml @@ -3,6 +3,6 @@ easyconfigs: - CUDA-12.1.1.eb - cuDNN-8.9.2.26-CUDA-12.1.1.eb: - options: - # Needed for support for --accept-uela-for option - include-easyblocks-from-commit: 11afb88ec55e0ca431cbe823696aa43e2a9bfca8 + options: + # Needed for support for --accept-uela-for option + include-easyblocks-from-commit: 11afb88ec55e0ca431cbe823696aa43e2a9bfca8 From 88bdb888586ff96a3a2335244327f7336adab2a5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 12:13:25 +0200 Subject: [PATCH 1444/1795] tweak variable expansion in test --- init/eessi_environment_variables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 60d69cc198..125f3e5a93 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -10,7 +10,7 @@ function error() { function show_msg { # only echo msg if EESSI_SILENT is unset msg=$1 - if [[ -z ${EESSI_SILENT+x} ]]; then + if [[ -z ${EESSI_SILENT:+x} ]]; then echo "$msg" fi } From 7b8ba8b8a1a6e3c99dce3b74ffa1f568140ccfa7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 13:24:12 +0200 Subject: [PATCH 1445/1795] dont use hooks when installing into host_injections --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 7d30c65def..3590cf3a86 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -99,6 +99,10 @@ SAVE_MODULEPATH=${MODULEPATH} for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do echo -e "Processing easystack file ${easystack_file}...\n\n" + # We don't want hooks used in this install, we need vanilla installations + touch "${tmpdir}"/none.py + export EASYBUILD_HOOKS="${tmpdir}/none.py" + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file eb_version=$(echo ${EASYSTACK_FILE} | sed 's/.*eb-\([0-9.]*\).*/\1/g') From cf5491b1f1cdc946af90c9b130b675ccec16c3f5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 13:58:37 +0200 Subject: [PATCH 1446/1795] Add accelerator check to Lmod init script --- .github/workflows/tests_eessi_module.yml | 35 ++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index cbcffe6385..3c29c6ee56 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -7,13 +7,13 @@ on: permissions: contents: read # to fetch code (actions/checkout) jobs: - build: + basic_checks: runs-on: ubuntu-latest strategy: fail-fast: false matrix: EESSI_VERSION: - - 2023.06 + - 2023.06 steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -60,10 +60,40 @@ jobs: fi unset EESSI_ARCHDETECT_OPTIONS set -e # Re-enable exit on non-zero status + + lmod_and_init_script_comparison: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: + - x86_64/amd/zen2 + EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE: + - none + - accel/nvidia/cc80 + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io - name: Test for expected variables while adding dummy cpu archs and loading EESSI module run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod + + # Set our path overrides according to our matrix + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} + if [[ "${{matrix.EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE}}" != "none" ]]; then + export EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE}} + fi + export MODULEPATH=init/modules CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" @@ -83,4 +113,3 @@ jobs: diff "${moduleoutfile}" "${sourceoutfile}" exit 1 fi - From 0e20f008382ca6cf00afb64a9d143debfa0e2c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 15 Oct 2024 14:00:08 +0200 Subject: [PATCH 1447/1795] individually remove every file of the installation dir --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 98576efcb0..1d38092004 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -127,7 +127,7 @@ if [ $EUID -eq 0 ]; then app_dir=${app_installprefix}/software/${app} app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - rm -rf ${app_dir} + find ${app_dir} -exec rm -rf {} \; rm -rf ${app_module} done else From e7eb8798181e71bde87077339893c45891c5f59c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 14:25:50 +0200 Subject: [PATCH 1448/1795] Match all variables that start with EESSI_ --- .github/workflows/tests_eessi_module.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 3c29c6ee56..c874b50403 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -100,10 +100,11 @@ jobs: moduleoutfile="moduleout.txt" sourceoutfile="sourceout.txt" module load EESSI/${{matrix.EESSI_VERSION}} - env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${moduleoutfile}" + unset EESSI_ARCHDETECT_OPTIONS + env | grep -E '^EESSI_' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} source ./init/bash - env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${sourceoutfile}" + env | grep -E '^EESSI_' | sort > "${sourceoutfile}" cat "${moduleoutfile}" cat "${sourceoutfile}" if (diff "${moduleoutfile}" "${sourceoutfile}" > /dev/null); then From 1091ab75ac474fb8e74c75cd5d93c8fb7f437b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 15 Oct 2024 14:27:27 +0200 Subject: [PATCH 1449/1795] only find and remove files, and then remove the entire install dir --- EESSI-remove-software.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 1d38092004..fbd2760596 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -127,7 +127,8 @@ if [ $EUID -eq 0 ]; then app_dir=${app_installprefix}/software/${app} app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - find ${app_dir} -exec rm -rf {} \; + find ${app_dir} -type f -exec rm -f {} \; + rm -rf ${app_dir} rm -rf ${app_module} done else From 1311055a365937a03b5577c6fda8ba19af1c2580 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 14:32:39 +0200 Subject: [PATCH 1450/1795] Lmod init uses archdetect, but init script still uses archspec --- .github/workflows/tests_eessi_module.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index c874b50403..f581bda3b7 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -69,7 +69,7 @@ jobs: EESSI_VERSION: - 2023.06 EESSI_SOFTWARE_SUBDIR_OVERRIDE: - - x86_64/amd/zen2 + - x86_64/amd/zen3 EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE: - none - accel/nvidia/cc80 @@ -101,9 +101,13 @@ jobs: sourceoutfile="sourceout.txt" module load EESSI/${{matrix.EESSI_VERSION}} unset EESSI_ARCHDETECT_OPTIONS + # module version uses archdetect (ignore this) + unset EESSI_USE_ARCHDETECT env | grep -E '^EESSI_' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} source ./init/bash + # source script version uses archspec (ignore this) + unset EESSI_USE_ARCHSPEC env | grep -E '^EESSI_' | sort > "${sourceoutfile}" cat "${moduleoutfile}" cat "${sourceoutfile}" From d4d5a796a7e2d5a3fe9fbaed7ca6a9fbc8d15bad Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 14:43:50 +0200 Subject: [PATCH 1451/1795] init script is the only one who sets EESSI_ARCHDETECT EESSI_ARCHSPEC --- .github/workflows/tests_eessi_module.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index f581bda3b7..693e293a04 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -101,13 +101,12 @@ jobs: sourceoutfile="sourceout.txt" module load EESSI/${{matrix.EESSI_VERSION}} unset EESSI_ARCHDETECT_OPTIONS - # module version uses archdetect (ignore this) - unset EESSI_USE_ARCHDETECT env | grep -E '^EESSI_' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} source ./init/bash - # source script version uses archspec (ignore this) + # source script version sets environment variables to force archdetect, ignore these unset EESSI_USE_ARCHSPEC + unset EESSI_USE_ARCHDETECT env | grep -E '^EESSI_' | sort > "${sourceoutfile}" cat "${moduleoutfile}" cat "${sourceoutfile}" From c7d423017d61e1cf1186358515ae0e9db49725c1 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 14:55:06 +0200 Subject: [PATCH 1452/1795] Use the correct override for when GPUs are expected --- .github/workflows/tests_eessi_module.yml | 28 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 693e293a04..dfb4a28842 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -70,7 +70,7 @@ jobs: - 2023.06 EESSI_SOFTWARE_SUBDIR_OVERRIDE: - x86_64/amd/zen3 - EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE: + EESSI_ACCELERATOR_TARGET_OVERRIDE: - none - accel/nvidia/cc80 steps: @@ -90,30 +90,44 @@ jobs: # Set our path overrides according to our matrix export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} - if [[ "${{matrix.EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE}}" != "none" ]]; then - export EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE}} + if [[ "${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}}" != "none" ]]; then + export EESSI_ACCELERATOR_TARGET_OVERRIDE=${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}} fi - + + moduleoutfile="moduleout.txt" + sourceoutfile="sourceout.txt" + + # First do (and undo) the Lmod initialisation export MODULEPATH=init/modules CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" - moduleoutfile="moduleout.txt" - sourceoutfile="sourceout.txt" module load EESSI/${{matrix.EESSI_VERSION}} + # EESSI_ARCHDETECT_OPTIONS only relevant for Lmod init unset EESSI_ARCHDETECT_OPTIONS + # Store all relevant environment variables env | grep -E '^EESSI_' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} + + # Now do the init script initialisation source ./init/bash # source script version sets environment variables to force archdetect, ignore these unset EESSI_USE_ARCHSPEC unset EESSI_USE_ARCHDETECT env | grep -E '^EESSI_' | sort > "${sourceoutfile}" + + # Now compare the two results + echo "" + echo "Lmod initialisation:" cat "${moduleoutfile}" + echo "" + echo "Source script initialisation:" cat "${sourceoutfile}" + echo "" + echo "" if (diff "${moduleoutfile}" "${sourceoutfile}" > /dev/null); then echo "Test for checking env variables PASSED" else echo "Test for checking env variables FAILED" >&2 - diff "${moduleoutfile}" "${sourceoutfile}" + diff --unified=0 "${moduleoutfile}" "${sourceoutfile}" exit 1 fi From a95d546371afa4892ca51841527a6cda4907402c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 15:09:32 +0200 Subject: [PATCH 1453/1795] revert variable expansion and unset certain variables instead --- EESSI-install-software.sh | 4 +++- init/eessi_environment_variables | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 84c96ad369..467a484fc7 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -173,7 +173,9 @@ fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages # $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT= EESSI_BASIC_ENV= source $TOPDIR/init/eessi_environment_variables +unset EESSI_SILENT +unset EESSI_BASIC_ENV +source $TOPDIR/init/eessi_environment_variables if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then fatal_error "Failed to determine software subdirectory?!" diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 125f3e5a93..60d69cc198 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -10,7 +10,7 @@ function error() { function show_msg { # only echo msg if EESSI_SILENT is unset msg=$1 - if [[ -z ${EESSI_SILENT:+x} ]]; then + if [[ -z ${EESSI_SILENT+x} ]]; then echo "$msg" fi } From 5fd01f8a2da5987075d364b947d150e47b83e9b5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 16:12:30 +0200 Subject: [PATCH 1454/1795] Add a load/unload test to make sure we return the environment to it's original state --- .github/workflows/tests_eessi_module.yml | 64 ++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index dfb4a28842..b629ae531f 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -71,7 +71,6 @@ jobs: EESSI_SOFTWARE_SUBDIR_OVERRIDE: - x86_64/amd/zen3 EESSI_ACCELERATOR_TARGET_OVERRIDE: - - none - accel/nvidia/cc80 steps: - name: Check out software-layer repository @@ -90,9 +89,7 @@ jobs: # Set our path overrides according to our matrix export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} - if [[ "${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}}" != "none" ]]; then - export EESSI_ACCELERATOR_TARGET_OVERRIDE=${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}} - fi + export EESSI_ACCELERATOR_TARGET_OVERRIDE=${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}} moduleoutfile="moduleout.txt" sourceoutfile="sourceout.txt" @@ -131,3 +128,62 @@ jobs: diff --unified=0 "${moduleoutfile}" "${sourceoutfile}" exit 1 fi + + make_sure_load_and_unload_work: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: + - x86_64/amd/zen2 + - x86_64/amd/zen3 + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + + - name: Test for expected variables while adding dummy cpu archs and loading EESSI module + run: | + . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod + + # Set our path overrides according to our matrix + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} + + initial_env_file="initial_env.txt" + module_cycled_file="load_unload_cycle.txt" + + # Store the initial environment + env | grep -E '^EESSI_' | sort > "${initial_env_file}" + + # Do (and undo) loading the EESSI module + export MODULEPATH=init/modules + CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) + export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" + module load EESSI/${{matrix.EESSI_VERSION}} + module unload EESSI/${{matrix.EESSI_VERSION}} + env | grep -E '^EESSI_' | sort > "${module_cycled_file}" + + # Now compare the two results + echo "" + echo "Initial environment:" + cat "${initial_env_file}" + echo "" + echo "Environment after load/unload cycle of EESSI:" + cat "${module_cycled_file}" + echo "" + echo "" + if (diff "${initial_env_file}" "${module_cycled_file}" > /dev/null); then + echo "Test for checking env variables PASSED" + else + echo "Test for checking env variables FAILED" >&2 + diff --unified=0 "${initial_env_file}" "${module_cycled_file}" + exit 1 + fi \ No newline at end of file From 45fa6b12ffa9a2d7b559f1c60ce6b71f94f4cc45 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 19:59:05 +0200 Subject: [PATCH 1455/1795] log if Lmod rc/SitePackage are being created --- EESSI-install-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 467a484fc7..f05e8ea7f9 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -161,11 +161,13 @@ _eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_ _lmod_cfg_dir=${_eessi_software_path}/.lmod _lmod_rc_file=${_lmod_cfg_dir}/lmodrc.lua if [ ! -f ${_lmod_rc_file} ]; then + echo "Lmod file '${_lmod_rc_file}' does not exist yet; creating it..." command -V python3 python3 ${TOPDIR}/create_lmodrc.py ${_eessi_software_path} fi _lmod_sitepackage_file=${_lmod_cfg_dir}/SitePackage.lua if [ ! -f ${_lmod_sitepackage_file} ]; then + echo "Lmod file '${_lmod_sitepackage_file}' does not exist yet; creating it..." command -V python3 python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path} fi From 7abcdc337b5416c3e3713d01aeeaf476eb512664 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 20:19:09 +0200 Subject: [PATCH 1456/1795] Try to make module work in load and unload modes --- .github/workflows/tests_eessi_module.yml | 1 - init/modules/EESSI/2023.06.lua | 46 +++++++++++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index b629ae531f..50578d7904 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -166,7 +166,6 @@ jobs: # Do (and undo) loading the EESSI module export MODULEPATH=init/modules CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) - export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" module load EESSI/${{matrix.EESSI_VERSION}} module unload EESSI/${{matrix.EESSI_VERSION}} env | grep -E '^EESSI_' | sort > "${module_cycled_file}" diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index d3ecf58813..57eb3e41d1 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -17,40 +17,54 @@ local eessi_os_type = "linux" setenv("EESSI_VERSION", eessi_version) setenv("EESSI_CVMFS_REPO", eessi_repo) setenv("EESSI_OS_TYPE", eessi_os_type) +function eessiDebug(text) + if os.getenv("EESSI_DEBUG_INIT") then + LmodMessage(text) + end +end function archdetect_cpu() local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper.sh') - if not os.getenv("EESSI_ARCHDETECT_OPTIONS") then + if not os.getenv("EESSI_ARCHDETECT_OPTIONS_OVERRIDE") then if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then - LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ARCHDETECT_OPTIONS to the available cpu architecture in the form of: x86_64/intel/haswell:x86_64/generic or aarch64/neoverse_v1:aarch64/generic") + LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ARCHDETECT_OPTIONS_OVERRIDE to the available cpu architecture in the form of: x86_64/intel/haswell:x86_64/generic or aarch64/neoverse_v1:aarch64/generic") end source_sh("bash", script) end - local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or "" - for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do - if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then - -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, - -- since optimized software installations for Zen4 are a work-in-progress, - -- see https://gitlab.com/eessi/support/-/issues/37 - if archdetect_filter_cpu == "x86_64/amd/zen4" then - archdetect_filter_cpu = "x86_64/amd/zen3" - if mode() == "load" then - LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.") + -- EESSI_ARCHDETECT_OPTIONS is set by the script (_if_ it was called) + local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or (os.getenv("EESSI_ARCHDETECT_OPTIONS_OVERRIDE") or "") + if archdetect_options then + eessiDebug("Got archdetect CPU options: " .. archdetect_options) + for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do + if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then + -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, + -- since optimized software installations for Zen4 are a work-in-progress, + -- see https://gitlab.com/eessi/support/-/issues/37 + if archdetect_filter_cpu == "x86_64/amd/zen4" then + archdetect_filter_cpu = "x86_64/amd/zen3" + if mode() == "load" then + LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.") + end end + eessiDebug("Selected archdetect CPU: " .. archdetect_filter_cpu) + return archdetect_filter_cpu end - return archdetect_filter_cpu end + LmodError("Software directory check for the detected architecture failed") + else + -- Still need to return something + return nil end - LmodError("Software directory check for the detected architecture failed") end function archdetect_accel() local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper_accel.sh') - if not os.getenv("EESSI_ACCEL_SUBDIR") then + if not os.getenv("EESSI_ACCELERATOR_TARGET_OVERRIDE ") then if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then - LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ACCEL_SUBDIR to the available accelerator architecture in the form of: accel/nvidia/cc80") + LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ACCELERATOR_TARGET_OVERRIDE to the available accelerator architecture in the form of: accel/nvidia/cc80") end source_sh("bash", script) end local archdetect_accel = os.getenv("EESSI_ACCEL_SUBDIR") or "" + eessiDebug("Got archdetect accel option: " .. archdetect_accel) return archdetect_accel end local archdetect = archdetect_cpu() From e4ac2fa59769f77fdbfc7b9a3de403e47c095b6c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 20:44:50 +0200 Subject: [PATCH 1457/1795] Fix lots of small issues --- init/modules/EESSI/2023.06.lua | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 57eb3e41d1..9ac6d5e6de 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -24,6 +24,8 @@ function eessiDebug(text) end function archdetect_cpu() local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper.sh') + -- make sure that we grab the value for architecture before the module unsets the environment variable (in unload mode) + local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or (os.getenv("EESSI_ARCHDETECT_OPTIONS_OVERRIDE") or "") if not os.getenv("EESSI_ARCHDETECT_OPTIONS_OVERRIDE") then if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ARCHDETECT_OPTIONS_OVERRIDE to the available cpu architecture in the form of: x86_64/intel/haswell:x86_64/generic or aarch64/neoverse_v1:aarch64/generic") @@ -31,7 +33,7 @@ function archdetect_cpu() source_sh("bash", script) end -- EESSI_ARCHDETECT_OPTIONS is set by the script (_if_ it was called) - local archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or (os.getenv("EESSI_ARCHDETECT_OPTIONS_OVERRIDE") or "") + archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or archdetect_options if archdetect_options then eessiDebug("Got archdetect CPU options: " .. archdetect_options) for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do @@ -57,13 +59,15 @@ function archdetect_cpu() end function archdetect_accel() local script = pathJoin(eessi_prefix, 'init', 'lmod_eessi_archdetect_wrapper_accel.sh') + -- for unload mode, we need to grab the value before it is unset + local archdetect_accel = os.getenv("EESSI_ACCEL_SUBDIR") or (os.getenv("EESSI_ACCELERATOR_TARGET_OVERRIDE") or "") if not os.getenv("EESSI_ACCELERATOR_TARGET_OVERRIDE ") then if convertToCanonical(LmodVersion()) < convertToCanonical("8.6") then LmodError("Loading this modulefile requires using Lmod version >= 8.6, but you can export EESSI_ACCELERATOR_TARGET_OVERRIDE to the available accelerator architecture in the form of: accel/nvidia/cc80") end source_sh("bash", script) end - local archdetect_accel = os.getenv("EESSI_ACCEL_SUBDIR") or "" + archdetect_accel = os.getenv("EESSI_ACCEL_SUBDIR") or archdetect_accel eessiDebug("Got archdetect accel option: " .. archdetect_accel) return archdetect_accel end @@ -101,8 +105,8 @@ setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) if not (archdetect_accel == nil or archdetect_accel == '') then eessi_accel_software_subdir = os.getenv("EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE") or eessi_software_subdir eessi_accel_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_accel_software_subdir) - eessi_module_path_accel = pathJoin(eessi_accel_software_path, eessi_accel_software_subdir, eessi_modules_subdir) - if isDir(eessi_modulepath_accel) then + eessi_module_path_accel = pathJoin(eessi_accel_software_path, archdetect_accel, eessi_modules_subdir) + if isDir(eessi_module_path_accel) then setenv("EESSI_MODULEPATH_ACCEL", eessi_module_path_accel) prepend_path("MODULEPATH", eessi_module_path_accel) end From c3482b269b892503a83080420f0317a3f98c5525 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 21:11:05 +0200 Subject: [PATCH 1458/1795] show full path to Lmod RC/SitePackage when created --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f05e8ea7f9..3ebc2a5f03 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -328,20 +328,20 @@ else done fi -echo ">> Creating/updating Lmod RC file..." export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?) if [ ! -f $lmod_rc_file ] || [ ${lmodrc_changed} == '0' ]; then + echo ">> Creating/updating Lmod RC file (${lmod_rc_file})..." python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH} check_exit_code $? "$lmod_rc_file created" "Failed to create $lmod_rc_file" fi -echo ">> Creating/updating Lmod SitePackage.lua ..." export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?) if [ ! -f "$lmod_sitepackage_file" ] || [ "${sitepackage_changed}" == '0' ]; then + echo ">> Creating/updating Lmod SitePackage.lua (${lmod_sitepackage_file})..." python3 $TOPDIR/create_lmodsitepackage.py ${EASYBUILD_INSTALLPATH} check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file" fi From 38297912b4ebb2355bfab94440d64ef54e494de1 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 21:33:23 +0200 Subject: [PATCH 1459/1795] Fix lots of small issues --- .github/workflows/tests_eessi_module.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 50578d7904..c4a56ba0aa 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -48,7 +48,7 @@ jobs: . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod export MODULEPATH=init/modules set +e # Do not exit immediately if a command exits with a non-zero status - export EESSI_ARCHDETECT_OPTIONS="dummy/cpu" + export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="dummy/cpu" outfile="outfile.txt" module load EESSI/${{matrix.EESSI_VERSION}} > "${outfile}" 2>&1 cat "${outfile}" @@ -58,7 +58,7 @@ jobs: echo "Test for picking up invalid path on \${archdetect_cpu} FAILED" >&2 exit 1 fi - unset EESSI_ARCHDETECT_OPTIONS + unset EESSI_ARCHDETECT_OPTIONS_OVERRIDE set -e # Re-enable exit on non-zero status lmod_and_init_script_comparison: @@ -97,10 +97,10 @@ jobs: # First do (and undo) the Lmod initialisation export MODULEPATH=init/modules CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) - export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" + export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" module load EESSI/${{matrix.EESSI_VERSION}} - # EESSI_ARCHDETECT_OPTIONS only relevant for Lmod init - unset EESSI_ARCHDETECT_OPTIONS + # EESSI_ARCHDETECT_OPTIONS_OVERRIDE only relevant for Lmod init + unset EESSI_ARCHDETECT_OPTIONS_OVERRIDE # Store all relevant environment variables env | grep -E '^EESSI_' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} From 443cdef1a8d88092c3d3cd90835d8a743ae0eb57 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 21:38:57 +0200 Subject: [PATCH 1460/1795] Fix lots of small issues --- .github/workflows/tests_eessi_module.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index c4a56ba0aa..6013003d26 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -83,7 +83,7 @@ jobs: cvmfs_http_proxy: DIRECT cvmfs_repositories: software.eessi.io - - name: Test for expected variables while adding dummy cpu archs and loading EESSI module + - name: Test for expected variables match between Lmod init script and original bash script run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod @@ -137,6 +137,7 @@ jobs: EESSI_VERSION: - 2023.06 EESSI_SOFTWARE_SUBDIR_OVERRIDE: + - none - x86_64/amd/zen2 - x86_64/amd/zen3 steps: @@ -155,7 +156,9 @@ jobs: . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod # Set our path overrides according to our matrix - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} + if [[ "${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}" != "none" ]]; then + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} + fi initial_env_file="initial_env.txt" module_cycled_file="load_unload_cycle.txt" From 78293e6a282d4405c35a432b2ccece1c4716f300 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 21:47:23 +0200 Subject: [PATCH 1461/1795] Use debug feature for module a bit more --- init/modules/EESSI/2023.06.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 9ac6d5e6de..b88f5b25c8 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -109,6 +109,7 @@ if not (archdetect_accel == nil or archdetect_accel == '') then if isDir(eessi_module_path_accel) then setenv("EESSI_MODULEPATH_ACCEL", eessi_module_path_accel) prepend_path("MODULEPATH", eessi_module_path_accel) + eessiDebug("Using acclerator modules at: " .. eessi_module_path_accel) end end From fa10c1a876b00600ccfb9fd13932f57ba9615b8d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 15 Oct 2024 21:56:22 +0200 Subject: [PATCH 1462/1795] Correct comment in action --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 6013003d26..0fb5c656f0 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -151,7 +151,7 @@ jobs: cvmfs_http_proxy: DIRECT cvmfs_repositories: software.eessi.io - - name: Test for expected variables while adding dummy cpu archs and loading EESSI module + - name: Test for identical environment after loading and unloading the EESSI module run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod From db90ca70d49da84a2283c1c805181835d055f00b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 15 Oct 2024 22:26:25 +0200 Subject: [PATCH 1463/1795] adjust path to lua files if building accelerator software --- EESSI-install-software.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 3ebc2a5f03..7c51727bef 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -330,6 +330,11 @@ fi export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" +if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then + # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_rc_file + lmod_rc_file=$(echo ${lmod_rc_file} | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@") + echo "Path to lmodrc.lua changed to '${lmod_rc_file}'" +fi lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?) if [ ! -f $lmod_rc_file ] || [ ${lmodrc_changed} == '0' ]; then echo ">> Creating/updating Lmod RC file (${lmod_rc_file})..." @@ -339,6 +344,11 @@ fi export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" +if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then + # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_sitepackage_file + lmod_sitepackage_file=$(echo ${lmod_sitepackage_file} | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@") + echo "Path to SitePackage.lua changed to '${lmod_sitepackage_file}'" +fi sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?) if [ ! -f "$lmod_sitepackage_file" ] || [ "${sitepackage_changed}" == '0' ]; then echo ">> Creating/updating Lmod SitePackage.lua (${lmod_sitepackage_file})..." From 9c4897ce56fae9952da7c4533e86a896bb2c869f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 09:55:12 +0200 Subject: [PATCH 1464/1795] Add more debug logging --- init/modules/EESSI/2023.06.lua | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index b88f5b25c8..55a77335fc 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -18,7 +18,7 @@ setenv("EESSI_VERSION", eessi_version) setenv("EESSI_CVMFS_REPO", eessi_repo) setenv("EESSI_OS_TYPE", eessi_os_type) function eessiDebug(text) - if os.getenv("EESSI_DEBUG_INIT") then + if (mode() == "load" and os.getenv("EESSI_DEBUG_INIT")) then LmodMessage(text) end end @@ -82,21 +82,35 @@ local eessi_module_path = pathJoin(eessi_software_path, eessi_modules_subdir) local eessi_site_software_path = string.gsub(eessi_software_path, "versions", "host_injections") local eessi_site_module_path = pathJoin(eessi_site_software_path, eessi_modules_subdir) setenv("EPREFIX", eessi_eprefix) +eessiDebug("Setting EPREFIX to " .. eessi_eprefix) setenv("EESSI_CPU_FAMILY", eessi_cpu_family) +eessiDebug("Setting EESSI_CPU_FAMILY to " .. eessi_cpu_family) setenv("EESSI_SITE_SOFTWARE_PATH", eessi_site_software_path) +eessiDebug("Setting EESSI_SITE_SOFTWARE_PATH to " .. eessi_site_software_path) setenv("EESSI_SITE_MODULEPATH", eessi_site_module_path) +eessiDebug("Setting EESSI_SITE_MODULEPATH to " .. eessi_site_module_path) setenv("EESSI_SOFTWARE_SUBDIR", eessi_software_subdir) +eessiDebug("Setting EESSI_SOFTWARE_SUBDIR to " .. eessi_software_subdir) setenv("EESSI_PREFIX", eessi_prefix) +eessiDebug("Setting EESSI_PREFIX to " .. eessi_prefix) setenv("EESSI_EPREFIX", eessi_eprefix) +eessiDebug("Setting EPREFIX to " .. eessi_eprefix) prepend_path("PATH", pathJoin(eessi_eprefix, "bin")) -prepend_path("PATH", pathJoin(eessi_eprefix, "usr/bin")) +eessiDebug("Adding " .. pathJoin(eessi_eprefix, "bin") .. " to PATH") +prepend_path("PATH", pathJoin(eessi_eprefix, "usr", "bin")) +eessiDebug("Adding " .. pathJoin(eessi_eprefix, "usr", "bin") .. " to PATH") setenv("EESSI_SOFTWARE_PATH", eessi_software_path) +eessiDebug("Setting EESSI_SOFTWARE_PATH to " .. eessi_software_path) setenv("EESSI_MODULEPATH", eessi_module_path) +eessiDebug("Setting EESSI_MODULEPATH to " .. eessi_module_path) if ( mode() ~= "spider" ) then prepend_path("MODULEPATH", eessi_module_path) + eessiDebug("Adding " .. eessi_module_path .. " to MODULEPATH") end -prepend_path("LMOD_RC", pathJoin(eessi_software_path, "/.lmod/lmodrc.lua")) +prepend_path("LMOD_RC", pathJoin(eessi_software_path, ".lmod", "lmodrc.lua")) +eessiDebug("Adding " .. pathJoin(eessi_software_path, ".lmod", "lmodrc.lua") .. " to LMOD_RC") setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) +eessiDebug("Setting LMOD_PACKAGE_PATH to " .. pathJoin(eessi_software_path, ".lmod")) -- the accelerator may have an empty value and we need to give some flexibility -- * construct the path we expect to find @@ -106,6 +120,7 @@ if not (archdetect_accel == nil or archdetect_accel == '') then eessi_accel_software_subdir = os.getenv("EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE") or eessi_software_subdir eessi_accel_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_accel_software_subdir) eessi_module_path_accel = pathJoin(eessi_accel_software_path, archdetect_accel, eessi_modules_subdir) + eessiDebug("Checking if " .. eessi_module_path_accel .. " exists") if isDir(eessi_module_path_accel) then setenv("EESSI_MODULEPATH_ACCEL", eessi_module_path_accel) prepend_path("MODULEPATH", eessi_module_path_accel) @@ -115,6 +130,7 @@ end -- prepend the site module path last so it has priority prepend_path("MODULEPATH", eessi_site_module_path) +eessiDebug("Adding " .. eessi_site_module_path .. " to MODULEPATH") if mode() == "load" then LmodMessage("EESSI/" .. eessi_version .. " loaded successfully") end From d27fcf4e516e31fd117497970077ca1d293f403c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 09:55:13 +0200 Subject: [PATCH 1465/1795] Modify EESSI-install-software.sh so that it loads the EESSI module to set the environment --- EESSI-install-software.sh | 105 ++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 44f524ebbc..2e032b35b4 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -112,16 +112,6 @@ fi TMPDIR=$(mktemp -d) -echo ">> Setting up environment..." - -source $TOPDIR/init/minimal_eessi_env - -if [ -d $EESSI_CVMFS_REPO ]; then - echo_green "$EESSI_CVMFS_REPO available, OK!" -else - fatal_error "$EESSI_CVMFS_REPO is not available!" -fi - # make sure we're in Prefix environment by checking $SHELL if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" @@ -129,9 +119,7 @@ else fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" fi -# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory -export PYTHONPYCACHEPREFIX=$TMPDIR/pycache - +# Get override subdir DETECTION_PARAMETERS='' GENERIC=0 EB='eb' @@ -148,10 +136,68 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" - # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) - mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # Run in a subshell, so that minimal_eessi_env doesn't change the shell environment for the rest of this script + ( + # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set + source $TOPDIR/init/minimal_eessi_env + + # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) + mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + ) fi +echo ">> Setting up environment..." + +# If EESSI_VERSION is not set, source the defaults script to set it +if [ -z ${EESSI_VERSION} ]; then + source $TOPDIR/init/eessi_defaults +fi + +# If module command does not exist, use the one from the compat layer +command -v module +module_cmd_exists=$? +if [[ "$module_cmd_exists" -ne 0 ]]; then + echo_green "No module command found, initializing lmod from the compatibility layer" + # Minimal initalization of the lmod from the compat layer + source $TOPDIR/init/lmod/bash +else + echo_green "Module command found" +fi +ml_version_out=$TMPDIR/ml.out +ml --version &> $ml_version_out +if [[ $? -eq 0 ]]; then + echo_green ">> Found Lmod ${LMOD_VERSION}" +else + fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" +fi + +# Make sure we start with no modules and clean $MODULEPATH +echo ">> Setting up \$MODULEPATH..." +module --force purge +module unuse $MODULEPATH + +# Initialize the EESSI environment +module use $TOPDIR/init/modules +module load EESSI/$EESSI_VERSION + +if [ -d $EESSI_CVMFS_REPO ]; then + echo_green "$EESSI_CVMFS_REPO available, OK!" +else + fatal_error "$EESSI_CVMFS_REPO is not available!" +fi + +# Check that EESSI_SOFTWARE_SUBDIR now matches EESSI_SOFTWARE_SUBDIR_OVERRIDE +if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then + fatal_error "Failed to determine software subdirectory?!" +elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" +else + echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" +fi + +# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory +export PYTHONPYCACHEPREFIX=$TMPDIR/pycache + # if we run the script for the first time, e.g., to start building for a new # stack, we need to ensure certain files are present in # ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} @@ -170,29 +216,6 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path} fi -# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) -# $EESSI_SILENT - don't print any messages -# $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables - -if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then - fatal_error "Failed to determine software subdirectory?!" -elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then - fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" -else - echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" -fi - -echo ">> Initializing Lmod..." -source $EPREFIX/usr/share/Lmod/init/bash -ml_version_out=$TMPDIR/ml.out -ml --version &> $ml_version_out -if [[ $? -eq 0 ]]; then - echo_green ">> Found Lmod ${LMOD_VERSION}" -else - fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" -fi - echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild @@ -202,12 +225,6 @@ if [ ! -z "${shared_fs_path}" ]; then export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} fi -echo ">> Setting up \$MODULEPATH..." -# make sure no modules are loaded -module --force purge -# ignore current $MODULEPATH entirely -module unuse $MODULEPATH - # if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all From 4d22c13fe7c3b6c9820bf10025cf315a9bd2ef8d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 10:00:06 +0200 Subject: [PATCH 1466/1795] Also replace source of easybuild configuration by loading the EESSI-extend module --- EESSI-install-software.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2e032b35b4..68ff4a1927 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -217,7 +217,8 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then fi echo ">> Configuring EasyBuild..." -source $TOPDIR/configure_easybuild +export EESSI_CVMFS_INSTALL=1 +module load EESSI-extend/${EESSI_VERSION}-easybuild if [ ! -z "${shared_fs_path}" ]; then shared_eb_sourcepath=${shared_fs_path}/easybuild/sources From 86e0c4f2813e1e9cdc2d4b18e939361148bf5c87 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 10:05:31 +0200 Subject: [PATCH 1467/1795] Add debug logging to CI --- .github/workflows/tests_eessi_module.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 0fb5c656f0..67c16259db 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -96,11 +96,14 @@ jobs: # First do (and undo) the Lmod initialisation export MODULEPATH=init/modules + # Turn on debug output in case we want to take a look + export EESSI_DEBUG_INIT=true CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="dummy/cpu:${CPU_ARCH}:dummy1/cpu1" module load EESSI/${{matrix.EESSI_VERSION}} - # EESSI_ARCHDETECT_OPTIONS_OVERRIDE only relevant for Lmod init + # EESSI_ARCHDETECT_OPTIONS_OVERRIDE/EESSI_DEBUG_INIT only relevant for Lmod init unset EESSI_ARCHDETECT_OPTIONS_OVERRIDE + unset EESSI_DEBUG_INIT # Store all relevant environment variables env | grep -E '^EESSI_' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} @@ -139,7 +142,7 @@ jobs: EESSI_SOFTWARE_SUBDIR_OVERRIDE: - none - x86_64/amd/zen2 - - x86_64/amd/zen3 + - x86_64/amd/zen4 steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -160,6 +163,9 @@ jobs: export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} fi + # Turn on debug output in case we want to take a look + export EESSI_DEBUG_INIT=true + initial_env_file="initial_env.txt" module_cycled_file="load_unload_cycle.txt" From 87c6cdf8ad3f275a59af6fc76c36eb1196e77582 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 10:08:17 +0200 Subject: [PATCH 1468/1795] Add LMOD_RC and LMOD_PACKAGE_PATH to our checks --- .github/workflows/tests_eessi_module.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 67c16259db..65a00a94a7 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -105,7 +105,7 @@ jobs: unset EESSI_ARCHDETECT_OPTIONS_OVERRIDE unset EESSI_DEBUG_INIT # Store all relevant environment variables - env | grep -E '^EESSI_' | sort > "${moduleoutfile}" + env | grep -E '(^EESSI_|^LMOD_RC|^LMOD_PACKAGE_PATH)' | sort > "${moduleoutfile}" module unload EESSI/${{matrix.EESSI_VERSION}} # Now do the init script initialisation From 913266293bfd2a88ead028ff96de10bc6decc7cb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 10:11:17 +0200 Subject: [PATCH 1469/1795] Use the same regex everywhere --- .github/workflows/tests_eessi_module.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 65a00a94a7..75d09b74ec 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -113,7 +113,7 @@ jobs: # source script version sets environment variables to force archdetect, ignore these unset EESSI_USE_ARCHSPEC unset EESSI_USE_ARCHDETECT - env | grep -E '^EESSI_' | sort > "${sourceoutfile}" + env | grep -E '(^EESSI_|^LMOD_RC|^LMOD_PACKAGE_PATH)' | sort > "${sourceoutfile}" # Now compare the two results echo "" @@ -170,14 +170,14 @@ jobs: module_cycled_file="load_unload_cycle.txt" # Store the initial environment - env | grep -E '^EESSI_' | sort > "${initial_env_file}" + env | grep -E '(^EESSI_|^LMOD_RC|^LMOD_PACKAGE_PATH)' | sort > "${initial_env_file}" # Do (and undo) loading the EESSI module export MODULEPATH=init/modules CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) module load EESSI/${{matrix.EESSI_VERSION}} module unload EESSI/${{matrix.EESSI_VERSION}} - env | grep -E '^EESSI_' | sort > "${module_cycled_file}" + env | grep -E '(^EESSI_|^LMOD_RC|^LMOD_PACKAGE_PATH)' | sort > "${module_cycled_file}" # Now compare the two results echo "" From b8c311d5254a5c7324f65dd5f090f45ab5ea2a1b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 10:17:17 +0200 Subject: [PATCH 1470/1795] zen4 is currently an exception, let's explicitly add it to the tests --- .github/workflows/tests_eessi_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 75d09b74ec..e74ad8587f 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -70,6 +70,7 @@ jobs: - 2023.06 EESSI_SOFTWARE_SUBDIR_OVERRIDE: - x86_64/amd/zen3 + - x86_64/amd/zen4 EESSI_ACCELERATOR_TARGET_OVERRIDE: - accel/nvidia/cc80 steps: From 6a0223c944d8dd3cbf8943fbc7af9bc41ff45bcf Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 16 Oct 2024 10:42:14 +0200 Subject: [PATCH 1471/1795] small typo fixed --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 3590cf3a86..0662dac761 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -234,5 +234,5 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do rm -rf "${tmpdir}" # Restore MODULEPATH for next loop iteration - MODUELPATH=${SAVE_MODULEPATH} + MODULEPATH=${SAVE_MODULEPATH} done From fe6a7def8986ad5bddae1abc71466fef0ce3aba8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 10:46:42 +0200 Subject: [PATCH 1472/1795] Allow overriding to use Zen4 --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 55a77335fc..598c436785 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -41,7 +41,7 @@ function archdetect_cpu() -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, -- since optimized software installations for Zen4 are a work-in-progress, -- see https://gitlab.com/eessi/support/-/issues/37 - if archdetect_filter_cpu == "x86_64/amd/zen4" then + if (archdetect_filter_cpu == "x86_64/amd/zen4" and not os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE") == "x86_64/amd/zen4") then archdetect_filter_cpu = "x86_64/amd/zen3" if mode() == "load" then LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.") From 660697a52edf1afa8162d651934935f8c3ac3a2b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 10:50:08 +0200 Subject: [PATCH 1473/1795] Add a quick build so we can test the EESSI-install-software.sh script --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml index 5325f2e553..c33d2e1ac6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -7,3 +7,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b + - patchelf-0.18.0-GCCcore-13.2.0.eb: From affe37ba5617909d0ad415cdcf3ce6f899150644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 16 Oct 2024 11:45:07 +0200 Subject: [PATCH 1474/1795] add comment to clarify setting of MODULEPATH Co-authored-by: ocaisa --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 0662dac761..f9d889c1a1 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -112,6 +112,8 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do module load EESSI-extend/${EESSI_VERSION}-easybuild # Install modules in hidden .modules dir to keep track of what was installed before + # (this action is temporary, and we do not call Lmod again within the current shell context, but in EasyBuild + # subshells, so loaded modules are not automatically unloaded) MODULEPATH=${EESSI_SITE_SOFTWARE_PATH}/.modules/all echo "set MODULEPATH=${MODULEPATH}" From 6d64c4283c1de2fa7b9c51a1d53e78e18c7b5ae3 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 16 Oct 2024 11:51:39 +0200 Subject: [PATCH 1475/1795] Add comment to loop for clarification --- init/modules/EESSI/2023.06.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 598c436785..83e3791947 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -36,6 +36,9 @@ function archdetect_cpu() archdetect_options = os.getenv("EESSI_ARCHDETECT_OPTIONS") or archdetect_options if archdetect_options then eessiDebug("Got archdetect CPU options: " .. archdetect_options) + -- archdetect_options is a colon-separated list of CPU architectures that are compatible with + -- the host CPU and ordered from most specific to least specific, e.g., + -- x86_64/intel/skylake_avx512:x86_64/intel/haswell:x86_64/generic for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, From d2d95e9a9b7a8e47cc574de8d4d41dfad338f6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 16 Oct 2024 11:52:57 +0200 Subject: [PATCH 1476/1795] clarify need for option Co-authored-by: Kenneth Hoste --- .../eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml index 677627eed3..83e68077a2 100644 --- a/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml +++ b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml @@ -4,5 +4,6 @@ easyconfigs: - CUDA-12.1.1.eb - cuDNN-8.9.2.26-CUDA-12.1.1.eb: options: - # Needed for support for --accept-uela-for option + # needed to enforce acceptance of EULA in cuDNN easyblock, + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3473 include-easyblocks-from-commit: 11afb88ec55e0ca431cbe823696aa43e2a9bfca8 From 77f3bc9cdaf6ab836616a7724d904e830a2e2d5e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 16 Oct 2024 12:03:51 +0200 Subject: [PATCH 1477/1795] revert to silent sourcing, keep initialising full environment and clarify in comments --- EESSI-install-software.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7c51727bef..c2900b9a30 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -173,11 +173,9 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) -# $EESSI_SILENT - don't print any messages -# $EESSI_BASIC_ENV - give a basic set of environment variables -unset EESSI_SILENT -unset EESSI_BASIC_ENV -source $TOPDIR/init/eessi_environment_variables +# $EESSI_SILENT - don't print any messages if set (use 'unset EESSI_SILENT' to let script show messages) +# $EESSI_BASIC_ENV - give a basic set of environment variables if set (use 'EESSI_BASIC_ENV=' to let script initialise a full environment) +EESSI_SILENT=1 EESSI_BASIC_ENV= source $TOPDIR/init/eessi_environment_variables if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then fatal_error "Failed to determine software subdirectory?!" From 7e4f901762cd7981740da8c2f8b5cc9413f0c2df Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 16 Oct 2024 12:15:20 +0200 Subject: [PATCH 1478/1795] Add comments with examples in lots of places --- init/modules/EESSI/2023.06.lua | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 83e3791947..878850aee3 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -38,7 +38,8 @@ function archdetect_cpu() eessiDebug("Got archdetect CPU options: " .. archdetect_options) -- archdetect_options is a colon-separated list of CPU architectures that are compatible with -- the host CPU and ordered from most specific to least specific, e.g., - -- x86_64/intel/skylake_avx512:x86_64/intel/haswell:x86_64/generic + -- x86_64/intel/skylake_avx512:x86_64/intel/haswell:x86_64/generic + -- We loop over the list, and return the highest matching arch for which a directory exists for this EESSI version for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, @@ -74,15 +75,23 @@ function archdetect_accel() eessiDebug("Got archdetect accel option: " .. archdetect_accel) return archdetect_accel end +-- archdetect finds the best compatible architecture, e.g., x86_64/amd/zen3 local archdetect = archdetect_cpu() +-- archdetect_accel() attempts to identify an accelerator, e.g., accel/nvidia/cc80 local archdetect_accel = archdetect_accel() +-- eessi_cpu_family is derived from the archdetect match, e.g., x86_64 local eessi_cpu_family = archdetect:match("([^/]+)") local eessi_software_subdir = archdetect +-- eessi_eprefix is the base location of the compat layer, e.g., /cvmfs/software.eessi.io/versions/2023.06/compat/linux/x86_64 local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family) +-- eessi_software_path is the location of the software installations, e.g., +-- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3 local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) local eessi_modules_subdir = pathJoin("modules", "all") local eessi_module_path = pathJoin(eessi_software_path, eessi_modules_subdir) local eessi_site_software_path = string.gsub(eessi_software_path, "versions", "host_injections") +-- Site module path is the same as the EESSI one, but with `versions` changed to `host_injections`, e.g., +-- /cvmfs/software.eessi.io/host_injections/2023.06/software/linux/x86_64/amd/zen3/modules/all local eessi_site_module_path = pathJoin(eessi_site_software_path, eessi_modules_subdir) setenv("EPREFIX", eessi_eprefix) eessiDebug("Setting EPREFIX to " .. eessi_eprefix) @@ -106,6 +115,7 @@ setenv("EESSI_SOFTWARE_PATH", eessi_software_path) eessiDebug("Setting EESSI_SOFTWARE_PATH to " .. eessi_software_path) setenv("EESSI_MODULEPATH", eessi_module_path) eessiDebug("Setting EESSI_MODULEPATH to " .. eessi_module_path) +-- We ship our spider cache, so this location does not need to be spider-ed if ( mode() ~= "spider" ) then prepend_path("MODULEPATH", eessi_module_path) eessiDebug("Adding " .. eessi_module_path .. " to MODULEPATH") @@ -120,8 +130,11 @@ eessiDebug("Setting LMOD_PACKAGE_PATH to " .. pathJoin(eessi_software_path, ".lm -- * then check it exists -- * then update the modulepath if not (archdetect_accel == nil or archdetect_accel == '') then + -- The CPU subdirectory of the accelerator installations is _usually_ the same as host CPU, but this can be overridden eessi_accel_software_subdir = os.getenv("EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE") or eessi_software_subdir eessi_accel_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_accel_software_subdir) + -- location of the accelerator modules, e.g., + -- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all eessi_module_path_accel = pathJoin(eessi_accel_software_path, archdetect_accel, eessi_modules_subdir) eessiDebug("Checking if " .. eessi_module_path_accel .. " exists") if isDir(eessi_module_path_accel) then From 089158a67beca03731fb280c347c72bc0fdd47e6 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 16 Oct 2024 12:15:52 +0200 Subject: [PATCH 1479/1795] Apply suggestions from code review --- init/modules/EESSI/2023.06.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 878850aee3..c45ae743f5 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -132,6 +132,8 @@ eessiDebug("Setting LMOD_PACKAGE_PATH to " .. pathJoin(eessi_software_path, ".lm if not (archdetect_accel == nil or archdetect_accel == '') then -- The CPU subdirectory of the accelerator installations is _usually_ the same as host CPU, but this can be overridden eessi_accel_software_subdir = os.getenv("EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE") or eessi_software_subdir + -- CPU location of the accelerator installations, e.g., + -- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3 eessi_accel_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_accel_software_subdir) -- location of the accelerator modules, e.g., -- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all From bea886b618515416f114ac267b177396412fda41 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 16 Oct 2024 12:16:38 +0200 Subject: [PATCH 1480/1795] Apply suggestions from code review --- init/modules/EESSI/2023.06.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index c45ae743f5..0c2f34280a 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -88,6 +88,8 @@ local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_ -- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3 local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir) local eessi_modules_subdir = pathJoin("modules", "all") +-- eessi_module_path is the location of the _CPU_ module files, e.g., +-- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/modules/all local eessi_module_path = pathJoin(eessi_software_path, eessi_modules_subdir) local eessi_site_software_path = string.gsub(eessi_software_path, "versions", "host_injections") -- Site module path is the same as the EESSI one, but with `versions` changed to `host_injections`, e.g., From 6e1ccaa36e028b81e9a58b1958ca2f3102ba1a1b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 12:28:34 +0200 Subject: [PATCH 1481/1795] Add more cases to load/unload tests, and check full env --- .github/workflows/tests_eessi_module.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index e74ad8587f..de99543f6d 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -144,6 +144,9 @@ jobs: - none - x86_64/amd/zen2 - x86_64/amd/zen4 + EESSI_ACCELERATOR_TARGET_OVERRIDE: + - none + - accel/nvidia/cc80 steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -159,11 +162,16 @@ jobs: run: | . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod - # Set our path overrides according to our matrix + # Set our cpu path overrides according to our matrix if [[ "${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}" != "none" ]]; then export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} fi + # Set our accelerator path overrides according to our matrix + if [[ "${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}}" != "none" ]]; then + export EESSI_ACCELERATOR_TARGET_OVERRIDE=${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}} + fi + # Turn on debug output in case we want to take a look export EESSI_DEBUG_INIT=true @@ -171,14 +179,14 @@ jobs: module_cycled_file="load_unload_cycle.txt" # Store the initial environment - env | grep -E '(^EESSI_|^LMOD_RC|^LMOD_PACKAGE_PATH)' | sort > "${initial_env_file}" + env | sort > "${initial_env_file}" # Do (and undo) loading the EESSI module export MODULEPATH=init/modules CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) module load EESSI/${{matrix.EESSI_VERSION}} module unload EESSI/${{matrix.EESSI_VERSION}} - env | grep -E '(^EESSI_|^LMOD_RC|^LMOD_PACKAGE_PATH)' | sort > "${module_cycled_file}" + env | sort > "${module_cycled_file}" # Now compare the two results echo "" From 84d9f1deabbb366ba8799dbc7d26f45f4b6ae157 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 12:46:55 +0200 Subject: [PATCH 1482/1795] Make sure Lmod has run once before storing environment settings --- .github/workflows/tests_eessi_module.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index de99543f6d..807c16eb8e 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -45,7 +45,8 @@ jobs: - name: Test for archdetect_cpu functionality with invalid path run: | - . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod + # Initialise Lmod + . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash export MODULEPATH=init/modules set +e # Do not exit immediately if a command exits with a non-zero status export EESSI_ARCHDETECT_OPTIONS_OVERRIDE="dummy/cpu" @@ -86,7 +87,8 @@ jobs: - name: Test for expected variables match between Lmod init script and original bash script run: | - . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod + # Initialise Lmod + . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Set our path overrides according to our matrix export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} @@ -160,7 +162,8 @@ jobs: - name: Test for identical environment after loading and unloading the EESSI module run: | - . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Initialise Lmod + # Initialise Lmod + . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash # Set our cpu path overrides according to our matrix if [[ "${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}" != "none" ]]; then @@ -178,11 +181,14 @@ jobs: initial_env_file="initial_env.txt" module_cycled_file="load_unload_cycle.txt" + # prepare Lmod + export MODULEPATH=init/modules + module list + # Store the initial environment env | sort > "${initial_env_file}" # Do (and undo) loading the EESSI module - export MODULEPATH=init/modules CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) module load EESSI/${{matrix.EESSI_VERSION}} module unload EESSI/${{matrix.EESSI_VERSION}} From 56518eac74c1d4790e74662a0562016394b16d3e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 12:54:25 +0200 Subject: [PATCH 1483/1795] Don't expose full environment in CI --- .github/workflows/tests_eessi_module.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 807c16eb8e..91760b6c04 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -194,15 +194,7 @@ jobs: module unload EESSI/${{matrix.EESSI_VERSION}} env | sort > "${module_cycled_file}" - # Now compare the two results - echo "" - echo "Initial environment:" - cat "${initial_env_file}" - echo "" - echo "Environment after load/unload cycle of EESSI:" - cat "${module_cycled_file}" - echo "" - echo "" + # Now compare the two results (do not expose the files, as they contain the full environment!) if (diff "${initial_env_file}" "${module_cycled_file}" > /dev/null); then echo "Test for checking env variables PASSED" else From 946528974bbb7a0492fa31649761dcd16ca52cd9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 12:58:29 +0200 Subject: [PATCH 1484/1795] Try harder to initialise Lmod before checking load/unload --- .github/workflows/tests_eessi_module.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 91760b6c04..acc903c878 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -183,6 +183,7 @@ jobs: # prepare Lmod export MODULEPATH=init/modules + module reset module list # Store the initial environment From e937d5b8bd549d3525dbe2149737473a75c39f57 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 13:04:56 +0200 Subject: [PATCH 1485/1795] Make sure a module reset does not return an error --- .github/workflows/tests_eessi_module.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index acc903c878..2c4dbca88c 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -181,9 +181,9 @@ jobs: initial_env_file="initial_env.txt" module_cycled_file="load_unload_cycle.txt" - # prepare Lmod + # prepare Lmod, including resetting it (even though this will likely return an error since we have no defaults) export MODULEPATH=init/modules - module reset + module reset || true module list # Store the initial environment From fb59a5537e76e2eb14e06c4fbda92cba0402e5b5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 13:12:44 +0200 Subject: [PATCH 1486/1795] Try another way to reset Lmod --- .github/workflows/tests_eessi_module.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 2c4dbca88c..8d9909bfdf 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -183,8 +183,8 @@ jobs: # prepare Lmod, including resetting it (even though this will likely return an error since we have no defaults) export MODULEPATH=init/modules - module reset || true - module list + module purge + module avail # Store the initial environment env | sort > "${initial_env_file}" From c02d894ce47608f6cf1b116ddedf1c22b3dd8c3f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 13:19:56 +0200 Subject: [PATCH 1487/1795] Add a fake module so we can initialise Lmod somehow --- .github/workflows/modules/fake_module.lua | 1 + .github/workflows/tests_eessi_module.yml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/modules/fake_module.lua diff --git a/.github/workflows/modules/fake_module.lua b/.github/workflows/modules/fake_module.lua new file mode 100644 index 0000000000..f23a69c008 --- /dev/null +++ b/.github/workflows/modules/fake_module.lua @@ -0,0 +1 @@ +setenv("INSIDE_GITHUB_ACTIONS", "true") diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 8d9909bfdf..89e48187cb 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -181,9 +181,11 @@ jobs: initial_env_file="initial_env.txt" module_cycled_file="load_unload_cycle.txt" - # prepare Lmod, including resetting it (even though this will likely return an error since we have no defaults) - export MODULEPATH=init/modules + # prepare Lmod, resetting it in a roundabout way + export MODULEPATH=init/modules:.github/workflows/modules + module load fake_module module purge + module unuse .github/workflows/modules module avail # Store the initial environment From 6b9d9809d6cadd4513d165bdc9213f5efa4883a0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 13:24:25 +0200 Subject: [PATCH 1488/1795] Keep trying to figure out how to control Lmod --- .github/workflows/modules/fake_module.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/modules/fake_module.lua b/.github/workflows/modules/fake_module.lua index f23a69c008..e45cb640d6 100644 --- a/.github/workflows/modules/fake_module.lua +++ b/.github/workflows/modules/fake_module.lua @@ -1 +1,3 @@ setenv("INSIDE_GITHUB_ACTIONS", "true") +-- Interfere with PATH so Lmod keeps a record +prepend_path("PATH", "/snap/bin") From 834ce38c5628a56d0057300fd046a63a5b17ef53 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Oct 2024 13:29:25 +0200 Subject: [PATCH 1489/1795] Ignore Lmod tables when doing environment comparison --- .github/workflows/tests_eessi_module.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 89e48187cb..2bf4b39bde 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -181,21 +181,21 @@ jobs: initial_env_file="initial_env.txt" module_cycled_file="load_unload_cycle.txt" - # prepare Lmod, resetting it in a roundabout way + # prepare Lmod, resetting it in a roundabout given we don't want defaults set export MODULEPATH=init/modules:.github/workflows/modules module load fake_module module purge module unuse .github/workflows/modules module avail - # Store the initial environment - env | sort > "${initial_env_file}" + # Store the initial environment (ignoring Lmod tables) + env | grep -v _ModuleTable | sort > "${initial_env_file}" # Do (and undo) loading the EESSI module CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath) module load EESSI/${{matrix.EESSI_VERSION}} module unload EESSI/${{matrix.EESSI_VERSION}} - env | sort > "${module_cycled_file}" + env | grep -v _ModuleTable | sort > "${module_cycled_file}" # Now compare the two results (do not expose the files, as they contain the full environment!) if (diff "${initial_env_file}" "${module_cycled_file}" > /dev/null); then From f97fd82ac8ca7d54e8f5565b7a2cb8d4c2a72cc0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Oct 2024 15:09:22 +0200 Subject: [PATCH 1490/1795] {2023.06}[2023a] mpl-ascii v0.10.0 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 170a639064..3ebc4ebc56 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -9,3 +9,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 + - mpl-ascii-0.10.0-gfbf-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21679 + from-commit: 7106f63160b1418d605882dd02ba151d099300bd From 41f8e028e1269d1b455553257cc66690342467b4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 16:31:45 +0200 Subject: [PATCH 1491/1795] Move test if we are in prefix --- EESSI-install-software.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 68ff4a1927..65ab8ec986 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -112,12 +112,6 @@ fi TMPDIR=$(mktemp -d) -# make sure we're in Prefix environment by checking $SHELL -if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then - echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" -else - fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" -fi # Get override subdir DETECTION_PARAMETERS='' @@ -180,6 +174,14 @@ module unuse $MODULEPATH module use $TOPDIR/init/modules module load EESSI/$EESSI_VERSION +# make sure we're in Prefix environment by checking $SHELL +# We can only do this after loading the EESSI module, as we need ${EPREFIX} +if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then + echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" +else + fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" +fi + if [ -d $EESSI_CVMFS_REPO ]; then echo_green "$EESSI_CVMFS_REPO available, OK!" else From 04c25738137a041a0bda39ba80f7e291c2e72833 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 16 Oct 2024 16:35:21 +0200 Subject: [PATCH 1492/1795] Update 2023.06.lua --- init/modules/EESSI/2023.06.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 0c2f34280a..348699c0f1 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -124,7 +124,8 @@ if ( mode() ~= "spider" ) then end prepend_path("LMOD_RC", pathJoin(eessi_software_path, ".lmod", "lmodrc.lua")) eessiDebug("Adding " .. pathJoin(eessi_software_path, ".lmod", "lmodrc.lua") .. " to LMOD_RC") -setenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) +-- Use pushenv for LMOD_PACKAGE_PATH as this may be set locally by the site +pushenv("LMOD_PACKAGE_PATH", pathJoin(eessi_software_path, ".lmod")) eessiDebug("Setting LMOD_PACKAGE_PATH to " .. pathJoin(eessi_software_path, ".lmod")) -- the accelerator may have an empty value and we need to give some flexibility From ab27742e903932214860424ced935b9ef2515f6c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 21:57:42 +0200 Subject: [PATCH 1493/1795] Unsetting all the other variables that EESSI-extend would consider, we can only set one --- EESSI-install-software.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index add7e7a9be..0cb05efddc 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -221,6 +221,9 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then fi echo ">> Configuring EasyBuild..." +unset EESSI_USER_INSTALL +unset EESSI_PROJECT_INSTALL +unset EESSI_SITE_INSTALL export EESSI_CVMFS_INSTALL=1 module load EESSI-extend/${EESSI_VERSION}-easybuild From abde580bdadb92d3fa1d1e6e2a7a3b0db4ca589a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 17 Oct 2024 19:41:45 +0200 Subject: [PATCH 1494/1795] use TARGET=ARMV8 when building OpenBLAS for aarch64/generic --- .../20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml | 9 +++++++++ eb_hooks.py | 14 +++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml new file mode 100644 index 0000000000..4ba7983da6 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml @@ -0,0 +1,9 @@ +# 2024.10.17 +# TARGET=ARMV8 must be used when building OpenBLAS for aarch64/generic, +# since otherwise "Illegal instruction" errors may happen in the driver part of OpenBLAS +# on systems that only support a minimal instruction set like Arm v8 (like Raspberry Pi SBCs); +# see also https://github.com/OpenMathLib/OpenBLAS/issues/4945 +easyconfigs: + - OpenBLAS-0.3.21-GCC-12.2.0.eb + - OpenBLAS-0.3.23-GCC-12.3.0.eb + - OpenBLAS-0.3.24-GCC-13.2.0.eb diff --git a/eb_hooks.py b/eb_hooks.py index 33bdf53c52..ba6f9ad5da 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -491,11 +491,19 @@ def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): for step in ('build', 'test', 'install'): self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1") - # use -mtune=generic rather than -mcpu=generic in $CFLAGS on aarch64, - # because -mcpu=generic implies a particular -march=armv* which clashes with those used by OpenBLAS - # when building with DYNAMIC_ARCH=1 if get_cpu_architecture() == AARCH64: + # when building for aarch64/generic, we also need to set TARGET=ARMV8 to make sure + # that the driver parts of OpenBLAS are compiled generically; + # see also https://github.com/OpenMathLib/OpenBLAS/issues/4945 + for step in ('build', 'test', 'install'): + self.cfg.update(f'{step}opts', "TARGET=ARMV8") + + # use -mtune=generic rather than -mcpu=generic in $CFLAGS for aarch64/generic, + # because -mcpu=generic implies a particular -march=armv* which clashes with those used by OpenBLAS + # when building with DYNAMIC_ARCH=1 cflags = os.getenv('CFLAGS').replace('-mcpu=generic', '-mtune=generic') + self.log.info("Replaced -mcpu=generic with -mtune=generic in $CFLAGS") + self.log.info("Updating $CFLAGS to: %s", cflags) env.setvar('CFLAGS', cflags) else: raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") From eb6b534a49491aee7f8909251b55a4a79228f7b2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 17 Oct 2024 20:00:53 +0200 Subject: [PATCH 1495/1795] {2023.06}[2023a] jedi 0.19.0 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 3ebc4ebc56..2fac9b8330 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -13,3 +13,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21679 from-commit: 7106f63160b1418d605882dd02ba151d099300bd + - jedi-0.19.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21650 + from-commit: 109998f6adcda7efb4174b1e5f73b41ee82d1f13 From 188938ad2204f596667d26a673214605c7541ddb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 17 Oct 2024 21:29:48 +0200 Subject: [PATCH 1496/1795] use updated OpenBLAS easyblock that sets DYNAMIC_ARCH=1 for generic builds + uses TARGET=ARMV8 for aarch64/generic --- ...1017-eb-4.9.4-OpenBLAS-aarch64-generic.yml | 15 +++++++++++--- eb_hooks.py | 20 +++++++++++++------ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml index 4ba7983da6..d6d8f70143 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241017-eb-4.9.4-OpenBLAS-aarch64-generic.yml @@ -4,6 +4,15 @@ # on systems that only support a minimal instruction set like Arm v8 (like Raspberry Pi SBCs); # see also https://github.com/OpenMathLib/OpenBLAS/issues/4945 easyconfigs: - - OpenBLAS-0.3.21-GCC-12.2.0.eb - - OpenBLAS-0.3.23-GCC-12.3.0.eb - - OpenBLAS-0.3.24-GCC-13.2.0.eb + - OpenBLAS-0.3.21-GCC-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3492 + include-easyblocks-from-commit: d06d9617d9bfb63d338b6879eab9da81c8a312d8 + - OpenBLAS-0.3.23-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3492 + include-easyblocks-from-commit: d06d9617d9bfb63d338b6879eab9da81c8a312d8 + - OpenBLAS-0.3.24-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3492 + include-easyblocks-from-commit: d06d9617d9bfb63d338b6879eab9da81c8a312d8 diff --git a/eb_hooks.py b/eb_hooks.py index ba6f9ad5da..a3ae6bbc05 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -486,25 +486,33 @@ def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): """ Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC """ + # note: OpenBLAS easyblock was updated in https://github.com/easybuilders/easybuild-easyblocks/pull/3492 + # to take care of this already, so at some point this hook can be removed... if self.name == 'OpenBLAS': if build_option('optarch') == OPTARCH_GENERIC: + dynamic_arch = 'DYNAMIC_ARCH=1' for step in ('build', 'test', 'install'): - self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1") + if dynamic_arch not in self.cfg[f'{step}opts']: + self.cfg.update(f'{step}opts', dynamic_arch) if get_cpu_architecture() == AARCH64: # when building for aarch64/generic, we also need to set TARGET=ARMV8 to make sure # that the driver parts of OpenBLAS are compiled generically; # see also https://github.com/OpenMathLib/OpenBLAS/issues/4945 + target_armv8 = 'TARGET=ARMV8' for step in ('build', 'test', 'install'): - self.cfg.update(f'{step}opts', "TARGET=ARMV8") + if target_armv8 not in self.cfg[f'{step}opts']: + self.cfg.update(f'{step}opts', target_armv8) # use -mtune=generic rather than -mcpu=generic in $CFLAGS for aarch64/generic, # because -mcpu=generic implies a particular -march=armv* which clashes with those used by OpenBLAS # when building with DYNAMIC_ARCH=1 - cflags = os.getenv('CFLAGS').replace('-mcpu=generic', '-mtune=generic') - self.log.info("Replaced -mcpu=generic with -mtune=generic in $CFLAGS") - self.log.info("Updating $CFLAGS to: %s", cflags) - env.setvar('CFLAGS', cflags) + mcpu_generic = '-mcpu=generic' + cflags = os.getenv('CFLAGS') + if mcpu_generic in cflags: + cflags = cflags.replace(mcpu_generic, '-mtune=generic') + self.log.info("Replaced -mcpu=generic with -mtune=generic in $CFLAGS") + env.setvar('CFLAGS', cflags) else: raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") From 0154d649eface117f71dc07b598b8dc780c61f2e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 13:23:11 +0200 Subject: [PATCH 1497/1795] Streamline the checks for missing installations --- .../scripts/only_latest_easystacks.sh | 42 +++++++++++++++++++ .github/workflows/test-software.eessi.io.yml | 6 +-- check_missing_installations.sh | 17 ++++---- 3 files changed, 54 insertions(+), 11 deletions(-) create mode 100755 .github/workflows/scripts/only_latest_easystacks.sh diff --git a/.github/workflows/scripts/only_latest_easystacks.sh b/.github/workflows/scripts/only_latest_easystacks.sh new file mode 100755 index 0000000000..c78829e724 --- /dev/null +++ b/.github/workflows/scripts/only_latest_easystacks.sh @@ -0,0 +1,42 @@ +#!/bin/bash +EESSI_VERSION=${EESSI_VERSION:-"2023.06"} + +directory="easystacks/software.eessi.io/${EESSI_VERSION}" +# List of example filenames +files=($(ls "$directory"/*.yml)) +[ -n "$DEBUG" ] && echo "${files[@]}" + +versions=() +# Loop over each filename +for filename in "${files[@]}"; do + # Extract the semantic version using grep + version=$(echo "$filename" | grep -oP '(?<=eb-)\d+\.\d+\.\d+?(?=-)') + + # Output the result + [ -n "$DEBUG" ] && echo "Filename: $filename" + [ -n "$DEBUG" ] && echo "Extracted version: $version" + [ -n "$DEBUG" ] && echo + versions+=("$version") +done +highest_version=$(printf "%s\n" "${versions[@]}" | sort -V | tail -n 1) + +[ -n "$DEBUG" ] && echo "Highest version: $highest_version" +[ -n "$DEBUG" ] && echo +[ -n "$DEBUG" ] && echo "Matching files:" +all_latest_easystacks=($(find $directory -type f -name "*eb-$highest_version*.yml")) + +accel_latest_easystacks=() +cpu_latest_easystacks=() + +# Loop through the array and split based on partial matching of string +accel="/accel/" +for item in "${all_latest_easystacks[@]}"; do + if [[ "$item" == *"$accel"* ]]; then + accel_latest_easystacks+=("$item") + else + cpu_latest_easystacks+=("$item") + fi +done + +# Output the results +[ -n "$ACCEL_EASYSTACKS" ] && echo "${accel_latest_easystacks[@]}" || echo "${cpu_latest_easystacks[@]}" diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 6f592cf4c4..525e16f99d 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -57,8 +57,8 @@ jobs: env | grep ^EESSI | sort # first check the CPU-only builds for this CPU target - echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" - for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" + for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then if grep -q 2022b <<<"${easystack_file}"; then # skip the check of installed software on zen4 for foss/2022b builds @@ -82,7 +82,7 @@ jobs: for accel in ${accelerators}; do module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" - for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/$(dirname ${accel})/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 280de294af..7edd2ebb39 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -25,16 +25,17 @@ easystack=$1 LOCAL_TMPDIR=$(mktemp -d) +# ~~No longer required as we use from-commit as of EB 4.9.something~~ # Clone the develop branch of EasyBuild and use that to search for easyconfigs - -if [[ -z ${EASYBUILD_ROBOT_PATHS} ]]; then - git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs - export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs -fi - +# +#if [[ -z ${EASYBUILD_ROBOT_PATHS} ]]; then +# git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs +# export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs +#fi +# # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr -tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) -grep -v from-pr ${easystack} > ${tmp_easystack} +# tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) +# grep -v from-pr ${easystack} > ${tmp_easystack} source $TOPDIR/scripts/utils.sh From b7e343d8bdba537ce01522831fa62d4e66dae2ab Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 13:32:46 +0200 Subject: [PATCH 1498/1795] Clean out all code relating to use of from-pr --- check_missing_installations.sh | 37 +--------------------------------- 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 7edd2ebb39..79f6acc733 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -25,18 +25,6 @@ easystack=$1 LOCAL_TMPDIR=$(mktemp -d) -# ~~No longer required as we use from-commit as of EB 4.9.something~~ -# Clone the develop branch of EasyBuild and use that to search for easyconfigs -# -#if [[ -z ${EASYBUILD_ROBOT_PATHS} ]]; then -# git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs -# export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs -#fi -# -# All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr -# tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) -# grep -v from-pr ${easystack} > ${tmp_easystack} - source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild @@ -46,34 +34,11 @@ ${EB:-eb} --show-config echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$exit_code" -ne 0 ] && [ ! -z "$pr_exceptions" ]; then - # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR - # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg - # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as - # well skip this check - unmerged PRs can not be the reason for the non-zero exit code in that scenario - - # Let's use awk so we can allow for exceptions if we are given a PR diff file - awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" - awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! - eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack} - - msg=">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}," - msg="${msg} allowing for --from-pr's that were added in this PR..." - echo ${msg} - eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out - ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} - exit_code_with_from_pr=${PIPESTATUS[0]} - - # If now we succeeded, the reason must be that we originally stripped the --from-pr's - if [ "$exit_code_with_from_pr" -eq 0 ]; then - fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" - fi -fi check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" From 0f49570976f75354e3ee0a5e9f25810d45638a55 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 13:41:46 +0200 Subject: [PATCH 1499/1795] Disable CUDA missing installation check for Zen4 --- .github/workflows/test-software.eessi.io.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 525e16f99d..3ed3289ed7 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -83,6 +83,11 @@ jobs: module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do + if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then + if grep -q CUDA <<<"${easystack_file}"; then + # skip the check of install CUDA software in the CPU path for zen4 + continue + fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 8af465370f5006376f30e7872643753a20d8d72a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 13:46:09 +0200 Subject: [PATCH 1500/1795] Typo --- .github/workflows/test-software.eessi.io.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 3ed3289ed7..2e7a8e2724 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -85,8 +85,9 @@ jobs: for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then if grep -q CUDA <<<"${easystack_file}"; then - # skip the check of install CUDA software in the CPU path for zen4 - continue + # skip the check of install CUDA software in the CPU path for zen4 + continue + fi fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} From 7b49df4d76d11b28a805992aca95a36d6fb011c3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 15:23:25 +0200 Subject: [PATCH 1501/1795] Shuffle some files --- .../rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml | 0 ...me.yml => 20240506-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{ => accel/nvidia}/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml (100%) rename easystacks/software.eessi.io/2023.06/rebuilds/{2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml => 20240506-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml rename to easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml From 94c04f85405e426c6eac64b67e0b2854ea860ee9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 15:24:52 +0200 Subject: [PATCH 1502/1795] Simplify things further --- .github/workflows/test-software.eessi.io.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 2e7a8e2724..025f797d52 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -63,9 +63,6 @@ jobs: if grep -q 2022b <<<"${easystack_file}"; then # skip the check of installed software on zen4 for foss/2022b builds continue - elif grep -q CUDA <<<"${easystack_file}"; then - # skip the check of install CUDA software in the CPU path for zen4 - continue fi fi echo "check missing installations for ${easystack_file}..." @@ -83,12 +80,6 @@ jobs: module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do - if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then - if grep -q CUDA <<<"${easystack_file}"; then - # skip the check of install CUDA software in the CPU path for zen4 - continue - fi - fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From f32a18cf1c1ade9bc411329a376904ecb4ea0db0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 16:15:04 +0200 Subject: [PATCH 1503/1795] Prune 2022b from easystacks when using Zen4 --- .github/workflows/test-software.eessi.io.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 025f797d52..851bd323e0 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -60,10 +60,11 @@ jobs: echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then - if grep -q 2022b <<<"${easystack_file}"; then - # skip the check of installed software on zen4 for foss/2022b builds - continue - fi + # Make a temporary EasyStack file where we clean out all 2022b stuff and use that + new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml) + # first clean out the options then clean out the .eb name + sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack + easystack_file="$new_easystack" fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} From 118ef43761d80830e82f7500199a2117a1e8653b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 16:39:59 +0200 Subject: [PATCH 1504/1795] Add a diff for pruned Zen4 easystacks --- .github/workflows/test-software.eessi.io.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 851bd323e0..e3d76351ea 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -64,6 +64,7 @@ jobs: new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml) # first clean out the options then clean out the .eb name sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack + diff --unified=0 "$easystack_file" "$new_easystack" || : easystack_file="$new_easystack" fi echo "check missing installations for ${easystack_file}..." From b4ef5da4c7282f57aa0ace944f20669d435accf1 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 18 Oct 2024 16:57:59 +0200 Subject: [PATCH 1505/1795] Revery some stuff and handle rebuilds for Zen4 --- .github/workflows/test-software.eessi.io.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index e3d76351ea..ca3792f6ef 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -60,12 +60,18 @@ jobs: echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then - # Make a temporary EasyStack file where we clean out all 2022b stuff and use that - new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml) - # first clean out the options then clean out the .eb name - sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack - diff --unified=0 "$easystack_file" "$new_easystack" || : - easystack_file="$new_easystack" + if grep -q 2022b <<<"${easystack_file}"; then + # skip the check of installed software on zen4 for foss/2022b builds + continue + fi + if [[ $easystack_file == *"rebuilds"* ]]; then + # Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that + new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml) + # first clean out the options then clean out the .eb name + sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack + diff --unified=0 "$easystack_file" "$new_easystack" || : + easystack_file="$new_easystack" + fi fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} From eb5f29b02739addb1e3db3ceb2cf03d3215a9f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 21 Oct 2024 10:44:09 +0200 Subject: [PATCH 1506/1795] easyblocks PR has been merged, use merge commit --- .../rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml index 36e887644d..5b90d44a89 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml @@ -4,13 +4,13 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb: options: - # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3484/commits/64060fa9b6a4d444406cc403ae00fbbc3f36c416 - include-easyblocks-from-commit: 64060fa9b6a4d444406cc403ae00fbbc3f36c416 + # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3484 + include-easyblocks-from-commit: 3671c5b7c238c7dc8aadd2c510329770ef1bdcdf - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 - # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3484/commits/64060fa9b6a4d444406cc403ae00fbbc3f36c416 - include-easyblocks-from-commit: 64060fa9b6a4d444406cc403ae00fbbc3f36c416 + # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3484 + include-easyblocks-from-commit: 3671c5b7c238c7dc8aadd2c510329770ef1bdcdf From c374bf451e11285eef343ed596f11c1c91bd7598 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 21 Oct 2024 15:17:44 +0200 Subject: [PATCH 1507/1795] Make sure to use the site installation path for install_cuda_libraries.sh --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index f9d889c1a1..69b11e26da 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -109,6 +109,13 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do # Load EasyBuild version for this easystack file _before_ loading EESSI-extend module avail EasyBuild module load EasyBuild/${eb_version} + # Make sure EESSI-extend does a site install here + # We need to reload it with the current environment variables set + unset EESSI_CVMFS_INSTALL + unset EESSI_PROJECT_INSTALL + unset EESSI_USER_INSTALL + export EESSI_SITE_INSTALL=1 + module unload EESSI-extend module load EESSI-extend/${EESSI_VERSION}-easybuild # Install modules in hidden .modules dir to keep track of what was installed before From 9c48e836aa4464d04f69922dbddeb8ec14d72802 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 21 Oct 2024 15:21:00 +0200 Subject: [PATCH 1508/1795] Make sure we reload EESSI-extend with the correct env vars set, since we want to install in the CVMFS prefix --- EESSI-install-software.sh | 85 +++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 0cb05efddc..e3b31d25e8 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -220,45 +220,6 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path} fi -echo ">> Configuring EasyBuild..." -unset EESSI_USER_INSTALL -unset EESSI_PROJECT_INSTALL -unset EESSI_SITE_INSTALL -export EESSI_CVMFS_INSTALL=1 -module load EESSI-extend/${EESSI_VERSION}-easybuild - -if [ ! -z "${shared_fs_path}" ]; then - shared_eb_sourcepath=${shared_fs_path}/easybuild/sources - echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" - export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} -fi - -# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available -if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then - CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all - if [ -d ${CPU_ONLY_MODULES_PATH} ]; then - module use ${CPU_ONLY_MODULES_PATH} - else - fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}" - fi -fi - -# If in dev.eessi.io, allow building on top of softw -if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then - module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all -fi - -module use $EASYBUILD_INSTALLPATH/modules/all - -if [[ -z ${MODULEPATH} ]]; then - fatal_error "Failed to set up \$MODULEPATH?!" -else - echo_green ">> MODULEPATH set up: ${MODULEPATH}" -fi - -# assume there's only one diff file that corresponds to the PR patch file -pr_diff=$(ls [0-9]*.diff | head -1) - # install any additional required scripts # order is important: these are needed to install a full CUDA SDK in host_injections # for now, this just reinstalls all scripts. Note the most elegant, but works @@ -301,6 +262,52 @@ if command_exists "nvidia-smi"; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh fi + +echo ">> Configuring EasyBuild..." + +# Make sure that we use the EESSI_CVMFS_INSTALL +# Since the path is set when loading EESSI-extend, we reload it to make sure it works - even if it is already loaded +# Note we need to do this after running install_cuda_and_libraries, since that does installations in the EESSI_SITE_INSTALL +unset EESSI_USER_INSTALL +unset EESSI_PROJECT_INSTALL +unset EESSI_SITE_INSTALL +export EESSI_CVMFS_INSTALL=1 +module unload EESSI-extend +module load EESSI-extend/${EESSI_VERSION}-easybuild + +if [ ! -z "${shared_fs_path}" ]; then + shared_eb_sourcepath=${shared_fs_path}/easybuild/sources + echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" + export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} +fi + +# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all + if [ -d ${CPU_ONLY_MODULES_PATH} ]; then + module use ${CPU_ONLY_MODULES_PATH} + else + fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}" + fi +fi + +# If in dev.eessi.io, allow building on top of softw +if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then + module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all +fi + +module use $EASYBUILD_INSTALLPATH/modules/all + +if [[ -z ${MODULEPATH} ]]; then + fatal_error "Failed to set up \$MODULEPATH?!" +else + echo_green ">> MODULEPATH set up: ${MODULEPATH}" +fi + +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + + # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then From ad42fad480c3b49a373069e2f323895bf0aa23e3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 22 Oct 2024 08:17:06 +0200 Subject: [PATCH 1509/1795] Remove patchelf, add util-linux. Patchelf is already in the stack, so not suitable as a test package --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 1 - .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml index c33d2e1ac6..5325f2e553 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -7,4 +7,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b - - patchelf-0.18.0-GCCcore-13.2.0.eb: diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 170a639064..afeed2404e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -9,3 +9,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 + - util-linux-2.39-GCCcore-13.2.0.eb: From ce5f2c8640b60d23cdebb527f4045549c0e1829e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 22 Oct 2024 09:09:09 +0200 Subject: [PATCH 1510/1795] Remove util-linux again, this was just to prove the PR worked --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index afeed2404e..170a639064 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -9,4 +9,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 - - util-linux-2.39-GCCcore-13.2.0.eb: From abf1aff4146f773a0c9e9a92f0c6eaf480e09a72 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 22 Oct 2024 23:21:22 +0200 Subject: [PATCH 1511/1795] Update EESSI-install-software.sh Co-authored-by: ocaisa --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index e3b31d25e8..65c43d5ac5 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -291,7 +291,7 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi -# If in dev.eessi.io, allow building on top of softw +# If in dev.eessi.io, allow building on top of software.eessi.io if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all fi From fb8bbd114deed64698faa5700ad22a518e7b019a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 23 Oct 2024 15:46:05 +0200 Subject: [PATCH 1512/1795] make 4.9.4 2023b easystack file, add Score-P --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml new file mode 100644 index 0000000000..3b6234ffce --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - Score-P-8.4-gompi-2023b.eb From 6e5bb5cadee76dfd93af74d3cc40243cea1bd142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 25 Oct 2024 16:36:02 +0200 Subject: [PATCH 1513/1795] add SIONlib (+fix) --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 3b6234ffce..e20d3daa38 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -1,2 +1,6 @@ easyconfigs: + - SIONlib-1.7.7-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21748 + from-commit: 253198299616e4069327797374e579077aa8dfa5 - Score-P-8.4-gompi-2023b.eb From ecf97e3a2c672f69e96da8331283f675703439e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 25 Oct 2024 16:36:53 +0200 Subject: [PATCH 1514/1795] add SIONlib from a specific commit --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 3b6234ffce..e20d3daa38 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -1,2 +1,6 @@ easyconfigs: + - SIONlib-1.7.7-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21748 + from-commit: 253198299616e4069327797374e579077aa8dfa5 - Score-P-8.4-gompi-2023b.eb From 5856c2e605e57b7bb2522f713eae5adb3c77b57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 25 Oct 2024 16:45:19 +0200 Subject: [PATCH 1515/1795] use correct filename for SIONlib easyconfig --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index e20d3daa38..d5ff8f31b9 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -1,5 +1,5 @@ easyconfigs: - - SIONlib-1.7.7-GCCcore-13.2.0.eb: + - SIONlib-1.7.7-GCCcore-13.2.0-tools.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21748 from-commit: 253198299616e4069327797374e579077aa8dfa5 From e2574ee73269338fa7258ff7eb6a820c2ed3a47f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 25 Oct 2024 17:23:54 +0200 Subject: [PATCH 1516/1795] add hook for Score-P for picking up libbfd from compat layer --- eb_hooks.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index a3ae6bbc05..03642656ea 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -414,6 +414,29 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") +def pre_configure_hook_score_p(self, *args, **kwargs): + """ + Pre-configure hook for Score-p + - specify correct path to binutils (in compat layer) + """ + if self.name == 'Score-P': + + # determine path to Prefix installation in compat layer via $EPREFIX + eprefix = get_eessi_envvar('EPREFIX') + + binutils_lib_path_glob_pattern = os.path.join(eprefix, 'usr', 'lib*', 'binutils', '*-linux-gnu', '2.*') + binutils_lib_path = glob.glob(binutils_lib_path_glob_pattern) + if len(binutils_lib_path) == 1: + self.cfg.update('configopts', '--with-libbfd-lib=' + binutils_lib_path[0]) + self.cfg.update('configopts', '--with-libbfd-include=' + os.path.join(binutils_lib_path[0], 'include')) + else: + raise EasyBuildError("Failed to isolate path for binutils libraries using %s, got %s", + binutils_lib_path_glob_pattern, binutils_lib_path) + + else: + raise EasyBuildError("Score-P-specific hook triggered for non-Score-P easyconfig?!") + + def pre_configure_hook_extrae(self, *args, **kwargs): """ Pre-configure hook for Extrae @@ -972,6 +995,7 @@ def inject_gpu_property(ec): 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, 'LAMMPS': pre_configure_hook_LAMMPS_zen4, + 'Score-P': pre_configure_hook_score_p, } PRE_TEST_HOOKS = { From 5901d41130c53d51563681622aba0419546ceacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 28 Oct 2024 11:02:15 +0100 Subject: [PATCH 1517/1795] use newer commit for SIONlib, includes a fix for non-x86-64 architectures --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index d5ff8f31b9..8edf3c84b0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -1,6 +1,6 @@ easyconfigs: - SIONlib-1.7.7-GCCcore-13.2.0-tools.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21748 - from-commit: 253198299616e4069327797374e579077aa8dfa5 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21752 + from-commit: 6b8b53493a1188a5baa56a133574daac239730e7 - Score-P-8.4-gompi-2023b.eb From a7044806eb149fec5b5aa1c128ef6ca114b0ec55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Oct 2024 09:46:15 +0100 Subject: [PATCH 1518/1795] use modified easyblock for Score-P --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 8edf3c84b0..a63ccf8bc0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -3,4 +3,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21752 from-commit: 6b8b53493a1188a5baa56a133574daac239730e7 - - Score-P-8.4-gompi-2023b.eb + - Score-P-8.4-gompi-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3496 + include-easyblocks-from-commit: 6b7efa90f35a71c6be2ccc37096915f8a1f57225 From 7b90c46732e6dff8929f13cf4d572ec80e60b96f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Oct 2024 11:48:09 +0100 Subject: [PATCH 1519/1795] use newer commit of easyblock --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index a63ccf8bc0..89312b1e49 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -6,4 +6,4 @@ easyconfigs: - Score-P-8.4-gompi-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyblocks/pull/3496 - include-easyblocks-from-commit: 6b7efa90f35a71c6be2ccc37096915f8a1f57225 + include-easyblocks-from-commit: f23d154eb75b72767557f78a178eb31ae9ac9d9c From 2bc14d1db0d418801a5ac3cbadad95bad55be28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Oct 2024 13:49:48 +0100 Subject: [PATCH 1520/1795] use newer commit of easyblock --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 89312b1e49..099df62c08 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -6,4 +6,4 @@ easyconfigs: - Score-P-8.4-gompi-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyblocks/pull/3496 - include-easyblocks-from-commit: f23d154eb75b72767557f78a178eb31ae9ac9d9c + include-easyblocks-from-commit: 60633b0acfd41a0732992d9e16800dae71a056eb From 3ec0157ef11001586de4eb72c0e3e7bbd5ac6732 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 31 Oct 2024 16:05:53 +0100 Subject: [PATCH 1521/1795] Address review --- .../scripts/only_latest_easystacks.sh | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/only_latest_easystacks.sh b/.github/workflows/scripts/only_latest_easystacks.sh index c78829e724..acc9d3279a 100755 --- a/.github/workflows/scripts/only_latest_easystacks.sh +++ b/.github/workflows/scripts/only_latest_easystacks.sh @@ -1,9 +1,21 @@ #!/bin/bash +# +# This script figures out the latest version of EasyBuild being used for the installation of easystack +# files. +# +# This file is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Alan O'Cais (CECAM) +# +# license: GPLv2 +# + EESSI_VERSION=${EESSI_VERSION:-"2023.06"} directory="easystacks/software.eessi.io/${EESSI_VERSION}" # List of example filenames -files=($(ls "$directory"/*.yml)) +files=($(find "$directory" -name "*.yml" | grep -e '-eb-')) [ -n "$DEBUG" ] && echo "${files[@]}" versions=() @@ -39,4 +51,8 @@ for item in "${all_latest_easystacks[@]}"; do done # Output the results -[ -n "$ACCEL_EASYSTACKS" ] && echo "${accel_latest_easystacks[@]}" || echo "${cpu_latest_easystacks[@]}" +if [ -n "$ACCEL_EASYSTACKS" ]; then + echo "${accel_latest_easystacks[@]}" +else + echo "${cpu_latest_easystacks[@]}" +fi From 004d39afe7c361ba1fa21f4e646029474c6ce61e Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 5 Nov 2024 14:58:28 +0000 Subject: [PATCH 1522/1795] {2023.06}[foss/2023a] Solids4foam-2.1 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 2fac9b8330..1a147af30d 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -17,3 +17,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21650 from-commit: 109998f6adcda7efb4174b1e5f73b41ee82d1f13 + - Solids4foam-2.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21606 + from-commit: 63562c58acf1be64407192b6862c3bd80253d2e0 From 6caca557e0fbf8c47f7959f691ce1ec0eb7ff575 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 7 Nov 2024 07:56:49 +0000 Subject: [PATCH 1523/1795] {2023.06}[foss/2023a] Cassiopeia v2.0.0 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 2fac9b8330..ca8f20c9e2 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -17,3 +17,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21650 from-commit: 109998f6adcda7efb4174b1e5f73b41ee82d1f13 + - Cassiopeia-2.0.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657 + from-commit: 7f1f0e60487e7e1fcb5c4e6bc4fbc4f89994e3fd From 3eae3f265480ab2f2343f313fbcc6664d02b4855 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 7 Nov 2024 10:16:24 +0100 Subject: [PATCH 1524/1795] Apply suggestions from code review Accepted all except one Co-authored-by: TopRichard <121792457+TopRichard@users.noreply.github.com> --- .../nvidia/link_nvidia_host_libraries.sh | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 967e7d3589..2218a92116 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -15,8 +15,8 @@ get_host_ldconfig() { local found_paths=() # Initialize an array to store found paths # Always attempt to use /sbin/ldconfig - if [ -x "/sbin/$command_name" ]; then - found_paths+=("/sbin/$command_name") + if [ -x "/sbin/${command_name}" ]; then + found_paths+=("/sbin/${command_name}") fi # Split the $PATH and iterate over each directory @@ -28,8 +28,8 @@ get_host_ldconfig() { # Check if directory does not start with the exclude prefix if [[ ! "$dir" =~ ^$exclude_prefix ]]; then - if [ -x "$dir/$command_name" ]; then - found_paths+=("$dir/$command_name") + if [ -x "${dir}/${command_name}" ]; then + found_paths+=("${dir}/${command_name}") fi fi done @@ -142,7 +142,7 @@ LIBS_LIST="" # Parse command-line options while [[ "$#" -gt 0 ]]; do - case $1 in + case "$1" in --ld-preload) LD_PRELOAD_MODE=1 ;; # Enable LD_PRELOAD mode --no-download) LIBS_LIST="default" ;; # Download latest list of CUDA libraries *) fatal_error "Unknown option: $1";; @@ -151,7 +151,7 @@ while [[ "$#" -gt 0 ]]; do done # Gather information about NVIDIA drivers (even if we are inside a Gentoo Prefix in a container) -export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH="/.singularity.d/libs:${LD_LIBRARY_PATH}" # Check for NVIDIA GPUs via nvidia-smi command nvidia_smi=$(command -v nvidia-smi) @@ -159,9 +159,9 @@ if [[ $? -eq 0 ]]; then nvidia_smi_out=$(mktemp -p /tmp nvidia_smi_out.XXXXX) nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader 2>&1 > $nvidia_smi_out if [[ $? -eq 0 ]]; then - nvidia_smi_info=$(head -1 $nvidia_smi_out) - host_cuda_version=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f4 -d,) - host_driver_version=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f3 -d,) + nvidia_smi_info=$(head -1 "${nvidia_smi_out}") + host_cuda_version=$(echo "${nvidia_smi_info}" | sed 's/, /,/g' | cut -f4 -d,) + host_driver_version=$(echo "${nvidia_smi_info}" | sed 's/, /,/g' | cut -f3 -d,) echo_green "Found host CUDA version ${host_cuda_version}" echo_green "Found NVIDIA GPU driver version ${host_driver_version}" rm -f $nvidia_smi_out @@ -180,7 +180,7 @@ fi # Find the host ldconfig host_ldconfig=$(get_host_ldconfig) # Gather libraries on the host (_must_ be host ldconfig) -host_libraries=$($host_ldconfig -p | awk '{print $NF}') +host_libraries=$("${host_ldconfig}" -p | awk '{print $NF}') singularity_libs=$(ls /.singularity.d/libs/* 2>/dev/null) # Now gather the list of possible CUDA libraries and make them into an array @@ -223,13 +223,13 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then compat_filtered_libraries=() for library in "${matched_libraries[@]}"; do # Run ldd on the given binary and filter for "not found" libraries - not_found_libs=$(ldd "$library" 2>/dev/null | grep "not found" | awk '{print $1}') + not_found_libs=$(ldd "${library}" 2>/dev/null | grep "not found" | awk '{print $1}') # Check if it is missing an so dep under EESSI if [[ -z "$not_found_libs" ]]; then # Resolve any symlink realpath_library=$(realpath "$library") if [[ ! " ${filtered_libraries[@]} " =~ " $realpath_library " ]]; then - filtered_libraries+=("$realpath_library") + filtered_libraries+=("${realpath_library}") # Also prepare compat only libraries for the short list for item in "${cuda_compat_nvlib_list[@]}"; do # Check if the current item is a substring of $library @@ -249,7 +249,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then found=false for listed_lib in "${matched_libraries[@]}"; do # Matching to the .so or a symlink target is enough - realpath_lib=$(realpath "$listed_lib") + realpath_lib=$(realpath "${listed_lib}") if [[ "$lib" == "$listed_lib"* || "$realpath_lib" == *"$lib" ]]; then found=true break @@ -266,16 +266,16 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # If we find all the missing libs in our list include it if [[ "$all_found" == true ]]; then # Resolve any symlink - realpath_library=$(realpath "$library") + realpath_library=$(realpath "${library}") if [[ ! " ${filtered_libraries[@]} " =~ " $realpath_library " ]]; then - filtered_libraries+=("$realpath_library") + filtered_libraries+=("${realpath_library}") # Also prepare compat only libraries for the short list for item in "${cuda_compat_nvlib_list[@]}"; do # Check if the current item is a substring of $library if [[ "$realpath_library" == *"$item"* ]]; then echo "Match found for $item for CUDA compat libraries" if [[ ! " ${compat_filtered_libraries[@]} " =~ " $realpath_library " ]]; then - compat_filtered_libraries+=("$realpath_library") + compat_filtered_libraries+=("${realpath_library}") fi break fi From 31de5d6bbaa190359155abceb936d7723cd335b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 11 Nov 2024 11:21:52 +0100 Subject: [PATCH 1525/1795] add Cython-3.0.10-GCCcore-13.2.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 099df62c08..77287f0daf 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -7,3 +7,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyblocks/pull/3496 include-easyblocks-from-commit: 60633b0acfd41a0732992d9e16800dae71a056eb + - Cython-3.0.10-GCCcore-13.2.0.eb From 78a3aa0ac7c5a3a64a7cdddeb07aa3136247fc48 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 12 Nov 2024 13:30:54 +0000 Subject: [PATCH 1526/1795] use `${tmpdir}/none.py` in `install_cuda_and_libraries.sh` --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 69b11e26da..e6343595ad 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -99,10 +99,6 @@ SAVE_MODULEPATH=${MODULEPATH} for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do echo -e "Processing easystack file ${easystack_file}...\n\n" - # We don't want hooks used in this install, we need vanilla installations - touch "${tmpdir}"/none.py - export EASYBUILD_HOOKS="${tmpdir}/none.py" - # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file eb_version=$(echo ${EASYSTACK_FILE} | sed 's/.*eb-\([0-9.]*\).*/\1/g') @@ -124,6 +120,10 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do MODULEPATH=${EESSI_SITE_SOFTWARE_PATH}/.modules/all echo "set MODULEPATH=${MODULEPATH}" + # We don't want hooks used in this install, we need vanilla installations + touch "${tmpdir}"/none.py + export EASYBUILD_HOOKS="${tmpdir}/none.py" + # show EasyBuild configuration echo "Show EasyBuild configuration" eb --show-config From a122c6e638ce8c0e909031c7e8fde96054317925 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 12 Nov 2024 20:34:52 +0100 Subject: [PATCH 1527/1795] adjust installpath if EESSI_ACCELERATOR_TARGET is set --- EESSI-extend-2023.06-easybuild.eb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 8e328c3ece..bfe7931c8f 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -87,6 +87,16 @@ if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then end eessi_cvmfs_install = true easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") + eessi_accelerator_target = os.getenv("EESSI_ACCELERATOR_TARGET") + if (eessi_accelerator_target ~= nil) then + cuda_compute_capability = string.match(eessi_accelerator_target, "^nvidia/cc([0-9][0-9])$") + if (cuda_compute_capability ~= nil) then + easybuild_installpath = pathJoin(easybuild_installpath, 'accel', eessi_accelerator_target) + easybuild_cuda_compute_capabilities = cuda_compute_capability:sub(1, 1) .. "." .. cuda_compute_capability:sub(2, 2) + else + LmodError("Incorrect value for $EESSI_ACCELERATOR_TARGET: " .. eessi_accelerator_target) + end + end elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then -- Make sure no other EESSI install environment variables are set if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then @@ -146,6 +156,11 @@ setenv ("EASYBUILD_UMASK", "022") -- Allow this module to be loaded when running EasyBuild setenv ("EASYBUILD_ALLOW_LOADED_MODULES", "EasyBuild,EESSI-extend") +-- Set environment variables if building for CUDA compute capabilities +if (easybuild_cuda_compute_capabilities ~= nil) then + setenv ("EASYBUILD_CUDA_COMPUTE_CAPABILITIES", easybuild_cuda_compute_capabilities) +end + -- Set all related environment variables if we have project or user installations (including extending MODULEPATH) if (user_modulepath ~= nil) then -- Use a more restrictive umask for this case From 7cead8e19dde11b17bf75ba721b63fad4f7b736f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 12 Nov 2024 20:44:36 +0100 Subject: [PATCH 1528/1795] add rebuild easystack for EESSI-extend --- .../2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml new file mode 100644 index 0000000000..e4c658784f --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml @@ -0,0 +1,6 @@ +# 2024.11.12 +# for installations under /cvmfs, if EESSI_ACCELERATOR_TARGET is set, +# EESSI-extend should adjust EASYBUILD_INSTALLPATH and set +# EASYBUILD_CUDA_COMPUTE_CAPABILITIES +easyconfigs: + - EESSI-extend-2023.06-easybuild.eb From e3e32d86ebd24c83bc421f65d0159b086d899e37 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 13 Nov 2024 09:19:03 +0000 Subject: [PATCH 1529/1795] GPU docs have moved --- create_lmodsitepackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 11ca614be5..e959572ab1 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -114,7 +114,7 @@ -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI - local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" + local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/site_specific_config/gpu/.\\n" if simpleName == 'CUDA' then -- get the full host_injections path local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') From 58c1da5ff401aea4d9829329f439cad7b1e28e1e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 11:00:15 +0100 Subject: [PATCH 1530/1795] adding changes from PR #812 to validate them here --- EESSI-install-software.sh | 9 ++- load_eessi_extend_module.sh | 106 ++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100755 load_eessi_extend_module.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 65c43d5ac5..b365edd71b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -273,7 +273,14 @@ unset EESSI_PROJECT_INSTALL unset EESSI_SITE_INSTALL export EESSI_CVMFS_INSTALL=1 module unload EESSI-extend -module load EESSI-extend/${EESSI_VERSION}-easybuild + +# The EESSI-extend module is being loaded (or installed if it doesn't exist yet). +# The script requires the EESSI_VERSION given as argument, a couple of +# environment variables set (TMPDIR, EB and EASYBUILD_INSTALLPATH) and the +# function check_exit_code defined. +# NOTE, the script exits if those variables/functions are undefined. +export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +source load_eessi_extend_module.sh ${EESSI_VERSION} if [ ! -z "${shared_fs_path}" ]; then shared_eb_sourcepath=${shared_fs_path}/easybuild/sources diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh new file mode 100755 index 0000000000..bc277c5f0a --- /dev/null +++ b/load_eessi_extend_module.sh @@ -0,0 +1,106 @@ +# Script to load the environment module for EESSI-extend. +# If that module is not available yet, a specific version will be installed using the latest EasyBuild. +# +# This script must be sourced, since it makes changes in the current environment, like loading an EESSI-extend module. +# +# Assumptions (if one is not satisfied the script prints a message and exits) +# - EESSI version is given as first argument +# - TMPDIR is set +# - EB is set +# - EASYBUILD_INSTALLPATH needs to be set +# - Function check_exit_code is defined; +# scripts/utils.sh in EESSI/software-layer repository defines this function, hence +# scripts/utils.sh shall be sourced before this script is run +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Kenneth Hoste (@boegel, HPC-UGent) +# author: Alan O'Cais (@ocaisa, CECAM) +# author: Thomas Roeblitz (@trz42, University of Bergen) +# +# license: GPLv2 +# +# +set -o pipefail + +# this script is *sourced*, not executed, so can't rely on $0 to determine path to self or script name +# $BASH_SOURCE points to correct path or script name, see also http://mywiki.wooledge.org/BashFAQ/028 +if [ $# -ne 1 ]; then + echo "Usage: source ${BASH_SOURCE} " >&2 + exit 1 +fi + +EESSI_EXTEND_VERSION="${1}-easybuild" + +# make sure that environment variables that we expect to be set are indeed set +if [ -z "${TMPDIR}" ]; then + echo "\$TMPDIR is not set; exiting" >&2 + exit 2 +fi + +# ${EB} is used to specify which 'eb' command should be used; +# can potentially be more than just 'eb', for example when using 'eb --optarch=GENERIC' +if [ -z "${EB}" ]; then + echo "\$EB is not set; exiting" >&2 + exit 2 +fi + +# ${EASYBUILD_INSTALLPATH} points to the installation path and needs to be set +if [ -z "${EASYBUILD_INSTALLPATH}" ]; then + echo "\$EASYBUILD_INSTALLPATH is not set; exiting" >&2 + exit 2 +fi + +# make sure that utility functions are defined (cfr. scripts/utils.sh script in EESSI/software-layer repo) +type check_exit_code +if [ $? -ne 0 ]; then + echo "check_exit_code function is not defined; exiting" >&2 + exit 3 +fi + +echo ">> Checking for EESSI-extend module..." + +ml_av_eessi_extend_out=${TMPDIR}/ml_av_eessi_extend.out +module avail 2>&1 | grep -i EESSI-extend/${EESSI_EXTEND_VERSION} &> ${ml_av_eessi_extend_out} + +if [[ $? -eq 0 ]]; then + echo_green ">> Module for EESSI-extend/${EESSI_EXTEND_VERSION} found!" +else + echo_yellow ">> No module yet for EESSI-extend/${EESSI_EXTEND_VERSION}, installing it..." + + EB_TMPDIR=${TMPDIR}/ebtmp + echo ">> Using temporary installation of EasyBuild (in ${EB_TMPDIR})..." + pip_install_out=${TMPDIR}/pip_install.out + pip3 install --prefix ${EB_TMPDIR} easybuild &> ${pip_install_out} + + # keep track of original $PATH and $PYTHONPATH values, so we can restore them + ORIG_PATH=${PATH} + ORIG_PYTHONPATH=${PYTHONPATH} + + echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..." + export PATH=${EB_TMPDIR}/bin:${PATH} + export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):${PYTHONPATH} + eb_install_out=${TMPDIR}/eb_install.out + ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" + fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" + ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} + check_exit_code $? "${ok_msg}" "${fail_msg}" + + # restore origin $PATH and $PYTHONPATH values, and clean up environment variables that are no longer needed + export PATH=${ORIG_PATH} + export PYTHONPATH=${ORIG_PYTHONPATH} + unset EB_TMPDIR ORIG_PATH ORIG_PYTHONPATH + + module --ignore-cache avail EESSI-extend/${EESSI_EXTEND_VERSION} &> ${ml_av_eessi_extend_out} + if [[ $? -eq 0 ]]; then + echo_green ">> EESSI-extend/${EESSI_EXTEND_VERSION} module installed!" + else + fatal_error "EESSI-extend/${EESSI_EXTEND_VERSION} module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'module avail EESSI-extend' in ${ml_av_eessi_extend_out})" + fi +fi + +echo ">> Loading EESSI-extend/${EESSI_EXTEND_VERSION} module..." +module --ignore-cache load EESSI-extend/${EESSI_EXTEND_VERSION} + +unset EESSI_EXTEND_VERSION From 679d18051080b38204aa400aeb393e2ab5a50de0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 11:54:00 +0100 Subject: [PATCH 1531/1795] use script that loads and installs EESSI-extend --- EESSI-install-software.sh | 4 ++++ scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index b365edd71b..50eacc52bf 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -248,6 +248,10 @@ fi temp_install_storage=${TMPDIR}/temp_install_storage mkdir -p ${temp_install_storage} if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then + # need to ensure that some variables will be available to the script + # TMPDIR, EB, EESSI_VERSION, for EASYBUILD_INSTALLPATH (EESSI_PREFIX, + # EESSI_OS_TYPE, EESSI_SOFTWARE_SUBDIR_OVERRIDE) + export TMPDIR EB EESSI_VERSION EESSI_PREFIX EESSI_OS_TYPE EESSI_SOFTWARE_SUBDIR_OVERRIDE ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \ -t ${temp_install_storage} \ --accept-cuda-eula \ diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index e6343595ad..51d139bec5 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -112,7 +112,14 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do unset EESSI_USER_INSTALL export EESSI_SITE_INSTALL=1 module unload EESSI-extend - module load EESSI-extend/${EESSI_VERSION}-easybuild + + # The EESSI-extend module is being loaded (or installed if it doesn't exist yet). + # The script requires the EESSI_VERSION given as argument, a couple of + # environment variables set (TMPDIR, EB and EASYBUILD_INSTALLPATH) and the + # function check_exit_code defined. + # NOTE, the script exits if those variables/functions are undefined. + export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + source load_eessi_extend_module.sh ${EESSI_VERSION} # Install modules in hidden .modules dir to keep track of what was installed before # (this action is temporary, and we do not call Lmod again within the current shell context, but in EasyBuild From f9243d8f85a616b83c599a01c8a7bf871a89c163 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 12:14:07 +0100 Subject: [PATCH 1532/1795] facilitate a bit debugging --- load_eessi_extend_module.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index bc277c5f0a..f424c5b79b 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -103,4 +103,7 @@ fi echo ">> Loading EESSI-extend/${EESSI_EXTEND_VERSION} module..." module --ignore-cache load EESSI-extend/${EESSI_EXTEND_VERSION} +echo ">> Determining how to load EESSI-extend/${EESSI_EXTEND_VERSION} module..." +module --ignore-cache spider EESSI-extend/${EESSI_EXTEND_VERSION} + unset EESSI_EXTEND_VERSION From 68a882f707e70ed74ac7fa5235550461604caf63 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 12:23:09 +0100 Subject: [PATCH 1533/1795] fix module command arg typo --- load_eessi_extend_module.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index f424c5b79b..366da66a5f 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -92,7 +92,7 @@ else export PYTHONPATH=${ORIG_PYTHONPATH} unset EB_TMPDIR ORIG_PATH ORIG_PYTHONPATH - module --ignore-cache avail EESSI-extend/${EESSI_EXTEND_VERSION} &> ${ml_av_eessi_extend_out} + module --ignore_cache avail EESSI-extend/${EESSI_EXTEND_VERSION} &> ${ml_av_eessi_extend_out} if [[ $? -eq 0 ]]; then echo_green ">> EESSI-extend/${EESSI_EXTEND_VERSION} module installed!" else @@ -101,9 +101,6 @@ else fi echo ">> Loading EESSI-extend/${EESSI_EXTEND_VERSION} module..." -module --ignore-cache load EESSI-extend/${EESSI_EXTEND_VERSION} - -echo ">> Determining how to load EESSI-extend/${EESSI_EXTEND_VERSION} module..." -module --ignore-cache spider EESSI-extend/${EESSI_EXTEND_VERSION} +module --ignore_cache load EESSI-extend/${EESSI_EXTEND_VERSION} unset EESSI_EXTEND_VERSION From 37c1e9c5f29f6d9696e087629ea5572f4da82649 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 12:33:26 +0100 Subject: [PATCH 1534/1795] run module avail with --ignore_cache --- load_eessi_extend_module.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index 366da66a5f..88ddcf7f46 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -62,7 +62,9 @@ fi echo ">> Checking for EESSI-extend module..." ml_av_eessi_extend_out=${TMPDIR}/ml_av_eessi_extend.out -module avail 2>&1 | grep -i EESSI-extend/${EESSI_EXTEND_VERSION} &> ${ml_av_eessi_extend_out} +# need to use --ignore_cache to avoid the case that the module was removed (to be +# rebuilt) but it is still in the cache +module --ignore_cache avail 2>&1 | grep -i EESSI-extend/${EESSI_EXTEND_VERSION} &> ${ml_av_eessi_extend_out} if [[ $? -eq 0 ]]; then echo_green ">> Module for EESSI-extend/${EESSI_EXTEND_VERSION} found!" From a83cde888ef1d341c96adce6563ce1c4f2f8d14f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 14:10:05 +0100 Subject: [PATCH 1535/1795] use previous overlay-upper dir(s) as left-most lowerdirs --- eessi_container.sh | 49 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index fc97f9877c..b6adc60503 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -746,9 +746,21 @@ do # to be able to see the contents of the read-write session we have to mount # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository - echo "While processing '${cvmfs_repo_name}' to be mounted 'read-only' we detected an overlay-upper" - echo " directory (${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper) likely from a previous" - echo " session. Will use it as left-most directory in 'lowerdir' argument for fuse-overlayfs." + echo "While processing '${cvmfs_repo_name}' to be mounted 'read-only'" + echo " we detected one or more overlay-upper* directories" + echo " (${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper*)" + echo " likely originating from a previous session. Will use then as" + echo " left-most directory in 'lowerdir' argument for fuse-overlayfs." + + lowerdirs=/cvmfs_ro/${cvmfs_repo_name} + # check if there are more overlay-upper directories, e.g., with three digit suffix + for dir in $(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n); do + lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper-${dir}:${lowerdirs} + done + # finally add most recent overlay-upper to lowerdirs + lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:${lowerdirs} + [[ ${VERBOSE} -eq 1 ]] && ls ${EESSI_TMPDIR}/${cvmfs_repo_name} + [[ ${VERBOSE} -eq 1 ]] && echo ${lowerdirs} # make the target CernVM-FS repository available under /cvmfs_ro export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" @@ -757,12 +769,7 @@ do # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs EESSI_READONLY_OVERLAY="container:fuse-overlayfs" - # The contents of the previous session are available under - # ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}. - # Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper - # the left-most directory given for the lowerdir argument is put on top, - # and with no upperdir=... the whole overlayfs is made available read-only - EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}" + EESSI_READONLY_OVERLAY+=" -o lowerdir=${lowerdirs}" EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" export EESSI_READONLY_OVERLAY @@ -778,7 +785,27 @@ do export EESSI_FUSE_MOUNTS fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then - # use repo-specific overlay directories + # use repo-specific overlay directories; if there is already an + # overlay-upper (e.g., from a previous run) move it to overlay-upper-SEQ + # and create a new one; all overlay-upper-SEQs must be added to lowerdir + # starting with the lowest number first and preprending it to the lowerdir + # setting + lowerdirs=/cvmfs_ro/${cvmfs_repo_name} + if [ -d ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ]; then + # determine next sequence number + last_seq_num=$(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n | tail -n 1 | sed -e 's/^0*//') + if [ -n ${last_seq_num} ]; then + last_seq_num=0 + fi + next_seq_num=$(($last_seq_num + 1)) + next_ovl_upper=$(printf "overlay-upper-%03d" ${next_seq_num}) + mv ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ${EESSI_TMPDIR}/${cvmfs_repo_name}/${next_ovl_upper} + for dir in $(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n); do + lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper-${dir}:${lowerdirs} + done + [[ ${VERBOSE} -eq 1 ]] && ls ${EESSI_TMPDIR}/${cvmfs_repo_name} + [[ ${VERBOSE} -eq 1 ]] && echo ${lowerdirs} + fi mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" @@ -789,7 +816,7 @@ do EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" - EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=${lowerdirs}" EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" From 86bdd5a16d4a4a1c7393c08416bc0aa20f9ba298 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 13 Nov 2024 17:50:20 +0000 Subject: [PATCH 1536/1795] {2023.06}[foss/2023b] Mustache v1.3.3 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 77287f0daf..cc2c109eff 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -8,3 +8,7 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyblocks/pull/3496 include-easyblocks-from-commit: 60633b0acfd41a0732992d9e16800dae71a056eb - Cython-3.0.10-GCCcore-13.2.0.eb + - Mustache-1.3.3-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/21783 + from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 From 16dce19146c8adf696aa8037dc691a2944d072ab Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 21:19:14 +0100 Subject: [PATCH 1537/1795] use alternative approach to remove software to be rebuilt --- EESSI-determine-rebuilds.sh | 123 ++++++++++++++++++++++++++++++++++++ EESSI-remove-software.sh | 13 ++-- bot/build.sh | 50 ++++++++++++++- eessi_container.sh | 54 ++++++++++------ 4 files changed, 216 insertions(+), 24 deletions(-) create mode 100755 EESSI-determine-rebuilds.sh diff --git a/EESSI-determine-rebuilds.sh b/EESSI-determine-rebuilds.sh new file mode 100755 index 0000000000..4f4d5ab713 --- /dev/null +++ b/EESSI-determine-rebuilds.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# +# Script to determine which parts of the EESSI software stack (version set through init/eessi_defaults) +# have to be rebuilt + +# see example parsing of command line arguments at +# https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " -g | --generic - instructs script to build for generic architecture target" + echo " -h | --help - display this usage information" +} + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -g|--generic) + DETECTION_PARAMETERS="--generic" + shift + ;; + -h|--help) + display_help # Call your function + # no shifting needed here, we're done. + exit 0 + ;; + -*|--*) + echo "Error: Unknown option: $1" >&2 + exit 1 + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +TOPDIR=$(dirname $(realpath $0)) + +export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX) + +source $TOPDIR/scripts/utils.sh + +echo ">> Determining software subdirectory to use for current build host..." +if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" +else + echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" +fi + +echo ">> Setting up environment..." + +source $TOPDIR/init/bash + +if [ -d $EESSI_CVMFS_REPO ]; then + echo_green "$EESSI_CVMFS_REPO available, OK!" +else + fatal_error "$EESSI_CVMFS_REPO is not available!" +fi + +if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then + fatal_error "Failed to determine software subdirectory?!" +elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" +else + echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" +fi + +echo ">> Configuring EasyBuild..." +EB="eb" +source $TOPDIR/configure_easybuild + +echo ">> Setting up \$MODULEPATH..." +# make sure no modules are loaded +module --force purge +# ignore current $MODULEPATH entirely +module unuse $MODULEPATH +module use $EASYBUILD_INSTALLPATH/modules/all +if [[ -z ${MODULEPATH} ]]; then + fatal_error "Failed to set up \$MODULEPATH?!" +else + echo_green ">> MODULEPATH set up: ${MODULEPATH}" +fi + +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# if this script is run as root, use PR patch file to determine if software needs to be removed first +changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") +if [ -z ${changed_easystacks_rebuilds} ]; then + echo "No software needs to be removed." +else + for easystack_file in ${changed_easystacks_rebuilds}; do + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + + if [ -f ${easystack_file} ]; then + echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." + # we need to remove existing installation directories first, + # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) + # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) + rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + for app in ${rebuild_apps}; do + app_dir=${EASYBUILD_INSTALLPATH}/software/${app} + app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua + echo_yellow "Removing ${app_dir} and ${app_module}..." + find ${app_dir} -type d | sed -e 's/^/REMOVE_DIRECTORY /' + find ${app_dir} -type f | sed -e 's/^/REMOVE_FILE /' + echo "REMOVE_MODULE ${app_module}" + done + else + fatal_error "Easystack file ${easystack_file} not found!" + fi + done +fi diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 98576efcb0..c0818db393 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -114,11 +114,14 @@ if [ $EUID -eq 0 ]; then source ${TOPDIR}/load_easybuild_module.sh ${eb_version} if [ -f ${easystack_file} ]; then - echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." + echo_green "Software rebuild(s) requested in ${easystack_file}, so" + echo_green " determining which existing installation have to be removed (assuming contents" + echo_green " have been made writable/deletable)..." # we need to remove existing installation directories first, # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + # rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') for app in ${rebuild_apps}; do # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all: app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://') @@ -126,9 +129,11 @@ if [ $EUID -eq 0 ]; then app_installprefix=$(dirname $(dirname ${app_modulepath})) app_dir=${app_installprefix}/software/${app} app_module=${app_installprefix}/modules/all/${app}.lua + # app_dir=${EASYBUILD_INSTALLPATH}/software/${app} + # app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - rm -rf ${app_dir} - rm -rf ${app_module} + rm -rdfv ${app_dir} + rm -rdfv ${app_module} done else fatal_error "Easystack file ${easystack_file} not found!" diff --git a/bot/build.sh b/bot/build.sh index 3fd343e96f..718fceafc0 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -200,6 +200,49 @@ changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else + # determine which software packages (and modules) have to be removed + TARBALL_TMP_DETERMINE_STEP_DIR=${PREVIOUS_TMP_DIR}/determine_step + mkdir -p ${TARBALL_TMP_DETERMINE_STEP_DIR} + + # prepare arguments to eessi_container.sh specific to determine step + declare -a DETERMINE_STEP_ARGS=() + DETERMINE_STEP_ARGS+=("--save" "${TARBALL_TMP_DETERMINE_STEP_DIR}") + DETERMINE_STEP_ARGS+=("--storage" "${STORAGE}") + + # create tmp file for output of determine step + determine_outerr=$(mktemp determine.outerr.XXXX) + + echo "Executing command to determine software to be removed:" + echo "${software_layer_dir}/eessi_container.sh ${COMMON_ARGS[@]} ${DETERMINE_STEP_ARGS[@]}" + echo " -- ${software_layer_dir}/EESSI-determine-rebuilds.sh \"${DETERMINE_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${determine_outerr}" + ${software_layer_dir}/eessi_container.sh "${COMMON_ARGS[@]}" "${DETERMINE_STEP_ARGS[@]}" \ + -- ${software_layer_dir}/EESSI-determine-rebuilds.sh "${DETERMINE_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${determine_outerr} + + # process output file + # for each line containing 'REMOVE_DIRECTORY some_path' + # create a new directory ${STORAGE}/lower_dirs/some_path_stripped + # where the prefix /cvmfs/repo_name is removed from some_path + # set permission of the directory to u+rwx + # for each line containing 'REMOVE_FILE some_file_path' + # touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped + # where the prefix /cvmfs/repo_name is removed from some_file_path + # set permission of the file to u+rw + + LOWER_DIRS="${STORAGE}/lower_dirs" + mkdir -p "${LOWER_DIRS}" + + grep ^REMOVE_DIRECTORY ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_dirs + cat ${determine_outerr}.rm_dirs | while read remove_dir; do + mkdir -p ${STORAGE}/lower_dirs/${remove_dir} + chmod u+rwx ${STORAGE}/lower_dirs/${remove_dir} + done + + grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files + cat ${determine_outerr}.rm_files | while read remove_file; do + touch ${STORAGE}/lower_dirs/${remove_file} + chmod u+rw ${STORAGE}/lower_dirs/${remove_file} + done + # prepare directory to store tarball of tmp for removal and build steps TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR} @@ -208,9 +251,14 @@ else declare -a REMOVAL_STEP_ARGS=() REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}") REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") + # add fakeroot option in order to be able to remove software, see: # https://github.com/EESSI/software-layer/issues/312 - REMOVAL_STEP_ARGS+=("--fakeroot") + # REMOVAL_STEP_ARGS+=("--fakeroot") + + if [[ ! -z ${LOWER_DIRS} ]]; then + REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}") + fi # create tmp file for output of removal step removal_outerr=$(mktemp remove.outerr.XXXX) diff --git a/eessi_container.sh b/eessi_container.sh index b6adc60503..661d0f63f1 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -89,6 +89,11 @@ display_help() { echo " -n | --nvidia MODE - configure the container to work with NVIDIA GPUs," echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" + echo " -o | --lower-dirs DIRS - list of ':' separated directories that are used" + echo " in front of the default lower dir (CVMFS repo);" + echo " fuse-overlayfs will merge all lower directories;" + echo " the option can be used to make certain directories" + echo " in the CVMFS repo writable [default: none]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use; can be given multiple times;" echo " CFG may include a suffix ',access={ro,rw}' to" @@ -125,6 +130,7 @@ FAKEROOT=0 VERBOSE=0 STORAGE= LIST_REPOS=0 +LOWER_DIRS= MODE="shell" SETUP_NVIDIA=0 REPOSITORIES=() @@ -182,6 +188,10 @@ while [[ $# -gt 0 ]]; do NVIDIA_MODE="$2" shift 2 ;; + -o|--lower-dirs) + LOWER_DIRS="$2" + shift 2 + ;; -r|--repository) REPOSITORIES+=("$2") shift 2 @@ -753,10 +763,10 @@ do echo " left-most directory in 'lowerdir' argument for fuse-overlayfs." lowerdirs=/cvmfs_ro/${cvmfs_repo_name} - # check if there are more overlay-upper directories, e.g., with three digit suffix - for dir in $(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n); do - lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper-${dir}:${lowerdirs} - done + # # check if there are more overlay-upper directories, e.g., with three digit suffix + # for dir in $(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n); do + # lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper-${dir}:${lowerdirs} + # done # finally add most recent overlay-upper to lowerdirs lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:${lowerdirs} [[ ${VERBOSE} -eq 1 ]] && ls ${EESSI_TMPDIR}/${cvmfs_repo_name} @@ -791,21 +801,21 @@ do # starting with the lowest number first and preprending it to the lowerdir # setting lowerdirs=/cvmfs_ro/${cvmfs_repo_name} - if [ -d ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ]; then - # determine next sequence number - last_seq_num=$(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n | tail -n 1 | sed -e 's/^0*//') - if [ -n ${last_seq_num} ]; then - last_seq_num=0 - fi - next_seq_num=$(($last_seq_num + 1)) - next_ovl_upper=$(printf "overlay-upper-%03d" ${next_seq_num}) - mv ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ${EESSI_TMPDIR}/${cvmfs_repo_name}/${next_ovl_upper} - for dir in $(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n); do - lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper-${dir}:${lowerdirs} - done - [[ ${VERBOSE} -eq 1 ]] && ls ${EESSI_TMPDIR}/${cvmfs_repo_name} - [[ ${VERBOSE} -eq 1 ]] && echo ${lowerdirs} - fi + # if [ -d ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ]; then + # # determine next sequence number + # last_seq_num=$(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n | tail -n 1 | sed -e 's/^0*//') + # if [ -n ${last_seq_num} ]; then + # last_seq_num=0 + # fi + # next_seq_num=$(($last_seq_num + 1)) + # next_ovl_upper=$(printf "overlay-upper-%03d" ${next_seq_num}) + # mv ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ${EESSI_TMPDIR}/${cvmfs_repo_name}/${next_ovl_upper} + # for dir in $(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n); do + # lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper-${dir}:${lowerdirs} + # done + # [[ ${VERBOSE} -eq 1 ]] && ls ${EESSI_TMPDIR}/${cvmfs_repo_name} + # [[ ${VERBOSE} -eq 1 ]] && echo ${lowerdirs} + # fi mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" @@ -816,6 +826,12 @@ do EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + if [[ ! -z ${LOWER_DIRS} ]]; then + # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as + # separator while the lowerdir overlayfs option uses ':' + export BIND_PATHS="${BIND_PATHS},${LOWER_DIRS/:/,}" + lowerdirs=${LOWER_DIRS}:${lowerdirs}" + fi EESSI_WRITABLE_OVERLAY+=" -o lowerdir=${lowerdirs}" EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" From bcb0b654bef3bbc1d5f8008a773054b15ac32292 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 21:38:57 +0100 Subject: [PATCH 1538/1795] fix syntax error --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 661d0f63f1..069da99d14 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -830,7 +830,7 @@ do # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as # separator while the lowerdir overlayfs option uses ':' export BIND_PATHS="${BIND_PATHS},${LOWER_DIRS/:/,}" - lowerdirs=${LOWER_DIRS}:${lowerdirs}" + lowerdirs=${LOWER_DIRS}:${lowerdirs} fi EESSI_WRITABLE_OVERLAY+=" -o lowerdir=${lowerdirs}" EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" From 0fb455d47477415fa34dff711c04abf5e32655db Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 21:53:38 +0100 Subject: [PATCH 1539/1795] remove script is not run by root anymore --- EESSI-remove-software.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index c0818db393..582ed61a7f 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -101,7 +101,7 @@ fi pr_diff=$(ls [0-9]*.diff | head -1) # if this script is run as root, use PR patch file to determine if software needs to be removed first -if [ $EUID -eq 0 ]; then +if [ $EUID -ne 0 ]; then changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") if [ -z ${changed_easystacks_rebuilds} ]; then echo "No software needs to be removed." @@ -141,5 +141,6 @@ if [ $EUID -eq 0 ]; then done fi else - fatal_error "This script can only be run by root!" + # fatal_error "This script can only be run by root!" + fatal_error "This script must not be run by root!" fi From 75e9c35720a83c722eb47c315d5456e2af151bfd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 22:06:13 +0100 Subject: [PATCH 1540/1795] add debug info --- load_eessi_extend_module.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index 88ddcf7f46..a712ab31af 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -81,6 +81,10 @@ else ORIG_PYTHONPATH=${PYTHONPATH} echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..." + ls -l ${EASYBUILD_INSTALLPATH} + ls -lR ${EASYBUILD_INSTALLPATH}/software/EESSI-extend + whoami + export PATH=${EB_TMPDIR}/bin:${PATH} export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):${PYTHONPATH} eb_install_out=${TMPDIR}/eb_install.out From 5ba88d383418be6cb49c808f7efa563456766c04 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 22:14:17 +0100 Subject: [PATCH 1541/1795] set 777 perms --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 718fceafc0..223891117d 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -234,13 +234,13 @@ else grep ^REMOVE_DIRECTORY ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_dirs cat ${determine_outerr}.rm_dirs | while read remove_dir; do mkdir -p ${STORAGE}/lower_dirs/${remove_dir} - chmod u+rwx ${STORAGE}/lower_dirs/${remove_dir} + chmod ugo+rwx ${STORAGE}/lower_dirs/${remove_dir} done grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files cat ${determine_outerr}.rm_files | while read remove_file; do touch ${STORAGE}/lower_dirs/${remove_file} - chmod u+rw ${STORAGE}/lower_dirs/${remove_file} + chmod ugo+rw ${STORAGE}/lower_dirs/${remove_file} done # prepare directory to store tarball of tmp for removal and build steps From 47fe73c87b8949c781c121095918148848bf5bbc Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 22:15:56 +0100 Subject: [PATCH 1542/1795] add more debug info --- load_eessi_extend_module.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index a712ab31af..fa3708f269 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -81,8 +81,8 @@ else ORIG_PYTHONPATH=${PYTHONPATH} echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..." - ls -l ${EASYBUILD_INSTALLPATH} - ls -lR ${EASYBUILD_INSTALLPATH}/software/EESSI-extend + ls -lisa ${EASYBUILD_INSTALLPATH} + ls -lisaR ${EASYBUILD_INSTALLPATH}/software/EESSI-extend whoami export PATH=${EB_TMPDIR}/bin:${PATH} From acf987c936e5396a6324656ca513ae72d0ae169d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 22:22:11 +0100 Subject: [PATCH 1543/1795] add even more debug info --- load_eessi_extend_module.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index fa3708f269..1ab50daba0 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -91,7 +91,9 @@ else ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} - check_exit_code $? "${ok_msg}" "${fail_msg}" + ec=$? + ls -lisaR ${EASYBUILD_INSTALLPATH}/software/EESSI-extend + check_exit_code $ec "${ok_msg}" "${fail_msg}" # restore origin $PATH and $PYTHONPATH values, and clean up environment variables that are no longer needed export PATH=${ORIG_PATH} From 32e6d69a0a6b87979f06e960f40f6a2602afe288 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 22:29:59 +0100 Subject: [PATCH 1544/1795] add lower dirs to build step --- bot/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 223891117d..6bf7827d66 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -296,6 +296,10 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi +if [[ ! -z ${LOWER_DIRS} ]]; then + BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}") +fi + # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) From afa1356c5cd55544efffae89532833711e056741 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 22:45:14 +0100 Subject: [PATCH 1545/1795] limit perm to 700 --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 6bf7827d66..132770fd01 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -234,13 +234,13 @@ else grep ^REMOVE_DIRECTORY ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_dirs cat ${determine_outerr}.rm_dirs | while read remove_dir; do mkdir -p ${STORAGE}/lower_dirs/${remove_dir} - chmod ugo+rwx ${STORAGE}/lower_dirs/${remove_dir} + chmod u+rwx ${STORAGE}/lower_dirs/${remove_dir} done grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files cat ${determine_outerr}.rm_files | while read remove_file; do touch ${STORAGE}/lower_dirs/${remove_file} - chmod ugo+rw ${STORAGE}/lower_dirs/${remove_file} + chmod u+rw ${STORAGE}/lower_dirs/${remove_file} done # prepare directory to store tarball of tmp for removal and build steps From 9eebad7be3b3cbc1bd3b14a8614a13cc79c25f82 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 22:55:27 +0100 Subject: [PATCH 1546/1795] source configure_easybuild and run in subshell --- load_eessi_extend_module.sh | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index 1ab50daba0..9d13f313f3 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -80,20 +80,20 @@ else ORIG_PATH=${PATH} ORIG_PYTHONPATH=${PYTHONPATH} - echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..." - ls -lisa ${EASYBUILD_INSTALLPATH} - ls -lisaR ${EASYBUILD_INSTALLPATH}/software/EESSI-extend - whoami - - export PATH=${EB_TMPDIR}/bin:${PATH} - export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):${PYTHONPATH} - eb_install_out=${TMPDIR}/eb_install.out - ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" - fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" - ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} - ec=$? - ls -lisaR ${EASYBUILD_INSTALLPATH}/software/EESSI-extend - check_exit_code $ec "${ok_msg}" "${fail_msg}" + # source configure_easybuild to use correct eb settings + ( + EESSI_MAIN_DIR=$(dirname $(readlink -f $BASH_SOURCE)) + source ${EESSI_MAIN_DIR}/configure_easybuild + + echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..." + export PATH=${EB_TMPDIR}/bin:${PATH} + export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):${PYTHONPATH} + eb_install_out=${TMPDIR}/eb_install.out + ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" + fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" + ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} + check_exit_code $? "${ok_msg}" "${fail_msg}" + ) # restore origin $PATH and $PYTHONPATH values, and clean up environment variables that are no longer needed export PATH=${ORIG_PATH} From 2af57ee8678a1b291f830a64e7e8c7e733aac5c2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 13 Nov 2024 23:11:48 +0100 Subject: [PATCH 1547/1795] set 770 perms --- bot/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 132770fd01..6a382b2fa4 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -222,11 +222,11 @@ else # for each line containing 'REMOVE_DIRECTORY some_path' # create a new directory ${STORAGE}/lower_dirs/some_path_stripped # where the prefix /cvmfs/repo_name is removed from some_path - # set permission of the directory to u+rwx + # set permission of the directory to ug+rwx # for each line containing 'REMOVE_FILE some_file_path' # touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped # where the prefix /cvmfs/repo_name is removed from some_file_path - # set permission of the file to u+rw + # set permission of the file to ug+rw LOWER_DIRS="${STORAGE}/lower_dirs" mkdir -p "${LOWER_DIRS}" @@ -234,13 +234,13 @@ else grep ^REMOVE_DIRECTORY ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_dirs cat ${determine_outerr}.rm_dirs | while read remove_dir; do mkdir -p ${STORAGE}/lower_dirs/${remove_dir} - chmod u+rwx ${STORAGE}/lower_dirs/${remove_dir} + chmod ug+rwx ${STORAGE}/lower_dirs/${remove_dir} done grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files cat ${determine_outerr}.rm_files | while read remove_file; do touch ${STORAGE}/lower_dirs/${remove_file} - chmod u+rw ${STORAGE}/lower_dirs/${remove_file} + chmod ug+rw ${STORAGE}/lower_dirs/${remove_file} done # prepare directory to store tarball of tmp for removal and build steps From d677c88765b2400faa6bd1cf4867bf6d1a3f13e2 Mon Sep 17 00:00:00 2001 From: TopRichard <121792457+TopRichard@users.noreply.github.com> Date: Thu, 14 Nov 2024 08:10:53 +0100 Subject: [PATCH 1548/1795] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index cc2c109eff..03c9ec8f98 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -10,5 +10,5 @@ easyconfigs: - Cython-3.0.10-GCCcore-13.2.0.eb - Mustache-1.3.3-foss-2023b.eb: options: - # see https://github.com/easybuilders/easybuild-easyblocks/pull/21783 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21783 from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 From 2184b303be49c5f0911ef65c70636a573a68b01c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 09:25:44 +0100 Subject: [PATCH 1549/1795] alternative approach to remove files/directories; one-by-one in depth-first order --- EESSI-remove-software.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 582ed61a7f..10f57fe678 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -132,8 +132,21 @@ if [ $EUID -ne 0 ]; then # app_dir=${EASYBUILD_INSTALLPATH}/software/${app} # app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - rm -rdfv ${app_dir} - rm -rdfv ${app_module} + # rm -rdfv ${app_dir} + # rm -rdfv ${app_module} + # 1st remove files in depth-first order + for filepath in $(find ${app_dir} -depth -type f); do + echo " removing file ${filepath}" + rm -fv ${filepath} + done + # 2nd remove directories in depth-first order + for dirpath in $(find ${app_dir} -depth -type d); do + echo " removing directory ${dirpath}" + rmdir -v ${dirpath} + done + # 3rd remove module file + echo " removing module file ${app_module}" + rm -fv ${app_module} done else fatal_error "Easystack file ${easystack_file} not found!" From 2438c476acc05930de540e3a89f38f8ffd7dc2c4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 10:44:32 +0100 Subject: [PATCH 1550/1795] pre-package hook for EESSI-extend that stats all files/dirs in installation directory --- eb_hooks.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 03642656ea..20f0281492 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -962,6 +962,29 @@ def inject_gpu_property(ec): return ec +def pre_package_hook(self, *args, **kwargs): + """Main pre-package hook: trigger custom functions based on software name.""" + if self.name in PRE_PACKAGE_HOOKS: + PRE_PACKAGE_HOOKS[self.name](self, *args, **kwargs) + + +def pre_package_eessi_extend(self, *args, **kwargs): + """ + Pre-package hook for EESSI-extend: ls/stat all files/directories to work around + 'permission denied' issue when package got removed (and this hook is run when + the package is being rebuilt) + """ + if self.name == 'EESSI-extend': + dir_tree = [] + for root, _, files in os.walk(self.installdir): + dir_tree.append(root) + for f in files: + tree.append(os.path.join(root, f)) + for entry in dir_tree: + print(entry) + os.stat(entry) + + PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, @@ -1020,4 +1043,7 @@ def inject_gpu_property(ec): POST_POSTPROC_HOOKS = { 'CUDA': post_postproc_cuda, 'cuDNN': post_postproc_cudnn, + +PRE_PACKAGE_HOOKS = { + 'EESSI-extend': pre_package_eessi_extend, } From 29004cee3f49e3c38a6953842fb6f71a465e13ca Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 11:03:48 +0100 Subject: [PATCH 1551/1795] fix syntax error --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index 20f0281492..ea722b8c04 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -1043,6 +1043,7 @@ def pre_package_eessi_extend(self, *args, **kwargs): POST_POSTPROC_HOOKS = { 'CUDA': post_postproc_cuda, 'cuDNN': post_postproc_cudnn, +} PRE_PACKAGE_HOOKS = { 'EESSI-extend': pre_package_eessi_extend, From cf076be39d8d4f4320f7397b6cf8f3c155fcb407 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 11:09:43 +0100 Subject: [PATCH 1552/1795] fix another code error --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index ea722b8c04..62c5ee3eb6 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -979,7 +979,7 @@ def pre_package_eessi_extend(self, *args, **kwargs): for root, _, files in os.walk(self.installdir): dir_tree.append(root) for f in files: - tree.append(os.path.join(root, f)) + dir_tree.append(os.path.join(root, f)) for entry in dir_tree: print(entry) os.stat(entry) From 60b3f31d7ff411adff4e3d7aa639d0d7ca54e5f8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 11:26:52 +0100 Subject: [PATCH 1553/1795] print file permissions --- eb_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 62c5ee3eb6..9601aa1264 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -3,6 +3,7 @@ import glob import os import re +import stat import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess @@ -982,7 +983,7 @@ def pre_package_eessi_extend(self, *args, **kwargs): dir_tree.append(os.path.join(root, f)) for entry in dir_tree: print(entry) - os.stat(entry) + print(stat.S_IMODE(os.lstat(entry).st_mode)) PARSE_HOOKS = { From ace7d7e81571dd3130120958e55a129ffd8281c0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 11:33:52 +0100 Subject: [PATCH 1554/1795] convert permissions to octal value --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9601aa1264..b4a9affddf 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -983,7 +983,7 @@ def pre_package_eessi_extend(self, *args, **kwargs): dir_tree.append(os.path.join(root, f)) for entry in dir_tree: print(entry) - print(stat.S_IMODE(os.lstat(entry).st_mode)) + print(oct(stat.S_IMODE(os.lstat(entry).st_mode))) PARSE_HOOKS = { From e969bae487d8147c910b4996a6836ffb41000f95 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Thu, 14 Nov 2024 10:37:01 +0000 Subject: [PATCH 1555/1795] clean `${tmpdir}` after iterations in `install_cuda_and_libraries.sh` --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index e6343595ad..e1ca42fc9a 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -239,9 +239,9 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do echo_green "all installations at ${EESSI_SITE_SOFTWARE_PATH}/software/... succeeded!" fi - # clean up tmpdir - rm -rf "${tmpdir}" - # Restore MODULEPATH for next loop iteration MODULEPATH=${SAVE_MODULEPATH} done + +# clean up tmpdir +rm -rf "${tmpdir}" From f760fd25b088e00a010f73b9b92f28cae5d62cb3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 11:55:14 +0100 Subject: [PATCH 1556/1795] show file creation time too --- eb_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b4a9affddf..5877d7c3b8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -4,6 +4,7 @@ import os import re import stat +import time import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess @@ -983,7 +984,8 @@ def pre_package_eessi_extend(self, *args, **kwargs): dir_tree.append(os.path.join(root, f)) for entry in dir_tree: print(entry) - print(oct(stat.S_IMODE(os.lstat(entry).st_mode))) + statinfo = os.lstat(entry) + print(f"file: {entry}\nperms: {oct(stat.S_IMODE(statinfo.st_mode))}\ncreated: {time.ctime(statinfo.st_ctime)}\n") PARSE_HOOKS = { From 731514599738160303d548053929f80ee33ab490 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 14:33:03 +0100 Subject: [PATCH 1557/1795] disable hook, use recursive rm and do ls on specific directory --- EESSI-remove-software.sh | 34 +++++++++++++++++++--------------- eb_hooks.py | 2 +- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 10f57fe678..29a47c0686 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -132,21 +132,25 @@ if [ $EUID -ne 0 ]; then # app_dir=${EASYBUILD_INSTALLPATH}/software/${app} # app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - # rm -rdfv ${app_dir} - # rm -rdfv ${app_module} - # 1st remove files in depth-first order - for filepath in $(find ${app_dir} -depth -type f); do - echo " removing file ${filepath}" - rm -fv ${filepath} - done - # 2nd remove directories in depth-first order - for dirpath in $(find ${app_dir} -depth -type d); do - echo " removing directory ${dirpath}" - rmdir -v ${dirpath} - done - # 3rd remove module file - echo " removing module file ${app_module}" - rm -fv ${app_module} + # suggestion: use the recursive rm's and ls a specific + # directory only (${app_dir}/easybuild) + rm -rdfv ${app_dir} + rm -rdfv ${app_module} + ls ${app_dir}/easybuild || true + + ## 1st remove files in depth-first order + #for filepath in $(find ${app_dir} -depth -type f); do + # echo " removing file ${filepath}" + # rm -fv ${filepath} + #done + ## 2nd remove directories in depth-first order + #for dirpath in $(find ${app_dir} -depth -type d); do + # echo " removing directory ${dirpath}" + # rmdir -v ${dirpath} + #done + ## 3rd remove module file + #echo " removing module file ${app_module}" + #rm -fv ${app_module} done else fatal_error "Easystack file ${easystack_file} not found!" diff --git a/eb_hooks.py b/eb_hooks.py index 5877d7c3b8..fd88d8399a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -1049,5 +1049,5 @@ def pre_package_eessi_extend(self, *args, **kwargs): } PRE_PACKAGE_HOOKS = { - 'EESSI-extend': pre_package_eessi_extend, + # 'EESSI-extend': pre_package_eessi_extend, } From ea6a842479893ef2bab7755b09906f6b30ebb7b4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 14:52:22 +0100 Subject: [PATCH 1558/1795] recreate 'installdir/easybuild' and use keeppreviousinstall option --- EESSI-remove-software.sh | 4 +++- .../2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml | 4 +++- load_eessi_extend_module.sh | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 29a47c0686..2e89a9fceb 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -136,7 +136,9 @@ if [ $EUID -ne 0 ]; then # directory only (${app_dir}/easybuild) rm -rdfv ${app_dir} rm -rdfv ${app_module} - ls ${app_dir}/easybuild || true + # ls didn't change the result (permission denied) + # ls ${app_dir}/easybuild || true + mkdir -p ${app_dir}/easybuild ## 1st remove files in depth-first order #for filepath in $(find ${app_dir} -depth -type f); do diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml index e4c658784f..86ec70d6ec 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml @@ -3,4 +3,6 @@ # EESSI-extend should adjust EASYBUILD_INSTALLPATH and set # EASYBUILD_CUDA_COMPUTE_CAPABILITIES easyconfigs: - - EESSI-extend-2023.06-easybuild.eb + - EESSI-extend-2023.06-easybuild.eb: + options: + keeppreviousinstall: True diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index 9d13f313f3..257570bc50 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -91,7 +91,7 @@ else eb_install_out=${TMPDIR}/eb_install.out ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" - ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} + ${EB} --keeppreviousinstall True "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" ) From e5d3692a9530bc098ec126823123051850913715 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 14 Nov 2024 15:01:28 +0100 Subject: [PATCH 1559/1795] remove using keeppreviousinstall --- .../2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml | 4 +--- load_eessi_extend_module.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml index 86ec70d6ec..e4c658784f 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml @@ -3,6 +3,4 @@ # EESSI-extend should adjust EASYBUILD_INSTALLPATH and set # EASYBUILD_CUDA_COMPUTE_CAPABILITIES easyconfigs: - - EESSI-extend-2023.06-easybuild.eb: - options: - keeppreviousinstall: True + - EESSI-extend-2023.06-easybuild.eb diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index 257570bc50..9d13f313f3 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -91,7 +91,7 @@ else eb_install_out=${TMPDIR}/eb_install.out ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" - ${EB} --keeppreviousinstall True "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} + ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" ) From 0fe18c28878b42043e715e859461db827c93df03 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 10:07:34 +0100 Subject: [PATCH 1560/1795] show contents and recreate specific/problematic directory --- EESSI-remove-software.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 2e89a9fceb..8b882b38a9 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -136,9 +136,14 @@ if [ $EUID -ne 0 ]; then # directory only (${app_dir}/easybuild) rm -rdfv ${app_dir} rm -rdfv ${app_module} + echo_yellow "Contents of ${app_dir}/easybuild (should not exist)" + ls -l ${app_dir}/easybuild || true # ls didn't change the result (permission denied) # ls ${app_dir}/easybuild || true + # 2nd idea: recreate some directory mkdir -p ${app_dir}/easybuild + echo_yellow "Contents of ${app_dir}/easybuild after it got recreated with 'mkdir -p' (should be empty)" + ls -l ${app_dir}/easybuild || true ## 1st remove files in depth-first order #for filepath in $(find ${app_dir} -depth -type f); do From ddfec84cc9b8333c54480c56f43388369d551fef Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 10:17:24 +0100 Subject: [PATCH 1561/1795] only make directories and module file writable --- bot/build.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 6a382b2fa4..7b4414983d 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -223,7 +223,11 @@ else # create a new directory ${STORAGE}/lower_dirs/some_path_stripped # where the prefix /cvmfs/repo_name is removed from some_path # set permission of the directory to ug+rwx - # for each line containing 'REMOVE_FILE some_file_path' + # SKIP for each line containing 'REMOVE_FILE some_file_path' + # SKIP touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped + # SKIP where the prefix /cvmfs/repo_name is removed from some_file_path + # SKIP set permission of the file to ug+rw + # for each line containing 'REMOVE_MODULE some_file_path' # touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped # where the prefix /cvmfs/repo_name is removed from some_file_path # set permission of the file to ug+rw @@ -237,10 +241,16 @@ else chmod ug+rwx ${STORAGE}/lower_dirs/${remove_dir} done - grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files - cat ${determine_outerr}.rm_files | while read remove_file; do - touch ${STORAGE}/lower_dirs/${remove_file} - chmod ug+rw ${STORAGE}/lower_dirs/${remove_file} + # grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files + # cat ${determine_outerr}.rm_files | while read remove_file; do + # touch ${STORAGE}/lower_dirs/${remove_file} + # chmod ug+rw ${STORAGE}/lower_dirs/${remove_file} + # done + + grep ^REMOVE_MODULE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_modules + cat ${determine_outerr}.rm_modules | while read remove_module; do + touch ${STORAGE}/lower_dirs/${remove_module} + chmod ug+rw ${STORAGE}/lower_dirs/${remove_module} done # prepare directory to store tarball of tmp for removal and build steps From c7916ec5ffd2939e6d0862f6b46477335dbe87b2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 10:18:23 +0100 Subject: [PATCH 1562/1795] remove hooks --- eb_hooks.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index fd88d8399a..03642656ea 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -3,8 +3,6 @@ import glob import os import re -import stat -import time import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess @@ -964,30 +962,6 @@ def inject_gpu_property(ec): return ec -def pre_package_hook(self, *args, **kwargs): - """Main pre-package hook: trigger custom functions based on software name.""" - if self.name in PRE_PACKAGE_HOOKS: - PRE_PACKAGE_HOOKS[self.name](self, *args, **kwargs) - - -def pre_package_eessi_extend(self, *args, **kwargs): - """ - Pre-package hook for EESSI-extend: ls/stat all files/directories to work around - 'permission denied' issue when package got removed (and this hook is run when - the package is being rebuilt) - """ - if self.name == 'EESSI-extend': - dir_tree = [] - for root, _, files in os.walk(self.installdir): - dir_tree.append(root) - for f in files: - dir_tree.append(os.path.join(root, f)) - for entry in dir_tree: - print(entry) - statinfo = os.lstat(entry) - print(f"file: {entry}\nperms: {oct(stat.S_IMODE(statinfo.st_mode))}\ncreated: {time.ctime(statinfo.st_ctime)}\n") - - PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, @@ -1047,7 +1021,3 @@ def pre_package_eessi_extend(self, *args, **kwargs): 'CUDA': post_postproc_cuda, 'cuDNN': post_postproc_cudnn, } - -PRE_PACKAGE_HOOKS = { - # 'EESSI-extend': pre_package_eessi_extend, -} From 1c24b7ab796e684131939ce034e07af0bd94f984 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 10:27:33 +0100 Subject: [PATCH 1563/1795] clean up lower dir changes --- eessi_container.sh | 48 +++++++++++----------------------------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 069da99d14..73244778cf 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -756,21 +756,9 @@ do # to be able to see the contents of the read-write session we have to mount # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository - echo "While processing '${cvmfs_repo_name}' to be mounted 'read-only'" - echo " we detected one or more overlay-upper* directories" - echo " (${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper*)" - echo " likely originating from a previous session. Will use then as" - echo " left-most directory in 'lowerdir' argument for fuse-overlayfs." - - lowerdirs=/cvmfs_ro/${cvmfs_repo_name} - # # check if there are more overlay-upper directories, e.g., with three digit suffix - # for dir in $(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n); do - # lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper-${dir}:${lowerdirs} - # done - # finally add most recent overlay-upper to lowerdirs - lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:${lowerdirs} - [[ ${VERBOSE} -eq 1 ]] && ls ${EESSI_TMPDIR}/${cvmfs_repo_name} - [[ ${VERBOSE} -eq 1 ]] && echo ${lowerdirs} + echo "While processing '${cvmfs_repo_name}' to be mounted 'read-only' we detected an overlay-upper" + echo " directory (${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper) likely from a previous" + echo " session. Will use it as left-most directory in 'lowerdir' argument for fuse-overlayfs." # make the target CernVM-FS repository available under /cvmfs_ro export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" @@ -779,7 +767,12 @@ do # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs EESSI_READONLY_OVERLAY="container:fuse-overlayfs" - EESSI_READONLY_OVERLAY+=" -o lowerdir=${lowerdirs}" + # The contents of the previous session are available under + # ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}. + # Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper + # the left-most directory given for the lowerdir argument is put on top, + # and with no upperdir=... the whole overlayfs is made available read-only + EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}" EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" export EESSI_READONLY_OVERLAY @@ -795,27 +788,7 @@ do export EESSI_FUSE_MOUNTS fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then - # use repo-specific overlay directories; if there is already an - # overlay-upper (e.g., from a previous run) move it to overlay-upper-SEQ - # and create a new one; all overlay-upper-SEQs must be added to lowerdir - # starting with the lowest number first and preprending it to the lowerdir - # setting - lowerdirs=/cvmfs_ro/${cvmfs_repo_name} - # if [ -d ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ]; then - # # determine next sequence number - # last_seq_num=$(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n | tail -n 1 | sed -e 's/^0*//') - # if [ -n ${last_seq_num} ]; then - # last_seq_num=0 - # fi - # next_seq_num=$(($last_seq_num + 1)) - # next_ovl_upper=$(printf "overlay-upper-%03d" ${next_seq_num}) - # mv ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ${EESSI_TMPDIR}/${cvmfs_repo_name}/${next_ovl_upper} - # for dir in $(ls ${EESSI_TMPDIR}/${cvmfs_repo_name} | grep -E "overlay-upper-[0-9]{3}" | cut -f3 -d- | sort -n); do - # lowerdirs=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper-${dir}:${lowerdirs} - # done - # [[ ${VERBOSE} -eq 1 ]] && ls ${EESSI_TMPDIR}/${cvmfs_repo_name} - # [[ ${VERBOSE} -eq 1 ]] && echo ${lowerdirs} - # fi + # use repo-specific overlay directories mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" @@ -826,6 +799,7 @@ do EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + lowerdirs=/cvmfs_ro/${cvmfs_repo_name} if [[ ! -z ${LOWER_DIRS} ]]; then # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as # separator while the lowerdir overlayfs option uses ':' From 5fd8fd71dc88b15cbfa5b7053ac0ec85be610606 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 10:41:55 +0100 Subject: [PATCH 1564/1795] need to create directory for module file in lower_dirs --- bot/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 7b4414983d..d07c5ab641 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -249,6 +249,9 @@ else grep ^REMOVE_MODULE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_modules cat ${determine_outerr}.rm_modules | while read remove_module; do + module_parent_dir=$(dirname ${STORAGE}/lower_dirs/${remove_module}) + mkdir -p ${module_parent_dir} + chmod ug+rw ${module_parent_dir} touch ${STORAGE}/lower_dirs/${remove_module} chmod ug+rw ${STORAGE}/lower_dirs/${remove_module} done From f7baeafec103cbe01c0aadebdec3fe3074178ebc Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 10:51:10 +0100 Subject: [PATCH 1565/1795] add try-amend keeppreviousinstal=True option in easystack --- .../2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml index e4c658784f..cd0ecd1108 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml @@ -3,4 +3,6 @@ # EESSI-extend should adjust EASYBUILD_INSTALLPATH and set # EASYBUILD_CUDA_COMPUTE_CAPABILITIES easyconfigs: - - EESSI-extend-2023.06-easybuild.eb + - EESSI-extend-2023.06-easybuild.eb: + options: + try-amend: keeppreviousinstall=True From 3db71ef615064b6a5a23c540d19f9ef9fe9e4dbc Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 11:05:53 +0100 Subject: [PATCH 1566/1795] need keeppreviousinstall already in load_eessi_... script --- load_eessi_extend_module.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index 9d13f313f3..f24cf98757 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -91,7 +91,7 @@ else eb_install_out=${TMPDIR}/eb_install.out ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" - ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" 2>&1 | tee ${eb_install_out} + ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" --try-amend=keeppreviousinstall=True 2>&1 | tee ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" ) From c56712e2d40d95219126de1c06d4bca24e1a6338 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 14:16:47 +0100 Subject: [PATCH 1567/1795] revert to using standard method (fakeroot) to remove existing installation --- EESSI-determine-rebuilds.sh | 123 ------------------ EESSI-remove-software.sh | 43 ++---- bot/build.sh | 66 +--------- .../20241112-eb-4.9.4-EESSI-extend.yml | 8 +- eessi_container.sh | 19 +-- load_eessi_extend_module.sh | 2 + 6 files changed, 17 insertions(+), 244 deletions(-) delete mode 100755 EESSI-determine-rebuilds.sh diff --git a/EESSI-determine-rebuilds.sh b/EESSI-determine-rebuilds.sh deleted file mode 100755 index 4f4d5ab713..0000000000 --- a/EESSI-determine-rebuilds.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -# -# Script to determine which parts of the EESSI software stack (version set through init/eessi_defaults) -# have to be rebuilt - -# see example parsing of command line arguments at -# https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop -# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash - -display_help() { - echo "usage: $0 [OPTIONS]" - echo " -g | --generic - instructs script to build for generic architecture target" - echo " -h | --help - display this usage information" -} - -POSITIONAL_ARGS=() - -while [[ $# -gt 0 ]]; do - case $1 in - -g|--generic) - DETECTION_PARAMETERS="--generic" - shift - ;; - -h|--help) - display_help # Call your function - # no shifting needed here, we're done. - exit 0 - ;; - -*|--*) - echo "Error: Unknown option: $1" >&2 - exit 1 - ;; - *) # No more options - POSITIONAL_ARGS+=("$1") # save positional arg - shift - ;; - esac -done - -set -- "${POSITIONAL_ARGS[@]}" - -TOPDIR=$(dirname $(realpath $0)) - -export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX) - -source $TOPDIR/scripts/utils.sh - -echo ">> Determining software subdirectory to use for current build host..." -if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" -else - echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" -fi - -echo ">> Setting up environment..." - -source $TOPDIR/init/bash - -if [ -d $EESSI_CVMFS_REPO ]; then - echo_green "$EESSI_CVMFS_REPO available, OK!" -else - fatal_error "$EESSI_CVMFS_REPO is not available!" -fi - -if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then - fatal_error "Failed to determine software subdirectory?!" -elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then - fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" -else - echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" -fi - -echo ">> Configuring EasyBuild..." -EB="eb" -source $TOPDIR/configure_easybuild - -echo ">> Setting up \$MODULEPATH..." -# make sure no modules are loaded -module --force purge -# ignore current $MODULEPATH entirely -module unuse $MODULEPATH -module use $EASYBUILD_INSTALLPATH/modules/all -if [[ -z ${MODULEPATH} ]]; then - fatal_error "Failed to set up \$MODULEPATH?!" -else - echo_green ">> MODULEPATH set up: ${MODULEPATH}" -fi - -# assume there's only one diff file that corresponds to the PR patch file -pr_diff=$(ls [0-9]*.diff | head -1) - -# if this script is run as root, use PR patch file to determine if software needs to be removed first -changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") -if [ -z ${changed_easystacks_rebuilds} ]; then - echo "No software needs to be removed." -else - for easystack_file in ${changed_easystacks_rebuilds}; do - # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') - - # load EasyBuild module (will be installed if it's not available yet) - source ${TOPDIR}/load_easybuild_module.sh ${eb_version} - - if [ -f ${easystack_file} ]; then - echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." - # we need to remove existing installation directories first, - # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) - # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') - for app in ${rebuild_apps}; do - app_dir=${EASYBUILD_INSTALLPATH}/software/${app} - app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua - echo_yellow "Removing ${app_dir} and ${app_module}..." - find ${app_dir} -type d | sed -e 's/^/REMOVE_DIRECTORY /' - find ${app_dir} -type f | sed -e 's/^/REMOVE_FILE /' - echo "REMOVE_MODULE ${app_module}" - done - else - fatal_error "Easystack file ${easystack_file} not found!" - fi - done -fi diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 8b882b38a9..1a03a7af98 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -101,7 +101,7 @@ fi pr_diff=$(ls [0-9]*.diff | head -1) # if this script is run as root, use PR patch file to determine if software needs to be removed first -if [ $EUID -ne 0 ]; then +if [ $EUID -eq 0 ]; then changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") if [ -z ${changed_easystacks_rebuilds} ]; then echo "No software needs to be removed." @@ -114,14 +114,11 @@ if [ $EUID -ne 0 ]; then source ${TOPDIR}/load_easybuild_module.sh ${eb_version} if [ -f ${easystack_file} ]; then - echo_green "Software rebuild(s) requested in ${easystack_file}, so" - echo_green " determining which existing installation have to be removed (assuming contents" - echo_green " have been made writable/deletable)..." + echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." # we need to remove existing installation directories first, # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - # rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') - rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') for app in ${rebuild_apps}; do # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all: app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://') @@ -129,35 +126,12 @@ if [ $EUID -ne 0 ]; then app_installprefix=$(dirname $(dirname ${app_modulepath})) app_dir=${app_installprefix}/software/${app} app_module=${app_installprefix}/modules/all/${app}.lua - # app_dir=${EASYBUILD_INSTALLPATH}/software/${app} - # app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - # suggestion: use the recursive rm's and ls a specific - # directory only (${app_dir}/easybuild) - rm -rdfv ${app_dir} - rm -rdfv ${app_module} - echo_yellow "Contents of ${app_dir}/easybuild (should not exist)" - ls -l ${app_dir}/easybuild || true - # ls didn't change the result (permission denied) - # ls ${app_dir}/easybuild || true - # 2nd idea: recreate some directory + rm -rf ${app_dir} + rm -rf ${app_module} + # recreate some directory to work around permission denied + # issues when rebuilding the package mkdir -p ${app_dir}/easybuild - echo_yellow "Contents of ${app_dir}/easybuild after it got recreated with 'mkdir -p' (should be empty)" - ls -l ${app_dir}/easybuild || true - - ## 1st remove files in depth-first order - #for filepath in $(find ${app_dir} -depth -type f); do - # echo " removing file ${filepath}" - # rm -fv ${filepath} - #done - ## 2nd remove directories in depth-first order - #for dirpath in $(find ${app_dir} -depth -type d); do - # echo " removing directory ${dirpath}" - # rmdir -v ${dirpath} - #done - ## 3rd remove module file - #echo " removing module file ${app_module}" - #rm -fv ${app_module} done else fatal_error "Easystack file ${easystack_file} not found!" @@ -165,6 +139,5 @@ if [ $EUID -ne 0 ]; then done fi else - # fatal_error "This script can only be run by root!" - fatal_error "This script must not be run by root!" + fatal_error "This script can only be run by root!" fi diff --git a/bot/build.sh b/bot/build.sh index d07c5ab641..81b3ef4660 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -200,62 +200,6 @@ changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed if [[ -z "${changed_easystacks_rebuilds}" ]]; then echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step." else - # determine which software packages (and modules) have to be removed - TARBALL_TMP_DETERMINE_STEP_DIR=${PREVIOUS_TMP_DIR}/determine_step - mkdir -p ${TARBALL_TMP_DETERMINE_STEP_DIR} - - # prepare arguments to eessi_container.sh specific to determine step - declare -a DETERMINE_STEP_ARGS=() - DETERMINE_STEP_ARGS+=("--save" "${TARBALL_TMP_DETERMINE_STEP_DIR}") - DETERMINE_STEP_ARGS+=("--storage" "${STORAGE}") - - # create tmp file for output of determine step - determine_outerr=$(mktemp determine.outerr.XXXX) - - echo "Executing command to determine software to be removed:" - echo "${software_layer_dir}/eessi_container.sh ${COMMON_ARGS[@]} ${DETERMINE_STEP_ARGS[@]}" - echo " -- ${software_layer_dir}/EESSI-determine-rebuilds.sh \"${DETERMINE_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${determine_outerr}" - ${software_layer_dir}/eessi_container.sh "${COMMON_ARGS[@]}" "${DETERMINE_STEP_ARGS[@]}" \ - -- ${software_layer_dir}/EESSI-determine-rebuilds.sh "${DETERMINE_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${determine_outerr} - - # process output file - # for each line containing 'REMOVE_DIRECTORY some_path' - # create a new directory ${STORAGE}/lower_dirs/some_path_stripped - # where the prefix /cvmfs/repo_name is removed from some_path - # set permission of the directory to ug+rwx - # SKIP for each line containing 'REMOVE_FILE some_file_path' - # SKIP touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped - # SKIP where the prefix /cvmfs/repo_name is removed from some_file_path - # SKIP set permission of the file to ug+rw - # for each line containing 'REMOVE_MODULE some_file_path' - # touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped - # where the prefix /cvmfs/repo_name is removed from some_file_path - # set permission of the file to ug+rw - - LOWER_DIRS="${STORAGE}/lower_dirs" - mkdir -p "${LOWER_DIRS}" - - grep ^REMOVE_DIRECTORY ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_dirs - cat ${determine_outerr}.rm_dirs | while read remove_dir; do - mkdir -p ${STORAGE}/lower_dirs/${remove_dir} - chmod ug+rwx ${STORAGE}/lower_dirs/${remove_dir} - done - - # grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files - # cat ${determine_outerr}.rm_files | while read remove_file; do - # touch ${STORAGE}/lower_dirs/${remove_file} - # chmod ug+rw ${STORAGE}/lower_dirs/${remove_file} - # done - - grep ^REMOVE_MODULE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_modules - cat ${determine_outerr}.rm_modules | while read remove_module; do - module_parent_dir=$(dirname ${STORAGE}/lower_dirs/${remove_module}) - mkdir -p ${module_parent_dir} - chmod ug+rw ${module_parent_dir} - touch ${STORAGE}/lower_dirs/${remove_module} - chmod ug+rw ${STORAGE}/lower_dirs/${remove_module} - done - # prepare directory to store tarball of tmp for removal and build steps TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR} @@ -267,11 +211,7 @@ else # add fakeroot option in order to be able to remove software, see: # https://github.com/EESSI/software-layer/issues/312 - # REMOVAL_STEP_ARGS+=("--fakeroot") - - if [[ ! -z ${LOWER_DIRS} ]]; then - REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}") - fi + REMOVAL_STEP_ARGS+=("--fakeroot") # create tmp file for output of removal step removal_outerr=$(mktemp remove.outerr.XXXX) @@ -309,10 +249,6 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi -if [[ ! -z ${LOWER_DIRS} ]]; then - BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}") -fi - # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml index cd0ecd1108..2129eebee0 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml @@ -3,6 +3,8 @@ # EESSI-extend should adjust EASYBUILD_INSTALLPATH and set # EASYBUILD_CUDA_COMPUTE_CAPABILITIES easyconfigs: - - EESSI-extend-2023.06-easybuild.eb: - options: - try-amend: keeppreviousinstall=True + - EESSI-extend-2023.06-easybuild.eb +# the options are added to load_eessi_extend_module.sh +# - EESSI-extend-2023.06-easybuild.eb: +# options: +# try-amend: keeppreviousinstall=True diff --git a/eessi_container.sh b/eessi_container.sh index 73244778cf..fc97f9877c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -89,11 +89,6 @@ display_help() { echo " -n | --nvidia MODE - configure the container to work with NVIDIA GPUs," echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" - echo " -o | --lower-dirs DIRS - list of ':' separated directories that are used" - echo " in front of the default lower dir (CVMFS repo);" - echo " fuse-overlayfs will merge all lower directories;" - echo " the option can be used to make certain directories" - echo " in the CVMFS repo writable [default: none]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use; can be given multiple times;" echo " CFG may include a suffix ',access={ro,rw}' to" @@ -130,7 +125,6 @@ FAKEROOT=0 VERBOSE=0 STORAGE= LIST_REPOS=0 -LOWER_DIRS= MODE="shell" SETUP_NVIDIA=0 REPOSITORIES=() @@ -188,10 +182,6 @@ while [[ $# -gt 0 ]]; do NVIDIA_MODE="$2" shift 2 ;; - -o|--lower-dirs) - LOWER_DIRS="$2" - shift 2 - ;; -r|--repository) REPOSITORIES+=("$2") shift 2 @@ -799,14 +789,7 @@ do EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" - lowerdirs=/cvmfs_ro/${cvmfs_repo_name} - if [[ ! -z ${LOWER_DIRS} ]]; then - # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as - # separator while the lowerdir overlayfs option uses ':' - export BIND_PATHS="${BIND_PATHS},${LOWER_DIRS/:/,}" - lowerdirs=${LOWER_DIRS}:${lowerdirs} - fi - EESSI_WRITABLE_OVERLAY+=" -o lowerdir=${lowerdirs}" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index f24cf98757..d3ba524053 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -91,6 +91,8 @@ else eb_install_out=${TMPDIR}/eb_install.out ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" + # while always adding --try-amend=keep... may do no harm, we could make + # an attempt to figure out if it is needed, e.g., when we are rebuilding ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" --try-amend=keeppreviousinstall=True 2>&1 | tee ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" ) From d1264e11e6dac91e3e87ba0b35a658426fa9e3c0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 15 Nov 2024 19:55:53 +0100 Subject: [PATCH 1568/1795] first load/install EESSI-extend module, then install CUDA and libraries --- EESSI-install-software.sh | 40 ++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 50eacc52bf..bb103d8c21 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -229,6 +229,27 @@ if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} fi +echo ">> Configuring EasyBuild..." + +# Make sure that we use the EESSI_CVMFS_INSTALL +# Since the path is set when loading EESSI-extend, we reload it to make sure it works - even if it is already loaded +# Note we need to do this after running install_cuda_and_libraries, since that does installations in the EESSI_SITE_INSTALL +unset EESSI_USER_INSTALL +unset EESSI_PROJECT_INSTALL +unset EESSI_SITE_INSTALL +export EESSI_CVMFS_INSTALL=1 +module unload EESSI-extend + +# The EESSI-extend module is being loaded (or installed if it doesn't exist yet). +# The script requires the EESSI_VERSION given as argument, a couple of +# environment variables set (TMPDIR, EB and EASYBUILD_INSTALLPATH) and the +# function check_exit_code defined. +# NOTE, the script exits if those variables/functions are undefined. +export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +# Loading the EESSI-extend module may adjust the value of EASYBUILD_INSTALLPATH, +# e.g., to point to the installation directory for accelerators. +source load_eessi_extend_module.sh ${EESSI_VERSION} + # Install full CUDA SDK and cu* libraries in host_injections # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install @@ -267,25 +288,6 @@ if command_exists "nvidia-smi"; then fi -echo ">> Configuring EasyBuild..." - -# Make sure that we use the EESSI_CVMFS_INSTALL -# Since the path is set when loading EESSI-extend, we reload it to make sure it works - even if it is already loaded -# Note we need to do this after running install_cuda_and_libraries, since that does installations in the EESSI_SITE_INSTALL -unset EESSI_USER_INSTALL -unset EESSI_PROJECT_INSTALL -unset EESSI_SITE_INSTALL -export EESSI_CVMFS_INSTALL=1 -module unload EESSI-extend - -# The EESSI-extend module is being loaded (or installed if it doesn't exist yet). -# The script requires the EESSI_VERSION given as argument, a couple of -# environment variables set (TMPDIR, EB and EASYBUILD_INSTALLPATH) and the -# function check_exit_code defined. -# NOTE, the script exits if those variables/functions are undefined. -export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -source load_eessi_extend_module.sh ${EESSI_VERSION} - if [ ! -z "${shared_fs_path}" ]; then shared_eb_sourcepath=${shared_fs_path}/easybuild/sources echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" From 47b2a2fdfc5de9728be37bada9d21a4657306070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 16 Nov 2024 23:06:22 +0100 Subject: [PATCH 1569/1795] add GROMACS-2024.4-foss-2023b.eb --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 03c9ec8f98..54bd7418ea 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -12,3 +12,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21783 from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 + - GROMACS-2024.4-foss-2023b.eb: + options: + # https://github.com/easybuilders/easybuild-easyconfigs/pull/21851 + from-commit: f0fa64b440deaf5fb0a6d26ff1bb3e9f36626c8a From 66c36c898a208fcdce1258f7bde53f08a5fa28f6 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 18 Nov 2024 12:59:16 +0000 Subject: [PATCH 1570/1795] clean `${tmpdir}` content for each iteration in `install_cuda_and_libraries.sh` --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index e1ca42fc9a..ee219fb444 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -239,9 +239,9 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do echo_green "all installations at ${EESSI_SITE_SOFTWARE_PATH}/software/... succeeded!" fi + # clean up tmpdir content + rm -rf "${tmpdir}"/* + # Restore MODULEPATH for next loop iteration MODULEPATH=${SAVE_MODULEPATH} done - -# clean up tmpdir -rm -rf "${tmpdir}" From fedfd77582a42fa81d60754bd4054e5f4a8a14b9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 18 Nov 2024 20:05:59 +0100 Subject: [PATCH 1571/1795] tweak script to just load EESSI-extend, and exit if it couldn't be loaded --- .../nvidia/install_cuda_and_libraries.sh | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 51d139bec5..87b7653749 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -112,14 +112,19 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do unset EESSI_USER_INSTALL export EESSI_SITE_INSTALL=1 module unload EESSI-extend - - # The EESSI-extend module is being loaded (or installed if it doesn't exist yet). - # The script requires the EESSI_VERSION given as argument, a couple of - # environment variables set (TMPDIR, EB and EASYBUILD_INSTALLPATH) and the - # function check_exit_code defined. - # NOTE, the script exits if those variables/functions are undefined. - export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - source load_eessi_extend_module.sh ${EESSI_VERSION} + ml_av_eessi_extend_out=${tmpdir}/ml_av_eessi_extend.out + # need to use --ignore_cache to avoid the case that the module was removed (to be + # rebuilt) but it is still in the cache and the rebuild failed + EESSI_EXTEND_VERSION=${EESSI_VERSION}-easybuild + module --ignore_cache avail 2>&1 | grep -i EESSI-extend/${EESSI_EXTEND_VERSION} &> ${ml_av_eessi_extend_out} + if [[ $? -eq 0 ]]; then + echo_green ">> Module for EESSI-extend/${EESSI_EXTEND_VERSION} found!" + else + error="\nNo module for EESSI-extend/${EESSI_EXTEND_VERSION} found\nwhile EESSI has been initialised to use software under ${EESSI_SOFTWARE_PATH}\n" + fatal_error "${error}" + fi + module --ignore_cache load EESSI-extend/${EESSI_EXTEND_VERSION} + unset EESSI_EXTEND_VERSION # Install modules in hidden .modules dir to keep track of what was installed before # (this action is temporary, and we do not call Lmod again within the current shell context, but in EasyBuild From 635fe8894523376d00666420b659e6d08b37ef25 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 19 Nov 2024 10:06:07 +0100 Subject: [PATCH 1572/1795] Remove tempdir after running `install_cuda_and_libraries.sh` --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index ee219fb444..119316dab2 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -245,3 +245,5 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do # Restore MODULEPATH for next loop iteration MODULEPATH=${SAVE_MODULEPATH} done +# Remove the temporary directory +rm -rf "${tmpdir}" From a1f407340339cee2b8517121ceebe56d90847b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Nov 2024 14:40:52 +0100 Subject: [PATCH 1573/1795] add CUDA 12.4.0 and UCX-CUDA 1.15.0 for 2023b --- .../2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml new file mode 100644 index 0000000000..3fd439d0dc --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -0,0 +1,3 @@ +easyconfigs: + - CUDA-12.4.0.eb + - UCX-CUDA-1.15.0.eb From 6c07b17be28c0263bab8f14e95b31a29eaff96ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Nov 2024 14:45:24 +0100 Subject: [PATCH 1574/1795] add easystack for 2023b, with CUDA 12.4.0 in it --- .../eessi-2023.06-eb-4.9.4-2023b-CUDA-host-injections.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023b-CUDA-host-injections.yml diff --git a/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023b-CUDA-host-injections.yml b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023b-CUDA-host-injections.yml new file mode 100644 index 0000000000..5cfec813f6 --- /dev/null +++ b/scripts/gpu_support/nvidia/easystacks/eessi-2023.06-eb-4.9.4-2023b-CUDA-host-injections.yml @@ -0,0 +1,4 @@ +# This EasyStack provides a list of all the EasyConfigs that should be installed in host_injections +# for nvidia GPU support, because they cannot (fully) be shipped as part of EESSI due to license constraints +easyconfigs: + - CUDA-12.4.0.eb From 0959f2db0fd820f0d335341d83f3207b559c51fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Nov 2024 14:46:17 +0100 Subject: [PATCH 1575/1795] add 2023b easystack for host injection installations --- install_scripts.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_scripts.sh b/install_scripts.sh index 8787888c79..c5a9a556c2 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -132,6 +132,7 @@ copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/script # Easystacks to be used to install software in host injections host_injections_easystacks=( eessi-2023.06-eb-4.9.4-2023a-CUDA-host-injections.yml + eessi-2023.06-eb-4.9.4-2023b-CUDA-host-injections.yml ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia/easystacks \ ${INSTALL_PREFIX}/scripts/gpu_support/nvidia/easystacks "${host_injections_easystacks[@]}" From 37d57715fcaac1fd5f8583bbc6cd6872bdc90754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Nov 2024 15:22:28 +0100 Subject: [PATCH 1576/1795] accept CUDA eula --- .../2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index 3fd439d0dc..0001b5db16 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -1,3 +1,5 @@ easyconfigs: - CUDA-12.4.0.eb + options: + accept-eula-for: CUDA - UCX-CUDA-1.15.0.eb From 1191204d620a041fde635d4757b92f04d96ce32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Nov 2024 15:40:15 +0100 Subject: [PATCH 1577/1795] add missing colon --- .../2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index 0001b5db16..c549ebdd70 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -1,5 +1,5 @@ easyconfigs: - - CUDA-12.4.0.eb + - CUDA-12.4.0.eb: options: accept-eula-for: CUDA - UCX-CUDA-1.15.0.eb From 4518029454bb51ca810a18dc275cdb9324131676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Nov 2024 15:47:40 +0100 Subject: [PATCH 1578/1795] fix name of UCX-CUDA easyconfig --- .../2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index c549ebdd70..5f195b3714 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -2,4 +2,4 @@ easyconfigs: - CUDA-12.4.0.eb: options: accept-eula-for: CUDA - - UCX-CUDA-1.15.0.eb + - UCX-CUDA-1.15.0-GCCcore-13.2.0-CUDA-12.4.0.eb From 83d501c8724e0877298a1ee49302864429ac7a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 19 Nov 2024 16:01:25 +0100 Subject: [PATCH 1579/1795] add GDRCopy-2.4-GCCcore-13.2.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 03c9ec8f98..d9b7dca5d5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -12,3 +12,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21783 from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 + - GDRCopy-2.4-GCCcore-13.2.0.eb From 8e87c33bd7d7015877daae975f94404e025cfb7a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 20 Nov 2024 21:51:40 +0100 Subject: [PATCH 1580/1795] various changes to address suggestions - EESSI-install-software.sh - rearranged setting up build environment and improved comments to make choices more clear - moved test if EasyBuild exists into install_cuda_and_libraries.sh, removed unnecessary export of environment variables and clarified comments accordingly - .../rebuilds/20241112-eb-4.9.4-EESSI-extend.yml - removed outdated comments - load_eessi_extend_module.sh - replaced sourcing configure_easybuild with two needed EASYBUILD_ environment settings - install_cuda_and_libraries.sh - removed comment and setting of EESSI_SITE_INSTALL - clarified comment when saving MODULEPATH - added a check if the required EasyBuild version exists and only if so use it to process the easystack file that required it --- EESSI-install-software.sh | 39 +++++++------------ .../20241112-eb-4.9.4-EESSI-extend.yml | 4 -- load_eessi_extend_module.sh | 4 +- .../nvidia/install_cuda_and_libraries.sh | 16 +++++--- 4 files changed, 25 insertions(+), 38 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index bb103d8c21..83c06c2184 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -231,54 +231,41 @@ fi echo ">> Configuring EasyBuild..." -# Make sure that we use the EESSI_CVMFS_INSTALL -# Since the path is set when loading EESSI-extend, we reload it to make sure it works - even if it is already loaded -# Note we need to do this after running install_cuda_and_libraries, since that does installations in the EESSI_SITE_INSTALL +# Make sure EESSI-extend is not loaded, and configure location variables for a +# CVMFS installation +module unload EESSI-extend unset EESSI_USER_INSTALL unset EESSI_PROJECT_INSTALL unset EESSI_SITE_INSTALL export EESSI_CVMFS_INSTALL=1 -module unload EESSI-extend -# The EESSI-extend module is being loaded (or installed if it doesn't exist yet). +# We now run 'source load_eessi_extend_module.sh' to load or install and load the +# EESSI-extend module which sets up all build environment settings. # The script requires the EESSI_VERSION given as argument, a couple of -# environment variables set (TMPDIR, EB and EASYBUILD_INSTALLPATH) and the -# function check_exit_code defined. -# NOTE, the script exits if those variables/functions are undefined. +# environment variables set (TMPDIR, EB and EASYBUILD_INSTALLPATH) and the +# function check_exit_code defined. +# NOTE 1, the script exits if those variables/functions are undefined. +# NOTE 2, loading the EESSI-extend module may adjust the value of EASYBUILD_INSTALLPATH, +# e.g., to point to the installation directory for accelerators. +# NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the +# EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -# Loading the EESSI-extend module may adjust the value of EASYBUILD_INSTALLPATH, -# e.g., to point to the installation directory for accelerators. source load_eessi_extend_module.sh ${EESSI_VERSION} # Install full CUDA SDK and cu* libraries in host_injections # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install # Allow skipping CUDA SDK install in e.g. CI environments -# The install_cuda... script uses EasyBuild. So, we need to check if we have EB -# or skip this step. echo "Going to install full CUDA SDK and cu* libraries under host_injections if necessary" -module_avail_out=$TMPDIR/ml.out -module avail 2>&1 | grep EasyBuild &> ${module_avail_out} -if [[ $? -eq 0 ]]; then - echo_green ">> Found an EasyBuild module" -else - echo_yellow ">> No EasyBuild module found: skipping step to install CUDA (see output in ${module_avail_out})" - export skip_cuda_install=True -fi - temp_install_storage=${TMPDIR}/temp_install_storage mkdir -p ${temp_install_storage} if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then - # need to ensure that some variables will be available to the script - # TMPDIR, EB, EESSI_VERSION, for EASYBUILD_INSTALLPATH (EESSI_PREFIX, - # EESSI_OS_TYPE, EESSI_SOFTWARE_SUBDIR_OVERRIDE) - export TMPDIR EB EESSI_VERSION EESSI_PREFIX EESSI_OS_TYPE EESSI_SOFTWARE_SUBDIR_OVERRIDE ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \ -t ${temp_install_storage} \ --accept-cuda-eula \ --accept-cudnn-eula else - echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found" + echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed" fi # Install NVIDIA drivers in host_injections (if they exist) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml index 2129eebee0..e4c658784f 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml @@ -4,7 +4,3 @@ # EASYBUILD_CUDA_COMPUTE_CAPABILITIES easyconfigs: - EESSI-extend-2023.06-easybuild.eb -# the options are added to load_eessi_extend_module.sh -# - EESSI-extend-2023.06-easybuild.eb: -# options: -# try-amend: keeppreviousinstall=True diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index d3ba524053..62b6e3f3ae 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -82,8 +82,8 @@ else # source configure_easybuild to use correct eb settings ( - EESSI_MAIN_DIR=$(dirname $(readlink -f $BASH_SOURCE)) - source ${EESSI_MAIN_DIR}/configure_easybuild + export EASYBUILD_PREFIX=${TMPDIR}/easybuild + export EASYBUILD_READ_ONLY_INSTALLDIR=1 echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..." export PATH=${EB_TMPDIR}/bin:${PATH} diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index f7405e3ca4..cd4d1daf38 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -77,10 +77,6 @@ done # Make sure EESSI is initialised check_eessi_initialised -# Make sure that `EESSI-extend` will install in the site installation path EESSI_SITE_SOFTWARE_PATH -export EESSI_SITE_INSTALL=1 -echo "EESSI_SITE_SOFTWARE_PATH=${EESSI_SITE_SOFTWARE_PATH}" - # we need a directory we can use for temporary storage if [[ -z "${TEMP_DIR}" ]]; then tmpdir=$(mktemp -d) @@ -93,7 +89,7 @@ else fi echo "Created temporary directory '${tmpdir}'" -# use EESSI_SITE_SOFTWARE_PATH/.modules/all as MODULEPATH +# Store MODULEPATH so it can be restored at the end of each loop iteration SAVE_MODULEPATH=${MODULEPATH} for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do @@ -103,8 +99,16 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do eb_version=$(echo ${EASYSTACK_FILE} | sed 's/.*eb-\([0-9.]*\).*/\1/g') # Load EasyBuild version for this easystack file _before_ loading EESSI-extend - module avail EasyBuild + module_avail_out=${tmpdir}/ml.out + module avail 2>&1 | grep EasyBuild/${eb_version} &> ${module_avail_out} + if [[ $? -eq 0 ]]; then + echo_green ">> Found an EasyBuild/${eb_version} module" + else + echo_yellow ">> No EasyBuild/${eb_version} module found: skipping step to install easystack file ${easystack_file} (see output in ${module_avail_out})" + continue + fi module load EasyBuild/${eb_version} + # Make sure EESSI-extend does a site install here # We need to reload it with the current environment variables set unset EESSI_CVMFS_INSTALL From 9812aa5cb5b6c168c03725fcb18d8b73257a0c68 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 21 Nov 2024 08:14:41 +0100 Subject: [PATCH 1581/1795] catch FATAL: messages when checking build result --- bot/check-build.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/bot/check-build.sh b/bot/check-build.sh index f185b18dda..41aeab258e 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -17,6 +17,7 @@ # - SUCCESS (all of) # - working directory contains slurm-JOBID.out file # - working directory contains eessi*tar.gz +# - no message FATAL # - no message ERROR # - no message FAILED # - no message ' required modules missing:' @@ -25,6 +26,7 @@ # - FAILED (one of ... implemented as NOT SUCCESS) # - no slurm-JOBID.out file # - no tarball +# - message with FATAL # - message with ERROR # - message with FAILED # - message with ' required modules missing:' @@ -105,6 +107,16 @@ else [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi +FATAL=-1 +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + GP_fatal='FATAL: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_fatal}") + [[ $? -eq 0 ]] && FATAL=1 || FATAL=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_fatal}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + ERROR=-1 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_error='ERROR: ' @@ -163,6 +175,7 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" [[ ${VERBOSE} -ne 0 ]] && echo " : ()" +[[ ${VERBOSE} -ne 0 ]] && echo " FATAL......: $([[ $FATAL -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" @@ -190,6 +203,7 @@ job_result_file=_bot_job${SLURM_JOB_ID}.result # Default reason: if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ + [[ ${FATAL} -eq 0 ]] && \ [[ ${ERROR} -eq 0 ]] && \ [[ ${FAILED} -eq 0 ]] && \ [[ ${MISSING} -eq 0 ]] && \ @@ -223,6 +237,7 @@ fi #
    _Details_
    #
    # :white_check_mark: job output file slurm-4682.out
    +# :white_check_mark: no message matching FATAL:
    # :white_check_mark: no message matching ERROR:
    # :white_check_mark: no message matching FAILED:
    # :white_check_mark: no message matching required modules missing:
    @@ -264,6 +279,7 @@ fi #
    _Details_
    #
    # :white_check_mark: job output file slurm-4682.out
    +# :x: no message matching FATAL:
    # :x: no message matching ERROR:
    # :white_check_mark: no message matching FAILED:
    # :x: no message matching required modules missing:
    @@ -381,6 +397,10 @@ success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") +success_msg="no message matching ${GP_fatal}" +failure_msg="found message matching ${GP_fatal}" +comment_details_list=${comment_details_list}$(add_detail ${FATAL} 0 "${success_msg}" "${failure_msg}") + success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") From 18b20c0e13b2447b954f305f90eb1bece673d63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 22 Nov 2024 16:18:40 +0100 Subject: [PATCH 1582/1795] add OpenFOAM-v2406-foss-2023a.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index ba050fe2fa..5ab2a09fd5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -25,3 +25,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657 from-commit: 7f1f0e60487e7e1fcb5c4e6bc4fbc4f89994e3fd + - OpenFOAM-v2406-foss-2023a.eb From 3091a3194c975cb1ccad0eb99b91c23731569400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Nov 2024 09:06:28 +0100 Subject: [PATCH 1583/1795] fix typo in EULA: replace libnvrtx-builtins_static by libnvrtc-builtins_static --- eb_hooks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 03642656ea..8a12d63d7e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -786,6 +786,11 @@ def post_postproc_cuda(self, *args, **kwargs): for word in line.split(): if any(ext in word for ext in file_extensions): allowlist.append(os.path.splitext(word)[0]) + # The EULA of CUDA 12.4 introduced a typo (confirmed by NVIDIA): + # libnvrtx-builtins_static.so should be libnvrtc-builtins_static.so + if 'libnvrtx-builtins_static' in allowlist: + allowlist.remove('libnvrtx-builtins_static') + allowlist.append('libnvrtc-builtins_static') allowlist = sorted(set(allowlist)) self.log.info("Allowlist for files in CUDA installation that can be redistributed: " + ', '.join(allowlist)) From 9d31114858bf2e235591e68562f0b4edf90c88e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Nov 2024 12:44:33 +0100 Subject: [PATCH 1584/1795] add SlurmViewer-1.0.1-GCCcore-13.2.0.eb --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index d9b7dca5d5..db665193a9 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -13,3 +13,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21783 from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 - GDRCopy-2.4-GCCcore-13.2.0.eb + - SlurmViewer-1.0.1-GCCcore-13.2.0.eb From c9baeda7d4eb352879b091ce72592bd10571720a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Nov 2024 13:10:56 +0100 Subject: [PATCH 1585/1795] use commit for SlurmViewer --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index db665193a9..88c0730800 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -13,4 +13,7 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21783 from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 - GDRCopy-2.4-GCCcore-13.2.0.eb - - SlurmViewer-1.0.1-GCCcore-13.2.0.eb + - SlurmViewer-1.0.1-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21899 + from-commit: 0bdeb23c9ea5a3caefd353ecd936919424c1bba4 From 1b24612a52805c19ca3115dd262e4a1896124be1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Nov 2024 13:14:01 +0100 Subject: [PATCH 1586/1795] add GROMACS-2024.4-foss-2023b-CUDA-12.4.0.eb --- .../accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index 5f195b3714..05025290f2 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -3,3 +3,7 @@ easyconfigs: options: accept-eula-for: CUDA - UCX-CUDA-1.15.0-GCCcore-13.2.0-CUDA-12.4.0.eb + - GROMACS-2024.4-foss-2023b-CUDA-12.4.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21549 + from-commit: 12b53341343967ce5a402fe8190a3c85bce7d49b From ccec8566637046a6d704c2d9be980cbb909f52e5 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 26 Nov 2024 12:49:00 +0000 Subject: [PATCH 1587/1795] {2023.06}[foss/2023a] LightGBM v4.5.0 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index ba050fe2fa..196af4ddbc 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -25,3 +25,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657 from-commit: 7f1f0e60487e7e1fcb5c4e6bc4fbc4f89994e3fd + - LightGBM-4.5.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21699 + from-commit: e3407bd127d248c08960f6b09c973da0fdecc2c3 From 610d760ef34c27b0c63353064a9da08e3a162b2f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Nov 2024 14:41:05 +0100 Subject: [PATCH 1588/1795] use GCC toolchain for Paraver 4.11.4 --- .../2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index eee9e16f40..2c7e80e467 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -22,6 +22,6 @@ easyconfigs: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb - - Paraver-4.11.4-foss-2023a.eb: + - Paraver-4.11.4-GCC-12.3.0.eb: options: - from-pr: 20515 + from-pr: 20230 From c7d76c72984b084a75d826f946fd7808bfe2b3cc Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 26 Nov 2024 15:42:27 +0000 Subject: [PATCH 1589/1795] {2023.06}[foss/2023a] LightGBM v4.5.0 w/ CUDA 12.1.1 --- .../accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml index 873c19aa33..7ac4ba6cca 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml @@ -1,3 +1,7 @@ easyconfigs: - CUDA-12.1.1.eb - cuDNN-8.9.2.26-CUDA-12.1.1.eb + - LightGBM-4.5.0-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21699 + from-commit: e3407bd127d248c08960f6b09c973da0fdecc2c3 From fbd85db3d36956cff9fe5841e31629cf7ade5e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Nov 2024 17:21:56 +0100 Subject: [PATCH 1590/1795] use easyblock from commit for OpenFOAM --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 5ab2a09fd5..0bfa601ba6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -25,4 +25,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657 from-commit: 7f1f0e60487e7e1fcb5c4e6bc4fbc4f89994e3fd - - OpenFOAM-v2406-foss-2023a.eb + - OpenFOAM-v2406-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3519 + include-easyblocks-from-commit: e4a3ff1932350d575dffc7597435609fad6dd691 From 9505031076b9e9240e23c91f92b9fd3b3bb08f0f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Nov 2024 11:22:49 +0100 Subject: [PATCH 1591/1795] use EasyBuild v4.9.4 to install Paraver --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 3 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 2c7e80e467..27c18a487e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -22,6 +22,3 @@ easyconfigs: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb - - Paraver-4.11.4-GCC-12.3.0.eb: - options: - from-pr: 20230 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index ba050fe2fa..4df8afdb25 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -25,3 +25,6 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657 from-commit: 7f1f0e60487e7e1fcb5c4e6bc4fbc4f89994e3fd + - Paraver-4.11.4-GCC-12.3.0.eb: + options: + from-pr: 20230 From 7b6d819ff3e6a09a17b46ffd86d61c29c6fbea52 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Nov 2024 13:50:50 +0100 Subject: [PATCH 1592/1795] use from-commit rather than from-pr to instal Paraver 4.11.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 37a3c1dd4e..7ea65ba94e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -35,4 +35,5 @@ easyconfigs: include-easyblocks-from-commit: e4a3ff1932350d575dffc7597435609fad6dd691 - Paraver-4.11.4-GCC-12.3.0.eb: options: - from-pr: 20230 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20230 + from-commit: 91c8df6b4c0810061e9f325427c9c79e961bc4b0 From 40001c3b90bc4ac7959d7209c753266b94c6c63a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 28 Nov 2024 16:34:39 +0100 Subject: [PATCH 1593/1795] add UCC-CUDA-1.2.0-GCCcore-13.2.0-CUDA-12.4.0.eb and OSU-Micro-Benchmarks-7.5-gompi-2023b-CUDA-12.4.0.eb --- .../accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index 5f195b3714..916fbedbe8 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -3,3 +3,13 @@ easyconfigs: options: accept-eula-for: CUDA - UCX-CUDA-1.15.0-GCCcore-13.2.0-CUDA-12.4.0.eb + - UCC-CUDA-1.2.0-GCCcore-13.2.0-CUDA-12.4.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21565 + from-commit: 46141a3f40e699433fac03af2d3ed81bd5a62da7 + - OSU-Micro-Benchmarks-7.5-gompi-2023b-CUDA-12.4.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21926 + from-commit: 3ac794d3d2a6e2bebc498976b98e2c61cefaf735 + + From ec73fdf9b6a2711f2f3b796931c052edf237f098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Nov 2024 09:46:03 +0100 Subject: [PATCH 1594/1795] new commit for OSU-Micro-Benchmarks --- .../accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index 916fbedbe8..1bee9257a8 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -10,6 +10,4 @@ easyconfigs: - OSU-Micro-Benchmarks-7.5-gompi-2023b-CUDA-12.4.0.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21926 - from-commit: 3ac794d3d2a6e2bebc498976b98e2c61cefaf735 - - + from-commit: de79ec74eb076e1aceda5e21235a73c05ed6764c From cb0b1ef40b48a8adf0dc8540a3a5c69fa3da01c3 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 29 Nov 2024 15:51:15 +0100 Subject: [PATCH 1595/1795] LAMMPS: Add 2Aug2023_update4 to zen4 hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8a12d63d7e..5bf42b6bbf 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -595,7 +595,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'LAMMPS': - if self.version in ('2Aug2023_update2', '29Aug2024'): + if self.version in ('2Aug2023_update2', '2Aug2023_update4', '29Aug2024'): if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 From 6dffc9f3f487abf379623cf17d1a0139b2f3e2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 30 Nov 2024 13:49:06 +0100 Subject: [PATCH 1596/1795] use CMakeMake easyblock from PR 3523 for GROMACS --- .../2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index 05025290f2..c3d057653d 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -7,3 +7,5 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21549 from-commit: 12b53341343967ce5a402fe8190a3c85bce7d49b + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3523 + include-easyblocks-from-commit: ee216590a83315ab7e6deaddcdb1030502a3e9fb From 862e6e0d70e8e9b6675d2d11602d2097db928f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Dec 2024 14:19:24 +0100 Subject: [PATCH 1597/1795] use newer cmakemake.py easyblock for GROMACS --- .../2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index f478b5cd30..dafc5c6a7d 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -16,4 +16,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21549 from-commit: 12b53341343967ce5a402fe8190a3c85bce7d49b # see https://github.com/easybuilders/easybuild-easyblocks/pull/3523 - include-easyblocks-from-commit: ee216590a83315ab7e6deaddcdb1030502a3e9fb + include-easyblocks-from-commit: 0857b9fbcab2f975408aa59623d128aa75e00d5c From 2c43034222d588ca4f7448a082aa6891775d6bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Dec 2024 15:21:09 +0100 Subject: [PATCH 1598/1795] use updated version of cmakemake easyblock --- .../2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml index dafc5c6a7d..e6562e68a3 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -16,4 +16,4 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21549 from-commit: 12b53341343967ce5a402fe8190a3c85bce7d49b # see https://github.com/easybuilders/easybuild-easyblocks/pull/3523 - include-easyblocks-from-commit: 0857b9fbcab2f975408aa59623d128aa75e00d5c + include-easyblocks-from-commit: 90495ed23d26b3d5fd8162bf5d7b4c073a0682fe From a740337067d4faadfed9aa4dc74c4d654716b3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 10:55:22 +0100 Subject: [PATCH 1599/1795] use variable for path to CVMFS repository --- init/lmod/bash | 6 ++++-- init/lmod/csh | 6 ++++-- init/lmod/fish | 6 ++++-- init/lmod/ksh | 6 ++++-- init/lmod/zsh | 6 ++++-- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/init/lmod/bash b/init/lmod/bash index b4941d6766..b2db9a8802 100644 --- a/init/lmod/bash +++ b/init/lmod/bash @@ -1,8 +1,10 @@ +# Choose an EESSI CVMFS repository +EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO:-/cvmfs/software.eessi.io}" # Choose an EESSI version EESSI_VERSION="${EESSI_VERSION:-2023.06}" # Path to top-level module tree -export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules -. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/bash +export MODULEPATH="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules" +. "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash" if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; diff --git a/init/lmod/csh b/init/lmod/csh index 8e50d5e5c8..f2e9100255 100644 --- a/init/lmod/csh +++ b/init/lmod/csh @@ -1,8 +1,10 @@ +# Choose an EESSI CVMFS repository +if (! $?EESSI_CVMFS_REPO) then; set EESSI_CVMFS_REPO = "/cvmfs/software.eessi.io"; endif # Choose an EESSI version if (! $?EESSI_VERSION) then; set EESSI_VERSION = "2023.06"; endif # Path to top-level module tree -setenv MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules -source /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/`uname -m`/usr/share/Lmod/init/csh +setenv MODULEPATH "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules" +source "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/csh" if (! $?__Init_Default_Modules ) then setenv __Init_Default_Modules 1; diff --git a/init/lmod/fish b/init/lmod/fish index d4252ef32a..e0c3b58b31 100644 --- a/init/lmod/fish +++ b/init/lmod/fish @@ -1,8 +1,10 @@ +# Choose an EESSI CVMFS repository +set EESSI_CVMFS_REPO (set -q EESSI_CVMFS_REPO; and echo "$EESSI_CVMFS_REPO"; or echo "/cvmfs/software.eessi.io") # Choose an EESSI version set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") # Path to top-level module tree -set -x MODULEPATH /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules -. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/(uname -m)/usr/share/Lmod/init/fish +set -x MODULEPATH MODULEPATH="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules" +. "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/fish" if test -z "$__Init_Default_Modules" export __Init_Default_Modules=1; diff --git a/init/lmod/ksh b/init/lmod/ksh index 71dc29542f..7d9a05d688 100644 --- a/init/lmod/ksh +++ b/init/lmod/ksh @@ -1,8 +1,10 @@ +# Choose an EESSI CVMFS repository +EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO:-/cvmfs/software.eessi.io}" # Choose an EESSI version EESSI_VERSION="${EESSI_VERSION:-2023.06}" # Path to top-level module tree -export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules -. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/ksh +export MODULEPATH="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules" +. "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/ksh" if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; diff --git a/init/lmod/zsh b/init/lmod/zsh index 5f605579c8..bc6e8e4deb 100644 --- a/init/lmod/zsh +++ b/init/lmod/zsh @@ -1,8 +1,10 @@ +# Choose an EESSI CVMFS repository +EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO:-/cvmfs/software.eessi.io}" # Choose an EESSI version EESSI_VERSION="${EESSI_VERSION:-2023.06}" # Path to top-level module tree -export MODULEPATH=/cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/init/modules -. /cvmfs/software.eessi.io/versions/"$EESSI_VERSION"/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh +export MODULEPATH="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules" +. "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh" if [ -z "$__Init_Default_Modules" ]; then export __Init_Default_Modules=1; From 33775dc20e0fa336c107aca016e291f836fc3c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:00:28 +0100 Subject: [PATCH 1600/1795] remove curly brackets, not supported in fish --- init/lmod/fish | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init/lmod/fish b/init/lmod/fish index e0c3b58b31..1098108b1b 100644 --- a/init/lmod/fish +++ b/init/lmod/fish @@ -3,8 +3,8 @@ set EESSI_CVMFS_REPO (set -q EESSI_CVMFS_REPO; and echo "$EESSI_CVMFS_REPO"; or # Choose an EESSI version set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") # Path to top-level module tree -set -x MODULEPATH MODULEPATH="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules" -. "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/fish" +set -x MODULEPATH MODULEPATH="$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/init/modules" +. "$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/compat/linux/$(uname -m)/usr/share/Lmod/init/fish" if test -z "$__Init_Default_Modules" export __Init_Default_Modules=1; From 8ef745d267e14e5847f542d881e0d7920bdb7b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:10:20 +0100 Subject: [PATCH 1601/1795] restore correct syntax for fish command substitution --- init/lmod/fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/lmod/fish b/init/lmod/fish index 1098108b1b..aceee3f725 100644 --- a/init/lmod/fish +++ b/init/lmod/fish @@ -4,7 +4,7 @@ set EESSI_CVMFS_REPO (set -q EESSI_CVMFS_REPO; and echo "$EESSI_CVMFS_REPO"; or set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") # Path to top-level module tree set -x MODULEPATH MODULEPATH="$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/init/modules" -. "$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/compat/linux/$(uname -m)/usr/share/Lmod/init/fish" +. "$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/compat/linux/(uname -m)/usr/share/Lmod/init/fish" if test -z "$__Init_Default_Modules" export __Init_Default_Modules=1; From 37f6f02d4dba950b2592ca5e851b6d867342d78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:13:25 +0100 Subject: [PATCH 1602/1795] fix syntax for strings with variables --- init/lmod/fish | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init/lmod/fish b/init/lmod/fish index aceee3f725..9e85ba8610 100644 --- a/init/lmod/fish +++ b/init/lmod/fish @@ -3,8 +3,8 @@ set EESSI_CVMFS_REPO (set -q EESSI_CVMFS_REPO; and echo "$EESSI_CVMFS_REPO"; or # Choose an EESSI version set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") # Path to top-level module tree -set -x MODULEPATH MODULEPATH="$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/init/modules" -. "$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/compat/linux/(uname -m)/usr/share/Lmod/init/fish" +set -x MODULEPATH MODULEPATH="$EESSI_CVMFS_REPO"/versions/"$EESSI_VERSION"/init/modules +. "$EESSI_CVMFS_REPO}"/versions/"$EESSI_VERSION"/compat/linux/(uname -m)/usr/share/Lmod/init/fish if test -z "$__Init_Default_Modules" export __Init_Default_Modules=1; From a8baf5bb666e7f8a7d43c18fc1152a1cf63694be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:17:20 +0100 Subject: [PATCH 1603/1795] fix typo --- init/lmod/fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/lmod/fish b/init/lmod/fish index 9e85ba8610..c7d729368f 100644 --- a/init/lmod/fish +++ b/init/lmod/fish @@ -4,7 +4,7 @@ set EESSI_CVMFS_REPO (set -q EESSI_CVMFS_REPO; and echo "$EESSI_CVMFS_REPO"; or set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") # Path to top-level module tree set -x MODULEPATH MODULEPATH="$EESSI_CVMFS_REPO"/versions/"$EESSI_VERSION"/init/modules -. "$EESSI_CVMFS_REPO}"/versions/"$EESSI_VERSION"/compat/linux/(uname -m)/usr/share/Lmod/init/fish +. "$EESSI_CVMFS_REPO"/versions/"$EESSI_VERSION"/compat/linux/(uname -m)/usr/share/Lmod/init/fish if test -z "$__Init_Default_Modules" export __Init_Default_Modules=1; From c0243806af1c42af534be5fffb8826a2446e8c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:22:09 +0100 Subject: [PATCH 1604/1795] fix copy-paste error --- init/lmod/fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/lmod/fish b/init/lmod/fish index c7d729368f..46cd7cacf4 100644 --- a/init/lmod/fish +++ b/init/lmod/fish @@ -3,7 +3,7 @@ set EESSI_CVMFS_REPO (set -q EESSI_CVMFS_REPO; and echo "$EESSI_CVMFS_REPO"; or # Choose an EESSI version set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06") # Path to top-level module tree -set -x MODULEPATH MODULEPATH="$EESSI_CVMFS_REPO"/versions/"$EESSI_VERSION"/init/modules +set -x MODULEPATH "$EESSI_CVMFS_REPO"/versions/"$EESSI_VERSION"/init/modules . "$EESSI_CVMFS_REPO"/versions/"$EESSI_VERSION"/compat/linux/(uname -m)/usr/share/Lmod/init/fish if test -z "$__Init_Default_Modules" From e214f53a98ebdd56e0282ba06f1650f8e100d76e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:45:26 +0100 Subject: [PATCH 1605/1795] remove version number from EESSI-extend easyconfig filename --- EESSI-extend-2023.06-easybuild.eb => EESSI-extend-easybuild.eb | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename EESSI-extend-2023.06-easybuild.eb => EESSI-extend-easybuild.eb (100%) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-easybuild.eb similarity index 100% rename from EESSI-extend-2023.06-easybuild.eb rename to EESSI-extend-easybuild.eb From 341bcbf0ec2fd16d73c80c2ada80286a62355212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:46:13 +0100 Subject: [PATCH 1606/1795] determine version with $EESSI_VERSION --- EESSI-extend-easybuild.eb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-extend-easybuild.eb b/EESSI-extend-easybuild.eb index bfe7931c8f..21b62461bf 100644 --- a/EESSI-extend-easybuild.eb +++ b/EESSI-extend-easybuild.eb @@ -1,7 +1,8 @@ easyblock = 'Bundle' name = 'EESSI-extend' -version = '2023.06' +import os +version = os.getenv('EESSI_VERSION', '2023.06') # May have different ways to extend EESSI in future (manually, other tools,...) versionsuffix = '-easybuild' From 20ac33b9e3689f99e4cdbf4277db06463e731dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:48:30 +0100 Subject: [PATCH 1607/1795] use renamed easyconfig without version number in filename --- load_eessi_extend_module.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index 62b6e3f3ae..aa8f659ef1 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -93,7 +93,7 @@ else fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" # while always adding --try-amend=keep... may do no harm, we could make # an attempt to figure out if it is needed, e.g., when we are rebuilding - ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" --try-amend=keeppreviousinstall=True 2>&1 | tee ${eb_install_out} + ${EB} "EESSI-extend-easybuild.eb" --try-amend=keeppreviousinstall=True 2>&1 | tee ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" ) From 7b4e4961325772c0d14484b9e78daa685dd2e23d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 11:56:21 +0100 Subject: [PATCH 1608/1795] remove 2023.06 from EESSI-extend easyconfig filename --- .../2023.06/eessi-2023.06-eb-4.9.1-001-system.yml | 2 +- .../rebuilds/20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml | 2 +- .../2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml | 2 +- .../2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml | 2 +- .../2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml index 866766a36b..748ecc8a02 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -2,7 +2,7 @@ easyconfigs: - EasyBuild-4.9.1.eb: options: from-pr: 20299 - - EESSI-extend-2023.06-easybuild.eb + - EESSI-extend-easybuild.eb - EasyBuild-4.9.2.eb: options: from-pr: 20818 diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml index 93c4950fc8..2e0030b05a 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-EESSI-extend-allow-loaded.yml @@ -2,4 +2,4 @@ # The module is an EasyBuild created module and therefore needs to be an allowed # module when running EasyBuild easyconfigs: - - EESSI-extend-2023.06-easybuild.eb + - EESSI-extend-easybuild.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml index 9cd1b451cd..5c1eb292e4 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml @@ -2,5 +2,5 @@ # EESSI-extend did not support LMOD_EXACT_MATCH # (see https://github.com/EESSI/software-layer/pull/747) easyconfigs: - - EESSI-extend-2023.06-easybuild.eb + - EESSI-extend-easybuild.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml index 5491ef8427..b1f69e3297 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241008-eb-4.9.4-EESSI-extend.yml @@ -1,5 +1,5 @@ # 2024.10.08 # EESSI-extend should use EESSI_SITE_INSTALLPATH, instead of recalculating this easyconfigs: - - EESSI-extend-2023.06-easybuild.eb + - EESSI-extend-easybuild.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml index e4c658784f..9b2910a83a 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml @@ -3,4 +3,4 @@ # EESSI-extend should adjust EASYBUILD_INSTALLPATH and set # EASYBUILD_CUDA_COMPUTE_CAPABILITIES easyconfigs: - - EESSI-extend-2023.06-easybuild.eb + - EESSI-extend-easybuild.eb From d80dd712209f0c31df1eca9cf6f8fc938ddd9752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 12:03:52 +0100 Subject: [PATCH 1609/1795] Yasm instead of yasm --- EESSI-extend-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-easybuild.eb b/EESSI-extend-easybuild.eb index 21b62461bf..d5921b5762 100644 --- a/EESSI-extend-easybuild.eb +++ b/EESSI-extend-easybuild.eb @@ -41,7 +41,7 @@ toolchain = SYSTEM # All the dependencies we filter in EESSI local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib" -local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''} +local_arch_specific_deps_to_filter = {'aarch64': ',Yasm', 'x86_64': ''} local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH] # Set the universal EasyBuild variables From 79d698a121e876bb1f7e08cc88804bf31e995df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 12:04:32 +0100 Subject: [PATCH 1610/1795] also filter Yasm for riscv64 --- EESSI-extend-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-easybuild.eb b/EESSI-extend-easybuild.eb index d5921b5762..f74f36aca7 100644 --- a/EESSI-extend-easybuild.eb +++ b/EESSI-extend-easybuild.eb @@ -41,7 +41,7 @@ toolchain = SYSTEM # All the dependencies we filter in EESSI local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib" -local_arch_specific_deps_to_filter = {'aarch64': ',Yasm', 'x86_64': ''} +local_arch_specific_deps_to_filter = {'aarch64': ',Yasm', 'riscv64': ',Yasm', 'x86_64': ''} local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH] # Set the universal EasyBuild variables From 2d0200132950253b82d47c2c92db593bce136cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 13:11:43 +0100 Subject: [PATCH 1611/1795] rebuild EESSI-extend --- .../2023.06/rebuilds/20241206-eb-4.9.4-EESSI-extend.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20241206-eb-4.9.4-EESSI-extend.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241206-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241206-eb-4.9.4-EESSI-extend.yml new file mode 100644 index 0000000000..04a60f595c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241206-eb-4.9.4-EESSI-extend.yml @@ -0,0 +1,6 @@ +# 2024.12.06 +# - Use $EESSI_VERSION to determine version of EESSI-extend module +# - Fix the filtered dependencies for aarch64: Yasm instead of yasm +# - Also add filtered dependencies for RISC-V +easyconfigs: + - EESSI-extend-easybuild.eb From 1ae2c0c5070f9383852ad0416bbdaee7ab3b6bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 13:24:14 +0100 Subject: [PATCH 1612/1795] use REPOSITORY_* variables for repo id, version, name, and use them throughout the entire script --- bot/build.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 81b3ef4660..b674acfd44 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -125,7 +125,9 @@ else fi # determine repository to be used from entry .repository in ${JOB_CFG_FILE} -REPOSITORY=$(cfg_get_value "repository" "repo_id") +REPOSITORY_ID=$(cfg_get_value "repository" "repo_id") +REPOSITORY_NAME=$(cfg_get_value "repository" "repo_name") +REPOSITORY_VERSION=$(cfg_get_value "repository" "repo_version") EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" @@ -134,13 +136,13 @@ echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE # here, just set & export EESSI_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +export EESSI_VERSION_OVERRIDE=${REPOSITORY_VERSION} echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME} echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine CPU architecture to be used from entry .architecture in ${JOB_CFG_FILE} @@ -169,11 +171,11 @@ COMMON_ARGS+=("--mode" "run") [[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") [[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") [[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") -[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") +[[ ! -z ${REPOSITORY_ID} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY_ID}") # Also expose software.eessi.io when configured for dev.eessi.io # Need software.eessi.io for the compat layer -if [[ "${REPOSITORY}" == dev.eessi.io ]]; then +if [[ "${REPOSITORY_NAME}" == "dev.eessi.io" ]]; then COMMON_ARGS+=("--repository" "software.eessi.io,access=ro") fi From e0b98ad4cef008b7563a429375ceb100731517b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 13:26:20 +0100 Subject: [PATCH 1613/1795] don't install CUDA on RISC-V build hosts --- bot/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index b674acfd44..01f971dde0 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -193,6 +193,12 @@ fi [[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") [[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") +# Skip CUDA installation for riscv.eessi.io +if [[ "${REPOSITORY_NAME}" == "riscv.eessi.io" ]]; then + echo "bot/build.sh: disabling CUDA installation for RISC-V repository (${REPOSITORY_NAME})" + INSTALL_SCRIPT_ARGS+=("--skip-cuda-install") +fi + # determine if the removal step has to be run # assume there's only one diff file that corresponds to the PR patch file pr_diff=$(ls [0-9]*.diff | head -1) From 2938e3cf3f21a10cb4051dbc6c12aa8c439aa876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 13:35:00 +0100 Subject: [PATCH 1614/1795] use $JOB_STORAGE as base dir for $SINGULARITY_TMPDIR --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 01f971dde0..29444a32c2 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -110,7 +110,7 @@ echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") export SINGULARITY_HOME="${PWD}:/eessi_bot_job" -export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" +export SINGULARITY_TMPDIR="${JOB_STORAGE}/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty From 5927dae59fc926333e5d05c7445ba8b2c6f72619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 13:38:40 +0100 Subject: [PATCH 1615/1795] use $JOB_STORAGE as base dir for $SINGULARITY_TMPDIR, if it is defined; fall back to $PWD otherwise --- bot/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/test.sh b/bot/test.sh index d3f3630ea8..820618b477 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -130,7 +130,7 @@ echo "bot/test.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") export SINGULARITY_HOME="${PWD}:/eessi_bot_job" -export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" +export SINGULARITY_TMPDIR="${JOB_STORAGE:-${PWD}}/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty From e4c8b0419c0e0560eace89a597de99fd70b72c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Dec 2024 13:48:50 +0100 Subject: [PATCH 1616/1795] move block of code that sets $STORAGE and $JOB_STORAGE out of if statement --- bot/test.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index 820618b477..2e2c838dfa 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -103,19 +103,6 @@ fi RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -1 | awk '{print $2}') if [[ -z ${RESUME_DIR} ]]; then - echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " - # replace any env variable in ${LOCAL_TMP} with its - # current value (e.g., a value that is local to the job) - STORAGE=$(envsubst <<< ${LOCAL_TMP}) - echo "'${STORAGE}'" - - # make sure ${STORAGE} exists - mkdir -p ${STORAGE} - - # make sure the base tmp storage is unique - JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) - echo "bot/test.sh: created unique base tmp storage directory at ${JOB_STORAGE}" - RESUME_TGZ=${PWD}/previous_tmp/build_step/$(ls previous_tmp/build_step) if [[ -z ${RESUME_TGZ} ]]; then echo "bot/test.sh: no information about tmp directory and tarball of build step; --> giving up" @@ -123,6 +110,19 @@ if [[ -z ${RESUME_DIR} ]]; then fi fi +echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " +# replace any env variable in ${LOCAL_TMP} with its +# current value (e.g., a value that is local to the job) +STORAGE=$(envsubst <<< ${LOCAL_TMP}) +echo "'${STORAGE}'" + +# make sure ${STORAGE} exists +mkdir -p ${STORAGE} + +# make sure the base tmp storage is unique +JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) +echo "bot/test.sh: created unique base tmp storage directory at ${JOB_STORAGE}" + # obtain list of modules to be loaded LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") echo "bot/test.sh: LOAD_MODULES='${LOAD_MODULES}'" From 658019fa9d225c3b0dee5a34c378f2624de2aa8c Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 9 Dec 2024 09:13:06 +0000 Subject: [PATCH 1617/1795] {2023.06}[foss/2023a] Tombo v1.5.1 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index e9011a0664..3b46097a05 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -71,3 +71,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21227 from-commit: 4c5e3455dec31e68e8383c7fd86d1f80c434676d + - Tombo-1.5.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21925 + from-commit: 522ca010ab11949ab9594037f72b975cf1cd0d33 From e4997f2aaf5e67204a973583fa537de1415d5d4a Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 9 Dec 2024 11:38:10 +0000 Subject: [PATCH 1618/1795] {2023.06}[foss/2023a] Tombo v1.5.1 --- .../2023.06/eessi-2023.06-eb-4.9.2-2023a.yml | 4 ---- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 3b46097a05..e9011a0664 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -71,7 +71,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21227 from-commit: 4c5e3455dec31e68e8383c7fd86d1f80c434676d - - Tombo-1.5.1-foss-2023a.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21925 - from-commit: 522ca010ab11949ab9594037f72b975cf1cd0d33 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 7ea65ba94e..cb8f1766c3 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -37,3 +37,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20230 from-commit: 91c8df6b4c0810061e9f325427c9c79e961bc4b0 + - Tombo-1.5.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21925 + from-commit: 522ca010ab11949ab9594037f72b975cf1cd0d33 From 25bf2db12f192d505dab8564f2ad78d7ecc56519 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 9 Dec 2024 12:52:38 +0000 Subject: [PATCH 1619/1795] {2023.06}[GCC/2023b] wxWidgets v3.2.6 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 69b05fb02a..ee161b8b9c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -21,3 +21,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21899 from-commit: 0bdeb23c9ea5a3caefd353ecd936919424c1bba4 + - wxWidgets-3.2.6-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21915 + from-commit: 58f16c0caf8c5494c68e9eda8cbf19e9145d3cfa From 986c9f11b88f3feb0d53ccad0a35bb42e7d4586e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Dec 2024 09:28:40 +0100 Subject: [PATCH 1620/1795] detect and redirect RISC-V clients --- init/modules/EESSI/2023.06.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 348699c0f1..04268f30e1 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -12,6 +12,13 @@ whatis("URL: https://www.eessi.io/docs/") conflict("EESSI") local eessi_version = myModuleVersion() local eessi_repo = "/cvmfs/software.eessi.io" +if (capture("uname -m"):gsub("\n$","") == "riscv64") then + eessi_version = os.getenv("EESSI_VERSION_OVERRIDE") or "20240402" + eessi_repo = "/cvmfs/riscv.eessi.io" + LmodMessage("RISC-V architecture detected, but there is no RISC-V support yet in the production repository.\n" .. + "Automatically switching to version " .. eessi_version .. " of the RISC-V development repository " .. eessi_repo .. ".\n" .. + "For more details about this repository, see https://www.eessi.io/docs/repositories/riscv.eessi.io/.") +end local eessi_prefix = pathJoin(eessi_repo, "versions", eessi_version) local eessi_os_type = "linux" setenv("EESSI_VERSION", eessi_version) From b6439951050dde21a52be08d48b93d426d56e986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Dec 2024 09:29:16 +0100 Subject: [PATCH 1621/1795] add 20240402 symlink for RISC-V repo --- init/modules/EESSI/20240402.lua | 1 + 1 file changed, 1 insertion(+) create mode 120000 init/modules/EESSI/20240402.lua diff --git a/init/modules/EESSI/20240402.lua b/init/modules/EESSI/20240402.lua new file mode 120000 index 0000000000..cbf80d1fcd --- /dev/null +++ b/init/modules/EESSI/20240402.lua @@ -0,0 +1 @@ +2023.06.lua \ No newline at end of file From c738b64fbb73b932aeea58ea908e262e65e45eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Dec 2024 09:55:56 +0100 Subject: [PATCH 1622/1795] use subprocess instead of capture --- init/modules/EESSI/2023.06.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index 04268f30e1..eb1cd1753b 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -12,7 +12,7 @@ whatis("URL: https://www.eessi.io/docs/") conflict("EESSI") local eessi_version = myModuleVersion() local eessi_repo = "/cvmfs/software.eessi.io" -if (capture("uname -m"):gsub("\n$","") == "riscv64") then +if (subprocess("uname -m"):gsub("\n$","") == "riscv64") then eessi_version = os.getenv("EESSI_VERSION_OVERRIDE") or "20240402" eessi_repo = "/cvmfs/riscv.eessi.io" LmodMessage("RISC-V architecture detected, but there is no RISC-V support yet in the production repository.\n" .. From d61f33d20b7d9d6657b8f4bc45657d4a63a1c243 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 10 Dec 2024 18:14:51 +0100 Subject: [PATCH 1623/1795] Initial attempt for an EasyBuild hook that makes modules print an LmodError when they are loaded --- eb_hooks.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 5bf42b6bbf..b445754d83 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -77,6 +77,10 @@ def parse_hook(ec, *args, **kwargs): if ec.name in PARSE_HOOKS: PARSE_HOOKS[ec.name](ec, eprefix) + # Always trigger this one, regardless of ec.name + # TODO: limit to only zen4 in the future + parse_hook_zen4_module_only(ec, eprefix) + # inject the GPU property (if required) ec = inject_gpu_property(ec) @@ -353,6 +357,28 @@ def parse_hook_CP2K_remove_deps_for_aarch64(ec, *args, **kwargs): raise EasyBuildError("CP2K-specific hook triggered for non-CP2K easyconfig?!") +def parse_hook_zen4_module_only(ec, eprefix): + """ + Use --force --module-only if building a foss-2022b-based EasyConfig for Zen4. + This toolchain will not be supported on Zen4, so we will generate a modulefile + and have it print an LmodError. + """ + ecname, ecversion = ec['name'], ec['version'] + tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] + if ( + (tcname in ['foss', 'gompi'] and tcversion == '2022b') or + (tcname in ['GCC', 'GCCcore'] and LooseVersion(tcversion) == LooseVersion('12.2.0')) or + (ecname in ['foss', 'gompi'] and ecversion == '2022b') or + (ecname in ['GCC', 'GCCcore'] and LooseVersion(ecversion) == LooseVersion('12.2.0')) + ): + update_build_option('force', 'True') + update_build_option('module_only', 'True') + # TODO: create a docs page to which we can refer for more info here + # TODO: then update the link to the known issues page to the _specific_ issue + errmsg = "Toolchains based on GCCcore-12.2.0 are not supported for the Zen4 architecture." + errmsg += " See https://www.eessi.io/docs/known_issues/eessi-2023.06/" + ec['modluafooter'] = 'LmodError(%s)' % errmsg + def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): """ Solve issues with compiling or running the tests on both From 9bb52dcbdcc61841739acc817135ba9a293f489b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 11 Dec 2024 12:03:37 +0100 Subject: [PATCH 1624/1795] This works, but we should probably set the environment variable in one of the step-hooks, so we can unset it at the end --- eb_hooks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b445754d83..0b30db9b4d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -371,13 +371,17 @@ def parse_hook_zen4_module_only(ec, eprefix): (ecname in ['foss', 'gompi'] and ecversion == '2022b') or (ecname in ['GCC', 'GCCcore'] and LooseVersion(ecversion) == LooseVersion('12.2.0')) ): + env_varname="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220" + # TODO: I need to think about how to unset this, at the end of the build for this module + # Maybe I shouldn't do it in the parse hook, but set it in a step hook, and unset it in a another step hook? + os.environ[env_varname] = "1" update_build_option('force', 'True') update_build_option('module_only', 'True') # TODO: create a docs page to which we can refer for more info here # TODO: then update the link to the known issues page to the _specific_ issue errmsg = "Toolchains based on GCCcore-12.2.0 are not supported for the Zen4 architecture." errmsg += " See https://www.eessi.io/docs/known_issues/eessi-2023.06/" - ec['modluafooter'] = 'LmodError(%s)' % errmsg + ec['modluafooter'] = 'if (not os.getenv("%s")) then LmodError("%s") end' % (env_varname, errmsg) def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): """ From c26a1e54362ef585fae4576df5998257fbed8aec Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 11 Dec 2024 14:18:03 +0100 Subject: [PATCH 1625/1795] Only set env var for prepare step --- eb_hooks.py | 75 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0b30db9b4d..cb8e759a7a 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -35,6 +35,20 @@ EESSI_INSTALLATION_REGEX = r"^/cvmfs/[^/]*.eessi.io/versions/" HOST_INJECTIONS_LOCATION = "/cvmfs/software.eessi.io/host_injections/" +# Make sure a single environment variable name is used for this throughout the hooks +EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220" + +def is_gcccore_1220_based(ecname, ecversion, tcname, tcversion): + """Checks if this easyconfig either _is_ or _uses_ a GCCcore-12.2.2 based toolchain""" + gcccore_based_names = ['GCCcore', 'GCC'] + foss_based_names = ['gfbf', 'gompi', 'foss'] + return ( + (tcname in foss_based_names and tcversion == '2022b') or + (tcname in gcccore_based_names and LooseVersion(tcversion) == LooseVersion('12.2.0')) or + (ecname in foss_based_names and ecversion == '2022b') or + (ecname in gcccore_based_names and LooseVersion(ecversion) == LooseVersion('12.2.0')) + ) + def get_eessi_envvar(eessi_envvar): """Get an EESSI environment variable from the environment""" @@ -78,8 +92,9 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) # Always trigger this one, regardless of ec.name - # TODO: limit to only zen4 in the future - parse_hook_zen4_module_only(ec, eprefix) + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if cpu_target == CPU_TARGET_ZEN4: + parse_hook_zen4_module_only(ec, eprefix) # inject the GPU property (if required) ec = inject_gpu_property(ec) @@ -134,6 +149,11 @@ def pre_prepare_hook(self, *args, **kwargs): if self.name in PRE_PREPARE_HOOKS: PRE_PREPARE_HOOKS[self.name](self, *args, **kwargs) + # Always trigger this one, regardless of ec.name + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if cpu_target == CPU_TARGET_ZEN4: + pre_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs) + def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ @@ -179,6 +199,11 @@ def post_prepare_hook(self, *args, **kwargs): if self.name in POST_PREPARE_HOOKS: POST_PREPARE_HOOKS[self.name](self, *args, **kwargs) + # Always trigger this one, regardless of ec.name + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if cpu_target == CPU_TARGET_ZEN4: + post_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs) + def parse_hook_casacore_disable_vectorize(ec, eprefix): """ @@ -363,26 +388,44 @@ def parse_hook_zen4_module_only(ec, eprefix): This toolchain will not be supported on Zen4, so we will generate a modulefile and have it print an LmodError. """ - ecname, ecversion = ec['name'], ec['version'] - tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version'] - if ( - (tcname in ['foss', 'gompi'] and tcversion == '2022b') or - (tcname in ['GCC', 'GCCcore'] and LooseVersion(tcversion) == LooseVersion('12.2.0')) or - (ecname in ['foss', 'gompi'] and ecversion == '2022b') or - (ecname in ['GCC', 'GCCcore'] and LooseVersion(ecversion) == LooseVersion('12.2.0')) - ): - env_varname="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220" - # TODO: I need to think about how to unset this, at the end of the build for this module - # Maybe I shouldn't do it in the parse hook, but set it in a step hook, and unset it in a another step hook? - os.environ[env_varname] = "1" + if is_gcccore_1220_based(ec['name'], ec['version'], ec['toolchain']['name'], ec['toolchain']['version']): + env_varname=EESSI_IGNORE_ZEN4_GCC1220_ENVVAR update_build_option('force', 'True') update_build_option('module_only', 'True') # TODO: create a docs page to which we can refer for more info here # TODO: then update the link to the known issues page to the _specific_ issue - errmsg = "Toolchains based on GCCcore-12.2.0 are not supported for the Zen4 architecture." - errmsg += " See https://www.eessi.io/docs/known_issues/eessi-2023.06/" + # Need to escape newline character so that the newline character actually ends up in the module file + # (otherwise, it splits the string, and a 2-line string ends up in the modulefile, resulting in syntax error) + errmsg = "EasyConfigs using toolchains based on GCCcore-12.2.0 are not supported for the Zen4 architecture.\\n" + errmsg += "See https://www.eessi.io/docs/known_issues/eessi-2023.06/" ec['modluafooter'] = 'if (not os.getenv("%s")) then LmodError("%s") end' % (env_varname, errmsg) + +def pre_fetch_hook(ec, *args, **kwargs): + """Main parse hook: trigger custom functions based on software name.""" + + if ec.name in PRE_FETCH_HOOKS: + PRE_FETCH_HOOKS[ec.name](ec, *args, **kwargs) + + # Always trigger this one, regardless of ec.name + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if cpu_target == CPU_TARGET_ZEN4: + pre_fetch_hook_ignore_zen4_gcccore1220_error(ec, *args, **kwargs) + + +# We do this as early as possible - and remove it all the way in the last step hook (post_testcases_hook) +def pre_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): + """Set environment variable to ignore the LmodError from parse_hook_zen4_module_only during build phase""" + if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): + os.environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR] = "1" + + +def post_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): + """Unset environment variable to ignore the LmodError from parse_hook_zen4_module_only during build phase""" + if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): + del os.environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR] + + def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): """ Solve issues with compiling or running the tests on both From 600adee5699660b5aa79f5d2c19c07ae8e3864ea Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 11 Dec 2024 14:28:51 +0100 Subject: [PATCH 1626/1795] Remove unused code --- eb_hooks.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index cb8e759a7a..f9580b46d0 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -401,18 +401,6 @@ def parse_hook_zen4_module_only(ec, eprefix): ec['modluafooter'] = 'if (not os.getenv("%s")) then LmodError("%s") end' % (env_varname, errmsg) -def pre_fetch_hook(ec, *args, **kwargs): - """Main parse hook: trigger custom functions based on software name.""" - - if ec.name in PRE_FETCH_HOOKS: - PRE_FETCH_HOOKS[ec.name](ec, *args, **kwargs) - - # Always trigger this one, regardless of ec.name - cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if cpu_target == CPU_TARGET_ZEN4: - pre_fetch_hook_ignore_zen4_gcccore1220_error(ec, *args, **kwargs) - - # We do this as early as possible - and remove it all the way in the last step hook (post_testcases_hook) def pre_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): """Set environment variable to ignore the LmodError from parse_hook_zen4_module_only during build phase""" From b3fade96749ea22e867ffac8ecd850edae95ef40 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 11 Dec 2024 15:40:19 +0100 Subject: [PATCH 1627/1795] Use step hooks to set and unset force and module-only --- eb_hooks.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index f9580b46d0..ddfbc37692 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -29,6 +29,8 @@ CPU_TARGET_ZEN4 = 'x86_64/amd/zen4' EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' +EESSI_MODULE_ONLY_ATTR = 'orig_module_only' +EESSI_FORCE_ATTR = 'orig_force' SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] @@ -389,9 +391,9 @@ def parse_hook_zen4_module_only(ec, eprefix): and have it print an LmodError. """ if is_gcccore_1220_based(ec['name'], ec['version'], ec['toolchain']['name'], ec['toolchain']['version']): - env_varname=EESSI_IGNORE_ZEN4_GCC1220_ENVVAR - update_build_option('force', 'True') - update_build_option('module_only', 'True') + env_varname = EESSI_IGNORE_ZEN4_GCC1220_ENVVAR + # update_build_option('force', 'True') + # update_build_option('module_only', 'True') # TODO: create a docs page to which we can refer for more info here # TODO: then update the link to the known issues page to the _specific_ issue # Need to escape newline character so that the newline character actually ends up in the module file @@ -401,6 +403,56 @@ def parse_hook_zen4_module_only(ec, eprefix): ec['modluafooter'] = 'if (not os.getenv("%s")) then LmodError("%s") end' % (env_varname, errmsg) +def pre_fetch_hook(self, *args, **kwargs): + """Main pre fetch hook: trigger custom functions based on software name.""" + if self.name in PRE_FETCH_HOOKS: + PRE_FETCH_HOOKS[ec.name](self, *args, **kwargs) + + # Always trigger this one, regardless of self.name + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if cpu_target == CPU_TARGET_ZEN4: + pre_fetch_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs) + + +def pre_fetch_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): + """Use --force --module-only if building a foss-2022b-based EasyConfig for Zen4. + This toolchain will not be supported on Zen4, so we will generate a modulefile + and have it print an LmodError. + """ + if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): + if hasattr(self, EESSI_MODULE_ONLY_ATTR): + raise EasyBuildError("'self' already has attribute %s! Can't use pre_fetch hook.", + EESSI_MODULE_ONLY_ATTR) + setattr(self, EESSI_MODULE_ONLY_ATTR, build_option('module_only')) + update_build_option('module_only', 'True') + print_msg("Updated build option 'module-only' to 'True'") + + if hasattr(self, EESSI_FORCE_ATTR): + raise EasyBuildError("'self' already has attribute %s! Can't use pre_fetch hook.", + EESSI_FORCE_ATTR) + setattr(self, EESSI_FORCE_ATTR, build_option('force')) + update_build_option('force', 'True') + print_msg("Updated build option 'force' to 'True'") + + +def post_module_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): + """Revert changes from pre_fetch_hook_ignore_zen4_gcccore1220_error""" + if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): + if hasattr(self, EESSI_MODULE_ONLY_ATTR): + update_build_option('module_only', getattr(self, EESSI_MODULE_ONLY_ATTR)) + print_msg("Restored original build option 'module_only' to %s" % getattr(self, EESSI_MODULE_ONLY_ATTR)) + else: + raise EasyBuildError("Cannot restore module_only to it's original value: 'self' is missing attribute %s.", + EESSI_MODULE_ONLY_ATTR) + + if hasattr(self, EESSI_FORCE_ATTR): + update_build_option('force', getattr(self, EESSI_FORCE_ATTR)) + print_msg("Restored original build option 'force' to %s" % getattr(self, EESSI_FORCE_ATTR)) + else: + raise EasyBuildError("Cannot restore force to it's original value: 'self' is misisng attribute %s.", + EESSI_FORCE_ATTR) + + # We do this as early as possible - and remove it all the way in the last step hook (post_testcases_hook) def pre_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): """Set environment variable to ignore the LmodError from parse_hook_zen4_module_only during build phase""" @@ -1028,6 +1080,17 @@ def inject_gpu_property(ec): return ec +def post_module_hook(self, *args, **kwargs): + """Main post module hook: trigger custom functions based on software name.""" + if self.name in POST_MODULE_HOOKS: + POST_MODULE_HOOKS[ec.name](self, *args, **kwargs) + + # Always trigger this one, regardless of self.name + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if cpu_target == CPU_TARGET_ZEN4: + post_module_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs) + + PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, @@ -1042,6 +1105,8 @@ def inject_gpu_property(ec): 'UCX': parse_hook_ucx_eprefix, } +PRE_FETCH_HOOKS = {} + PRE_PREPARE_HOOKS = { 'Highway': pre_prepare_hook_highway_handle_test_compilation_issues, } @@ -1087,3 +1152,5 @@ def inject_gpu_property(ec): 'CUDA': post_postproc_cuda, 'cuDNN': post_postproc_cudnn, } + +POST_MODULE_HOOKS = {} From 6a1700ef648e8e0026a352aa63f4e855bc2d664c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 11 Dec 2024 15:46:07 +0100 Subject: [PATCH 1628/1795] Rename hooks to make more sense --- eb_hooks.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index ddfbc37692..3223286f82 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -392,8 +392,6 @@ def parse_hook_zen4_module_only(ec, eprefix): """ if is_gcccore_1220_based(ec['name'], ec['version'], ec['toolchain']['name'], ec['toolchain']['version']): env_varname = EESSI_IGNORE_ZEN4_GCC1220_ENVVAR - # update_build_option('force', 'True') - # update_build_option('module_only', 'True') # TODO: create a docs page to which we can refer for more info here # TODO: then update the link to the known issues page to the _specific_ issue # Need to escape newline character so that the newline character actually ends up in the module file @@ -411,10 +409,10 @@ def pre_fetch_hook(self, *args, **kwargs): # Always trigger this one, regardless of self.name cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if cpu_target == CPU_TARGET_ZEN4: - pre_fetch_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs) + pre_fetch_hook_zen4_gcccore1220(self, *args, **kwargs) -def pre_fetch_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): +def pre_fetch_hook_zen4_gcccore1220(self, *args, **kwargs): """Use --force --module-only if building a foss-2022b-based EasyConfig for Zen4. This toolchain will not be supported on Zen4, so we will generate a modulefile and have it print an LmodError. @@ -435,8 +433,8 @@ def pre_fetch_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): print_msg("Updated build option 'force' to 'True'") -def post_module_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): - """Revert changes from pre_fetch_hook_ignore_zen4_gcccore1220_error""" +def post_module_hook_zen4_gcccore1220(self, *args, **kwargs): + """Revert changes from pre_fetch_hook_zen4_gcccore1220""" if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): if hasattr(self, EESSI_MODULE_ONLY_ATTR): update_build_option('module_only', getattr(self, EESSI_MODULE_ONLY_ATTR)) @@ -1088,7 +1086,7 @@ def post_module_hook(self, *args, **kwargs): # Always trigger this one, regardless of self.name cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if cpu_target == CPU_TARGET_ZEN4: - post_module_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs) + post_module_hook_zen4_gcccore1220(self, *args, **kwargs) PARSE_HOOKS = { From f93855743d4eda38ff8f35dc594f1ff57b396edd Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 5 Jan 2025 16:21:09 +0100 Subject: [PATCH 1629/1795] add support for alternative artefacts check --- bot/check-build.sh | 287 ++++++++++++++++++++++++--------------------- 1 file changed, 153 insertions(+), 134 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 41aeab258e..85816519ef 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -8,6 +8,7 @@ # https://github.com/EESSI/software-layer.git # # author: Thomas Roeblitz (@trz42) +# author: Samuel Moors (@smoors) # # license: GPLv2 # @@ -59,10 +60,12 @@ display_help() { echo " OPTIONS:" echo " -h | --help - display this usage information [default: false]" echo " -v | --verbose - display more information [default: false]" + echo " -a | --alt-artefacts - alternative artefacts check (sources file check-build-artefacts.sh if exists) [default: false]" } # set defaults for command line arguments VERBOSE=0 +ALT_ARTEFACTS=0 POSITIONAL_ARGS=() @@ -76,6 +79,10 @@ while [[ $# -gt 0 ]]; do VERBOSE=1 shift 1 ;; + -a|--alt-artefacts) + ALT_ARTEFACTS=1 + shift 1 + ;; --) shift POSITIONAL_ARGS+=("$@") # save positional args @@ -157,20 +164,22 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi -TGZ=-1 -TARBALL= -if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then - GP_tgz_created="\.tar\.gz created!" - grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) - if [[ $? -eq 0 ]]; then - TGZ=1 - TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') - else - TGZ=0 - fi - # have to be careful to not add searched for pattern into slurm out file - [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" - [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +if [[ $ALT_ARTEFACTS -eq 0 ]]; then + TGZ=-1 + TARBALL= + if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + GP_tgz_created="\.tar\.gz created!" + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) + if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') + else + TGZ=0 + fi + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + fi fi [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" @@ -180,7 +189,9 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +if [[ $ALT_ARTEFACTS -eq 0 ]]; then + [[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +fi # Here, we try to do some additional analysis on the output file # to see if we can print a more clear 'reason' for the failure @@ -208,8 +219,8 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ [[ ${FAILED} -eq 0 ]] && \ [[ ${MISSING} -eq 0 ]] && \ [[ ${NO_MISSING} -eq 1 ]] && \ - [[ ${TGZ} -eq 1 ]] && \ - [[ ! -z ${TARBALL} ]]; then + [[ $ALT_ARTEFACTS -ne 0 || ${TGZ} -eq 1 ]] && \ + [[ $ALT_ARTEFACTS -ne 0 || -n ${TARBALL} ]]; then # SUCCESS status="SUCCESS" reason="" @@ -417,127 +428,142 @@ success_msg="found message(s) matching ${GP_no_missing}" failure_msg="no message matching ${GP_no_missing}" comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") -success_msg="found message matching ${GP_tgz_created}" -failure_msg="no message matching ${GP_tgz_created}" -comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +if [[ $ALT_ARTEFACTS -eq 0 ]]; then + success_msg="found message matching ${GP_tgz_created}" + failure_msg="no message matching ${GP_tgz_created}" + comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +fi # Now, do the actual replacement of __DETAILS_FMT__ comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} -# first construct comment_artefacts_list -# then use it to set comment_artefacts -comment_artifacts_list="" - -# TARBALL should only contain a single tarball -if [[ ! -z ${TARBALL} ]]; then - # Example of the detailed information for a tarball. The actual result MUST be a - # single line (no '\n') or it would break the structure of the markdown table - # that holds status updates of a bot job. - # - #
    - #
    - # eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz - # size: 234 MiB (245366784 bytes)
    - # entries: 1234
    - # modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_
    - #
    -    #       GCC/9.3.0.lua
    - # GCC/10.3.0.lua
    - # OpenSSL/1.1.lua - #
    - # software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_ - #
    -    #       GCC/9.3.0/
    - # CMake/3.20.1-GCCcore-10.3.0/
    - # OpenMPI/4.1.1-GCC-10.3.0/ - #
    - # other under _2023.06/software/linux/x86_64/intel/cascadelake/_ - #
    -    #       .lmod/cache/spiderT.lua
    - # .lmod/cache/spiderT.luac_5.1
    - # .lmod/cache/timestamp - #
    - #
    - #
    - size="$(stat --dereference --printf=%s ${TARBALL})" - size_mib=$((${size} >> 20)) - tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) - tar tf ${TARBALL} > ${tmpfile} - entries=$(cat ${tmpfile} | wc -l) - # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE - # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512 - # cfg/job.cfg contains (only the attributes to be used are shown below): - # [repository] - # repo_version = 2023.06 - # [architecture] - # os_type = linux - # software_subdir = x86_64/intel/skylake_avx512 - repo_version=$(cfg_get_value "repository" "repo_version") - os_type=$(cfg_get_value "architecture" "os_type") - software_subdir=$(cfg_get_value "architecture" "software_subdir") - accelerator=$(cfg_get_value "architecture" "accelerator") - prefix="${repo_version}/software/${os_type}/${software_subdir}" - - # if we build for an accelerator, the prefix is different - if [[ ! -z ${accelerator} ]]; then - prefix="${prefix}/accel/${accelerator}" - fi - - # extract directories/entries from tarball content - modules_entries=$(grep "${prefix}/modules" ${tmpfile}) - software_entries=$(grep "${prefix}/software" ${tmpfile}) - other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") - other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) - modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) - software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) - - artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" +if [[ $ALT_ARTEFACTS -eq 0 ]]; then + # first construct comment_artefacts_list + # then use it to set comment_artefacts comment_artifacts_list="" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" - comment_artifacts_list="${comment_artifacts_list}
    "
    -    if [[ ! -z ${modules} ]]; then
    -        while IFS= read -r mod ; do
    -            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
    -        done <<< "${modules}"
    -    else
    -        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
    -    fi
    -    comment_artifacts_list="${comment_artifacts_list}
    " - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" - comment_artifacts_list="${comment_artifacts_list}
    "
    -    if [[ ! -z ${software_pkgs} ]]; then
    -        while IFS= read -r sw_pkg ; do
    -            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
    -        done <<< "${software_pkgs}"
    -    else
    -        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
    -    fi
    -    comment_artifacts_list="${comment_artifacts_list}
    " - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})" - comment_artifacts_list="${comment_artifacts_list}
    "
    -    if [[ ! -z ${other_shortened} ]]; then
    -        while IFS= read -r other ; do
    -            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
    -        done <<< "${other_shortened}"
    +
    +    # TARBALL should only contain a single tarball
    +    if [[ ! -z ${TARBALL} ]]; then
    +        # Example of the detailed information for a tarball. The actual result MUST be a
    +        # single line (no '\n') or it would break the structure of the markdown table
    +        # that holds status updates of a bot job.
    +        # 
    +        # 
    + #
    + # eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz + # size: 234 MiB (245366784 bytes)
    + # entries: 1234
    + # modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_
    + #
    +        #       GCC/9.3.0.lua
    + # GCC/10.3.0.lua
    + # OpenSSL/1.1.lua + #
    + # software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_ + #
    +        #       GCC/9.3.0/
    + # CMake/3.20.1-GCCcore-10.3.0/
    + # OpenMPI/4.1.1-GCC-10.3.0/ + #
    + # other under _2023.06/software/linux/x86_64/intel/cascadelake/_ + #
    +        #       .lmod/cache/spiderT.lua
    + # .lmod/cache/spiderT.luac_5.1
    + # .lmod/cache/timestamp + #
    + #
    + #
    + size="$(stat --dereference --printf=%s ${TARBALL})" + size_mib=$((${size} >> 20)) + tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) + tar tf ${TARBALL} > ${tmpfile} + entries=$(cat ${tmpfile} | wc -l) + # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE + # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512 + # cfg/job.cfg contains (only the attributes to be used are shown below): + # [repository] + # repo_version = 2023.06 + # [architecture] + # os_type = linux + # software_subdir = x86_64/intel/skylake_avx512 + repo_version=$(cfg_get_value "repository" "repo_version") + os_type=$(cfg_get_value "architecture" "os_type") + software_subdir=$(cfg_get_value "architecture" "software_subdir") + accelerator=$(cfg_get_value "architecture" "accelerator") + prefix="${repo_version}/software/${os_type}/${software_subdir}" + + # if we build for an accelerator, the prefix is different + if [[ ! -z ${accelerator} ]]; then + prefix="${prefix}/accel/${accelerator}" + fi + + # extract directories/entries from tarball content + modules_entries=$(grep "${prefix}/modules" ${tmpfile}) + software_entries=$(grep "${prefix}/software" ${tmpfile}) + other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") + other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) + modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) + software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) + + artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" + comment_artifacts_list="" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})" + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" + comment_artifacts_list="${comment_artifacts_list}
    "
    +        if [[ ! -z ${modules} ]]; then
    +            while IFS= read -r mod ; do
    +                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
    +            done <<< "${modules}"
    +        else
    +            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
    +        fi
    +        comment_artifacts_list="${comment_artifacts_list}
    " + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" + comment_artifacts_list="${comment_artifacts_list}
    "
    +        if [[ ! -z ${software_pkgs} ]]; then
    +            while IFS= read -r sw_pkg ; do
    +                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
    +            done <<< "${software_pkgs}"
    +        else
    +            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
    +        fi
    +        comment_artifacts_list="${comment_artifacts_list}
    " + comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})" + comment_artifacts_list="${comment_artifacts_list}
    "
    +        if [[ ! -z ${other_shortened} ]]; then
    +            while IFS= read -r other ; do
    +                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
    +            done <<< "${other_shortened}"
    +        else
    +            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
    +        fi
    +        comment_artifacts_list="${comment_artifacts_list}
    " else - comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')" + comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')" fi - comment_artifacts_list="${comment_artifacts_list}
    " -else - comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')" -fi -comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " -comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" -comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}" + comment_artefact_details_fmt="
    __ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
    " + comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" + comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}" + + comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " + comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" + comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} -comment_artefacts_fmt="
    _Artefacts_
    __ARTEFACTS_LIST__
    " -comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" -comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} + echo "artefacts = " >> ${job_result_file} + echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} + + # remove tmpfile + if [[ -f ${tmpfile} ]]; then + rm ${tmpfile} + fi + +elif [[ -f "$TOPDIR/check-build-artefacts.sh" ]]; then + source "$TOPDIR/check-build-artefacts.sh" +fi echo "${comment_description}" >> ${job_result_file} @@ -545,13 +571,6 @@ echo "${comment_description}" >> ${job_result_file} # - this should make use of subsequent steps such as deploying a tarball more # efficient echo "status = ${status}" >> ${job_result_file} -echo "artefacts = " >> ${job_result_file} -echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} - -# remove tmpfile -if [[ -f ${tmpfile} ]]; then - rm ${tmpfile} -fi # exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1) test "${status}" == "SUCCESS" From 78542d1dfdaee9f56f5bd8fa49ea375c9279a3d4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 Jan 2025 10:33:03 +0100 Subject: [PATCH 1630/1795] Processed Alan's comments --- eb_hooks.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 3223286f82..59e19e4f3c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -40,8 +40,23 @@ # Make sure a single environment variable name is used for this throughout the hooks EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220" -def is_gcccore_1220_based(ecname, ecversion, tcname, tcversion): - """Checks if this easyconfig either _is_ or _uses_ a GCCcore-12.2.2 based toolchain""" +def is_gcccore_1220_based(**kwargs): +# ecname, ecversion, tcname, tcversion): + """ + Checks if this easyconfig either _is_ or _uses_ a GCCcore-12.2.0 based toolchain. + This function is, for example, used to generate errors in GCCcore-12.2.0 based modules for the zen4 architecture + since zen4 is not fully supported with that toolchain. + + :param str ecname: Name of the software specified in the EasyConfig + :param str ecversion: Version of the software specified in the EasyConfig + :param str tcname: Toolchain name specified in the EasyConfig + :param str tcversion: Toolchain version specified in the EasyConfig + """ + ecname = kwargs.get['ecname', None] + ecversion = kwargs.get['ecversion', None] + tcname = kwargs.get['tcname', None] + tcversion = kwargs.get['tcversion', None] + gcccore_based_names = ['GCCcore', 'GCC'] foss_based_names = ['gfbf', 'gompi', 'foss'] return ( @@ -397,7 +412,7 @@ def parse_hook_zen4_module_only(ec, eprefix): # Need to escape newline character so that the newline character actually ends up in the module file # (otherwise, it splits the string, and a 2-line string ends up in the modulefile, resulting in syntax error) errmsg = "EasyConfigs using toolchains based on GCCcore-12.2.0 are not supported for the Zen4 architecture.\\n" - errmsg += "See https://www.eessi.io/docs/known_issues/eessi-2023.06/" + errmsg += "See https://www.eessi.io/docs/known_issues/eessi-2023.06/#gcc-1220-and-foss-2022b-based-modules-cannot-be-loaded-on-zen4-architecture" ec['modluafooter'] = 'if (not os.getenv("%s")) then LmodError("%s") end' % (env_varname, errmsg) @@ -451,7 +466,9 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs): EESSI_FORCE_ATTR) -# We do this as early as possible - and remove it all the way in the last step hook (post_testcases_hook) +# Modules for dependencies are loaded in the prepare step. Thus, that's where we need this variable to be set +# so that the modules can be succesfully loaded without printing the error (so that we can create a module +# _with_ the warning for the current software being installed) def pre_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): """Set environment variable to ignore the LmodError from parse_hook_zen4_module_only during build phase""" if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): From 54abb683507f0782863be6718b72fd99581a57c8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 Jan 2025 12:11:36 +0100 Subject: [PATCH 1631/1795] Fix formatting --- eb_hooks.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 59e19e4f3c..08eb5ea62e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -52,10 +52,10 @@ def is_gcccore_1220_based(**kwargs): :param str tcname: Toolchain name specified in the EasyConfig :param str tcversion: Toolchain version specified in the EasyConfig """ - ecname = kwargs.get['ecname', None] - ecversion = kwargs.get['ecversion', None] - tcname = kwargs.get['tcname', None] - tcversion = kwargs.get['tcversion', None] + ecname = kwargs.get('ecname', None) + ecversion = kwargs.get('ecversion', None) + tcname = kwargs.get('tcname', None) + tcversion = kwargs.get('tcversion', None) gcccore_based_names = ['GCCcore', 'GCC'] foss_based_names = ['gfbf', 'gompi', 'foss'] @@ -405,7 +405,8 @@ def parse_hook_zen4_module_only(ec, eprefix): This toolchain will not be supported on Zen4, so we will generate a modulefile and have it print an LmodError. """ - if is_gcccore_1220_based(ec['name'], ec['version'], ec['toolchain']['name'], ec['toolchain']['version']): + if is_gcccore_1220_based(ecname=ec['name'], ecversion=ec['version'], tcname=ec['toolchain']['name'], + tcversion=ec['toolchain']['version']): env_varname = EESSI_IGNORE_ZEN4_GCC1220_ENVVAR # TODO: create a docs page to which we can refer for more info here # TODO: then update the link to the known issues page to the _specific_ issue @@ -432,7 +433,8 @@ def pre_fetch_hook_zen4_gcccore1220(self, *args, **kwargs): This toolchain will not be supported on Zen4, so we will generate a modulefile and have it print an LmodError. """ - if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): + if is_gcccore_1220_based(ecname=self.name, ecversion=self.version, tcname=self.toolchain.name, + tcversion=self.toolchain.version): if hasattr(self, EESSI_MODULE_ONLY_ATTR): raise EasyBuildError("'self' already has attribute %s! Can't use pre_fetch hook.", EESSI_MODULE_ONLY_ATTR) @@ -450,7 +452,8 @@ def pre_fetch_hook_zen4_gcccore1220(self, *args, **kwargs): def post_module_hook_zen4_gcccore1220(self, *args, **kwargs): """Revert changes from pre_fetch_hook_zen4_gcccore1220""" - if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): + if is_gcccore_1220_based(ecname=self.name, ecversion=self.version, tcname=self.toolchain.name, + tcversion=self.toolchain.version): if hasattr(self, EESSI_MODULE_ONLY_ATTR): update_build_option('module_only', getattr(self, EESSI_MODULE_ONLY_ATTR)) print_msg("Restored original build option 'module_only' to %s" % getattr(self, EESSI_MODULE_ONLY_ATTR)) @@ -471,13 +474,15 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs): # _with_ the warning for the current software being installed) def pre_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): """Set environment variable to ignore the LmodError from parse_hook_zen4_module_only during build phase""" - if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): + if is_gcccore_1220_based(ecname=self.name, ecversion=self.version, tcname=self.toolchain.name, + tcversion=self.toolchain.version): os.environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR] = "1" def post_prepare_hook_ignore_zen4_gcccore1220_error(self, *args, **kwargs): """Unset environment variable to ignore the LmodError from parse_hook_zen4_module_only during build phase""" - if is_gcccore_1220_based(self.name, self.version, self.toolchain.name, self.toolchain.version): + if is_gcccore_1220_based(ecname=self.name, ecversion=self.version, tcname=self.toolchain.name, + tcversion=self.toolchain.version): del os.environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR] From 5bad2d9b5b6d11e481f0c5f36f4c1ad00ba483cf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jan 2025 11:00:26 +0100 Subject: [PATCH 1632/1795] use 0x46 as 'CPU implementer' field for A64FX --- init/arch_specs/eessi_arch_arm.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index 476d3d6119..7f7dc7330b 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -1,6 +1,6 @@ # ARM CPU architecture specifications (see https://gpages.juszkiewicz.com.pl/arm-socs-table/arm-socs.html for guidance) -# Software path in EESSI | Vendor ID | List of defining CPU features -"aarch64/a64fx" "" "asimdhp sve" # Fujitsu A64FX +# Software path in EESSI | 'Vendor ID' or 'CPU implementer' | List of defining CPU features +"aarch64/a64fx" "0x46" "asimdhp sve" # Fujitsu A64FX "aarch64/neoverse_n1" "ARM" "asimddp" # Ampere Altra "aarch64/neoverse_n1" "" "asimddp" # AWS Graviton2 "aarch64/neoverse_v1" "ARM" "asimddp svei8mm" From e53ac16396afe46154e20a2a72819cbdff948d4b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jan 2025 11:01:54 +0100 Subject: [PATCH 1633/1795] fall back to 'CPU implementer' field from /proc/cpuinfo if 'Vendor ID' field is not available --- init/eessi_archdetect.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index 2b1534ce62..cca3764ee8 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -111,15 +111,17 @@ cpupath(){ update_arch_specs "$base_dir/arch_specs/${spec_file}" # Identify the host CPU vendor - local cpu_vendor_tag="vendor[ _]id" - local cpu_vendor=$(get_cpuinfo "$cpu_vendor_tag") + local cpu_vendor=$(get_cpuinfo "vendor[ _]id") + if [ "${cpu_vendor}" == "" ]; then + cpu_vendor=$(get_cpuinfo "cpu[ _]implementer") + fi log "DEBUG" "cpupath: CPU vendor of host system: '$cpu_vendor'" # Identify the host CPU flags or features local cpu_flag_tag='flags' # cpuinfo systems print different line identifiers, eg features, instead of flags [ "${cpu_vendor}" == "ARM" ] && cpu_flag_tag='flags' - [ "${machine_type}" == "aarch64" ] && [ "${cpu_vendor}x" == "x" ] && cpu_flag_tag='features' + [ "${machine_type}" == "aarch64" ] && [ "${cpu_flag_tag}" == "flags" ] && cpu_flag_tag='features' [ "${machine_type}" == "ppc64le" ] && cpu_flag_tag='cpu' local cpu_flags=$(get_cpuinfo "$cpu_flag_tag") From f1124066502152b71f48b672d3a446593b608ec2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jan 2025 11:34:27 +0100 Subject: [PATCH 1634/1795] add Deucalion-Rocky85.all.output for archdetect tests --- tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.all.output | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.all.output diff --git a/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.all.output b/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.all.output new file mode 100644 index 0000000000..f6f97c2aaa --- /dev/null +++ b/tests/archdetect/aarch64/a64fx/Deucalion-Rocky85.all.output @@ -0,0 +1 @@ +aarch64/a64fx:aarch64/generic From d82a188865503fb177e1555d0ddb208c5c39aa18 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jan 2025 13:08:23 +0100 Subject: [PATCH 1635/1795] set correct 'CPU implementer' for graviton2/graviton3 --- init/arch_specs/eessi_arch_arm.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index 7f7dc7330b..595dddb800 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -2,6 +2,6 @@ # Software path in EESSI | 'Vendor ID' or 'CPU implementer' | List of defining CPU features "aarch64/a64fx" "0x46" "asimdhp sve" # Fujitsu A64FX "aarch64/neoverse_n1" "ARM" "asimddp" # Ampere Altra -"aarch64/neoverse_n1" "" "asimddp" # AWS Graviton2 +"aarch64/neoverse_n1" "0x41" "asimddp" # AWS Graviton2 "aarch64/neoverse_v1" "ARM" "asimddp svei8mm" -"aarch64/neoverse_v1" "" "asimddp svei8mm" # AWS Graviton3 +"aarch64/neoverse_v1" "0x41" "asimddp svei8mm" # AWS Graviton3 From 562fd4d1886f04d21911c1e3305cfac1cb9f4d67 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jan 2025 13:24:57 +0100 Subject: [PATCH 1636/1795] fix logic for determining field to use to determine CPU flags --- init/arch_specs/eessi_arch_arm.spec | 2 ++ init/eessi_archdetect.sh | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/init/arch_specs/eessi_arch_arm.spec b/init/arch_specs/eessi_arch_arm.spec index 595dddb800..c0d74bd4ad 100755 --- a/init/arch_specs/eessi_arch_arm.spec +++ b/init/arch_specs/eessi_arch_arm.spec @@ -1,4 +1,6 @@ # ARM CPU architecture specifications (see https://gpages.juszkiewicz.com.pl/arm-socs-table/arm-socs.html for guidance) +# CPU implementers: 0x41 (ARM), 0x46 (Fujitsu) - also see https://github.com/hrw/arm-socs-table/blob/main/data/socs.yml + # Software path in EESSI | 'Vendor ID' or 'CPU implementer' | List of defining CPU features "aarch64/a64fx" "0x46" "asimdhp sve" # Fujitsu A64FX "aarch64/neoverse_n1" "ARM" "asimddp" # Ampere Altra diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index cca3764ee8..4fd979cea5 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -118,11 +118,20 @@ cpupath(){ log "DEBUG" "cpupath: CPU vendor of host system: '$cpu_vendor'" # Identify the host CPU flags or features - local cpu_flag_tag='flags' # cpuinfo systems print different line identifiers, eg features, instead of flags - [ "${cpu_vendor}" == "ARM" ] && cpu_flag_tag='flags' - [ "${machine_type}" == "aarch64" ] && [ "${cpu_flag_tag}" == "flags" ] && cpu_flag_tag='features' - [ "${machine_type}" == "ppc64le" ] && cpu_flag_tag='cpu' + local cpu_flag_tag; + if [ "${cpu_vendor}" == "ARM" ]; then + # if CPU vendor field is ARM, then we should be able to determine CPU microarchitecture based on 'flags' field + cpu_flag_tag='flags' + # if 64-bit Arm CPU without "ARM" as vendor ID, we need to take into account 'features' field + elif [ "${machine_type}" == "aarch64" ]; then + cpu_flag_tag='features' + # on 64-bit POWER, we need to look at 'cpu' field + elif [ "${machine_type}" == "ppc64le" ]; then + cpu_flag_tag='cpu' + else + cpu_flag_tag='flags' + fi local cpu_flags=$(get_cpuinfo "$cpu_flag_tag") log "DEBUG" "cpupath: CPU flags of host system: '$cpu_flags'" From 733af6ddca05cdfdcac01eb23067d229fba00248 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Jan 2025 09:33:09 +0100 Subject: [PATCH 1637/1795] {2023.06}[2023a] elfx86exts 0.6.2 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index cb8f1766c3..19d685b109 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -41,3 +41,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21925 from-commit: 522ca010ab11949ab9594037f72b975cf1cd0d33 + - elfx86exts-0.6.2-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22145 + from-commit: 31478e5c9869de3add74d0a02dd5df01ea65b21e From b3bb5226951adad6c3867e30740d9e077a17a7fc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 13 Jan 2025 13:14:35 +0100 Subject: [PATCH 1638/1795] Make sure eessi_container.sh will add --nv flag for the test step, when GPUs are available. This is needed to run the GPU test --- bot/test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/test.sh b/bot/test.sh index 2e2c838dfa..4205ea8564 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -208,6 +208,12 @@ fi # Reframe configuration TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro") +# add options required to handle NVIDIA support +if command_exists "nvidia-smi"; then + echo "Command 'nvidia-smi' found, using available GPU" + TEST_STEP_ARGS+=("--nvidia" "run") +fi + # prepare arguments to test_suite.sh (specific to test step) declare -a TEST_SUITE_ARGS=() if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then From a94ac7efc14353647cda39c895aa60409af9534e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 13 Jan 2025 15:41:05 +0100 Subject: [PATCH 1639/1795] use ubuntu-22.04 instead of ubuntu-latest --- .github/workflows/test_eessi_container_script.yml | 2 +- .github/workflows/tests.yml | 2 +- .github/workflows/tests_init.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index fb587ec489..dc9468741e 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -9,7 +9,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: eessi_container_script: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 940614b6bb..04af57810c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: python: [3.7, 3.8, 3.9, '3.10'] diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 8581681d63..e2aa9007b7 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -8,7 +8,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: python: [3.7, 3.8, 3.9, '3.10'] From 028a70a4d635ce5dbcf78796211fdebf9163ddf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 13 Jan 2025 15:48:48 +0100 Subject: [PATCH 1640/1795] change ubuntu-latest to ubuntu-22.04 --- .github/workflows/scorecards.yml | 2 +- .github/workflows/test-software.eessi.io.yml | 2 +- .github/workflows/test_licenses.yml | 2 +- .github/workflows/tests_archdetect.yml | 2 +- .github/workflows/tests_archdetect_nvidia_gpu.yml | 2 +- .github/workflows/tests_eessi_module.yml | 8 ++++---- .github/workflows/tests_init_module.yml | 2 +- .github/workflows/tests_readme.yml | 2 +- .github/workflows/tests_scripts.yml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 91c2576b7d..f94503e076 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -19,7 +19,7 @@ jobs: analysis: if: github.repository_owner == 'EESSI' # Prevent running on forks name: Scorecards analysis - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: # Needed to upload the results to code-scanning dashboard. security-events: write diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index ca3792f6ef..a4d0bf327e 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -15,7 +15,7 @@ env: - nvidia/cc80 jobs: check_missing: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/test_licenses.yml b/.github/workflows/test_licenses.yml index bf6965c620..2517b42a9c 100644 --- a/.github/workflows/test_licenses.yml +++ b/.github/workflows/test_licenses.yml @@ -8,7 +8,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 860efb57d4..ca64442d01 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -8,7 +8,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: proc_cpuinfo: diff --git a/.github/workflows/tests_archdetect_nvidia_gpu.yml b/.github/workflows/tests_archdetect_nvidia_gpu.yml index 8ad5f4fb36..fc08f576b2 100644 --- a/.github/workflows/tests_archdetect_nvidia_gpu.yml +++ b/.github/workflows/tests_archdetect_nvidia_gpu.yml @@ -7,7 +7,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: fake_nvidia_smi_script: diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 2bf4b39bde..23699622fe 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -8,7 +8,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: basic_checks: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -63,7 +63,7 @@ jobs: set -e # Re-enable exit on non-zero status lmod_and_init_script_comparison: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -136,7 +136,7 @@ jobs: fi make_sure_load_and_unload_work: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -204,4 +204,4 @@ jobs: echo "Test for checking env variables FAILED" >&2 diff --unified=0 "${initial_env_file}" "${module_cycled_file}" exit 1 - fi \ No newline at end of file + fi diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index cfc4ae7b3d..9686bc4a09 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -9,7 +9,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index 76f46a8abe..ac92d8130d 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -14,7 +14,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 6413fcf86f..fc4b622bb8 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -28,7 +28,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 From 8ae64a348e837a4fa0e34ab53a23f84b3b3c7571 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 13 Jan 2025 16:17:40 +0100 Subject: [PATCH 1641/1795] Transition CI workflows to Ubuntu 24.04 --- .github/workflows/scorecards.yml | 2 +- .github/workflows/test-software.eessi.io.yml | 2 +- .github/workflows/test_eessi_container_script.yml | 2 +- .github/workflows/test_licenses.yml | 2 +- .github/workflows/tests.yml | 4 ++-- .github/workflows/tests_archdetect.yml | 2 +- .github/workflows/tests_archdetect_nvidia_gpu.yml | 2 +- .github/workflows/tests_eessi_module.yml | 2 +- .github/workflows/tests_init.yml | 4 ++-- .github/workflows/tests_init_module.yml | 2 +- .github/workflows/tests_readme.yml | 2 +- .github/workflows/tests_scripts.yml | 2 +- install_apptainer_ubuntu.sh | 15 +++++++++++++++ 13 files changed, 29 insertions(+), 14 deletions(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index f94503e076..d761d5f1ff 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -19,7 +19,7 @@ jobs: analysis: if: github.repository_owner == 'EESSI' # Prevent running on forks name: Scorecards analysis - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: # Needed to upload the results to code-scanning dashboard. security-events: write diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index a4d0bf327e..f569c664c9 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -15,7 +15,7 @@ env: - nvidia/cc80 jobs: check_missing: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index dc9468741e..6f7055a546 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -9,7 +9,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: eessi_container_script: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/test_licenses.yml b/.github/workflows/test_licenses.yml index 2517b42a9c..1770d8719a 100644 --- a/.github/workflows/test_licenses.yml +++ b/.github/workflows/test_licenses.yml @@ -8,7 +8,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 04af57810c..ecfa9a7ba5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,10 +8,10 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: - python: [3.7, 3.8, 3.9, '3.10'] + python: [3.8, 3.9, '3.10'] fail-fast: false steps: - name: checkout diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index ca64442d01..82438d8e92 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -8,7 +8,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: proc_cpuinfo: diff --git a/.github/workflows/tests_archdetect_nvidia_gpu.yml b/.github/workflows/tests_archdetect_nvidia_gpu.yml index fc08f576b2..54827eccad 100644 --- a/.github/workflows/tests_archdetect_nvidia_gpu.yml +++ b/.github/workflows/tests_archdetect_nvidia_gpu.yml @@ -7,7 +7,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: fake_nvidia_smi_script: diff --git a/.github/workflows/tests_eessi_module.yml b/.github/workflows/tests_eessi_module.yml index 23699622fe..a7b38e2205 100644 --- a/.github/workflows/tests_eessi_module.yml +++ b/.github/workflows/tests_eessi_module.yml @@ -136,7 +136,7 @@ jobs: fi make_sure_load_and_unload_work: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index e2aa9007b7..9ab373eec0 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -8,10 +8,10 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: - python: [3.7, 3.8, 3.9, '3.10'] + python: [3.8, 3.9, '3.10'] fail-fast: false steps: - name: checkout diff --git a/.github/workflows/tests_init_module.yml b/.github/workflows/tests_init_module.yml index 9686bc4a09..d30da61c84 100644 --- a/.github/workflows/tests_init_module.yml +++ b/.github/workflows/tests_init_module.yml @@ -9,7 +9,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index ac92d8130d..1a838e06d6 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -14,7 +14,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index fc4b622bb8..de227dfc25 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -28,7 +28,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index 5c4f37ac2d..6de75b35db 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -10,6 +10,21 @@ apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/${epel_subdir}/Eve curl -OL https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/${apptainer_rpm} sudo alien -d ${apptainer_rpm} sudo apt install ./apptainer*.deb +# No unpriviledged user name spaces in Ubuntu 23.10+ +if [ "$(lsb_release -r | awk '{print $2}')" \ge "23.10" ]; then + sudo tee /etc/apparmor.d/apptainer << 'EOF' +# Permit unprivileged user namespace creation for apptainer starter +abi , +include +profile apptainer /usr/local/libexec/apptainer/bin/starter{,-suid} + flags=(unconfined) { + userns, + # Site-specific additions and overrides. See local/README for details. + include if exists +} +EOF + sudo systemctl reload apparmor +fi apptainer --version # also check whether 'singularity' command is still provided by Apptainer installation singularity --version From 061db02afb598d8390b0fdc39ec72a51acab2abb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 13 Jan 2025 16:26:06 +0100 Subject: [PATCH 1642/1795] Comparison was no good --- install_apptainer_ubuntu.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index 6de75b35db..209a63a0c3 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -11,7 +11,8 @@ curl -OL https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/ sudo alien -d ${apptainer_rpm} sudo apt install ./apptainer*.deb # No unpriviledged user name spaces in Ubuntu 23.10+ -if [ "$(lsb_release -r | awk '{print $2}')" \ge "23.10" ]; then +ubuntu_version=$(lsb_release -r | awk '{print $2}') +if [[ $(echo -e "$ubuntu_version\n23.10" | sort -V | head -n 1) == "23.10" ]]; then sudo tee /etc/apparmor.d/apptainer << 'EOF' # Permit unprivileged user namespace creation for apptainer starter abi , From ff0dd71dd04912c1af60e577f3fd432c71c7b7f5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 13 Jan 2025 16:34:44 +0100 Subject: [PATCH 1643/1795] Is it this? --- install_apptainer_ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index 209a63a0c3..bba4c07532 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -17,7 +17,7 @@ if [[ $(echo -e "$ubuntu_version\n23.10" | sort -V | head -n 1) == "23.10" ]]; t # Permit unprivileged user namespace creation for apptainer starter abi , include -profile apptainer /usr/local/libexec/apptainer/bin/starter{,-suid} +profile apptainer /usr/local/libexec/apptainer/bin/starter{,-suid} flags=(unconfined) { userns, # Site-specific additions and overrides. See local/README for details. From 78e0eb9d40ce380a8a8a4e39cd744ac9f1400138 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 13 Jan 2025 16:39:05 +0100 Subject: [PATCH 1644/1795] apptainer is not where I expect it --- install_apptainer_ubuntu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index bba4c07532..7ceb20eec0 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -26,6 +26,7 @@ profile apptainer /usr/local/libexec/apptainer/bin/starter{,-suid} EOF sudo systemctl reload apparmor fi +which apptainer apptainer --version # also check whether 'singularity' command is still provided by Apptainer installation singularity --version From c1dda1b090d4c73a56098cea6a4bcb811d34f5e7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 13 Jan 2025 16:42:40 +0100 Subject: [PATCH 1645/1795] Use correct location for apptainer starter --- install_apptainer_ubuntu.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index 7ceb20eec0..192a36a483 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -17,7 +17,7 @@ if [[ $(echo -e "$ubuntu_version\n23.10" | sort -V | head -n 1) == "23.10" ]]; t # Permit unprivileged user namespace creation for apptainer starter abi , include -profile apptainer /usr/local/libexec/apptainer/bin/starter{,-suid} +profile apptainer /usr/libexec/apptainer/bin/starter{,-suid} flags=(unconfined) { userns, # Site-specific additions and overrides. See local/README for details. @@ -26,7 +26,6 @@ profile apptainer /usr/local/libexec/apptainer/bin/starter{,-suid} EOF sudo systemctl reload apparmor fi -which apptainer apptainer --version # also check whether 'singularity' command is still provided by Apptainer installation singularity --version From 65e4c36a790e3bdda73807f7baa393077a195d4a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 13 Jan 2025 18:04:12 +0100 Subject: [PATCH 1646/1795] Replace the use of a ReFrame template config file for a manually created one. This means the user deploying a bot to build for software-layer will have to create those ReFrame config files and set the RFM_CONFIG_FILES environment variable in the session running the bot app --- reframe_config_bot.py.tmpl | 57 ---------------------- test_suite.sh | 99 ++++++++++++++++++++++---------------- 2 files changed, 58 insertions(+), 98 deletions(-) delete mode 100644 reframe_config_bot.py.tmpl diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl deleted file mode 100644 index 323aafd5ec..0000000000 --- a/reframe_config_bot.py.tmpl +++ /dev/null @@ -1,57 +0,0 @@ -# WARNING: this file is intended as template and the __X__ template variables need to be replaced -# before it can act as a configuration file -# Once replaced, this is a config file for running tests after the build phase, by the bot - -from eessi.testsuite.common_config import common_logging_config -from eessi.testsuite.constants import * # noqa: F403 - - -site_configuration = { - 'systems': [ - { - 'name': 'BotBuildTests', - 'descr': 'Software-layer bot', - 'hostnames': ['.*'], - 'modules_system': 'lmod', - 'partitions': [ - { - 'name': '__RFM_PARTITION__', - 'scheduler': 'local', - 'launcher': 'mpirun', - 'environs': ['default'], - 'features': [ - FEATURES[CPU] - ] + list(SCALES.keys()), - 'resources': [ - { - 'name': 'memory', - 'options': ['--mem={size}'], - } - ], - 'extras': { - # Make sure to round down, otherwise a job might ask for more mem than is available - # per node - 'mem_per_node': __MEM_PER_NODE__, - }, - 'max_jobs': 1 - } - ] - } - ], - 'environments': [ - { - 'name': 'default', - 'cc': 'cc', - 'cxx': '', - 'ftn': '' - } - ], - 'general': [ - { - 'purge_environment': True, - 'resolve_module_conflicts': False, # avoid loading the module before submitting the job - 'remote_detect': True, - } - ], - 'logging': common_logging_config(), -} diff --git a/test_suite.sh b/test_suite.sh index 1f0b91c477..a337ce48b6 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -132,56 +132,73 @@ else fi # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration -export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py -export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl +# RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame +# config file that matches the bot config. See https://gitlab.com/eessi/support/-/issues/114#note_2293660921 +if [ -z "$RFM_CONFIG_FILES" ]; then + err_msg = "Please set RFM_CONFIG_FILES in the environment of this bot instance to point to a valid" + err_msg = "${err_msg} ReFrame configuration file that matches the bot config." + err_msg = "${err_msg} For more information, see https://gitlab.com/eessi/support/-/issues/114#note_2293660921" + fatal_error "${err_msg}" +fi export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests export RFM_CHECK_SEARCH_RECURSIVE=1 export RFM_PREFIX=$PWD/reframe_runs +# Get the correct partition name +REFRAME_PARTITION_NAME=${EESSI_SOFTWARE_SUBDIR//\//_} +if [ ! -z "$EESSI_ACCELERATOR_TARGET" ]; then + REFRAME_PARTITION_NAME=${REFRAME_PARTITION_NAME}_${EESSI_ACCELERATOR_TARGET//\//_} +fi +echo "Constructed partition name based on EESSI_SOFTWARE_SUBDIR and EESSI_ACCELERATOR_TARGET: ${REFRAME_PARTITION_NAME}" + +# Set the reframe system name, including partition +export RFM_SYSTEM="BotBuildTests:${REFRAME_PARTITION_NAME}" + echo "Configured reframe with the following environment variables:" env | grep "RFM_" -# The /sys inside the container is not the same as the /sys of the host -# We want to extract the memory limit from the cgroup on the host (which is typically set by SLURM). -# Thus, bot/test.sh bind-mounts the host's /sys/fs/cgroup into /hostsys/fs/cgroup -# and that's the prefix we use to extract the memory limit from -cgroup_v1_mem_limit="/hostsys/fs/cgroup/memory/$( Date: Tue, 14 Jan 2025 15:27:39 +0100 Subject: [PATCH 1647/1795] add all EB versions --- .../sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml new file mode 100644 index 0000000000..ae9de1d067 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml @@ -0,0 +1,6 @@ +easyconfigs: + - EasyBuild-4.8.2.eb + - EasyBuild-4.9.0.eb + - EasyBuild-4.9.1.eb + - EasyBuild-4.9.2.eb + - EasyBuild-4.9.4.eb From bcc9380fc578db8071dcb96afb86a3e91db717cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 14 Jan 2025 15:29:40 +0100 Subject: [PATCH 1648/1795] add 4.9.3 --- .../sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml index ae9de1d067..87e4cd720d 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml @@ -3,4 +3,5 @@ easyconfigs: - EasyBuild-4.9.0.eb - EasyBuild-4.9.1.eb - EasyBuild-4.9.2.eb + - EasyBuild-4.9.3.eb - EasyBuild-4.9.4.eb From 1f0f4f2d4e657fdc433d7c0f281cf73c03df2d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 14 Jan 2025 15:55:49 +0100 Subject: [PATCH 1649/1795] make modules and software directories for new CPU targets --- EESSI-install-software.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 83c06c2184..2f05953158 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -135,8 +135,9 @@ else # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set source $TOPDIR/init/minimal_eessi_env - # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) - mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # make sure the the software and modules directory exist + # (since it's expected by init/eessi_environment_variables when using archdetect and by the EESSI module) + mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software} ) fi From e85f839d0a701c1fc4277d261b518a0090682453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 14 Jan 2025 17:06:21 +0100 Subject: [PATCH 1650/1795] also install the latest EB release as module before installing EESSI-extend --- load_eessi_extend_module.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index aa8f659ef1..bd46c3a7eb 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -91,6 +91,8 @@ else eb_install_out=${TMPDIR}/eb_install.out ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" + # EESSI-extend also needs EasyBuild to be installed as a module, so install the latest release + ${EB} --install-latest-eb-release 2>&1 | tee ${eb_install_out} # while always adding --try-amend=keep... may do no harm, we could make # an attempt to figure out if it is needed, e.g., when we are rebuilding ${EB} "EESSI-extend-easybuild.eb" --try-amend=keeppreviousinstall=True 2>&1 | tee ${eb_install_out} From eb3cb49800b2a808354b56602776aa76f9cdc020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 14 Jan 2025 17:13:20 +0100 Subject: [PATCH 1651/1795] Update load_eessi_extend_module.sh Co-authored-by: ocaisa --- load_eessi_extend_module.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/load_eessi_extend_module.sh b/load_eessi_extend_module.sh index bd46c3a7eb..b5ad47de4e 100755 --- a/load_eessi_extend_module.sh +++ b/load_eessi_extend_module.sh @@ -88,14 +88,19 @@ else echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..." export PATH=${EB_TMPDIR}/bin:${PATH} export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):${PYTHONPATH} - eb_install_out=${TMPDIR}/eb_install.out - ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" - fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})" # EESSI-extend also needs EasyBuild to be installed as a module, so install the latest release + eb_install_out=${TMPDIR}/eb_install.out + ok_msg="Latest EasyBuild installed, let's go!" + fail_msg="Installing latest EasyBuild failed, that's not good... (output: ${eb_install_out})" ${EB} --install-latest-eb-release 2>&1 | tee ${eb_install_out} + check_exit_code $? "${ok_msg}" "${fail_msg}" + # Now install EESSI-extend + eessi_install_out=${TMPDIR}/eessi_install.out + ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!" + fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eessi_install_out})" # while always adding --try-amend=keep... may do no harm, we could make # an attempt to figure out if it is needed, e.g., when we are rebuilding - ${EB} "EESSI-extend-easybuild.eb" --try-amend=keeppreviousinstall=True 2>&1 | tee ${eb_install_out} + ${EB} "EESSI-extend-easybuild.eb" --try-amend=keeppreviousinstall=True 2>&1 | tee ${eessi_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" ) From 9d6f6946bc738053e7b2690b2cee7bbdb99999ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 14 Jan 2025 17:32:24 +0100 Subject: [PATCH 1652/1795] add --from-commit for EB 4.9.4 --- .../sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml index 87e4cd720d..4317faf9a4 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml @@ -5,3 +5,6 @@ easyconfigs: - EasyBuild-4.9.2.eb - EasyBuild-4.9.3.eb - EasyBuild-4.9.4.eb + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 From b5e4a919859f79d8f17c877c5273a4fb7e63921d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 14 Jan 2025 17:32:45 +0100 Subject: [PATCH 1653/1795] missing colon --- .../sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml index 4317faf9a4..c92d62717b 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml @@ -4,7 +4,7 @@ easyconfigs: - EasyBuild-4.9.1.eb - EasyBuild-4.9.2.eb - EasyBuild-4.9.3.eb - - EasyBuild-4.9.4.eb + - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 From d8794a1f3b0decf07e9ebff7808c53ac562d147b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 16 Jan 2025 10:49:59 +0100 Subject: [PATCH 1654/1795] Make the ReFrame args configurable through environment in which the bot instance runs. --- test_suite.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index a337ce48b6..d587e5a626 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -234,7 +234,18 @@ else fatal_error "Failed to extract names of tests to run: ${REFRAME_NAME_ARGS}" exit ${test_selection_exit_code} fi -export REFRAME_ARGS="--tag CI --tag 1_node --nocolor ${REFRAME_NAME_ARGS}" +# Allow people deploying the bot to overrwide this +if [ -z "$REFRAME_SCALE_TAG" ]; then + REFRAME_SCALE_TAGS="--tag 1_node" +fi +if [ -z "$REFRAME_CI_TAG" ]; then + REFRAME_CI_TAG="--tag CI" +fi +# Allow bot-deployers to add additional args through the environment +if [ -z "$REFRAME_ADDITIONAL_ARGS" ]; then + REFRAME_ADDITIONAL_ARGS="" +fi +export REFRAME_ARGS="${REFRAME_CI_TAG} ${REFRAME_SCALE_TAG} ${REFRAME_ADDITIONAL_ARGS} --nocolor ${REFRAME_NAME_ARGS}" # List the tests we want to run echo "Listing tests: reframe ${REFRAME_ARGS} --list" From 4c9f1a9a9c60671efe1ae82e634d761374de6060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 17 Jan 2025 11:52:25 +0100 Subject: [PATCH 1655/1795] first batch of builds for sapphire rapids --- .../sapphire_rapids/eessi-2023.06-eb-4.8.2-001-system.yml | 4 ++++ .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml | 1 + 2 files changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-001-system.yml create mode 120000 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-001-system.yml new file mode 100644 index 0000000000..a033316c1e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-001-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - Nextflow-23.10.0.eb: + options: + from-pr: 19172 diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml new file mode 120000 index 0000000000..d4f46cb297 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml @@ -0,0 +1 @@ +../eessi-2023.06-eb-4.8.2-2022b.yml \ No newline at end of file From f14b1110139ffc546d99c117018308d433e4d074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 17 Jan 2025 14:12:35 +0100 Subject: [PATCH 1656/1795] only build deps for OpenBLAS --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) mode change 120000 => 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml deleted file mode 120000 index d4f46cb297..0000000000 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml +++ /dev/null @@ -1 +0,0 @@ -../eessi-2023.06-eb-4.8.2-2022b.yml \ No newline at end of file diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml new file mode 100644 index 0000000000..571a376317 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml @@ -0,0 +1,4 @@ +easyconfigs: + - GCC-12.2.0.eb + - Python-3.10.8-GCCcore-12.2.0-bare.eb + - make-4.3-GCCcore-12.2.0.eb From cec1c16d9af77e136afc7fbd75bc100b75a5f24b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 17 Jan 2025 14:20:37 +0100 Subject: [PATCH 1657/1795] add GCC for foss 2023a --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml new file mode 100644 index 0000000000..24a45ad731 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -0,0 +1,4 @@ +easyconfigs: + - GCCcore-12.3.0.eb: + options: + from-pr: 20218 From f5649d2bdc44110f3644a4be26d29c231b4c525a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 17 Jan 2025 14:20:43 +0100 Subject: [PATCH 1658/1795] add GCC for foss 2023b --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml new file mode 100644 index 0000000000..e585992fc1 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -0,0 +1,4 @@ +easyconfigs: + - GCCcore-13.2.0.eb: + options: + from-pr: 19974 From 99443929b40b37718210b65bfa0236c774ce83e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 17 Jan 2025 14:20:56 +0100 Subject: [PATCH 1659/1795] add openblas --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml new file mode 100644 index 0000000000..ac5139290c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml @@ -0,0 +1,5 @@ +easyconfigs: + - OpenBLAS-0.3.21-GCC-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3492 + include-easyblocks-from-commit: d06d9617d9bfb63d338b6879eab9da81c8a312d8 From 835a49766efac08c30b0c704b070532c5c7c04f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 00:34:49 +0100 Subject: [PATCH 1660/1795] add dependencies of GObject-Introspection --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml index 571a376317..c5159f0afa 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml @@ -2,3 +2,4 @@ easyconfigs: - GCC-12.2.0.eb - Python-3.10.8-GCCcore-12.2.0-bare.eb - make-4.3-GCCcore-12.2.0.eb + - cairo-1.17.4-GCCcore-12.2.0.eb From 9dbaf2dc9ba2b0688187f4cab1d0ffee048e049e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 00:35:04 +0100 Subject: [PATCH 1661/1795] add GObject-Introspection --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml new file mode 100644 index 0000000000..8ec0067465 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GObject-Introspection-1.74.0-GCCcore-12.2.0.eb From 82ac85ba9c9764afc0f77fe18acf3fe356e7e541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 09:34:58 +0100 Subject: [PATCH 1662/1795] add missing eb 4.8.2 2022b software --- .../sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml index c5159f0afa..36a2cf5a0b 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml @@ -3,3 +3,9 @@ easyconfigs: - Python-3.10.8-GCCcore-12.2.0-bare.eb - make-4.3-GCCcore-12.2.0.eb - cairo-1.17.4-GCCcore-12.2.0.eb + - foss-2022b.eb + - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: + options: + from-pr: 19339 + - Qt5-5.15.7-GCCcore-12.2.0.eb + - QuantumESPRESSO-7.2-foss-2022b.eb From 5370a645013f4fa5b8221c059c251a6fcd1e997b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 10:15:26 +0100 Subject: [PATCH 1663/1795] add deps for OpenMPI --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml index c5159f0afa..ef2dc8742a 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml @@ -3,3 +3,6 @@ easyconfigs: - Python-3.10.8-GCCcore-12.2.0-bare.eb - make-4.3-GCCcore-12.2.0.eb - cairo-1.17.4-GCCcore-12.2.0.eb + - UCC-1.1.0-GCCcore-12.2.0.eb + - PMIx-4.2.2-GCCcore-12.2.0.eb + - libfabric-1.16.1-GCCcore-12.2.0.eb From 7d6f04c1f8b049aa6cc3ded9d542bef3453f61a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 10:16:18 +0100 Subject: [PATCH 1664/1795] add OpenMPI 4.1.4 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml new file mode 100644 index 0000000000..4aa98f6c4b --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OpenMPI-4.1.4-GCC-12.2.0.eb From bf2268c5dfff0ac0e4b75ec3ebca5279e9e59f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 10:17:25 +0100 Subject: [PATCH 1665/1795] add OpenMPI 4.1.5 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 24a45ad731..b2cf9bf86c 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -2,3 +2,6 @@ easyconfigs: - GCCcore-12.3.0.eb: options: from-pr: 20218 + - OpenMPI-4.1.5-GCC-12.3.0: + options: + from-pr: 19940 From a9c7e20cd19b7926daa02cef965b29bbf3221524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 10:17:35 +0100 Subject: [PATCH 1666/1795] add OpenMPI 4.1.6 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index e585992fc1..c103e5e368 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -2,3 +2,6 @@ easyconfigs: - GCCcore-13.2.0.eb: options: from-pr: 19974 + - OpenMPI-4.1.6-GCC-13.2.0: + options: + from-pr: 19940 From a11c44e91a2621db9486668691a166c6e1e0b69e Mon Sep 17 00:00:00 2001 From: ocaisa Date: Sat, 18 Jan 2025 14:20:17 +0100 Subject: [PATCH 1667/1795] Update eessi-2023.06-eb-4.8.2-2022b.yml --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml index f261e467b1..cb2748982c 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml @@ -9,6 +9,7 @@ easyconfigs: - foss-2022b.eb - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: options: - from-pr: 19339 + # See https://github.com/easybuilders/easybuild-easyconfigs/pull/19339 + from-commit: 42319adacfc6b998f510ddddcc4cac77c8684783 - Qt5-5.15.7-GCCcore-12.2.0.eb - QuantumESPRESSO-7.2-foss-2022b.eb From 95dedd043972241ef9127d587e83d83c41ea4598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 14:49:15 +0100 Subject: [PATCH 1668/1795] dependencies for Python-3.11.3-GCCcore-12.3.0.eb --- .../sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml new file mode 100644 index 0000000000..0caec51abb --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - XZ-5.4.2-GCCcore-12.3.0.eb + - libffi-3.4.4-GCCcore-12.3.0.eb + - SQLite-3.42.0-GCCcore-12.3.0.eb + - bzip2-1.0.8-GCCcore-12.3.0.eb + - UnZip-6.0-GCCcore-12.3.0.eb From 5a4cb2526a2045734951f83e703f364af71f6cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 14:51:06 +0100 Subject: [PATCH 1669/1795] add Python and hatchling --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml new file mode 100644 index 0000000000..ab558cfbfa --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml @@ -0,0 +1,3 @@ +easyconfigs: + - Python-3.11.3-GCCcore-12.3.0.eb + - hatchling-1.18.0-GCCcore-12.3.0.eb From fa4d32aa9f569ad3ae26854cd2ba4c571c37f16c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 14:59:27 +0100 Subject: [PATCH 1670/1795] add dependencies for Python --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index c103e5e368..13875918e7 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -5,3 +5,7 @@ easyconfigs: - OpenMPI-4.1.6-GCC-13.2.0: options: from-pr: 19940 + - XZ-5.4.4-GCCcore-13.2.0.eb + - SQLite-3.43.1-GCCcore-13.2.0.eb + - UnZip-6.0-GCCcore-13.2.0.eb + - libffi-3.4.4-GCCcore-13.2.0.eb From 7063de43041e9b501431580e7d236367812a7281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 15:00:03 +0100 Subject: [PATCH 1671/1795] add Python and hatchling --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml new file mode 100644 index 0000000000..8d57c5b898 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml @@ -0,0 +1,3 @@ +easyconfigs: + - Python-3.11.5-GCCcore-13.2.0.eb + - hatchling-1.18.0-GCCcore-13.2.0.eb From 20730686b45b06b181633a0f9c932dc69da2600e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 15:56:44 +0100 Subject: [PATCH 1672/1795] from-commit does not work with EB 4.8.2 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml index cb2748982c..f261e467b1 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml @@ -9,7 +9,6 @@ easyconfigs: - foss-2022b.eb - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: options: - # See https://github.com/easybuilders/easybuild-easyconfigs/pull/19339 - from-commit: 42319adacfc6b998f510ddddcc4cac77c8684783 + from-pr: 19339 - Qt5-5.15.7-GCCcore-12.2.0.eb - QuantumESPRESSO-7.2-foss-2022b.eb From 351346e0a1d4b4974a44161fb67980a2d674df95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 22:29:21 +0100 Subject: [PATCH 1673/1795] pypi bundle dependencies --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml index 0caec51abb..b38acdbd0a 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -4,3 +4,6 @@ easyconfigs: - SQLite-3.42.0-GCCcore-12.3.0.eb - bzip2-1.0.8-GCCcore-12.3.0.eb - UnZip-6.0-GCCcore-12.3.0.eb + - poetry-1.5.1-GCCcore-12.3.0.eb + - git-2.41.0-GCCcore-12.3.0-nodocs.eb + - flit-3.9.0-GCCcore-12.3.0.eb From d7861961d7437411df62b2d0ed9d5806dc39ebf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 22:29:59 +0100 Subject: [PATCH 1674/1795] add Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml index ab558cfbfa..57eda0aec4 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,3 +1,4 @@ easyconfigs: - Python-3.11.3-GCCcore-12.3.0.eb - hatchling-1.18.0-GCCcore-12.3.0.eb + - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb From d158e9fd4c12e40239f1ddaee5d2a300aed4e463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 22:32:21 +0100 Subject: [PATCH 1675/1795] deps for pypi bundle --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index 13875918e7..f0d6273214 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -9,3 +9,7 @@ easyconfigs: - SQLite-3.43.1-GCCcore-13.2.0.eb - UnZip-6.0-GCCcore-13.2.0.eb - libffi-3.4.4-GCCcore-13.2.0.eb + - poetry-1.6.1-GCCcore-13.2.0.eb + - git-2.42.0-GCCcore-13.2.0.eb + - flit-3.9.0-GCCcore-13.2.0.eb + - expat-2.5.0-GCCcore-13.2.0.eb From a2a2d4c78b4d4ad5f0373087a064f666ed0087fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 22:32:40 +0100 Subject: [PATCH 1676/1795] add Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml index 8d57c5b898..813a7f87c8 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml @@ -1,3 +1,4 @@ easyconfigs: - Python-3.11.5-GCCcore-13.2.0.eb - hatchling-1.18.0-GCCcore-13.2.0.eb + - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb From 37ea0eba04f74425f5c10f8f14e2836545c21bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 22:52:27 +0100 Subject: [PATCH 1677/1795] add cairo --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml index 0caec51abb..8a33563256 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -4,3 +4,4 @@ easyconfigs: - SQLite-3.42.0-GCCcore-12.3.0.eb - bzip2-1.0.8-GCCcore-12.3.0.eb - UnZip-6.0-GCCcore-12.3.0.eb + - cairo-1.17.8-GCCcore-12.3.0.eb From 60dd47a564d91288bb3c7b2713b78d0b61a2d079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 22:53:34 +0100 Subject: [PATCH 1678/1795] GObject-Introspection and at-spi2-core --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml new file mode 100644 index 0000000000..e4e3238ce7 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml @@ -0,0 +1,3 @@ +easyconfigs: + - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb + - at-spi2-core-2.49.91-GCCcore-12.3.0.eb From f71211e68cc600530688cff543acd9b3d7c6373e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 22:58:21 +0100 Subject: [PATCH 1679/1795] add cairo-1.18.0-GCCcore-13.2.0.eb --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index 13875918e7..3475fe0efb 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -9,3 +9,4 @@ easyconfigs: - SQLite-3.43.1-GCCcore-13.2.0.eb - UnZip-6.0-GCCcore-13.2.0.eb - libffi-3.4.4-GCCcore-13.2.0.eb + - cairo-1.18.0-GCCcore-13.2.0.eb From c15fd9726920c4b3a8affd51b21ac7a767bdc1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 22:58:52 +0100 Subject: [PATCH 1680/1795] add GObject-Introspection-1.78.1-GCCcore-13.2.0.eb --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml new file mode 100644 index 0000000000..d4614af159 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb From 0bace6bff0f4bee02e6e713e1b2a445f749ac81b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 23:01:55 +0100 Subject: [PATCH 1681/1795] add at-spi2-core --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml index d4614af159..4d9f5411ed 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,2 +1,3 @@ easyconfigs: - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb + - at-spi2-core-2.50.0-GCCcore-13.2.0.eb From 5b8fa3237d362aa82ecb40aaa0c96f539ccfaa5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 23:08:38 +0100 Subject: [PATCH 1682/1795] ReFrame 4.3.3 --- .../sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml new file mode 100644 index 0000000000..855d65f458 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ReFrame-4.3.3.eb From 01247bac172c596db77d86cdcf9de71336635cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 18 Jan 2025 23:09:03 +0100 Subject: [PATCH 1683/1795] ReFrame 4.6.2 --- .../sapphire_rapids/eessi-2023.06-eb-4.9.2-001-system.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-001-system.yml new file mode 100644 index 0000000000..18e5f0a3e9 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-001-system.yml @@ -0,0 +1,7 @@ +easyconfigs: + - ReFrame-4.6.2.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21307 + from-commit: 0c4bd5c5a80f571a8932fbc38880d72455406816 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3431 + include-easyblocks-from-commit: efddeb02abe1a679324ac01ef19601dedbe79cc0 From d7c168b61ab40b604cd9109e9ad244d5f049658f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 19 Jan 2025 11:03:54 +0100 Subject: [PATCH 1684/1795] include easyblock for Rust --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml index 4ce2d62114..e2a4aed631 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -5,6 +5,9 @@ easyconfigs: - bzip2-1.0.8-GCCcore-12.3.0.eb - UnZip-6.0-GCCcore-12.3.0.eb - cairo-1.17.8-GCCcore-12.3.0.eb + - Rust-1.70.0-GCCcore-12.3.0.eb: + options: + include-easyblocks-from-pr: 3038 - poetry-1.5.1-GCCcore-12.3.0.eb - git-2.41.0-GCCcore-12.3.0-nodocs.eb - flit-3.9.0-GCCcore-12.3.0.eb From 46e35449c09cbfa0c7f0b579a16a9a61cf2fee0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 19 Jan 2025 17:48:42 +0100 Subject: [PATCH 1685/1795] add scikit-build-core-0.9.3-GCCcore-13.2.0.eb --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml index 813a7f87c8..69ff4697ea 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml @@ -2,3 +2,4 @@ easyconfigs: - Python-3.11.5-GCCcore-13.2.0.eb - hatchling-1.18.0-GCCcore-13.2.0.eb - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb + - scikit-build-core-0.9.3-GCCcore-13.2.0.eb From 0e21835be505dd1907d0586ef03627b54aac907c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 19 Jan 2025 17:49:54 +0100 Subject: [PATCH 1686/1795] add GROMACS-2024.1-foss-2023b.eb --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml index 69ff4697ea..087e491f49 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml @@ -3,3 +3,4 @@ easyconfigs: - hatchling-1.18.0-GCCcore-13.2.0.eb - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb - scikit-build-core-0.9.3-GCCcore-13.2.0.eb + - GROMACS-2024.1-foss-2023b.eb From 2db27b56529cf4311feda1a7f1ef9341c8188821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 19 Jan 2025 17:52:44 +0100 Subject: [PATCH 1687/1795] add dependenies of GROMACS 2024.1 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index c2c3b7fef2..6136329821 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -14,3 +14,7 @@ easyconfigs: - git-2.42.0-GCCcore-13.2.0.eb - flit-3.9.0-GCCcore-13.2.0.eb - expat-2.5.0-GCCcore-13.2.0.eb + - foss-2023b.eb + - SciPy-bundle-2023.11-gfbf-2023b.eb + - networkx-3.2.1-gfbf-2023b.eb + - mpi4py-3.1.5-gompi-2023b.eb From 28afcbc6bc03d5cc72147cbad23a16d3d60620a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 19 Jan 2025 19:35:40 +0100 Subject: [PATCH 1688/1795] rebuild Python versions --- ...eb-4.9.2-Python-ctypes-sapphire_rapids.yml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250119-eb-4.9.2-Python-ctypes-sapphire_rapids.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250119-eb-4.9.2-Python-ctypes-sapphire_rapids.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250119-eb-4.9.2-Python-ctypes-sapphire_rapids.yml new file mode 100644 index 0000000000..1d9acf0988 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250119-eb-4.9.2-Python-ctypes-sapphire_rapids.yml @@ -0,0 +1,24 @@ +# 2025.01.19 +# Python ctypes relies on LD_LIBRARY_PATH and doesn't respect rpath linking. There is a workaround +# for the EasyBuild context in https://github.com/easybuilders/easybuild-easyblocks/pull/3352. +# +# This rebuild ensures this fix is available for all Python versions shipped with EESSI. +# +# See https://gitlab.com/eessi/support/-/issues/77 +easyconfigs: + - Python-3.10.8-GCCcore-12.2.0-bare: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + - Python-3.10.8-GCCcore-12.2.0: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + - Python-3.11.3-GCCcore-12.3.0: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + - Python-3.11.5-GCCcore-13.2.0: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f From 4520cb622ee5048c10994bc32e64a1a6258a2b23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 19 Jan 2025 23:39:54 +0100 Subject: [PATCH 1689/1795] forgot the from-pr options for scitkit-build-core and GROMACS --- .../sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml index 087e491f49..4ad5c6d95a 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml @@ -2,5 +2,11 @@ easyconfigs: - Python-3.11.5-GCCcore-13.2.0.eb - hatchling-1.18.0-GCCcore-13.2.0.eb - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb - - scikit-build-core-0.9.3-GCCcore-13.2.0.eb - - GROMACS-2024.1-foss-2023b.eb + - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: + options: + # from-commit: 61d07bff09afe63cfe1ae35dc58a0c8be01eed62 + from-pr: 20526 + - GROMACS-2024.1-foss-2023b.eb: + options: + # from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 + from-pr: 20522 From 56f727b734f4166bdc47580c8fe40da6e93806fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 07:37:12 +0100 Subject: [PATCH 1690/1795] networkx should be done with EB 4.9.1 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index 6136329821..104ea1d461 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -16,5 +16,4 @@ easyconfigs: - expat-2.5.0-GCCcore-13.2.0.eb - foss-2023b.eb - SciPy-bundle-2023.11-gfbf-2023b.eb - - networkx-3.2.1-gfbf-2023b.eb - mpi4py-3.1.5-gompi-2023b.eb From 897500a769827bf0022f64a59993fb78f44a0165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 11:08:55 +0100 Subject: [PATCH 1691/1795] mpi4py should also be built with EB 4.9.1 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index 104ea1d461..331881efdd 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -16,4 +16,3 @@ easyconfigs: - expat-2.5.0-GCCcore-13.2.0.eb - foss-2023b.eb - SciPy-bundle-2023.11-gfbf-2023b.eb - - mpi4py-3.1.5-gompi-2023b.eb From a1b650075687422769c592d0cf6487763fa3d67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 13:05:02 +0100 Subject: [PATCH 1692/1795] use easyconfigs PR #22183 --- .../sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml index 855d65f458..1e6233a041 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml @@ -1,2 +1,4 @@ easyconfigs: - - ReFrame-4.3.3.eb + - ReFrame-4.3.3.eb: + options: + from-pr: 22183 From 208979a65e5092ccb2b610930e498ba342c5e31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 13:25:55 +0100 Subject: [PATCH 1693/1795] rebuild hatchling 1.18.0 --- ...tchling-1.18.0-updated-easyconfig-sapphire_rapids.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250120-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphire_rapids.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250120-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphire_rapids.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250120-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphire_rapids.yml new file mode 100644 index 0000000000..a86eb166bd --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250120-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphire_rapids.yml @@ -0,0 +1,9 @@ +# 2025.01.20 +# hatchling-1.18.0 rebuild to account for easyconfig changed upstream +# see https://gitlab.com/eessi/support/-/issues/85 and +# https://github.com/easybuilders/easybuild-easyconfigs/pull/20389 +easyconfigs: + - hatchling-1.18.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20389 + from-commit: 9580c0d67d6dd97b160b768a839bfcba6d5b21b9 From 0ea815fae665019ff82aea0c636ae055334757cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 13:31:38 +0100 Subject: [PATCH 1694/1795] add all missing EB 4.9.0 2022b apps --- .../sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml index 4aa98f6c4b..bc31f6d512 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,2 +1,14 @@ easyconfigs: - OpenMPI-4.1.4-GCC-12.2.0.eb + - SciPy-bundle-2023.02-gfbf-2022b.eb + - GDAL-3.6.2-foss-2022b.eb + - waLBerla-6.1-foss-2022b.eb: + options: + from-pr: 19324 + - WRF-4.4.1-foss-2022b-dmpar.eb + - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb: + options: + from-pr: 20086 + - R-4.2.2-foss-2022b.eb: + options: + from-pr: 20238 From 3851d868d39dbbb44ea187e1bcddd2d3160f2cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 14:48:55 +0100 Subject: [PATCH 1695/1795] add fix for Highway --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml index bc31f6d512..a8dc0693a3 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,5 +1,8 @@ easyconfigs: - OpenMPI-4.1.4-GCC-12.2.0.eb + - Highway-1.0.3-GCCcore-12.2.0.eb: + options: + from-pr: 20298 - SciPy-bundle-2023.02-gfbf-2022b.eb - GDAL-3.6.2-foss-2022b.eb - waLBerla-6.1-foss-2022b.eb: From 9238a031b245c4e023ef1bcd96dcc2e3f6a39800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 16:35:49 +0100 Subject: [PATCH 1696/1795] add TensorFlow 2.13.0 (and some dependencies) --- .../sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml index e2a4aed631..395a0e685d 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -11,3 +11,15 @@ easyconfigs: - poetry-1.5.1-GCCcore-12.3.0.eb - git-2.41.0-GCCcore-12.3.0-nodocs.eb - flit-3.9.0-GCCcore-12.3.0.eb + - foss-2023a.eb + - pybind11-2.11.1-GCCcore-12.3.0.eb: + # avoid indirect dependency on old CMake version built with GCCcore/10.2.0 via Catch2 build dependency; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 + options: + from-pr: 19270 + - SciPy-bundle-2023.07-gfbf-2023a.eb + - TensorFlow-2.13.0-foss-2023a.eb: + # patch setup.py for grpcio extension in TensorFlow 2.13.0 easyconfigs to take into account alternate sysroot; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19268 + options: + from-pr: 19268 From 6ba93988b4806b8a99d4bbae73c16f78ce6fe4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 16:42:03 +0100 Subject: [PATCH 1697/1795] add bunch of apps from 2023b easystack --- .../eessi-2023.06-eb-4.9.0-2023b.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index 331881efdd..b58a56a2b4 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -16,3 +16,41 @@ easyconfigs: - expat-2.5.0-GCCcore-13.2.0.eb - foss-2023b.eb - SciPy-bundle-2023.11-gfbf-2023b.eb + - netCDF-4.9.2-gompi-2023b.eb: + options: + from-pr: 19534 + - matplotlib-3.8.2-gfbf-2023b.eb: + options: + from-pr: 19552 + - AOFlagger-3.4.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 + - arpack-ng-3.9.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 + - Armadillo-12.8.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 + - casacore-3.5.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 + - IDG-1.2.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 + - EveryBeam-0.5.2-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 + - DP3-6.0-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 + - WSClean-3.4-foss-2023b.eb: + options: + from-pr: 19840 + include-easyblocks-from-pr: 3088 From 4e9b74dcf972baf433a73e730b6290f82b36223f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 20:58:08 +0100 Subject: [PATCH 1698/1795] use from-pr for OpenBLAS (required for Sapphire Rapids support) --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml index 395a0e685d..d1dc67660d 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -11,6 +11,11 @@ easyconfigs: - poetry-1.5.1-GCCcore-12.3.0.eb - git-2.41.0-GCCcore-12.3.0-nodocs.eb - flit-3.9.0-GCCcore-12.3.0.eb + - OpenBLAS-0.3.23-GCC-12.3.0.eb: + options: + # required for Intel Sapphire Rapids support + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19159 + from-pr: 19159 - foss-2023a.eb - pybind11-2.11.1-GCCcore-12.3.0.eb: # avoid indirect dependency on old CMake version built with GCCcore/10.2.0 via Catch2 build dependency; From 1d097dd297477bd2f948fe5c5b1006e4d81d9ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 20 Jan 2025 22:24:08 +0100 Subject: [PATCH 1699/1795] include fixes for SciPy-bundle from PR 21693 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml index d1dc67660d..5b22019772 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -22,7 +22,9 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19270 options: from-pr: 19270 - - SciPy-bundle-2023.07-gfbf-2023a.eb + - SciPy-bundle-2023.07-gfbf-2023a.eb: + options: + from-pr: 21693 - TensorFlow-2.13.0-foss-2023a.eb: # patch setup.py for grpcio extension in TensorFlow 2.13.0 easyconfigs to take into account alternate sysroot; # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19268 From 5fc7332110c3be11efa499f00ae19a1061d59231 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 21 Jan 2025 03:03:29 +0100 Subject: [PATCH 1700/1795] Apply suggestions from code review Co-authored-by: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> --- test_suite.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index d587e5a626..79986d9188 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -135,9 +135,9 @@ fi # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame # config file that matches the bot config. See https://gitlab.com/eessi/support/-/issues/114#note_2293660921 if [ -z "$RFM_CONFIG_FILES" ]; then - err_msg = "Please set RFM_CONFIG_FILES in the environment of this bot instance to point to a valid" - err_msg = "${err_msg} ReFrame configuration file that matches the bot config." - err_msg = "${err_msg} For more information, see https://gitlab.com/eessi/support/-/issues/114#note_2293660921" + err_msg="Please set RFM_CONFIG_FILES in the environment of this bot instance to point to a valid" + err_msg="${err_msg} ReFrame configuration file that matches the bot config." + err_msg="${err_msg} For more information, see https://gitlab.com/eessi/support/-/issues/114#note_2293660921" fatal_error "${err_msg}" fi export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests @@ -236,7 +236,7 @@ else fi # Allow people deploying the bot to overrwide this if [ -z "$REFRAME_SCALE_TAG" ]; then - REFRAME_SCALE_TAGS="--tag 1_node" + REFRAME_SCALE_TAG="--tag 1_node" fi if [ -z "$REFRAME_CI_TAG" ]; then REFRAME_CI_TAG="--tag CI" From ebe999ff58e375b7b4c6e920238508f5ca9416b7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 21 Jan 2025 03:21:31 +0100 Subject: [PATCH 1701/1795] Make sure the EESSI_ACCELERATOR_TARGET is also set for the test step, as we need it to append the right path to the MODULEPATH for the test_suite.sh --- bot/test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/test.sh b/bot/test.sh index 4205ea8564..597c277292 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -172,6 +172,10 @@ EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE echo "bot/test.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" +# determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} +export EESSI_ACCELERATOR_TARGET=$(cfg_get_value "architecture" "accelerator") +echo "bot/test.sh: EESSI_ACCELERATOR_TARGET='${EESSI_ACCELERATOR_TARGET}'" + # get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} From 24bd69af2bc5755189a02d62c335bdc30cc2dc85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 09:12:29 +0100 Subject: [PATCH 1702/1795] rebuild SciPy-bundle with additional patches --- ...121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml new file mode 100644 index 0000000000..96a334cc3c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml @@ -0,0 +1,14 @@ +# 2025.01.21 +# While adding support for Intel Sapphire Rapids, additional patches applied to the +# original easyconfig were required to successfully build SciPy-bundle 2023.07. +# In order to keep the stack consistent across the different CPUs, +# a rebuild is done for all CPU targets with this updated easyconfig. +# See: +# - https://github.com/easybuilders/easybuild-easyconfigs/pull/19419 +# - https://github.com/easybuilders/easybuild-easyconfigs/pull/20817 +# - https://github.com/easybuilders/easybuild-easyconfigs/pull/21693 +easyconfigs: + - SciPy-bundle-2023.07-gfbf-2023a.eb + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21693 + from-commit: 7c5144d2c1a061cd9f08b5901970b7f6ec5eb5c0 From 27978a9e9364a6d8ee9e876dd7e6d33be44d1f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 09:39:23 +0100 Subject: [PATCH 1703/1795] add missing colon --- .../20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml index 96a334cc3c..f7db7a994c 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250121-eb-4.9.4-SciPy-bundle-2023.07-bug-fixes.yml @@ -8,7 +8,7 @@ # - https://github.com/easybuilders/easybuild-easyconfigs/pull/20817 # - https://github.com/easybuilders/easybuild-easyconfigs/pull/21693 easyconfigs: - - SciPy-bundle-2023.07-gfbf-2023a.eb + - SciPy-bundle-2023.07-gfbf-2023a.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21693 from-commit: 7c5144d2c1a061cd9f08b5901970b7f6ec5eb5c0 From 02369ebd4c011d8c8db940dea88065fcad723c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 09:50:55 +0100 Subject: [PATCH 1704/1795] add remaining EB 4.9.0 2023b builds --- .../eessi-2023.06-eb-4.9.0-2023b.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml index b58a56a2b4..23fc934645 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -54,3 +54,21 @@ easyconfigs: options: from-pr: 19840 include-easyblocks-from-pr: 3088 + - CDO-2.2.2-gompi-2023b.eb: + options: + from-pr: 19792 + - python-casacore-3.5.2-foss-2023b.eb: + options: + from-pr: 20089 + - libspatialindex-1.9.3-GCCcore-13.2.0.eb: + options: + from-pr: 19922 + - LittleCMS-2.15-GCCcore-13.2.0.eb + - giflib-5.2.1-GCCcore-13.2.0.eb + - OpenJPEG-2.5.0-GCCcore-13.2.0.eb + - libwebp-1.3.2-GCCcore-13.2.0.eb + - Wayland-1.22.0-GCCcore-13.2.0.eb + - Qt5-5.15.13-GCCcore-13.2.0.eb: + options: + from-pr: 20201 + - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb From a5e9c3301475256f6bb7972433ac27964c4e63ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 09:55:07 +0100 Subject: [PATCH 1705/1795] add remaining EB 4.8.2 2023a apps --- .../eessi-2023.06-eb-4.8.2-2023a.yml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml index 5b22019772..bd34313a10 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -30,3 +30,30 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19268 options: from-pr: 19268 + - X11-20230603-GCCcore-12.3.0.eb + - HarfBuzz-5.3.1-GCCcore-12.3.0.eb: + options: + from-pr: 19339 + - Qt5-5.15.10-GCCcore-12.3.0.eb + - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb + - LHAPDF-6.5.4-GCC-12.3.0.eb: + options: + from-pr: 19363 + - LoopTools-2.15-GCC-12.3.0.eb: + options: + from-pr: 19397 + - R-4.3.2-gfbf-2023a.eb: + options: + from-pr: 19185 + - Boost-1.82.0-GCC-12.3.0.eb + - netCDF-4.9.2-gompi-2023a.eb + - FFmpeg-6.0-GCCcore-12.3.0.eb + - ALL-0.9.2-foss-2023a.eb: + options: + from-pr: 19455 + - CDO-2.2.2-gompi-2023a.eb: + options: + from-pr: 19735 + - BWA-0.7.17-20220923-GCCcore-12.3.0.eb: + options: + from-pr: 19820 From 5abe39ebbb24d064a8544debfc513e4e22088b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 12:26:39 +0100 Subject: [PATCH 1706/1795] try workaround for permission issues with rebuilds --- EESSI-install-software.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2f05953158..bd289e7f91 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -336,7 +336,11 @@ else if [ -f ${easystack_file} ]; then echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - ${EB} --easystack ${easystack_file} --robot + if [[ $stack == *"/rebuilds/"* ]]; then + ${EB} --easystack ${easystack_file} --robot --try-amend=keeppreviousinstall=True + else + ${EB} --easystack ${easystack_file} --robot + fi ec=$? # copy EasyBuild log file if EasyBuild exited with an error From 2c31511e0cbbbcc96fcd8d69635890a66514c845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 13:37:12 +0100 Subject: [PATCH 1707/1795] ls on the new dir --- EESSI-remove-software.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 1a03a7af98..f512fb61d6 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -131,7 +131,10 @@ if [ $EUID -eq 0 ]; then rm -rf ${app_module} # recreate some directory to work around permission denied # issues when rebuilding the package + mkdir -p ${app_dir} + ls ${app_dir} mkdir -p ${app_dir}/easybuild + ls ${app_dir}/easybuild done else fatal_error "Easystack file ${easystack_file} not found!" From b8313337c9cc708ae14cf34a650c21b9abe857ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 13:51:48 +0100 Subject: [PATCH 1708/1795] add echo command for debugging --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index f512fb61d6..98b25c394d 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -132,8 +132,8 @@ if [ $EUID -eq 0 ]; then # recreate some directory to work around permission denied # issues when rebuilding the package mkdir -p ${app_dir} - ls ${app_dir} mkdir -p ${app_dir}/easybuild + echo "Running ls ${app_dir}/easybuild" ls ${app_dir}/easybuild done else From 5fefebc7a76047024fdcd5532f7dcde28bcbea27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 14:08:54 +0100 Subject: [PATCH 1709/1795] recreate all subdirs of removed app --- EESSI-remove-software.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 98b25c394d..b2b0e59f7f 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -127,14 +127,17 @@ if [ $EUID -eq 0 ]; then app_dir=${app_installprefix}/software/${app} app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." + app_subdirs=$(find ${app_dir} -mindepth 1 -maxdepth 1 -type d) rm -rf ${app_dir} rm -rf ${app_module} # recreate some directory to work around permission denied # issues when rebuilding the package mkdir -p ${app_dir} - mkdir -p ${app_dir}/easybuild - echo "Running ls ${app_dir}/easybuild" - ls ${app_dir}/easybuild + for app_subdir in ${app_subdirs}; do + echo "Recreating ${app_subdir}..." + mkdir -p ${app_subdir} + ls ${app_subdir} + done done else fatal_error "Easystack file ${easystack_file} not found!" From 1219b3438acda853f2d9c7cf662dc4893d985d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 14:22:25 +0100 Subject: [PATCH 1710/1795] fix check for rebuilds easystacks --- EESSI-install-software.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index bd289e7f91..aa7e5dd729 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -336,7 +336,8 @@ else if [ -f ${easystack_file} ]; then echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - if [[ $stack == *"/rebuilds/"* ]]; then + if [[ ${easystack_file} == *"/rebuilds/"* ]]; then + echo_yellow "This is a rebuild, so using --try-amend=keeppreviousinstall=True to reuse the already created directory" ${EB} --easystack ${easystack_file} --robot --try-amend=keeppreviousinstall=True else ${EB} --easystack ${easystack_file} --robot From 9d3d177681a68c9bed2623e89ff1c8589935b984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 14:53:14 +0100 Subject: [PATCH 1711/1795] add easystack for EB 4.9.1 / 2022b --- .../sapphire_rapids/eessi-2023.06-eb-4.9.1-2022b.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2022b.yml new file mode 100644 index 0000000000..1805c581c3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2022b.yml @@ -0,0 +1,8 @@ +easyconfigs: + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: + options: + from-pr: 20379 + - ParaView-5.11.1-foss-2022b.eb + - ASE-3.22.1-gfbf-2022b.eb + - SEPP-4.5.1-foss-2022b.eb + - Valgrind-3.21.0-gompi-2022b.eb From 98588e66b172866a339bb2af63a3e2a51fefd2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 14:55:59 +0100 Subject: [PATCH 1712/1795] add remaining apps for EB 4.9.1 / 2023b --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml index 4ad5c6d95a..987d8c2625 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml @@ -10,3 +10,7 @@ easyconfigs: options: # from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 from-pr: 20522 + - NLTK-3.8.1-foss-2023b.eb + - Valgrind-3.23.0-gompi-2023b.eb: + options: + from-pr: 20792 From ab82e6d84c514176fc2e9cf1afc978139f2da65e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 15:42:11 +0100 Subject: [PATCH 1713/1795] reuse existing directory for rebuilds --- EESSI-install-software.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2f05953158..3a9ba175c9 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -336,7 +336,15 @@ else if [ -f ${easystack_file} ]; then echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - ${EB} --easystack ${easystack_file} --robot + if [[ ${easystack_file} == *"/rebuilds/"* ]]; then + # the removal script should have removed the original directory and created a new and empty one + # to work around permission issues: + # https://github.com/EESSI/software-layer/issues/556 + echo_yellow "This is a rebuild, so using --try-amend=keeppreviousinstall=True to reuse the already created directory" + ${EB} --easystack ${easystack_file} --robot --try-amend=keeppreviousinstall=True + else + ${EB} --easystack ${easystack_file} --robot + fi ec=$? # copy EasyBuild log file if EasyBuild exited with an error From 0a85fd21eb0fa37605e8976d928edc19ab3a6008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 15:45:21 +0100 Subject: [PATCH 1714/1795] recreate installation dir and first-level subdirs after it has been removed --- EESSI-remove-software.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 1a03a7af98..e4b377fd16 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -125,13 +125,18 @@ if [ $EUID -eq 0 ]; then # Two dirname invocations, so returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2 app_installprefix=$(dirname $(dirname ${app_modulepath})) app_dir=${app_installprefix}/software/${app} + app_subdirs=$(find ${app_dir} -mindepth 1 -maxdepth 1 -type d) app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${app_dir} rm -rf ${app_module} - # recreate some directory to work around permission denied - # issues when rebuilding the package - mkdir -p ${app_dir}/easybuild + # recreate the installation directories and first-level subdirectories to work around permission denied + # issues when rebuilding the package (see https://github.com/EESSI/software-layer/issues/556) + echo_yellow "Recreating an empty ${app_dir}..." + mkdir -p ${app_dir} + for app_subdir in ${app_subdirs}; do + mkdir -p ${app_subdir} + done done else fatal_error "Easystack file ${easystack_file} not found!" From 30659d62e6f807095b1558475eca5c7499085aff Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 21 Jan 2025 19:44:15 +0100 Subject: [PATCH 1715/1795] OSU test got renamed in the test suite. This changes the name accordingly in the test mapping --- tests/eessi_test_mapping/software_to_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/eessi_test_mapping/software_to_tests.yml b/tests/eessi_test_mapping/software_to_tests.yml index 626477781f..3a162666fc 100644 --- a/tests/eessi_test_mapping/software_to_tests.yml +++ b/tests/eessi_test_mapping/software_to_tests.yml @@ -25,11 +25,11 @@ mappings: LAMMPS/*: - EESSI_LAMMPS OSU-Micro-Benchmarks/*: - - EESSI_OSU_Micro_Benchmarks + - EESSI_OSU GROMACS/*: - EESSI_GROMACS default_tests: # Low level tests - - EESSI_OSU_Micro_Benchmarks + - EESSI_OSU # A very quick-to-run high level application test - EESSI_LAMMPS From 0cf0cedecc96730daca251ea6226632d3710432f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 23:37:13 +0100 Subject: [PATCH 1716/1795] add dependencies of OpenFOAM --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index b2cf9bf86c..19f80d71fe 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -5,3 +5,8 @@ easyconfigs: - OpenMPI-4.1.5-GCC-12.3.0: options: from-pr: 19940 + - METIS-5.1.0-GCCcore-12.3.0.eb + - SCOTCH-7.0.3-gompi-2023a.eb + - CGAL-5.6-GCCcore-12.3.0.eb + - ParaView-5.11.2-foss-2023a.eb + - gnuplot-5.4.8-GCCcore-12.3.0.eb From b66853809c7af7c0d6531e66af28c28671e5da81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 21 Jan 2025 23:38:01 +0100 Subject: [PATCH 1717/1795] add OpenFOAM 10 and 11 --- .../sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml index e4e3238ce7..7257ec4dae 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,3 +1,12 @@ easyconfigs: - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb - at-spi2-core-2.49.91-GCCcore-12.3.0.eb + - OpenFOAM-10-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - OpenFOAM-11-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + From f2ad957d26da2d839922b33e6ee7822987e59da5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 22 Jan 2025 00:35:20 +0100 Subject: [PATCH 1718/1795] Create logic to hide 2022b-based modules on zen4. Can be tested by running EESSI_SOFTWARE_SUBDIR_OVERRIDE=x86_64/amd/zen4 python3 create_lmodsitepackage.py and then using LMOD_PACKAGE_PATH=/.lmod module --show_hidden av GCCcore-12.2.2. This will show that all these modules are indeed hidden --- create_lmodsitepackage.py | 41 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index e959572ab1..4f3d6771ff 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -8,10 +8,13 @@ DOT_LMOD = '.lmod' -hook_txt = """require("strict") +hook_prologue = """require("strict") local hook = require("Hook") local open = io.open +""" + +hook_txt = """ local function read_file(path) local file = open(path, "rb") -- r read mode and b binary mode if not file then return nil end @@ -203,11 +206,47 @@ hook.register("load", eessi_load_hook) +""" + +hook_epilogue = """ -- Note that this needs to happen at the end, so that any EESSI specific hooks can be overwritten by the site load_site_specific_hooks() """ +# This hook is only for zen4. +hook_txt_zen4 = """ +local function hide_2022b_modules(modT) + -- modT is a table with: fullName, sn, fn and isVisible + -- The latter is a boolean to determine if a module is visible or not + + local tcver = modT.fullName:match("gfbf%-(20[0-9][0-9][ab])") or + modT.fullName:match("gompi%-(20[0-9][0-9][ab])") or + modT.fullName:match("foss%-(20[0-9][0-9][ab])") or + modT.fullName:match("GCC%-([0-9]*.[0-9]*.[0-9]*)") or + modT.fullName:match("GCCcore%-([0-9]*.[0-9]*.[0-9]*)") + + -- if nothing matches, return + if tcver == nil then return end + + -- if we have matches, check if the toolchain version is either 2022b or 12.2.0 + if parseVersion(tcver) == parseVersion("2022b") or parseVersion(tcver) == parseVersion("12.2.0") then + modT.isVisible = false + end +end + +hook.register("isVisibleHook", hide_2022b_modules) +""" + +# Append conditionally for zen4 +eessi_software_subdir_override = os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE") +if eessi_software_subdir_override == "x86_64/amd/zen4": + hook_txt = hook_txt + hook_txt_zen4 + +# Concatenate hook prologue, body and epilogue +# Note that this has to happen after any conditional items have been added to the hook_txt +hook_txt = hook_prologue + hook_txt + hook_epilogue + def error(msg): sys.stderr.write("ERROR: %s\n" % msg) sys.exit(1) From 0fbaadf5323e6007af1366bc85df34bf0957584c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 22 Jan 2025 00:38:29 +0100 Subject: [PATCH 1719/1795] add all 2022b things to zen4 --- .../zen4/eessi-2023.06-eb-4.8.2-2022b.yml | 7 +++ .../zen4/eessi-2023.06-eb-4.9.0-2022b.yml | 13 +++++ .../zen4/eessi-2023.06-eb-4.9.1-2022b.yml | 8 +++ .../zen4/eessi-2023.06-eb-4.9.2-2022b.yml | 52 +++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml new file mode 100644 index 0000000000..fd88fafb0c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml @@ -0,0 +1,7 @@ +easyconfigs: + - foss-2022b.eb + - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: + options: + from-pr: 19339 + - Qt5-5.15.7-GCCcore-12.2.0.eb + - QuantumESPRESSO-7.2-foss-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml new file mode 100644 index 0000000000..3962f63bda --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml @@ -0,0 +1,13 @@ +easyconfigs: + - SciPy-bundle-2023.02-gfbf-2022b.eb + - GDAL-3.6.2-foss-2022b.eb + - waLBerla-6.1-foss-2022b.eb: + options: + from-pr: 19324 + - WRF-4.4.1-foss-2022b-dmpar.eb + - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb: + options: + from-pr: 20086 + - R-4.2.2-foss-2022b.eb: + options: + from-pr: 20238 diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2022b.yml new file mode 100644 index 0000000000..1805c581c3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2022b.yml @@ -0,0 +1,8 @@ +easyconfigs: + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: + options: + from-pr: 20379 + - ParaView-5.11.1-foss-2022b.eb + - ASE-3.22.1-gfbf-2022b.eb + - SEPP-4.5.1-foss-2022b.eb + - Valgrind-3.21.0-gompi-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2022b.yml new file mode 100644 index 0000000000..969b0d469b --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2022b.yml @@ -0,0 +1,52 @@ +easyconfigs: + - BLAST+-2.14.0-gompi-2022b.eb + - BioPerl-1.7.8-GCCcore-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + - gnuplot-5.4.6-GCCcore-12.2.0.eb + - h5py-3.8.0-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - ncbi-vdb-3.0.5-gompi-2022b.eb + - Bio-DB-HTS-3.01-GCC-12.2.0.eb + - MAFFT-7.505-GCC-12.2.0-with-extensions.eb + - MetaEuk-6-GCC-12.2.0.eb + - BamTools-2.5.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - Mash-2.3-GCC-12.2.0.eb + - CapnProto-0.10.3-GCCcore-12.2.0.eb + - WhatsHap-2.1-foss-2022b.eb + - SAMtools-1.17-GCC-12.2.0.eb + - Bowtie2-2.5.1-GCC-12.2.0.eb + - CD-HIT-4.8.1-GCC-12.2.0.eb + - VCFtools-0.1.16-GCC-12.2.0.eb + - GenomeTools-1.6.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - parallel-20230722-GCCcore-12.2.0.eb + - BCFtools-1.17-GCC-12.2.0.eb + - lpsolve-5.5.2.11-GCC-12.2.0.eb + - fastp-0.23.4-GCC-12.2.0.eb + - KronaTools-2.8.1-GCCcore-12.2.0.eb + - MultiQC-1.14-foss-2022b.eb + - CGAL-5.5.2-GCCcore-12.2.0.eb + - KaHIP-3.14-gompi-2022b.eb + - MPC-1.3.1-GCCcore-12.2.0.eb + - MUMPS-5.6.1-foss-2022b-metis.eb + - GL2PS-1.4.2-GCCcore-12.2.0.eb + - GST-plugins-base-1.22.1-GCC-12.2.0.eb + - wxWidgets-3.2.2.1-GCC-12.2.0.eb + - Archive-Zip-1.68-GCCcore-12.2.0.eb + - jemalloc-5.3.0-GCCcore-12.2.0.eb + - Judy-1.0.5-GCCcore-12.2.0.eb + - libaio-0.3.113-GCCcore-12.2.0.eb + - Z3-4.12.2-GCCcore-12.2.0.eb + - tbb-2021.10.0-GCCcore-12.2.0.eb + - dask-2023.7.1-foss-2022b.eb + - netcdf4-python-1.6.3-foss-2022b.eb + - Ruby-3.2.2-GCCcore-12.2.0.eb + - ROOT-6.26.10-foss-2022b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From 8e6816f608305be146e327b4b71dd5f9780e2945 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 22 Jan 2025 02:18:55 +0100 Subject: [PATCH 1720/1795] Add Python and OpenMPI which were rebuild in the rebuilds dir for other architectures --- .../2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml | 4 ++++ .../2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml index fd88fafb0c..ddc783d11a 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml @@ -1,4 +1,8 @@ easyconfigs: + # Adding OpenMPI explicitely, as defined in 20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml + - OpenMPI-4.1.4-GCC-12.2.0.eb: + options: + from-pr: 19940 - foss-2022b.eb - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: options: diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml index 3962f63bda..3fcfd03048 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,4 +1,13 @@ easyconfigs: + # Adding Python explicitely, as defined in 20240729-eb-4.9.2-Python-ctypes.yml + - Python-3.10.8-GCCcore-12.2.0-bare: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + - Python-3.10.8-GCCcore-12.2.0: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f - SciPy-bundle-2023.02-gfbf-2022b.eb - GDAL-3.6.2-foss-2022b.eb - waLBerla-6.1-foss-2022b.eb: From e1b52db44d89cf93d186b2db80630d741a5a2bd8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 22 Jan 2025 02:24:52 +0100 Subject: [PATCH 1721/1795] Make sure we do proper CI check on zen4, including the 2022b stuff, since we now put that in place with hidden modules that print an error if loaded --- .github/workflows/test-software.eessi.io.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index f569c664c9..b8104a543f 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -59,20 +59,6 @@ jobs: # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do - if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then - if grep -q 2022b <<<"${easystack_file}"; then - # skip the check of installed software on zen4 for foss/2022b builds - continue - fi - if [[ $easystack_file == *"rebuilds"* ]]; then - # Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that - new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml) - # first clean out the options then clean out the .eb name - sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack - diff --unified=0 "$easystack_file" "$new_easystack" || : - easystack_file="$new_easystack" - fi - fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 361956677954eee2406d74ef74f12440c3c0c8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 22 Jan 2025 11:26:57 +0100 Subject: [PATCH 1722/1795] add ESPResSO, Rivet, PyTorch --- .../sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 19f80d71fe..2f6c344123 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -10,3 +10,13 @@ easyconfigs: - CGAL-5.6-GCCcore-12.3.0.eb - ParaView-5.11.2-foss-2023a.eb - gnuplot-5.4.8-GCCcore-12.3.0.eb + - ESPResSo-4.2.1-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19592 + options: + from-pr: 19592 + - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb: + options: + from-pr: 19679 + - PyTorch-2.1.2-foss-2023a.eb: + options: + from-pr: 19573 From e4bfc2cb5e9889e03e43744d9b94c8d658224070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 22 Jan 2025 11:30:29 +0100 Subject: [PATCH 1723/1795] add remaining apps from EB 4.9.2 2023b easystack --- .../eessi-2023.06-eb-4.9.2-2023b.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml index 4d9f5411ed..f719fb7bab 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,3 +1,32 @@ easyconfigs: - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb - at-spi2-core-2.50.0-GCCcore-13.2.0.eb + - IPython-8.17.2-GCCcore-13.2.0.eb + - dlb-3.4-gompi-2023b.eb + - pystencils-1.3.4-gfbf-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 + from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 + - Extrae-4.2.0-gompi-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21017 + from-commit: 120f4d56efebd2bc61382db4c84a664a339c66cf + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3393 + include-easyblocks-from-commit: c4951c78d62fa5cf8e9f6fe0ead212d2a4d7cb9c + - pyMBE-0.8.0-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 + from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 + - STAR-2.7.11b-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 + from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00 + - HPL-2.3-foss-2023b.eb + - Brunsli-0.1-GCCcore-13.2.0.eb: + options: + # https://github.com/easybuilders/easybuild-easyconfigs/pull/21366 + from-commit: 1736a123b1685836452587a5c51793257570bb2d + - R-bundle-CRAN-2024.06-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21285 + from-commit: 41a2cd83f9fb017b76f0693f6a264d8acb548317 From 5164681ef8cd5a6c3b3daa09a29b4c5e3e5803fc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 22 Jan 2025 15:50:45 +0100 Subject: [PATCH 1724/1795] Use include-easyblocks-from-pr instead of from-commit, since that wasnt known yet to this eb version --- .../2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml index 3fcfd03048..763fcb20a8 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml @@ -3,11 +3,17 @@ easyconfigs: - Python-3.10.8-GCCcore-12.2.0-bare: options: # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 - include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + # Can't use include-easyblocks-from-commit here, as was done for rebuilds for the other archs + # because the eb version from this easystack isn't new enough to know it + # include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + include-easyblocks-from-pr: 3352 - Python-3.10.8-GCCcore-12.2.0: options: # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 - include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + # Can't use include-easyblocks-from-commit here, as was done for rebuilds for the other archs + # because the eb version from this easystack isn't new enough to know it + # include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + include-easyblocks-from-pr: 3352 - SciPy-bundle-2023.02-gfbf-2022b.eb - GDAL-3.6.2-foss-2022b.eb - waLBerla-6.1-foss-2022b.eb: From 9ef1033315d7d751fd3df65fc073de3f56fd98f0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 22 Jan 2025 16:47:43 +0100 Subject: [PATCH 1725/1795] Added DP3, EveryBeam and WSClean from EB PR #21765 to test build against EESSI --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index ee161b8b9c..5d20176dc8 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -25,3 +25,15 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21915 from-commit: 58f16c0caf8c5494c68e9eda8cbf19e9145d3cfa + - DP3-6.2-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: bf9fe67ec7d14f823c32081d52eb9d2763e8dcd0 + - WSClean-3.5-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: bf9fe67ec7d14f823c32081d52eb9d2763e8dcd0 + - EveryBeam-0.6.1-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: bf9fe67ec7d14f823c32081d52eb9d2763e8dcd0 From faa4d38859c24131e71eed50b21c585e454ad7f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 22 Jan 2025 23:42:13 +0100 Subject: [PATCH 1726/1795] add remaining apps from EB 4.9.2 2022b easystack --- .../eessi-2023.06-eb-4.9.2-2022b.yml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml index 8ec0067465..e5464dc4e2 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml @@ -1,2 +1,53 @@ easyconfigs: - GObject-Introspection-1.74.0-GCCcore-12.2.0.eb + - BLAST+-2.14.0-gompi-2022b.eb + - BioPerl-1.7.8-GCCcore-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + - gnuplot-5.4.6-GCCcore-12.2.0.eb + - h5py-3.8.0-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - ncbi-vdb-3.0.5-gompi-2022b.eb + - Bio-DB-HTS-3.01-GCC-12.2.0.eb + - MAFFT-7.505-GCC-12.2.0-with-extensions.eb + - MetaEuk-6-GCC-12.2.0.eb + - BamTools-2.5.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - Mash-2.3-GCC-12.2.0.eb + - CapnProto-0.10.3-GCCcore-12.2.0.eb + - WhatsHap-2.1-foss-2022b.eb + - SAMtools-1.17-GCC-12.2.0.eb + - Bowtie2-2.5.1-GCC-12.2.0.eb + - CD-HIT-4.8.1-GCC-12.2.0.eb + - VCFtools-0.1.16-GCC-12.2.0.eb + - GenomeTools-1.6.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - parallel-20230722-GCCcore-12.2.0.eb + - BCFtools-1.17-GCC-12.2.0.eb + - lpsolve-5.5.2.11-GCC-12.2.0.eb + - fastp-0.23.4-GCC-12.2.0.eb + - KronaTools-2.8.1-GCCcore-12.2.0.eb + - MultiQC-1.14-foss-2022b.eb + - CGAL-5.5.2-GCCcore-12.2.0.eb + - KaHIP-3.14-gompi-2022b.eb + - MPC-1.3.1-GCCcore-12.2.0.eb + - MUMPS-5.6.1-foss-2022b-metis.eb + - GL2PS-1.4.2-GCCcore-12.2.0.eb + - GST-plugins-base-1.22.1-GCC-12.2.0.eb + - wxWidgets-3.2.2.1-GCC-12.2.0.eb + - Archive-Zip-1.68-GCCcore-12.2.0.eb + - jemalloc-5.3.0-GCCcore-12.2.0.eb + - Judy-1.0.5-GCCcore-12.2.0.eb + - libaio-0.3.113-GCCcore-12.2.0.eb + - Z3-4.12.2-GCCcore-12.2.0.eb + - tbb-2021.10.0-GCCcore-12.2.0.eb + - dask-2023.7.1-foss-2022b.eb + - netcdf4-python-1.6.3-foss-2022b.eb + - Ruby-3.2.2-GCCcore-12.2.0.eb + - ROOT-6.26.10-foss-2022b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From 9dac8c3b96cf8aa4e2ec96225dc4c7681fc3e6fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 22 Jan 2025 23:47:41 +0100 Subject: [PATCH 1727/1795] remove PyTorch --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 2f6c344123..7661bf3166 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -17,6 +17,3 @@ easyconfigs: - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb: options: from-pr: 19679 - - PyTorch-2.1.2-foss-2023a.eb: - options: - from-pr: 19573 From f3cd59cebb17e724152b6f818deaa03016935b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 10:00:33 +0100 Subject: [PATCH 1728/1795] add bunch of dependencies for PyTorch 2.1.2 --- .../sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 7661bf3166..c995222178 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -17,3 +17,11 @@ easyconfigs: - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb: options: from-pr: 19679 + - Pillow-10.0.0-GCCcore-12.3.0.eb + - sympy-1.12-gfbf-2023a.eb + - networkx-3.1-gfbf-2023a.eb + - expecttest-0.1.5-GCCcore-12.3.0.eb + - PyYAML-6.0-GCCcore-12.3.0.eb + - pytest-flakefinder-1.1.0-GCCcore-12.3.0.eb + - pytest-rerunfailures-12.0-GCCcore-12.3.0.eb + - pytest-shard-0.1.2-GCCcore-12.3.0.eb From 2ab2b6f60fb2278adb9918138343ffca4647df8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 11:08:42 +0100 Subject: [PATCH 1729/1795] use pytest-* easyconfigs from PR 20164 --- .../eessi-2023.06-eb-4.9.0-2023a.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index c995222178..79b371f229 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -22,6 +22,15 @@ easyconfigs: - networkx-3.1-gfbf-2023a.eb - expecttest-0.1.5-GCCcore-12.3.0.eb - PyYAML-6.0-GCCcore-12.3.0.eb - - pytest-flakefinder-1.1.0-GCCcore-12.3.0.eb - - pytest-rerunfailures-12.0-GCCcore-12.3.0.eb - - pytest-shard-0.1.2-GCCcore-12.3.0.eb + - pytest-flakefinder-1.1.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20164 + from-pr: 20164 + - pytest-rerunfailures-12.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20164 + from-pr: 20164 + - pytest-shard-0.1.2-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20164 + from-pr: 20164 From 522b2cad7a3e7220307a3c50e062dd99ef083d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 11:11:07 +0100 Subject: [PATCH 1730/1795] actually, use the PyTorch PR itself for its dependencies --- .../sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 79b371f229..9717a1a039 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -24,13 +24,13 @@ easyconfigs: - PyYAML-6.0-GCCcore-12.3.0.eb - pytest-flakefinder-1.1.0-GCCcore-12.3.0.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20164 - from-pr: 20164 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19573 + from-pr: 19573 - pytest-rerunfailures-12.0-GCCcore-12.3.0.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20164 - from-pr: 20164 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19573 + from-pr: 19573 - pytest-shard-0.1.2-GCCcore-12.3.0.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20164 - from-pr: 20164 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19573 + from-pr: 19573 From e84f1e4ae2d454eac6b88be41cf8a7621a7aa13c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 12:11:05 +0100 Subject: [PATCH 1731/1795] use brunsli commit for R-bundle-CRAN --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml index f719fb7bab..4511a604a9 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml @@ -29,4 +29,4 @@ easyconfigs: - R-bundle-CRAN-2024.06-foss-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21285 - from-commit: 41a2cd83f9fb017b76f0693f6a264d8acb548317 + from-commit: 1736a123b1685836452587a5c51793257570bb2d From f82d3e7aaea20e21c8a1578ccff19a8b66dc80c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 12:22:38 +0100 Subject: [PATCH 1732/1795] update comment about R-bundle-CRAN commit --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml index 4511a604a9..c5f4110ac3 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml @@ -29,4 +29,7 @@ easyconfigs: - R-bundle-CRAN-2024.06-foss-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21285 + # the commit we use is not from the PR , but the same one as used for Brunsli, + # because otherwise the missing installation script still complains about a missing Highway + # note: this commit has the same version of R-bundle-CRAN as merged by PR #21285 from-commit: 1736a123b1685836452587a5c51793257570bb2d From b2d094d61daa036c2cea63b248b8bd11bea0f924 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 23 Jan 2025 13:42:59 +0100 Subject: [PATCH 1733/1795] Make sure the latest release of the test suite is used --- run_tests.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 1dbb47db9d..cd57476f2b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -17,8 +17,14 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults +# Make sure we clone the latest version. This assumes versions are of the format "v1.2.3", then picks the latest +# then checks it out +TEST_CLONE="git clone https://github.com/EESSI/test-suite EESSI-test-suite && cd EESSI-test-suite" +LATEST_VERSION="VERSION=$(git tag | grep "^v[0-9]\+\.[0-9]\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)" +CHECKOUT_LATEST="git checkout ${VERSION}" + # Git clone has to be run in compat layer, to make the git command available -./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite" +./run_in_compat_layer_env.sh "${TEST_CLONE} && ${LATEST_VERSION} && ${CHECKOUT_LATEST}" # Run the test suite ./test_suite.sh "$@" From 5e06dffc08d7da047451c2b7de67f20c7994c4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 13:45:55 +0100 Subject: [PATCH 1734/1795] using the PyTorch PR for deps doesn't work, stick to the original PR --- .../eessi-2023.06-eb-4.9.0-2023a.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 9717a1a039..bd2d75744f 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -24,13 +24,10 @@ easyconfigs: - PyYAML-6.0-GCCcore-12.3.0.eb - pytest-flakefinder-1.1.0-GCCcore-12.3.0.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19573 - from-pr: 19573 - - pytest-rerunfailures-12.0-GCCcore-12.3.0.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19573 - from-pr: 19573 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19480 + from-pr: 19480 + - pytest-rerunfailures-12.0-GCCcore-12.3.0.eb - pytest-shard-0.1.2-GCCcore-12.3.0.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19573 - from-pr: 19573 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19480 + from-pr: 19480 From 3d848aaf0881db51632b7451349bcabf0c1c1609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 14:46:03 +0100 Subject: [PATCH 1735/1795] reduce number of builds --- .../eessi-2023.06-eb-4.9.2-2023b.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml index c5f4110ac3..35ccab6beb 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml @@ -17,19 +17,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 - - STAR-2.7.11b-GCC-13.2.0.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 - from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00 - - HPL-2.3-foss-2023b.eb - - Brunsli-0.1-GCCcore-13.2.0.eb: - options: - # https://github.com/easybuilders/easybuild-easyconfigs/pull/21366 - from-commit: 1736a123b1685836452587a5c51793257570bb2d - - R-bundle-CRAN-2024.06-foss-2023b.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21285 - # the commit we use is not from the PR , but the same one as used for Brunsli, - # because otherwise the missing installation script still complains about a missing Highway - # note: this commit has the same version of R-bundle-CRAN as merged by PR #21285 - from-commit: 1736a123b1685836452587a5c51793257570bb2d + From 33f4321eed4f88155f2c8cd21dd56be7ba2f5c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 16:26:24 +0100 Subject: [PATCH 1736/1795] add PyTorch --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index bd2d75744f..8ba30ddf8d 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -31,3 +31,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19480 from-pr: 19480 + - PyTorch-2.1.2-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19573 + from-pr: 19573 From da98c52225192dd1bcc872aa40abee2202fa50b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 16:30:47 +0100 Subject: [PATCH 1737/1795] increase max_failed_tests for PyTorch 2.1.2 to 4 for Sapphire Rapids --- eb_hooks.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 08eb5ea62e..9f3793a0db 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -26,6 +26,7 @@ CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' CPU_TARGET_A64FX = 'aarch64/a64fx' +CPU_TARGET_SAPPHIRE_RAPIDS = 'x86_64/intel/sapphire_rapids' CPU_TARGET_ZEN4 = 'x86_64/amd/zen4' EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -817,11 +818,16 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): """ - Pre-test hook for PyTorch: increase max failing tests for ARM for PyTorch 2.1.2 - See https://github.com/EESSI/software-layer/pull/444#issuecomment-1890416171 + Pre-test hook for PyTorch: increase max failing tests for ARM and Intel Sapphire Rapids for PyTorch 2.1.2 + See https://github.com/EESSI/software-layer/pull/444#issuecomment-1890416171 and + https://github.com/EESSI/software-layer/pull/875#issuecomment-2606854400 """ - if self.name == 'PyTorch' and self.version == '2.1.2' and get_cpu_architecture() == AARCH64: - self.cfg['max_failed_tests'] = 10 + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'PyTorch' and self.version == '2.1.2': + if get_cpu_architecture() == AARCH64: + self.cfg['max_failed_tests'] = 10 + if cpu_target == CPU_TARGET_SAPPHIRE_RAPIDS: + self.cfg['max_failed_tests'] = 4 def pre_single_extension_hook(ext, *args, **kwargs): From 960dba9055958eef946930aed58d1bbfd90577e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 16:49:49 +0100 Subject: [PATCH 1738/1795] add scikit-learn, snakemake, LAMMPS --- .../sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index bd2d75744f..2b53315214 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -31,3 +31,13 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19480 from-pr: 19480 + - scikit-learn-1.3.1-gfbf-2023a.eb + - snakemake-8.4.2-foss-2023a.eb: + options: + from-pr: 19646 + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19471 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3036 + options: + from-pr: 19471 + include-easyblocks-from-pr: 3036 From 12b30f650edf10c32166a7f5ac88144a9e23c93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 23 Jan 2025 17:02:09 +0100 Subject: [PATCH 1739/1795] add STAR, HPL, R-bundle-CRAN --- .../sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml index 4d9f5411ed..42d6b748ef 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,3 +1,13 @@ easyconfigs: - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb - at-spi2-core-2.50.0-GCCcore-13.2.0.eb + - STAR-2.7.11b-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 + from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00 + - HPL-2.3-foss-2023b.eb + - R-bundle-CRAN-2024.06-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21366 + # we use a commit from the Brunsli PR here to get rid of the Highway dependency + from-commit: 1736a123b1685836452587a5c51793257570bb2d From a2fc9e7d81f6b32f8f7fdfaa4363512d46edb458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 09:28:07 +0100 Subject: [PATCH 1740/1795] add PyTorch 2.1.2 issue (too many failing tests) for Sapphire Rapids --- eessi-2023.06-known-issues.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index c5cdc68941..bdef076dcd 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -56,3 +56,7 @@ - SciPy-bundle-2023.11-gfbf-2023b: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "2 failing tests (vs 54876 passed) in scipy test suite" +- x86_64/intel/sapphire_rapids: + - PyTorch-2.1.2-foss-2023a: + - issue: https://github.com/EESSI/software-layer/issues/461 + - info: "4 failing tests (out of 209567) on x86_64/intel/sapphire_rapids" From 081dd59bd5cd8580c500a47e3b875acdb26e8a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 10:55:13 +0100 Subject: [PATCH 1741/1795] restore original version --- run_tests.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index cd57476f2b..1dbb47db9d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -17,14 +17,8 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -# Make sure we clone the latest version. This assumes versions are of the format "v1.2.3", then picks the latest -# then checks it out -TEST_CLONE="git clone https://github.com/EESSI/test-suite EESSI-test-suite && cd EESSI-test-suite" -LATEST_VERSION="VERSION=$(git tag | grep "^v[0-9]\+\.[0-9]\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)" -CHECKOUT_LATEST="git checkout ${VERSION}" - # Git clone has to be run in compat layer, to make the git command available -./run_in_compat_layer_env.sh "${TEST_CLONE} && ${LATEST_VERSION} && ${CHECKOUT_LATEST}" +./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite" # Run the test suite ./test_suite.sh "$@" From 9030cdb6d1a849200a2470ff915e322411822a1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 11:21:01 +0100 Subject: [PATCH 1742/1795] add changes from PR#880 --- run_tests.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 1dbb47db9d..cd57476f2b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -17,8 +17,14 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults +# Make sure we clone the latest version. This assumes versions are of the format "v1.2.3", then picks the latest +# then checks it out +TEST_CLONE="git clone https://github.com/EESSI/test-suite EESSI-test-suite && cd EESSI-test-suite" +LATEST_VERSION="VERSION=$(git tag | grep "^v[0-9]\+\.[0-9]\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)" +CHECKOUT_LATEST="git checkout ${VERSION}" + # Git clone has to be run in compat layer, to make the git command available -./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite" +./run_in_compat_layer_env.sh "${TEST_CLONE} && ${LATEST_VERSION} && ${CHECKOUT_LATEST}" # Run the test suite ./test_suite.sh "$@" From f70476b880d184c271809ab5a520af3c023c1050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 11:24:15 +0100 Subject: [PATCH 1743/1795] don't expand variables that depend on command substitution --- run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index cd57476f2b..a2395eb5f9 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -20,8 +20,8 @@ source ${base_dir}/init/eessi_defaults # Make sure we clone the latest version. This assumes versions are of the format "v1.2.3", then picks the latest # then checks it out TEST_CLONE="git clone https://github.com/EESSI/test-suite EESSI-test-suite && cd EESSI-test-suite" -LATEST_VERSION="VERSION=$(git tag | grep "^v[0-9]\+\.[0-9]\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)" -CHECKOUT_LATEST="git checkout ${VERSION}" +LATEST_VERSION="VERSION=\$(git tag | grep "^v[0-9]\+\.[0-9]\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)" +CHECKOUT_LATEST="git checkout \${VERSION}" # Git clone has to be run in compat layer, to make the git command available ./run_in_compat_layer_env.sh "${TEST_CLONE} && ${LATEST_VERSION} && ${CHECKOUT_LATEST}" From ee6835e37821438cd794ef4e9d002601bc729f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 11:41:08 +0100 Subject: [PATCH 1744/1795] another batch of apps for EB 4.9.0 and 2023a --- .../eessi-2023.06-eb-4.9.0-2023a.yml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 2b53315214..2a2b753cd4 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -41,3 +41,26 @@ easyconfigs: options: from-pr: 19471 include-easyblocks-from-pr: 3036 + - matplotlib-3.7.2-gfbf-2023a.eb + - PyQt5-5.15.10-GCCcore-12.3.0.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 + options: + from-pr: 19554 + - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996 + from-pr: 19996 + - dask-2023.9.2-foss-2023a.eb + - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb + - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb: + options: + from-pr: 20086 + - Z3-4.12.2-GCCcore-12.3.0.eb: + options: + # The Z3 dependency of PyTorch had it's versionsuffix removed + # and we need to workaround the problem this creates, + # see https://github.com/EESSI/software-layer/pull/501 for details + from-pr: 20050 + - PyOpenGL-3.1.7-GCCcore-12.3.0.eb: + options: + from-pr: 20007 From 86ae593aa6a93ffae57d7ad13ae8cb4e42f5c1da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 11:54:12 +0100 Subject: [PATCH 1745/1795] use single quotes for grep command contained in double-quoted string --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index a2395eb5f9..f6264c3cc8 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -20,7 +20,7 @@ source ${base_dir}/init/eessi_defaults # Make sure we clone the latest version. This assumes versions are of the format "v1.2.3", then picks the latest # then checks it out TEST_CLONE="git clone https://github.com/EESSI/test-suite EESSI-test-suite && cd EESSI-test-suite" -LATEST_VERSION="VERSION=\$(git tag | grep "^v[0-9]\+\.[0-9]\+\.[0-9]\+$" | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)" +LATEST_VERSION="VERSION=\$(git tag | grep '^v[0-9]\+\.[0-9]\+\.[0-9]\+$' | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)" CHECKOUT_LATEST="git checkout \${VERSION}" # Git clone has to be run in compat layer, to make the git command available From 20070e65a76a6cc58caba6c9b60cd963a80831a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 13:40:02 +0100 Subject: [PATCH 1746/1795] add GROMACS/2024.3-foss-2023b --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml new file mode 100644 index 0000000000..8e19788425 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml @@ -0,0 +1,5 @@ +easyconfigs: + - GROMACS-2024.3-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 + from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b From cfa969511b9f817d63455b398541533f003b90ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 19:44:21 +0100 Subject: [PATCH 1747/1795] add EB 4.9.4 2023b easystack --- .../eessi-2023.06-eb-4.9.4-2023b.yml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023b.yml new file mode 100644 index 0000000000..ee161b8b9c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023b.yml @@ -0,0 +1,27 @@ +easyconfigs: + - SIONlib-1.7.7-GCCcore-13.2.0-tools.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21752 + from-commit: 6b8b53493a1188a5baa56a133574daac239730e7 + - Score-P-8.4-gompi-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3496 + include-easyblocks-from-commit: 60633b0acfd41a0732992d9e16800dae71a056eb + - Cython-3.0.10-GCCcore-13.2.0.eb + - Mustache-1.3.3-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21783 + from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 + - GDRCopy-2.4-GCCcore-13.2.0.eb + - GROMACS-2024.4-foss-2023b.eb: + options: + # https://github.com/easybuilders/easybuild-easyconfigs/pull/21851 + from-commit: f0fa64b440deaf5fb0a6d26ff1bb3e9f36626c8a + - SlurmViewer-1.0.1-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21899 + from-commit: 0bdeb23c9ea5a3caefd353ecd936919424c1bba4 + - wxWidgets-3.2.6-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21915 + from-commit: 58f16c0caf8c5494c68e9eda8cbf19e9145d3cfa From 34928cb2b4d72c387e6c9445fdd10ac5de6c6881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 19:48:08 +0100 Subject: [PATCH 1748/1795] another batch of apps --- .../eessi-2023.06-eb-4.9.0-2023a.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 05d3ead91e..5d2c2b4acb 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -68,3 +68,17 @@ easyconfigs: - PyOpenGL-3.1.7-GCCcore-12.3.0.eb: options: from-pr: 20007 + - OpenJPEG-2.5.0-GCCcore-12.3.0.eb + - Highway-1.0.4-GCCcore-12.3.0.eb + - ELPA-2023.05.001-foss-2023a.eb + - libxc-6.2.2-GCC-12.3.0.eb + - SuperLU_DIST-8.1.2-foss-2023a.eb: + options: + from-pr: 20162 + - PETSc-3.20.3-foss-2023a.eb: + options: + include-easyblocks-from-pr: 3086 + from-pr: 19686 + - MODFLOW-6.4.4-foss-2023a.eb: + options: + from-pr: 20142 From 5799125568aa2da7822622abe9ebd17eb8932755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 24 Jan 2025 23:41:16 +0100 Subject: [PATCH 1749/1795] add R-bundle-CRAN-2023.12-foss-2023a.eb --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml index 05d3ead91e..1a7e02293d 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -68,3 +68,7 @@ easyconfigs: - PyOpenGL-3.1.7-GCCcore-12.3.0.eb: options: from-pr: 20007 + - R-bundle-CRAN-2023.12-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 + from-pr: 20238 From 91169047826f26a1713b9ab876c9735e2d50f1a3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Jan 2025 10:35:09 +0100 Subject: [PATCH 1750/1795] disable using x86_64/amd/zen3 when x86_64/amd/zen4 in EESSI/2023.06 module --- init/modules/EESSI/2023.06.lua | 9 --------- 1 file changed, 9 deletions(-) diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index eb1cd1753b..d5105e89fc 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -49,15 +49,6 @@ function archdetect_cpu() -- We loop over the list, and return the highest matching arch for which a directory exists for this EESSI version for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then - -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, - -- since optimized software installations for Zen4 are a work-in-progress, - -- see https://gitlab.com/eessi/support/-/issues/37 - if (archdetect_filter_cpu == "x86_64/amd/zen4" and not os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE") == "x86_64/amd/zen4") then - archdetect_filter_cpu = "x86_64/amd/zen3" - if mode() == "load" then - LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.") - end - end eessiDebug("Selected archdetect CPU: " .. archdetect_filter_cpu) return archdetect_filter_cpu end From 5dd60c498a1da2d1175827f1e2ffad6277a5da0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 25 Jan 2025 23:49:06 +0100 Subject: [PATCH 1751/1795] add remaining apps from EB 4.9.1 2023a easystack --- .../eessi-2023.06-eb-4.9.1-2023a.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml index 57eda0aec4..d35be0c6c2 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml @@ -2,3 +2,21 @@ easyconfigs: - Python-3.11.3-GCCcore-12.3.0.eb - hatchling-1.18.0-GCCcore-12.3.0.eb - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb + - ncdu-1.18-GCC-12.3.0.eb + - SAMtools-1.18-GCC-12.3.0.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + from-pr: 20379 + - ipympl-0.9.3-gfbf-2023a.eb: + options: + from-pr: 18852 + - ESPResSo-4.2.2-foss-2023a.eb: + options: + from-pr: 20595 + - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb + - WhatsHap-2.2-foss-2023a.eb + - BLAST+-2.14.1-gompi-2023a.eb: + options: + from-pr: 20784 + - Valgrind-3.21.0-gompi-2023a.eb + - OrthoFinder-2.5.5-foss-2023a.eb From 30fa1e59e006aa198e69618259671a8458a310bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 26 Jan 2025 14:27:26 +0100 Subject: [PATCH 1752/1795] first batch of apps from EB 4.9.2 2023a easystack --- .../eessi-2023.06-eb-4.9.2-2023a.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml index 7257ec4dae..9a6072070e 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml @@ -9,4 +9,20 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - BCFtools-1.18-GCC-12.3.0.eb + - BWA-0.7.18-GCCcore-12.3.0.eb + - CapnProto-1.0.1-GCCcore-12.3.0.eb + - DendroPy-4.6.1-GCCcore-12.3.0.eb + - DIAMOND-2.1.8-GCC-12.3.0.eb + - FastME-2.1.6.3-GCC-12.3.0.eb + - fastp-0.23.4-GCC-12.3.0.eb + - HMMER-3.4-gompi-2023a.eb + - IQ-TREE-2.3.5-gompi-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20955 + from-commit: 185f88b9a03d65a7fb74edc7acb4221e87e90784 + - KronaTools-2.8.1-GCCcore-12.3.0.eb + - LSD2-2.4.1-GCCcore-12.3.0.eb + - MAFFT-7.520-GCC-12.3.0-with-extensions.eb + - ncbi-vdb-3.0.10-gompi-2023a.eb From 4b45c24b661fbb98142837e9dfcad6c29f1507a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 26 Jan 2025 16:08:04 +0100 Subject: [PATCH 1753/1795] remove fastp and DIAMOND --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml index 9a6072070e..9d4dc4d4f4 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml @@ -13,9 +13,7 @@ easyconfigs: - BWA-0.7.18-GCCcore-12.3.0.eb - CapnProto-1.0.1-GCCcore-12.3.0.eb - DendroPy-4.6.1-GCCcore-12.3.0.eb - - DIAMOND-2.1.8-GCC-12.3.0.eb - FastME-2.1.6.3-GCC-12.3.0.eb - - fastp-0.23.4-GCC-12.3.0.eb - HMMER-3.4-gompi-2023a.eb - IQ-TREE-2.3.5-gompi-2023a.eb: options: From cd2904554334feded88f31be8f7cebc46fd83db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 26 Jan 2025 17:11:25 +0100 Subject: [PATCH 1754/1795] restore original easystack, and add some more apps --- .../sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml index 9d4dc4d4f4..17468ad9f6 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml @@ -13,14 +13,22 @@ easyconfigs: - BWA-0.7.18-GCCcore-12.3.0.eb - CapnProto-1.0.1-GCCcore-12.3.0.eb - DendroPy-4.6.1-GCCcore-12.3.0.eb + - DIAMOND-2.1.8-GCC-12.3.0.eb - FastME-2.1.6.3-GCC-12.3.0.eb + - fastp-0.23.4-GCC-12.3.0.eb - HMMER-3.4-gompi-2023a.eb - IQ-TREE-2.3.5-gompi-2023a.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20955 - from-commit: 185f88b9a03d65a7fb74edc7acb4221e87e90784 + from-commit: 185f88b9a03d65a7fb74edc7acb4221e87e90784 - KronaTools-2.8.1-GCCcore-12.3.0.eb - LSD2-2.4.1-GCCcore-12.3.0.eb - MAFFT-7.520-GCC-12.3.0-with-extensions.eb - ncbi-vdb-3.0.10-gompi-2023a.eb - + - MetalWalls-21.06.1-foss-2023a.eb + - QuantumESPRESSO-7.3.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138 + from-commit: dbdaacc0739fdee91baa9123864ea4428cf21273 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3338 + include-easyblocks-from-commit: 32e45bd1f2d916732ca5852d55d17fa4d99e388b From bf13ff4dedba0b29a3b8d23b921b359417d1f699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 26 Jan 2025 18:35:16 +0100 Subject: [PATCH 1755/1795] last batch of apps for EB 4.9.2 2023a --- .../eessi-2023.06-eb-4.9.2-2023a.yml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml index 17468ad9f6..7166348bde 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml @@ -32,3 +32,52 @@ easyconfigs: from-commit: dbdaacc0739fdee91baa9123864ea4428cf21273 # see https://github.com/easybuilders/easybuild-easyblocks/pull/3338 include-easyblocks-from-commit: 32e45bd1f2d916732ca5852d55d17fa4d99e388b + - CP2K-2023.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 + from-commit: a92667fe32396bbd4106243658625f7ff2adcd68 + - amdahl-0.3.1-gompi-2023a.eb + - librosa-0.10.1-foss-2023a.eb + - xarray-2023.9.0-gfbf-2023a.eb + - SciTools-Iris-3.9.0-foss-2023a.eb + - OpenFOAM-v2312-foss-2023a.eb: + options: + # https://github.com/easybuilders/easybuild-easyblocks/pull/3388 + include-easyblocks-from-commit: c8256a36e7062bc09f5ce30552a9de9827054c9e + # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841 + from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 + - BioPerl-1.7.8-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + - grpcio-1.57.0-GCCcore-12.3.0.eb + - orjson-3.9.15-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 + from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 + - wradlib-2.0.3-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21094 + from-commit: 3a2e0b8e6ee45277d01fb7e2eb93027a28c9461a + - MBX-1.1.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21155 + from-commit: 6929a67401f2a2ec58f91fb306332a77497d73ff + - Transrate-1.0.3-GCC-12.3.0.eb: + options: + # https://github.com/easybuilders/easybuild-easyblocks/pull/3381 + include-easyblocks-from-commit: bb86f05d4917b29e022023f152efdf0ca5c14ded + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20964 + from-commit: 7d539a9e599d8bc5ac2bda6ee9587ef62351ee03 + - Critic2-1.2-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20833 + from-commit: 78426c2383fc7e4b9b9e77d7a77f336e1bee3843 + - LRBinner-0.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21310 + from-commit: 799d9101df2cf81aabe252f00cc82a7246363f53 + - Redland-1.0.17-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21227 + from-commit: 4c5e3455dec31e68e8383c7fd86d1f80c434676d From 8f01e37992d6041bbcaacd0c379c2dfb9b43bc84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 27 Jan 2025 07:11:42 +0100 Subject: [PATCH 1756/1795] remove librosa --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml index 7166348bde..3caa1edf62 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml @@ -37,7 +37,6 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 from-commit: a92667fe32396bbd4106243658625f7ff2adcd68 - amdahl-0.3.1-gompi-2023a.eb - - librosa-0.10.1-foss-2023a.eb - xarray-2023.9.0-gfbf-2023a.eb - SciTools-Iris-3.9.0-foss-2023a.eb - OpenFOAM-v2312-foss-2023a.eb: From 06f420a9cdec4d8423ee76cb2d69309a0145a905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 27 Jan 2025 12:00:29 +0100 Subject: [PATCH 1757/1795] build librosa with EB 4.9.4 --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml new file mode 100644 index 0000000000..9182b8e126 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - librosa-0.10.1-foss-2023a.eb From df7a5ee110352784655f74f5247b7e21210e0c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 27 Jan 2025 19:45:29 +0100 Subject: [PATCH 1758/1795] add easystack for EB 4.9.3 / 2023a --- .../sapphire_rapids/eessi-2023.06-eb-4.9.3-2023a.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..0c863f0025 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,8 @@ +easyconfigs: + - ccache-4.9-GCCcore-12.3.0.eb + - GDB-13.2-GCCcore-12.3.0.eb + - tmux-3.3a-GCCcore-12.3.0.eb + - Vim-9.1.0004-GCCcore-12.3.0.eb + - gmsh-4.12.2-foss-2023a.eb + - basemap-1.3.9-foss-2023a.eb + - geopandas-0.14.2-foss-2023a.eb \ No newline at end of file From 84653ca5f9f96da6e7239c58ad9939dc410ca5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 28 Jan 2025 10:30:14 +0100 Subject: [PATCH 1759/1795] add EB 4.9.4 2023a easystack --- .../eessi-2023.06-eb-4.9.4-2023a.yml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml new file mode 100644 index 0000000000..19d685b109 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2023a.yml @@ -0,0 +1,47 @@ +easyconfigs: + - ROOT-6.30.06-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 + - waLBerla-6.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 + from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 + - mpl-ascii-0.10.0-gfbf-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21679 + from-commit: 7106f63160b1418d605882dd02ba151d099300bd + - jedi-0.19.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21650 + from-commit: 109998f6adcda7efb4174b1e5f73b41ee82d1f13 + - Solids4foam-2.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21606 + from-commit: 63562c58acf1be64407192b6862c3bd80253d2e0 + - Cassiopeia-2.0.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657 + from-commit: 7f1f0e60487e7e1fcb5c4e6bc4fbc4f89994e3fd + - LightGBM-4.5.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21699 + from-commit: e3407bd127d248c08960f6b09c973da0fdecc2c3 + - OpenFOAM-v2406-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3519 + include-easyblocks-from-commit: e4a3ff1932350d575dffc7597435609fad6dd691 + - Paraver-4.11.4-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20230 + from-commit: 91c8df6b4c0810061e9f325427c9c79e961bc4b0 + - Tombo-1.5.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21925 + from-commit: 522ca010ab11949ab9594037f72b975cf1cd0d33 + - elfx86exts-0.6.2-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22145 + from-commit: 31478e5c9869de3add74d0a02dd5df01ea65b21e From 6aef77f612d232f841e586f7a0812d4667270967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 28 Jan 2025 17:03:26 +0100 Subject: [PATCH 1760/1795] add LAMMPS-29Aug2024-foss-2023b --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml index 8e19788425..9703a8b9bd 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml @@ -3,3 +3,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b + - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 + options: + from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 From 1462d25d6d408b53a7a9ff0afa977d1addfb1ac6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 18:12:03 +0100 Subject: [PATCH 1761/1795] Add BCFTools so that we have SOMETHING to build in this PR --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index ee161b8b9c..5664a67e8b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -25,3 +25,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21915 from-commit: 58f16c0caf8c5494c68e9eda8cbf19e9145d3cfa + - BCFtools-1.19-GCC-13.2.0.eb From 673861b983c02f01dcf851bd390a0f7d49163d43 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 18:30:38 +0100 Subject: [PATCH 1762/1795] Add explicit path to reframe config file for AWS bot for testing --- test_suite.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_suite.sh b/test_suite.sh index 79986d9188..0b32b7fbaa 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -131,6 +131,9 @@ else fatal_error "Failed to import ${testsuite_import}" fi +# WARNING: REMOVE BEFORE MERGING, THIS IS JUST TO TEST WITH THE AWS BOT: +export RFM_CONFIG_FILES="$HOME/example_reframe_config.py" + # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame # config file that matches the bot config. See https://gitlab.com/eessi/support/-/issues/114#note_2293660921 From 189cc79ac2f0f3b621d1f56a759909374fec00ce Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 18:39:04 +0100 Subject: [PATCH 1763/1795] /home/casparl gets set to something else, so lets try this --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 0b32b7fbaa..8048b3fa2f 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -132,7 +132,7 @@ else fi # WARNING: REMOVE BEFORE MERGING, THIS IS JUST TO TEST WITH THE AWS BOT: -export RFM_CONFIG_FILES="$HOME/example_reframe_config.py" +export RFM_CONFIG_FILES="/home/$USER/example_reframe_config.py" # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame From d93dee52db8b35645470dd87e908c5c1f3502af2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 19:25:06 +0100 Subject: [PATCH 1764/1795] This needs to be a path that is actually accessible in the container. This dir is the shared dir for bot jobs, so should work --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 8048b3fa2f..6113447729 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -132,7 +132,7 @@ else fi # WARNING: REMOVE BEFORE MERGING, THIS IS JUST TO TEST WITH THE AWS BOT: -export RFM_CONFIG_FILES="/home/$USER/example_reframe_config.py" +export RFM_CONFIG_FILES="/project/def-users/$user/shared/example_reframe_config.py" # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame From 7b04794cd14ac9bb960776b7e0f7acbb54594712 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 19:35:52 +0100 Subject: [PATCH 1765/1795] Needs to be caps, of course --- test_suite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.sh b/test_suite.sh index 6113447729..a0f5ccc32a 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -132,7 +132,7 @@ else fi # WARNING: REMOVE BEFORE MERGING, THIS IS JUST TO TEST WITH THE AWS BOT: -export RFM_CONFIG_FILES="/project/def-users/$user/shared/example_reframe_config.py" +export RFM_CONFIG_FILES="/project/def-users/$USER/shared/example_reframe_config.py" # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame From cc637e8b3387f89da21a058f901e72d0ca41fc53 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 19:57:15 +0100 Subject: [PATCH 1766/1795] Allow using the envrionemn t variable if it is set, so Lara can test --- test_suite.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index a0f5ccc32a..fd49409b9b 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -131,13 +131,14 @@ else fatal_error "Failed to import ${testsuite_import}" fi -# WARNING: REMOVE BEFORE MERGING, THIS IS JUST TO TEST WITH THE AWS BOT: -export RFM_CONFIG_FILES="/project/def-users/$USER/shared/example_reframe_config.py" # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame # config file that matches the bot config. See https://gitlab.com/eessi/support/-/issues/114#note_2293660921 if [ -z "$RFM_CONFIG_FILES" ]; then + # WARNING: REMOVE BEFORE MERGING, THIS IS JUST TO TEST WITH THE AWS BOT: + export RFM_CONFIG_FILES="/project/def-users/$USER/shared/example_reframe_config.py" + err_msg="Please set RFM_CONFIG_FILES in the environment of this bot instance to point to a valid" err_msg="${err_msg} ReFrame configuration file that matches the bot config." err_msg="${err_msg} For more information, see https://gitlab.com/eessi/support/-/issues/114#note_2293660921" From 7b9c90c2e16c898d558f016c707f421a69eadaff Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 21:09:31 +0100 Subject: [PATCH 1767/1795] Look for bot config in the SHARED_FS_PATH by default --- test_suite.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index fd49409b9b..916a6affa1 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -136,13 +136,19 @@ fi # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame # config file that matches the bot config. See https://gitlab.com/eessi/support/-/issues/114#note_2293660921 if [ -z "$RFM_CONFIG_FILES" ]; then - # WARNING: REMOVE BEFORE MERGING, THIS IS JUST TO TEST WITH THE AWS BOT: - export RFM_CONFIG_FILES="/project/def-users/$USER/shared/example_reframe_config.py" - - err_msg="Please set RFM_CONFIG_FILES in the environment of this bot instance to point to a valid" - err_msg="${err_msg} ReFrame configuration file that matches the bot config." - err_msg="${err_msg} For more information, see https://gitlab.com/eessi/support/-/issues/114#note_2293660921" - fatal_error "${err_msg}" + if [ -z "${SHARED_FS_PATH}" ]; then + fatal_error "SHARED_FS_PATH was expected, but was not set" + fi + # Try to find a config file at $SHARED_FS_PATH/reframe_config.py + export RFM_CONFIG_FILES="${SHARED_FS_PATH}/reframe_config.py" + if [ ! -f "${RFM_CONFIG_FILES}" ]; then + # If we haven't found the ReFrame config, print an informative error + err_msg="Please put a ReFrame configuration file in ${SHARED_FS_PATH}/reframe_config.py" + err_msg="${err_msg} or set RFM_CONFIG_FILES in the environment of this bot instance to point to a valid" + err_msg="${err_msg} ReFrame configuration file that matches the bot config." + err_msg="${err_msg} For more information, see https://gitlab.com/eessi/support/-/issues/114#note_2293660921" + fatal_error "${err_msg}" + fi fi export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests export RFM_CHECK_SEARCH_RECURSIVE=1 From b0aaed2b2d782adb9832fd7e18c7ef6380e45362 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 21:21:36 +0100 Subject: [PATCH 1768/1795] export SHARED_FS_PATH so that it is available to --- bot/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/test.sh b/bot/test.sh index 597c277292..6461df2d4a 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -79,7 +79,7 @@ fi # check if path to directory on shared filesystem is specified, # and use it as location for source tarballs used by EasyBuild if so -SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") +export SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") echo "bot/test.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'" # if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container if [[ ! -z ${SHARED_FS_PATH} ]]; then From 28abaf6408fac34f8c7b57006f2c4ee223e10110 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 21:26:43 +0100 Subject: [PATCH 1769/1795] Explicitely pass shared-fs-path as argument to the test_suite.sh --- bot/test.sh | 5 ++++- test_suite.sh | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index 6461df2d4a..8f8bc9f82a 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -79,7 +79,7 @@ fi # check if path to directory on shared filesystem is specified, # and use it as location for source tarballs used by EasyBuild if so -export SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") +SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path") echo "bot/test.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'" # if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container if [[ ! -z ${SHARED_FS_PATH} ]]; then @@ -223,6 +223,9 @@ declare -a TEST_SUITE_ARGS=() if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then TEST_SUITE_ARGS+=("--generic") fi +if [[ ${SHARED_FS_PATH} ]]; then + TEST_SUITE_ARGS+=("--shared-fs-path ${SHARED_FS_PATH}") +fi # [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") # [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") diff --git a/test_suite.sh b/test_suite.sh index 916a6affa1..227ba8e218 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -136,8 +136,8 @@ fi # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame # config file that matches the bot config. See https://gitlab.com/eessi/support/-/issues/114#note_2293660921 if [ -z "$RFM_CONFIG_FILES" ]; then - if [ -z "${SHARED_FS_PATH}" ]; then - fatal_error "SHARED_FS_PATH was expected, but was not set" + if [ -z "${shared_fs_path}" ]; then + fatal_error "Environment variable 'shared_fs_path' was expected, but was not set" fi # Try to find a config file at $SHARED_FS_PATH/reframe_config.py export RFM_CONFIG_FILES="${SHARED_FS_PATH}/reframe_config.py" From 5ca5a76dcc112970029bb40922be6cbff5e3efa9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 21:29:03 +0100 Subject: [PATCH 1770/1795] Make all shared_fs_path env vars lower case --- test_suite.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 227ba8e218..e7cae2f965 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -139,11 +139,11 @@ if [ -z "$RFM_CONFIG_FILES" ]; then if [ -z "${shared_fs_path}" ]; then fatal_error "Environment variable 'shared_fs_path' was expected, but was not set" fi - # Try to find a config file at $SHARED_FS_PATH/reframe_config.py - export RFM_CONFIG_FILES="${SHARED_FS_PATH}/reframe_config.py" + # Try to find a config file at $shared_fs_path/reframe_config.py + export RFM_CONFIG_FILES="${shared_fs_path}/reframe_config.py" if [ ! -f "${RFM_CONFIG_FILES}" ]; then # If we haven't found the ReFrame config, print an informative error - err_msg="Please put a ReFrame configuration file in ${SHARED_FS_PATH}/reframe_config.py" + err_msg="Please put a ReFrame configuration file in ${shared_fs_path}/reframe_config.py" err_msg="${err_msg} or set RFM_CONFIG_FILES in the environment of this bot instance to point to a valid" err_msg="${err_msg} ReFrame configuration file that matches the bot config." err_msg="${err_msg} For more information, see https://gitlab.com/eessi/support/-/issues/114#note_2293660921" From 99c5f88c8557609b7684245f2de042f437fe7f3b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Jan 2025 21:44:05 +0100 Subject: [PATCH 1771/1795] Make this two separate strings, so that they are two separate arguments (in a shell sense) --- bot/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/test.sh b/bot/test.sh index 8f8bc9f82a..464c4817a9 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -224,7 +224,7 @@ if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then TEST_SUITE_ARGS+=("--generic") fi if [[ ${SHARED_FS_PATH} ]]; then - TEST_SUITE_ARGS+=("--shared-fs-path ${SHARED_FS_PATH}") + TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") fi # [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") # [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") From 732e6d423c1b253582b6619d4520d956b0323634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 28 Jan 2025 21:58:29 +0100 Subject: [PATCH 1772/1795] use updated easyblock with sapphire rapids fix --- .../2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml index 9703a8b9bd..2cf72ca098 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.3-2023b.yml @@ -4,6 +4,8 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3569 + include-easyblocks-from-commit: 362b4679193612e04abe336fa041e2a34d183991 From 61d4617c11d9cdee11cae23c049da4d44837e85a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 29 Jan 2025 00:05:55 +0100 Subject: [PATCH 1773/1795] Remove BCFtools from this PR, it was just meant to demonstrate the functionality of the test step --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 5664a67e8b..ee161b8b9c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -25,4 +25,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21915 from-commit: 58f16c0caf8c5494c68e9eda8cbf19e9145d3cfa - - BCFtools-1.19-GCC-13.2.0.eb From 896f2d4590ef62f5dd96d7b698725c19ed7b39d8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 29 Jan 2025 00:08:46 +0100 Subject: [PATCH 1774/1795] Removed white line and comments --- test_suite.sh | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index e7cae2f965..4121a37c2e 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -131,7 +131,6 @@ else fatal_error "Failed to import ${testsuite_import}" fi - # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration # RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame # config file that matches the bot config. See https://gitlab.com/eessi/support/-/issues/114#note_2293660921 @@ -167,46 +166,6 @@ export RFM_SYSTEM="BotBuildTests:${REFRAME_PARTITION_NAME}" echo "Configured reframe with the following environment variables:" env | grep "RFM_" -# THIS WHOLE BLOCK SHOULD NO LONGER BE NEEDED IF WE HAVE SITE-SPECIFIC CONFIG FILES -# # The /sys inside the container is not the same as the /sys of the host -# # We want to extract the memory limit from the cgroup on the host (which is typically set by SLURM). -# # Thus, bot/test.sh bind-mounts the host's /sys/fs/cgroup into /hostsys/fs/cgroup -# # and that's the prefix we use to extract the memory limit from -# cgroup_v1_mem_limit="/hostsys/fs/cgroup/memory/$( Date: Thu, 30 Jan 2025 15:59:53 +0100 Subject: [PATCH 1775/1795] rename to USE_CHECK_BUILD_ARTEFACTS_SCRIPT; error if script is not found --- bot/check-build.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 85816519ef..001906b632 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -60,12 +60,12 @@ display_help() { echo " OPTIONS:" echo " -h | --help - display this usage information [default: false]" echo " -v | --verbose - display more information [default: false]" - echo " -a | --alt-artefacts - alternative artefacts check (sources file check-build-artefacts.sh if exists) [default: false]" + echo " -a | --use-check-build-artefacts-script - alternative build artefacts check (sources file check-build-artefacts.sh if exists) [default: false]" } # set defaults for command line arguments VERBOSE=0 -ALT_ARTEFACTS=0 +USE_CHECK_BUILD_ARTEFACTS_SCRIPT=0 POSITIONAL_ARGS=() @@ -80,7 +80,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; -a|--alt-artefacts) - ALT_ARTEFACTS=1 + USE_CHECK_BUILD_ARTEFACTS_SCRIPT=1 shift 1 ;; --) @@ -164,7 +164,7 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi -if [[ $ALT_ARTEFACTS -eq 0 ]]; then +if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then TGZ=-1 TARBALL= if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then @@ -189,7 +189,7 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -if [[ $ALT_ARTEFACTS -eq 0 ]]; then +if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then [[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" fi @@ -219,8 +219,8 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ [[ ${FAILED} -eq 0 ]] && \ [[ ${MISSING} -eq 0 ]] && \ [[ ${NO_MISSING} -eq 1 ]] && \ - [[ $ALT_ARTEFACTS -ne 0 || ${TGZ} -eq 1 ]] && \ - [[ $ALT_ARTEFACTS -ne 0 || -n ${TARBALL} ]]; then + [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \ + [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then # SUCCESS status="SUCCESS" reason="" @@ -428,7 +428,7 @@ success_msg="found message(s) matching ${GP_no_missing}" failure_msg="no message matching ${GP_no_missing}" comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") -if [[ $ALT_ARTEFACTS -eq 0 ]]; then +if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then success_msg="found message matching ${GP_tgz_created}" failure_msg="no message matching ${GP_tgz_created}" comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") @@ -439,7 +439,7 @@ comment_details_fmt="
    _Details_
    __DETAILS_LIST__
    " comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} -if [[ $ALT_ARTEFACTS -eq 0 ]]; then +if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then # first construct comment_artefacts_list # then use it to set comment_artefacts comment_artifacts_list="" @@ -563,6 +563,9 @@ if [[ $ALT_ARTEFACTS -eq 0 ]]; then elif [[ -f "$TOPDIR/check-build-artefacts.sh" ]]; then source "$TOPDIR/check-build-artefacts.sh" +else + echo "ERROR: Required script $TOPDIR/check-build-artefacts.sh not found!" >&2 + exit 1 fi echo "${comment_description}" >> ${job_result_file} From be0977d43a14077c7debe7e8812fc6894dd2f81d Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 30 Jan 2025 16:09:35 +0100 Subject: [PATCH 1776/1795] fix --- bot/check-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 001906b632..49fc0d99a9 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -60,7 +60,7 @@ display_help() { echo " OPTIONS:" echo " -h | --help - display this usage information [default: false]" echo " -v | --verbose - display more information [default: false]" - echo " -a | --use-check-build-artefacts-script - alternative build artefacts check (sources file check-build-artefacts.sh if exists) [default: false]" + echo " --use-check-build-artefacts-script - alternative build artefacts check (sources file check-build-artefacts.sh if exists) [default: false]" } # set defaults for command line arguments @@ -79,7 +79,7 @@ while [[ $# -gt 0 ]]; do VERBOSE=1 shift 1 ;; - -a|--alt-artefacts) + --use-check-build-artefacts-script) USE_CHECK_BUILD_ARTEFACTS_SCRIPT=1 shift 1 ;; From e5990d213eb27485d0bca990f1d9736d3c75459a Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 31 Jan 2025 09:52:01 +0100 Subject: [PATCH 1777/1795] fix ordering --- bot/check-build.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 49fc0d99a9..a3c3a3a1d2 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -553,6 +553,12 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} + echo "${comment_description}" >> ${job_result_file} + + # add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts + # - this should make use of subsequent steps such as deploying a tarball more + # efficient + echo "status = ${status}" >> ${job_result_file} echo "artefacts = " >> ${job_result_file} echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} @@ -568,13 +574,6 @@ else exit 1 fi -echo "${comment_description}" >> ${job_result_file} - -# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts -# - this should make use of subsequent steps such as deploying a tarball more -# efficient -echo "status = ${status}" >> ${job_result_file} - # exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1) test "${status}" == "SUCCESS" exit $? From 5889b2dbe7745bc1acb279878aef5b637e16eda4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 2 Feb 2025 14:01:06 +0100 Subject: [PATCH 1778/1795] add archspec 0.2.5 --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 19d685b109..c6c735f2b9 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -45,3 +45,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22145 from-commit: 31478e5c9869de3add74d0a02dd5df01ea65b21e + - archspec-0.2.5-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22235 + from-commit: 01dd97ea62fe4d7d0df040ede3af03eb2f1b8641 From 5be1e039c6652a0866023c500c331173f5d44390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 3 Feb 2025 13:55:51 +0100 Subject: [PATCH 1779/1795] easystack for rebuilding LAMMPS 2Aug2023 (for Sapphire Rapids) --- ...MMPS-2Aug2023-add-support-for-sapphire_rapids.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250203-eb-4.9.4-LAMMPS-2Aug2023-add-support-for-sapphire_rapids.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250203-eb-4.9.4-LAMMPS-2Aug2023-add-support-for-sapphire_rapids.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250203-eb-4.9.4-LAMMPS-2Aug2023-add-support-for-sapphire_rapids.yml new file mode 100644 index 0000000000..fb612ed513 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250203-eb-4.9.4-LAMMPS-2Aug2023-add-support-for-sapphire_rapids.yml @@ -0,0 +1,12 @@ +# 2025.02.03 +# Rebuild LAMMPS 2Aug2023_update2 for Sapphire Rapids, +# as the existing version used an old version of archspec that detected the CPU as Icelake. +# See https://github.com/easybuilders/easybuild-easyconfigs/pull/22235 +# and https://github.com/easybuilders/easybuild-easyblocks/pull/3569. +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22235 + from-commit: 01dd97ea62fe4d7d0df040ede3af03eb2f1b8641 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3569 + include-easyblocks-from-commit: b0ce5ec66c8f797220b2c2a773acfc4aeae7a0c7 From 8d5ef69a02d34d1dba5c09a141fab0758672dbf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 3 Feb 2025 14:02:11 +0100 Subject: [PATCH 1780/1795] remove find command --- EESSI-remove-software.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 5f540b8061..e4b377fd16 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -128,7 +128,6 @@ if [ $EUID -eq 0 ]; then app_subdirs=$(find ${app_dir} -mindepth 1 -maxdepth 1 -type d) app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." - find ${app_dir} -type f -exec rm -f {} \; rm -rf ${app_dir} rm -rf ${app_module} # recreate the installation directories and first-level subdirectories to work around permission denied From aa31e15c22833412e74ebd0e8a729dd4a7fc8992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 3 Feb 2025 15:21:16 +0100 Subject: [PATCH 1781/1795] do an ls on the original subdirs of the installation dir to work around permission issues (instead of recreating them) --- EESSI-remove-software.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index e4b377fd16..8c48a3e362 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -130,12 +130,12 @@ if [ $EUID -eq 0 ]; then echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${app_dir} rm -rf ${app_module} - # recreate the installation directories and first-level subdirectories to work around permission denied - # issues when rebuilding the package (see https://github.com/EESSI/software-layer/issues/556) + # recreate the installation directory and do an ls on the first-level subdirectories to work around + # permission denied issues when rebuilding the package (see https://github.com/EESSI/software-layer/issues/556) echo_yellow "Recreating an empty ${app_dir}..." mkdir -p ${app_dir} for app_subdir in ${app_subdirs}; do - mkdir -p ${app_subdir} + ls ${app_subdir} >& /dev/null || true done done else From ecbc3612891ef89ca98ce4d666ca962f14bfffbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 3 Feb 2025 15:39:29 +0100 Subject: [PATCH 1782/1795] no need to do the ls on subdirs in a for loop, do them all at once --- EESSI-remove-software.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 8c48a3e362..928d162ae6 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -134,9 +134,7 @@ if [ $EUID -eq 0 ]; then # permission denied issues when rebuilding the package (see https://github.com/EESSI/software-layer/issues/556) echo_yellow "Recreating an empty ${app_dir}..." mkdir -p ${app_dir} - for app_subdir in ${app_subdirs}; do - ls ${app_subdir} >& /dev/null || true - done + ls ${app_subdirs} >& /dev/null || true done else fatal_error "Easystack file ${easystack_file} not found!" From 0560af8eb7ddea0aa31377527fb5f3501bd32bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 3 Feb 2025 16:49:15 +0100 Subject: [PATCH 1783/1795] better workaround for permission issues with the recreated installation dir --- EESSI-remove-software.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index e4b377fd16..c35c7bc469 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -130,13 +130,13 @@ if [ $EUID -eq 0 ]; then echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${app_dir} rm -rf ${app_module} - # recreate the installation directories and first-level subdirectories to work around permission denied - # issues when rebuilding the package (see https://github.com/EESSI/software-layer/issues/556) + # recreate the installation directory and do an ls on the first-level subdirectories to work around + # permission issues when reinstalling the application (see https://github.com/EESSI/software-layer/issues/556) echo_yellow "Recreating an empty ${app_dir}..." mkdir -p ${app_dir} - for app_subdir in ${app_subdirs}; do - mkdir -p ${app_subdir} - done + # these subdirs don't (and shouldn't) exist, but we need to do the ls anyway as a workaround, + # so redirect to /dev/null and ignore the exit code + ls ${app_subdirs} >& /dev/null || true done else fatal_error "Easystack file ${easystack_file} not found!" From 9eacba5ccab66fe48cf28cfa3c23f57de3f50a39 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 7 Feb 2025 17:21:33 +0100 Subject: [PATCH 1784/1795] Update from-commit to merge commit --- .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index 5d20176dc8..0ff41ec4fa 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -28,12 +28,12 @@ easyconfigs: - DP3-6.2-foss-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 - from-commit: bf9fe67ec7d14f823c32081d52eb9d2763e8dcd0 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 - WSClean-3.5-foss-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 - from-commit: bf9fe67ec7d14f823c32081d52eb9d2763e8dcd0 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 - EveryBeam-0.6.1-foss-2023b.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 - from-commit: bf9fe67ec7d14f823c32081d52eb9d2763e8dcd0 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 From 2dbe9319e0a2d9db0d0f78c80fc916793bf41132 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 11 Feb 2025 16:06:29 +0000 Subject: [PATCH 1785/1795] GH200-test --- bot/build.sh | 14 +++++++++++++- create_tarball.sh | 4 ++++ .../GH/eessi-2023.06-eb-4.9.4-001-system.yml | 5 +++++ eessi_container.sh | 9 +++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml diff --git a/bot/build.sh b/bot/build.sh index 29444a32c2..5c6ac8c185 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -93,7 +93,19 @@ fi echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " # replace any env variable in ${LOCAL_TMP} with its # current value (e.g., a value that is local to the job) -STORAGE=$(envsubst <<< ${LOCAL_TMP}) +#STORAGE=$(envsubst <<< ${LOCAL_TMP}) +# We should have set TMPDIR via $LOCAL_TMP already in the bot's slurm script. +# To be sure, we test if TMPDIR is set and simply use that, otherwise we expand +# it here. +if [[ -n ${TMPDIR} && -d ${TMPDIR} ]]; then + # TMPDIR defined and directory exists + STORAGE=${TMPDIR} +else + if [[ -z ${TMPDIR} ]]; then + # TMPDIR not defined + STORAGE=$(envsubst <<< ${LOCAL_TMP}) + fi +fi echo "'${STORAGE}'" # make sure ${STORAGE} exists diff --git a/create_tarball.sh b/create_tarball.sh index 01f498e1ac..86160cb751 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -14,6 +14,10 @@ cpu_arch_subdir=$3 accel_subdir=$4 target_tgz=$5 +echo "create_tarball.sh: TMPDIR='${TMPDIR}'" +if [[ ! -z ${TMPDIR} ]]; then + ls -l ${TMPDIR} +fi tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml new file mode 100644 index 0000000000..d9c6075561 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -0,0 +1,5 @@ +easyconfigs: + - EasyBuild-4.9.4.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 diff --git a/eessi_container.sh b/eessi_container.sh index fc97f9877c..7259fb69a8 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -382,6 +382,7 @@ else # note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to # create a temporary directory if [[ ! -z ${STORAGE} ]]; then + [[ ${VERBOSE} -eq 1 ]] && echo "export TMPDIR=${STORAGE}" export TMPDIR=${STORAGE} # mktemp fails if TMPDIR does not exist, so let's create it mkdir -p ${TMPDIR} @@ -529,6 +530,14 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru # provide a '/tmp' inside the container BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" + +# if TMPDIR is not empty and if TMP_IN_CONTAINER is not a prefix of TMPDIR, we need to add a bind mount for TMPDIR +if [[ ! -z ${TMPDIR} && ${TMP_IN_CONTAINER} != ${TMPDIR}* ]]; then + echo "TMPDIR is not empty (${TMPDIR}) and TMP_IN_CONTAINER (${TMP_IN_CONTAINER}) is not a prefix of TMPDIR: adding bind mount for TMPDIR" + BIND_PATHS="${BIND_PATHS},${TMPDIR}" +fi + +# add bind mount for extra bind paths if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then BIND_PATHS="${BIND_PATHS},${EXTRA_BIND_PATHS}" fi From be3c40164c5fb10ae5930638e413a0331668cf17 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 07:32:06 +0000 Subject: [PATCH 1786/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index d9c6075561..6f3946ca16 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,4 +1,5 @@ easyconfigs: + - EasyBuild-4.9.3.eb - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 From 36410891e0729b1d8c1e7b2e8a3bfd7e56aa38cf Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 07:41:26 +0000 Subject: [PATCH 1787/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index 6f3946ca16..d9c6075561 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,5 +1,4 @@ easyconfigs: - - EasyBuild-4.9.3.eb - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 From 2f7daaafdbd74901c1159e5a90ff8993a5bfe419 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 08:05:35 +0000 Subject: [PATCH 1788/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index d9c6075561..3b41b00bc4 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,4 +1,5 @@ easyconfigs: + - EasyBuild-4.9.3.eb: - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 From 310c67ad2efe4a14188439a48f9909cd85aed4b1 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 08:17:11 +0000 Subject: [PATCH 1789/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index 3b41b00bc4..d9c6075561 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,5 +1,4 @@ easyconfigs: - - EasyBuild-4.9.3.eb: - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 From d5680f6800a8869e233fcf2b7e5208d5890537b3 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 08:27:48 +0000 Subject: [PATCH 1790/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index d9c6075561..3b41b00bc4 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,4 +1,5 @@ easyconfigs: + - EasyBuild-4.9.3.eb: - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 From 7c4311699fb02e7f5ae496be717dd9b8d0d7f63e Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 08:30:34 +0000 Subject: [PATCH 1791/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index 3b41b00bc4..d9c6075561 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,5 +1,4 @@ easyconfigs: - - EasyBuild-4.9.3.eb: - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 From 8076409921235a05ea78d212425624926ca24bab Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 08:35:01 +0000 Subject: [PATCH 1792/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index d9c6075561..3b41b00bc4 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,4 +1,5 @@ easyconfigs: + - EasyBuild-4.9.3.eb: - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 From e12bff8e95fe20f281a327d82bd4fe7b9923caff Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 08:50:33 +0000 Subject: [PATCH 1793/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index 3b41b00bc4..c5efeaa48a 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,5 +1,6 @@ easyconfigs: - - EasyBuild-4.9.3.eb: + - EasyBuild-4.9.2.eb + - EasyBuild-4.9.3.eb - EasyBuild-4.9.4.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 From 22779e4cd0e7157d06bd808f26d3c5113e613a59 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 08:58:06 +0000 Subject: [PATCH 1794/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index c5efeaa48a..efa0916b2d 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,4 +1,5 @@ easyconfigs: + - EasyBuild-4.9.1.eb - EasyBuild-4.9.2.eb - EasyBuild-4.9.3.eb - EasyBuild-4.9.4.eb: From bbf6c8151a2e2535e747205e0a7b2defe788d9f9 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 12 Feb 2025 09:00:24 +0000 Subject: [PATCH 1795/1795] test-pr --- .../2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml index efa0916b2d..c5efeaa48a 100644 --- a/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/GH/eessi-2023.06-eb-4.9.4-001-system.yml @@ -1,5 +1,4 @@ easyconfigs: - - EasyBuild-4.9.1.eb - EasyBuild-4.9.2.eb - EasyBuild-4.9.3.eb - EasyBuild-4.9.4.eb: