From 3b8f6f51164ba7236c89d8347c574cd37e48df65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Aug 2025 17:28:46 +0200 Subject: [PATCH 01/15] add possibility for generating zstd-compressed tarballs --- create_tarball.sh | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index d104e210..c39134be 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -12,7 +12,7 @@ eessi_tmpdir=$1 eessi_version=$2 cpu_arch_subdir=$3 accel_subdir=$4 -target_tgz=$5 +target_tarball=$5 tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" @@ -114,10 +114,29 @@ fi topdir=${cvmfs_repo}/versions/ -echo ">> Creating tarball ${target_tgz} from ${topdir}..." -tar cfvz ${target_tgz} -C ${topdir} --files-from=${files_list} - -echo ${target_tgz} created! +echo ">> Creating tarball ${target_tarball} from ${topdir}..." +target_tarball_ext="${target_tarball##*.}" +if [ "${target_tarball_ext}" = "zst" ]; then + if [ -x "$(command -v zstd)" ]; then + tar -cf - -C ${topdir} --files-from=${files_list} | zstd -T0 > ${target_tarball} + else + echo "Zstd compression requested, but zstd command is not available." >&2 + exit 4 + fi +elif [ "${target_tarball_ext}" = "gz" ]; then + if [[ -x "$(command -v gzip)" ]]; then + tar -cfvz ${target_tarball} -C ${topdir} --files-from=${files_list} + else + echo "Gzip compression requested, but gzip command is not available." >&2 + exit 4 + fi +elif [ "${target_tarball_ext}" = "tar" ]; then + tar -cfv ${target_tarball} -C ${topdir} --files-from=${files_list} +else + echo "Unknown tarball type (${target_tarball_ext}) requested." + exit 4 +fi +echo ${target_tarball} created! echo ">> Cleaning up tmpdir ${tmpdir}..." rm -r ${tmpdir} From 460cd1d7be75769f992d910f6d05cbf11cc90729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Aug 2025 17:37:22 +0200 Subject: [PATCH 02/15] add dummy easystack for testing --- .../2023.06/eessi-2023.06-eb-5.1.1-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.1.1-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.1.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.1.1-001-system.yml new file mode 100644 index 00000000..44fe4d92 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.1.1-001-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - cowsay-3.04.eb From 0ba48f7dda6767d8ff1ef879d4a6bd4de1edf2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 8 Aug 2025 21:11:22 +0200 Subject: [PATCH 03/15] use correct options for tar command --- create_tarball.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index c39134be..4cf49b8c 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -118,14 +118,14 @@ echo ">> Creating tarball ${target_tarball} from ${topdir}..." target_tarball_ext="${target_tarball##*.}" if [ "${target_tarball_ext}" = "zst" ]; then if [ -x "$(command -v zstd)" ]; then - tar -cf - -C ${topdir} --files-from=${files_list} | zstd -T0 > ${target_tarball} + tar -cvf - -C ${topdir} --files-from=${files_list} | zstd -T0 > ${target_tarball} else echo "Zstd compression requested, but zstd command is not available." >&2 exit 4 fi elif [ "${target_tarball_ext}" = "gz" ]; then if [[ -x "$(command -v gzip)" ]]; then - tar -cfvz ${target_tarball} -C ${topdir} --files-from=${files_list} + tar -czvf ${target_tarball} -C ${topdir} --files-from=${files_list} else echo "Gzip compression requested, but gzip command is not available." >&2 exit 4 From f4e9fef0a68442b7b20f59fef3550fbf29cae965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Aug 2025 15:08:01 +0200 Subject: [PATCH 04/15] use tar's auto-compress option --- create_tarball.sh | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 4cf49b8c..55247868 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -115,27 +115,7 @@ fi topdir=${cvmfs_repo}/versions/ echo ">> Creating tarball ${target_tarball} from ${topdir}..." -target_tarball_ext="${target_tarball##*.}" -if [ "${target_tarball_ext}" = "zst" ]; then - if [ -x "$(command -v zstd)" ]; then - tar -cvf - -C ${topdir} --files-from=${files_list} | zstd -T0 > ${target_tarball} - else - echo "Zstd compression requested, but zstd command is not available." >&2 - exit 4 - fi -elif [ "${target_tarball_ext}" = "gz" ]; then - if [[ -x "$(command -v gzip)" ]]; then - tar -czvf ${target_tarball} -C ${topdir} --files-from=${files_list} - else - echo "Gzip compression requested, but gzip command is not available." >&2 - exit 4 - fi -elif [ "${target_tarball_ext}" = "tar" ]; then - tar -cfv ${target_tarball} -C ${topdir} --files-from=${files_list} -else - echo "Unknown tarball type (${target_tarball_ext}) requested." - exit 4 -fi +tar cavf ${target_tgz} -C ${topdir} --files-from=${files_list} echo ${target_tarball} created! echo ">> Cleaning up tmpdir ${tmpdir}..." From a9baa425b7b4e632b8332e3c7013feec93e3758f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Aug 2025 15:39:06 +0200 Subject: [PATCH 05/15] use correct var name --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 55247868..3e79a571 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -115,7 +115,7 @@ fi topdir=${cvmfs_repo}/versions/ echo ">> Creating tarball ${target_tarball} from ${topdir}..." -tar cavf ${target_tgz} -C ${topdir} --files-from=${files_list} +tar cavf ${target_tarball} -C ${topdir} --files-from=${files_list} echo ${target_tarball} created! echo ">> Cleaning up tmpdir ${tmpdir}..." From 55aacf65f8c10c98b0b4f69eefe08b5f5dbe5cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Aug 2025 15:46:40 +0200 Subject: [PATCH 06/15] dummy easystack for 2025.06 --- .../2025.06/eessi-2025.06-eb-5.1.1-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml diff --git a/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml b/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml new file mode 100644 index 00000000..44fe4d92 --- /dev/null +++ b/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - cowsay-3.04.eb From eeec4da5651231639986bee76a36c8e43452f9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Aug 2025 15:51:31 +0200 Subject: [PATCH 07/15] remove 2023.06 easystack --- ...-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml | 13 ------------- .../2023.06/eessi-2023.06-eb-5.1.1-001-system.yml | 2 -- 2 files changed, 15 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml delete mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.1.1-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml deleted file mode 100644 index 839125de..00000000 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml +++ /dev/null @@ -1,13 +0,0 @@ -# In https://github.com/EESSI/software-layer-scripts/pull/59 we introduced a new location for -# installing the CUDA toolkit within the host_injections directory. This requires reinstallation -# of CUDA and cuDNN to make sure all symlinks point to these new locations -easyconfigs: - - CUDA-12.1.1.eb: - options: - accept-eula-for: CUDA - - CUDA-12.4.0.eb: - options: - accept-eula-for: CUDA - - cuDNN-8.9.2.26-CUDA-12.1.1.eb: - options: - accept-eula-for: cuDNN diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.1.1-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.1.1-001-system.yml deleted file mode 100644 index 44fe4d92..00000000 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.1.1-001-system.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - cowsay-3.04.eb From f9727e8eec58c15e6b31f59084b96cd4a9263031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Aug 2025 15:57:42 +0200 Subject: [PATCH 08/15] use zst instead of gz for tarballs --- bot/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index eb7c68c5..d5d8cc7a 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -272,9 +272,9 @@ source $software_layer_dir/init/eessi_defaults # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is # appended if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then - export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.tar.zst" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) else - export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.zst" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball @@ -288,8 +288,8 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT} TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr} exit 0 From fc5d5ad02aef6ec9bb8db263c65a03c8a011d2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 26 Aug 2025 16:03:29 +0200 Subject: [PATCH 09/15] search for any kind of (compressed) tarball --- bot/check-build.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 507d962b..34d50f16 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -17,7 +17,7 @@ # - SUCCESS (all of) # - working directory contains slurm-JOBID.out file -# - working directory contains eessi*tar.gz +# - working directory contains eessi*tar* # - no message FATAL # - no message ERROR # - no message FAILED @@ -165,19 +165,19 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then fi if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then - TGZ=-1 + TARBALL_CREATED=-1 TARBALL= if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then - GP_tgz_created="\.tar\.gz created!" - grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) + GP_tarball_created="\.tar.* created!" + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tarball_created}" | sort -u) if [[ $? -eq 0 ]]; then - TGZ=1 + TARBALL_CREATED=1 TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') else - TGZ=0 + TARBALL_CREATED=0 fi # have to be careful to not add searched for pattern into slurm out file - [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tarball_created}"'" [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi fi @@ -190,7 +190,7 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then - [[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" + [[ ${VERBOSE} -ne 0 ]] && echo " TARBALL_CREATED: $([[ $TARBALL -eq 1 ]] && echo 'yes' || echo 'no') (yes)" fi # Here, we try to do some additional analysis on the output file @@ -219,7 +219,7 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ [[ ${FAILED} -eq 0 ]] && \ [[ ${MISSING} -eq 0 ]] && \ [[ ${NO_MISSING} -eq 1 ]] && \ - [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \ + [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TARBALL_CREATED} -eq 1 ]] && \ [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then # SUCCESS status="SUCCESS" @@ -429,9 +429,9 @@ failure_msg="no message matching ${GP_no_missing}" comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then - success_msg="found message matching ${GP_tgz_created}" - failure_msg="no message matching ${GP_tgz_created}" - comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") + success_msg="found message matching ${GP_tarball_created}" + failure_msg="no message matching ${GP_tarball_created}" + comment_details_list=${comment_details_list}$(add_detail ${TARBALL_CREATED} 1 "${success_msg}" "${failure_msg}") fi # Now, do the actual replacement of __DETAILS_FMT__ From 5adbacb8ee9c5611d57643c40426d17faf5b129b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 11 Nov 2025 21:44:05 +0100 Subject: [PATCH 10/15] set tarball extension based on availability of compression tools --- bot/build.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index d5d8cc7a..747fdc38 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -265,6 +265,13 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") timestamp=$(date +%s) +if [[ -x "$(command -v zstd)" ]]; then + tarball_extension="tar.zst" +elif [[ -x "$(command -v gzip)" ]]; then + tarball_extension="tar.gz" +else + tarball_extension="tar" +fi # to set EESSI_VERSION we need to source init/eessi_defaults now source $software_layer_dir/init/eessi_defaults # Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we @@ -272,9 +279,9 @@ source $software_layer_dir/init/eessi_defaults # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is # appended if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then - export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.tar.zst" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) else - export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.zst" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) fi # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball From 5e5518c9e4fafa59e297ee555cdca29da930fb5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 11 Nov 2025 21:44:25 +0100 Subject: [PATCH 11/15] tar tf needs to check the compression --- bot/check-build.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 866c11c0..0b26a54c 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -478,7 +478,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then size="$(stat --dereference --printf=%s ${TARBALL})" size_mib=$((${size} >> 20)) tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) - tar tf ${TARBALL} > ${tmpfile} + if [[ "${TARBALL}" == *.tar.zst ]]; then + tar tf --use-compress-program=zstd ${TARBALL} > ${tmpfile} + elif [[ "${TARBALL}" == *.tar.gz ]]; then + tar tf --use-compress-program=gzip ${TARBALL} > ${tmpfile} + elif [[ "${TARBALL}" == *.tar ]]; then + tar tf ${TARBALL} > ${tmpfile} + else + echo "ERROR: Unsupported tarball extension!" >&2 + exit 1 + fi entries=$(cat ${tmpfile} | wc -l) # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512 From e54a99f95d906df5cdd94a9951215a69807e32eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 11 Nov 2025 21:55:44 +0100 Subject: [PATCH 12/15] fix tar command, order matters, use -tf instead of tf --- bot/check-build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 0b26a54c..b3bd798c 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -479,11 +479,11 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then size_mib=$((${size} >> 20)) tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) if [[ "${TARBALL}" == *.tar.zst ]]; then - tar tf --use-compress-program=zstd ${TARBALL} > ${tmpfile} + tar --use-compress-program=zstd -tf ${TARBALL} > ${tmpfile} elif [[ "${TARBALL}" == *.tar.gz ]]; then - tar tf --use-compress-program=gzip ${TARBALL} > ${tmpfile} + tar --use-compress-program=gzip -tf ${TARBALL} > ${tmpfile} elif [[ "${TARBALL}" == *.tar ]]; then - tar tf ${TARBALL} > ${tmpfile} + tar -tf ${TARBALL} > ${tmpfile} else echo "ERROR: Unsupported tarball extension!" >&2 exit 1 From 33076f9a40f7a8fcdd5f89f120f1ed319209974a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 11 Nov 2025 22:08:33 +0100 Subject: [PATCH 13/15] add comment about tarball extension --- bot/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/build.sh b/bot/build.sh index 747fdc38..91c66c78 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -265,6 +265,7 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") timestamp=$(date +%s) +# determine compression/extension for tarball, check in order of preference if [[ -x "$(command -v zstd)" ]]; then tarball_extension="tar.zst" elif [[ -x "$(command -v gzip)" ]]; then From 973b8144f6579d948a259ef9be8def93df33a286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 11 Nov 2025 22:09:03 +0100 Subject: [PATCH 14/15] remove easystack --- .../2025.06/eessi-2025.06-eb-5.1.1-001-system.yml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml diff --git a/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml b/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml deleted file mode 100644 index 44fe4d92..00000000 --- a/easystacks/software.eessi.io/2025.06/eessi-2025.06-eb-5.1.1-001-system.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - cowsay-3.04.eb From bfa88845454306cbc579163375b39a87c2f73d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 11 Nov 2025 22:15:30 +0100 Subject: [PATCH 15/15] restore deleted easystack --- ...-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml new file mode 100644 index 00000000..839125de --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250807-eb-5.1.1-CUDA-cuDNN-new-host-injections-dir.yml @@ -0,0 +1,13 @@ +# In https://github.com/EESSI/software-layer-scripts/pull/59 we introduced a new location for +# installing the CUDA toolkit within the host_injections directory. This requires reinstallation +# of CUDA and cuDNN to make sure all symlinks point to these new locations +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA + - CUDA-12.4.0.eb: + options: + accept-eula-for: CUDA + - cuDNN-8.9.2.26-CUDA-12.1.1.eb: + options: + accept-eula-for: cuDNN