From e72f2cdd2047836037d3badabf0efb6d911222c8 Mon Sep 17 00:00:00 2001 From: Mauro Rodriguez Date: Tue, 9 Dec 2025 11:51:34 -0500 Subject: [PATCH 1/7] added updated modules for Oscar supercomputer --- toolchain/modules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/modules b/toolchain/modules index 65686085b6..69757c3c03 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -54,7 +54,7 @@ d-gpu MFC_CUDA_CC=80,86 o Brown Oscar o-all python -o-cpu hpcx-mpi +o-cpu hpcx-mpi python/3.13.10s o-gpu nvhpc cuda/12.3.0 cmake/3.26.3 o-gpu CC=nvc CXX=nvc++ FC=nvfortran From 8df72f846245a20ff3268d88e52362cec1ea9479 Mon Sep 17 00:00:00 2001 From: Mauro Rodriguez Date: Tue, 9 Dec 2025 16:04:56 -0500 Subject: [PATCH 2/7] fix to oscar mako file --- toolchain/templates/oscar.mako | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/toolchain/templates/oscar.mako b/toolchain/templates/oscar.mako index af54ca7187..fca54836d4 100644 --- a/toolchain/templates/oscar.mako +++ b/toolchain/templates/oscar.mako @@ -15,9 +15,8 @@ #SBATCH --account="${account}" % endif % if gpu_enabled: -#SBATCH --gpus-per-node=${tasks_per_node} -#SBATCH --mem=64G -#SBATCH --gpu-bind=closest +#SBATCH --gpu-bind=verbose,closest +#SBATCH --gres=gpu:v100-16:${tasks_per_node} % endif #SBATCH --output="${name}.out" #SBATCH --error="${name}.err" @@ -31,7 +30,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" -cd "${MFC_ROOTDIR}" +cd "${MFC_ROOT_DIR}" . ./mfc.sh load -c o -m ${'g' if gpu_enabled else 'c'} cd - > /dev/null echo @@ -42,9 +41,8 @@ echo % if not mpi: (set -x; ${profiler} "${target.get_install_binpath(case)}") % else: - (set -x; ${profiler} \ - mpirun -np ${nodes*tasks_per_node} \ - ${' '.join([f"'{x}'" for x in ARG('--') ])} \ + (set -x; ${profiler} \ + mpirun -np ${nodes*tasks_per_node} \ "${target.get_install_binpath(case)}") % endif From 0056072ad54011343dd1f4048fd2f53cf8c728c9 Mon Sep 17 00:00:00 2001 From: Mauro Rodriguez Date: Tue, 9 Dec 2025 16:30:45 -0500 Subject: [PATCH 3/7] merger --- toolchain/templates/oscar.mako | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/templates/oscar.mako b/toolchain/templates/oscar.mako index fca54836d4..223c0e6ae2 100644 --- a/toolchain/templates/oscar.mako +++ b/toolchain/templates/oscar.mako @@ -4,7 +4,7 @@ % if engine == 'batch': #SBATCH --nodes=${nodes} -#SBATCH --ntasks-per-node=${tasks_per_node} +#SBATCH --tasks-per-node=${tasks_per_node} #SBATCH --cpus-per-task=1 #SBATCH --job-name="${name}" #SBATCH --time=${walltime} From 93bc50ab62f1ce0fd0b2ec7a0e785f03f957f889 Mon Sep 17 00:00:00 2001 From: Mauro Rodriguez Date: Fri, 12 Dec 2025 00:39:56 -0500 Subject: [PATCH 4/7] Oscar mako to account for srun --- toolchain/templates/oscar.mako | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchain/templates/oscar.mako b/toolchain/templates/oscar.mako index 223c0e6ae2..1533e87fbf 100644 --- a/toolchain/templates/oscar.mako +++ b/toolchain/templates/oscar.mako @@ -42,7 +42,7 @@ echo (set -x; ${profiler} "${target.get_install_binpath(case)}") % else: (set -x; ${profiler} \ - mpirun -np ${nodes*tasks_per_node} \ + srun \ "${target.get_install_binpath(case)}") % endif From 9a998580ca9be77e51e182da697eba1425041f6c Mon Sep 17 00:00:00 2001 From: Mauro Rodriguez Date: Wed, 28 Jan 2026 20:02:46 -0500 Subject: [PATCH 5/7] added Purdue Anvil --- toolchain/modules | 8 ++++++ toolchain/templates/anvil.mako | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 toolchain/templates/anvil.mako diff --git a/toolchain/modules b/toolchain/modules index c8ed12940b..f1ad6131d0 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -64,6 +64,14 @@ o-cpu hpcx-mpi python/3.13.10s o-gpu nvhpc cuda/12.3.0 cmake/3.26.3 o-gpu CC=nvc CXX=nvc++ FC=nvfortran +pa Purdue Anvil +pa-cpu gcc/11.2.0 +pa-cpu openmpi/4.1.6 +pa-cpu python/3.9.5 +pa-cpu fftw/3.3.8 +pa-cpu cmake/3.20.0 +pa-all PYTHONPATH="" + dai NCSA DeltaAI dai-all python cmake nvhpc-openmpi3/24.3 cuda dai-all CC=nvc CXX=nvc++ FC=nvfortran diff --git a/toolchain/templates/anvil.mako b/toolchain/templates/anvil.mako new file mode 100644 index 0000000000..3a9d7a4605 --- /dev/null +++ b/toolchain/templates/anvil.mako @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +<%namespace name="helpers" file="helpers.mako"/> + +% if engine == 'batch': +#SBATCH --nodes=${nodes} +#SBATCH --ntasks-per-node=${tasks_per_node} +#SBATCH --cpus-per-task=1 +#SBATCH --job-name="${name}" +#SBATCH --time=${walltime} +% if partition: +##SBATCH --partition=${partition} +% endif +% if account: +#SBATCH --account="${account}" +% endif +#SBATCH --output="${name}.out" +#SBATCH --error="${name}.err" +#SBATCH --export=ALL +% if email: +#SBATCH --mail-user=${email} +#SBATCH --mail-type="BEGIN, END, FAIL" +% endif +% endif + +${helpers.template_prologue()} + +ok ":) Loading modules:\n" +cd "${MFC_ROOT_DIR}" +. ./mfc.sh load -c pa -m ${'g' if gpu_enabled else 'c'} +cd - > /dev/null +echo + +% for target in targets: + ${helpers.run_prologue(target)} + + % if not mpi: + (set -x; ${profiler} "${target.get_install_binpath(case)}") + % else: + (set -x; ${profiler} \ + mpirun -np ${nodes*tasks_per_node} \ + "${target.get_install_binpath(case)}") + % endif + + ${helpers.run_epilogue(target)} + + echo +% endfor + +${helpers.template_epilogue()} From 3e5a88dabc498b40b66a81dc1431e31bfcbaf465 Mon Sep 17 00:00:00 2001 From: Mauro Rodriguez Date: Wed, 4 Feb 2026 14:06:37 -0500 Subject: [PATCH 6/7] added Purdue Anvil mako file --- toolchain/bootstrap/modules.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/toolchain/bootstrap/modules.sh b/toolchain/bootstrap/modules.sh index bc83d503a6..9c2db8a62e 100644 --- a/toolchain/bootstrap/modules.sh +++ b/toolchain/bootstrap/modules.sh @@ -43,7 +43,8 @@ if [ -v $u_c ]; then log "$C""ACCESS$W: Bridges2 (b) | Expanse (e) | Delta (d) | DeltaAI (dai)" log "$Y""Gatech$W: Phoenix (p)" log "$R""Caltech$W: Richardson (r)" - log "$BR""Brown$W: Oscar (o)" + log "$BR""Brown$W: Oscar (o)" + log "$BR""Purdue$W: Anvil (pa)" log "$B""DoD$W: Carpenter Cray (cc) | Carpenter GNU (c) | Nautilus (n)" log "$OR""Florida$W: HiPerGator (h)" log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$B""tuo$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""cc$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"/$OR""h"$CR"): " From da17a2484fd0387a923b8ee1e6c1a4c484a95a7f Mon Sep 17 00:00:00 2001 From: Mauro Rodriguez Date: Fri, 3 Apr 2026 00:21:24 -0400 Subject: [PATCH 7/7] updating mako and modules for Brown Oscar --- toolchain/modules | 2 +- toolchain/templates/oscar.mako | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/toolchain/modules b/toolchain/modules index f1766d72d1..1885e54306 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -65,7 +65,7 @@ d-gpu MFC_CUDA_CC=80,86 o Brown Oscar o-all python -o-cpu hpcx-mpi python/3.13.10s +o-cpu hpcx-mpi python/3.13.5 o-gpu nvhpc cuda/12.3.0 cmake/3.26.3 o-gpu CC=nvc CXX=nvc++ FC=nvfortran diff --git a/toolchain/templates/oscar.mako b/toolchain/templates/oscar.mako index 1533e87fbf..9fdc6831fa 100644 --- a/toolchain/templates/oscar.mako +++ b/toolchain/templates/oscar.mako @@ -6,6 +6,7 @@ #SBATCH --nodes=${nodes} #SBATCH --tasks-per-node=${tasks_per_node} #SBATCH --cpus-per-task=1 +#SBATCH --mem-per-cpu=4g #SBATCH --job-name="${name}" #SBATCH --time=${walltime} % if partition: