From 3b24b318b1aaa0bbd8826fc0b038f4647143f260 Mon Sep 17 00:00:00 2001 From: Daniel Vickers Date: Mon, 15 Dec 2025 12:13:55 -0500 Subject: [PATCH 1/4] works on 2 ranks with GPU --- src/post_process/m_data_input.f90 | 1 - src/pre_process/m_data_output.fpp | 10 +++++++--- src/simulation/m_start_up.fpp | 16 +++++++++------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/post_process/m_data_input.f90 b/src/post_process/m_data_input.f90 index 98c85e8612..174a95272d 100644 --- a/src/post_process/m_data_input.f90 +++ b/src/post_process/m_data_input.f90 @@ -311,7 +311,6 @@ impure subroutine s_read_serial_data_files(t_step) STATUS='old', ACTION='read') read (1) q_cons_vf(i)%sf(0:m, 0:n, 0:p) close (1) - print *, q_cons_vf(i)%sf(:, 0, 0) else call s_mpi_abort('File q_cons_vf'//trim(file_num)// & '.dat is missing in '//trim(t_step_dir)// & diff --git a/src/pre_process/m_data_output.fpp b/src/pre_process/m_data_output.fpp index ca5fb3c56a..efaa1467e6 100644 --- a/src/pre_process/m_data_output.fpp +++ b/src/pre_process/m_data_output.fpp @@ -587,9 +587,9 @@ contains #ifdef MFC_MPI - integer :: ifile, ierr, data_size + integer :: ifile, ierr, data_size, count_int integer, dimension(MPI_STATUS_SIZE) :: status - integer(KIND=MPI_OFFSET_KIND) :: disp + integer(KIND=MPI_OFFSET_KIND) :: disp, file_pos integer(KIND=MPI_OFFSET_KIND) :: m_MOK, n_MOK, p_MOK integer(KIND=MPI_OFFSET_KIND) :: WP_MOK, var_MOK, str_MOK integer(KIND=MPI_OFFSET_KIND) :: NVARS_MOK @@ -792,13 +792,17 @@ contains call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & 'native', mpi_info_int, ierr) - call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + + call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size, & mpi_io_p, status, ierr) + call MPI_GET_COUNT(status, mpi_io_p, count_int, ierr) end do end if call MPI_FILE_CLOSE(ifile, ierr) + + inquire(FILE=trim(file_loc), SIZE=file_pos) end if ! IB Markers diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index eb5fd3179e..8e71348000 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -517,9 +517,9 @@ contains real(wp), allocatable, dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb - integer :: ifile, ierr, data_size + integer :: ifile, ierr, data_size, count_int integer, dimension(MPI_STATUS_SIZE) :: status - integer(KIND=MPI_OFFSET_KIND) :: disp + integer(KIND=MPI_OFFSET_KIND) :: disp, file_pos integer(KIND=MPI_OFFSET_KIND) :: m_MOK, n_MOK, p_MOK integer(KIND=MPI_OFFSET_KIND) :: WP_MOK, var_MOK, str_MOK integer(KIND=MPI_OFFSET_KIND) :: NVARS_MOK @@ -679,7 +679,7 @@ contains do i = 1, sys_size!adv_idx%end var_MOK = int(i, MPI_OFFSET_KIND) - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size*mpi_io_type, & mpi_io_p, status, ierr) end do !Read pb and mv for non-polytropic qbmm @@ -687,7 +687,7 @@ contains do i = sys_size + 1, sys_size + 2*nb*nnode var_MOK = int(i, MPI_OFFSET_KIND) - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size*mpi_io_type, & mpi_io_p, status, ierr) end do end if @@ -824,7 +824,7 @@ contains call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & 'native', mpi_info_int, ierr) - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size*mpi_io_type, & mpi_io_p, status, ierr) end do !Read pb and mv for non-polytropic qbmm @@ -836,7 +836,7 @@ contains call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & 'native', mpi_info_int, ierr) - call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size*mpi_io_type, & mpi_io_p, status, ierr) end do end if @@ -849,8 +849,10 @@ contains call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & 'native', mpi_info_int, ierr) - call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + + call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size, & mpi_io_p, status, ierr) + call MPI_GET_COUNT(status, mpi_io_p, count_int, ierr) end do end if From 45bdac56bce6b921b2ea247513958bb0caa06e12 Mon Sep 17 00:00:00 2001 From: Daniel Vickers Date: Mon, 15 Dec 2025 12:17:15 -0500 Subject: [PATCH 2/4] Added more writes --- src/pre_process/m_data_output.fpp | 11 ++++------- src/simulation/m_start_up.fpp | 5 ++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/pre_process/m_data_output.fpp b/src/pre_process/m_data_output.fpp index efaa1467e6..2bf1b3108a 100644 --- a/src/pre_process/m_data_output.fpp +++ b/src/pre_process/m_data_output.fpp @@ -587,9 +587,9 @@ contains #ifdef MFC_MPI - integer :: ifile, ierr, data_size, count_int + integer :: ifile, ierr, data_size integer, dimension(MPI_STATUS_SIZE) :: status - integer(KIND=MPI_OFFSET_KIND) :: disp, file_pos + integer(KIND=MPI_OFFSET_KIND) :: disp integer(KIND=MPI_OFFSET_KIND) :: m_MOK, n_MOK, p_MOK integer(KIND=MPI_OFFSET_KIND) :: WP_MOK, var_MOK, str_MOK integer(KIND=MPI_OFFSET_KIND) :: NVARS_MOK @@ -765,7 +765,7 @@ contains call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & 'native', mpi_info_int, ierr) - call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size*mpi_io_type, & mpi_io_p, status, ierr) end do !Additional variables pb and mv for non-polytropic qbmm @@ -778,7 +778,7 @@ contains call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), & 'native', mpi_info_int, ierr) - call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, & + call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size*mpi_io_type, & mpi_io_p, status, ierr) end do end if @@ -795,14 +795,11 @@ contains call MPI_FILE_WRITE_ALL(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size, & mpi_io_p, status, ierr) - call MPI_GET_COUNT(status, mpi_io_p, count_int, ierr) end do end if call MPI_FILE_CLOSE(ifile, ierr) - - inquire(FILE=trim(file_loc), SIZE=file_pos) end if ! IB Markers diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 8e71348000..01191cc151 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -517,9 +517,9 @@ contains real(wp), allocatable, dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb - integer :: ifile, ierr, data_size, count_int + integer :: ifile, ierr, data_size integer, dimension(MPI_STATUS_SIZE) :: status - integer(KIND=MPI_OFFSET_KIND) :: disp, file_pos + integer(KIND=MPI_OFFSET_KIND) :: disp integer(KIND=MPI_OFFSET_KIND) :: m_MOK, n_MOK, p_MOK integer(KIND=MPI_OFFSET_KIND) :: WP_MOK, var_MOK, str_MOK integer(KIND=MPI_OFFSET_KIND) :: NVARS_MOK @@ -852,7 +852,6 @@ contains call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf(1, 1, 1), data_size, & mpi_io_p, status, ierr) - call MPI_GET_COUNT(status, mpi_io_p, count_int, ierr) end do end if From 9d8edcc111d8e71368f49c4c204d339a8e0fae3f Mon Sep 17 00:00:00 2001 From: Daniel Vickers Date: Mon, 15 Dec 2025 12:18:07 -0500 Subject: [PATCH 3/4] Updated modules: --- toolchain/modules | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/toolchain/modules b/toolchain/modules index 65686085b6..bca6e71e7f 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -84,10 +84,7 @@ san-gpu nvhpc cuda cray-mpich h hipergator h-gpu nvhpc/25.9 -h-gpu CUDA_HOME="/apps/compilers/cuda/12.8.1" -h-all HPC_OMPI_DIR="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7" -h-all HPC_OMPI_BIN="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin" -h-all OMPI_MCA_pml=ob1 OMPI_MCA_coll_hcoll_enable=0 -h-gpu PATH="/apps/mpi/cuda/12.8.1/nvhpc/25.3/openmpi/5.0.7/bin:${PATH}" -h-all LD_LIBRARY_PATH=/apps/compilers/cuda/12.8.1/lib64:$LD_LIBRARY_PATH -h-gpu MFC_CUDA_CC=100 NVHPC_CUDA_HOME="/apps/compilers/cuda/12.8.1" +h-gpu openmpi/5.0.7 +h-gpu MFC_CUDA_CC=100 +h-gpu LD_LIBRARY_PATH=/apps/compilers/nvhpc/25.9/Linux_x86_64/25.9/math_libs/12.9/lib64:$LD_LIBRARY_PATH +h-gpu UCX_NET_DEVICES="mlx5_4:1,mlx5_7:1,mlx5_8:1,mlx5_9:1,mlx5_10:1,mlx5_13:1,mlx5_14:1,mlx5_15:1" From 4e57faccc677c1924a0ebaaa1d230b1c951868ca Mon Sep 17 00:00:00 2001 From: Daniel Vickers Date: Mon, 15 Dec 2025 13:43:16 -0500 Subject: [PATCH 4/4] Added python requirement --- toolchain/modules | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolchain/modules b/toolchain/modules index bca6e71e7f..22f12fe155 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -85,6 +85,7 @@ san-gpu nvhpc cuda cray-mpich h hipergator h-gpu nvhpc/25.9 h-gpu openmpi/5.0.7 +h-gpu python/3.12 h-gpu MFC_CUDA_CC=100 -h-gpu LD_LIBRARY_PATH=/apps/compilers/nvhpc/25.9/Linux_x86_64/25.9/math_libs/12.9/lib64:$LD_LIBRARY_PATH -h-gpu UCX_NET_DEVICES="mlx5_4:1,mlx5_7:1,mlx5_8:1,mlx5_9:1,mlx5_10:1,mlx5_13:1,mlx5_14:1,mlx5_15:1" +h-all LD_LIBRARY_PATH=/apps/compilers/nvhpc/25.9/Linux_x86_64/25.9/math_libs/12.9/lib64:$LD_LIBRARY_PATH +h-all UCX_NET_DEVICES="mlx5_4:1,mlx5_7:1,mlx5_8:1,mlx5_9:1,mlx5_10:1,mlx5_13:1,mlx5_14:1,mlx5_15:1"