Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 69 additions & 10 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,24 @@ option( BUILD_UNITY "enables unity build for faster compile times" ON )
option( BUILD_CODE_COV "enables compiler option required for code coverage analysis" OFF )
option( BUILD_ML "enables build with tensorflow backend access" OFF )
option( BUILD_MPI "enables build with MPI access" OFF )
option( BUILD_CUDA_HPC "enables CUDA backend for SN HPC solver (single GPU)" OFF )
#################################################


### COMPILER ####################################
set( CMAKE_CXX_STANDARD 17 )
set( CMAKE_CXX_STANDARD_REQUIRED ON )
if( BUILD_CUDA_HPC )
enable_language( CUDA )
if( CMAKE_VERSION VERSION_LESS 3.18 )
# CMake 3.16 does not ship CUDA17 mappings by default.
set( CMAKE_CUDA17_STANDARD_COMPILE_OPTION "--std=c++17" )
set( CMAKE_CUDA17_EXTENSION_COMPILE_OPTION "--std=c++17" )
endif()
set( CMAKE_CUDA_STANDARD 17 )
set( CMAKE_CUDA_STANDARD_REQUIRED ON )
set( CMAKE_CUDA_EXTENSIONS OFF )
endif()
set( KITRT_RELEASE_OPTIONS -march=native -w )
set( KITRT_RELWITHDEBINFO_OPTIONS -march=native -pg -no-pie )
set( KITRT_DEBUG_OPTIONS -Wall -Wextra -Wpedantic )
Expand All @@ -30,6 +42,21 @@ set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})

find_package( OpenMP REQUIRED )

if( BUILD_CUDA_HPC )
add_compile_definitions( KITRT_ENABLE_CUDA_HPC )
if( CMAKE_VERSION VERSION_GREATER_EQUAL 3.17 )
find_package( CUDAToolkit QUIET )
endif()
if( DEFINED CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES )
include_directories( ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} )
elseif( EXISTS "/usr/local/cuda/include" )
include_directories( /usr/local/cuda/include )
endif()
message( STATUS "CUDA HPC solver: enabled" )
else()
message( STATUS "CUDA HPC solver: disabled" )
endif()

message(STATUS "MPI build flag: ${BUILD_MPI}")
if( BUILD_MPI )
add_definitions(-DIMPORT_MPI)
Expand Down Expand Up @@ -72,6 +99,20 @@ else()
set( CORE_LIBRARIES ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${VTK_LIBRARIES} OpenMP::OpenMP_CXX -lstdc++fs )
endif()

if( BUILD_CUDA_HPC )
if( TARGET CUDA::cudart )
list( APPEND CORE_LIBRARIES CUDA::cudart )
else()
find_library( CUDART_LIBRARY
NAMES cudart
HINTS ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} /usr/local/cuda/lib64 /usr/lib/x86_64-linux-gnu )
if( NOT CUDART_LIBRARY )
message( FATAL_ERROR "Could not find CUDA runtime library (cudart)." )
endif()
list( APPEND CORE_LIBRARIES ${CUDART_LIBRARY} )
endif()
endif()


#################################################

Expand All @@ -89,6 +130,12 @@ add_compile_definitions( GIT_HASH="${GIT_HASH}" )

### BUILD KIT-RT ################################
file( GLOB_RECURSE SRCS RELATIVE ${CMAKE_SOURCE_DIR} "src/*.cpp" "include/*.h" )
if( BUILD_CUDA_HPC )
list( APPEND SRCS "src/solvers/snsolver_hpc.cu" "include/solvers/snsolver_hpc_cuda.hpp" )
# Force nvcc to parse this translation unit as C++17 even if transitive
# imported target features inject an older fallback standard.
set_source_files_properties( src/solvers/snsolver_hpc.cu PROPERTIES COMPILE_FLAGS "--std=c++17" )
endif()
set( EXCLUDE_DIR "/gui/" )
foreach( TMP_PATH ${SRCS} )
string( FIND ${TMP_PATH} ${EXCLUDE_DIR} EXCLUDE_DIR_FOUND )
Expand All @@ -110,9 +157,15 @@ if( BUILD_ML )
target_link_libraries( ${CMAKE_PROJECT_NAME} ${CORE_LIBRARIES} ${TENSORFLOW_LIB})
endif()
target_link_libraries( ${CMAKE_PROJECT_NAME} ${CORE_LIBRARIES} )
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<CONFIG:DEBUG>:${KITRT_DEBUG_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<CONFIG:RELWITHDEBINFO>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<CONFIG:RELEASE>:${KITRT_RELEASE_OPTIONS}>" )
if( BUILD_CUDA_HPC )
set_target_properties(
${CMAKE_PROJECT_NAME}
PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON CUDA_EXTENSIONS OFF
)
endif()
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>:${KITRT_RELEASE_OPTIONS}>" )
#################################################

### BUILD UNIT TESTS ############################
Expand Down Expand Up @@ -145,20 +198,26 @@ if( BUILD_TESTING )
target_compile_definitions( unit_tests PUBLIC BUILD_TESTING )
target_compile_definitions( unit_tests PUBLIC TESTS_PATH="${CMAKE_SOURCE_DIR}/tests/" )
target_link_libraries( unit_tests Catch ${CORE_LIBRARIES} )
target_compile_options( unit_tests PUBLIC "$<$<CONFIG:DEBUG>:${KITRT_DEBUG_OPTIONS}>" )
if( BUILD_CUDA_HPC )
set_target_properties(
unit_tests
PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON CUDA_EXTENSIONS OFF
)
endif()
target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
if( BUILD_CODE_COV )
if( CMAKE_COMPILER_IS_GNUCXX )
set( CODE_COVERAGE_OPTIONS --coverage -g -O0 -w )
target_compile_options( unit_tests PUBLIC "$<$<CONFIG:DEBUG>:${CODE_COVERAGE_OPTIONS}>" )
target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${CODE_COVERAGE_OPTIONS}>" )
target_link_libraries( unit_tests Catch gcov )
else()
message( FATAL_ERROR "Code coverage is currently only supported for gcc!" )
endif()
else()
target_link_libraries( unit_tests Catch )
endif()
target_compile_options( unit_tests PUBLIC "$<$<CONFIG:RELWITHDEBINFO>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
target_compile_options( unit_tests PUBLIC "$<$<CONFIG:RELEASE>:${KITRT_RELEASE_OPTIONS}>" )
target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>:${KITRT_RELEASE_OPTIONS}>" )
catch_discover_tests( unit_tests )
endif()
#################################################
Expand All @@ -179,9 +238,9 @@ if( BUILD_GUI )
set( GUI_LIBRARIES Qt5::Core Qt5::Widgets Qt5::Gui Qt5::OpenGL ${VTK_LIBRARIES} )
target_link_libraries( ${CMAKE_PROJECT_NAME}_gui ${CORE_LIBRARIES} ${GUI_LIBRARIES} )
set_target_properties( ${CMAKE_PROJECT_NAME}_gui PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}/bin" )
target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<CONFIG:DEBUG>:${KITRT_DEBUG_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<CONFIG:RELWITHDEBINFO>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<CONFIG:RELEASE>:${KITRT_RELEASE_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
target_compile_options( ${CMAKE_PROJECT_NAME}_gui PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>:${KITRT_RELEASE_OPTIONS}>" )
endif()
#################################################

Expand Down
143 changes: 104 additions & 39 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ KiT-RT is an open-source, multi-fidelity **C++ PDE solver for radiative transpor

* **Modular, HPC-ready** architecture. Supports hybrid **MPI/OpenMP distributed parallelism**.
* **Containerized** for portable deployment across HPC systems (Docker & Singularity).
* **Python-wrapped** via [CharmKIT](https://github.com/KiT-RT/CharmKIT)
* **Python-wrapped** via [charm_kit](https://github.com/KiT-RT/charm_kit)
* Downstream applications:
- Data generation for scientific **foundation models**.
- high-resolution **reference solutions** for AI-based surrogate modeling.
Expand Down Expand Up @@ -66,79 +66,144 @@ Applications include:



## Installation
## Installation and Run (By Parallelization)

### Plain cpp setup
One-time setup:
```bash
# Clone repository
git clone https://github.com/KiT-RT/kitrt_code.git
cd kitrt_code
git submodule update --init --recursive
```

# Build with CMake
mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ../
make -j
Then run all commands from the repository root.

```
### Docker setup
A preconfigured docker container can also be used to run the code.
By running
### 1. CPU (OpenMP only)

#### 1a) Plain installation (no container)
```bash
docker run --rm -ti -v $(pwd):/home kitrt/test:latest
mkdir -p build_omp
cd build_omp
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
make -j
cd ..
./build_omp/KiT-RT tests/input/validation_tests/SN_solver/checkerboard_SN.cfg
```

Bash scripts are provided in the folder tools/CI to get started with the docker environments. To start an interactive docker environment, execute
#### 1b) Docker installation
```bash
docker run -i -t --rm -v $(pwd)/../..:/mnt kitrt/test:latest /bin/bash
docker run --rm -it -v $(pwd):/mnt -w /mnt kitrt/test:latest /bin/bash
mkdir -p build_docker_omp
cd build_docker_omp
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
make -j
cd ..
./build_docker_omp/KiT-RT tests/input/validation_tests/SN_solver/checkerboard_SN.cfg
```

### Singularity setup
Create the singularity container
#### 1c) Singularity installation
```bash
mkdir build_singularity
cd tools/singularity
sudo sh build_container.sh
chmod +x install_kitrt_singularity.sh
singularity exec kit_rt.sif ./install_kitrt_singularity.sh
sudo singularity build kit_rt.sif kit_rt.def
cd ../..
mkdir -p build_singularity_omp
cd build_singularity_omp
singularity exec ../tools/singularity/kit_rt.sif \
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
singularity exec ../tools/singularity/kit_rt.sif make -j
cd ..
singularity exec tools/singularity/kit_rt.sif \
./build_singularity_omp/KiT-RT tests/input/validation_tests/SN_solver/checkerboard_SN.cfg
```

### 2. CPU (OpenMP + MPI)

#### 2a) Plain installation (no container)
```bash
mkdir -p build_mpi
cd build_mpi
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=ON -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
make -j
cd ..
mpirun -np 4 ./build_mpi/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cpu_order2.cfg
```
Run the singularity container

#### 2b) Singularity installation
```bash
singularity shell --bind $(pwd)/../..:/mnt kit_rt.sif
cd tools/singularity
sudo singularity build kit_rt_MPI.sif kit_rt_MPI.def
cd ../..
mkdir -p build_singularity_mpi
cd build_singularity_mpi
singularity exec ../tools/singularity/kit_rt_MPI.sif \
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=ON -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
singularity exec ../tools/singularity/kit_rt_MPI.sif make -j
cd ..
singularity exec tools/singularity/kit_rt_MPI.sif \
mpirun -np 4 ./build_singularity_mpi/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cpu_order2.cfg
```

## Running simulations
Within any of the above setups, navigate to the example folder and execute KiT-RT
### 3. CPU + single GPU (OpenMP + CUDA)

#### 3a) Singularity installation
```bash
cd examples
../<build_folder_name>/KiT-RT configs/lattice_SN.cfg
cd tools/singularity
sudo singularity build kit_rt_MPI_cuda.sif kit_rt_MPI_cuda.def
cd ../..
mkdir -p build_singularity_cuda
cd build_singularity_cuda
singularity exec --nv ../tools/singularity/kit_rt_MPI_cuda.sif \
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=ON -DBUILD_ML=OFF ..
singularity exec --nv ../tools/singularity/kit_rt_MPI_cuda.sif make -j
cd ..
singularity exec --nv tools/singularity/kit_rt_MPI_cuda.sif \
./build_singularity_cuda/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cuda_order2.cfg
```

## Tensorflow backend
If you choose to enable the integrated machine learning tools via the BUILD_ML option, you need to install the tensorflow C-API:
When compiled with `-DBUILD_CUDA_HPC=ON`, HPC runs use the CUDA backend if a GPU is visible, and fall back to CPU if no GPU is detected.

### 4. Build with TensorFlow backend (CPU + OpenMP only)

```bash
FILENAME=libtensorflow-cpu-linux-x86_64-2.7.0.tar.gz
wget -q --no-check-certificate https://storage.googleapis.com/tensorflow/libtensorflow/${FILENAME}
tar -C /usr/local -xzf ${FILENAME}
ldconfig /usr/local/lib
sudo tar -C /usr/local -xzf ${FILENAME}
sudo ldconfig /usr/local/lib
rm ${FILENAME}
mkdir -p build_ml
cd build_ml
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=ON ..
make -j
cd ..
./build_ml/KiT-RT tests/input/validation_tests/MN_solver/checkerboard_MN_neural.cfg
```
and for a gpu based version (you need supported hardware and gpu drivers, see here ):

### 5. Debug build

```bash
FILENAME=libtensorflow-gpu-linux-x86_64-2.7.0.tar.gz
wget -q --no-check-certificate https://storage.googleapis.com/tensorflow/libtensorflow/${FILENAME}
tar -C /usr/local -xzf ${FILENAME}
ldconfig /usr/local/lib
mkdir -p build_debug
cd build_debug
cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_MPI=OFF -DBUILD_CUDA_HPC=OFF -DBUILD_ML=OFF ..
make -j
cd ..
./build_debug/KiT-RT tests/input/validation_tests/SN_solver/checkerboard_SN.cfg
```
Or use the docker container

### 6. Test + coverage build

```bash
docker run --rm -ti -v $(pwd):/home kitrt/test_ml:latest
mkdir -p build_coverage
cd build_coverage
cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DBUILD_CODE_COV=ON -DBUILD_UNITY=OFF ..
make -j
./unit_tests
ctest --output-on-failure
gcovr -r .. --html-details coverage.html
```


## Python API

The Python interface is provided via [CharmKIT](https://github.com/KiT-RT/CharmKIT), allowing seamless integration into AI and outer-loop (UQ, Optimization) workflows.
The Python interface is provided via [charm_kit](https://github.com/KiT-RT/charm_kit), allowing seamless integration into AI and outer-loop (UQ, Optimization) workflows.
Check the corresponding readme for further info


Expand Down
3 changes: 2 additions & 1 deletion include/solvers/snsolver_hpc.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ class SNSolverHPC
std::vector<double> _flux; /*!< @brief dim = _nCells x _nSys */

// Output related members
std::vector<double> _scalarFlux; /*!< @brief dim = _nCells */
std::vector<double> _scalarFlux; /*!< @brief dim = _nCells */
std::vector<double> _scalarFluxPrevIter; /*!< @brief scalar flux snapshot before current solver iteration */

// Lattice QOIS
unsigned _nOutputMoments;
Expand Down
Loading