Merge branch 'master' into MTHINC

sbryngelson · web-flow · commit f3caca5d3d1e · 2026-04-02T14:51:19.000-04:00
diff --git a/.github/workflows/common/build.sh b/.github/workflows/common/build.sh
@@ -40,4 +40,4 @@ if [ "$job_cluster" = "phoenix" ]; then
 fi
 
 RETRY_VALIDATE_CMD="$validate_cmd" \
-    retry_build ./mfc.sh test -v --dry-run -j 8 $build_opts || exit 1
+    retry_build ./mfc.sh test -v --dry-run -a -j 8 $build_opts || exit 1
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -213,25 +213,25 @@ jobs:
       fail-fast: false
     continue-on-error: true
     runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }}
-    container:
-      image: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }}
-      options: ${{ matrix.nvhpc && '--security-opt seccomp=unconfined' || '' }}
     env:
-      CC:  ${{ matrix.nvhpc && 'nvc' || '' }}
-      CXX: ${{ matrix.nvhpc && 'nvc++' || '' }}
-      FC:  ${{ matrix.nvhpc && 'nvfortran' || '' }}
-      OMPI_ALLOW_RUN_AS_ROOT:         ${{ matrix.nvhpc && '1' || '' }}
-      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: ${{ matrix.nvhpc && '1' || '' }}
-      PMIX_MCA_gds:                   ${{ matrix.nvhpc && 'hash' || '' }}
-      OMPI_MCA_hwloc_base_binding_policy: ${{ matrix.nvhpc && 'none' || '' }}
-      FFLAGS:  ${{ matrix.nvhpc && '-tp=px -Kieee -noswitcherror' || '' }}
-      CFLAGS:  ${{ matrix.nvhpc && '-tp=px' || '' }}
-      CXXFLAGS: ${{ matrix.nvhpc && '-tp=px' || '' }}
+      # Image tag for NVHPC jobs; empty for non-NVHPC jobs.
+      NVHPC_IMAGE: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }}
 
     steps:
-      - name: Git safe directory
+      # ── NVHPC: free disk before pulling the ~25-30 GB cuda_multi image ──
+      - name: Free disk space
         if:   matrix.nvhpc
-        run:  git config --global --add safe.directory /__w/MFC/MFC
+        run: |
+          echo "=== Disk before cleanup ==="
+          df -h /
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android \
+            /opt/ghc /usr/local/share/boost /opt/hostedtoolcache \
+            /usr/local/graalvm /usr/local/.ghcup \
+            /usr/local/share/chromium /usr/local/lib/node_modules
+          sudo docker image prune -af
+          sudo apt-get clean
+          echo "=== Disk after cleanup ==="
+          df -h /
 
       - name: Clone
         uses: actions/checkout@v4
@@ -274,6 +274,67 @@ jobs:
             echo "Coverage cache: none available — full test suite will run"
           fi
 
+      # ── NVHPC: pull image and start a long-lived container ──────────────
+      # Replaces the container: directive so we can free disk space first.
+      # Uses "docker run -d ... sleep infinity" + "docker exec" to preserve
+      # installed packages and env vars across steps.
+      - name: Pull NVHPC container
+        if:   matrix.nvhpc
+        run:  docker pull "$NVHPC_IMAGE"
+
+      - name: Start NVHPC container
+        if:   matrix.nvhpc
+        run: |
+          docker run -d --name nvhpc \
+            --security-opt seccomp=unconfined \
+            -v "${{ github.workspace }}:/workspace" \
+            -w /workspace \
+            -e CC=nvc \
+            -e CXX=nvc++ \
+            -e FC=nvfortran \
+            -e OMPI_ALLOW_RUN_AS_ROOT=1 \
+            -e OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \
+            -e PMIX_MCA_gds=hash \
+            -e OMPI_MCA_hwloc_base_binding_policy=none \
+            -e "FFLAGS=-tp=px -Kieee -noswitcherror" \
+            -e CFLAGS=-tp=px \
+            -e CXXFLAGS=-tp=px \
+            "$NVHPC_IMAGE" sleep infinity
+
+      - name: Setup NVHPC
+        if:   matrix.nvhpc
+        run: |
+          docker exec nvhpc bash -c '
+            set -e
+            apt-get update -y
+            apt-get install -y cmake python3 python3-venv python3-pip \
+                               libfftw3-dev libhdf5-dev hdf5-tools git
+
+            # The repo is bind-mounted from the host so git sees a different
+            # owner.  Mark it safe to suppress "dubious ownership" errors that
+            # otherwise spam 80 000+ lines into the CI log.
+            git config --global --add safe.directory /workspace
+
+            # Set up NVHPC HPC-X MPI runtime paths
+            HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../..
+            MPI_LIB=$(mpifort --showme:link | grep -oP "(?<=-L)\S+" | head -1)
+
+            # Persist env vars for subsequent docker exec calls
+            cat > /etc/nvhpc-env.sh <<EOF
+          export LD_LIBRARY_PATH=${MPI_LIB}:${HPCX_DIR}/ucx/lib:${HPCX_DIR}/ucc/lib:\$LD_LIBRARY_PATH
+          export OMPI_MCA_rmaps_base_oversubscribe=1
+          EOF
+
+            # Debug: confirm compiler flags are set
+            echo "=== NVHPC Environment ==="
+            echo "FFLAGS=$FFLAGS"
+            echo "CFLAGS=$CFLAGS"
+            echo "CXXFLAGS=$CXXFLAGS"
+            nvfortran --version
+            cat /proc/cpuinfo | grep "model name" | head -1
+          '
+
+      # ── Standard (non-NVHPC) setup ─────────────────────────────────────
       - name: Setup MacOS
         if:   matrix.os == 'macos' && !matrix.nvhpc
         run:  |
@@ -313,30 +374,7 @@ jobs:
           echo "MPICC=mpiicx" >> $GITHUB_ENV
           echo "MPICXX=mpiicpx" >> $GITHUB_ENV
 
-      # --- NVHPC container setup ---
-      - name: Setup NVHPC
-        if:   matrix.nvhpc
-        run: |
-          apt-get update -y
-          apt-get install -y cmake python3 python3-venv python3-pip \
-                             libfftw3-dev libhdf5-dev hdf5-tools git
-          # Set up NVHPC HPC-X MPI runtime paths
-          HPCX_DIR=$(dirname "$(find /opt/nvidia/hpc_sdk -path "*/hpcx/hpcx-*/ompi/bin/mpirun" | head -1)")/../..
-          MPI_LIB=$(mpifort --showme:link | grep -oP '(?<=-L)\S+' | head -1)
-          echo "LD_LIBRARY_PATH=${MPI_LIB}:${HPCX_DIR}/ucx/lib:${HPCX_DIR}/ucc/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-          # Container MPI fixes: PMIx shared-memory, hwloc binding
-          echo "PMIX_MCA_gds=hash" >> $GITHUB_ENV
-          echo "OMPI_MCA_hwloc_base_binding_policy=none" >> $GITHUB_ENV
-          echo "OMPI_MCA_rmaps_base_oversubscribe=1" >> $GITHUB_ENV
-          # Debug: confirm compiler flags are set
-          echo "=== NVHPC Environment ==="
-          echo "FFLAGS=$FFLAGS"
-          echo "CFLAGS=$CFLAGS"
-          echo "CXXFLAGS=$CXXFLAGS"
-          nvfortran --version
-          cat /proc/cpuinfo | grep "model name" | head -1
-
-      # --- Standard build + test ---
+      # ── Standard build + test ───────────────────────────────────────────
       - name: Build
         if:   '!matrix.nvhpc'
         run:  |
@@ -354,22 +392,37 @@ jobs:
           TEST_PCT: ${{ matrix.debug == 'reldebug' && '-% 20' || '' }}
           ONLY_CHANGES: ${{ github.event_name == 'pull_request' && '--only-changes' || '' }}
 
-      # --- NVHPC build + test ---
+      # ── NVHPC build + test (via docker exec into long-lived container) ──
       - name: Build (NVHPC)
         if:   matrix.nvhpc && matrix.target == 'cpu'
-        run:  /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all
+        run: |
+          docker exec nvhpc bash -c '
+            source /etc/nvhpc-env.sh
+            /bin/bash mfc.sh test -v --dry-run -j $(nproc) --test-all
+          '
 
       - name: Build (NVHPC GPU)
         if:   matrix.nvhpc && matrix.target == 'gpu'
-        run:  |
-          /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu acc
-          /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp
+        run: |
+          docker exec nvhpc bash -c '
+            source /etc/nvhpc-env.sh
+            /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu acc
+            /bin/bash mfc.sh test -v --dry-run -j 2 --test-all --gpu mp
+          '
 
       - name: Test (NVHPC)
         if:   matrix.nvhpc && matrix.target == 'cpu'
-        run:  |
-          ulimit -s unlimited || ulimit -s 65536 || true
-          /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all
+        run: |
+          docker exec nvhpc bash -c '
+            source /etc/nvhpc-env.sh
+            ulimit -s unlimited || ulimit -s 65536 || true
+            /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all
+          '
+
+      # ── Cleanup ─────────────────────────────────────────────────────────
+      - name: Stop NVHPC container
+        if:   always() && matrix.nvhpc
+        run:  docker rm -f nvhpc || true
 
   self:
     name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }}${{ matrix.shard != '' && format(' [{0}]', matrix.shard) || '' }})"
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -10,7 +10,7 @@
 cmake_minimum_required(VERSION 3.20)
 
 
-# We include C as a language because - for some reason -
+# We include C as a language because - for some reason
 # FIND_LIBRARY_USE_LIB64_PATHS is otherwise ignored.
 
 project(MFC LANGUAGES C CXX Fortran)
@@ -491,7 +491,32 @@ function(MFC_SETUP_TARGET)
     # Here we need to split into "library" and "executable" to perform IPO on the NVIDIA compiler.
     # A little hacky, but it *is* an edge-case for *one* compiler.
     if (NVHPC_USE_TWO_PASS_IPO AND NOT(MFC_OpenMP AND ARGS_OpenMP))
+        # nvfortran -Mextract does not produce .o files, only inline library
+        # data. An OBJECT library with -Mextract causes CMake to rebuild
+        # everything on every build because the expected .o outputs never
+        # exist. We use a wrapper script as RULE_LAUNCH_COMPILE that runs
+        # the compiler and then touches the expected .o output file.
+        set(_ipo_wrapper "${CMAKE_BINARY_DIR}/${ARGS_TARGET}_extract_wrapper.sh")
+        file(WRITE "${_ipo_wrapper}" [=[#!/bin/sh
+# Find the -o argument (the object file CMake expects)
+out=
+prev=
+for arg do
+  if [ "$prev" = "-o" ]; then out="$arg"; break; fi
+  prev="$arg"
+done
+# Run the compiler; propagate its exit status on failure
+"$@"
+status=$?
+[ "$status" -eq 0 ] || exit "$status"
+# Touch the .o so CMake's dependency tracking sees it
+[ -n "$out" ] && touch "$out"
+exit 0
+]=])
+        file(CHMOD "${_ipo_wrapper}" PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
         add_library(${ARGS_TARGET}_lib OBJECT ${ARGS_SOURCES})
+        set_target_properties(${ARGS_TARGET}_lib PROPERTIES
+            RULE_LAUNCH_COMPILE "${_ipo_wrapper}")
         target_compile_options(${ARGS_TARGET}_lib PRIVATE
 		    $<$<COMPILE_LANGUAGE:Fortran>:-Mextract=lib:${ARGS_TARGET}_lib>
 	        $<$<COMPILE_LANGUAGE:Fortran>:-Minline>
