From 47e30d5e5667f0a6cc533d43a7ebde0cc190b6b0 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 19 Jan 2026 11:54:11 +0000 Subject: [PATCH 1/3] ci: Cache Zenodo data, and retry on http 502, 503 errors Also: - reduce max number of tries to avoid hammering Zenodo - update moved URL --- .github/workflows/tests.yml | 23 ++++++++++++++++++++ tests/integrated/test-fci-mpi/CMakeLists.txt | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 314ee196ae..3124a11762 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -181,6 +181,29 @@ jobs: python -m pip install --upgrade pip setuptools python -m pip install -r requirements.txt + - name: Cache Zenodo test data + uses: actions/cache@v4 + with: + path: ./zenodo_data + # Data doesn't depend on matrix job, so this should cache between all jobs + key: zenodo-data + + - name: Download Zenodo test data + run: | + if [[ ! -e zenodo_data/grid.fci.nc ]]; then + echo "Downloading Zenodo data" + mkdir -p zenodo_data + wget \ + https://zenodo.org/records/7614499/files/W7X-conf4-36x8x128.fci.nc?download=1 \ + --output-document=zenodo_data/grid.fci.nc \ + --retry-on-http-error=502,503,504 \ + --tries 3 + else + echo "Using cached Zenodo data" + fi + mkdir -pv build/tests/integrated/test-fci-mpi + cp -v zenodo_data/grid.fci.nc build/tests/integrated/test-fci-mpi + - name: Cache SUNDIALS build uses: actions/cache@v4 with: diff --git a/tests/integrated/test-fci-mpi/CMakeLists.txt b/tests/integrated/test-fci-mpi/CMakeLists.txt index 0dd38487a3..aea0421c0a 100644 --- a/tests/integrated/test-fci-mpi/CMakeLists.txt +++ b/tests/integrated/test-fci-mpi/CMakeLists.txt @@ -3,7 +3,7 @@ bout_add_mms_test(test-fci-mpi USE_RUNTEST USE_DATA_BOUT_INP PROCESSORS 6 - DOWNLOAD https://zenodo.org/record/7614499/files/W7X-conf4-36x8x128.fci.nc?download=1 + DOWNLOAD https://zenodo.org/records/7614499/files/W7X-conf4-36x8x128.fci.nc?download=1 DOWNLOAD_NAME grid.fci.nc REQUIRES BOUT_HAS_PETSC ) From 51d2ceb79105055d84ac4f60d1636b0fe073f53d Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 19 Jan 2026 13:37:47 +0000 Subject: [PATCH 2/3] ci: Use wget flags from env var This avoids duplicating the download in all jobs --- .github/workflows/tests.yml | 44 ++++++++++++++++++++----------------- cmake/BOUT++functions.cmake | 2 +- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3124a11762..e50154e615 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,6 +30,7 @@ jobs: OMPI_MCA_rmaps_base_oversubscribe: yes PRTE_MCA_rmaps_default_mapping_policy: ":oversubscribe" MPIRUN: mpiexec -np + BOUT_TEST_DOWNLOAD_FLAGS: --retry-on-http-error=502,503,504 --tries 3 strategy: fail-fast: true matrix: @@ -184,25 +185,9 @@ jobs: - name: Cache Zenodo test data uses: actions/cache@v4 with: - path: ./zenodo_data - # Data doesn't depend on matrix job, so this should cache between all jobs - key: zenodo-data - - - name: Download Zenodo test data - run: | - if [[ ! -e zenodo_data/grid.fci.nc ]]; then - echo "Downloading Zenodo data" - mkdir -p zenodo_data - wget \ - https://zenodo.org/records/7614499/files/W7X-conf4-36x8x128.fci.nc?download=1 \ - --output-document=zenodo_data/grid.fci.nc \ - --retry-on-http-error=502,503,504 \ - --tries 3 - else - echo "Using cached Zenodo data" - fi - mkdir -pv build/tests/integrated/test-fci-mpi - cp -v zenodo_data/grid.fci.nc build/tests/integrated/test-fci-mpi + path: build/tests/integrated/test-fci-mpi/grid.fci.nc + # If we update the test, invalidate the cache + key: zenodo-data-${{ hashFiles('tests/integrated/test-fci-mpi/CMakeLists.txt') }} - name: Cache SUNDIALS build uses: actions/cache@v4 @@ -227,10 +212,20 @@ jobs: # standard_tests timeout-minutes: 120 runs-on: ubuntu-latest + env: + BOUT_TEST_DOWNLOAD_FLAGS: --retry-on-http-error=502,503,504 --tries 3 steps: - uses: actions/checkout@v5 with: submodules: true + + - name: Cache Zenodo test data + uses: actions/cache@v4 + with: + path: build/tests/integrated/test-fci-mpi/grid.fci.nc + # If we update the test, invalidate the cache + key: zenodo-data-${{ hashFiles('tests/integrated/test-fci-mpi/CMakeLists.txt') }} + - name: Build Fedora run: ./.ci_fedora.sh setup openmpi shell: bash @@ -240,11 +235,20 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-latest container: ghcr.io/ggeorgakoudis/boutdev-cuda:latest - + env: + BOUT_TEST_DOWNLOAD_FLAGS: --retry-on-http-error=502,503,504 --tries 3 steps: - uses: actions/checkout@v5 with: submodules: true + + - name: Cache Zenodo test data + uses: actions/cache@v4 + with: + path: build/tests/integrated/test-fci-mpi/grid.fci.nc + # If we update the test, invalidate the cache + key: zenodo-data-${{ hashFiles('tests/integrated/test-fci-mpi/CMakeLists.txt') }} + - name: Build minimal CUDA 12.2 @ GCC9.4.0 @ Ubuntu 20.04 run: | . /spack/share/spack/setup-env.sh diff --git a/cmake/BOUT++functions.cmake b/cmake/BOUT++functions.cmake index 77279dfd4b..074d803fb6 100644 --- a/cmake/BOUT++functions.cmake +++ b/cmake/BOUT++functions.cmake @@ -208,7 +208,7 @@ function(bout_add_integrated_or_mms_test BUILD_CHECK_TARGET TESTNAME) endif() set(output ) add_custom_command(OUTPUT ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} - COMMAND wget ${BOUT_TEST_OPTIONS_DOWNLOAD} -O ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} + COMMAND wget ${BOUT_TEST_OPTIONS_DOWNLOAD} -O ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} $ENV{BOUT_TEST_DOWNLOAD_FLAGS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Downloading ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME}" ) From bbaeeaf05ea80488d295313eca12ed3dbd22a51f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 19 Jan 2026 13:39:44 +0000 Subject: [PATCH 3/3] ci: Cache Python packages --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e50154e615..47c053744f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -176,6 +176,7 @@ jobs: - uses: actions/setup-python@v6 with: python-version: '3.x' + cache: 'pip' - name: Install pip packages run: |