Merge pull request #3254 from boutproject/cache-zenodo

ci: Cache Zenodo data, and retry on http 502, 503 errors

Author: ZedThree

.github/workflows/tests.yml

@@ -30,6 +30,7 @@ jobs:
       OMPI_MCA_rmaps_base_oversubscribe: yes
       PRTE_MCA_rmaps_default_mapping_policy: ":oversubscribe"
       MPIRUN: mpiexec -np
+      BOUT_TEST_DOWNLOAD_FLAGS: --retry-on-http-error=502,503,504 --tries 3
     strategy:
       fail-fast: true
       matrix:
@@ -175,12 +176,20 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: '3.x'
+          cache: 'pip'
 
       - name: Install pip packages
         run: |
           python -m pip install --upgrade pip setuptools
           python -m pip install -r requirements.txt
 
+      - name: Cache Zenodo test data
+        uses: actions/cache@v4
+        with:
+          path: build/tests/integrated/test-fci-mpi/grid.fci.nc
+          # If we update the test, invalidate the cache
+          key: zenodo-data-${{ hashFiles('tests/integrated/test-fci-mpi/CMakeLists.txt') }}
+
       - name: Cache SUNDIALS build
         uses: actions/cache@v5
         with:
@@ -204,10 +213,20 @@ jobs:
     # standard_tests
     timeout-minutes: 120
     runs-on: ubuntu-latest
+    env:
+      BOUT_TEST_DOWNLOAD_FLAGS: --retry-on-http-error=502,503,504 --tries 3
     steps:
       - uses: actions/checkout@v6
         with:
           submodules: true
+
+      - name: Cache Zenodo test data
+        uses: actions/cache@v4
+        with:
+          path: build/tests/integrated/test-fci-mpi/grid.fci.nc
+          # If we update the test, invalidate the cache
+          key: zenodo-data-${{ hashFiles('tests/integrated/test-fci-mpi/CMakeLists.txt') }}
+
       - name: Build Fedora
         run: ./.ci_fedora.sh setup openmpi
         shell: bash
@@ -217,11 +236,20 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     container: ghcr.io/ggeorgakoudis/boutdev-cuda:latest
-
+    env:
+      BOUT_TEST_DOWNLOAD_FLAGS: --retry-on-http-error=502,503,504 --tries 3
     steps:
       - uses: actions/checkout@v6
         with:
           submodules: true
+
+      - name: Cache Zenodo test data
+        uses: actions/cache@v4
+        with:
+          path: build/tests/integrated/test-fci-mpi/grid.fci.nc
+          # If we update the test, invalidate the cache
+          key: zenodo-data-${{ hashFiles('tests/integrated/test-fci-mpi/CMakeLists.txt') }}
+
       - name: Build minimal CUDA 12.2 @ GCC9.4.0 @ Ubuntu 20.04
         run: |
           . /spack/share/spack/setup-env.sh

cmake/BOUT++functions.cmake

@@ -208,7 +208,7 @@ function(bout_add_integrated_or_mms_test BUILD_CHECK_TARGET TESTNAME)
     endif()
     set(output )
     add_custom_command(OUTPUT ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME}
-      COMMAND wget ${BOUT_TEST_OPTIONS_DOWNLOAD} -O ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME}
+      COMMAND wget ${BOUT_TEST_OPTIONS_DOWNLOAD} -O ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME} $ENV{BOUT_TEST_DOWNLOAD_FLAGS}
       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
       COMMENT "Downloading ${BOUT_TEST_OPTIONS_DOWNLOAD_NAME}"
       )

tests/integrated/test-fci-mpi/CMakeLists.txt

@@ -3,7 +3,7 @@ bout_add_mms_test(test-fci-mpi
   USE_RUNTEST
   USE_DATA_BOUT_INP
   PROCESSORS 6
-  DOWNLOAD https://zenodo.org/record/7614499/files/W7X-conf4-36x8x128.fci.nc?download=1
+  DOWNLOAD https://zenodo.org/records/7614499/files/W7X-conf4-36x8x128.fci.nc?download=1
   DOWNLOAD_NAME grid.fci.nc
   REQUIRES BOUT_HAS_PETSC
 )