Add --suites filter to generate_reference_results

.github/workflows/generate_reference_results_manual.yml

@@ -10,6 +10,11 @@ on:
         description: 'Commit msg for commit that adds the reference results'
         default: "Adding reference results"
         type: string
+      suites:
+        description: 'Comma-separated test suites to generate reference results for (leave empty for all)'
+        default: ''
+        required: false
+        type: string
       log_level:
         description: 'Logging verbosity level used for the systemtests'
         default: 'INFO'
@@ -28,4 +33,5 @@ jobs:
     with: 
       from_ref: ${{ inputs.from_ref }}
       commit_msg: ${{ inputs.commit_msg }}
+      suites: ${{ inputs.suites }}
       log_level: ${{ inputs.log_level }}

.github/workflows/generate_reference_results_workflow.yml

@@ -10,6 +10,11 @@ on:
         description: 'Commit msg for commit that adds the reference results'
         default: "Adding reference results"
         type: string
+      suites:
+        description: 'Comma-separated test suites to generate reference results for. If empty, all suites are generated.'
+        default: ''
+        required: false
+        type: string
       log_level:
         description: 'Logging verbosity level used for the systemtests'
         default: 'INFO'
@@ -24,6 +29,7 @@ jobs:
         echo "Initiated by: ${{ github.actor }}"
         echo "Running generate_reference_results.py --log-level ${{inputs.log_level}}"
         echo "Using Ref: ${{ inputs.from_ref }}"
+        echo "Suites filter: ${{ inputs.suites || 'all (no filter)' }}"
         echo "Commit message on success:  ${{ inputs.commit_msg }}"
     - name: Move LFS URL to local LFS server
       run: |
@@ -56,15 +62,26 @@ jobs:
         test -f generate_reference_results.py && export GENERATE_REF_RESULTS=generate_reference_results.py
         test -f generate_reference_data.py && export GENERATE_REF_RESULTS=generate_reference_data.py
         echo "Selected $GENERATE_REF_RESULTS to run"
-        python $GENERATE_REF_RESULTS --log-level=${{inputs.log_level}}
+        SUITES_ARG=""
+        if [ -n "${{ inputs.suites }}" ]; then
+          SUITES_ARG="--suites=${{ inputs.suites }}"
+        fi
+        python $GENERATE_REF_RESULTS --log-level=${{inputs.log_level}} $SUITES_ARG
         cd ../../
     - name: Create commit
       if: success()
       run: |
         git checkout ${{ inputs.from_ref }}
-        git add ./*/*/*.tar.gz
-        git add ./*/*.tar.gz
-        git add ./*/reference-results/*.metadata
+        # Add only the changed reference results.
+        # When no suite filter is given, add all reference result files.
+        # When a suite filter is given, only add files that were actually modified.
+        if [ -z "${{ inputs.suites }}" ]; then
+          git add ./*/*/*.tar.gz
+          git add ./*/*.tar.gz
+          git add ./*/reference-results/*.metadata
+        else
+          git add --update '*.tar.gz' '*.metadata'
+        fi
         git commit -m "${{inputs.commit_msg}}"
         git push
     - name: Upload artifacts for debugging

changelog-entries/711.md

@@ -0,0 +1 @@
+- Added optional `--suites` filter to `generate_reference_results.py` and the corresponding workflow, allowing selective regeneration of reference results for specific test suites instead of always regenerating all of them. https://github.com/precice/tutorials/pull/711

tools/tests/generate_reference_results.py

@@ -86,6 +86,9 @@ def main():
     parser = argparse.ArgumentParser(description='Generate reference data for systemtests')
     parser.add_argument('--rundir', type=str, help='Directory to run the systemstests in.',
                         nargs='?', const=PRECICE_TESTS_RUN_DIR, default=PRECICE_TESTS_RUN_DIR)
+    parser.add_argument('--suites', type=str,
+                        help='Comma-separated test suites to generate reference results for. '
+                             'If not specified, all suites are used.')
     parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
                         default='INFO', help='Set the logging level')
 
@@ -99,7 +102,25 @@ def main():
 
     available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR)
 
-    test_suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", available_tutorials)
+    all_test_suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", available_tutorials)
+
+    if args.suites:
+        test_suites_requested = args.suites.split(',')
+        test_suites = []
+        for name in test_suites_requested:
+            found = all_test_suites.get_by_name(name)
+            if not found:
+                logging.error(f"Did not find the testsuite with name {name}")
+            else:
+                test_suites.append(found)
+        if not test_suites:
+            raise RuntimeError(
+                f"No matching test suites with names {test_suites_requested} found. "
+                "Use print_test_suites.py to get an overview")
+        logging.info(f"Filtering to requested suites: {[s.name for s in test_suites]}")
+    else:
+        test_suites = all_test_suites
+        logging.info("No --suites filter specified, generating reference results for all suites.")
 
     # Read in parameters
     build_args = SystemtestArguments.from_yaml(PRECICE_TESTS_DIR / "reference_versions.yaml")