diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..93a5b1d90 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -107,6 +107,50 @@ The easiest way to debug a systemtest run is first to have a look at the output If this does not provide enough hints, the next step is to download the generated `system_tests_run__` artifact. Note that by default this will only be generated if the systemtests fail. Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. +### Re-running system tests from CI artifacts + +System test artifacts are produced by the GitHub Actions workflows as archives named +`system_tests_run__`. Each archive contains a `runs/` folder with one +subdirectory per executed system test. Every such subdirectory is self-contained and can be +re-run locally using Docker. + +To re-run a failing test locally: + +1. Download `system_tests_run__.zip` from the failed GitHub Actions run. +2. Extract the archive and change into the `runs/` directory: + + ```bash + unzip system_tests_run__.zip + cd system_tests_run__/runs + ls + # pick the test you are interested in + cd + ``` + +3. In that directory you will find: + + - a copy of the tutorial, + - a `tools/` folder with the required helper scripts and Dockerfiles, + - generated Docker Compose files (`docker-compose.tutorial.yaml`, + `docker-compose.field_compare.yaml`), and + - a helper script `rerun_systemtest.sh`. + +4. Re-run the test using Docker: + + ```bash + # if the script is executable + ./rerun_systemtest.sh + + # otherwise + sh rerun_systemtest.sh + ``` + +The script rebuilds the Docker images, re-runs the tutorial containers, and (if present) +executes the field comparison step. All paths used inside the generated Docker Compose +files are relative to the system test directory, so you can move the extracted `runs/` +folder anywhere on a Linux machine with Docker installed and still re-run the tests +in the same way as on the CI runner. + ## Adding new tests ### Adding tutorials diff --git a/tools/tests/components.yaml b/tools/tests/components.yaml index 35f5d1739..2f76a5120 100644 --- a/tools/tests/components.yaml +++ b/tools/tests/components.yaml @@ -13,7 +13,7 @@ bare: # A default component used when the solver does not have any dependencies default: "production-audit" TUTORIALS_REF: description: Tutorial git reference to use - default: "master" + default: "develop" python-bindings: repository: https://github.com/precice/python-bindings @@ -30,7 +30,7 @@ python-bindings: default: "production-audit" TUTORIALS_REF: description: Tutorial git reference to use - default: "master" + default: "develop" PYTHON_BINDINGS_REF: semnantic: Git ref of the Python bindings to use default: "master" @@ -50,7 +50,7 @@ openfoam-adapter: default: "production-audit" TUTORIALS_REF: description: Tutorial git reference to use - default: "master" + default: "develop" OPENFOAM_EXECUTABLE: description: exectuable of openfoam to use default: "openfoam2512" @@ -73,7 +73,7 @@ fenics-adapter: default: "production-audit" TUTORIALS_REF: description: Tutorial git reference to use - default: "master" + default: "develop" PYTHON_BINDINGS_REF: semnantic: Git ref of the Python bindings to use default: "master" @@ -96,7 +96,7 @@ nutils-adapter: default: "production-audit" TUTORIALS_REF: description: Tutorial git reference to use - default: "master" + default: "develop" PYTHON_BINDINGS_REF: semnantic: Git ref of the Python bindings to use default: "master" @@ -116,7 +116,7 @@ calculix-adapter: default: "production-audit" TUTORIALS_REF: description: Tutorial git reference to use - default: "master" + default: "develop" CALCULIX_VERSION: description: Version of Calculix to use default: "2.20" @@ -139,7 +139,7 @@ su2-adapter: default: "production-audit" TUTORIALS_REF: description: Tutorial git reference to use - default: "master" + default: "develop" SU2_VERSION: description: Version of SU2 to use default: "7.5.1" @@ -162,7 +162,7 @@ dealii-adapter: default: "production-audit" TUTORIALS_REF: description: Tutorial git reference to use - default: "master" + default: "develop" DEALII_ADAPTER_REF: description: Version of deal.ii-adapter to use default: "master" \ No newline at end of file diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index bfb1151cf..2c1cf3e8e 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -187,20 +187,37 @@ def __get_docker_services(self) -> Dict[str, str]: except Exception as exc: raise KeyError("Please specify a PLATFORM argument") from exc + # Use an absolute path here only for validation that the requested + # dockerfile context exists on the machine running the system tests. self.dockerfile_context = PRECICE_TESTS_DIR / "dockerfiles" / Path(plaform_requested) if not self.dockerfile_context.exists(): raise ValueError( f"The path {self.dockerfile_context.resolve()} resulting from argument PLATFORM={plaform_requested} could not be found in the system") def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) -> str: + # Inside the individual system test directory (`self.system_test_dir`) + # we copy a full `tools/` tree into the parent run directory + # (see __copy_tools). From the point of view of the system test + # directory we therefore need to go one level up to reach the + # shared `tools/` folder: + # /tools/tests/dockerfiles/ + # ^-------------^ parent of self.system_test_dir + dockerfile_context_relative = ( + Path("..") / "tools" / "tests" / "dockerfiles" / Path(plaform_requested) + ) + render_dict = { - 'run_directory': self.run_directory.resolve(), + # Use a relative path to the *parent* run directory so that + # containers still see /runs/ like before, + # while keeping the compose file independent of the CI + # runner's absolute paths. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'build_arguments': params_to_use, 'params': params_to_use, 'case_folder': case.path, 'run': case.run_cmd, - 'dockerfile_context': self.dockerfile_context, + 'dockerfile_context': dockerfile_context_relative, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template(case.component.template) @@ -215,12 +232,20 @@ def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) def __get_docker_compose_file(self): rendered_services = self.__get_docker_services() render_dict = { - 'run_directory': self.run_directory.resolve(), + # See __get_docker_services: keep the docker-compose file + # portable by referring to the parent run directory only. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'tutorial': self.tutorial.path.name, 'services': rendered_services, 'build_arguments': self.params_to_use, - 'dockerfile_context': self.dockerfile_context, + # The dockerfile_context value inside the templates is only + # used as a build context path and does not need to be + # absolute – it will be resolved relative to the system test + # directory. + 'dockerfile_context': ( + Path("..") / "tools" / "tests" / "dockerfiles" / Path(self.params_to_use.get("PLATFORM")) + ), 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) @@ -229,7 +254,10 @@ def __get_docker_compose_file(self): def __get_field_compare_compose_file(self): render_dict = { - 'run_directory': self.run_directory.resolve(), + # Fieldcompare should also use only relative paths from inside + # the system test directory so that the run directory can be + # moved and re-executed elsewhere. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, 'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""), @@ -420,9 +448,42 @@ def _build_docker(self): logging.debug(f"Building docker image for {self}") time_start = time.perf_counter() docker_compose_content = self.__get_docker_compose_file() - with open(self.system_test_dir / "docker-compose.tutorial.yaml", 'w') as file: + docker_compose_path = self.system_test_dir / "docker-compose.tutorial.yaml" + with open(docker_compose_path, 'w') as file: file.write(docker_compose_content) + # Provide a small helper script inside the system test directory so + # that a user downloading the corresponding `runs/` artifact can + # re-run the exact docker-compose setup locally without having to + # reconstruct the commands by hand. + rerun_script_path = self.system_test_dir / "rerun_systemtest.sh" + if not rerun_script_path.exists(): + rerun_script_path.write_text( + "#!/usr/bin/env sh\n" + "set -e -u\n" + "\n" + "cd \"$(dirname \"$0\")\"\n" + "\n" + "echo \"[systemtests] Building tutorial images...\"\n" + "docker compose --file docker-compose.tutorial.yaml build\n" + "\n" + "echo \"[systemtests] Running tutorial containers...\"\n" + "docker compose --file docker-compose.tutorial.yaml up\n" + "\n" + "if [ -f docker-compose.field_compare.yaml ]; then\n" + " echo \"[systemtests] Running fieldcompare...\"\n" + " docker compose --file docker-compose.field_compare.yaml up --exit-code-from field-compare\n" + "fi\n" + ) + # Make the script executable for convenience; even if this bit + # does not survive archiving, users can still run it via + # `sh rerun_systemtest.sh`. + try: + rerun_script_path.chmod(rerun_script_path.stat().st_mode | 0o111) + except Exception: + logging.debug( + f"Could not mark {rerun_script_path} as executable; continuing anyway.") + stdout_data = [] stderr_data = []