Systemtest.py

import subprocess
from typing import List, Dict, Optional
from jinja2 import Environment, FileSystemLoader
from dataclasses import dataclass, field
import shutil
from pathlib import Path
from paths import PRECICE_REL_OUTPUT_DIR, PRECICE_TOOLS_DIR, PRECICE_REL_REFERENCE_DIR, PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR

from metadata_parser.metdata import Tutorial, CaseCombination, Case, ReferenceResult
from .SystemtestArguments import SystemtestArguments

from datetime import datetime
import tarfile
import time

import unicodedata
import re
import logging
import os


GLOBAL_TIMEOUT = 600
SHORT_TIMEOUT = 10


def slugify(value, allow_unicode=False):
    """
    Taken from https://github.com/django/django/blob/master/django/utils/text.py
    Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
    dashes to single dashes. Remove characters that aren't alphanumerics,
    underscores, or hyphens. Convert to lowercase. Also strip leading and
    trailing whitespace, dashes, and underscores.
    """
    value = str(value)
    if allow_unicode:
        value = unicodedata.normalize('NFKC', value)
    else:
        value = unicodedata.normalize('NFKD', value).encode(
            'ascii', 'ignore').decode('ascii')
    value = re.sub(r'[^\w\s-]', '', value.lower())
    return re.sub(r'[-\s]+', '-', value).strip('-_')


class Systemtest:
    pass


@dataclass
class DockerComposeResult:
    exit_code: int
    stdout_data: List[str]
    stderr_data: List[str]
    systemtest: Systemtest
    runtime: float  # in seconds


@dataclass
class FieldCompareResult:
    exit_code: int
    stdout_data: List[str]
    stderr_data: List[str]
    systemtest: Systemtest
    runtime: float  # in seconds


@dataclass
class SystemtestResult:
    success: bool
    stdout_data: List[str]
    stderr_data: List[str]
    systemtest: Systemtest
    build_time: float  # in seconds
    solver_time: float  # in seconds
    fieldcompare_time: float  # in seconds


def display_systemtestresults_as_table(results: List[SystemtestResult]):
    """
    Prints the result in a nice tabluated way to get an easy overview
    """
    def _get_length_of_name(results: List[SystemtestResult]) -> int:
        return max(len(str(result.systemtest)) for result in results)

    max_name_length = _get_length_of_name(results)

    header = f"| {'systemtest':<{max_name_length + 2}} "\
        f"| {'success':^7} "\
        f"| {'building time [s]':^17} "\
        f"| {'solver time [s]':^15} "\
        f"| {'fieldcompare time [s]':^21} |"
    separator_plaintext = "+-" + "-" * (max_name_length + 2) + \
        "-+---------+-------------------+-----------------+-----------------------+"
    separator_markdown = "| --- | --- | --- | --- | --- |"

    print(separator_plaintext)
    print(header)
    print(separator_plaintext)

    if "GITHUB_STEP_SUMMARY" in os.environ:
        with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
            print(header, file=f)
            print(separator_markdown, file=f)

    for result in results:
        row = f"| {str(result.systemtest):<{max_name_length + 2}} "\
            f"| {result.success:^7} "\
            f"| {result.build_time:^17.1f} "\
            f"| {result.solver_time:^15.1f} "\
            f"| {result.fieldcompare_time:^21.1f} |"
        print(row)
        print(separator_plaintext)
        if "GITHUB_STEP_SUMMARY" in os.environ:
            with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
                print(row, file=f)

    if "GITHUB_STEP_SUMMARY" in os.environ:
        with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
            print("\n\n", file=f)
            print(
                "In case a test fails, download the archive from the bottom of this page and look into each `stdout.log` and `stderr.log`. The time spent in each step might already give useful hints.",
                file=f)
            print(
                "See the [documentation](https://precice.org/dev-docs-system-tests.html#understanding-what-went-wrong).",
                file=f)


@dataclass
class Systemtest:
    """
    Represents a system test by specifing the cases and the corresponding Tutorial
    """

    tutorial: Tutorial
    arguments: SystemtestArguments
    case_combination: CaseCombination
    reference_result: ReferenceResult
    params_to_use: Dict[str, str] = field(init=False)
    env: Dict[str, str] = field(init=False)

    def __eq__(self, other) -> bool:
        if isinstance(other, Systemtest):
            return (
                self.tutorial == other.tutorial) and (
                self.arguments == other.arguments) and (
                self.case_combination == other.case_combination)
        return False

    def __hash__(self) -> int:
        return hash(f"{self.tutorial, self.arguments, self.case_combination}")

    def __post_init__(self):
        self.__init_args_to_use()
        self.env = {}

    def __init_args_to_use(self):
        """
        Forwards the command-line arguments to the params_to_use dictionary, substituting any missing arguments with their defaults.

        Previously, this function was also checking if all required parameters were provided, and was raising exceptions for parameters not provided and not having a default value. This check made adding optional parameters with empty defaults (e.g., the TUTORIALS_PR) complicated, and it was removed.
        """

        # Forward all provided arguments to params_to_use
        provided_arguments = self.arguments.arguments
        self.params_to_use = provided_arguments

        # Find out which parameters are needed
        needed_parameters = set()
        for case in self.case_combination.cases:
            needed_parameters.update(case.component.parameters)

        # Substitute defaults for non-provided, needed arguments
        for needed_param in needed_parameters:
            if not needed_param.key in provided_arguments:
                logging.warning(
                    f"No argument provided for needed parameter {needed_param.key}. Substituting with {needed_param.default}")
                self.params_to_use[needed_param.key] = needed_param.default

    def __get_docker_services(self) -> Dict[str, str]:
        """
        Renders the service templates for each case using the parameters to use.

        Returns:
            A dictionary of rendered services per case name.
        """
        try:
            plaform_requested = self.params_to_use.get("PLATFORM")
        except Exception as exc:
            raise KeyError("Please specify a PLATFORM argument") from exc

        # Use an absolute path here only for validation that the requested
        # dockerfile context exists on the machine running the system tests.
        self.dockerfile_context = PRECICE_TESTS_DIR / "dockerfiles" / Path(plaform_requested)
        if not self.dockerfile_context.exists():
            raise ValueError(
                f"The path {self.dockerfile_context.resolve()} resulting from argument PLATFORM={plaform_requested} could not be found in the system")

        def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) -> str:
            # Inside the individual system test directory (`self.system_test_dir`)
            # we copy a full `tools/` tree into the parent run directory
            # (see __copy_tools). From the point of view of the system test
            # directory we therefore need to go one level up to reach the
            # shared `tools/` folder:
            #   <run_directory>/tools/tests/dockerfiles/<PLATFORM>
            #   ^-------------^ parent of self.system_test_dir
            dockerfile_context_relative = (
                Path("..") / "tools" / "tests" / "dockerfiles" / Path(plaform_requested)
            )

            render_dict = {
                # Use a relative path to the *parent* run directory so that
                # containers still see /runs/<tutorial_folder> like before,
                # while keeping the compose file independent of the CI
                # runner's absolute paths.
                'run_directory': "..",
                'tutorial_folder': self.tutorial_folder,
                'build_arguments': params_to_use,
                'params': params_to_use,
                'case_folder': case.path,
                'run': case.run_cmd,
                'dockerfile_context': dockerfile_context_relative,
            }
            jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR))
            template = jinja_env.get_template(case.component.template)
            return template.render(render_dict)

        rendered_services = {}
        for case in self.case_combination.cases:
            rendered_services[case.name] = render_service_template_per_case(
                case, self.params_to_use)
        return rendered_services

    def __get_docker_compose_file(self):
        rendered_services = self.__get_docker_services()
        render_dict = {
            # See __get_docker_services: keep the docker-compose file
            # portable by referring to the parent run directory only.
            'run_directory': "..",
            'tutorial_folder': self.tutorial_folder,
            'tutorial': self.tutorial.path.name,
            'services': rendered_services,
            'build_arguments': self.params_to_use,
            # The dockerfile_context value inside the templates is only
            # used as a build context path and does not need to be
            # absolute – it will be resolved relative to the system test
            # directory.
            'dockerfile_context': (
                Path("..") / "tools" / "tests" / "dockerfiles" / Path(self.params_to_use.get("PLATFORM"))
            ),
            'precice_output_folder': PRECICE_REL_OUTPUT_DIR,
        }
        jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR))
        template = jinja_env.get_template("docker-compose.template.yaml")
        return template.render(render_dict)

    def __get_field_compare_compose_file(self):
        render_dict = {
            # Fieldcompare should also use only relative paths from inside
            # the system test directory so that the run directory can be
            # moved and re-executed elsewhere.
            'run_directory': "..",
            'tutorial_folder': self.tutorial_folder,
            'precice_output_folder': PRECICE_REL_OUTPUT_DIR,
            'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""),
        }
        jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR))
        template = jinja_env.get_template(
            "docker-compose.field_compare.template.yaml")
        return template.render(render_dict)

    def _get_git_ref(self, repository: Path, abbrev_ref=False) -> Optional[str]:
        try:
            result = subprocess.run([
                "git",
                "-C", os.fspath(repository.resolve()),
                "rev-parse",
                "--abbrev-ref" if abbrev_ref else
                "HEAD"], stdout=subprocess.PIPE,
                stderr=subprocess.PIPE, text=True, check=True, timeout=60)
            current_ref = result.stdout.strip()
            return current_ref
        except Exception as e:
            raise RuntimeError(f"An error occurred while getting the current Git ref: {e}") from e

    def _fetch_pr(self, repository: Path, pr: str):
        try:
            result = subprocess.run([
                "git",
                "-C", os.fspath(repository.resolve()),
                "fetch",
                "origin",
                "pull/" + pr + "/head"
            ], check=True, timeout=60)

        except Exception as e:
            raise RuntimeError(f"git command returned code {result.returncode}")

    def _fetch_ref(self, repository: Path, ref: str):
        try:
            result = subprocess.run([
                "git",
                "-C", os.fspath(repository.resolve()),
                "fetch"
            ], check=True, timeout=60)
            if result.returncode != 0:
                raise RuntimeError(f"git command returned code {result.returncode}")

        except Exception as e:
            raise RuntimeError(f"An error occurred while fetching origin '{ref}':  {e}")

    def _checkout_ref_in_subfolder(self, repository: Path, subfolder: Path, ref: str):
        try:
            result = subprocess.run([
                "git",
                "-C", os.fspath(repository.resolve()),
                "checkout", ref,
                "--", os.fspath(subfolder.resolve())
            ], check=True, timeout=60)
            if result.returncode != 0:
                raise RuntimeError(f"git command returned code {result.returncode}")

        except Exception as e:
            raise RuntimeError(f"An error occurred while checking out '{ref}' for folder '{repository}': {e}")

    def __copy_tutorial_into_directory(self, run_directory: Path):
        """
        Checks out the requested tutorial ref and copies the entire tutorial into a folder to prepare for running.
        """
        current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        self.run_directory = run_directory
        pr_requested = self.params_to_use.get("TUTORIALS_PR")
        if pr_requested:
            logging.debug(f"Fetching the PR {pr_requested} HEAD reference")
            self._fetch_pr(PRECICE_TUTORIAL_DIR, pr_requested)
        current_ref = self._get_git_ref(PRECICE_TUTORIAL_DIR)
        ref_requested = self.params_to_use.get("TUTORIALS_REF")
        if ref_requested:
            logging.debug(f"Checking out tutorials {ref_requested} before copying")
            self._fetch_ref(PRECICE_TUTORIAL_DIR, ref_requested)
            self._checkout_ref_in_subfolder(PRECICE_TUTORIAL_DIR, self.tutorial.path, ref_requested)

        self.tutorial_folder = slugify(f'{self.tutorial.path.name}_{self.case_combination.cases}_{current_time_string}')
        destination = run_directory / self.tutorial_folder
        src = self.tutorial.path
        self.system_test_dir = destination
        shutil.copytree(src, destination)

        if ref_requested:
            with open(destination / "tutorials_ref", 'w') as file:
                file.write(ref_requested)
            self._checkout_ref_in_subfolder(PRECICE_TUTORIAL_DIR, self.tutorial.path, current_ref)

    def __copy_tools(self, run_directory: Path):
        destination = run_directory / "tools"
        src = PRECICE_TOOLS_DIR
        try:
            shutil.copytree(src, destination)
        except FileExistsError as e:
            logging.debug(f"Tools directory has already been copied to the workspace - skipping.")
        except Exception as e:
            logging.warning(f"Something went wrong while copying the tools directory to the workspace: {e}")

    def __put_gitignore(self, run_directory: Path):
        # Create the .gitignore file with a single asterisk
        gitignore_file = run_directory / ".gitignore"
        with gitignore_file.open("w") as file:
            file.write("*")

    def __cleanup(self):
        shutil.rmtree(self.run_directory)

    def __get_uid_gid(self):
        try:
            uid = int(subprocess.check_output(["id", "-u"]).strip())
            gid = int(subprocess.check_output(["id", "-g"]).strip())
            return uid, gid
        except Exception as e:
            logging.error("Error getting group and user id: ", e)

    def __write_env_file(self):
        with open(self.system_test_dir / ".env", "w") as env_file:
            for key, value in self.env.items():
                env_file.write(f"{key}={value}\n")

    def __unpack_reference_results(self):
        with tarfile.open(self.reference_result.path) as reference_results_tared:
            # specify which folder to extract to
            reference_results_tared.extractall(self.system_test_dir / PRECICE_REL_REFERENCE_DIR)
        logging.debug(
            f"extracting {self.reference_result.path} into {self.system_test_dir / PRECICE_REL_REFERENCE_DIR}")

    def _run_field_compare(self):
        """
        Writes the Docker Compose file to disk, executes docker-compose up, and handles the process output.

        Args:
            docker_compose_content: The content of the Docker Compose file.

        Returns:
            A SystemtestResult object containing the state.
        """
        logging.debug(f"Running fieldcompare for {self}")
        time_start = time.perf_counter()
        self.__unpack_reference_results()
        docker_compose_content = self.__get_field_compare_compose_file()
        stdout_data = []
        stderr_data = []

        with open(self.system_test_dir / "docker-compose.field_compare.yaml", 'w') as file:
            file.write(docker_compose_content)
        try:
            # Execute docker-compose command
            process = subprocess.Popen(['docker',
                                        'compose',
                                        '--file',
                                        'docker-compose.field_compare.yaml',
                                        'up',
                                        '--exit-code-from',
                                        'field-compare'],
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE,
                                       start_new_session=True,
                                       cwd=self.system_test_dir)

            try:
                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
            except KeyboardInterrupt as k:
                process.kill()
                raise KeyboardInterrupt from k
            except Exception as e:
                logging.critical(
                    f"Systemtest {self} had serious issues executing the docker compose command about to kill the docker compose command. Please check the logs! {e}")
                process.kill()
                process.communicate(timeout=SHORT_TIMEOUT)
            stdout_data.extend(stdout.decode().splitlines())
            stderr_data.extend(stderr.decode().splitlines())
            process.poll()
            elapsed_time = time.perf_counter() - time_start
            return FieldCompareResult(process.returncode, stdout_data, stderr_data, self, elapsed_time)
        except Exception as e:
            logging.CRITICAL("Error executing docker compose command:", e)
            elapsed_time = time.perf_counter() - time_start
            return FieldCompareResult(1, stdout_data, stderr_data, self, elapsed_time)

    def _build_docker(self):
        """
        Builds the docker image
        """
        logging.debug(f"Building docker image for {self}")
        time_start = time.perf_counter()
        docker_compose_content = self.__get_docker_compose_file()
        docker_compose_path = self.system_test_dir / "docker-compose.tutorial.yaml"
        with open(docker_compose_path, 'w') as file:
            file.write(docker_compose_content)

        # Provide a small helper script inside the system test directory so
        # that a user downloading the corresponding `runs/` artifact can
        # re-run the exact docker-compose setup locally without having to
        # reconstruct the commands by hand.
        rerun_script_path = self.system_test_dir / "rerun_systemtest.sh"
        if not rerun_script_path.exists():
            rerun_script_path.write_text(
                "#!/usr/bin/env sh\n"
                "set -e -u\n"
                "\n"
                "cd \"$(dirname \"$0\")\"\n"
                "\n"
                "echo \"[systemtests] Building tutorial images...\"\n"
                "docker compose --file docker-compose.tutorial.yaml build\n"
                "\n"
                "echo \"[systemtests] Running tutorial containers...\"\n"
                "docker compose --file docker-compose.tutorial.yaml up\n"
                "\n"
                "if [ -f docker-compose.field_compare.yaml ]; then\n"
                "  echo \"[systemtests] Running fieldcompare...\"\n"
                "  docker compose --file docker-compose.field_compare.yaml up --exit-code-from field-compare\n"
                "fi\n"
            )
            # Make the script executable for convenience; even if this bit
            # does not survive archiving, users can still run it via
            # `sh rerun_systemtest.sh`.
            try:
                rerun_script_path.chmod(rerun_script_path.stat().st_mode | 0o111)
            except Exception:
                logging.debug(
                    f"Could not mark {rerun_script_path} as executable; continuing anyway.")

        stdout_data = []
        stderr_data = []

        try:
            # Execute docker-compose command
            process = subprocess.Popen(['docker',
                                        'compose',
                                        '--file',
                                        'docker-compose.tutorial.yaml',
                                        'build'],
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE,
                                       start_new_session=True,
                                       cwd=self.system_test_dir)

            try:
                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
            except KeyboardInterrupt as k:
                process.kill()
                # process.send_signal(9)
                raise KeyboardInterrupt from k
            except Exception as e:
                logging.critical(
                    f"systemtest {self} had serious issues building the docker images via the `docker compose build` command. About to kill the docker compose command. Please check the logs! {e}")
                process.communicate(timeout=SHORT_TIMEOUT)
                process.kill()

            stdout_data.extend(stdout.decode().splitlines())
            stderr_data.extend(stderr.decode().splitlines())
            elapsed_time = time.perf_counter() - time_start
            return DockerComposeResult(process.returncode, stdout_data, stderr_data, self, elapsed_time)
        except Exception as e:
            logging.critical(f"Error executing docker compose build command: {e}")
            elapsed_time = time.perf_counter() - time_start
            return DockerComposeResult(1, stdout_data, stderr_data, self, elapsed_time)

    def _run_tutorial(self):
        """
        Runs precice couple

        Returns:
            A DockerComposeResult object containing the state.
        """
        logging.debug(f"Running tutorial {self}")
        time_start = time.perf_counter()
        stdout_data = []
        stderr_data = []
        try:
            # Execute docker-compose command
            process = subprocess.Popen(['docker',
                                        'compose',
                                        '--file',
                                        'docker-compose.tutorial.yaml',
                                        'up'],
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE,
                                       start_new_session=True,
                                       cwd=self.system_test_dir)

            try:
                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
            except KeyboardInterrupt as k:
                process.kill()
                # process.send_signal(9)
                raise KeyboardInterrupt from k
            except Exception as e:
                logging.critical(
                    f"Systemtest {self} had serious issues executing the docker compose command about to kill the docker compose command. Please check the logs! {e}")
                process.kill()
                stdout, stderr = process.communicate(timeout=SHORT_TIMEOUT)
                process.kill()

            stdout_data.extend(stdout.decode().splitlines())
            stderr_data.extend(stderr.decode().splitlines())
            elapsed_time = time.perf_counter() - time_start
            return DockerComposeResult(process.returncode, stdout_data, stderr_data, self, elapsed_time)
        except Exception as e:
            logging.critical(f"Error executing docker compose up command: {e}")
            elapsed_time = time.perf_counter() - time_start
            return DockerComposeResult(1, stdout_data, stderr_data, self, elapsed_time)

    def __repr__(self):
        return f"{self.tutorial.name} {self.case_combination}"

    def __write_logs(self, stdout_data: List[str], stderr_data: List[str]):
        with open(self.system_test_dir / "stdout.log", 'w') as stdout_file:
            stdout_file.write("\n".join(stdout_data))
        with open(self.system_test_dir / "stderr.log", 'w') as stderr_file:
            stderr_file.write("\n".join(stderr_data))

    def __prepare_for_run(self, run_directory: Path):
        """
        Prepares the run_directory with folders and datastructures needed for every systemtest execution
        """
        self.__copy_tutorial_into_directory(run_directory)
        self.__copy_tools(run_directory)
        self.__put_gitignore(run_directory)
        host_uid, host_gid = self.__get_uid_gid()
        self.params_to_use['PRECICE_UID'] = host_uid
        self.params_to_use['PRECICE_GID'] = host_gid

    def run(self, run_directory: Path):
        """
        Runs the system test by generating the Docker Compose file, copying everything into a run folder, and executing docker-compose up.
        """
        self.__prepare_for_run(run_directory)
        std_out: List[str] = []
        std_err: List[str] = []

        docker_build_result = self._build_docker()
        std_out.extend(docker_build_result.stdout_data)
        std_err.extend(docker_build_result.stderr_data)
        if docker_build_result.exit_code != 0:
            self.__write_logs(std_out, std_err)
            logging.critical(f"Could not build the docker images, {self} failed")
            return SystemtestResult(
                False,
                std_out,
                std_err,
                self,
                build_time=docker_build_result.runtime,
                solver_time=0,
                fieldcompare_time=0)

        docker_run_result = self._run_tutorial()
        std_out.extend(docker_run_result.stdout_data)
        std_err.extend(docker_run_result.stderr_data)
        if docker_run_result.exit_code != 0:
            self.__write_logs(std_out, std_err)
            logging.critical(f"Could not run the tutorial, {self} failed")
            return SystemtestResult(
                False,
                std_out,
                std_err,
                self,
                build_time=docker_build_result.runtime,
                solver_time=docker_run_result.runtime,
                fieldcompare_time=0)

        fieldcompare_result = self._run_field_compare()
        std_out.extend(fieldcompare_result.stdout_data)
        std_err.extend(fieldcompare_result.stderr_data)
        if fieldcompare_result.exit_code != 0:
            self.__write_logs(std_out, std_err)
            logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed")
            return SystemtestResult(
                False,
                std_out,
                std_err,
                self,
                build_time=docker_build_result.runtime,
                solver_time=docker_run_result.runtime,
                fieldcompare_time=fieldcompare_result.runtime)

        # self.__cleanup()
        self.__write_logs(std_out, std_err)
        return SystemtestResult(
            True,
            std_out,
            std_err,
            self,
            build_time=docker_build_result.runtime,
            solver_time=docker_run_result.runtime,
            fieldcompare_time=fieldcompare_result.runtime)

    def run_for_reference_results(self, run_directory: Path):
        """
        Runs the system test by generating the Docker Compose files to generate the reference results
        """
        self.__prepare_for_run(run_directory)
        std_out: List[str] = []
        std_err: List[str] = []
        docker_build_result = self._build_docker()
        std_out.extend(docker_build_result.stdout_data)
        std_err.extend(docker_build_result.stderr_data)
        if docker_build_result.exit_code != 0:
            self.__write_logs(std_out, std_err)
            logging.critical(f"Could not build the docker images, {self} failed")
            return SystemtestResult(
                False,
                std_out,
                std_err,
                self,
                build_time=docker_build_result.runtime,
                solver_time=0,
                fieldcompare_time=0)

        docker_run_result = self._run_tutorial()
        std_out.extend(docker_run_result.stdout_data)
        std_err.extend(docker_run_result.stderr_data)
        if docker_run_result.exit_code != 0:
            self.__write_logs(std_out, std_err)
            logging.critical(f"Could not run the tutorial, {self} failed")
            return SystemtestResult(
                False,
                std_out,
                std_err,
                self,
                build_time=docker_build_result.runtime,
                solver_time=docker_run_result.runtime,
                fieldcompare_time=0)

        self.__write_logs(std_out, std_err)
        return SystemtestResult(
            True,
            std_out,
            std_err,
            self,
            build_time=docker_build_result.runtime,
            solver_time=docker_run_result.runtime,
            fieldcompare_time=0)

    def run_only_build(self, run_directory: Path):
        """
        Runs only the build commmand, for example to preheat the caches of the docker builder.
        """
        self.__prepare_for_run(run_directory)
        std_out: List[str] = []
        std_err: List[str] = []
        docker_build_result = self._build_docker()
        std_out.extend(docker_build_result.stdout_data)
        std_err.extend(docker_build_result.stderr_data)
        if docker_build_result.exit_code != 0:
            self.__write_logs(std_out, std_err)
            logging.critical(f"Could not build the docker images, {self} failed")
            return SystemtestResult(
                False,
                std_out,
                std_err,
                self,
                build_time=docker_build_result.runtime,
                solver_time=0,
                fieldcompare_time=0)

        self.__write_logs(std_out, std_err)
        return SystemtestResult(
            True,
            std_out,
            std_err,
            self,
            build_time=docker_build_result.runtime,
            solver_time=0,
            fieldcompare_time=0)

    def get_system_test_dir(self) -> Path:
        return self.system_test_dir