Merge branch 'main' into exe-pipeline

Author: rboni-dk

dk-installer.py

@@ -224,7 +224,7 @@ class Console:
 
     def __init__(self):
         self._last_is_space = False
-        self._partial_msg = ""
+        self._partial_msg = None
 
     def title(self, text):
         LOG.info("Console title: [%s]", text)
@@ -259,22 +259,27 @@ def print_log(self, log_path: pathlib.Path) -> None:
             self._last_is_space = True
 
     @contextlib.contextmanager
-    def partial(self):
+    def start_partial(self):
         print(self.MARGIN, end="")
 
-        def console_partial(text):
-            print(text, end="")
-            sys.stdout.flush()
-            self._partial_msg += text
-
+        if self._partial_msg is not None:
+            raise ValueError("Console partial is already started.")
+        self._partial_msg = ""
         try:
-            yield console_partial
+            yield self.partial
         finally:
             print("")
             LOG.info("Console message: [%s]", self._partial_msg)
-            self._partial_msg = ""
+            self._partial_msg = None
             self._last_is_space = False
 
+    def partial(self, text):
+        if self._partial_msg is None:
+            raise ValueError("Console partial has not been started.")
+        print(text, end="")
+        sys.stdout.flush()
+        self._partial_msg += text
+
     @contextlib.contextmanager
     def tee(self, file_path, append=False):
         tee_lines = ["" if append else None]
@@ -797,7 +802,7 @@ def execute(self, args):
         action_fail_exception = None
         action_fail_step = None
         for step in action_steps:
-            with CONSOLE.partial() as partial:
+            with CONSOLE.start_partial() as partial:
                 partial(f"{step.label}... ")
                 try:
                     if action_fail_exception:
@@ -1921,22 +1926,42 @@ def create_compose_file(self, action, args, username, password, ssl_cert_file, s
 
 class TestGenPullImagesStep(Step):
     label = "Pulling docker images"
+    required = False
 
     def execute(self, action, args):
         action.analytics.additional_properties["pull_timeout"] = args.pull_timeout
 
         try:
-            action.run_cmd_retries(
+            with action.start_cmd(
                 "docker",
                 "compose",
                 "-f",
                 action.docker_compose_file_path,
                 "pull",
                 "--policy",
                 "always",
-                timeout=args.pull_timeout * 60,
-                retries=TESTGEN_PULL_RETRIES,
-            )
+            ) as (proc, _, stderr):
+                complete_re = re.compile(r"^ ([0-9a-f]{12}) (Already exists|Pull complete)")
+                hash_discovery_re = re.compile(r"^ ([0-9a-f]{12}) (Already exists|Pulling fs layer|Waiting)")
+                discovering = True
+                hashes: set[str] = set()
+                completed_count = 0
+                reported = 0
+                try:
+                    for line in stderr:
+                        if disc_match := hash_discovery_re.match(line):
+                            hashes.add(disc_match.group(1))
+                        elif hashes and discovering:
+                            discovering = False
+                        if complete_re.match(line):
+                            completed_count += 1
+                        if not discovering:
+                            to_be_reported = list(range(reported, int(completed_count * 100 / len(hashes)) + 1, 20))[1:]
+                            for progress in to_be_reported:
+                                CONSOLE.partial(f"{progress}% ")
+                                reported = progress
+                except Exception:
+                    pass
         except CommandFailed:
             # Pulling the images before starting is not mandatory, so we just proceed if it fails
             raise SkipStep

tests/test_tg_install.py

@@ -25,16 +25,16 @@ def tg_install_action(action_cls, args_mock, tmp_data_folder, start_cmd_mock):
 def test_tg_install(tg_install_action, start_cmd_mock, stdout_mock, tmp_data_folder, compose_path):
     tg_install_action.execute()
 
-    docker_call_retry = partial(call, "docker", "compose", "-f", compose_path, raise_on_non_zero=True, env=None)
-    docker_call = partial(docker_call_retry, raise_on_non_zero=True)
+    docker_call_start = partial(call, "docker", "compose", "-f", compose_path)
+    docker_call_run = partial(docker_call_start, raise_on_non_zero=True, env=None)
 
     start_cmd_mock.assert_has_calls(
         [
-            docker_call_retry("pull", "--policy", "always"),
-            docker_call("up", "--wait"),
-            docker_call("exec", "engine", "testgen", "setup-system-db", "--yes"),
-            docker_call("exec", "engine", "testgen", "upgrade-system-version"),
-            docker_call("exec", "engine", "testgen", "--help"),
+            docker_call_start("pull", "--policy", "always"),
+            docker_call_run("up", "--wait"),
+            docker_call_run("exec", "engine", "testgen", "setup-system-db", "--yes"),
+            docker_call_run("exec", "engine", "testgen", "upgrade-system-version"),
+            docker_call_run("exec", "engine", "testgen", "--help"),
         ],
         any_order=True,
     )

tests/test_tg_upgrade.py

@@ -101,7 +101,7 @@ def test_tg_upgrade(
         [
             call(*compose_args, "exec", "engine", "testgen", "--help", **compose_kwargs),
             call(*compose_args, "down", **compose_kwargs),
-            call(*compose_args, "pull", "--policy", "always", raise_on_non_zero=True, env=None),
+            call(*compose_args, "pull", "--policy", "always"),
             call(*compose_args, "up", "--wait", **compose_kwargs),
             call(*compose_args, "exec", "engine", "testgen", "upgrade-system-version", **compose_kwargs),
         ],