Skip to content

Commit 6ceaa86

Browse files
pluskymimi1vx
authored andcommitted
perf(reboot): reconnect hosts concurrently after a reboot
reboot()/_reboot() dispatched the reboot to every host (fire-and-forget) and then reconnected them one at a time. Connection.reconnect begins with a fixed pre-sleep (~10s on the first attempt) before probing the host, so the serial loop stacked that wait: N hosts paid ~N*10s even though they all rebooted at once. Fan the reconnects out via the file's own run_parallel helper so the waits overlap instead of accumulating. The per-host sleep is kept -- it guards against reconnecting to a still-shutting-down sshd -- and the reconnect call is unchanged; only the serialization is removed. Reconnects are independent (each Target owns its Connection), and the first failure still surfaces to the caller. Most visible on transactional (SL Micro) updates, which reboot every host. Test: a Barrier-based concurrency proof (fails on a serial revert) that also pins the keyword call convention.
1 parent 98fc717 commit 6ceaa86

3 files changed

Lines changed: 80 additions & 6 deletions

File tree

CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht
2828
updates. `refhosts.yml` is now parsed once per report and shared across every
2929
testplatform (and with the product-drift check) instead of being re-parsed
3030
(~1s each) for every testplatform during autoconnect / `add_host`.
31+
- Rebooting a host group is faster. After the reboot is dispatched, the hosts
32+
are now reconnected concurrently instead of one at a time, so the fixed
33+
per-host reconnect wait (~10s) overlaps across hosts rather than stacking.
34+
The wait itself is kept (it guards against reconnecting to a
35+
still-shutting-down sshd); only the serialization is removed. Most visible on
36+
transactional (SL Micro) updates, which reboot every host.
3137

3238
## 19.0.1 - 2026-07-17
3339

mtui/hosts/target/hostgroup.py

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -214,9 +214,7 @@ def _reboot(self, reboot: dict[str, str]) -> None:
214214
# host with retries + backoff while it comes back up.
215215
for hn, command in reboot.items():
216216
self.data[hn].reboot(command)
217-
for hn in sorted(reboot):
218-
self.data[hn].reconnect(retry=10, backoff=True)
219-
logger.info("%s is back up", hn)
217+
self._reconnect_all(sorted(reboot))
220218

221219
def reboot(
222220
self, command: str = "systemctl reboot", relock_comment: str = ""
@@ -245,9 +243,7 @@ def reboot(
245243
boot_ids = {hn: t.boot_id() for hn, t in self.data.items()}
246244
for t in self.data.values():
247245
t.reboot(command)
248-
for hn in sorted(self.data):
249-
self.data[hn].reconnect(retry=10, backoff=True)
250-
logger.info("%s is back up", hn)
246+
self._reconnect_all(sorted(self.data))
251247

252248
for hn, t in self.data.items():
253249
self._verify_reboot(t, boot_ids[hn])
@@ -282,6 +278,29 @@ def _verify_reboot(target, old_boot_id: str) -> None:
282278
new_boot_id,
283279
)
284280

281+
def _reconnect_all(self, hostnames: list[str]) -> None:
282+
"""Reconnect the given hosts concurrently after a reboot.
283+
284+
Each ``Connection.reconnect`` begins with a fixed pre-sleep (~10s on
285+
the first attempt) before probing the host. Fanning the reconnects
286+
out overlaps those waits instead of stacking them (N hosts would
287+
otherwise pay ~N*10s even though they all rebooted at once); the
288+
per-host sleep is kept -- it guards against reconnecting to a
289+
still-shutting-down sshd -- and only its accumulation is removed.
290+
Reconnects are independent: each :class:`Target` owns its own
291+
connection. The first failure surfaces to the caller, as a serial
292+
loop would.
293+
"""
294+
295+
def _reconnect(hn: str) -> None:
296+
self.data[hn].reconnect(retry=10, backoff=True)
297+
logger.info("%s is back up", hn)
298+
299+
run_parallel(
300+
[(_reconnect, (hn,)) for hn in hostnames],
301+
desc="reconnect" if self.interactive else None,
302+
)
303+
285304
def update_lock(self) -> None:
286305
"""Locks all hosts in the group for an update."""
287306
skipped = False

tests/test_hostgroup.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,33 @@ def test_reboot_all_logs_clean_host_list_and_back_up(caplog):
548548
assert not any("['" in m for m in msgs)
549549

550550

551+
def test_reboot_reconnects_hosts_concurrently():
552+
"""Post-reboot reconnects run in parallel, not one host at a time.
553+
554+
Each host's reconnect blocks on a Barrier of width N; it trips only if
555+
every host's reconnect is in flight at once. A revert to the serial loop
556+
makes the first reconnect block alone until the Barrier times out, and
557+
run_parallel re-raises the resulting BrokenBarrierError -- so this test
558+
fails on that revert. It also pins the keyword call convention.
559+
"""
560+
import threading
561+
562+
hosts = ["h1", "h2", "h3"]
563+
barrier = threading.Barrier(len(hosts))
564+
targets = []
565+
for hn in hosts:
566+
t = _stub_target(hn, transactional=True)
567+
t.reconnect.side_effect = lambda retry, backoff: barrier.wait(timeout=10)
568+
targets.append(t)
569+
570+
hg = HostsGroup(targets)
571+
hg._reboot(dict.fromkeys(hosts, "systemctl reboot"))
572+
573+
for t in targets:
574+
t.reboot.assert_called_once_with("systemctl reboot")
575+
t.reconnect.assert_called_once_with(retry=10, backoff=True)
576+
577+
551578
# ---------------------------------------------------------------------------
552579
# update_lock — comment-logging branch
553580
# ---------------------------------------------------------------------------
@@ -1027,6 +1054,28 @@ def test_hostgroup_interactive_passes_desc_label_to_run_parallel():
10271054
assert mock_rp.call_args.kwargs["desc"] == "set_repo add"
10281055

10291056

1057+
def test_hostgroup_non_interactive_reconnect_passes_desc_none():
1058+
"""``_reconnect_all`` on a headless group (mtui-mcp) must pass ``desc=None``."""
1059+
t1 = _stub_target("h1")
1060+
hg = HostsGroup([t1], interactive=False)
1061+
1062+
with patch("mtui.hosts.target.hostgroup.run_parallel") as mock_rp:
1063+
hg._reconnect_all(["h1"]) # noqa: SLF001
1064+
1065+
assert mock_rp.call_args.kwargs["desc"] is None
1066+
1067+
1068+
def test_hostgroup_interactive_reconnect_passes_desc_label():
1069+
"""The REPL path keeps the ``reconnect`` spinner label."""
1070+
t1 = _stub_target("h1")
1071+
hg = HostsGroup([t1]) # interactive=True by default
1072+
1073+
with patch("mtui.hosts.target.hostgroup.run_parallel") as mock_rp:
1074+
hg._reconnect_all(["h1"]) # noqa: SLF001
1075+
1076+
assert mock_rp.call_args.kwargs["desc"] == "reconnect"
1077+
1078+
10301079
def test_hostgroup_non_interactive_run_passes_through_to_runcommand():
10311080
"""``HostsGroup.run`` forwards ``interactive`` into ``RunCommand``."""
10321081
from mtui.types import ExecutionMode

0 commit comments

Comments
 (0)