burn/install: route through pod-side fastboot when power=rack (#88)

widgetii · claude · web-flow · commit deabdc21d34b · 2026-05-11T22:10:04.000+03:00
## Summary `defib burn` and `defib install` drove the HiSilicon SPL upload from the host even when the transport went through a rack pod's WiFi-bridged UART, where the per-frame ACK loop (150 ms × dozens of frames per upload) doesn't survive the round-trip latency — both commands failed at the very first PRESTEP0 frame. Now, when `power_controller` is a `RackController`, the CLI calls the new `defib.recovery.rack_fastboot.run_rack_fastboot()` helper instead of `session.run()`. The helper packages profile + SPL + agent into the binary blob the pod's `POST /fastboot` expects, posts it, and turns the pod's phase-by-phase JSON into a `RecoveryResult` so the rest of the CLI (terminal mode, download_process detection, TFTP scripting) stays unchanged. The pod takes exclusive UART access during the upload, so the host transport is opened only **after** fastboot returns. ## Live verification on the prototype ``` $ DEFIB_POWER_TYPE=rack DEFIB_RACK_HOST=10.216.128.69 \ defib burn -c hi3516ev300 -p tcp://10.216.128.69:9000 \ --power-cycle --break Power: rack pod HTTP API Pod-side fastboot in progress… rack fastboot: spl=17408 agent=236195 spl_addr=0x4010500 ddr_addr=0x4013000 uboot_addr=0x41000000 Done! (25678ms) ``` ``` $ # camera halted at the freshly-uploaded U-Boot prompt > version U-Boot 2016.11-g131d3f2 (May 08 2026 - 11:58:25 +0000) hi3516ev300 OpenIPC # ``` Build `g131d3f2` is **distinct from** the in-flash build (`g6d2ed0c-dirty`, Mar 2023) — proves the burn landed in RAM and the chip jumped to the new image rather than falling through to flash. ## Install + restore scope - **`install`'s Phase 1** (burn-to-RAM) now uses the same fastboot path; Phase 2 (U-Boot `tftp` + `sf write` scripting) goes over the bridge as ordinary text commands and is already known to work — TFTP-through-pod-NAPT was verified during the earlier manual kernel restore at 167 KB/s. - **`restore`** has its own shape (frame-blast started before power-on, then power-on triggers the catch) that doesn't map cleanly onto fastboot's all-in-one semantics. Left out of scope for this PR; can be a follow-up if needed. ## Architecture note The SPL-boundary detection (`HiSiliconStandard._detect_spl_size`) and the 0xFF-run zeroing (`_zero_long_ff_runs`) stay on the host. The pod gets ready-to-send bytes. This keeps the pod firmware minimal and ensures the two paths (host-driven and pod-driven) stay byte-identical for any chip we test. ## Test plan - [ ] `uv run pytest tests/ -x -v --ignore=tests/fuzz` (**461 passed / 2 skipped**) - [ ] `uv run ruff check src/defib/ tests/` - [ ] `uv run mypy src/defib/cli/app.py src/defib/recovery/rack_fastboot.py --ignore-missing-imports` - [ ] 4 new `TestRunRackFastboot` cases cover success path, PRESTEP0 failure attribution, profile-address packing, and the `agent_payload` override used by agent-flash. - [ ] Regression: existing local-UART burn / install paths unchanged — both still go through `session.run` when power controller is RouterOS / Vectis / None. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Dmitry Ilyin <widgetii@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
diff --git a/src/defib/cli/app.py b/src/defib/cli/app.py
@@ -145,6 +145,13 @@ def _dl_progress(done: int, total: int) -> None:
             if output == "human":
                 console.print(f"Power: [cyan]{power_controller.name()}[/cyan]")
 
+    # Detect rack-pod power early: the pod runs the entire HiSilicon SPL
+    # upload locally over its UART, so we MUST NOT have a TCP client on
+    # tcp://<pod>:9000 during the upload.  Defer transport-open until
+    # after fastboot returns.
+    from defib.power.rack import RackController
+    use_rack_fastboot = isinstance(power_controller, RackController)
+
     try:
         session = RecoverySession(
             chip=chip, firmware_path=firmware_path,
@@ -163,21 +170,24 @@ def _dl_progress(done: int, total: int) -> None:
         console.print(f"Protocol: [cyan]{session.protocol_name}[/cyan]")
         console.print(f"Port: [cyan]{port}[/cyan]")
 
-    # Use platform-aware transport factory
-    try:
-        from defib.transport.serial_platform import create_transport, normalize_port_name
-        transport = await create_transport(normalize_port_name(port))
-    except Exception as e:
-        if output == "json":
-            print(json_mod.dumps({"event": "error", "message": f"Serial port error: {e}"}))
-        else:
-            console.print(f"[red]Failed to open serial port:[/red] {e}")
-        raise typer.Exit(2)
+    # Use platform-aware transport factory.  Skip on rack-fastboot path —
+    # the pod needs exclusive UART access during the upload.
+    from defib.transport.serial_platform import create_transport, normalize_port_name
+    transport = None
+    if not use_rack_fastboot:
+        try:
+            transport = await create_transport(normalize_port_name(port))
+        except Exception as e:
+            if output == "json":
+                print(json_mod.dumps({"event": "error", "message": f"Serial port error: {e}"}))
+            else:
+                console.print(f"[red]Failed to open serial port:[/red] {e}")
+            raise typer.Exit(2)
 
     # Vectis: hand the live RFC 2217 transport (or legacy raw TCP) to
     # the controller so RTS/DTR toggles ride the same connection that
     # the UART data uses — Vectis only allows one client at a time.
-    if power_controller is not None:
+    if power_controller is not None and transport is not None:
         from defib.power.vectis import VectisController
         from defib.transport.rfc2217 import Rfc2217Transport
         from defib.transport.socket import SocketTransport
@@ -238,12 +248,42 @@ def on_log(event: LogEvent) -> None:
             console.print(f"[{style}]{event.message}[/{style}]")
 
     try:
-        result = await session.run(
-            transport,
-            on_progress=on_progress,
-            on_log=on_log,
-            send_break=send_break,
-        )
+        if use_rack_fastboot:
+            # Pod-side fastboot: pod runs handshake + DDR + SPL + U-Boot
+            # locally on its UART (microsecond ACK latency).  No host
+            # transport during the upload.
+            from pathlib import Path
+            from defib.recovery.rack_fastboot import run_rack_fastboot
+            assert isinstance(power_controller, RackController)
+            on_log(LogEvent(level="info", message="Pod-side fastboot in progress…"))
+            firmware_bytes = Path(firmware_path).read_bytes()
+            result = await run_rack_fastboot(
+                power_controller, chip, firmware_bytes,
+            )
+            # Now open the transport for the post-burn flow (terminal /
+            # download_process detection / etc.).
+            if result.success:
+                transport = await create_transport(normalize_port_name(port))
+                if send_break:
+                    # Spam Ctrl-C briefly to break U-Boot autoboot — the
+                    # host-side path does this inside send_firmware, but
+                    # we skipped that.
+                    import asyncio as _aio
+                    end = _aio.get_event_loop().time() + 2.0
+                    while _aio.get_event_loop().time() < end:
+                        try:
+                            await transport.write(b"\x03")
+                        except Exception:
+                            break
+                        await _aio.sleep(0.05)
+        else:
+            assert transport is not None  # opened above when not use_rack_fastboot
+            result = await session.run(
+                transport,
+                on_progress=on_progress,
+                on_log=on_log,
+                send_break=send_break,
+            )
     finally:
         if progress_ctx is not None:
             progress_ctx.stop()
@@ -264,8 +304,10 @@ def on_log(event: LogEvent) -> None:
             console.print(f"\n[red bold]Failed:[/red bold] {result.error}")
 
     if not result.success:
-        await transport.close()
+        if transport is not None:
+            await transport.close()
         raise typer.Exit(1)
+    assert transport is not None  # success ⇒ transport opened
 
     # Terminal mode: stream serial output until Ctrl-C
     # Auto-detects download_process mode and bridges XHEAD/XCMD framing.
@@ -1954,6 +1996,11 @@ async def _install_async(
             if output == "human":
                 console.print(f"  Power: [cyan]{power_controller.name()}[/cyan]")
 
+    # Detect rack-pod power: the pod runs the SPL/DDR/U-Boot upload
+    # locally, requires exclusive UART, so we open the transport AFTER.
+    from defib.power.rack import RackController
+    use_rack_fastboot = isinstance(power_controller, RackController)
+
     session = RecoverySession(
         chip=chip, firmware_path=str(cached),
         power_controller=power_controller, poe_port=poe_port,
@@ -1964,16 +2011,18 @@ async def _install_async(
         if not power_cycle:
             console.print("  [yellow]Power-cycle the camera now![/yellow]")
 
-    transport = await create_transport(normalize_port_name(port))
+    transport = None
+    if not use_rack_fastboot:
+        transport = await create_transport(normalize_port_name(port))
 
-    # Vectis: share the TCP transport for Ctrl+P delivery (see burn).
-    if power_controller is not None:
-        from defib.power.vectis import VectisController
-        from defib.transport.socket import SocketTransport
-        if isinstance(power_controller, VectisController) and isinstance(
-            transport, SocketTransport
-        ):
-            power_controller.attach_transport(transport)
+        # Vectis: share the TCP transport for Ctrl+P delivery (see burn).
+        if power_controller is not None:
+            from defib.power.vectis import VectisController
+            from defib.transport.socket import SocketTransport
+            if isinstance(power_controller, VectisController) and isinstance(
+                transport, SocketTransport
+            ):
+                power_controller.attach_transport(transport)
 
     def on_log(event: LogEvent) -> None:
         if output == "human":
@@ -1984,19 +2033,32 @@ def on_progress(event: ProgressEvent) -> None:
         if output == "human" and event.message:
             console.print(f"  {event.message}")
 
-    result = await session.run(
-        transport,
-        on_progress=on_progress,
-        on_log=on_log,
-        send_break=False,
-    )
+    if use_rack_fastboot:
+        from defib.recovery.rack_fastboot import run_rack_fastboot
+        assert isinstance(power_controller, RackController)
+        on_log(LogEvent(level="info", message="Pod-side fastboot in progress…"))
+        result = await run_rack_fastboot(
+            power_controller, chip, cached.read_bytes(),
+        )
+        if result.success:
+            transport = await create_transport(normalize_port_name(port))
+    else:
+        assert transport is not None  # opened above when not use_rack_fastboot
+        result = await session.run(
+            transport,
+            on_progress=on_progress,
+            on_log=on_log,
+            send_break=False,
+        )
 
     if not result.success:
         console.print(f"[red]Burn failed:[/red] {result.error}")
-        await transport.close()
+        if transport is not None:
+            await transport.close()
         if power_controller:
             await power_controller.close()
         raise typer.Exit(1)
+    assert transport is not None  # success ⇒ transport opened
 
     if output == "human":
         console.print(f"  [green]U-Boot loaded in {result.elapsed_ms:.0f}ms[/green]")
diff --git a/src/defib/recovery/rack_fastboot.py b/src/defib/recovery/rack_fastboot.py
@@ -0,0 +1,129 @@
+"""Pod-side fastboot bring-up for rack-pod-controlled cameras.
+
+When a rack pod is the power controller, the host can't reliably drive
+the HiSilicon SPL boot protocol over its WiFi-bridged UART — the
+per-frame ACK loop (150 ms timeout × dozens of frames per upload)
+doesn't survive WiFi RTT. Instead the pod runs the entire upload
+sequence locally on its UART and returns a phase-by-phase JSON
+summary; this module is the host-side adapter that:
+
+1. Loads the SoC profile (PRESTEP0 / DDRSTEP0 / optional PRESTEP1 +
+   load addresses).
+2. Detects the SPL boundary in the firmware blob with the same logic
+   `HiSiliconStandard._send_spl` uses on the host path, then zeroes
+   long 0xFF runs (cv500-family bootrom RX bug).
+3. Calls :meth:`RackController.fastboot` with the assembled bundle.
+4. Returns a :class:`RecoveryResult`-shaped object so callers can drop
+   it into the same post-burn flow they use for `session.run()`.
+
+The CLI's burn / install / agent-upload paths use this when
+``power_controller`` is a :class:`RackController`. Note that the pod
+takes exclusive UART access during the upload, so callers MUST NOT
+have a TCP client connected to ``tcp://<pod>:9000`` when this runs —
+open the transport only after this function returns.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from pathlib import Path
+
+from defib.power.rack import RackController
+from defib.profiles.loader import load_profile
+from defib.protocol.hisilicon_standard import HiSiliconStandard
+from defib.recovery.events import RecoveryResult, Stage
+
+logger = logging.getLogger(__name__)
+
+
+async def run_rack_fastboot(
+    rack: RackController,
+    chip: str,
+    firmware: bytes | str | Path,
+    agent_payload: bytes | None = None,
+    timeout: float = 180.0,
+) -> RecoveryResult:
+    """Run the pod-side SPL/DDR/U-Boot upload via ``POST /fastboot``.
+
+    Args:
+        rack: configured :class:`RackController`.
+        chip: SoC name (e.g. ``"hi3516ev300"``) for profile lookup.
+        firmware: u-boot bytes (or path) used to derive the SPL portion
+            and — if ``agent_payload`` is omitted — as the U-Boot blob
+            loaded at ``profile.uboot_address``.  Matches the host-path
+            behaviour of ``defib burn``: the same binary contains both
+            the SPL boundary and the U-Boot to run.
+        agent_payload: optional override for the blob loaded at
+            ``profile.uboot_address``.  Used by the agent-flash path,
+            which sends ``u-boot.bin`` as SPL and the flash agent as
+            U-Boot.  ``None`` (the default) uses ``firmware`` for both.
+        timeout: HTTP timeout for the fastboot POST.
+
+    Returns:
+        :class:`RecoveryResult` with ``success`` / ``elapsed_ms`` /
+        ``error`` populated.  ``stages_completed`` reflects the phases
+        the pod reported reaching.
+    """
+    firmware_bytes = (
+        firmware if isinstance(firmware, (bytes, bytearray))
+        else Path(firmware).read_bytes()
+    )
+    profile = load_profile(chip)
+
+    # Same SPL-boundary detection the host path uses — keep both paths
+    # byte-identical so a chip that works on one works on the other.
+    scan_buf = agent_payload if agent_payload is not None else firmware_bytes
+    spl_size = HiSiliconStandard._detect_spl_size(
+        scan_buf, profile.spl_max_size, sram_limit=profile.spl_sram_limit,
+    )
+    spl_bytes = firmware_bytes[:spl_size].ljust(spl_size, b"\x00")
+    spl_bytes = HiSiliconStandard._zero_long_ff_runs(spl_bytes)
+    uboot_bytes = agent_payload if agent_payload is not None else firmware_bytes
+
+    logger.info(
+        "rack fastboot: spl=%d agent=%d profile=%s spl_addr=0x%x ddr_addr=0x%x uboot_addr=0x%x",
+        len(spl_bytes), len(uboot_bytes), profile.name,
+        profile.spl_address, profile.ddr_step_address, profile.uboot_address,
+    )
+
+    t0 = time.monotonic()
+    response = await rack.fastboot(
+        spl_address=profile.spl_address,
+        ddr_step_address=profile.ddr_step_address,
+        uboot_address=profile.uboot_address,
+        prestep0=profile.prestep_data or b"",
+        ddrstep0=profile.ddr_step_data,
+        prestep1=profile.prestep1_data,
+        spl=spl_bytes,
+        agent=uboot_bytes,
+        timeout=timeout,
+    )
+    elapsed_ms = (time.monotonic() - t0) * 1000.0
+
+    stages: list[Stage] = []
+    last_phase = str(response.get("last_phase", ""))
+    # Pod's phase names map onto defib's Stage enum where they exist.
+    if last_phase in ("frame_for_start", "prestep0", "ddrstep0", "prestep1", "spl", "agent", "done"):
+        stages.append(Stage.HANDSHAKE)
+    if last_phase in ("prestep0", "ddrstep0", "prestep1", "spl", "agent", "done"):
+        stages.append(Stage.DDR_INIT)
+    if last_phase in ("spl", "agent", "done"):
+        stages.append(Stage.SPL)
+    if last_phase in ("agent", "done"):
+        stages.append(Stage.UBOOT)
+    if last_phase == "done":
+        stages.append(Stage.COMPLETE)
+
+    if response.get("success"):
+        return RecoveryResult(
+            success=True, stages_completed=stages, elapsed_ms=elapsed_ms,
+        )
+
+    failed = response.get("failed_phase", "unknown")
+    err = response.get("error", "unknown")
+    return RecoveryResult(
+        success=False, stages_completed=stages,
+        error=f"rack fastboot failed at {failed}: {err}",
+        elapsed_ms=elapsed_ms,
+    )
diff --git a/tests/test_recovery_rack_fastboot.py b/tests/test_recovery_rack_fastboot.py
