fix(ungrub): replicate GRUB EFI core to all boot-pool members

install_grub installed the core to only the target device's ESP, while
grub-install rebuilt the shared modules under /boot/grub. On a mirror that
left every other member's ESP with a stale core that no longer matched the
rebuilt modules, so those members failed to boot with
"symbol 'grub_memcpy' not found" -- silently destroying boot redundancy.

- install_grub now copies the freshly built core to every other member's ESP
  (no-op for a single-device pool).
- new "mkbootable sync <good-device>" op re-distributes the core from a known
  good member to all others, to repair members that already drifted (eg a
  device added via a bare "zpool attach").

install_efi_core writes safely:
- if the target ESP is already mounted, write through that mount (a raw mtools
  write under a live mount can be clobbered by the mount's cache flush);
  otherwise write the raw FAT with mtools, no mount needed.
- stage as BOOTX64.EFI.new then rename, so an interrupted write can't leave a
  torn core.

The core uses a dynamic ($root) prefix so the same image is valid on every
member. Split into member_esp / install_efi_core / replicate_grub. Both write
paths and the mtools round-trip verified byte-faithful and idempotent.

Author: elibosley

ungrub/mkbootable

@@ -13,6 +13,9 @@
 # Reconfigure grub.cfg and theme.txt after UUID change
 # mkbootable reconfigure
 
+# Copy the EFI core from a known-good member to every other member's ESP
+# mkbootable sync <good-device>
+
 #set -euo pipefail
 #set -x
 source /etc/rc.d/rc.runlog
@@ -82,6 +85,59 @@ resize_partitions() {
   udevadm settle
 }
 
+# resolve a boot-pool member partition to that disk's EFI System Partition,
+# eg /dev/nvme1n1p3 -> /dev/nvme1n1p2, /dev/sdb3 -> /dev/sdb2
+member_esp() {
+  local part="$1" disk sep
+  disk="$(lsblk -no pkname "$part" 2>/dev/null | head -1)"
+  [[ -n "$disk" ]] || return 1
+  [[ $disk == *[0-9] ]] && sep="p" || sep=""
+  printf '/dev/%s%s2\n' "$disk" "$sep"
+}
+
+# write the EFI core ($2) onto a single ESP ($1). Staged as BOOTX64.EFI.new then
+# renamed, so an interrupted write can't leave a torn core. If the ESP is already
+# mounted we write through that mount -- a raw mtools write under a live mount is
+# unsafe (the mount's cache can later flush stale data over it). Only when it is
+# not mounted do we write the raw FAT with mtools (no mount needed).
+install_efi_core() {
+  local esp="$1" src="$2" mnt dir
+  mnt="$(findmnt -nro TARGET -S "$esp" 2>/dev/null | head -1)"
+  if [[ -n "$mnt" ]]; then
+    dir="$mnt/EFI/BOOT"
+    mkdir -p "$dir" || return 1
+    cp -f "$src" "$dir/BOOTX64.EFI.new" || return 1
+    mv -f "$dir/BOOTX64.EFI.new" "$dir/BOOTX64.EFI" || return 1
+    sync -f "$mnt"
+  else
+    mmd -i "$esp" ::/EFI 2>/dev/null        # ensure dir tree exists (ok if already)
+    mmd -i "$esp" ::/EFI/BOOT 2>/dev/null
+    mcopy -D o -i "$esp" "$src" ::/EFI/BOOT/BOOTX64.EFI.new || return 1
+    mdel -i "$esp" ::/EFI/BOOT/BOOTX64.EFI 2>/dev/null
+    mmove -i "$esp" ::/EFI/BOOT/BOOTX64.EFI.new ::/EFI/BOOT/BOOTX64.EFI || return 1
+  fi
+}
+
+# Copy a known-good EFI core to the ESP of every boot pool member except the one
+# it came from. grub-install rebuilds the shared modules under $BOOT/grub but
+# writes the core to only one ESP; a member left with a stale core no longer
+# matches those modules and fails to boot with "symbol 'grub_memcpy' not found",
+# silently breaking the mirror's boot redundancy. The core uses a dynamic
+# ($root) prefix, so the same image is valid on every member.
+#   $1 = path to the known-good BOOTX64.EFI to distribute (on a mounted ESP)
+replicate_grub() {
+  local src="$1" part esp
+  for part in $(zpool list -v -H -P "$POOL" | awk '/\/dev\// {print $1}'); do
+    esp="$(member_esp "$part")" || { log "replicate_grub: cannot resolve ESP for $part, skipping"; continue; }
+    [[ "$esp" == "$EFI_PART" || ! -b "$esp" ]] && continue
+    if install_efi_core "$esp" "$src"; then
+      log "replicate_grub: synced EFI core to $esp"
+    else
+      log "replicate_grub: failed to sync EFI core to $esp"
+    fi
+  done
+}
+
 install_grub() {
   # format the EFI System Partition (partition 2) with FAT32 and mount
   mkfs.fat -F32 -n EFI "$EFI_PART"
@@ -103,6 +159,11 @@ install_grub() {
     --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \
     --no-floppy --no-rs-codes
 
+  # replicate the core we just built to every other member's ESP so all mirror
+  # members stay in lockstep with the shared modules rebuilt above (no-op while
+  # this is the only member)
+  replicate_grub "$BOOT/efi/EFI/BOOT/BOOTX64.EFI"
+
   # we don't need this mounted anymore
   umount "$BOOT/efi"
 }
@@ -162,6 +223,19 @@ remove_device() {
   fi
 }
 
+# Re-distribute the EFI core from a known-good member to all other members.
+# Use this to repair members whose ESP holds a stale core (eg a device added
+# with a bare "zpool attach", or a member missed by an earlier grub refresh).
+sync_device() {
+  mkdir -p -m 0700 "$BOOT/efi"
+  if ! mount -o ro "$EFI_PART" "$BOOT/efi" 2>/dev/null; then
+    log "sync: cannot mount source ESP $EFI_PART"
+    exit 1
+  fi
+  replicate_grub "$BOOT/efi/EFI/BOOT/BOOTX64.EFI"
+  umount "$BOOT/efi"
+}
+
 case "$OPER" in
 'add')
   add_device
@@ -175,6 +249,9 @@ case "$OPER" in
 'reconfigure')
   configure_grub
   ;;
+'sync')
+  sync_device
+  ;;
 *)
   log "error 2" # shouldn't happen
   exit 2