fix(ungrub): sync BOOTX64.EFI to all boot-pool members

mkbootable installed the GRUB core to only the target device's ESP, while
grub-install refreshed the shared modules under /boot/grub. On a mirror that
left every other member's ESP with a stale core that no longer matched the
rebuilt modules, so those members failed to boot with
"symbol 'grub_memcpy' not found" -- silently destroying boot redundancy.

- install_grub now replicates the freshly built core to every pool member's
  ESP (no-op for a single-device pool; fixes add/replace/refresh on mirrors).
- new "mkbootable sync" op re-distributes the canonical core
  (/boot/grub/x86_64-efi/core.efi) to all members, to repair systems whose
  members already drifted (eg a device added via a bare "zpool attach").

The core uses a dynamic ($root) prefix, so the same image is valid on every
member. Device->ESP mapping verified against a live nvme mirror.

Author: elibosley

ungrub/mkbootable

@@ -13,6 +13,9 @@
 # Reconfigure grub.cfg and theme.txt after UUID change
 # mkbootable reconfigure
 
+# Re-distribute the current GRUB EFI core to every boot-pool member's ESP
+# mkbootable sync
+
 #set -euo pipefail
 #set -x
 source /etc/rc.d/rc.runlog
@@ -82,6 +85,50 @@ resize_partitions() {
   udevadm settle
 }
 
+# map a boot-pool member partition (eg /dev/nvme1n1p3 or /dev/sdb3) to that
+# disk's EFI System Partition (partition 2, eg /dev/nvme1n1p2 or /dev/sdb2)
+member_to_esp() {
+  local part="$1" disk sep
+  disk="$(lsblk -no pkname "$part" 2>/dev/null | head -1)"
+  [[ -n "$disk" ]] || return 1
+  [[ $disk == *[0-9] ]] && sep="p" || sep=""
+  printf '/dev/%s%s2\n' "$disk" "$sep"
+}
+
+# Replicate a known-good EFI core (BOOTX64.EFI) to the ESP of every device in
+# the boot pool. The core uses a dynamic "($root)" prefix and is rebuilt in
+# lockstep with the shared modules under $BOOT/grub, so the same image is valid
+# on every mirror member. Without this, a grub-install against one member
+# refreshes the shared modules but leaves sibling ESPs with a stale core,
+# yielding "symbol 'grub_memcpy' not found" on those members at boot.
+#   $1 = path to a known-good BOOTX64.EFI to distribute
+sync_efi_to_members() {
+  local src="$1" part esp mnt
+  if [[ ! -s "$src" ]]; then
+    log "sync_efi_to_members: source EFI '$src' missing or empty, skipping"
+    return 1
+  fi
+  for part in $(zpool list -v -H -P "$POOL" 2>/dev/null | awk '/\/dev\// {print $1}'); do
+    esp="$(member_to_esp "$part")" || { log "sync_efi_to_members: cannot map $part to an ESP, skipping"; continue; }
+    [[ -b "$esp" ]] || { log "sync_efi_to_members: $esp is not a block device, skipping"; continue; }
+    mnt="$(mktemp -d)"
+    if ! mount "$esp" "$mnt" 2>/dev/null; then
+      log "sync_efi_to_members: $esp not mountable (unformatted ESP?), skipping"
+      rmdir "$mnt"
+      continue
+    fi
+    mkdir -p "$mnt/EFI/BOOT"
+    if cp -f "$src" "$mnt/EFI/BOOT/BOOTX64.EFI"; then
+      sync -f "$mnt"
+      log "sync_efi_to_members: wrote BOOTX64.EFI to $esp"
+    else
+      log "sync_efi_to_members: failed to write BOOTX64.EFI to $esp"
+    fi
+    umount "$mnt"
+    rmdir "$mnt"
+  done
+}
+
 install_grub() {
   # format the EFI System Partition (partition 2) with FAT32 and mount
   mkfs.fat -F32 -n EFI "$EFI_PART"
@@ -103,8 +150,20 @@ install_grub() {
     --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \
     --no-floppy --no-rs-codes
 
+  # stash the freshly built core before unmounting so it can be replicated to
+  # every other pool member's ESP (keeps all mirror members in lockstep with
+  # the shared modules just rebuilt above)
+  local core_img
+  core_img="$(mktemp)"
+  cp -f "$BOOT/efi/EFI/BOOT/BOOTX64.EFI" "$core_img"
+
   # we don't need this mounted anymore
   umount "$BOOT/efi"
+
+  # propagate the new core to all members (no-op for the first device, which is
+  # the only member at this point; required for every subsequent add/refresh)
+  sync_efi_to_members "$core_img"
+  rm -f "$core_img"
 }
 
 add_device() {
@@ -175,6 +234,19 @@ case "$OPER" in
 'reconfigure')
   configure_grub
   ;;
+'sync')
+  # Re-distribute the current GRUB EFI core to every boot-pool member's ESP.
+  # Source is the canonical core built alongside the shared modules; it is
+  # guaranteed to match them. Use this to repair members whose ESP holds a
+  # stale core (eg a device added with a bare "zpool attach", or any member
+  # missed by a prior single-disk grub refresh).
+  if [[ -s "$BOOT/grub/x86_64-efi/core.efi" ]]; then
+    sync_efi_to_members "$BOOT/grub/x86_64-efi/core.efi"
+  else
+    log "sync: $BOOT/grub/x86_64-efi/core.efi not found; run 'mkbootable add' or a grub refresh first"
+    exit 1
+  fi
+  ;;
 *)
   log "error 2" # shouldn't happen
   exit 2