Fixes to get SDHCI stable (DISK_TEST=1) passing and image hash/verify passing. Only issue is test-app EL level. Peer review fixes.

Author: dgarske

config/examples/zynqmp_sdcard.config

@@ -2,7 +2,7 @@
 # Zynq UltraScale+ MPSoC ZU9EG - Quad-core ARM Cortex-A53
 #
 # This configuration enables SD card boot for the ZynqMP:
-#   FSBL -> BL31 (EL3) -> wolfBoot (EL2) -> Linux (EL1)
+#   FSBL -> PMUFW -> BL31 (EL3) -> wolfBoot (EL2) -> Linux (EL1)
 #
 # wolfBoot loads firmware images from MBR partitions on SD card.
 # Uses the generic SDHCI driver with SD1 controller (external SD slot on ZCU102).
@@ -29,6 +29,8 @@ DISK_EMMC?=0
 # ZynqMP Arasan SDHCI does not support CDSS/CDTL card detect test level.
 # Since FSBL booted from the same SD card, we know it is present.
 CFLAGS_EXTRA+=-DSDHCI_FORCE_CARD_DETECT
+# Note: Arasan SDHCI v3.0 does not support HV4E mode. The platform layer
+# in hal/zynq.c transparently redirects SRS22/SRS23 to SRS00 for legacy SDMA.
 
 # Disable QSPI flash when using SD card
 EXT_FLASH?=0
@@ -37,6 +39,9 @@ NO_XIP=1
 # ELF loading support
 ELF?=1
 
+# Boot Exception Level: transition from EL2 -> EL1 before jumping to app
+BOOT_EL1?=1
+
 # General options
 VTOR?=1
 CORTEX_M0?=0

docs/Targets.md

@@ -1991,7 +1991,7 @@ wolfBoot replaces U-Boot in the ZynqMP boot flow:
 FSBL -> PMUFW -> BL31 (EL3) -> wolfBoot (EL2) -> Linux (EL1)
 ```
 
-wolfBoot runs from DDR at `0x7FF00000` (EL2, non-secure). All clock, MIO, and DDR initialization is handled by the FSBL/PMUFW before wolfBoot starts.
+For QSPI boot, wolfBoot runs from DDR at `0x7FF00000` (EL2, non-secure). For SD card boot, wolfBoot runs at `0x8000000`. All clock, MIO, and DDR initialization is handled by the FSBL/PMUFW before wolfBoot starts.
 
 This target supports **two boot paths**:
 - **QSPI boot** (primary, production-style): `config/examples/zynqmp.config`
@@ -2101,8 +2101,8 @@ make clean
 make
 
 make test-app/image.bin
-./tools/keytools/sign --rsa4096 --sha3 test-app/image.bin wolfboot_signing_private_key.der 1
-./tools/keytools/sign --rsa4096 --sha3 test-app/image.bin wolfboot_signing_private_key.der 2
+IMAGE_HEADER_SIZE=1024 ./tools/keytools/sign --rsa4096 --sha3 test-app/image.elf wolfboot_signing_private_key.der 1
+IMAGE_HEADER_SIZE=1024 ./tools/keytools/sign --rsa4096 --sha3 test-app/image.elf wolfboot_signing_private_key.der 2
 ```
 
 **Build BOOT.BIN for SD card**
@@ -2154,6 +2154,17 @@ sudo umount /mnt
 sudo fdisk -l /dev/sdX
 ```
 
+Or just mount and copy the BOOT.BIN and "dd" the partitions
+
+```sh
+# Mount the FAT boot partition and copy just the updated BOOT.BIN
+sudo mount /dev/sdc1 /mnt && sudo cp BOOT.BIN /mnt/ && sudo umount /mnt && sync
+
+# Write the signed image to partition 2 (P:A) and partition 3 (P:B)
+sudo dd if=test-app/image_v1_signed.bin of=/dev/sdc2 bs=4k
+sudo dd if=test-app/image_v1_signed.bin of=/dev/sdc3 bs=4k
+```
+
 **Boot Mode**
 
 Set the ZCU102 boot mode switches (SW6) for SD card boot:
@@ -2459,7 +2470,6 @@ make test-app/image.bin
 dd if=/dev/zero of=sdcard.img bs=1M count=1024
 sfdisk sdcard.img <<EOF
 label: dos
-unit: sectors
 
 1 : start=2048, size=128M, type=c, bootable
 2 : size=200M, type=83

hal/zynq.c

@@ -1596,6 +1596,24 @@ void hal_prepare_boot(void)
         mDev.qnx = NULL;
     }
 #endif
+
+    /* Clean and invalidate caches for the loaded application.
+     * The application was written to RAM via D-cache, but the CPU will
+     * fetch instructions via I-cache from main memory. We must:
+     * 1. Clean D-cache (flush dirty data to memory)
+     * 2. Invalidate I-cache (ensure fresh instruction fetch) */
+    __asm__ volatile("dsb sy");
+    {
+        uintptr_t addr;
+        uintptr_t end = WOLFBOOT_LOAD_ADDRESS + APP_CACHE_FLUSH_SIZE;
+        for (addr = WOLFBOOT_LOAD_ADDRESS; addr < end; addr += CACHE_LINE_SIZE) {
+            __asm__ volatile("dc cvac, %0" : : "r"(addr));
+        }
+    }
+    __asm__ volatile("dsb sy");
+    __asm__ volatile("ic iallu");
+    __asm__ volatile("dsb sy");
+    __asm__ volatile("isb");
 }
 
 /* Flash functions must be relocated to RAM for execution */
@@ -1866,6 +1884,16 @@ static int test_ext_flash(QspiDev_t* dev)
 
 #define CADENCE_SRS_OFFSET      0x200
 
+/* Legacy SDMA system address register (standard SDHCI offset 0x00).
+ * The Arasan SDHCI v3.0 on ZynqMP does not support Host Version 4 Enable
+ * (HV4E) mode. The generic SDHCI driver uses HV4E-style 64-bit DMA
+ * addressing via SRS22/SRS23 (offsets 0x58/0x5C), but on this controller
+ * we must use the legacy SRS00 register (offset 0x00) for SDMA addresses.
+ * The platform reg_read/reg_write functions transparently redirect
+ * SRS22 <-> SRS00, making legacy SDMA work without changes to sdhci.c. */
+#define STD_SDHCI_SDMA_ADDR     0x00  /* SDMA System Address (32-bit) */
+#define STD_SDHCI_HOST_CTRL2    0x3C  /* Auto CMD Err(16) + Host Ctrl 2(16) */
+
 /* Standard SDHCI register offsets (byte addresses within the controller) */
 #define STD_SDHCI_HOST_CTRL1    0x28  /* Host Control 1 (8-bit) */
 #define STD_SDHCI_POWER_CTRL    0x29  /* Power Control (8-bit) */
@@ -1939,8 +1967,25 @@ uint32_t sdhci_reg_read(uint32_t offset)
 
     /* Cadence SRS registers (0x200+) -> standard SDHCI (subtract 0x200) */
     if (offset >= CADENCE_SRS_OFFSET) {
-        /* 32-bit reads work for all SDHCI registers */
-        return *((volatile uint32_t *)(base + offset - CADENCE_SRS_OFFSET));
+        uint32_t std_off = offset - CADENCE_SRS_OFFSET;
+
+        /* SRS22 (0x58) -> SRS00 (0x00): Legacy SDMA address register */
+        if (std_off == 0x58) {
+            return *((volatile uint32_t *)(base + STD_SDHCI_SDMA_ADDR));
+        }
+        /* SRS23 (0x5C) -> 0: No 64-bit addressing on SDHCI v3.0 */
+        if (std_off == 0x5C) {
+            return 0;
+        }
+
+        {
+            uint32_t val = *((volatile uint32_t *)(base + std_off));
+            /* Mask out A64S from Capabilities to prevent HV4E init */
+            if (std_off == 0x40) { /* SRS16 - Capabilities */
+                val &= ~SDHCI_SRS16_A64S;
+            }
+            return val;
+        }
     }
     /* Cadence HRS registers (0x000-0x1FF) -> translate to standard equivalents */
     return zynqmp_sdhci_hrs_read(offset);
@@ -1984,6 +2029,27 @@ void sdhci_reg_write(uint32_t offset, uint32_t val)
             return;
         }
 
+        /* SRS22 (0x58) -> SRS00 (0x00): Legacy SDMA address register.
+         * The generic driver writes the DMA buffer address here for HV4E mode.
+         * Redirect to SRS00 which is the legacy SDMA system address register.
+         * Writing SRS00 also restarts DMA after a boundary interrupt. */
+        if (std_off == 0x58) {
+            *((volatile uint32_t *)(base + STD_SDHCI_SDMA_ADDR)) = val;
+            return;
+        }
+        /* SRS23 (0x5C) -> no-op: No 64-bit addressing on SDHCI v3.0 */
+        if (std_off == 0x5C) {
+            return;
+        }
+
+        /* SRS15 (0x3C) -> mask out HV4E and A64 bits.
+         * The generic driver enables HV4E for SDMA, but the Arasan SDHCI v3.0
+         * does not support it. These are reserved bits on v3.0 and must not
+         * be set. */
+        if (std_off == STD_SDHCI_HOST_CTRL2) {
+            val &= ~(SDHCI_SRS15_HV4E | SDHCI_SRS15_A64);
+        }
+
         /* All other SRS registers: 32-bit write */
         *((volatile uint32_t *)(base + std_off)) = val;
         return;
@@ -2010,22 +2076,35 @@ void sdhci_platform_init(void)
 {
     uint32_t reg;
     volatile int i;
+    uint32_t slot_mask;
+    uint32_t slot_shift;
+    uint32_t reset_bit;
 
-    /* Set SD1 slot type to "Embedded Slot for One Device" (01).
+    /* Set the selected SDx slot type to "Embedded Slot for One Device" (01).
      * This feeds into the SDHCI Capabilities register bits 31:30 and makes
      * the controller report card as always present, bypassing the physical
      * CD pin that is not connected to the SDHCI controller on ZCU102. */
+    if (ZYNQMP_SDHCI_BASE == ZYNQMP_SD0_BASE) {
+        slot_mask  = SD_CONFIG_REG2_SD0_SLOTTYPE_MASK;
+        slot_shift = SD_CONFIG_REG2_SD0_SLOTTYPE_SHIFT;
+        reset_bit  = RST_LPD_IOU2_SDIO0;
+    } else {
+        slot_mask  = SD_CONFIG_REG2_SD1_SLOTTYPE_MASK;
+        slot_shift = SD_CONFIG_REG2_SD1_SLOTTYPE_SHIFT;
+        reset_bit  = RST_LPD_IOU2_SDIO1;
+    }
+
     reg = IOU_SLCR_SD_CONFIG_REG2;
-    reg &= ~SD_CONFIG_REG2_SD1_SLOTTYPE_MASK;
-    reg |= (1UL << SD_CONFIG_REG2_SD1_SLOTTYPE_SHIFT); /* 01 = Embedded */
+    reg &= ~slot_mask;
+    reg |= (1UL << slot_shift); /* 01 = Embedded */
     IOU_SLCR_SD_CONFIG_REG2 = reg;
 
     /* The SDHCI Capabilities register latches IOU_SLCR values on controller
-     * reset. Issue SDIO1 reset via CRL_APB so the controller picks up the
+     * reset. Issue SDIOx reset via CRL_APB so the controller picks up the
      * new slot type configuration. */
-    RST_LPD_IOU2 |= RST_LPD_IOU2_SDIO1;   /* Assert SDIO1 reset */
+    RST_LPD_IOU2 |= reset_bit;              /* Assert SDIOx reset */
     for (i = 0; i < 100; i++) {}             /* Brief delay */
-    RST_LPD_IOU2 &= ~RST_LPD_IOU2_SDIO1;  /* De-assert SDIO1 reset */
+    RST_LPD_IOU2 &= ~reset_bit;             /* De-assert SDIOx reset */
     for (i = 0; i < 1000; i++) {}            /* Wait for controller ready */
 
 #ifdef DEBUG_SDHCI
@@ -2068,6 +2147,45 @@ void sdhci_platform_set_bus_mode(int is_emmc)
     wolfBoot_printf("sdhci_platform_set_bus_mode: is_emmc=%d\n", is_emmc);
 #endif
 }
+
+/* DMA cache maintenance - called from sdhci_transfer() around SDMA operations.
+ * The SDMA engine transfers data directly to/from physical memory, bypassing
+ * the CPU's L1/L2 caches. We must ensure cache coherency:
+ *   - Before DMA write (card←memory): clean D-cache so DMA reads correct data
+ *   - After DMA read (card→memory): invalidate D-cache so CPU sees new data */
+void sdhci_platform_dma_prepare(void *buf, uint32_t sz, int is_write)
+{
+    uintptr_t addr;
+    uintptr_t start = (uintptr_t)buf & ~(CACHE_LINE_SIZE - 1);
+    uintptr_t end = (uintptr_t)buf + sz;
+
+    if (is_write) {
+        /* Clean D-cache: flush dirty lines to memory for DMA to read */
+        for (addr = start; addr < end; addr += CACHE_LINE_SIZE) {
+            __asm__ volatile("dc cvac, %0" : : "r"(addr) : "memory");
+        }
+    } else {
+        /* Invalidate D-cache: discard stale lines before DMA writes to memory */
+        for (addr = start; addr < end; addr += CACHE_LINE_SIZE) {
+            __asm__ volatile("dc civac, %0" : : "r"(addr) : "memory");
+        }
+    }
+    __asm__ volatile("dsb sy" : : : "memory");
+}
+
+void sdhci_platform_dma_complete(void *buf, uint32_t sz, int is_write)
+{
+    (void)is_write;
+    /* After DMA: clean and invalidate to ensure CPU sees DMA-written data */
+    uintptr_t addr;
+    uintptr_t start = (uintptr_t)buf & ~(CACHE_LINE_SIZE - 1);
+    uintptr_t end = (uintptr_t)buf + sz;
+
+    for (addr = start; addr < end; addr += CACHE_LINE_SIZE) {
+        __asm__ volatile("dc civac, %0" : : "r"(addr) : "memory");
+    }
+    __asm__ volatile("dsb sy" : : : "memory");
+}
 #endif /* DISK_SDCARD || DISK_EMMC */
 
 

hal/zynq.h

@@ -32,6 +32,11 @@
 #ifndef EL2_HYPERVISOR
 #define EL2_HYPERVISOR 1
 #endif
+
+/* Cache line size for Cortex-A53 (AArch64) */
+#define CACHE_LINE_SIZE         64
+/* Region size to flush before booting application */
+#define APP_CACHE_FLUSH_SIZE    (1 * 1024 * 1024)  /* 1MB */
 #ifndef EL1_NONSECURE
 #define EL1_NONSECURE  0
 #endif

hal/zynq.ld

@@ -16,7 +16,7 @@ MEMORY
     *   Loaded by FSBL to end of DDR0
     *   For SD card boot see zynq_sd.ld
     */
-   psu_ddr_0_MEM_0 : ORIGIN = 0x7FF00000, LENGTH = 0x200000
+   psu_ddr_0_MEM_0 : ORIGIN = 0x7FE00000, LENGTH = 0x200000
    psu_ddr_1_MEM_0 : ORIGIN = 0x800000000, LENGTH = 0x80000000
    psu_ocm_ram_0_MEM_0 : ORIGIN = 0xFFFC0000, LENGTH = 0x40000
    psu_qspi_linear_0_MEM_0 : ORIGIN = 0xC0000000, LENGTH = 0x20000000

src/sdhci.c

@@ -34,6 +34,22 @@
 #include "hal.h"
 #include "disk.h"
 
+/* ============================================================================
+ * Platform DMA cache maintenance (weak defaults - override in HAL)
+ * ============================================================================ */
+#ifndef SDHCI_SDMA_DISABLED
+void __attribute__((weak)) sdhci_platform_dma_prepare(
+    void *buf, uint32_t sz, int is_write)
+{
+    (void)buf; (void)sz; (void)is_write;
+}
+void __attribute__((weak)) sdhci_platform_dma_complete(
+    void *buf, uint32_t sz, int is_write)
+{
+    (void)buf; (void)sz; (void)is_write;
+}
+#endif
+
 /* ============================================================================
  * Internal state
  * ============================================================================ */
@@ -368,7 +384,7 @@ static uint32_t sdhci_set_clock(uint32_t clock_khz)
     reg = SDHCI_REG(SDHCI_SRS11);
     reg &= ~(SDHCI_SRS11_SDCFSL_MASK | SDHCI_SRS11_SDCFSH_MASK);
     reg |= (((mclk & 0x0FF) << SDHCI_SRS11_SDCFSL_SHIFT) & SDHCI_SRS11_SDCFSL_MASK);  /* lower 8 bits */
-    reg |= (((mclk & 0x300) << SDHCI_SRS11_SDCFSH_SHIFT) & SDHCI_SRS11_SDCFSH_MASK); /* upper 2 bits */
+    reg |= (((mclk & 0x300) >> 2) & SDHCI_SRS11_SDCFSH_MASK);  /* upper 2 bits */
     reg |= SDHCI_SRS11_ICE; /* clock enable */
     reg &= ~SDHCI_SRS11_CGS; /* select clock */
     SDHCI_REG_SET(SDHCI_SRS11, reg);
@@ -1190,10 +1206,16 @@ static int sdhci_transfer(int dir, uint32_t cmd_index, uint32_t block_addr,
             /* SDMA mode with Host Version 4 enable.
              * HV4E is required for SDMA to use the 64-bit address registers
              * (SRS22/SRS23) instead of the legacy 32-bit register (SRS00).
-             * A64 is cleared in SRS15 to use 32-bit DMA addressing. */
+             * A64 is cleared in SRS15 to use 32-bit DMA addressing.
+             * Note: Platform may redirect SRS22/SRS23 to SRS00 for legacy
+             * SDMA on controllers that don't support HV4E. */
             sdhci_reg_or(SDHCI_SRS10, SDHCI_SRS10_DMA_SDMA);
             sdhci_reg_or(SDHCI_SRS15, SDHCI_SRS15_HV4E);
             sdhci_reg_and(SDHCI_SRS15, ~SDHCI_SRS15_A64);
+
+            /* Platform DMA cache maintenance before transfer */
+            sdhci_platform_dma_prepare(buf, sz, dir == SDHCI_DIR_WRITE);
+
             /* Set SDMA address */
             SDHCI_REG_SET(SDHCI_SRS22, (uint32_t)(uintptr_t)buf);
             SDHCI_REG_SET(SDHCI_SRS23, (uint32_t)(((uint64_t)(uintptr_t)buf) >> 32));
@@ -1230,6 +1252,9 @@ static int sdhci_transfer(int dir, uint32_t cmd_index, uint32_t block_addr,
             }
         }
         sdhci_disable_sdma_interrupts();
+
+        /* Platform DMA cache maintenance after transfer */
+        sdhci_platform_dma_complete(buf, sz, dir == SDHCI_DIR_WRITE);
     }
     else {
         /* Blocking mode - buffer ready flag differs for read vs write */