Move other uses of asm::inner to modules using them.

This removes the inner module entirely.

Author: wt

cortex-m/src/asm.rs

@@ -7,8 +7,6 @@ use core::arch::asm;
 #[cfg(cortex_m)]
 use core::sync::atomic::{Ordering, compiler_fence};
 
-pub mod inner;
-
 /// Puts the processor in Debug state. Debuggers can pick this up as a "breakpoint".
 ///
 /// **NOTE** calling `bkpt` when the processor is not connected to a debugger will cause an

cortex-m/src/interrupt.rs

@@ -1,5 +1,8 @@
 //! Interrupts
 
+use core::arch::asm;
+use core::sync::atomic::{Ordering, compiler_fence};
+
 pub use bare_metal::{CriticalSection, Mutex, Nr};
 
 /// Trait for enums of external interrupt numbers.
@@ -35,7 +38,10 @@ unsafe impl<T: Nr + Copy> InterruptNumber for T {
 /// Disables all interrupts
 #[inline]
 pub fn disable() {
-    unsafe { crate::asm::inner::__cpsid() };
+    unsafe { asm!("cpsid i", options(nomem, nostack, preserves_flags)) };
+
+    // Ensure no subsequent memory accesses are reordered to before interrupts are disabled.
+    compiler_fence(Ordering::SeqCst);
 }
 
 /// Enables all the interrupts
@@ -44,8 +50,12 @@ pub fn disable() {
 ///
 /// - Do not call this function inside an `interrupt::free` critical section
 #[inline]
+#[cortex_m_macros::asm_cfg(any(armv6m, armv7m, armv7em, armv8m))]
 pub unsafe fn enable() {
-    unsafe { crate::asm::inner::__cpsie() };
+    // Ensure no preceeding memory accesses are reordered to after interrupts are enabled.
+    compiler_fence(Ordering::SeqCst);
+
+    unsafe { asm!("cpsie i", options(nomem, nostack, preserves_flags)) };
 }
 
 /// Execute closure `f` in an interrupt-free context.

cortex-m/src/peripheral/scb.rs

@@ -1,6 +1,10 @@
 //! System Control Block
 
+#[cfg(any(armv7m, armv8m))]
+use core::arch::asm;
 use core::ptr;
+#[cfg(any(armv7m, armv8m))]
+use core::sync::atomic::{Ordering, compiler_fence};
 
 use volatile_register::RW;
 
@@ -304,6 +308,7 @@ impl SCB {
     ///
     /// This operation first invalidates the entire I-cache.
     #[inline]
+    #[cortex_m_macros::asm_cfg(any(armv7m, armv7em, armv8m))]
     pub fn enable_icache(&mut self) {
         // Don't do anything if I-cache is already enabled
         if Self::icache_enabled() {
@@ -318,10 +323,25 @@ impl SCB {
 
         // NOTE(unsafe): The asm routine manages exclusive access to the SCB
         // registers and applies the proper barriers; it is technically safe on
-        // its own, and is only `unsafe` here because it's `extern "C"`.
+        // its own, and is only `unsafe` here because it's asm.
         unsafe {
-            crate::asm::inner::__enable_icache();
-        }
+            asm!(
+                "ldr {0}, =0xE000ED14",         // CCR
+                "mrs {2}, PRIMASK",             // save critical nesting info
+                "cpsid i",                      // mask interrupts
+                "ldr {1}, [{0}]",               // read CCR
+                "orr.w {1}, {1}, #(1 << 17)",   // Set bit 17, IC
+                "str {1}, [{0}]",               // write it back
+                "dsb",                          // ensure store completes
+                "isb",                          // synchronize pipeline
+                "msr PRIMASK, {2}",             // unnest critical section
+                out(reg) _,
+                out(reg) _,
+                out(reg) _,
+                options(nostack),
+            )
+        };
+        compiler_fence(Ordering::SeqCst);
     }
 
     /// Disables I-cache if currently enabled.
@@ -360,6 +380,7 @@ impl SCB {
 
     /// Invalidates the entire I-cache.
     #[inline]
+    #[cortex_m_macros::asm_cfg(any(armv6m, armv7m, armv7em, armv8m))]
     pub fn invalidate_icache(&mut self) {
         // NOTE(unsafe): No races as all CBP registers are write-only and stateless
         let mut cbp = unsafe { CBP::new() };
@@ -376,6 +397,7 @@ impl SCB {
     /// This operation first invalidates the entire D-cache, ensuring it does
     /// not contain stale values before being enabled.
     #[inline]
+    #[cortex_m_macros::asm_cfg(any(armv6m, armv7m, armv7em, armv8m))]
     pub fn enable_dcache(&mut self, cpuid: &mut CPUID) {
         // Don't do anything if D-cache is already enabled
         if Self::dcache_enabled() {
@@ -387,10 +409,26 @@ impl SCB {
 
         // NOTE(unsafe): The asm routine manages exclusive access to the SCB
         // registers and applies the proper barriers; it is technically safe on
-        // its own, and is only `unsafe` here because it's `extern "C"`.
+        // its own, and is only `unsafe` here because it's asm.
         unsafe {
-            crate::asm::inner::__enable_dcache();
-        }
+            asm!(
+                // Should this be replaced with a register modify?
+                "ldr {0}, =0xE000ED14",         // CCR
+                "mrs {2}, PRIMASK",             // save critical nesting info
+                "cpsid i",                      // mask interrupts
+                "ldr {1}, [{0}]",               // read CCR
+                "orr.w {1}, {1}, #(1 << 16)",   // Set bit 16, DC
+                "str {1}, [{0}]",               // write it back
+                "dsb",                          // ensure store completes
+                "isb",                          // synchronize pipeline
+                "msr PRIMASK, {2}",             // unnest critical section
+                out(reg) _,
+                out(reg) _,
+                out(reg) _,
+                options(nostack),
+            )
+        };
+        compiler_fence(Ordering::SeqCst);
     }
 
     /// Disables D-cache if currently enabled.
@@ -429,6 +467,7 @@ impl SCB {
     ///
     /// It's used immediately before enabling the dcache, but not exported publicly.
     #[inline]
+    #[cfg(cortex_m)]
     unsafe fn invalidate_dcache(&mut self, cpuid: &mut CPUID) {
         unsafe {
             // NOTE(unsafe): No races as all CBP registers are write-only and stateless

cortex-m/src/psp.rs

@@ -79,8 +79,7 @@ impl<const N: usize> core::default::Default for Stack<N> {
 /// In Unprivileged Mode, code can no longer perform privileged operations,
 /// such as disabling interrupts.
 ///
-pub fn switch_to_unprivileged_psp(psp_stack: StackHandle, function: extern "C" fn() -> !) -> ! {
-    let mut psp_stack = psp_stack;
+pub fn switch_to_unprivileged_psp(mut psp_stack: StackHandle, function: extern "C" fn() -> !) -> ! {
     // set the stack limit
     #[cfg(armv8m_main)]
     unsafe {
@@ -93,8 +92,7 @@ pub fn switch_to_unprivileged_psp(psp_stack: StackHandle, function: extern "C" f
 }
 
 /// Switch to running on the Process Stack Pointer (PSP), but remain in privileged mode
-pub fn switch_to_privileged_psp(psp_stack: StackHandle, function: extern "C" fn() -> !) -> ! {
-    let mut psp_stack = psp_stack;
+pub fn switch_to_privileged_psp(mut psp_stack: StackHandle, function: extern "C" fn() -> !) -> ! {
     // set the stack limit
     #[cfg(armv8m_main)]
     unsafe {

cortex-m/src/register/apsr.rs

@@ -1,5 +1,8 @@
 //! Application Program Status Register
 
+#[cfg(any(armv6m, armv7m, armv7em, armv8m))]
+use core::arch::asm;
+
 /// Application Program Status Register
 #[derive(Clone, Copy, Debug)]
 pub struct Apsr {
@@ -48,7 +51,9 @@ impl Apsr {
 ///
 /// **NOTE** This function is available if `cortex-m` is built with the `"inline-asm"` feature.
 #[inline]
+#[cortex_m_macros::asm_cfg(any(armv6m, armv7m, armv7em, armv8m))]
 pub fn read() -> Apsr {
-    let bits = unsafe { crate::asm::inner::__apsr_r() };
+    let bits;
+    unsafe { asm!("mrs {}, APSR", out(reg) bits, options(nomem, nostack, preserves_flags)) };
     Apsr { bits }
 }

cortex-m/src/register/basepri.rs

@@ -1,24 +1,46 @@
 //! Base Priority Mask Register
 
+#[cfg(any(armv7m, armv8m))]
+use core::arch::asm;
+
 /// Reads the CPU register
 #[inline]
+#[cortex_m_macros::asm_cfg(any(armv7m, armv8m_main))]
 pub fn read() -> u8 {
-    unsafe { crate::asm::inner::__basepri_r() }
+    let r;
+    unsafe { asm!("mrs {}, BASEPRI", out(reg) r, options(nomem, nostack, preserves_flags)) };
+    r
 }
 
 /// Writes to the CPU register
 ///
 /// **IMPORTANT** If you are using a Cortex-M7 device with revision r0p1 you MUST enable the
 /// `cm7-r0p1` Cargo feature or this function WILL misbehave.
 #[inline]
+#[cortex_m_macros::asm_cfg(any(armv7m, armv8m_main))]
 pub unsafe fn write(basepri: u8) {
-    #[cfg(feature = "cm7-r0p1")]
+    #[cfg(not(feature = "cm7-r0p1"))]
     {
-        unsafe { crate::asm::inner::__basepri_w_cm7_r0p1(basepri) }
+        unsafe {
+            asm!("msr BASEPRI, {}", in(reg) basepri, options(nomem, nostack, preserves_flags))
+        };
     }
 
-    #[cfg(not(feature = "cm7-r0p1"))]
+    #[cfg(feature = "cm7-r0p1")]
     {
-        unsafe { crate::asm::inner::__basepri_w(basepri) }
+        unsafe {
+            asm!(
+                "mrs {1}, PRIMASK",
+                "cpsid i",
+                "tst.w {1}, #1",
+                "msr BASEPRI, {0}",
+                "it ne",
+                "bxne lr",
+                "cpsie i",
+                in(reg) basepri,
+                out(reg) _,
+                options(nomem, nostack, preserves_flags),
+            )
+        };
     }
 }

cortex-m/src/register/basepri_max.rs

@@ -1,5 +1,8 @@
 //! Base Priority Mask Register (conditional write)
 
+#[cfg(any(armv7m, armv8m))]
+use core::arch::asm;
+
 /// Writes to BASEPRI *if*
 ///
 /// - `basepri != 0` AND `basepri::read() == 0`, OR
@@ -8,14 +11,30 @@
 /// **IMPORTANT** If you are using a Cortex-M7 device with revision r0p1 you MUST enable the
 /// `cm7-r0p1` Cargo feature or this function WILL misbehave.
 #[inline]
+#[cortex_m_macros::asm_cfg(any(armv7m, armv8m_main))]
 pub fn write(basepri: u8) {
-    #[cfg(feature = "cm7-r0p1")]
+    #[cfg(not(feature = "cm7-r0p1"))]
     {
-        unsafe { crate::asm::inner::__basepri_max_cm7_r0p1(basepri) }
+        unsafe {
+            asm!("msr BASEPRI_MAX, {}", in(reg) basepri, options(nomem, nostack, preserves_flags))
+        };
     }
 
-    #[cfg(not(feature = "cm7-r0p1"))]
+    #[cfg(feature = "cm7-r0p1")]
     {
-        unsafe { crate::asm::inner::__basepri_max(basepri) }
+        unsafe {
+            asm!(
+                "mrs {1}, PRIMASK",
+                "cpsid i",
+                "tst.w {1}, #1",
+                "msr BASEPRI_MAX, {0}",
+                "it ne",
+                "bxne lr",
+                "cpsie i",
+                in(reg) basepri,
+                out(reg) _,
+                options(nomem, nostack, preserves_flags),
+            )
+        };
     }
 }

cortex-m/src/register/control.rs

@@ -1,5 +1,10 @@
 //! Control register
 
+#[cfg(any(armv6m, armv7m, armv7em, armv8m))]
+use core::arch::asm;
+#[cfg(any(armv6m, armv7m, armv7em, armv8m))]
+use core::sync::atomic::{Ordering, compiler_fence};
+
 /// Control register
 #[derive(Clone, Copy, Debug)]
 pub struct Control {
@@ -173,14 +178,28 @@ impl Fpca {
 
 /// Reads the CPU register
 #[inline]
+#[cortex_m_macros::asm_cfg(any(armv6m, armv7m, armv7em, armv8m))]
 pub fn read() -> Control {
-    let bits = unsafe { crate::asm::inner::__control_r() };
+    let bits;
+    unsafe { asm!("mrs {}, CONTROL", out(reg) bits, options(nomem, nostack, preserves_flags)) };
     Control { bits }
 }
 
 /// Writes to the CPU register.
 #[inline]
+#[cortex_m_macros::asm_cfg(any(armv6m, armv7m, armv7em, armv8m))]
 pub unsafe fn write(control: Control) {
     let control = control.bits();
-    unsafe { crate::asm::inner::__control_w(control) };
+
+    // ISB is required after writing to CONTROL,
+    // per ARM architectural requirements (see Application Note 321).
+    unsafe {
+        asm!(
+            "msr CONTROL, {}",
+            "isb",
+            in(reg) control,
+            options(nomem, nostack, preserves_flags),
+        );
+        compiler_fence(Ordering::SeqCst);
+    }
 }

cortex-m/src/register/faultmask.rs

@@ -1,5 +1,8 @@
 //! Fault Mask Register
 
+#[cfg(any(armv6m, armv7m, armv7em, armv8m))]
+use core::arch::asm;
+
 /// All exceptions are ...
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum Faultmask {
@@ -25,8 +28,10 @@ impl Faultmask {
 
 /// Reads the CPU register
 #[inline]
+#[cortex_m_macros::asm_cfg(any(armv7m, armv8m_main))]
 pub fn read() -> Faultmask {
-    let r = unsafe { crate::asm::inner::__faultmask_r() };
+    let r: u32;
+    unsafe { asm!("mrs {}, FAULTMASK", out(reg) r, options(nomem, nostack, preserves_flags)) };
     if r & (1 << 0) == (1 << 0) {
         Faultmask::Inactive
     } else {

cortex-m/src/register/fpscr.rs

@@ -1,5 +1,8 @@
 //! Floating-point Status Control Register
 
+#[cfg(has_fpu)]
+use core::arch::asm;
+
 /// Floating-point Status Control Register
 #[derive(Clone, Copy, Debug)]
 pub struct Fpscr {
@@ -292,14 +295,17 @@ impl RMode {
 
 /// Read the FPSCR register
 #[inline]
+#[cfg(has_fpu)]
 pub fn read() -> Fpscr {
-    let r = unsafe { crate::asm::inner::__fpscr_r() };
+    let r;
+    unsafe { asm!("vmrs {}, fpscr", out(reg) r, options(nomem, nostack, preserves_flags)) };
     Fpscr::from_bits(r)
 }
 
 /// Set the value of the FPSCR register
 #[inline]
+#[cfg(has_fpu)]
 pub unsafe fn write(fpscr: Fpscr) {
     let fpscr = fpscr.bits();
-    unsafe { crate::asm::inner::__fpscr_w(fpscr) };
+    unsafe { asm!("vmsr fpscr, {}", in(reg) fpscr, options(nomem, nostack)) };
 }