alloc: userspace: Introduce MMU shared memory heap

In case of CONFIG_USERSPACE enabled, all memory is accessible form kernel
space. For accessing it from user space we need to grant access to its
explicitly. However there are some infrastructure components that could be
shared between kernel and user spaces.

This patch splits system heap into two sections. One section which will
contatin shareable data will be located in separate memory partition.
Each non-privilidged module will obtain it in its private memory domain.

Signed-off-by: Jaroslaw Stelter <Jaroslaw.Stelter@intel.com>
Signed-off-by: Adrian Warecki <adrian.warecki@intel.com>

Author: jxstelter

posix/include/rtos/alloc.h

@@ -33,19 +33,21 @@
  */
 
  /** \brief Indicates we should return DMA-able memory. */
-#define SOF_MEM_FLAG_DMA	BIT(0)
+#define SOF_MEM_FLAG_DMA		BIT(0)
 /** \brief Indicates that original content should not be copied by realloc. */
-#define SOF_MEM_FLAG_NO_COPY	BIT(1)
+#define SOF_MEM_FLAG_NO_COPY		BIT(1)
 /** \brief Indicates that if we should return uncached address. */
-#define SOF_MEM_FLAG_COHERENT	BIT(2)
+#define SOF_MEM_FLAG_COHERENT		BIT(2)
 /** \brief Indicates that if we should return L3 address. */
-#define SOF_MEM_FLAG_L3		BIT(3)
+#define SOF_MEM_FLAG_L3			BIT(3)
 /** \brief Indicates that if we should return Low power memory address. */
-#define SOF_MEM_FLAG_LOW_POWER	BIT(4)
+#define SOF_MEM_FLAG_LOW_POWER		BIT(4)
 /** \brief Indicates that if we should return kernel memory address. */
-#define SOF_MEM_FLAG_KERNEL	BIT(5)
+#define SOF_MEM_FLAG_KERNEL		BIT(5)
 /** \brief Indicates that if we should return user memory address. */
-#define SOF_MEM_FLAG_USER	BIT(6)
+#define SOF_MEM_FLAG_USER		BIT(6)
+/** \brief Indicates that if we should return shared user memory address. */
+#define SOF_MEM_FLAG_USER_SHARED	BIT(7)
 
 /** @} */
 

zephyr/Kconfig

@@ -41,6 +41,18 @@ config SOF_ZEPHYR_HEAP_SIZE
 	 NOTE: Keep in mind that the heap size should not be greater than the physical
 	 memory size of the system defined in DT (and this includes baseFW text/data).
 
+config SOF_ZEPHYR_SHARED_HEAP_SIZE
+	hex "Size of the Zephyr shared heap for SOF userspace modules"
+	default 0x1E000 if SOC_INTEL_ACE15_MTPM || SOC_INTEL_ACE20_LNL
+	default 0x1A000 if SOC_INTEL_ACE30
+	default 0x0
+	help
+	 The size of the zephyr heap portion designated as the shared heap. This heap
+	 is shared between the sof and userspace modules. It is used exclusively for
+	 allocating audio buffers between the kernel and userspace modules.
+	 NOTE: Keep in mind that the shared heap size should not be greater than the
+	 heap size.
+
 config SOF_ZEPHYR_VIRTUAL_HEAP_SIZE
 	hex "Size of the Zephyr virtual heap for SOF"
 	depends on VIRTUAL_HEAP

zephyr/include/rtos/alloc.h

@@ -24,19 +24,21 @@
  */
 
  /** \brief Indicates we should return DMA-able memory. */
-#define SOF_MEM_FLAG_DMA	BIT(0)
+#define SOF_MEM_FLAG_DMA		BIT(0)
 /** \brief Indicates that original content should not be copied by realloc. */
-#define SOF_MEM_FLAG_NO_COPY	BIT(1)
+#define SOF_MEM_FLAG_NO_COPY		BIT(1)
 /** \brief Indicates that if we should return uncached address. */
-#define SOF_MEM_FLAG_COHERENT	BIT(2)
+#define SOF_MEM_FLAG_COHERENT		BIT(2)
 /** \brief Indicates that if we should return L3 address. */
-#define SOF_MEM_FLAG_L3		BIT(3)
+#define SOF_MEM_FLAG_L3			BIT(3)
 /** \brief Indicates that if we should return Low power memory address. */
-#define SOF_MEM_FLAG_LOW_POWER	BIT(4)
+#define SOF_MEM_FLAG_LOW_POWER		BIT(4)
 /** \brief Indicates that if we should return kernel memory address. */
-#define SOF_MEM_FLAG_KERNEL	BIT(5)
+#define SOF_MEM_FLAG_KERNEL		BIT(5)
 /** \brief Indicates that if we should return user memory address. */
-#define SOF_MEM_FLAG_USER	BIT(6)
+#define SOF_MEM_FLAG_USER		BIT(6)
+/** \brief Indicates that if we should return shared user memory address. */
+#define SOF_MEM_FLAG_USER_SHARED	BIT(7)
 
 /** @} */
 
@@ -111,4 +113,21 @@ static inline void heap_trace_all(int force) {}
 
 /** @}*/
 
+#if CONFIG_USERSPACE
+/**
+ * Returns the start address of shared memory heap.
+ *
+ * @return pointer to shared memory heap start
+ */
+uintptr_t get_shared_heap_start(void);
+
+/**
+ * Returns the size of shared memory heap.
+ *
+ * @return size of shared memory heap start
+ */
+size_t get_shared_heap_size(void);
+
+#endif
+
 #endif /* __ZEPHYR_RTOS_ALLOC_H__ */

zephyr/lib/alloc.c

@@ -19,6 +19,8 @@
 #include <rtos/symbol.h>
 #include <rtos/wait.h>
 
+#define SHARED_HEAP_MEM_SIZE	0
+
 #if CONFIG_VIRTUAL_HEAP
 #include <sof/lib/regions_mm.h>
 
@@ -98,7 +100,14 @@ static uint8_t __aligned(PLATFORM_DCACHE_ALIGN) heapmem[HEAPMEM_SIZE];
  * to allow memory management driver to control unused
  * memory pages.
  */
-__section(".heap_mem") static uint8_t __aligned(PLATFORM_DCACHE_ALIGN) heapmem[HEAPMEM_SIZE];
+#if CONFIG_USERSPACE
+#undef SHARED_HEAP_MEM_SIZE
+#define SHARED_HEAP_MEM_SIZE	ROUND_UP(CONFIG_SOF_ZEPHYR_SHARED_HEAP_SIZE, HOST_PAGE_SIZE)
+__section(".shared_heap_mem")
+static uint8_t __aligned(HOST_PAGE_SIZE) shared_heapmem[SHARED_HEAP_MEM_SIZE];
+#endif /* CONFIG_USERSPACE */
+__section(".heap_mem") static uint8_t __aligned(HOST_PAGE_SIZE) heapmem[HEAPMEM_SIZE -
+									SHARED_HEAP_MEM_SIZE];
 
 #elif defined(CONFIG_ARCH_POSIX)
 
@@ -125,6 +134,40 @@ extern char _end[], _heap_sentry[];
 
 static struct k_heap sof_heap;
 
+#if CONFIG_USERSPACE
+static struct k_heap shared_heap;
+
+static bool is_shd_heap_pointer(void *ptr)
+{
+	uintptr_t shd_heap_start = POINTER_TO_UINT(shared_heapmem);
+	uintptr_t shd_heap_end = POINTER_TO_UINT(shared_heapmem + SHARED_HEAP_MEM_SIZE);
+
+	if (is_cached(ptr))
+		ptr = sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *)ptr);
+
+	return (POINTER_TO_UINT(ptr) >= shd_heap_start) && (POINTER_TO_UINT(ptr) < shd_heap_end);
+}
+
+/**
+ * Returns the start of HPSRAM Shared memory heap.
+ * @return Pointer to the HPSRAM Shared memory location which can be used
+ * for HPSRAM Shared heap.
+ */
+uintptr_t get_shared_heap_start(void)
+{
+	return ROUND_UP(POINTER_TO_UINT(shared_heapmem), HOST_PAGE_SIZE);
+}
+
+/**
+ * Returns the size of HPSRAM Shared memory heap.
+ * @return Size of the HPSRAM Shared memory region which can be used for HPSRAM Shared heap.
+ */
+size_t get_shared_heap_size(void)
+{
+	return ROUND_DOWN(SHARED_HEAP_MEM_SIZE, HOST_PAGE_SIZE);
+}
+#endif /* CONFIG_USERSPACE */
+
 #if CONFIG_L3_HEAP
 static struct k_heap l3_heap;
 static struct k_heap l3_heap_copy __imrdata;
@@ -243,15 +286,17 @@ static void *virtual_heap_alloc(struct vmh_heap *heap, uint32_t flags, size_t by
 	return mem;
 }
 
+extern int _unused_ram_start_marker;
+
 /**
  * Checks whether pointer is from virtual memory range.
  * @param ptr Pointer to memory being checked.
  * @return True if pointer falls into virtual memory region, false otherwise.
  */
 static bool is_virtual_heap_pointer(void *ptr)
 {
-	uintptr_t virtual_heap_start = POINTER_TO_UINT(sys_cache_cached_ptr_get(&heapmem)) +
-				       HEAPMEM_SIZE;
+	uintptr_t virtual_heap_start = POINTER_TO_UINT(
+		sys_cache_cached_ptr_get(&_unused_ram_start_marker));
 	uintptr_t virtual_heap_end = CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_VM_SIZE;
 
 	if (!is_cached(ptr))
@@ -393,6 +438,10 @@ void *rmalloc(uint32_t flags, size_t bytes)
 		return ptr;
 #else
 		k_panic();
+#endif
+#if CONFIG_USERSPACE
+	} else if (flags & SOF_MEM_FLAG_USER_SHARED) {
+		heap = &shared_heap;
 #endif
 	} else {
 		heap = &sof_heap;
@@ -477,16 +526,20 @@ void *rballoc_align(uint32_t flags, size_t bytes,
 		tr_err(&zephyr_tr, "L3_HEAP not available.");
 		return NULL;
 #endif
+#if CONFIG_USERSPACE
+	} else if (flags & SOF_MEM_FLAG_USER_SHARED) {
+		heap = &shared_heap;
+#endif /* CONFIG_USERSPACE */
 	} else {
-		heap = &sof_heap;
-	}
-
 #if CONFIG_VIRTUAL_HEAP
 	/* Use virtual heap if it is available */
 	if (virtual_buffers_heap)
 		return virtual_heap_alloc(virtual_buffers_heap, flags, bytes, align);
 #endif /* CONFIG_VIRTUAL_HEAP */
 
+		heap = &sof_heap;
+	}
+
 	if (flags & SOF_MEM_FLAG_COHERENT)
 		return heap_alloc_aligned(heap, align, bytes);
 
@@ -516,13 +569,24 @@ void rfree(void *ptr)
 	}
 #endif
 
+#if CONFIG_USERSPACE
+	if (is_shd_heap_pointer(ptr)) {
+		heap_free(&shared_heap, ptr);
+		return;
+	}
+#endif
+
 	heap_free(&sof_heap, ptr);
 }
 EXPORT_SYMBOL(rfree);
 
 static int heap_init(void)
 {
-	sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE);
+	sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE - SHARED_HEAP_MEM_SIZE);
+
+#if CONFIG_USERSPACE
+	sys_heap_init(&shared_heap.heap, shared_heapmem, SHARED_HEAP_MEM_SIZE);
+#endif
 
 #if CONFIG_L3_HEAP
 	if (l3_heap_copy.heap.heap)