mem: allow virtual heaps to be cross-core

In case of cross core buffers a buffer may be
allocated and freed by diffrent core, depending on
which component is deleted second.
As all control structures of virtual heaps are
stored in uncached aliases, there's no technical
problems with allowing virtual heaps to work cross
core.
The only consideration is that in case of cross core
allocate/free the cache invalidation MUST be performed
on the core that was storing data. It is up to the
memory user to ensure this

Signed-off-by: Marcin Szkudlinski <marcin.szkudlinski@intel.com>

Author: marcinszkudlinski

zephyr/include/sof/lib/regions_mm.h

@@ -58,7 +58,6 @@
  *  governed by sys_mem_blocks API.
  *  @var allocation_sizes[] a table of bit arrays representing sizes of allocations
  *  made in physical_blocks_allocators directly related to physical_blocks_allocators
- *  @var core_id id of the core that heap was created on
  *  @var allocating_continuously configuration value deciding if heap allocations
  *  will be contiguous or single block.
  */
@@ -67,7 +66,6 @@ struct vmh_heap {
 	const struct sys_mm_drv_region *virtual_region;
 	struct sys_mem_blocks *physical_blocks_allocators[MAX_MEMORY_ALLOCATORS_COUNT];
 	struct sys_bitarray *allocation_sizes[MAX_MEMORY_ALLOCATORS_COUNT];
-	int core_id;
 	bool allocating_continuously;
 };
 
@@ -105,15 +103,15 @@ struct vmh_heap_config {
 };
 
 struct vmh_heap *vmh_init_heap(const struct vmh_heap_config *cfg,
-		int memory_region_attribute, int core_id, bool allocating_continuously);
+		int memory_region_attribute, bool allocating_continuously);
 void *vmh_alloc(struct vmh_heap *heap, uint32_t alloc_size);
 int vmh_free_heap(struct vmh_heap *heap);
 int vmh_free(struct vmh_heap *heap, void *ptr);
 struct vmh_heap *vmh_reconfigure_heap(struct vmh_heap *heap,
-		struct vmh_heap_config *cfg, int core_id, bool allocating_continuously);
+		struct vmh_heap_config *cfg, bool allocating_continuously);
 void vmh_get_default_heap_config(const struct sys_mm_drv_region *region,
 		struct vmh_heap_config *cfg);
-struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr, uint32_t core_id);
+struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr);
 /**
  * @brief Checks if ptr is in range of given memory range
  *

zephyr/lib/alloc.c

@@ -21,7 +21,7 @@
 #if CONFIG_VIRTUAL_HEAP
 #include <sof/lib/regions_mm.h>
 
-struct vmh_heap *virtual_buffers_heap[CONFIG_MP_MAX_NUM_CPUS];
+struct vmh_heap *virtual_buffers_heap;
 struct k_spinlock vmh_lock;
 
 #undef	HEAPMEM_SIZE
@@ -251,12 +251,11 @@ static bool is_virtual_heap_pointer(void *ptr)
 
 static void virtual_heap_free(void *ptr)
 {
-	struct vmh_heap *const heap = virtual_buffers_heap[cpu_get_id()];
 	int ret;
 
 	ptr = (__sparse_force void *)sys_cache_cached_ptr_get(ptr);
 
-	ret = vmh_free(heap, ptr);
+	ret = vmh_free(virtual_buffers_heap, ptr);
 	if (ret) {
 		tr_err(&zephyr_tr, "Unable to free %p! %d", ptr, ret);
 		k_panic();
@@ -279,17 +278,11 @@ static const struct vmh_heap_config static_hp_buffers = {
 
 static int virtual_heap_init(void)
 {
-	int core;
-
 	k_spinlock_init(&vmh_lock);
-
-	for (core = 0; core < CONFIG_MP_MAX_NUM_CPUS; core++) {
-		struct vmh_heap *heap = vmh_init_heap(&static_hp_buffers, MEM_REG_ATTR_CORE_HEAP,
-						      core, false);
-		if (!heap)
-			tr_err(&zephyr_tr, "Unable to init virtual heap for core %d!", core);
-
-		virtual_buffers_heap[core] = heap;
+	virtual_buffers_heap = vmh_init_heap(&static_hp_buffers, MEM_REG_ATTR_CORE_HEAP, false);
+	if (!virtual_buffers_heap) {
+		tr_err(&zephyr_tr, "Unable to init virtual heap");
+		return -ENOMEM;
 	}
 
 	return 0;
@@ -490,9 +483,6 @@ EXPORT_SYMBOL(rzalloc);
 void *rballoc_align(uint32_t flags, uint32_t caps, size_t bytes,
 		    uint32_t align)
 {
-#if CONFIG_VIRTUAL_HEAP
-	struct vmh_heap *virtual_heap;
-#endif
 	struct k_heap *heap;
 
 	/* choose a heap */
@@ -510,9 +500,8 @@ void *rballoc_align(uint32_t flags, uint32_t caps, size_t bytes,
 
 #if CONFIG_VIRTUAL_HEAP
 	/* Use virtual heap if it is available */
-	virtual_heap = virtual_buffers_heap[cpu_get_id()];
-	if (virtual_heap)
-		return virtual_heap_alloc(virtual_heap, flags, caps, bytes, align);
+	if (virtual_buffers_heap)
+		return virtual_heap_alloc(virtual_buffers_heap, flags, caps, bytes, align);
 #endif /* CONFIG_VIRTUAL_HEAP */
 
 	if (flags & SOF_MEM_FLAG_COHERENT)

zephyr/lib/regions_mm.c

@@ -28,14 +28,14 @@ static struct list_item vmh_list = LIST_INIT(vmh_list);
  * @retval NULL on creation failure.
  */
 struct vmh_heap *vmh_init_heap(const struct vmh_heap_config *cfg,
-	int memory_region_attribute, int core_id, bool allocating_continuously)
+	int memory_region_attribute, bool allocating_continuously)
 {
 	const struct sys_mm_drv_region *virtual_memory_regions =
 		sys_mm_drv_query_memory_regions();
 	int i;
 
 	/* Check if we haven't created heap for this region already */
-	if (vmh_get_heap_by_attribute(memory_region_attribute, core_id))
+	if (vmh_get_heap_by_attribute(memory_region_attribute))
 		return NULL;
 
 	struct vmh_heap *new_heap =
@@ -44,7 +44,6 @@ struct vmh_heap *vmh_init_heap(const struct vmh_heap_config *cfg,
 	if (!new_heap)
 		return NULL;
 
-	new_heap->core_id = core_id;
 	list_init(&new_heap->node);
 	struct vmh_heap_config new_config = {0};
 
@@ -54,7 +53,7 @@ struct vmh_heap *vmh_init_heap(const struct vmh_heap_config *cfg,
 	 * available cores
 	 */
 	if (memory_region_attribute == MEM_REG_ATTR_CORE_HEAP) {
-		new_heap->virtual_region = &virtual_memory_regions[core_id];
+		new_heap->virtual_region = &virtual_memory_regions[0];
 	} else {
 		for (i = CONFIG_MP_MAX_NUM_CPUS;
 			i < CONFIG_MP_MAX_NUM_CPUS + VIRTUAL_REGION_COUNT; i++) {
@@ -393,8 +392,6 @@ void *vmh_alloc(struct vmh_heap *heap, uint32_t alloc_size)
 	if (!alloc_size)
 		return NULL;
 	/* Only operations on the same core are allowed */
-	if (heap->core_id != cpu_get_id())
-		return NULL;
 
 	void *ptr = NULL;
 	int mem_block_iterator, allocation_error_code = -ENOMEM;
@@ -560,9 +557,6 @@ int vmh_free(struct vmh_heap *heap, void *ptr)
 {
 	int retval;
 
-	if (heap->core_id != cpu_get_id())
-		return -EINVAL;
-
 	size_t mem_block_iter, i, size_to_free, block_size, ptr_bit_array_offset,
 		ptr_bit_array_position, blocks_to_free;
 	bool ptr_range_found;
@@ -691,15 +685,14 @@ int vmh_free(struct vmh_heap *heap, void *ptr)
  * @retval NULL when reconfiguration failed
  */
 struct vmh_heap *vmh_reconfigure_heap(
-	struct vmh_heap *heap, struct vmh_heap_config *cfg,
-	int core_id, bool allocating_continuously)
+	struct vmh_heap *heap, struct vmh_heap_config *cfg, bool allocating_continuously)
 {
 	uint32_t region_attribute = heap->virtual_region->attr;
 
 	if (vmh_free_heap(heap))
 		return NULL;
 
-	return vmh_init_heap(cfg, region_attribute, core_id, allocating_continuously);
+	return vmh_init_heap(cfg, region_attribute, allocating_continuously);
 }
 
 /**
@@ -740,7 +733,7 @@ void vmh_get_default_heap_config(const struct sys_mm_drv_region *region,
  * @retval heap ptr on success
  * @retval NULL if there was no heap created fitting the attr.
  */
-struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr, uint32_t core_id)
+struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr)
 {
 	struct list_item *vm_heaps_iterator;
 	struct vmh_heap *retval;
@@ -753,7 +746,7 @@ struct vmh_heap *vmh_get_heap_by_attribute(uint32_t attr, uint32_t core_id)
 		const struct sys_mm_drv_region *virtual_memory_region =
 			sys_mm_drv_query_memory_regions();
 		/* we move ptr to cpu vmr */
-		virtual_memory_region = &virtual_memory_region[core_id];
+		virtual_memory_region = &virtual_memory_region[0];
 
 		list_for_item(vm_heaps_iterator, &vmh_list) {
 			retval =

zephyr/test/vmh.c

@@ -21,12 +21,11 @@ LOG_MODULE_DECLARE(sof_boot_test, CONFIG_SOF_LOG_LEVEL);
 /* Test creating and freeing a virtual memory heap */
 static void test_vmh_init_and_free_heap(int memory_region_attribute,
 					struct vmh_heap_config *config,
-					int core_id,
 					bool allocating_continuously,
 					bool expect_success)
 {
 	struct vmh_heap *heap = vmh_init_heap(config, memory_region_attribute,
-		core_id, allocating_continuously);
+					      allocating_continuously);
 	if (expect_success) {
 		zassert_not_null(heap,
 		"Heap initialization expected to succeed but failed");
@@ -146,7 +145,7 @@ static void test_vmh_multiple_allocs(struct vmh_heap *heap, int num_allocs,
 static void test_vmh_alloc_multiple_times(bool allocating_continuously)
 {
 	struct vmh_heap *heap =
-	vmh_init_heap(NULL, MEM_REG_ATTR_CORE_HEAP, 0, allocating_continuously);
+	vmh_init_heap(NULL, MEM_REG_ATTR_CORE_HEAP, allocating_continuously);
 
 	zassert_not_null(heap, "Heap initialization failed");
 
@@ -170,7 +169,7 @@ static void test_vmh_alloc_multiple_times(bool allocating_continuously)
 static void test_vmh_alloc_free(bool allocating_continuously)
 {
 	struct vmh_heap *heap =
-	vmh_init_heap(NULL, MEM_REG_ATTR_CORE_HEAP, 0, allocating_continuously);
+	vmh_init_heap(NULL, MEM_REG_ATTR_CORE_HEAP, allocating_continuously);
 
 	zassert_not_null(heap, "Heap initialization failed");
 
@@ -223,7 +222,7 @@ static void test_alloc_on_configured_heap(bool allocating_continuously)
 
 	/* Create continuous allocation heap for success test */
 	struct vmh_heap *heap =
-	vmh_init_heap(&config, MEM_REG_ATTR_CORE_HEAP, 0, allocating_continuously);
+	vmh_init_heap(&config, MEM_REG_ATTR_CORE_HEAP, allocating_continuously);
 
 	/* Will succeed on continuous and fail with single block alloc */
 	test_vmh_alloc_free_check(heap, 512, allocating_continuously);
@@ -248,8 +247,7 @@ static void test_vmh_init_all_heaps(void)
 		if (!virtual_memory_region[i].size)
 			break;
 
-		struct vmh_heap *heap = vmh_init_heap(NULL, virtual_memory_region[i].attr,
-		i, true);
+		struct vmh_heap *heap = vmh_init_heap(NULL, virtual_memory_region[i].attr, true);
 
 		zassert_not_null(heap, "Heap initialization expected to succeed but failed");