audio: make shared buffers uncached for user-space threads

To ease moving audio code to user-space, modify the logic for allocating
shared buffers so that if a buffer is shared (accessed by multiple
cores, or by DSP core and separate hardware blocks), and code is
run in user-space, the buffer is allocated uncached.

Similarly, in invalidate and writeback functions, check for matching
conditions and skip the cache operations accordingly.

This approach allows to reuse most of the audio module code in
user-space and even have shared buffers between different user-space
threads running on different cores.

There is no impact to builds without user-space, or when running
modules in user-space without shared buffers.

Signed-off-by: Kai Vehmanen <kai.vehmanen@linux.intel.com>

Author: kv2019i

src/audio/buffers/comp_buffer.c

@@ -234,6 +234,15 @@ struct comp_buffer *buffer_alloc(size_t size, uint32_t flags, uint32_t align,
 
 	tr_dbg(&buffer_tr, "buffer_alloc()");
 
+#if CONFIG_USERSPACE
+	/*
+	 * cache control is not possible in user-space, so cross-core
+	 * buffers must be allocated as coherent
+	 */
+	if (is_shared && _buffer_is_user_thread())
+		flags |= SOF_MEM_FLAG_COHERENT;
+#endif
+
 	/* validate request */
 	if (size == 0) {
 		tr_err(&buffer_tr, "new size = %zu is invalid", size);
@@ -272,6 +281,15 @@ struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_siz
 		return NULL;
 	}
 
+#if CONFIG_USERSPACE
+	/*
+	 * cache control is not possible in user-space, so cross-core
+	 * buffers must be allocated as coherent
+	 */
+	if (is_shared && _buffer_is_user_thread())
+		flags |= SOF_MEM_FLAG_COHERENT;
+#endif
+
 	/* Align preferred size to a multiple of the minimum size */
 	if (preferred_size % minimum_size)
 		preferred_size += minimum_size - preferred_size % minimum_size;

src/audio/buffers/ring_buffer.c

@@ -46,13 +46,30 @@ static inline uint8_t __sparse_cache *ring_buffer_buffer_end(struct ring_buffer
 	return ring_buffer->_data_buffer + ring_buffer->data_buffer_size;
 }
 
+#if CONFIG_USERSPACE
+#include <zephyr/kernel.h>
+
+static inline bool _ring_is_user_thread(void)
+{
+	return (k_current_get()->base.user_options & K_USER) != 0;
+}
+#endif
+
 static inline void ring_buffer_invalidate_shared(struct ring_buffer *ring_buffer,
 						 void __sparse_cache *ptr, size_t size)
 {
 	/* no cache required in case of not shared queue */
 	if (!ring_buffer_is_shared(ring_buffer))
 		return;
 
+#if CONFIG_USERSPACE
+	/* user-space shared buffers are allocated as uncached */
+	if (_ring_is_user_thread()) {
+		__ASSERT_NO_MSG(sys_cache_is_ptr_cached(ptr) == false);
+		return;
+	}
+#endif
+
 	/* wrap-around? */
 	if ((uintptr_t)ptr + size > (uintptr_t)ring_buffer_buffer_end(ring_buffer)) {
 		/* writeback till the end of circular buffer */
@@ -72,6 +89,14 @@ static inline void ring_buffer_writeback_shared(struct ring_buffer *ring_buffer,
 	if (!ring_buffer_is_shared(ring_buffer))
 		return;
 
+#if CONFIG_USERSPACE
+	/* user-space shared buffers are allocated as uncached */
+	if (_ring_is_user_thread()) {
+		__ASSERT_NO_MSG(sys_cache_is_ptr_cached(ptr) == false);
+		return;
+	}
+#endif
+
 	/* wrap-around? */
 	if ((uintptr_t)ptr + size > (uintptr_t)ring_buffer_buffer_end(ring_buffer)) {
 		/* writeback till the end of circular buffer */
@@ -355,6 +380,20 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl
 	 */
 	ring_buffer->data_buffer_size = 3 * max_ibs_obs;
 
+#if CONFIG_USERSPACE
+	/*
+	 * cache control is not possible in user-space, so cross-core
+	 * buffers must be allocated as coherent
+	 */
+	if (is_shared && _ring_is_user_thread())
+		memory_flags |= SOF_MEM_FLAG_COHERENT;
+
+	/*
+	 * TODO: _data_buffer is not cached anymore so will confuse
+	 *       sparse
+	 */
+#endif
+
 	/* allocate data buffer - always in cached memory alias */
 	ring_buffer->data_buffer_size =
 			ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN);

src/include/sof/audio/audio_stream.h

@@ -716,6 +716,15 @@ static inline void audio_stream_reset(struct audio_stream *buffer)
  */
 void audio_stream_init(struct audio_stream *audio_stream, void *buff_addr, uint32_t size);
 
+#if CONFIG_USERSPACE
+#include <zephyr/kernel.h>
+
+static inline bool _audio_stream_is_user_thread(void)
+{
+	return (k_current_get()->base.user_options & K_USER) != 0;
+}
+#endif
+
 /**
  * Invalidates (in DSP d-cache) the buffer in range [r_ptr, r_ptr+bytes],
  * with rollover if necessary.
@@ -733,6 +742,14 @@ static inline void audio_stream_invalidate(struct audio_stream *buffer, uint32_t
 		tail_size = bytes - head_size;
 	}
 
+#if CONFIG_USERSPACE
+	/* user-space shared buffers are allocated as uncached */
+	if (_audio_stream_is_user_thread()) {
+		__ASSERT_NO_MSG(sys_cache_is_ptr_cached(buffer->addr) == false);
+		return;
+	}
+#endif
+
 	dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->r_ptr, head_size);
 	if (tail_size)
 		dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->addr,
@@ -756,6 +773,14 @@ static inline void audio_stream_writeback(struct audio_stream *buffer, uint32_t
 		tail_size = bytes - head_size;
 	}
 
+#if CONFIG_USERSPACE
+	/* user-space shared buffers are allocated as uncached */
+	if (_audio_stream_is_user_thread()) {
+		__ASSERT_NO_MSG(sys_cache_is_ptr_cached(buffer->addr) == false);
+		return;
+	}
+#endif
+
 	dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, head_size);
 	if (tail_size)
 		dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->addr,

src/include/sof/audio/buffer.h

@@ -150,6 +150,15 @@ struct comp_buffer {
 	struct list_item buffers_list;
 };
 
+#if CONFIG_USERSPACE
+#include <zephyr/kernel.h>
+
+static inline bool _buffer_is_user_thread(void)
+{
+	return (k_current_get()->base.user_options & K_USER) != 0;
+}
+#endif
+
 /*
  * get a component providing data to the buffer
  */
@@ -261,20 +270,32 @@ bool buffer_params_match(struct comp_buffer *buffer,
 static inline void buffer_stream_invalidate(struct comp_buffer *buffer, uint32_t bytes)
 {
 #if CONFIG_INCOHERENT
-	if (audio_buffer_is_shared(&buffer->audio_buffer))
+	if (audio_buffer_is_shared(&buffer->audio_buffer)) {
+#if CONFIG_USERSPACE
+		/* user-space shared buffers are allocated as uncached */
+		if (_buffer_is_user_thread())
+			return;
+#endif
 		audio_stream_invalidate(&buffer->stream, bytes);
+	}
 #endif
 }
 
 static inline void buffer_stream_writeback(struct comp_buffer *buffer, uint32_t bytes)
 {
 #if CONFIG_INCOHERENT
-	if (audio_buffer_is_shared(&buffer->audio_buffer))
+	if (audio_buffer_is_shared(&buffer->audio_buffer)) {
+#if CONFIG_USERSPACE
+		/* user-space shared buffers are allocated as uncached */
+		if (_buffer_is_user_thread())
+			return;
+#endif
+
 		audio_stream_writeback(&buffer->stream, bytes);
+	}
 #endif
 }
 
-
 /*
  * Attach a new buffer at the beginning of the list. Note, that "head" must
  * really be the head of the list, not a list head within another buffer. We