Moving Background Blur to GPU

REUSE.toml

@@ -34,3 +34,9 @@ path = "app/src/main/res/raw/next_voice_message_doodle.ogg"
 precedence = "aggregate"
 SPDX-FileCopyrightText = "2024 Paciosoft"
 SPDX-License-Identifier = "CC-BY-4.0"
+
+[[annotations]]
+path = "app/src/main/assets/selfie_segmenter.tflite"
+precedence = "aggregate"
+SPDX-FileCopyrightText = "2023 Google LLC"
+SPDX-License-Identifier = "Apache-2.0"

app/build.gradle

@@ -369,6 +369,10 @@ dependencies {
     testImplementation("com.squareup.okhttp3:mockwebserver:$okhttpVersion")
     testImplementation("com.google.dagger:hilt-android-testing:2.59.2")
     testImplementation("org.robolectric:robolectric:4.16.1")
+
+    // Computer Vision - for background effects during video calls
+    implementation 'com.google.mediapipe:tasks-vision:0.10.26'
+    implementation "io.github.crow-misia.libyuv:libyuv-android:0.43.2"
 }
 
 tasks.register('installGitHooks', Copy) {

app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt

@@ -73,6 +73,9 @@ import com.nextcloud.talk.call.ReactionAnimator
 import com.nextcloud.talk.call.components.ParticipantGrid
 import com.nextcloud.talk.call.components.SelfVideoView
 import com.nextcloud.talk.call.components.screenshare.ScreenShareComponent
+import com.nextcloud.talk.camera.BackgroundBlurFrameProcessor
+import com.nextcloud.talk.camera.BlurBackgroundViewModel
+import com.nextcloud.talk.camera.BlurBackgroundViewModel.BackgroundBlurOn
 import com.nextcloud.talk.chat.ChatActivity
 import com.nextcloud.talk.data.user.model.User
 import com.nextcloud.talk.databinding.CallActivityBinding
@@ -185,7 +188,6 @@ import java.util.Objects
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicInteger
 import javax.inject.Inject
-import kotlin.String
 import kotlin.math.abs
 
 @AutoInjector(NextcloudTalkApplication::class)
@@ -214,6 +216,7 @@ class CallActivity : CallBaseActivity() {
     var audioManager: WebRtcAudioManager? = null
     var callRecordingViewModel: CallRecordingViewModel? = null
     var raiseHandViewModel: RaiseHandViewModel? = null
+    val blurBackgroundViewModel: BlurBackgroundViewModel = BlurBackgroundViewModel()
     private var mReceiver: BroadcastReceiver? = null
     private var peerConnectionFactory: PeerConnectionFactory? = null
     private var screenSharePeerConnectionFactory: PeerConnectionFactory? = null
@@ -539,6 +542,20 @@ class CallActivity : CallBaseActivity() {
         }
     }
 
+    private fun initBackgroundBlurViewModel(surfaceTextureHelper: SurfaceTextureHelper) {
+        blurBackgroundViewModel.viewState.observe(this) { state ->
+            val isOn = state == BackgroundBlurOn
+
+            val processor = if (isOn) {
+                BackgroundBlurFrameProcessor(context, surfaceTextureHelper)
+            } else {
+                null
+            }
+
+            videoSource?.setVideoProcessor(processor)
+        }
+    }
+
     private fun processExtras(extras: Bundle) {
         roomId = extras.getString(KEY_ROOM_ID, "")
         roomToken = extras.getString(KEY_ROOM_TOKEN, "")
@@ -1116,6 +1133,7 @@ class CallActivity : CallBaseActivity() {
             videoSource = peerConnectionFactory!!.createVideoSource(false)
 
             videoCapturer!!.initialize(surfaceTextureHelper, applicationContext, videoSource!!.capturerObserver)
+            initBackgroundBlurViewModel(surfaceTextureHelper)
         }
         localVideoTrack = peerConnectionFactory!!.createVideoTrack("NCv0", videoSource)
         localStream!!.addTrack(localVideoTrack)
@@ -1250,6 +1268,7 @@ class CallActivity : CallBaseActivity() {
                 binding!!.cameraButton.setImageResource(R.drawable.ic_videocam_white_24px)
             } else {
                 binding!!.cameraButton.setImageResource(R.drawable.ic_videocam_off_white_24px)
+                blurBackgroundViewModel.turnOffBlur()
             }
             toggleMedia(videoOn, true)
         } else if (shouldShowRequestPermissionRationale(Manifest.permission.CAMERA)) {
@@ -1326,6 +1345,10 @@ class CallActivity : CallBaseActivity() {
         raiseHandViewModel!!.clickHandButton()
     }
 
+    fun toggleBackgroundBlur() {
+        blurBackgroundViewModel.toggleBackgroundBlur()
+    }
+
     public override fun onDestroy() {
         if (signalingMessageReceiver != null) {
             signalingMessageReceiver!!.removeListener(localParticipantMessageListener)

app/src/main/java/com/nextcloud/talk/camera/BackgroundBlurFrameProcessor.kt

@@ -0,0 +1,223 @@
+/*
+ * Nextcloud Talk - Android Client
+ *
+ * SPDX-FileCopyrightText: 2025 Julius Linus <juliuslinus1@gmail.com>
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+package com.nextcloud.talk.camera
+
+import android.content.Context
+import android.os.Handler
+import android.os.HandlerThread
+import android.util.Log
+import android.util.LruCache
+import io.github.crow_misia.libyuv.AbgrBuffer
+import io.github.crow_misia.libyuv.I420Buffer
+import io.github.crow_misia.libyuv.PlanePrimitive
+import org.webrtc.JavaI420Buffer
+import org.webrtc.SurfaceTextureHelper
+import org.webrtc.VideoFrame
+import org.webrtc.VideoProcessor
+import org.webrtc.VideoSink
+import org.webrtc.YuvHelper
+import java.nio.ByteBuffer
+
+@Suppress("TooGenericExceptionCaught")
+class BackgroundBlurFrameProcessor(val context: Context, val surfaceTextureHelper: SurfaceTextureHelper) :
+    VideoProcessor,
+    ImageSegmenterHelper.SegmenterListener {
+
+    companion object {
+        val TAG: String = this::class.java.simpleName
+        const val GPU_THREAD: String = "BackgroundBlur"
+        const val FLOAT_ROTATION = 180.0f
+        const val INT_4 = 4
+        const val MAX_NUM_FRAMES = 10
+    }
+
+    private var sink: VideoSink? = null
+    private var segmenterHelper: ImageSegmenterHelper? = null
+    private var backgroundBlurGPUProcessor: BackgroundBlurGPUProcessor? = null
+
+    /* This is to hold meta information between MediaPipe and GPU Render threads, in a thread safe way
+    A LRU (least recently used) cache, holds up to MAX_NUM_FRAMES, before evicting the least recently used
+    if full. This is used because in case an error occurs with MediaPipe, and a frame is dropped, the frame might stay
+    in the map indefinitely, unable to be cleaned up by the garbage collector, therefore causing a memory leak. With the
+    LRU Cache, the frame would end up being cleaned eventually as the program runs */
+    private var rotationMap = LruCache<Long, Float>(MAX_NUM_FRAMES)
+    private val frameBufferMap = LruCache<Long, ByteBuffer>(MAX_NUM_FRAMES)
+
+    // Dedicated Thread for OpenGL Operations
+    private var glThread: HandlerThread? = null
+    private var glHandler: Handler? = null
+
+    // SegmentationListener Interface
+
+    override fun onError(error: String, errorCode: Int) {
+        Log.e(TAG, "Error $errorCode: $error")
+    }
+
+    override fun onResults(resultBundle: ImageSegmenterHelper.ResultBundle) {
+        val rotation = rotationMap[resultBundle.inferenceTime] ?: 0f
+        val frameBuffer = frameBufferMap[resultBundle.inferenceTime]
+
+        // Remove once used to prevent mem leaks
+        rotationMap.synchronizedRemove(resultBundle.inferenceTime)
+        frameBufferMap.synchronizedRemove(resultBundle.inferenceTime)
+
+        if (frameBuffer == null) {
+            Log.e(TAG, "Critical Error in onResults: FrameBufferMap[${resultBundle.inferenceTime}] was null")
+            return
+        }
+
+        glHandler?.post {
+            // This block runs safely on gpu thread
+            backgroundBlurGPUProcessor?.let { scaler ->
+                try {
+                    val drawArray = scaler.process(
+                        resultBundle.mask,
+                        frameBuffer,
+                        resultBundle.width,
+                        resultBundle.height,
+                        rotation
+                    )
+
+                    val webRTCBuffer = drawArray.convertToWebRTCBuffer(resultBundle.width, resultBundle.height)
+                    val videoFrame = VideoFrame(webRTCBuffer, 0, resultBundle.inferenceTime)
+
+                    // This should run on the CaptureThread
+                    surfaceTextureHelper.handler.post {
+                        Log.d(TAG, "Sent VideoFrame to sink on :${Thread.currentThread().name}")
+                        sink?.onFrame(videoFrame)
+
+                        // webRTCBuffer usually needs release() if it's not a JavaI420Buffer wrapper that auto-GCs,
+                        // but JavaI420Buffer.wrap() relies on GC.
+                        videoFrame.release()
+                    }
+                } catch (e: Exception) {
+                    Log.e(TAG, "Error processing frame on GL Thread", e)
+                }
+            }
+        }
+    }
+
+    // Video Processor Interface
+
+    override fun onCapturerStarted(success: Boolean) {
+        segmenterHelper = ImageSegmenterHelper(context = context, imageSegmenterListener = this)
+
+        glThread = HandlerThread(GPU_THREAD).apply { start() }
+        glHandler = Handler(glThread!!.looper)
+        glHandler?.post {
+            backgroundBlurGPUProcessor = BackgroundBlurGPUProcessor(context)
+            backgroundBlurGPUProcessor?.init()
+        }
+    }
+
+    override fun onCapturerStopped() {
+        segmenterHelper?.destroyImageSegmenter()
+        glHandler?.post {
+            backgroundBlurGPUProcessor?.release()
+            backgroundBlurGPUProcessor = null
+
+            // Quit thread after cleanup
+            glThread?.quitSafely()
+            glThread = null
+            glHandler = null
+        }
+    }
+
+    override fun onFrameCaptured(videoFrame: VideoFrame) {
+        val i420WebRTCBuffer = videoFrame.buffer.toI420()
+        val width = videoFrame.buffer.width
+        val height = videoFrame.buffer.height
+        val rotation = FLOAT_ROTATION - videoFrame.rotation
+        val videoFrameBuffer = i420WebRTCBuffer?.convertToABGR()
+
+        i420WebRTCBuffer?.release()
+
+        videoFrameBuffer?.let {
+            rotationMap.synchronizedPut(videoFrame.timestampNs, rotation)
+            frameBufferMap.synchronizedPut(videoFrame.timestampNs, it)
+            segmenterHelper?.segmentFrame(it, width, height, videoFrame.timestampNs)
+        } ?: {
+            Log.e(TAG, "onFrameCaptured:: Video Frame was null!")
+            sink?.onFrame(videoFrame)
+        }
+    }
+
+    override fun setSink(sink: VideoSink?) {
+        this.sink = sink
+    }
+
+    fun VideoFrame.I420Buffer.convertToABGR(): ByteBuffer {
+        val dataYSize = dataY.limit() - dataY.position()
+        val dataUSize = dataU.limit() - dataU.position()
+        val dataVSize = dataV.limit() - dataV.position()
+
+        val planeY = PlanePrimitive.create(strideY, dataY, dataYSize)
+        val planeU = PlanePrimitive.create(strideU, dataU, dataUSize)
+        val planeV = PlanePrimitive.create(strideV, dataV, dataVSize)
+
+        val libYuvI420Buffer = I420Buffer.wrap(planeY, planeU, planeV, width, height)
+        val libYuvABGRBuffer = AbgrBuffer.allocate(width, height)
+        libYuvI420Buffer.convertTo(libYuvABGRBuffer)
+
+        return libYuvABGRBuffer.asBuffer()
+    }
+
+    inline fun <reified K, V> LruCache<K, V>.synchronizedPut(key: K, value: V) {
+        synchronized(this) {
+            this.put(key, value)
+        }
+    }
+
+    inline fun <reified K, V> LruCache<K, V>.synchronizedRemove(key: K) {
+        synchronized(this) {
+            this.remove(key)
+        }
+    }
+
+    fun ByteArray.convertToWebRTCBuffer(width: Int, height: Int): JavaI420Buffer {
+        val src = ByteBuffer.allocateDirect(this.size)
+        src.put(this)
+
+        val srcStride = width * INT_4
+        val yPlaneSize = width * height
+        val uvPlaneSize = (width / 2) * (height / 2)
+
+        val dstYStride = width
+        val dstUStride = width / 2
+        val dstVStride = width / 2
+
+        val dstYBuffer = ByteBuffer.allocateDirect(yPlaneSize)
+        val dstUBuffer = ByteBuffer.allocateDirect(uvPlaneSize)
+        val dstVBuffer = ByteBuffer.allocateDirect(uvPlaneSize)
+
+        YuvHelper.ABGRToI420(
+            src,
+            srcStride,
+            dstYBuffer,
+            dstYStride,
+            dstUBuffer,
+            dstUStride,
+            dstVBuffer,
+            dstVStride,
+            width,
+            height
+        )
+
+        return JavaI420Buffer.wrap(
+            width,
+            height,
+            dstYBuffer,
+            dstYStride,
+            dstUBuffer,
+            dstUStride,
+            dstVBuffer,
+            dstVStride,
+            null
+        )
+    }
+}