From f187d9ab2d25076d5f95f82fd27fe82f5e34bf9f Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 7 Nov 2025 13:52:28 -0800 Subject: [PATCH 01/11] wip --- LICENSE.txt.meta | 7 + Runtime/Scripts/VideoStream.cs | 203 +++++++++++++++++++++++++-- Runtime/Shaders.meta | 8 ++ Runtime/Shaders/YuvToRgb.shader | 129 +++++++++++++++++ Runtime/Shaders/YuvToRgb.shader.meta | 9 ++ 5 files changed, 345 insertions(+), 11 deletions(-) create mode 100644 LICENSE.txt.meta create mode 100644 Runtime/Shaders.meta create mode 100644 Runtime/Shaders/YuvToRgb.shader create mode 100644 Runtime/Shaders/YuvToRgb.shader.meta diff --git a/LICENSE.txt.meta b/LICENSE.txt.meta new file mode 100644 index 00000000..828fb869 --- /dev/null +++ b/LICENSE.txt.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: b7eb024736a4f4e79a75759f9c733718 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index cad060f0..ddf74061 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -9,14 +9,35 @@ namespace LiveKit { public class VideoStream { + public enum ColorStandard + { + Auto = 0, + BT601 = 1, + BT709 = 2, + } + public delegate void FrameReceiveDelegate(VideoFrame frame); - public delegate void TextureReceiveDelegate(Texture2D tex2d); + public delegate void TextureReceiveDelegate(Texture tex); public delegate void TextureUploadDelegate(); internal readonly FfiHandle Handle; private VideoStreamInfo _info; private bool _disposed = false; private bool _dirty = false; + private bool _useGpuYuvToRgb = true; + private string _lastColorConversionPathLog; + private bool _swapUV = false; + private bool _fullRange = false; + private ColorStandard _colorStandard = ColorStandard.BT709; + private bool _invertU = false; + private bool _invertV = false; + private int _debugMode = 0; + + private Material _yuvToRgbMaterial; + private Texture2D _planeY; + private Texture2D _planeU; + private Texture2D _planeV; + private RenderTexture _convertRt; /// Called when we receive a new frame from the VideoTrack public event FrameReceiveDelegate FrameReceived; @@ -29,7 +50,7 @@ public class VideoStream /// The texture changes every time the video resolution changes. /// Can be null if UpdateRoutine isn't started - public Texture2D Texture { private set; get; } + public RenderTexture Texture { private set; get; } public VideoFrameBuffer VideoBuffer { private set; get; } protected bool _playing = false; @@ -70,8 +91,19 @@ private void Dispose(bool disposing) if (!_disposed) { if (disposing) + { VideoBuffer?.Dispose(); - if (Texture != null) UnityEngine.Object.Destroy(Texture); + } + // Unity objects must be destroyed on main thread; RT is destroyed below + if (_planeY != null) UnityEngine.Object.Destroy(_planeY); + if (_planeU != null) UnityEngine.Object.Destroy(_planeU); + if (_planeV != null) UnityEngine.Object.Destroy(_planeV); + if (_convertRt != null) + { + _convertRt.Release(); + UnityEngine.Object.Destroy(_convertRt); + } + if (_yuvToRgbMaterial != null) UnityEngine.Object.Destroy(_yuvToRgbMaterial); _disposed = true; } } @@ -87,6 +119,51 @@ public virtual void Stop() _playing = false; } + public bool SwapUV + { + get => _swapUV; + set => _swapUV = value; + } + + public bool FullRangeYuv + { + get => _fullRange; + set => _fullRange = value; + } + + public ColorStandard ColorStandardMode + { + get => _colorStandard; + set => _colorStandard = value; + } + + public bool InvertU + { + get => _invertU; + set => _invertU = value; + } + + public bool InvertV + { + get => _invertV; + set => _invertV = value; + } + + /// 0 = normal, 1 = Y, 2 = U, 3 = V + public int DebugMode + { + get => _debugMode; + set => _debugMode = value; + } + + private void LogConversionPath(string path) + { + if (_lastColorConversionPathLog == path) + return; + _lastColorConversionPathLog = path; + Debug.Log($"[LiveKit] VideoStream color conversion: {path}"); + } + public IEnumerator Update() { while (_playing) @@ -104,24 +181,128 @@ public IEnumerator Update() var rHeight = VideoBuffer.Height; var textureChanged = false; - if (Texture == null || Texture.width != rWidth || Texture.height != rHeight) + if (_convertRt == null || _convertRt.width != rWidth || _convertRt.height != rHeight) { - if (Texture != null) UnityEngine.Object.Destroy(Texture); - Texture = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false); - Texture.ignoreMipmapLimit = false; + if (_convertRt != null) + { + _convertRt.Release(); + UnityEngine.Object.Destroy(_convertRt); + } + _convertRt = new RenderTexture((int)rWidth, (int)rHeight, 0, RenderTextureFormat.ARGB32); + _convertRt.Create(); + Texture = _convertRt; textureChanged = true; } - var rgba = VideoBuffer.ToRGBA(); + + if (_useGpuYuvToRgb) + { + if (_yuvToRgbMaterial == null) + { + var shader = Shader.Find("Hidden/LiveKit/YUV2RGB"); + if (shader != null) + _yuvToRgbMaterial = new Material(shader); + } + + // _convertRt ensured above + + // Ensure YUV plane textures + if (_planeY == null || _planeY.width != rWidth || _planeY.height != rHeight) + { + if (_planeY != null) UnityEngine.Object.Destroy(_planeY); + _planeY = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.R8, false, true); + _planeY.filterMode = FilterMode.Bilinear; + _planeY.wrapMode = TextureWrapMode.Clamp; + } + var chromaW = (int)(rWidth / 2); + var chromaH = (int)(rHeight / 2); + if (_planeU == null || _planeU.width != chromaW || _planeU.height != chromaH) + { + if (_planeU != null) UnityEngine.Object.Destroy(_planeU); + _planeU = new Texture2D(chromaW, chromaH, TextureFormat.R8, false, true); + _planeU.filterMode = FilterMode.Point; + _planeU.wrapMode = TextureWrapMode.Clamp; + } + if (_planeV == null || _planeV.width != chromaW || _planeV.height != chromaH) + { + if (_planeV != null) UnityEngine.Object.Destroy(_planeV); + _planeV = new Texture2D(chromaW, chromaH, TextureFormat.R8, false, true); + _planeV.filterMode = FilterMode.Point; + _planeV.wrapMode = TextureWrapMode.Clamp; + } + + // Upload planes (assuming NormalizeStride = true) + var info = VideoBuffer.Info; + if (info.Components.Count >= 3) + { + var yComp = info.Components[0]; + var uComp = info.Components[1]; + var vComp = info.Components[2]; + + _planeY.LoadRawTextureData((IntPtr)yComp.DataPtr, (int)yComp.Size); + _planeY.Apply(false, false); + _planeU.LoadRawTextureData((IntPtr)uComp.DataPtr, (int)uComp.Size); + _planeU.Apply(false, false); + _planeV.LoadRawTextureData((IntPtr)vComp.DataPtr, (int)vComp.Size); + _planeV.Apply(false, false); + } + + if (_yuvToRgbMaterial != null) + { + // Select color matrix, preferring explicit override when set + bool useBt709; + if (_colorStandard == ColorStandard.BT601) + useBt709 = false; + else if (_colorStandard == ColorStandard.BT709) + useBt709 = true; + else + useBt709 = (rWidth >= 1280) || (rHeight >= 720); // heuristic + _yuvToRgbMaterial.SetFloat("_ColorStd", useBt709 ? 1.0f : 0.0f); + _yuvToRgbMaterial.SetFloat("_SwapUV", _swapUV ? 1.0f : 0.0f); + _yuvToRgbMaterial.SetFloat("_FullRange", _fullRange ? 1.0f : 0.0f); + _yuvToRgbMaterial.SetFloat("_InvertU", _invertU ? 1.0f : 0.0f); + _yuvToRgbMaterial.SetFloat("_InvertV", _invertV ? 1.0f : 0.0f); + _yuvToRgbMaterial.SetFloat("_DebugMode", _debugMode); + var path = useBt709 ? "GPU shader YUV->RGB (BT.709)" : "GPU shader YUV->RGB (BT.601)"; + if (_swapUV) path += " +SwapUV"; + if (_fullRange) path += " +FullRange"; + if (_invertU) path += " +InvertU"; + if (_invertV) path += " +InvertV"; + if (_debugMode != 0) path += $" +Debug({(DebugMode==1?"Y":DebugMode==2?"U":"V")})"; + LogConversionPath(path); + _yuvToRgbMaterial.SetTexture("_TexY", _planeY); + _yuvToRgbMaterial.SetTexture("_TexU", _planeU); + _yuvToRgbMaterial.SetTexture("_TexV", _planeV); + Graphics.Blit(Texture2D.blackTexture, _convertRt, _yuvToRgbMaterial); + } + else + { + LogConversionPath("CPU conversion (shader not found)"); + // Fallback to CPU conversion if shader not found + var rgba = VideoBuffer.ToRGBA(); + var tempTex = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false); + tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); + tempTex.Apply(); + Graphics.Blit(tempTex, _convertRt); + UnityEngine.Object.Destroy(tempTex); + rgba.Dispose(); + } + } + else { - Texture.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); + LogConversionPath("CPU conversion (_useGpuYuvToRgb=false)"); + var rgba = VideoBuffer.ToRGBA(); + var tempTex = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false); + tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); + tempTex.Apply(); + Graphics.Blit(tempTex, _convertRt); + UnityEngine.Object.Destroy(tempTex); + rgba.Dispose(); } - Texture.Apply(); if (textureChanged) TextureReceived?.Invoke(Texture); TextureUploaded?.Invoke(); - rgba.Dispose(); } yield break; diff --git a/Runtime/Shaders.meta b/Runtime/Shaders.meta new file mode 100644 index 00000000..44491489 --- /dev/null +++ b/Runtime/Shaders.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 9de6f1e3731fd43659219ea7cc944c89 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Shaders/YuvToRgb.shader b/Runtime/Shaders/YuvToRgb.shader new file mode 100644 index 00000000..658f06da --- /dev/null +++ b/Runtime/Shaders/YuvToRgb.shader @@ -0,0 +1,129 @@ +Shader "Hidden/LiveKit/YUV2RGB" +{ + SubShader + { + Tags { "RenderType" = "Opaque" "Queue" = "Geometry" } + Pass + { + ZTest Always Cull Off ZWrite Off + + HLSLPROGRAM + #pragma vertex vert + #pragma fragment frag + #include "UnityCG.cginc" + + sampler2D _TexY; + sampler2D _TexU; + sampler2D _TexV; + float _ColorStd; // 0 = BT.601 limited, 1 = BT.709 limited + float _SwapUV; // 0 = normal, 1 = swap U and V + float _FullRange; // 0 = limited range (16-235/240), 1 = full range (0-255) + float _InvertU; // 0 = normal, 1 = invert U (u = 1-u) + float _InvertV; // 0 = normal, 1 = invert V (v = 1-v) + float _DebugMode; // 0 = normal, 1 = show Y, 2 = show U, 3 = show V + + struct appdata + { + float4 vertex : POSITION; + float2 uv : TEXCOORD0; + }; + + struct v2f + { + float4 pos : SV_POSITION; + float2 uv : TEXCOORD0; + }; + + v2f vert(appdata v) + { + v2f o; + o.pos = UnityObjectToClipPos(v.vertex); + o.uv = v.uv; + return o; + } + + float3 yuvToRgb601Limited(float y, float u, float v) + { + // BT.601 limited range + float c = y - 16.0 / 255.0; + float d = u - 128.0 / 255.0; + float e = v - 128.0 / 255.0; + + float3 rgb; + rgb.r = saturate(1.16438356 * c + 1.59602678 * e); + rgb.g = saturate(1.16438356 * c - 0.39176229 * d - 0.81296765 * e); + rgb.b = saturate(1.16438356 * c + 2.01723214 * d); + return rgb; + } + + float3 yuvToRgb709Limited(float y, float u, float v) + { + // BT.709 limited range + float c = y - 16.0 / 255.0; + float d = u - 128.0 / 255.0; + float e = v - 128.0 / 255.0; + + float3 rgb; + rgb.r = saturate(1.16438356 * c + 1.79274107 * e); + rgb.g = saturate(1.16438356 * c - 0.21324861 * d - 0.53290933 * e); + rgb.b = saturate(1.16438356 * c + 2.11240179 * d); + return rgb; + } + + float3 yuvToRgb601Full(float y, float u, float v) + { + // BT.601 full range + float d = u - 0.5; + float e = v - 0.5; + float3 rgb; + rgb.r = saturate(y + 1.40200000 * e); + rgb.g = saturate(y - 0.34413629 * d - 0.71413629 * e); + rgb.b = saturate(y + 1.77200000 * d); + return rgb; + } + + float3 yuvToRgb709Full(float y, float u, float v) + { + // BT.709 full range + float d = u - 0.5; + float e = v - 0.5; + float3 rgb; + rgb.r = saturate(y + 1.57480000 * e); + rgb.g = saturate(y - 0.18732427 * d - 0.46812427 * e); + rgb.b = saturate(y + 1.85560000 * d); + return rgb; + } + + float4 frag(v2f i) : SV_Target + { + float y = tex2D(_TexY, i.uv).r; + float u = tex2D(_TexU, i.uv).r; + float v = tex2D(_TexV, i.uv).r; + if (_SwapUV >= 0.5) + { + float t = u; u = v; v = t; + } + if (_InvertU >= 0.5) { u = 1.0 - u; } + if (_InvertV >= 0.5) { v = 1.0 - v; } + // Debug views + if (_DebugMode >= 0.5 && _DebugMode < 1.5) { return float4(y, y, y, 1.0); } + if (_DebugMode >= 1.5 && _DebugMode < 2.5) { return float4(u, u, u, 1.0); } + if (_DebugMode >= 2.5 && _DebugMode < 3.5) { return float4(v, v, v, 1.0); } + // Select matrix by _ColorStd and _FullRange + float use709 = step(0.5, _ColorStd); + float useFull = step(0.5, _FullRange); + float3 rgb601Lim = yuvToRgb601Limited(y, u, v); + float3 rgb709Lim = yuvToRgb709Limited(y, u, v); + float3 rgb601Full = yuvToRgb601Full(y, u, v); + float3 rgb709Full = yuvToRgb709Full(y, u, v); + float3 rgbLim = lerp(rgb601Lim, rgb709Lim, use709); + float3 rgbFull = lerp(rgb601Full, rgb709Full, use709); + float3 rgb = lerp(rgbLim, rgbFull, useFull); + return float4(rgb, 1.0); + } + ENDHLSL + } + } +} + + diff --git a/Runtime/Shaders/YuvToRgb.shader.meta b/Runtime/Shaders/YuvToRgb.shader.meta new file mode 100644 index 00000000..ce19c17d --- /dev/null +++ b/Runtime/Shaders/YuvToRgb.shader.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 5759033c525bd426f9663af70853816f +ShaderImporter: + externalObjects: {} + defaultTextures: [] + nonModifiableTextures: [] + userData: + assetBundleName: + assetBundleVariant: From 1a178fc3197ac8474fc8aa7cc02b87674a20356c Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 7 Nov 2025 14:01:57 -0800 Subject: [PATCH 02/11] cleanup and refactor some code --- Runtime/Scripts/VideoStream.cs | 236 ++++++++++++-------------------- Runtime/Shaders/YuvToRgb.shader | 65 +-------- 2 files changed, 89 insertions(+), 212 deletions(-) diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index ddf74061..9fb7cb2c 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -9,13 +9,6 @@ namespace LiveKit { public class VideoStream { - public enum ColorStandard - { - Auto = 0, - BT601 = 1, - BT709 = 2, - } - public delegate void FrameReceiveDelegate(VideoFrame frame); public delegate void TextureReceiveDelegate(Texture tex); public delegate void TextureUploadDelegate(); @@ -26,12 +19,7 @@ public enum ColorStandard private bool _dirty = false; private bool _useGpuYuvToRgb = true; private string _lastColorConversionPathLog; - private bool _swapUV = false; - private bool _fullRange = false; - private ColorStandard _colorStandard = ColorStandard.BT709; - private bool _invertU = false; - private bool _invertV = false; - private int _debugMode = 0; + // Fixed baseline: BT.709 limited, no UV swap private Material _yuvToRgbMaterial; private Texture2D _planeY; @@ -119,42 +107,88 @@ public virtual void Stop() _playing = false; } - public bool SwapUV - { - get => _swapUV; - set => _swapUV = value; - } + private bool EnsureRenderTexture(int width, int height) + { + var textureChanged = false; + if (_convertRt == null || _convertRt.width != width || _convertRt.height != height) + { + if (_convertRt != null) + { + _convertRt.Release(); + UnityEngine.Object.Destroy(_convertRt); + } + _convertRt = new RenderTexture(width, height, 0, RenderTextureFormat.ARGB32); + _convertRt.Create(); + Texture = _convertRt; + textureChanged = true; + } + return textureChanged; + } - public bool FullRangeYuv - { - get => _fullRange; - set => _fullRange = value; - } + private void EnsureGpuMaterial() + { + if (_yuvToRgbMaterial == null) + { + var shader = Shader.Find("Hidden/LiveKit/YUV2RGB"); + if (shader != null) + _yuvToRgbMaterial = new Material(shader); + } + } - public ColorStandard ColorStandardMode - { - get => _colorStandard; - set => _colorStandard = value; - } + private static void EnsurePlaneTexture(ref Texture2D tex, int width, int height, TextureFormat format, FilterMode filterMode) + { + if (tex == null || tex.width != width || tex.height != height) + { + if (tex != null) UnityEngine.Object.Destroy(tex); + tex = new Texture2D(width, height, format, false, true); + tex.filterMode = filterMode; + tex.wrapMode = TextureWrapMode.Clamp; + } + } - public bool InvertU - { - get => _invertU; - set => _invertU = value; - } + private void EnsureYuvPlaneTextures(int width, int height) + { + EnsurePlaneTexture(ref _planeY, width, height, TextureFormat.R8, FilterMode.Bilinear); + var chromaW = width / 2; + var chromaH = height / 2; + EnsurePlaneTexture(ref _planeU, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); + EnsurePlaneTexture(ref _planeV, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); + } - public bool InvertV - { - get => _invertV; - set => _invertV = value; - } + private void UploadYuvPlanes() + { + var info = VideoBuffer.Info; + if (info.Components.Count < 3) return; + var yComp = info.Components[0]; + var uComp = info.Components[1]; + var vComp = info.Components[2]; + + _planeY.LoadRawTextureData((IntPtr)yComp.DataPtr, (int)yComp.Size); + _planeY.Apply(false, false); + _planeU.LoadRawTextureData((IntPtr)uComp.DataPtr, (int)uComp.Size); + _planeU.Apply(false, false); + _planeV.LoadRawTextureData((IntPtr)vComp.DataPtr, (int)vComp.Size); + _planeV.Apply(false, false); + } - /// 0 = normal, 1 = Y, 2 = U, 3 = V - public int DebugMode - { - get => _debugMode; - set => _debugMode = value; - } + private void CpuConvertToRenderTarget(int width, int height) + { + var rgba = VideoBuffer.ToRGBA(); + var tempTex = new Texture2D(width, height, TextureFormat.RGBA32, false); + tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); + tempTex.Apply(); + Graphics.Blit(tempTex, _convertRt); + UnityEngine.Object.Destroy(tempTex); + rgba.Dispose(); + } + + private void GpuConvertToRenderTarget() + { + _yuvToRgbMaterial.SetTexture("_TexY", _planeY); + _yuvToRgbMaterial.SetTexture("_TexU", _planeU); + _yuvToRgbMaterial.SetTexture("_TexV", _planeV); + Graphics.Blit(Texture2D.blackTexture, _convertRt, _yuvToRgbMaterial); + } private void LogConversionPath(string path) { @@ -180,123 +214,29 @@ public IEnumerator Update() var rWidth = VideoBuffer.Width; var rHeight = VideoBuffer.Height; - var textureChanged = false; - if (_convertRt == null || _convertRt.width != rWidth || _convertRt.height != rHeight) - { - if (_convertRt != null) - { - _convertRt.Release(); - UnityEngine.Object.Destroy(_convertRt); - } - _convertRt = new RenderTexture((int)rWidth, (int)rHeight, 0, RenderTextureFormat.ARGB32); - _convertRt.Create(); - Texture = _convertRt; - textureChanged = true; - } + var textureChanged = EnsureRenderTexture((int)rWidth, (int)rHeight); if (_useGpuYuvToRgb) { - if (_yuvToRgbMaterial == null) - { - var shader = Shader.Find("Hidden/LiveKit/YUV2RGB"); - if (shader != null) - _yuvToRgbMaterial = new Material(shader); - } - - // _convertRt ensured above - - // Ensure YUV plane textures - if (_planeY == null || _planeY.width != rWidth || _planeY.height != rHeight) - { - if (_planeY != null) UnityEngine.Object.Destroy(_planeY); - _planeY = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.R8, false, true); - _planeY.filterMode = FilterMode.Bilinear; - _planeY.wrapMode = TextureWrapMode.Clamp; - } - var chromaW = (int)(rWidth / 2); - var chromaH = (int)(rHeight / 2); - if (_planeU == null || _planeU.width != chromaW || _planeU.height != chromaH) - { - if (_planeU != null) UnityEngine.Object.Destroy(_planeU); - _planeU = new Texture2D(chromaW, chromaH, TextureFormat.R8, false, true); - _planeU.filterMode = FilterMode.Point; - _planeU.wrapMode = TextureWrapMode.Clamp; - } - if (_planeV == null || _planeV.width != chromaW || _planeV.height != chromaH) - { - if (_planeV != null) UnityEngine.Object.Destroy(_planeV); - _planeV = new Texture2D(chromaW, chromaH, TextureFormat.R8, false, true); - _planeV.filterMode = FilterMode.Point; - _planeV.wrapMode = TextureWrapMode.Clamp; - } - - // Upload planes (assuming NormalizeStride = true) - var info = VideoBuffer.Info; - if (info.Components.Count >= 3) - { - var yComp = info.Components[0]; - var uComp = info.Components[1]; - var vComp = info.Components[2]; - - _planeY.LoadRawTextureData((IntPtr)yComp.DataPtr, (int)yComp.Size); - _planeY.Apply(false, false); - _planeU.LoadRawTextureData((IntPtr)uComp.DataPtr, (int)uComp.Size); - _planeU.Apply(false, false); - _planeV.LoadRawTextureData((IntPtr)vComp.DataPtr, (int)vComp.Size); - _planeV.Apply(false, false); - } + EnsureGpuMaterial(); + EnsureYuvPlaneTextures((int)rWidth, (int)rHeight); + UploadYuvPlanes(); if (_yuvToRgbMaterial != null) { - // Select color matrix, preferring explicit override when set - bool useBt709; - if (_colorStandard == ColorStandard.BT601) - useBt709 = false; - else if (_colorStandard == ColorStandard.BT709) - useBt709 = true; - else - useBt709 = (rWidth >= 1280) || (rHeight >= 720); // heuristic - _yuvToRgbMaterial.SetFloat("_ColorStd", useBt709 ? 1.0f : 0.0f); - _yuvToRgbMaterial.SetFloat("_SwapUV", _swapUV ? 1.0f : 0.0f); - _yuvToRgbMaterial.SetFloat("_FullRange", _fullRange ? 1.0f : 0.0f); - _yuvToRgbMaterial.SetFloat("_InvertU", _invertU ? 1.0f : 0.0f); - _yuvToRgbMaterial.SetFloat("_InvertV", _invertV ? 1.0f : 0.0f); - _yuvToRgbMaterial.SetFloat("_DebugMode", _debugMode); - var path = useBt709 ? "GPU shader YUV->RGB (BT.709)" : "GPU shader YUV->RGB (BT.601)"; - if (_swapUV) path += " +SwapUV"; - if (_fullRange) path += " +FullRange"; - if (_invertU) path += " +InvertU"; - if (_invertV) path += " +InvertV"; - if (_debugMode != 0) path += $" +Debug({(DebugMode==1?"Y":DebugMode==2?"U":"V")})"; - LogConversionPath(path); - _yuvToRgbMaterial.SetTexture("_TexY", _planeY); - _yuvToRgbMaterial.SetTexture("_TexU", _planeU); - _yuvToRgbMaterial.SetTexture("_TexV", _planeV); - Graphics.Blit(Texture2D.blackTexture, _convertRt, _yuvToRgbMaterial); + LogConversionPath("GPU shader YUV->RGB (BT.709 limited)"); + GpuConvertToRenderTarget(); } else { - LogConversionPath("CPU conversion (shader not found)"); - // Fallback to CPU conversion if shader not found - var rgba = VideoBuffer.ToRGBA(); - var tempTex = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false); - tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); - tempTex.Apply(); - Graphics.Blit(tempTex, _convertRt); - UnityEngine.Object.Destroy(tempTex); - rgba.Dispose(); + LogConversionPath("CPU conversion (shader not found)"); + CpuConvertToRenderTarget((int)rWidth, (int)rHeight); } } else { - LogConversionPath("CPU conversion (_useGpuYuvToRgb=false)"); - var rgba = VideoBuffer.ToRGBA(); - var tempTex = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false); - tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); - tempTex.Apply(); - Graphics.Blit(tempTex, _convertRt); - UnityEngine.Object.Destroy(tempTex); - rgba.Dispose(); + LogConversionPath("CPU conversion (_useGpuYuvToRgb=false)"); + CpuConvertToRenderTarget((int)rWidth, (int)rHeight); } if (textureChanged) diff --git a/Runtime/Shaders/YuvToRgb.shader b/Runtime/Shaders/YuvToRgb.shader index 658f06da..69631b30 100644 --- a/Runtime/Shaders/YuvToRgb.shader +++ b/Runtime/Shaders/YuvToRgb.shader @@ -15,12 +15,6 @@ Shader "Hidden/LiveKit/YUV2RGB" sampler2D _TexY; sampler2D _TexU; sampler2D _TexV; - float _ColorStd; // 0 = BT.601 limited, 1 = BT.709 limited - float _SwapUV; // 0 = normal, 1 = swap U and V - float _FullRange; // 0 = limited range (16-235/240), 1 = full range (0-255) - float _InvertU; // 0 = normal, 1 = invert U (u = 1-u) - float _InvertV; // 0 = normal, 1 = invert V (v = 1-v) - float _DebugMode; // 0 = normal, 1 = show Y, 2 = show U, 3 = show V struct appdata { @@ -42,20 +36,6 @@ Shader "Hidden/LiveKit/YUV2RGB" return o; } - float3 yuvToRgb601Limited(float y, float u, float v) - { - // BT.601 limited range - float c = y - 16.0 / 255.0; - float d = u - 128.0 / 255.0; - float e = v - 128.0 / 255.0; - - float3 rgb; - rgb.r = saturate(1.16438356 * c + 1.59602678 * e); - rgb.g = saturate(1.16438356 * c - 0.39176229 * d - 0.81296765 * e); - rgb.b = saturate(1.16438356 * c + 2.01723214 * d); - return rgb; - } - float3 yuvToRgb709Limited(float y, float u, float v) { // BT.709 limited range @@ -70,55 +50,12 @@ Shader "Hidden/LiveKit/YUV2RGB" return rgb; } - float3 yuvToRgb601Full(float y, float u, float v) - { - // BT.601 full range - float d = u - 0.5; - float e = v - 0.5; - float3 rgb; - rgb.r = saturate(y + 1.40200000 * e); - rgb.g = saturate(y - 0.34413629 * d - 0.71413629 * e); - rgb.b = saturate(y + 1.77200000 * d); - return rgb; - } - - float3 yuvToRgb709Full(float y, float u, float v) - { - // BT.709 full range - float d = u - 0.5; - float e = v - 0.5; - float3 rgb; - rgb.r = saturate(y + 1.57480000 * e); - rgb.g = saturate(y - 0.18732427 * d - 0.46812427 * e); - rgb.b = saturate(y + 1.85560000 * d); - return rgb; - } - float4 frag(v2f i) : SV_Target { float y = tex2D(_TexY, i.uv).r; float u = tex2D(_TexU, i.uv).r; float v = tex2D(_TexV, i.uv).r; - if (_SwapUV >= 0.5) - { - float t = u; u = v; v = t; - } - if (_InvertU >= 0.5) { u = 1.0 - u; } - if (_InvertV >= 0.5) { v = 1.0 - v; } - // Debug views - if (_DebugMode >= 0.5 && _DebugMode < 1.5) { return float4(y, y, y, 1.0); } - if (_DebugMode >= 1.5 && _DebugMode < 2.5) { return float4(u, u, u, 1.0); } - if (_DebugMode >= 2.5 && _DebugMode < 3.5) { return float4(v, v, v, 1.0); } - // Select matrix by _ColorStd and _FullRange - float use709 = step(0.5, _ColorStd); - float useFull = step(0.5, _FullRange); - float3 rgb601Lim = yuvToRgb601Limited(y, u, v); - float3 rgb709Lim = yuvToRgb709Limited(y, u, v); - float3 rgb601Full = yuvToRgb601Full(y, u, v); - float3 rgb709Full = yuvToRgb709Full(y, u, v); - float3 rgbLim = lerp(rgb601Lim, rgb709Lim, use709); - float3 rgbFull = lerp(rgb601Full, rgb709Full, use709); - float3 rgb = lerp(rgbLim, rgbFull, useFull); + float3 rgb = yuvToRgb709Limited(y, u, v); return float4(rgb, 1.0); } ENDHLSL From 104617630c11e8116b7de5d287b0af84cca084e5 Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 7 Nov 2025 14:05:43 -0800 Subject: [PATCH 03/11] clean up add comments --- Runtime/Scripts/VideoStream.cs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index 9fb7cb2c..49d170d1 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -17,9 +17,7 @@ public class VideoStream private VideoStreamInfo _info; private bool _disposed = false; private bool _dirty = false; - private bool _useGpuYuvToRgb = true; - private string _lastColorConversionPathLog; - // Fixed baseline: BT.709 limited, no UV swap + private bool _useGpuShader = true; private Material _yuvToRgbMaterial; private Texture2D _planeY; @@ -107,6 +105,7 @@ public virtual void Stop() _playing = false; } + // Ensure the output render texture is created and sized correctly private bool EnsureRenderTexture(int width, int height) { var textureChanged = false; @@ -125,6 +124,7 @@ private bool EnsureRenderTexture(int width, int height) return textureChanged; } + // Ensure the GPU YUV->RGB material is available private void EnsureGpuMaterial() { if (_yuvToRgbMaterial == null) @@ -135,6 +135,7 @@ private void EnsureGpuMaterial() } } + // Ensure or recreate a plane texture with given format and filter private static void EnsurePlaneTexture(ref Texture2D tex, int width, int height, TextureFormat format, FilterMode filterMode) { if (tex == null || tex.width != width || tex.height != height) @@ -146,6 +147,7 @@ private static void EnsurePlaneTexture(ref Texture2D tex, int width, int height, } } + // Ensure Y, U, V plane textures exist with correct dimensions private void EnsureYuvPlaneTextures(int width, int height) { EnsurePlaneTexture(ref _planeY, width, height, TextureFormat.R8, FilterMode.Bilinear); @@ -155,6 +157,7 @@ private void EnsureYuvPlaneTextures(int width, int height) EnsurePlaneTexture(ref _planeV, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); } + // Upload raw Y, U, V plane bytes from VideoBuffer to textures private void UploadYuvPlanes() { var info = VideoBuffer.Info; @@ -171,6 +174,7 @@ private void UploadYuvPlanes() _planeV.Apply(false, false); } + // CPU-side conversion to RGBA and blit to the render target private void CpuConvertToRenderTarget(int width, int height) { var rgba = VideoBuffer.ToRGBA(); @@ -182,6 +186,7 @@ private void CpuConvertToRenderTarget(int width, int height) rgba.Dispose(); } + // GPU-side YUV->RGB conversion using shader material private void GpuConvertToRenderTarget() { _yuvToRgbMaterial.SetTexture("_TexY", _planeY); @@ -190,14 +195,6 @@ private void GpuConvertToRenderTarget() Graphics.Blit(Texture2D.blackTexture, _convertRt, _yuvToRgbMaterial); } - private void LogConversionPath(string path) - { - if (_lastColorConversionPathLog == path) - return; - _lastColorConversionPathLog = path; - Debug.Log($"[LiveKit] VideoStream color conversion: {path}"); - } - public IEnumerator Update() { while (_playing) @@ -216,7 +213,7 @@ public IEnumerator Update() var textureChanged = EnsureRenderTexture((int)rWidth, (int)rHeight); - if (_useGpuYuvToRgb) + if (_useGpuShader) { EnsureGpuMaterial(); EnsureYuvPlaneTextures((int)rWidth, (int)rHeight); @@ -224,18 +221,15 @@ public IEnumerator Update() if (_yuvToRgbMaterial != null) { - LogConversionPath("GPU shader YUV->RGB (BT.709 limited)"); GpuConvertToRenderTarget(); } else { - LogConversionPath("CPU conversion (shader not found)"); CpuConvertToRenderTarget((int)rWidth, (int)rHeight); } } else { - LogConversionPath("CPU conversion (_useGpuYuvToRgb=false)"); CpuConvertToRenderTarget((int)rWidth, (int)rHeight); } @@ -248,6 +242,7 @@ public IEnumerator Update() yield break; } + // Handle new video stream events private void OnVideoStreamEvent(VideoStreamEvent e) { if (e.StreamHandle != (ulong)Handle.DangerousGetHandle()) From 269d26ebeec99c6a28ed0cbe3bb43b8809260483 Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 7 Nov 2025 14:39:50 -0800 Subject: [PATCH 04/11] refactor the conversion code into new class --- Runtime/Scripts/Video.meta | 8 + Runtime/Scripts/Video/YuvToRgbConverter.cs | 149 ++++++++++++++++++ .../Scripts/Video/YuvToRgbConverter.cs.meta | 2 + Runtime/Scripts/VideoStream.cs | 136 ++-------------- 4 files changed, 168 insertions(+), 127 deletions(-) create mode 100644 Runtime/Scripts/Video.meta create mode 100644 Runtime/Scripts/Video/YuvToRgbConverter.cs create mode 100644 Runtime/Scripts/Video/YuvToRgbConverter.cs.meta diff --git a/Runtime/Scripts/Video.meta b/Runtime/Scripts/Video.meta new file mode 100644 index 00000000..0071aac6 --- /dev/null +++ b/Runtime/Scripts/Video.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 983e1a7c3ee694548a8ac02eb7fd6a20 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Scripts/Video/YuvToRgbConverter.cs b/Runtime/Scripts/Video/YuvToRgbConverter.cs new file mode 100644 index 00000000..49268311 --- /dev/null +++ b/Runtime/Scripts/Video/YuvToRgbConverter.cs @@ -0,0 +1,149 @@ +using System; +using UnityEngine; + +namespace LiveKit +{ + // Converts I420 YUV frames to RGBA into an output RenderTexture, via GPU shader or CPU fallback. + internal sealed class YuvToRgbConverter : IDisposable + { + public bool UseGpuShader { get; set; } = true; + public RenderTexture Output { get; private set; } + + private Material _yuvToRgbMaterial; + private Texture2D _planeY; + private Texture2D _planeU; + private Texture2D _planeV; + + // Ensure Output exists and matches the given size; returns true if created or resized. + public bool EnsureOutput(int width, int height) + { + var changed = false; + if (Output == null || Output.width != width || Output.height != height) + { + if (Output != null) + { + Output.Release(); + UnityEngine.Object.Destroy(Output); + } + Output = new RenderTexture(width, height, 0, RenderTextureFormat.ARGB32); + Output.Create(); + changed = true; + } + return changed; + } + + // Convert the given buffer to RGBA and write into Output. + public void Convert(VideoFrameBuffer buffer) + { + if (buffer == null || !buffer.IsValid) + return; + + int width = (int)buffer.Width; + int height = (int)buffer.Height; + + EnsureOutput(width, height); + + if (UseGpuShader) + { + EnsureGpuMaterial(); + EnsureYuvPlaneTextures(width, height); + UploadYuvPlanes(buffer); + + if (_yuvToRgbMaterial != null) + { + GpuConvertToRenderTarget(); + return; + } + // fall through to CPU if shader missing + } + + CpuConvertToRenderTarget(buffer, width, height); + } + + // Release all Unity resources (RT, material, textures). + public void Dispose() + { + if (_planeY != null) UnityEngine.Object.Destroy(_planeY); + if (_planeU != null) UnityEngine.Object.Destroy(_planeU); + if (_planeV != null) UnityEngine.Object.Destroy(_planeV); + if (Output != null) + { + Output.Release(); + UnityEngine.Object.Destroy(Output); + } + if (_yuvToRgbMaterial != null) UnityEngine.Object.Destroy(_yuvToRgbMaterial); + } + + // Ensure the GPU YUV->RGB material exists. + private void EnsureGpuMaterial() + { + if (_yuvToRgbMaterial == null) + { + var shader = Shader.Find("Hidden/LiveKit/YUV2RGB"); + if (shader != null) + _yuvToRgbMaterial = new Material(shader); + } + } + + // Ensure or recreate a plane texture with given format and filter settings. + private static void EnsurePlaneTexture(ref Texture2D tex, int width, int height, TextureFormat format, FilterMode filterMode) + { + if (tex == null || tex.width != width || tex.height != height) + { + if (tex != null) UnityEngine.Object.Destroy(tex); + tex = new Texture2D(width, height, format, false, true); + tex.filterMode = filterMode; + tex.wrapMode = TextureWrapMode.Clamp; + } + } + + // Ensure Y, U, V plane textures exist with correct dimensions. + private void EnsureYuvPlaneTextures(int width, int height) + { + EnsurePlaneTexture(ref _planeY, width, height, TextureFormat.R8, FilterMode.Bilinear); + var chromaW = width / 2; + var chromaH = height / 2; + EnsurePlaneTexture(ref _planeU, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); + EnsurePlaneTexture(ref _planeV, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); + } + + // Upload raw Y, U, V plane bytes from buffer to textures. + private void UploadYuvPlanes(VideoFrameBuffer buffer) + { + var info = buffer.Info; + if (info.Components.Count < 3) return; + var yComp = info.Components[0]; + var uComp = info.Components[1]; + var vComp = info.Components[2]; + + _planeY.LoadRawTextureData((IntPtr)yComp.DataPtr, (int)yComp.Size); + _planeY.Apply(false, false); + _planeU.LoadRawTextureData((IntPtr)uComp.DataPtr, (int)uComp.Size); + _planeU.Apply(false, false); + _planeV.LoadRawTextureData((IntPtr)vComp.DataPtr, (int)vComp.Size); + _planeV.Apply(false, false); + } + + // CPU-side conversion to RGBA and blit to the output render target. + private void CpuConvertToRenderTarget(VideoFrameBuffer buffer, int width, int height) + { + var rgba = buffer.ToRGBA(); + var tempTex = new Texture2D(width, height, TextureFormat.RGBA32, false); + tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); + tempTex.Apply(); + Graphics.Blit(tempTex, Output); + UnityEngine.Object.Destroy(tempTex); + rgba.Dispose(); + } + + // GPU-side YUV->RGB conversion using shader material. + private void GpuConvertToRenderTarget() + { + _yuvToRgbMaterial.SetTexture("_TexY", _planeY); + _yuvToRgbMaterial.SetTexture("_TexU", _planeU); + _yuvToRgbMaterial.SetTexture("_TexV", _planeV); + Graphics.Blit(Texture2D.blackTexture, Output, _yuvToRgbMaterial); + } + } +} + diff --git a/Runtime/Scripts/Video/YuvToRgbConverter.cs.meta b/Runtime/Scripts/Video/YuvToRgbConverter.cs.meta new file mode 100644 index 00000000..549d7c17 --- /dev/null +++ b/Runtime/Scripts/Video/YuvToRgbConverter.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: de214c4761ddb4f38b8aa37b1f25df40 \ No newline at end of file diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index 49d170d1..49403c01 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -18,12 +18,7 @@ public class VideoStream private bool _disposed = false; private bool _dirty = false; private bool _useGpuShader = true; - - private Material _yuvToRgbMaterial; - private Texture2D _planeY; - private Texture2D _planeU; - private Texture2D _planeV; - private RenderTexture _convertRt; + private YuvToRgbConverter _converter; /// Called when we receive a new frame from the VideoTrack public event FrameReceiveDelegate FrameReceived; @@ -80,16 +75,9 @@ private void Dispose(bool disposing) { VideoBuffer?.Dispose(); } - // Unity objects must be destroyed on main thread; RT is destroyed below - if (_planeY != null) UnityEngine.Object.Destroy(_planeY); - if (_planeU != null) UnityEngine.Object.Destroy(_planeU); - if (_planeV != null) UnityEngine.Object.Destroy(_planeV); - if (_convertRt != null) - { - _convertRt.Release(); - UnityEngine.Object.Destroy(_convertRt); - } - if (_yuvToRgbMaterial != null) UnityEngine.Object.Destroy(_yuvToRgbMaterial); + // Unity objects must be destroyed on main thread + _converter?.Dispose(); + _converter = null; _disposed = true; } } @@ -105,96 +93,6 @@ public virtual void Stop() _playing = false; } - // Ensure the output render texture is created and sized correctly - private bool EnsureRenderTexture(int width, int height) - { - var textureChanged = false; - if (_convertRt == null || _convertRt.width != width || _convertRt.height != height) - { - if (_convertRt != null) - { - _convertRt.Release(); - UnityEngine.Object.Destroy(_convertRt); - } - _convertRt = new RenderTexture(width, height, 0, RenderTextureFormat.ARGB32); - _convertRt.Create(); - Texture = _convertRt; - textureChanged = true; - } - return textureChanged; - } - - // Ensure the GPU YUV->RGB material is available - private void EnsureGpuMaterial() - { - if (_yuvToRgbMaterial == null) - { - var shader = Shader.Find("Hidden/LiveKit/YUV2RGB"); - if (shader != null) - _yuvToRgbMaterial = new Material(shader); - } - } - - // Ensure or recreate a plane texture with given format and filter - private static void EnsurePlaneTexture(ref Texture2D tex, int width, int height, TextureFormat format, FilterMode filterMode) - { - if (tex == null || tex.width != width || tex.height != height) - { - if (tex != null) UnityEngine.Object.Destroy(tex); - tex = new Texture2D(width, height, format, false, true); - tex.filterMode = filterMode; - tex.wrapMode = TextureWrapMode.Clamp; - } - } - - // Ensure Y, U, V plane textures exist with correct dimensions - private void EnsureYuvPlaneTextures(int width, int height) - { - EnsurePlaneTexture(ref _planeY, width, height, TextureFormat.R8, FilterMode.Bilinear); - var chromaW = width / 2; - var chromaH = height / 2; - EnsurePlaneTexture(ref _planeU, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); - EnsurePlaneTexture(ref _planeV, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); - } - - // Upload raw Y, U, V plane bytes from VideoBuffer to textures - private void UploadYuvPlanes() - { - var info = VideoBuffer.Info; - if (info.Components.Count < 3) return; - var yComp = info.Components[0]; - var uComp = info.Components[1]; - var vComp = info.Components[2]; - - _planeY.LoadRawTextureData((IntPtr)yComp.DataPtr, (int)yComp.Size); - _planeY.Apply(false, false); - _planeU.LoadRawTextureData((IntPtr)uComp.DataPtr, (int)uComp.Size); - _planeU.Apply(false, false); - _planeV.LoadRawTextureData((IntPtr)vComp.DataPtr, (int)vComp.Size); - _planeV.Apply(false, false); - } - - // CPU-side conversion to RGBA and blit to the render target - private void CpuConvertToRenderTarget(int width, int height) - { - var rgba = VideoBuffer.ToRGBA(); - var tempTex = new Texture2D(width, height, TextureFormat.RGBA32, false); - tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); - tempTex.Apply(); - Graphics.Blit(tempTex, _convertRt); - UnityEngine.Object.Destroy(tempTex); - rgba.Dispose(); - } - - // GPU-side YUV->RGB conversion using shader material - private void GpuConvertToRenderTarget() - { - _yuvToRgbMaterial.SetTexture("_TexY", _planeY); - _yuvToRgbMaterial.SetTexture("_TexU", _planeU); - _yuvToRgbMaterial.SetTexture("_TexV", _planeV); - Graphics.Blit(Texture2D.blackTexture, _convertRt, _yuvToRgbMaterial); - } - public IEnumerator Update() { while (_playing) @@ -211,27 +109,11 @@ public IEnumerator Update() var rWidth = VideoBuffer.Width; var rHeight = VideoBuffer.Height; - var textureChanged = EnsureRenderTexture((int)rWidth, (int)rHeight); - - if (_useGpuShader) - { - EnsureGpuMaterial(); - EnsureYuvPlaneTextures((int)rWidth, (int)rHeight); - UploadYuvPlanes(); - - if (_yuvToRgbMaterial != null) - { - GpuConvertToRenderTarget(); - } - else - { - CpuConvertToRenderTarget((int)rWidth, (int)rHeight); - } - } - else - { - CpuConvertToRenderTarget((int)rWidth, (int)rHeight); - } + if (_converter == null) _converter = new YuvToRgbConverter(); + _converter.UseGpuShader = _useGpuShader; + var textureChanged = _converter.EnsureOutput((int)rWidth, (int)rHeight); + _converter.Convert(VideoBuffer); + if (textureChanged) Texture = _converter.Output; if (textureChanged) TextureReceived?.Invoke(Texture); From ff897484140cedb4788dd1bae99e498a48faddd4 Mon Sep 17 00:00:00 2001 From: David Chen Date: Wed, 19 Nov 2025 09:18:07 -0800 Subject: [PATCH 05/11] Update Runtime/Scripts/VideoStream.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Runtime/Scripts/VideoStream.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index 49403c01..0dce265e 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -78,6 +78,8 @@ private void Dispose(bool disposing) // Unity objects must be destroyed on main thread _converter?.Dispose(); _converter = null; + // Texture is owned and cleaned up by _converter. Set to null to avoid holding a reference to a disposed RenderTexture. + Texture = null; _disposed = true; } } From 18bdf278650a2c751ef389b2d2f66490618adaf4 Mon Sep 17 00:00:00 2001 From: David Chen Date: Wed, 19 Nov 2025 09:18:35 -0800 Subject: [PATCH 06/11] Update Runtime/Scripts/Video/YuvToRgbConverter.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Runtime/Scripts/Video/YuvToRgbConverter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Runtime/Scripts/Video/YuvToRgbConverter.cs b/Runtime/Scripts/Video/YuvToRgbConverter.cs index 49268311..0f1acdad 100644 --- a/Runtime/Scripts/Video/YuvToRgbConverter.cs +++ b/Runtime/Scripts/Video/YuvToRgbConverter.cs @@ -103,8 +103,8 @@ private void EnsureYuvPlaneTextures(int width, int height) EnsurePlaneTexture(ref _planeY, width, height, TextureFormat.R8, FilterMode.Bilinear); var chromaW = width / 2; var chromaH = height / 2; - EnsurePlaneTexture(ref _planeU, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); - EnsurePlaneTexture(ref _planeV, chromaW, chromaH, TextureFormat.R8, FilterMode.Point); + EnsurePlaneTexture(ref _planeU, chromaW, chromaH, TextureFormat.R8, FilterMode.Bilinear); + EnsurePlaneTexture(ref _planeV, chromaW, chromaH, TextureFormat.R8, FilterMode.Bilinear); } // Upload raw Y, U, V plane bytes from buffer to textures. From 5505ea9db0b1ebb4cdb72a095995f06eab7df8ee Mon Sep 17 00:00:00 2001 From: David Chen Date: Wed, 19 Nov 2025 09:18:56 -0800 Subject: [PATCH 07/11] Update Runtime/Scripts/Video/YuvToRgbConverter.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Runtime/Scripts/Video/YuvToRgbConverter.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Runtime/Scripts/Video/YuvToRgbConverter.cs b/Runtime/Scripts/Video/YuvToRgbConverter.cs index 0f1acdad..32f9072b 100644 --- a/Runtime/Scripts/Video/YuvToRgbConverter.cs +++ b/Runtime/Scripts/Video/YuvToRgbConverter.cs @@ -129,11 +129,17 @@ private void CpuConvertToRenderTarget(VideoFrameBuffer buffer, int width, int he { var rgba = buffer.ToRGBA(); var tempTex = new Texture2D(width, height, TextureFormat.RGBA32, false); - tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); - tempTex.Apply(); - Graphics.Blit(tempTex, Output); - UnityEngine.Object.Destroy(tempTex); - rgba.Dispose(); + try + { + tempTex.LoadRawTextureData((IntPtr)rgba.Info.DataPtr, (int)rgba.GetMemorySize()); + tempTex.Apply(); + Graphics.Blit(tempTex, Output); + } + finally + { + UnityEngine.Object.Destroy(tempTex); + rgba.Dispose(); + } } // GPU-side YUV->RGB conversion using shader material. From 1840ea5ff060c5a77a059b56f02d6804f8031530 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 2 Dec 2025 11:45:08 -0800 Subject: [PATCH 08/11] remove useGpuShader bool, always try to use shader first --- Runtime/Scripts/VideoStream.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index 0dce265e..eadd9ce8 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -17,7 +17,6 @@ public class VideoStream private VideoStreamInfo _info; private bool _disposed = false; private bool _dirty = false; - private bool _useGpuShader = true; private YuvToRgbConverter _converter; /// Called when we receive a new frame from the VideoTrack @@ -112,7 +111,6 @@ public IEnumerator Update() var rHeight = VideoBuffer.Height; if (_converter == null) _converter = new YuvToRgbConverter(); - _converter.UseGpuShader = _useGpuShader; var textureChanged = _converter.EnsureOutput((int)rWidth, (int)rHeight); _converter.Convert(VideoBuffer); if (textureChanged) Texture = _converter.Output; From ebfe92925ea130829a5b21b334e378f154b44b64 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 2 Dec 2025 15:21:23 -0800 Subject: [PATCH 09/11] flip texture horizontally --- Runtime/Shaders/YuvToRgb.shader | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Runtime/Shaders/YuvToRgb.shader b/Runtime/Shaders/YuvToRgb.shader index 69631b30..0bef4ef7 100644 --- a/Runtime/Shaders/YuvToRgb.shader +++ b/Runtime/Shaders/YuvToRgb.shader @@ -52,9 +52,12 @@ Shader "Hidden/LiveKit/YUV2RGB" float4 frag(v2f i) : SV_Target { - float y = tex2D(_TexY, i.uv).r; - float u = tex2D(_TexU, i.uv).r; - float v = tex2D(_TexV, i.uv).r; + // Flip horizontally to match Unity's texture orientation with incoming YUV data + float2 uv = float2(1.0 - i.uv.x, i.uv.y); + + float y = tex2D(_TexY, uv).r; + float u = tex2D(_TexU, uv).r; + float v = tex2D(_TexV, uv).r; float3 rgb = yuvToRgb709Limited(y, u, v); return float4(rgb, 1.0); } From 62b9be0cc5ea92e2333a42c18912c5e45591b2fe Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 2 Dec 2025 15:29:39 -0800 Subject: [PATCH 10/11] use half in shader --- Runtime/Shaders/YuvToRgb.shader | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Runtime/Shaders/YuvToRgb.shader b/Runtime/Shaders/YuvToRgb.shader index 0bef4ef7..2f9489a8 100644 --- a/Runtime/Shaders/YuvToRgb.shader +++ b/Runtime/Shaders/YuvToRgb.shader @@ -36,30 +36,31 @@ Shader "Hidden/LiveKit/YUV2RGB" return o; } - float3 yuvToRgb709Limited(float y, float u, float v) + inline half3 yuvToRgb709Limited(half y, half u, half v) { // BT.709 limited range - float c = y - 16.0 / 255.0; - float d = u - 128.0 / 255.0; - float e = v - 128.0 / 255.0; + half c = y - half(16.0 / 255.0); + half d = u - half(128.0 / 255.0); + half e = v - half(128.0 / 255.0); - float3 rgb; - rgb.r = saturate(1.16438356 * c + 1.79274107 * e); - rgb.g = saturate(1.16438356 * c - 0.21324861 * d - 0.53290933 * e); - rgb.b = saturate(1.16438356 * c + 2.11240179 * d); - return rgb; + half Y = half(1.16438356) * c; + + half3 rgb; + rgb.r = Y + half(1.79274107) * e; + rgb.g = Y - half(0.21324861) * d - half(0.53290933) * e; + rgb.b = Y + half(2.11240179) * d; + return saturate(rgb); } - float4 frag(v2f i) : SV_Target + half4 frag(v2f i) : SV_Target { // Flip horizontally to match Unity's texture orientation with incoming YUV data - float2 uv = float2(1.0 - i.uv.x, i.uv.y); + half2 uv = half2(1.0h - i.uv.x, i.uv.y); - float y = tex2D(_TexY, uv).r; - float u = tex2D(_TexU, uv).r; - float v = tex2D(_TexV, uv).r; - float3 rgb = yuvToRgb709Limited(y, u, v); - return float4(rgb, 1.0); + half y = tex2D(_TexY, uv).r; + half u = tex2D(_TexU, uv).r; + half v = tex2D(_TexV, uv).r; + return half4(yuvToRgb709Limited(y, u, v), 1.0h); } ENDHLSL } From bdfdaaec5764a798269cab22e0a6ce051b66d991 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 2 Dec 2025 15:43:43 -0800 Subject: [PATCH 11/11] use half for vertex --- Runtime/Shaders/YuvToRgb.shader | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Runtime/Shaders/YuvToRgb.shader b/Runtime/Shaders/YuvToRgb.shader index 2f9489a8..1ef01af5 100644 --- a/Runtime/Shaders/YuvToRgb.shader +++ b/Runtime/Shaders/YuvToRgb.shader @@ -25,14 +25,14 @@ Shader "Hidden/LiveKit/YUV2RGB" struct v2f { float4 pos : SV_POSITION; - float2 uv : TEXCOORD0; + half2 uv : TEXCOORD0; }; v2f vert(appdata v) { v2f o; o.pos = UnityObjectToClipPos(v.vertex); - o.uv = v.uv; + o.uv = half2(v.uv); return o; }