From 31ce9d8c393b86529c92b98ec5fa43dbdd9b5a1b Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Tue, 19 May 2026 17:13:09 +0200 Subject: [PATCH 1/5] Expose FrameMetadata send/receive for video frames Surfaces the Rust FFI Frame Packet Trailer feature in the Unity SDK: attach FrameMetadata (user_timestamp, frame_id) to outgoing frames via RtcVideoSource.MetadataProvider, and read it off VideoFrame.Metadata on received frames. PacketTrailerFeatures on TrackPublishOptions is already proto-passthrough so no wrapper change is needed there. Adds a PlayMode E2E test that publishes with both packet-trailer features enabled, attaches known metadata to outgoing frames, and asserts the subscriber sees matching frame_id and user_timestamp. Co-Authored-By: Claude Opus 4.7 (1M context) --- Runtime/Scripts/RtcVideoSource.cs | 8 ++ Runtime/Scripts/VideoFrame.cs | 4 +- Runtime/Scripts/VideoStream.cs | 2 +- Tests/PlayMode/Utils/CoroutineRunner.cs | 12 ++ Tests/PlayMode/Utils/CoroutineRunner.cs.meta | 11 ++ .../PlayMode/Utils/MetadataTestVideoSource.cs | 43 ++++++++ .../Utils/MetadataTestVideoSource.cs.meta | 11 ++ Tests/PlayMode/VideoFrameMetadataTests.cs | 103 ++++++++++++++++++ .../PlayMode/VideoFrameMetadataTests.cs.meta | 11 ++ 9 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 Tests/PlayMode/Utils/CoroutineRunner.cs create mode 100644 Tests/PlayMode/Utils/CoroutineRunner.cs.meta create mode 100644 Tests/PlayMode/Utils/MetadataTestVideoSource.cs create mode 100644 Tests/PlayMode/Utils/MetadataTestVideoSource.cs.meta create mode 100644 Tests/PlayMode/VideoFrameMetadataTests.cs create mode 100644 Tests/PlayMode/VideoFrameMetadataTests.cs.meta diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs index 5f62c671..1ef072d2 100644 --- a/Runtime/Scripts/RtcVideoSource.cs +++ b/Runtime/Scripts/RtcVideoSource.cs @@ -30,6 +30,12 @@ public enum VideoStreamSource /// Called when we receive a new texture (first texture or the resolution changed) public event TextureReceiveDelegate TextureReceived; + public delegate FrameMetadata FrameMetadataDelegate(); + /// Invoked once per outgoing frame. Return null (default) to send no trailer. + /// To actually serialize the trailer onto RTP, also enable the matching + /// PacketTrailerFeatures on the TrackPublishOptions used at publish time. + public FrameMetadataDelegate MetadataProvider { get; set; } + protected Texture2D _previewTexture; protected NativeArray _captureBuffer; protected VideoStreamSource _sourceType; @@ -175,6 +181,8 @@ protected virtual bool SendFrame() var now = DateTimeOffset.UtcNow; capture.TimestampUs = now.ToUnixTimeMilliseconds() * 1000 + (now.Ticks % TimeSpan.TicksPerMillisecond) / 10; capture.Buffer = buffer; + var metadata = MetadataProvider?.Invoke(); + if (metadata != null) capture.Metadata = metadata; using var response = request.Send(); _reading = false; _requestPending = false; diff --git a/Runtime/Scripts/VideoFrame.cs b/Runtime/Scripts/VideoFrame.cs index 7d263dd7..e2bf66d1 100644 --- a/Runtime/Scripts/VideoFrame.cs +++ b/Runtime/Scripts/VideoFrame.cs @@ -12,12 +12,14 @@ public sealed class VideoFrame public long Timestamp; public VideoRotation Rotation; + public FrameMetadata Metadata; - public VideoFrame(VideoBufferInfo info, long timeStamp, VideoRotation rotation) + public VideoFrame(VideoBufferInfo info, long timeStamp, VideoRotation rotation, FrameMetadata metadata = null) { _info = info; Timestamp = timeStamp; Rotation = rotation; + Metadata = metadata; } } diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index 8d2fa5e9..0c1a148d 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -242,7 +242,7 @@ private void OnVideoStreamEvent(VideoStreamEvent e) // Avoid allocating VideoFrame objects when nobody is observing them. if (FrameReceived != null) { - var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation); + var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation, e.FrameReceived.Metadata); FrameReceived.Invoke(frame); } } diff --git a/Tests/PlayMode/Utils/CoroutineRunner.cs b/Tests/PlayMode/Utils/CoroutineRunner.cs new file mode 100644 index 00000000..b8f6db60 --- /dev/null +++ b/Tests/PlayMode/Utils/CoroutineRunner.cs @@ -0,0 +1,12 @@ +using UnityEngine; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// Empty MonoBehaviour used by tests to host long-running coroutines + /// (e.g. and ) + /// that the test itself cannot host because [UnityTest] bodies must yield + /// back to the test runner. + /// + public class CoroutineRunner : MonoBehaviour { } +} diff --git a/Tests/PlayMode/Utils/CoroutineRunner.cs.meta b/Tests/PlayMode/Utils/CoroutineRunner.cs.meta new file mode 100644 index 00000000..76434df5 --- /dev/null +++ b/Tests/PlayMode/Utils/CoroutineRunner.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 85dedc359b4346d1bedd01817cedf75e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/Utils/MetadataTestVideoSource.cs b/Tests/PlayMode/Utils/MetadataTestVideoSource.cs new file mode 100644 index 00000000..3d53f655 --- /dev/null +++ b/Tests/PlayMode/Utils/MetadataTestVideoSource.cs @@ -0,0 +1,43 @@ +using LiveKit.Proto; +using Unity.Collections; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// Test-only that pushes a continuous stream of + /// zero-filled RGBA frames at the resolution given to the constructor. Used + /// by tests that need actual media flow (e.g. validating per-frame metadata + /// round-trips through the FFI / RTP path). + /// + public sealed class MetadataTestVideoSource : RtcVideoSource + { + private readonly int _width; + private readonly int _height; + + public override int GetWidth() => _width; + public override int GetHeight() => _height; + + protected override VideoRotation GetVideoRotation() => VideoRotation._0; + + protected override bool ReadBuffer() + { + if (!_captureBuffer.IsCreated) + { + _captureBuffer = new NativeArray( + _width * _height * 4, + Allocator.Persistent, + NativeArrayOptions.ClearMemory); + } + _requestPending = true; + return false; + } + + public MetadataTestVideoSource(int width = 16, int height = 16) + : base(VideoStreamSource.Texture, VideoBufferType.Rgba) + { + _width = width; + _height = height; + Init(); + } + } +} diff --git a/Tests/PlayMode/Utils/MetadataTestVideoSource.cs.meta b/Tests/PlayMode/Utils/MetadataTestVideoSource.cs.meta new file mode 100644 index 00000000..31ee6d4c --- /dev/null +++ b/Tests/PlayMode/Utils/MetadataTestVideoSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: defea528e82a4c8d85faf7574ea6f698 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/VideoFrameMetadataTests.cs b/Tests/PlayMode/VideoFrameMetadataTests.cs new file mode 100644 index 00000000..807e55ae --- /dev/null +++ b/Tests/PlayMode/VideoFrameMetadataTests.cs @@ -0,0 +1,103 @@ +using System.Collections; +using LiveKit.PlayModeTests.Utils; +using LiveKit.Proto; +using NUnit.Framework; +using UnityEngine; +using UnityEngine.TestTools; + +namespace LiveKit.PlayModeTests +{ + public class VideoFrameMetadataTests + { + const string VideoTrackName = "metadata-video-track"; + + static (TestRoomContext.ConnectionOptions publisher, TestRoomContext.ConnectionOptions subscriber) TwoPeers() + { + var publisher = TestRoomContext.ConnectionOptions.Default; + publisher.Identity = "metadata-publisher"; + var subscriber = TestRoomContext.ConnectionOptions.Default; + subscriber.Identity = "metadata-subscriber"; + return (publisher, subscriber); + } + + [UnityTest, Category("E2E")] + public IEnumerator VideoFrame_AttachedMetadata_ReceivedOnSubscriber() + { + var (publisher, subscriber) = TwoPeers(); + using var context = new TestRoomContext(new[] { publisher, subscriber }); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + + var publisherRoom = context.Rooms[0]; + var subscriberRoom = context.Rooms[1]; + + const uint expectedFrameId = 42u; + const ulong expectedUserTs = 0x1122334455667788UL; + + var source = new MetadataTestVideoSource(); + source.MetadataProvider = () => new FrameMetadata + { + FrameId = expectedFrameId, + UserTimestamp = expectedUserTs, + }; + + var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, source, publisherRoom); + + var subscribedExp = new Expectation(timeoutSeconds: 10f); + RemoteVideoTrack receivedRemoteTrack = null; + subscriberRoom.TrackSubscribed += (track, _, _) => + { + if (track is RemoteVideoTrack rv) + { + receivedRemoteTrack = rv; + subscribedExp.Fulfill(); + } + }; + + var options = new TrackPublishOptions + { + Source = TrackSource.SourceCamera, + }.WithPacketTrailerFeatures( + PacketTrailerFeature.PtfUserTimestamp, + PacketTrailerFeature.PtfFrameId); + var pub = publisherRoom.LocalParticipant.PublishTrack(localTrack, options); + yield return pub; + Assert.IsFalse(pub.IsError); + + // Host the source's Update coroutine on a throwaway MonoBehaviour so the + // test body can yield on Expectations without owning the producer loop. + var runnerObj = new GameObject("metadata-test-runner"); + var runner = runnerObj.AddComponent(); + source.Start(); + runner.StartCoroutine(source.Update()); + + yield return subscribedExp.Wait(); + Assert.IsNull(subscribedExp.Error); + Assert.IsNotNull(receivedRemoteTrack); + + var stream = new VideoStream(receivedRemoteTrack); + var metadataExp = new Expectation(timeoutSeconds: 10f); + FrameMetadata receivedMetadata = null; + stream.FrameReceived += frame => + { + if (receivedMetadata != null || frame.Metadata == null) return; + receivedMetadata = frame.Metadata; + metadataExp.Fulfill(); + }; + stream.Start(); + runner.StartCoroutine(stream.Update()); + + yield return metadataExp.Wait(); + Assert.IsNull(metadataExp.Error, "expected a frame with metadata within timeout"); + Assert.IsNotNull(receivedMetadata); + Assert.AreEqual(expectedFrameId, receivedMetadata.FrameId, "frame_id mismatch"); + Assert.AreEqual(expectedUserTs, receivedMetadata.UserTimestamp, "user_timestamp mismatch"); + + source.Stop(); + stream.Stop(); + stream.Dispose(); + source.Dispose(); + Object.Destroy(runnerObj); + } + } +} diff --git a/Tests/PlayMode/VideoFrameMetadataTests.cs.meta b/Tests/PlayMode/VideoFrameMetadataTests.cs.meta new file mode 100644 index 00000000..35011034 --- /dev/null +++ b/Tests/PlayMode/VideoFrameMetadataTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b2926ee0c31a473294e3ecdcddd0a227 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From d041daba86cb8814f31c0bc3fbe92726026a1876 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Tue, 19 May 2026 17:18:16 +0200 Subject: [PATCH 2/5] Add TrackPublishOptionsExtensions.WithPacketTrailerFeatures Provides a Google.Protobuf-free entry point for enabling packet trailer features on TrackPublishOptions. Required because the LiveKit SDK ships Google.Protobuf.dll as an explicitly-referenced plugin (deliberate, to avoid conflicts with com.unity.ai.assistant), so a Unity project's default Assembly-CSharp cannot use the proto's repeated-field collection initializer or call .Add on PacketTrailerFeatures directly. Co-Authored-By: Claude Opus 4.7 (1M context) --- Runtime/Scripts/Participant.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Runtime/Scripts/Participant.cs b/Runtime/Scripts/Participant.cs index bc49710b..4da028c0 100644 --- a/Runtime/Scripts/Participant.cs +++ b/Runtime/Scripts/Participant.cs @@ -1001,4 +1001,21 @@ public LocalDataTrack Track public PublishDataTrackError Error { get; private set; } } + + /// Helpers for setting fields whose underlying type + /// lives in Google.Protobuf (e.g. RepeatedField<T>). Unity's default Assembly-CSharp + /// does not auto-reference Google.Protobuf, so callers without an asmdef cannot use a + /// collection initializer or call .Add on the repeated field directly. These + /// helpers keep Google.Protobuf types out of the caller's signature. + public static class TrackPublishOptionsExtensions + { + public static TrackPublishOptions WithPacketTrailerFeatures( + this TrackPublishOptions options, + params PacketTrailerFeature[] features) + { + foreach (var feature in features) + options.PacketTrailerFeatures.Add(feature); + return options; + } + } } From 08d39e4dd1c1a72ede4410b8a0fe9df27387413f Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Tue, 19 May 2026 17:18:37 +0200 Subject: [PATCH 3/5] Use frame metadata in the Meet sample Publishes the local camera with PTF_USER_TIMESTAMP and PTF_FRAME_ID packet trailers and attaches a monotonic frame_id plus UnixTimeUs user_timestamp to each outgoing frame via the new MetadataProvider hook. Logs received trailer values (throttled to ~1 second per stream) for both main and extra-video tiles, so the Unity console / on-screen ScrollingLog shows the metadata round-tripping from remote publishers. Co-Authored-By: Claude Opus 4.7 (1M context) --- Samples~/Meet/Assets/Runtime/MeetManager.cs | 42 ++++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 225c7a0c..8eab3ff7 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -1,3 +1,4 @@ +using System; using System.Collections; using System.Collections.Generic; using UnityEngine; @@ -46,6 +47,7 @@ public class MeetManager : MonoBehaviour private LocalAudioTrack _localAudioTrack; private bool _cameraActive; private bool _microphoneActive; + private uint _nextOutgoingFrameId; #region Lifecycle @@ -236,6 +238,7 @@ private void BindRemoteCameraToTile(RemoteVideoTrack video, string identity) var stream = new VideoStream(video); stream.TextureReceived += tex => tile.BindLiveSource(tex); + stream.FrameReceived += MakeFrameMetadataLogger(identity); _videoStreams[sid] = stream; stream.Start(); StartCoroutine(stream.Update()); @@ -268,6 +271,7 @@ private void AddExtraVideoTile(RemoteVideoTrack video, string identity) var stream = new VideoStream(video); stream.TextureReceived += tex => tile.BindLiveSource(tex); + stream.FrameReceived += MakeFrameMetadataLogger($"{identity} (screen)"); _extraVideoTiles[sid] = tile; _extraVideoOwners[sid] = identity; @@ -276,6 +280,33 @@ private void AddExtraVideoTile(RemoteVideoTrack video, string identity) StartCoroutine(stream.Update()); } + /// Returns a per-stream that logs the trailer + /// metadata (frame_id, user_timestamp) at most once per second per stream. The throttle + /// clock uses Environment.TickCount because FrameReceived may fire off the main thread. + private static VideoStream.FrameReceiveDelegate MakeFrameMetadataLogger(string label) + { + var nextLogTick = 0; + return frame => + { + var now = Environment.TickCount; + if (now - nextLogTick < 0) return; + nextLogTick = now + 1000; + if (frame.Metadata == null) + { + Debug.Log($"[Meet RX {label}] no trailer"); + return; + } + Debug.Log($"[Meet RX {label}] frame_id={frame.Metadata.FrameId} user_ts={frame.Metadata.UserTimestamp}"); + }; + } + + private static ulong UnixTimeUs() + { + var now = DateTimeOffset.UtcNow; + return (ulong)(now.ToUnixTimeMilliseconds() * 1000 + + (now.Ticks % TimeSpan.TicksPerMillisecond) / 10); + } + private void RemoveExtraVideoTile(string sid) { if (_extraVideoTiles.TryGetValue(sid, out var tile)) @@ -403,6 +434,11 @@ private IEnumerator PublishLocalCamera() var tile = _participantTiles[_localId]; var source = new WebCameraSource(_webCamTexture); + source.MetadataProvider = () => new FrameMetadata + { + UserTimestamp = UnixTimeUs(), + FrameId = unchecked(_nextOutgoingFrameId++), + }; _localVideoTrack = LocalVideoTrack.CreateVideoTrack(LocalVideoTrackName, source, _room); var options = new TrackPublishOptions @@ -410,8 +446,10 @@ private IEnumerator PublishLocalCamera() VideoCodec = VideoCodec.H265, VideoEncoding = new VideoEncoding { MaxBitrate = 512000, MaxFramerate = frameRate }, Simulcast = false, - Source = TrackSource.SourceCamera - }; + Source = TrackSource.SourceCamera, + }.WithPacketTrailerFeatures( + PacketTrailerFeature.PtfUserTimestamp, + PacketTrailerFeature.PtfFrameId); var publish = _room.LocalParticipant.PublishTrack(_localVideoTrack, options); yield return publish; From e982fc10fd905ff2d2d0a9aad0e0f9cbf5325846 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Wed, 20 May 2026 10:18:28 +0200 Subject: [PATCH 4/5] Consolidate StubVideoSource and MetadataTestVideoSource into TestVideoSource Single test helper with a pushFrames flag covers both modes: signaling-only publication propagation (default) and continuous media flow for round-trip tests. Co-Authored-By: Claude Opus 4.7 (1M context) --- Tests/PlayMode/TrackTests.cs | 6 ++-- Tests/PlayMode/Utils/StubVideoSource.cs | 31 ------------------- Tests/PlayMode/Utils/StubVideoSource.cs.meta | 11 ------- ...aTestVideoSource.cs => TestVideoSource.cs} | 22 +++++++++---- ...Source.cs.meta => TestVideoSource.cs.meta} | 2 +- Tests/PlayMode/VideoFrameMetadataTests.cs | 2 +- 6 files changed, 21 insertions(+), 53 deletions(-) delete mode 100644 Tests/PlayMode/Utils/StubVideoSource.cs delete mode 100644 Tests/PlayMode/Utils/StubVideoSource.cs.meta rename Tests/PlayMode/Utils/{MetadataTestVideoSource.cs => TestVideoSource.cs} (51%) rename Tests/PlayMode/Utils/{MetadataTestVideoSource.cs.meta => TestVideoSource.cs.meta} (83%) diff --git a/Tests/PlayMode/TrackTests.cs b/Tests/PlayMode/TrackTests.cs index 88d5416d..62a723bb 100644 --- a/Tests/PlayMode/TrackTests.cs +++ b/Tests/PlayMode/TrackTests.cs @@ -149,7 +149,7 @@ public IEnumerator RemoteTrackPublication_SetVideoQuality_DoesNotThrow() var publisherRoom = context.Rooms[0]; var subscriberRoom = context.Rooms[1]; - var videoSource = new StubVideoSource(); + var videoSource = new TestVideoSource(); var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, videoSource, publisherRoom); // Video track uses a stub source that never pushes frames. TrackSubscribed may not @@ -283,10 +283,10 @@ public IEnumerator RemoteTrackPublication_PublisherDisablesCamera_UpdatesFlagAnd var publisherRoom = context.Rooms[0]; var subscriberRoom = context.Rooms[1]; - var videoSource = new StubVideoSource(); + var videoSource = new TestVideoSource(); var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, videoSource, publisherRoom); - // StubVideoSource never pushes frames, so TrackSubscribed may not fire on the + // TestVideoSource (pushFrames=false) never pushes frames, so TrackSubscribed may not fire on the // subscriber. The RemoteTrackPublication still propagates via TrackPublished. var publishedExp = new Expectation(timeoutSeconds: 10f); subscriberRoom.TrackPublished += (_, _) => publishedExp.Fulfill(); diff --git a/Tests/PlayMode/Utils/StubVideoSource.cs b/Tests/PlayMode/Utils/StubVideoSource.cs deleted file mode 100644 index 865258e1..00000000 --- a/Tests/PlayMode/Utils/StubVideoSource.cs +++ /dev/null @@ -1,31 +0,0 @@ -using LiveKit.Proto; - -namespace LiveKit.PlayModeTests.Utils -{ - /// - /// Test-only that registers with FFI but never - /// pushes frames. Used for tests that only require the video publication - /// to propagate to subscribers via signaling (e.g. RemoteTrackPublication - /// APIs that operate on metadata) without actual media flow. - /// - public sealed class StubVideoSource : RtcVideoSource - { - private readonly int _width; - private readonly int _height; - - public override int GetWidth() => _width; - public override int GetHeight() => _height; - - protected override VideoRotation GetVideoRotation() => VideoRotation._0; - - protected override bool ReadBuffer() => false; - - public StubVideoSource(int width = 16, int height = 16) - : base(VideoStreamSource.Texture, VideoBufferType.Rgba) - { - _width = width; - _height = height; - Init(); - } - } -} diff --git a/Tests/PlayMode/Utils/StubVideoSource.cs.meta b/Tests/PlayMode/Utils/StubVideoSource.cs.meta deleted file mode 100644 index 791a3108..00000000 --- a/Tests/PlayMode/Utils/StubVideoSource.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 158c662279bb240c387fa142f26314df -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Tests/PlayMode/Utils/MetadataTestVideoSource.cs b/Tests/PlayMode/Utils/TestVideoSource.cs similarity index 51% rename from Tests/PlayMode/Utils/MetadataTestVideoSource.cs rename to Tests/PlayMode/Utils/TestVideoSource.cs index 3d53f655..2689b837 100644 --- a/Tests/PlayMode/Utils/MetadataTestVideoSource.cs +++ b/Tests/PlayMode/Utils/TestVideoSource.cs @@ -4,15 +4,22 @@ namespace LiveKit.PlayModeTests.Utils { /// - /// Test-only that pushes a continuous stream of - /// zero-filled RGBA frames at the resolution given to the constructor. Used - /// by tests that need actual media flow (e.g. validating per-frame metadata - /// round-trips through the FFI / RTP path). + /// Test-only registered with FFI at a fixed + /// resolution. Two modes via : + /// + /// false (default): never pushes frames. Use when the test only + /// needs the publication to propagate via signaling (e.g. + /// APIs that operate on metadata). + /// true: pushes a continuous stream of zero-filled RGBA frames. + /// Use when the test needs actual media flow (e.g. validating per-frame + /// metadata round-trips through the FFI / RTP path). + /// /// - public sealed class MetadataTestVideoSource : RtcVideoSource + public sealed class TestVideoSource : RtcVideoSource { private readonly int _width; private readonly int _height; + private readonly bool _pushFrames; public override int GetWidth() => _width; public override int GetHeight() => _height; @@ -21,6 +28,8 @@ public sealed class MetadataTestVideoSource : RtcVideoSource protected override bool ReadBuffer() { + if (!_pushFrames) return false; + if (!_captureBuffer.IsCreated) { _captureBuffer = new NativeArray( @@ -32,11 +41,12 @@ protected override bool ReadBuffer() return false; } - public MetadataTestVideoSource(int width = 16, int height = 16) + public TestVideoSource(bool pushFrames = false, int width = 16, int height = 16) : base(VideoStreamSource.Texture, VideoBufferType.Rgba) { _width = width; _height = height; + _pushFrames = pushFrames; Init(); } } diff --git a/Tests/PlayMode/Utils/MetadataTestVideoSource.cs.meta b/Tests/PlayMode/Utils/TestVideoSource.cs.meta similarity index 83% rename from Tests/PlayMode/Utils/MetadataTestVideoSource.cs.meta rename to Tests/PlayMode/Utils/TestVideoSource.cs.meta index 31ee6d4c..2b80cd2b 100644 --- a/Tests/PlayMode/Utils/MetadataTestVideoSource.cs.meta +++ b/Tests/PlayMode/Utils/TestVideoSource.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: defea528e82a4c8d85faf7574ea6f698 +guid: 7e4ca10e199b4df7ba2eb6de53497882 MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Tests/PlayMode/VideoFrameMetadataTests.cs b/Tests/PlayMode/VideoFrameMetadataTests.cs index 807e55ae..c4c8108c 100644 --- a/Tests/PlayMode/VideoFrameMetadataTests.cs +++ b/Tests/PlayMode/VideoFrameMetadataTests.cs @@ -34,7 +34,7 @@ public IEnumerator VideoFrame_AttachedMetadata_ReceivedOnSubscriber() const uint expectedFrameId = 42u; const ulong expectedUserTs = 0x1122334455667788UL; - var source = new MetadataTestVideoSource(); + var source = new TestVideoSource(pushFrames: true); source.MetadataProvider = () => new FrameMetadata { FrameId = expectedFrameId, From cef0645fdae51f61f58d52bfae6d3f20b6b87dd0 Mon Sep 17 00:00:00 2001 From: Max Heimbrock <43608204+MaxHeimbrock@users.noreply.github.com> Date: Wed, 20 May 2026 10:29:49 +0200 Subject: [PATCH 5/5] Remove meet sample changes --- Samples~/Meet/Assets/Runtime/MeetManager.cs | 42 +-------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/Samples~/Meet/Assets/Runtime/MeetManager.cs b/Samples~/Meet/Assets/Runtime/MeetManager.cs index 8eab3ff7..225c7a0c 100644 --- a/Samples~/Meet/Assets/Runtime/MeetManager.cs +++ b/Samples~/Meet/Assets/Runtime/MeetManager.cs @@ -1,4 +1,3 @@ -using System; using System.Collections; using System.Collections.Generic; using UnityEngine; @@ -47,7 +46,6 @@ public class MeetManager : MonoBehaviour private LocalAudioTrack _localAudioTrack; private bool _cameraActive; private bool _microphoneActive; - private uint _nextOutgoingFrameId; #region Lifecycle @@ -238,7 +236,6 @@ private void BindRemoteCameraToTile(RemoteVideoTrack video, string identity) var stream = new VideoStream(video); stream.TextureReceived += tex => tile.BindLiveSource(tex); - stream.FrameReceived += MakeFrameMetadataLogger(identity); _videoStreams[sid] = stream; stream.Start(); StartCoroutine(stream.Update()); @@ -271,7 +268,6 @@ private void AddExtraVideoTile(RemoteVideoTrack video, string identity) var stream = new VideoStream(video); stream.TextureReceived += tex => tile.BindLiveSource(tex); - stream.FrameReceived += MakeFrameMetadataLogger($"{identity} (screen)"); _extraVideoTiles[sid] = tile; _extraVideoOwners[sid] = identity; @@ -280,33 +276,6 @@ private void AddExtraVideoTile(RemoteVideoTrack video, string identity) StartCoroutine(stream.Update()); } - /// Returns a per-stream that logs the trailer - /// metadata (frame_id, user_timestamp) at most once per second per stream. The throttle - /// clock uses Environment.TickCount because FrameReceived may fire off the main thread. - private static VideoStream.FrameReceiveDelegate MakeFrameMetadataLogger(string label) - { - var nextLogTick = 0; - return frame => - { - var now = Environment.TickCount; - if (now - nextLogTick < 0) return; - nextLogTick = now + 1000; - if (frame.Metadata == null) - { - Debug.Log($"[Meet RX {label}] no trailer"); - return; - } - Debug.Log($"[Meet RX {label}] frame_id={frame.Metadata.FrameId} user_ts={frame.Metadata.UserTimestamp}"); - }; - } - - private static ulong UnixTimeUs() - { - var now = DateTimeOffset.UtcNow; - return (ulong)(now.ToUnixTimeMilliseconds() * 1000 - + (now.Ticks % TimeSpan.TicksPerMillisecond) / 10); - } - private void RemoveExtraVideoTile(string sid) { if (_extraVideoTiles.TryGetValue(sid, out var tile)) @@ -434,11 +403,6 @@ private IEnumerator PublishLocalCamera() var tile = _participantTiles[_localId]; var source = new WebCameraSource(_webCamTexture); - source.MetadataProvider = () => new FrameMetadata - { - UserTimestamp = UnixTimeUs(), - FrameId = unchecked(_nextOutgoingFrameId++), - }; _localVideoTrack = LocalVideoTrack.CreateVideoTrack(LocalVideoTrackName, source, _room); var options = new TrackPublishOptions @@ -446,10 +410,8 @@ private IEnumerator PublishLocalCamera() VideoCodec = VideoCodec.H265, VideoEncoding = new VideoEncoding { MaxBitrate = 512000, MaxFramerate = frameRate }, Simulcast = false, - Source = TrackSource.SourceCamera, - }.WithPacketTrailerFeatures( - PacketTrailerFeature.PtfUserTimestamp, - PacketTrailerFeature.PtfFrameId); + Source = TrackSource.SourceCamera + }; var publish = _room.LocalParticipant.PublishTrack(_localVideoTrack, options); yield return publish;