Implement VideoFrame, Local/RemoteVideoTrack, video publication

CMakeLists.txt

@@ -163,6 +163,7 @@ add_library(livekit
   include/livekit/ffi_handle.h
   include/livekit/ffi_client.h
   include/livekit/local_audio_track.h
+  include/livekit/remote_audio_track.h
   include/livekit/participant.h
   include/livekit/local_participant.h
   include/livekit/livekit.h
@@ -171,11 +172,16 @@ add_library(livekit
   include/livekit/track_publication.h
   include/livekit/local_track_publication.h
   include/livekit/remote_track_publication.h
+  include/livekit/video_frame.h
+  include/livekit/video_source.h
+  include/livekit/local_video_track.h
+  include/livekit/remote_video_track.h
   src/audio_frame.cpp
   src/audio_source.cpp
   src/ffi_handle.cpp
   src/ffi_client.cpp
   src/local_audio_track.cpp
+  src/remote_audio_track.cpp
   src/room.cpp
   src/room_proto_converter.cpp
   src/room_proto_converter.h
@@ -187,6 +193,12 @@ add_library(livekit
   src/track_publication.cpp
   src/local_track_publication.cpp
   src/remote_track_publication.cpp
+  src/video_frame.cpp
+  src/video_source.cpp
+  src/local_video_track.cpp
+  src/remote_video_track.cpp
+  src/video_utils.cpp
+  src/video_utils.h
 )
 
 # Add generated proto objects to the wrapper

examples/simple_room/main.cpp

@@ -13,6 +13,9 @@
 // TODO(shijing), remove this livekit_ffi.h as it should be internal only.
 #include "livekit_ffi.h"
 
+// Consider expose this video_utils.h to public ?
+#include "video_utils.h"
+
 using namespace livekit;
 
 namespace {
@@ -160,6 +163,58 @@ void runNoiseCaptureLoop(const std::shared_ptr<AudioSource> &source) {
     std::cout << "Error in clearQueue" << std::endl;
   }
 }
+
+void runFakeVideoCaptureLoop(const std::shared_ptr<VideoSource> &source) {
+  auto frame = LKVideoFrame::create(1280, 720, VideoBufferType::ARGB);
+  double framerate = 1.0 / 30;
+  while (g_running.load(std::memory_order_relaxed)) {
+    static auto start = std::chrono::high_resolution_clock::now();
+    float t = std::chrono::duration<float>(
+                  std::chrono::high_resolution_clock::now() - start)
+                  .count();
+    // Cycle every 4 seconds: 0=red, 1=green, 2=blue, 3 black
+    int stage = static_cast<int>(t) % 4;
+    std::vector<int> rgb(4);
+    switch (stage) {
+    case 0: // red
+      rgb[0] = 255;
+      rgb[1] = 0;
+      rgb[2] = 0;
+      break;
+    case 1: // green
+      rgb[0] = 0;
+      rgb[1] = 255;
+      rgb[2] = 0;
+      break;
+    case 2: // blue
+      rgb[0] = 0;
+      rgb[1] = 0;
+      rgb[2] = 255;
+      break;
+    case 4: // black
+      rgb[0] = 0;
+      rgb[1] = 0;
+      rgb[2] = 0;
+    }
+    for (size_t i = 0; i < frame.dataSize(); i += 4) {
+      frame.data()[i] = 255;
+      frame.data()[i + 1] = rgb[0];
+      frame.data()[i + 2] = rgb[1];
+      frame.data()[i + 3] = rgb[2];
+    }
+    LKVideoFrame i420 = convertViaFfi(frame, VideoBufferType::I420, false);
+    try {
+      source->captureFrame(frame, 0, VideoRotation::VIDEO_ROTATION_0);
+    } catch (const std::exception &e) {
+      // If something goes wrong, log and break out
+      std::cerr << "Error in captureFrame: " << e.what() << std::endl;
+      break;
+    }
+
+    std::this_thread::sleep_for(std::chrono::duration<double>(framerate));
+  }
+}
+
 } // namespace
 
 int main(int argc, char *argv[]) {
@@ -210,35 +265,64 @@ int main(int argc, char *argv[]) {
             << info.reliable_dc_buffered_amount_low_threshold << "\n"
             << "  Creation time (ms): " << info.creation_time << "\n";
 
+  // Setup Audio Source / Track
   auto audioSource = std::make_shared<AudioSource>(44100, 1, 10);
   auto audioTrack =
       LocalAudioTrack::createLocalAudioTrack("micTrack", audioSource);
 
-  TrackPublishOptions opts;
-  opts.source = TrackSource::SOURCE_MICROPHONE;
-  opts.dtx = false;
-  opts.simulcast = false;
-
+  TrackPublishOptions audioOpts;
+  audioOpts.source = TrackSource::SOURCE_MICROPHONE;
+  audioOpts.dtx = false;
+  audioOpts.simulcast = false;
+  std::shared_ptr<LocalTrackPublication> audioPub;
   try {
     // publishTrack takes std::shared_ptr<Track>, LocalAudioTrack derives from
     // Track
-    auto pub = room.local_participant()->publishTrack(audioTrack, opts);
+    audioPub = room.local_participant()->publishTrack(audioTrack, audioOpts);
 
     std::cout << "Published track:\n"
-              << "  SID: " << pub->sid() << "\n"
-              << "  Name: " << pub->name() << "\n"
-              << "  Kind: " << static_cast<int>(pub->kind()) << "\n"
-              << "  Source: " << static_cast<int>(pub->source()) << "\n"
-              << "  Simulcasted: " << std::boolalpha << pub->simulcasted()
+              << "  SID: " << audioPub->sid() << "\n"
+              << "  Name: " << audioPub->name() << "\n"
+              << "  Kind: " << static_cast<int>(audioPub->kind()) << "\n"
+              << "  Source: " << static_cast<int>(audioPub->source()) << "\n"
+              << "  Simulcasted: " << std::boolalpha << audioPub->simulcasted()
               << "\n"
-              << "  Muted: " << std::boolalpha << pub->muted() << "\n";
+              << "  Muted: " << std::boolalpha << audioPub->muted() << "\n";
   } catch (const std::exception &e) {
     std::cerr << "Failed to publish track: " << e.what() << std::endl;
   }
 
   // TODO, if we have pre-buffering feature, we might consider starting the
   // thread right after creating the source.
   std::thread audioThread(runNoiseCaptureLoop, audioSource);
+
+  // Setup Video Source / Track
+  auto videoSource = std::make_shared<VideoSource>(1280, 720);
+  std::shared_ptr<LocalVideoTrack> videoTrack =
+      LocalVideoTrack::createLocalVideoTrack("cam", videoSource);
+  TrackPublishOptions videoOpts;
+  videoOpts.source = TrackSource::SOURCE_CAMERA;
+  videoOpts.dtx = false;
+  videoOpts.simulcast = true;
+  std::shared_ptr<LocalTrackPublication> videoPub;
+  try {
+    // publishTrack takes std::shared_ptr<Track>, LocalAudioTrack derives from
+    // Track
+    videoPub = room.local_participant()->publishTrack(videoTrack, videoOpts);
+
+    std::cout << "Published track:\n"
+              << "  SID: " << videoPub->sid() << "\n"
+              << "  Name: " << videoPub->name() << "\n"
+              << "  Kind: " << static_cast<int>(videoPub->kind()) << "\n"
+              << "  Source: " << static_cast<int>(videoPub->source()) << "\n"
+              << "  Simulcasted: " << std::boolalpha << videoPub->simulcasted()
+              << "\n"
+              << "  Muted: " << std::boolalpha << videoPub->muted() << "\n";
+  } catch (const std::exception &e) {
+    std::cerr << "Failed to publish track: " << e.what() << std::endl;
+  }
+  std::thread videoThread(runFakeVideoCaptureLoop, videoSource);
+
   // Keep the app alive until Ctrl-C so we continue receiving events,
   // similar to asyncio.run(main()) keeping the loop running.
   while (g_running.load()) {
@@ -249,6 +333,14 @@ int main(int argc, char *argv[]) {
   if (audioThread.joinable()) {
     audioThread.join();
   }
+  // Clean up the audio track publishment
+  room.local_participant()->unpublishTrack(audioPub->sid());
+
+  if (videoThread.joinable()) {
+    videoThread.join();
+  }
+  // Clean up the video track publishment
+  room.local_participant()->unpublishTrack(videoPub->sid());
 
   FfiClient::instance().shutdown();
   std::cout << "Exiting.\n";

include/livekit/livekit.h

@@ -19,7 +19,10 @@
 #include "local_audio_track.h"
 #include "local_participant.h"
 #include "local_track_publication.h"
+#include "local_video_track.h"
 #include "participant.h"
 #include "room.h"
 #include "room_delegate.h"
-#include "track_publication.h"
+#include "track_publication.h"
+#include "video_frame.h"
+#include "video_source.h"

include/livekit/local_video_track.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "track.h"
+#include <memory>
+#include <string>
+
+namespace livekit {
+
+namespace proto {
+class OwnedTrack;
+}
+
+class VideoSource;
+
+// ============================================================
+// LocalAudioTrack
+// ============================================================
+class LocalVideoTrack : public Track {
+public:
+  explicit LocalVideoTrack(FfiHandle handle, const proto::OwnedTrack &track);
+
+  static std::shared_ptr<LocalVideoTrack>
+  createLocalVideoTrack(const std::string &name,
+                        const std::shared_ptr<VideoSource> &source);
+
+  // Mute/unmute
+  void mute();
+  void unmute();
+
+  std::string to_string() const;
+
+private:
+  // Optional: you may add private helpers if needed
+};
+
+} // namespace livekit

include/livekit/remote_audio_track.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "track.h"
+#include <memory>
+#include <string>
+
+namespace livekit {
+
+namespace proto {
+class OwnedTrack;
+}
+
+class AudioSource;
+
+// ============================================================
+// RemoteAudioTrack
+// ============================================================
+class RemoteAudioTrack : public Track {
+public:
+  explicit RemoteAudioTrack(FfiHandle handle, const proto::OwnedTrack &track);
+
+  static std::shared_ptr<RemoteAudioTrack>
+  createRemoteAudioTrack(const std::string &name,
+                         const std::shared_ptr<AudioSource> &source);
+
+  std::string to_string() const;
+
+private:
+};
+
+} // namespace livekit

include/livekit/remote_video_track.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "track.h"
+#include <memory>
+#include <string>
+
+namespace livekit {
+
+namespace proto {
+class OwnedTrack;
+}
+
+class VideoSource;
+
+// ============================================================
+// RemoteVideoTrack
+// ============================================================
+class RemoteVideoTrack : public Track {
+public:
+  explicit RemoteVideoTrack(FfiHandle handle, const proto::OwnedTrack &track);
+
+  static std::shared_ptr<RemoteVideoTrack>
+  createRemoteVideoTrack(const std::string &name,
+                         const std::shared_ptr<VideoSource> &source);
+
+  std::string to_string() const;
+
+private:
+};
+
+} // namespace livekit