Merge pull request #76 from shiguredo/feature/switch-camera

接続中にキャプチャラを切り替える機能を実装
shiguredo · Sep 5, 2023 · 5d710be · 5d710be
2 parents 78a0be3 + 3f7bf5d
commit 5d710be
Show file tree

Hide file tree

Showing 9 changed files with 218 additions and 55 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -11,6 +11,12 @@
 
 ## develop
 
+- [CHANGE] `Sora.Config` 中にあるキャプチャラに関するフィールドを `Sora.CameraConfig` に移動する
+    - 修正方法は （TODO: @miosakuma がドキュメントへのリンクに差し替える） を参照して下さい
+    - @melpon
+- [ADD] 接続中にキャプチャラを切り替える機能を実装
+    - @melpon
+
 ## 2023.3.0 (2023-08-08)
 
 - [UPDATE] Sora C++ SDK を `2023.10.0` に上げる

diff --git a/Sora/Sora.cs b/Sora/Sora.cs
@@ -83,6 +83,44 @@ public class Rule
         public List<List<Rule>> Rules = new List<List<Rule>>();
     }
 
+    public class CameraConfig
+    {
+        public CapturerType CapturerType = Sora.CapturerType.DeviceCamera;
+        public UnityEngine.Camera UnityCamera = null;
+        public int UnityCameraRenderTargetDepthBuffer = 16;
+        public string VideoCapturerDevice = "";
+        public int VideoWidth = 640;
+        public int VideoHeight = 480;
+        public int VideoFps = 30;
+
+        public static CameraConfig FromUnityCamera(UnityEngine.Camera unityCamera, int unityCameraRenderTargetDepthBuffer, int videoWidth, int videoHeight, int videoFps)
+        {
+            return new CameraConfig()
+            {
+                CapturerType = Sora.CapturerType.UnityCamera,
+                UnityCamera = unityCamera,
+                UnityCameraRenderTargetDepthBuffer = unityCameraRenderTargetDepthBuffer,
+                VideoCapturerDevice = "",
+                VideoWidth = videoWidth,
+                VideoHeight = videoHeight,
+                VideoFps = videoFps,
+            };
+        }
+        public static CameraConfig FromDeviceCamera(string videoCapturerDevice, int videoWidth, int videoHeight, int videoFps)
+        {
+            return new CameraConfig()
+            {
+                CapturerType = Sora.CapturerType.DeviceCamera,
+                UnityCamera = null,
+                UnityCameraRenderTargetDepthBuffer = 16,
+                VideoCapturerDevice = videoCapturerDevice,
+                VideoWidth = videoWidth,
+                VideoHeight = videoHeight,
+                VideoFps = videoFps,
+            };
+        }
+    }
+
     public class Config
     {
         public string SignalingUrl = "";
@@ -100,15 +138,9 @@ public class Config
         public SpotlightFocusRidType? SpotlightUnfocusRid;
         public bool? Simulcast;
         public SimulcastRidType? SimulcastRid = null;
-        public CapturerType CapturerType = Sora.CapturerType.DeviceCamera;
-        public UnityEngine.Camera UnityCamera = null;
-        public int UnityCameraRenderTargetDepthBuffer = 16;
+        public CameraConfig CameraConfig = new CameraConfig();
         public bool Video = true;
         public bool Audio = true;
-        public string VideoCapturerDevice = "";
-        public int VideoWidth = 640;
-        public int VideoHeight = 480;
-        public int VideoFps = 30;
         public VideoCodecType VideoCodecType = VideoCodecType.VP9;
         public string VideoVp9Params = "";
         public string VideoAv1Params = "";
@@ -237,10 +269,10 @@ public Sora()
     public void Connect(Config config)
     {
         IntPtr unityCameraTexture = IntPtr.Zero;
-        if (config.CapturerType == CapturerType.UnityCamera)
+        if (config.CameraConfig.CapturerType == CapturerType.UnityCamera)
         {
-            unityCamera = config.UnityCamera;
-            var texture = new UnityEngine.RenderTexture(config.VideoWidth, config.VideoHeight, config.UnityCameraRenderTargetDepthBuffer, UnityEngine.RenderTextureFormat.BGRA32);
+            unityCamera = config.CameraConfig.UnityCamera;
+            var texture = new UnityEngine.RenderTexture(config.CameraConfig.VideoWidth, config.CameraConfig.VideoHeight, config.CameraConfig.UnityCameraRenderTargetDepthBuffer, UnityEngine.RenderTextureFormat.BGRA32);
             unityCamera.targetTexture = texture;
             unityCamera.enabled = true;
             unityCameraTexture = texture.GetNativeTexturePtr();
@@ -280,14 +312,14 @@ public void Connect(Config config)
         cc.simulcast = config.Simulcast == null ? false : config.Simulcast.Value;
         cc.simulcast_rid = config.SimulcastRid == null ? "" : config.SimulcastRid.Value.ToString().ToLower();
         cc.insecure = config.Insecure;
-        cc.capturer_type = (int)config.CapturerType;
-        cc.unity_camera_texture = unityCameraTexture.ToInt64();
         cc.video = config.Video;
         cc.audio = config.Audio;
-        cc.video_capturer_device = config.VideoCapturerDevice;
-        cc.video_width = config.VideoWidth;
-        cc.video_height = config.VideoHeight;
-        cc.video_fps = config.VideoFps;
+        cc.camera_config.capturer_type = (int)config.CameraConfig.CapturerType;
+        cc.camera_config.unity_camera_texture = unityCameraTexture.ToInt64();
+        cc.camera_config.video_capturer_device = config.CameraConfig.VideoCapturerDevice;
+        cc.camera_config.video_width = config.CameraConfig.VideoWidth;
+        cc.camera_config.video_height = config.CameraConfig.VideoHeight;
+        cc.camera_config.video_fps = config.CameraConfig.VideoFps;
         cc.video_codec_type = config.VideoCodecType.ToString();
         cc.video_vp9_params = config.VideoVp9Params;
         cc.video_av1_params = config.VideoAv1Params;
@@ -380,6 +412,34 @@ public void Disconnect()
         sora_disconnect(p);
     }
 
+    public void SwitchCamera(CameraConfig config)
+    {
+        if (unityCamera != null)
+        {
+            unityCamera.enabled = false;
+            unityCamera.targetTexture = null;
+            unityCamera = null;
+        }
+
+        IntPtr unityCameraTexture = IntPtr.Zero;
+        if (config.CapturerType == CapturerType.UnityCamera)
+        {
+            unityCamera = config.UnityCamera;
+            var texture = new UnityEngine.RenderTexture(config.VideoWidth, config.VideoHeight, config.UnityCameraRenderTargetDepthBuffer, UnityEngine.RenderTextureFormat.BGRA32);
+            unityCamera.targetTexture = texture;
+            unityCamera.enabled = true;
+            unityCameraTexture = texture.GetNativeTexturePtr();
+        }
+        var cc = new SoraConf.Internal.CameraConfig();
+        cc.capturer_type = (int)config.CapturerType;
+        cc.unity_camera_texture = unityCameraTexture.ToInt64();
+        cc.video_capturer_device = config.VideoCapturerDevice;
+        cc.video_width = config.VideoWidth;
+        cc.video_height = config.VideoHeight;
+        cc.video_fps = config.VideoFps;
+        sora_switch_camera(p, Jsonif.Json.ToJson(cc));
+    }
+
     // Unity 側でレンダリングが完了した時（yield return new WaitForEndOfFrame() の後）に呼ぶイベント
     // 指定した Unity カメラの映像を Sora 側のテクスチャにレンダリングしたりする
     public void OnRender()
@@ -791,6 +851,8 @@ public bool VideoEnabled
     [DllImport(DllName)]
     private static extern void sora_disconnect(IntPtr p);
     [DllImport(DllName)]
+    private static extern void sora_switch_camera(IntPtr p, string config);
+    [DllImport(DllName)]
     private static extern IntPtr sora_get_texture_update_callback();
     [DllImport(DllName)]
     private static extern void sora_destroy(IntPtr p);

diff --git a/proto/sora_conf_internal.proto b/proto/sora_conf_internal.proto
@@ -37,6 +37,15 @@ message ForwardingFilter {
     repeated Rules rules = 2;
 }
 
+message CameraConfig {
+    int32 capturer_type = 17;
+    int64 unity_camera_texture = 18;
+    string video_capturer_device = 19;
+    int32 video_width = 22;
+    int32 video_height = 23;
+    int32 video_fps = 24;
+}
+
 message ConnectConfig {
     string unity_version = 1;
     repeated string signaling_url = 2;
@@ -54,14 +63,9 @@ message ConnectConfig {
     bool enable_simulcast = 14;
     bool simulcast = 15;
     string simulcast_rid = 16;
-    int32 capturer_type = 17;
-    int64 unity_camera_texture = 18;
-    string video_capturer_device = 19;
+    CameraConfig camera_config = 17;
     bool video = 20;
     bool audio = 21;
-    int32 video_width = 22;
-    int32 video_height = 23;
-    int32 video_fps = 24;
     string video_codec_type = 25;
     string video_vp9_params = 250;
     string video_av1_params = 251;

diff --git a/src/sora.cpp b/src/sora.cpp
@@ -66,6 +66,7 @@ Sora::~Sora() {
   capturer_ = nullptr;
   unity_adm_ = nullptr;
 
+  video_sender_ = nullptr;
   audio_track_ = nullptr;
   video_track_ = nullptr;
   connection_context_ = nullptr;
@@ -147,6 +148,40 @@ void Sora::DispatchEvents() {
   }
 }
 
+static sora_conf::VideoFrame VideoFrameToConfig(
+    const webrtc::VideoFrame& frame) {
+  sora_conf::VideoFrame f;
+  f.baseptr = reinterpret_cast<int64_t>(&frame);
+  f.id = frame.id();
+  f.timestamp_us = frame.timestamp_us();
+  f.timestamp = frame.timestamp();
+  f.ntp_time_ms = frame.ntp_time_ms();
+  f.rotation = (int)frame.rotation();
+  auto& v = f.video_frame_buffer;
+  auto vfb = frame.video_frame_buffer();
+  v.baseptr = reinterpret_cast<int64_t>(vfb.get());
+  v.type = (sora_conf::VideoFrameBuffer::Type)vfb->type();
+  v.width = vfb->width();
+  v.height = vfb->height();
+  if (vfb->type() == webrtc::VideoFrameBuffer::Type::kI420) {
+    auto p = vfb->GetI420();
+    v.i420_stride_y = p->StrideY();
+    v.i420_stride_u = p->StrideU();
+    v.i420_stride_v = p->StrideV();
+    v.i420_data_y = reinterpret_cast<int64_t>(p->DataY());
+    v.i420_data_u = reinterpret_cast<int64_t>(p->DataU());
+    v.i420_data_v = reinterpret_cast<int64_t>(p->DataV());
+  }
+  if (vfb->type() == webrtc::VideoFrameBuffer::Type::kNV12) {
+    auto p = vfb->GetNV12();
+    v.nv12_stride_y = p->StrideY();
+    v.nv12_stride_uv = p->StrideUV();
+    v.nv12_data_y = reinterpret_cast<int64_t>(p->DataY());
+    v.nv12_data_uv = reinterpret_cast<int64_t>(p->DataUV());
+  }
+  return f;
+}
+
 void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) {
   auto on_disconnect = [this](int error_code, std::string reason) {
     PushEvent([this, error_code, reason = std::move(reason)]() {
@@ -302,51 +337,25 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc,
     if (on_capturer_frame_) {
       on_frame = [on_frame =
                       on_capturer_frame_](const webrtc::VideoFrame& frame) {
-        sora_conf::VideoFrame f;
-        f.baseptr = reinterpret_cast<int64_t>(&frame);
-        f.id = frame.id();
-        f.timestamp_us = frame.timestamp_us();
-        f.timestamp = frame.timestamp();
-        f.ntp_time_ms = frame.ntp_time_ms();
-        f.rotation = (int)frame.rotation();
-        auto& v = f.video_frame_buffer;
-        auto vfb = frame.video_frame_buffer();
-        v.baseptr = reinterpret_cast<int64_t>(vfb.get());
-        v.type = (sora_conf::VideoFrameBuffer::Type)vfb->type();
-        v.width = vfb->width();
-        v.height = vfb->height();
-        if (vfb->type() == webrtc::VideoFrameBuffer::Type::kI420) {
-          auto p = vfb->GetI420();
-          v.i420_stride_y = p->StrideY();
-          v.i420_stride_u = p->StrideU();
-          v.i420_stride_v = p->StrideV();
-          v.i420_data_y = reinterpret_cast<int64_t>(p->DataY());
-          v.i420_data_u = reinterpret_cast<int64_t>(p->DataU());
-          v.i420_data_v = reinterpret_cast<int64_t>(p->DataV());
-        }
-        if (vfb->type() == webrtc::VideoFrameBuffer::Type::kNV12) {
-          auto p = vfb->GetNV12();
-          v.nv12_stride_y = p->StrideY();
-          v.nv12_stride_uv = p->StrideUV();
-          v.nv12_data_y = reinterpret_cast<int64_t>(p->DataY());
-          v.nv12_data_uv = reinterpret_cast<int64_t>(p->DataUV());
-        }
+        sora_conf::VideoFrame f = VideoFrameToConfig(frame);
         on_frame(jsonif::to_json(f));
       };
     }
 
     auto capturer = CreateVideoCapturer(
-        cc.capturer_type, (void*)cc.unity_camera_texture,
-        cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps,
-        on_frame, signaling_thread_.get(), env, android_context);
+        cc.camera_config.capturer_type,
+        (void*)cc.camera_config.unity_camera_texture,
+        cc.camera_config.video_capturer_device, cc.camera_config.video_width,
+        cc.camera_config.video_height, cc.camera_config.video_fps, on_frame,
+        signaling_thread_.get(), env, android_context);
     if (!capturer) {
       on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR,
                     "Capturer Init Failed");
       return;
     }
 
     capturer_ = capturer;
-    capturer_type_ = cc.capturer_type;
+    capturer_type_ = cc.camera_config.capturer_type;
 
     std::string audio_track_id = rtc::CreateRandomString(16);
     audio_track_ = factory_->CreateAudioTrack(
@@ -530,6 +539,54 @@ void Sora::Disconnect() {
   signaling_->Disconnect();
 }
 
+void Sora::SwitchCamera(const sora_conf::internal::CameraConfig& cc) {
+  RTC_LOG(LS_INFO) << "SwitchCamera: " << jsonif::to_json(cc);
+  boost::asio::post(*ioc_, [self = shared_from_this(), cc = cc]() {
+    self->DoSwitchCamera(cc);
+  });
+}
+void Sora::DoSwitchCamera(const sora_conf::internal::CameraConfig& cc) {
+  std::function<void(const webrtc::VideoFrame& frame)> on_frame;
+  if (on_capturer_frame_) {
+    on_frame = [on_frame =
+                    on_capturer_frame_](const webrtc::VideoFrame& frame) {
+      sora_conf::VideoFrame f = VideoFrameToConfig(frame);
+      on_frame(jsonif::to_json(f));
+    };
+  }
+
+  void* env = sora::GetJNIEnv();
+  void* android_context = GetAndroidApplicationContext(env);
+
+#if defined(SORA_UNITY_SDK_ANDROID)
+  if (capturer_ != nullptr && capturer_type_ == 0) {
+    static_cast<sora::AndroidCapturer*>(capturer_.get())->Stop();
+  }
+#endif
+  if (capturer_ != nullptr && capturer_type_ != 0) {
+    static_cast<UnityCameraCapturer*>(capturer_.get())->Stop();
+  }
+  capturer_ = nullptr;
+
+  auto capturer = CreateVideoCapturer(
+      cc.capturer_type, (void*)cc.unity_camera_texture,
+      cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps,
+      on_frame, signaling_thread_.get(), env, android_context);
+  if (!capturer) {
+    RTC_LOG(LS_ERROR) << "Failed to CreateVideoCapturer";
+    return;
+  }
+
+  capturer_ = capturer;
+  capturer_type_ = cc.capturer_type;
+
+  std::string video_track_id = rtc::CreateRandomString(16);
+  auto video_track = factory_->CreateVideoTrack(video_track_id, capturer.get());
+  renderer_->ReplaceTrack(video_track_.get(), video_track.get());
+  video_sender_->SetTrack(video_track.get());
+  video_track_ = video_track;
+}
+
 void Sora::RenderCallbackStatic(int event_id) {
   auto sora = (Sora*)IdPointer::Instance().Lookup(event_id);
   if (sora == nullptr) {
@@ -791,6 +848,7 @@ void Sora::OnSetOffer(std::string offer) {
     webrtc::RTCErrorOr<rtc::scoped_refptr<webrtc::RtpSenderInterface>>
         video_result = signaling_->GetPeerConnection()->AddTrack(video_track_,
                                                                  {stream_id});
+    video_sender_ = video_result.value();
   }
 
   if (video_track_ != nullptr) {

diff --git a/src/sora.h b/src/sora.h
@@ -53,6 +53,7 @@ class Sora : public std::enable_shared_from_this<Sora>,
 
   void Connect(const sora_conf::internal::ConnectConfig& cc);
   void Disconnect();
+  void SwitchCamera(const sora_conf::internal::CameraConfig& cc);
 
   static void UNITY_INTERFACE_API RenderCallbackStatic(int event_id);
   int GetRenderCallbackEventID() const;
@@ -91,6 +92,7 @@ class Sora : public std::enable_shared_from_this<Sora>,
  private:
   void DoConnect(const sora_conf::internal::ConnectConfig& config,
                  std::function<void(int, std::string)> on_disconnect);
+  void DoSwitchCamera(const sora_conf::internal::CameraConfig& cc);
 
   static rtc::scoped_refptr<UnityAudioDevice> CreateADM(
       webrtc::TaskQueueFactory* task_queue_factory,
@@ -140,6 +142,7 @@ class Sora : public std::enable_shared_from_this<Sora>,
   std::unique_ptr<UnityRenderer> renderer_;
   rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track_;
   rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track_;
+  rtc::scoped_refptr<webrtc::RtpSenderInterface> video_sender_;
   std::function<void(ptrid_t, std::string)> on_add_track_;
   std::function<void(ptrid_t, std::string)> on_remove_track_;
   std::function<void(std::string)> on_set_offer_;

diff --git a/src/unity.cpp b/src/unity.cpp
@@ -142,6 +142,13 @@ void sora_disconnect(void* p) {
   wsora->sora->Disconnect();
 }
 
+void sora_switch_camera(void* p, const char* config_json) {
+  auto wsora = (SoraWrapper*)p;
+  auto config =
+      jsonif::from_json<sora_conf::internal::CameraConfig>(config_json);
+  wsora->sora->SwitchCamera(config);
+}
+
 void* sora_get_texture_update_callback() {
   return (void*)&sora_unity_sdk::UnityRenderer::Sink::TextureUpdateCallback;
 }