diff --git a/CHANGES.md b/CHANGES.md index eec1496..4fdf94e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,12 +14,22 @@ - [CHANGE] `Sora.Config` 中にあるキャプチャラに関するフィールドを `Sora.CameraConfig` に移動する - 修正方法は （TODO: @miosakuma がドキュメントへのリンクに差し替える） を参照して下さい - @melpon +- [UPDATE] SoraClientContext を利用してコードを短くする + - @melpon - [UPDATE] Sora C++ SDK を `2023.12.0` に上げる - @melpon - [UPDATE] libwebrtc を `m116.5845.6.1` に上げる - @melpon - [ADD] 接続中にキャプチャラを切り替える機能を実装 - @melpon +- [ADD] デバイスを掴まないようにする `NoVideoDevice`, `NoAudioDevice` を追加 + - @melpon +- [ADD] ハードウェアエンコーダを利用するかどうかを設定する `UseHardwareEncoder` を追加 + - @melpon +- [ADD] `SelectedSignalingURL` と `ConnectedSignalingURL` プロパティを追加 + - @melpon +- [FIX] IosAudioInit を初回接続の場合のみ呼び出すようにすることで、iOS で連続して接続しようとすると落ちることがあったのを修正 + - @melpon ## 2023.3.0 (2023-08-08) diff --git a/Sora/Sora.cs b/Sora/Sora.cs index 2cdfce0..16659be 100644 --- a/Sora/Sora.cs +++ b/Sora/Sora.cs @@ -138,6 +138,8 @@ public class Config public SpotlightFocusRidType? SpotlightUnfocusRid; public bool? Simulcast; public SimulcastRidType? SimulcastRid = null; + public bool NoVideoDevice = false; + public bool NoAudioDevice = false; public CameraConfig CameraConfig = new CameraConfig(); public bool Video = true; public bool Audio = true; @@ -185,6 +187,8 @@ public class Config public string ProxyAgent = ""; public ForwardingFilter ForwardingFilter; + + public bool? UseHardwareEncoder; } IntPtr p; @@ -302,16 +306,18 @@ public void Connect(Config config) cc.signaling_notify_metadata = config.SignalingNotifyMetadata; cc.role = role; cc.enable_multistream = config.Multistream != null; - cc.multistream = config.Multistream == null ? false : config.Multistream.Value; + cc.multistream = config.Multistream.GetValueOrDefault(); cc.enable_spotlight = config.Spotlight != null; - cc.spotlight = config.Spotlight == null ? false : config.Spotlight.Value; + cc.spotlight = config.Spotlight.GetValueOrDefault(); cc.spotlight_number = config.SpotlightNumber; cc.spotlight_focus_rid = config.SpotlightFocusRid == null ? "" : config.SpotlightFocusRid.Value.ToString().ToLower(); cc.spotlight_unfocus_rid = config.SpotlightUnfocusRid == null ? "" : config.SpotlightUnfocusRid.Value.ToString().ToLower(); cc.enable_simulcast = config.Simulcast != null; - cc.simulcast = config.Simulcast == null ? false : config.Simulcast.Value; + cc.simulcast = config.Simulcast.GetValueOrDefault(); cc.simulcast_rid = config.SimulcastRid == null ? "" : config.SimulcastRid.Value.ToString().ToLower(); cc.insecure = config.Insecure; + cc.no_video_device = config.NoVideoDevice; + cc.no_audio_device = config.NoAudioDevice; cc.video = config.Video; cc.audio = config.Audio; cc.camera_config.capturer_type = (int)config.CameraConfig.CapturerType; @@ -332,7 +338,7 @@ public void Connect(Config config) cc.audio_codec_type = config.AudioCodecType.ToString(); cc.audio_codec_lyra_bitrate = config.AudioCodecLyraBitrate; cc.enable_audio_codec_lyra_usedtx = config.AudioCodecLyraUsedtx != null; - cc.audio_codec_lyra_usedtx = config.AudioCodecLyraUsedtx == null ? false : config.AudioCodecLyraUsedtx.Value; + cc.audio_codec_lyra_usedtx = config.AudioCodecLyraUsedtx.GetValueOrDefault(); cc.check_lyra_version = config.CheckLyraVersion; cc.audio_bit_rate = config.AudioBitRate; cc.audio_streaming_language_code = config.AudioStreamingLanguageCode; @@ -404,6 +410,8 @@ public void Connect(Config config) cc.forwarding_filter.rules.Add(ccrs); } } + cc.enable_use_hardware_encoder = config.UseHardwareEncoder != null; + cc.use_hardware_encoder = config.UseHardwareEncoder.GetValueOrDefault(); sora_connect(p, Jsonif.Json.ToJson(cc)); } @@ -820,6 +828,28 @@ public bool VideoEnabled set { sora_set_video_enabled(p, value ? 1 : 0); } } + public string SelectedSignalingURL + { + get + { + int size = sora_get_selected_signaling_url_size(p); + byte[] buf = new byte[size]; + sora_get_selected_signaling_url(p, buf, size); + return System.Text.Encoding.UTF8.GetString(buf); + } + } + + public string ConnectedSignalingURL + { + get + { + int size = sora_get_connected_signaling_url_size(p); + byte[] buf = new byte[size]; + sora_get_connected_signaling_url(p, buf, size); + return System.Text.Encoding.UTF8.GetString(buf); + } + } + #if UNITY_IOS && !UNITY_EDITOR private const string DllName = "__Internal"; #else @@ -888,4 +918,12 @@ public bool VideoEnabled private static extern int sora_get_video_enabled(IntPtr p); [DllImport(DllName)] private static extern void sora_set_video_enabled(IntPtr p, int enabled); + [DllImport(DllName)] + private static extern int sora_get_selected_signaling_url_size(IntPtr p); + [DllImport(DllName)] + private static extern int sora_get_connected_signaling_url_size(IntPtr p); + [DllImport(DllName)] + private static extern void sora_get_selected_signaling_url(IntPtr p, [Out] byte[] buf, int size); + [DllImport(DllName)] + private static extern void sora_get_connected_signaling_url(IntPtr p, [Out] byte[] buf, int size); } diff --git a/proto/sora_conf_internal.proto b/proto/sora_conf_internal.proto index 83f86c7..2a3a58a 100644 --- a/proto/sora_conf_internal.proto +++ b/proto/sora_conf_internal.proto @@ -63,6 +63,8 @@ message ConnectConfig { bool enable_simulcast = 14; bool simulcast = 15; string simulcast_rid = 16; + bool no_video_device = 160; + bool no_audio_device = 161; CameraConfig camera_config = 17; bool video = 20; bool audio = 21; @@ -98,4 +100,6 @@ message ConnectConfig { string signaling_notify_metadata = 49; bool enable_forwarding_filter = 50; ForwardingFilter forwarding_filter = 51; + bool enable_use_hardware_encoder = 52; + bool use_hardware_encoder = 53; } \ No newline at end of file diff --git a/src/sora.cpp b/src/sora.cpp index 0d55d08..e871cde 100644 --- a/src/sora.cpp +++ b/src/sora.cpp @@ -44,7 +44,7 @@ namespace sora_unity_sdk { -Sora::Sora(UnityContext* context) : context_(context) { +Sora::Sora(UnityContext* context) : unity_context_(context) { ptrid_ = IdPointer::Instance().Register(this); #if defined(SORA_UNITY_SDK_ANDROID) auto env = sora::GetJNIEnv(); @@ -79,8 +79,6 @@ Sora::~Sora() { video_sender_ = nullptr; audio_track_ = nullptr; video_track_ = nullptr; - connection_context_ = nullptr; - factory_ = nullptr; if (ioc_ != nullptr) { ioc_->stop(); @@ -91,18 +89,7 @@ Sora::~Sora() { io_thread_->Stop(); io_thread_.reset(); } - if (network_thread_) { - network_thread_->Stop(); - network_thread_.reset(); - } - if (worker_thread_) { - worker_thread_->Stop(); - worker_thread_.reset(); - } - if (signaling_thread_) { - signaling_thread_->Stop(); - signaling_thread_.reset(); - } + sora_context_ = nullptr; RTC_LOG(LS_INFO) << "Sora object destroy finished"; } @@ -210,14 +197,18 @@ void Sora::Connect(const sora_conf::internal::ConnectConfig& cc) { return; } - IosAudioInit( - [this, on_disconnect = std::move(on_disconnect)](std::string error) { - if (!error.empty()) { - RTC_LOG(LS_ERROR) << "Failed to IosAudioInit: error=" << error; - on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, - "Failed to IosAudioInit: error=" + error); - } - }); + static bool ios_audio_init = false; + if (!ios_audio_init) { + IosAudioInit( + [this, on_disconnect = std::move(on_disconnect)](std::string error) { + if (!error.empty()) { + RTC_LOG(LS_ERROR) << "Failed to IosAudioInit: error=" << error; + on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, + "Failed to IosAudioInit: error=" + error); + } + ios_audio_init = true; + }); + } DoConnect(cc, std::move(on_disconnect)); #else DoConnect(cc, std::move(on_disconnect)); @@ -241,98 +232,54 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, return; } - rtc::InitializeSSL(); - - network_thread_ = rtc::Thread::CreateWithSocketServer(); - network_thread_->Start(); - worker_thread_ = rtc::Thread::Create(); - worker_thread_->Start(); - signaling_thread_ = rtc::Thread::Create(); - signaling_thread_->Start(); - - webrtc::PeerConnectionFactoryDependencies dependencies; - dependencies.network_thread = network_thread_.get(); - dependencies.worker_thread = worker_thread_.get(); - dependencies.signaling_thread = signaling_thread_.get(); - dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); - dependencies.call_factory = webrtc::CreateCallFactory(); - dependencies.event_log_factory = - absl::make_unique( - dependencies.task_queue_factory.get()); - - // worker 上の env - auto worker_env = - worker_thread_->BlockingCall([] { return sora::GetJNIEnv(); }); - // worker 上の context + // このスレッド上の env と context + void* env = sora::GetJNIEnv(); + void* android_context = GetAndroidApplicationContext(env); + + sora::SoraClientContextConfig client_config; + client_config.use_audio_device = false; + client_config.use_hardware_encoder = cc.use_hardware_encoder; + client_config.configure_media_dependencies = + [&, this](const webrtc::PeerConnectionFactoryDependencies& dependencies, + cricket::MediaEngineDependencies& media_dependencies) { + // worker 上の env + auto worker_env = dependencies.worker_thread->BlockingCall( + [] { return sora::GetJNIEnv(); }); + // worker 上の context #if defined(SORA_UNITY_SDK_ANDROID) - void* worker_context = worker_thread_->BlockingCall([worker_env] { - return ::sora_unity_sdk::GetAndroidApplicationContext((JNIEnv*)worker_env) - .Release(); - }); + void* worker_context = + dependencies.worker_thread->BlockingCall([worker_env] { + return ::sora_unity_sdk::GetAndroidApplicationContext( + (JNIEnv*)worker_env) + .Release(); + }); #else - void* worker_context = nullptr; + void* worker_context = nullptr; #endif - // media_dependencies - cricket::MediaEngineDependencies media_dependencies; - media_dependencies.task_queue_factory = dependencies.task_queue_factory.get(); - unity_adm_ = - CreateADM(media_dependencies.task_queue_factory, false, - cc.unity_audio_input, cc.unity_audio_output, on_handle_audio_, - cc.audio_recording_device, cc.audio_playout_device, - worker_thread_.get(), worker_env, worker_context); - media_dependencies.adm = unity_adm_; + unity_adm_ = CreateADM( + media_dependencies.task_queue_factory, cc.no_audio_device, + cc.unity_audio_input, cc.unity_audio_output, on_handle_audio_, + cc.audio_recording_device, cc.audio_playout_device, + dependencies.worker_thread, worker_env, worker_context); + media_dependencies.adm = unity_adm_; #if defined(SORA_UNITY_SDK_ANDROID) - worker_thread_->BlockingCall([worker_env, worker_context] { - ((JNIEnv*)worker_env)->DeleteLocalRef((jobject)worker_context); - }); + dependencies.worker_thread->BlockingCall([worker_env, worker_context] { + ((JNIEnv*)worker_env)->DeleteLocalRef((jobject)worker_context); + }); #endif + }; - media_dependencies.audio_encoder_factory = - sora::CreateBuiltinAudioEncoderFactory(); - media_dependencies.audio_decoder_factory = - sora::CreateBuiltinAudioDecoderFactory(); - - void* env = sora::GetJNIEnv(); - void* android_context = GetAndroidApplicationContext(env); - - auto cuda_context = sora::CudaContext::Create(); - { - auto config = sora::GetDefaultVideoEncoderFactoryConfig(cuda_context, env); - config.use_simulcast_adapter = true; - media_dependencies.video_encoder_factory = - absl::make_unique(std::move(config)); - } - { - auto config = sora::GetDefaultVideoDecoderFactoryConfig(cuda_context, env); - media_dependencies.video_decoder_factory = - absl::make_unique(std::move(config)); - } - - media_dependencies.audio_mixer = nullptr; - media_dependencies.audio_processing = - webrtc::AudioProcessingBuilder().Create(); - - dependencies.media_engine = - cricket::CreateMediaEngine(std::move(media_dependencies)); - - factory_ = sora::CreateModularPeerConnectionFactoryWithContext( - std::move(dependencies), connection_context_); + sora_context_ = sora::SoraClientContext::Create(client_config); - if (factory_ == nullptr) { + if (sora_context_ == nullptr) { RTC_LOG(LS_ERROR) << "Failed to create PeerConnectionFactory"; on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Failed to create PeerConnectionFactory"); return; } - webrtc::PeerConnectionFactoryInterface::Options factory_options; - factory_options.disable_encryption = false; - factory_options.ssl_max_version = rtc::SSL_PROTOCOL_DTLS_12; - factory_options.crypto_options.srtp.enable_gcm_crypto_suites = true; - factory_->SetOptions(factory_options); - if (!InitADM(unity_adm_, cc.audio_recording_device, cc.audio_playout_device)) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, @@ -354,11 +301,12 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, auto capturer = CreateVideoCapturer( cc.camera_config.capturer_type, - (void*)cc.camera_config.unity_camera_texture, + (void*)cc.camera_config.unity_camera_texture, cc.no_video_device, cc.camera_config.video_capturer_device, cc.camera_config.video_width, cc.camera_config.video_height, cc.camera_config.video_fps, on_frame, - signaling_thread_.get(), env, android_context); - if (!capturer) { + sora_context_->signaling_thread(), env, android_context, + unity_context_); + if (!cc.no_video_device && !capturer) { on_disconnect((int)sora_conf::ErrorCode::INTERNAL_ERROR, "Capturer Init Failed"); return; @@ -368,11 +316,16 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, capturer_type_ = cc.camera_config.capturer_type; std::string audio_track_id = rtc::CreateRandomString(16); - audio_track_ = factory_->CreateAudioTrack( - audio_track_id, - factory_->CreateAudioSource(cricket::AudioOptions()).get()); - std::string video_track_id = rtc::CreateRandomString(16); - video_track_ = factory_->CreateVideoTrack(video_track_id, capturer.get()); + audio_track_ = sora_context_->peer_connection_factory()->CreateAudioTrack( + audio_track_id, sora_context_->peer_connection_factory() + ->CreateAudioSource(cricket::AudioOptions()) + .get()); + + if (capturer) { + std::string video_track_id = rtc::CreateRandomString(16); + video_track_ = sora_context_->peer_connection_factory()->CreateVideoTrack( + video_track_id, capturer.get()); + } } { @@ -380,7 +333,7 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, ioc_.reset(new boost::asio::io_context(1)); sora::SoraSignalingConfig config; config.observer = shared_from_this(); - config.pc_factory = factory_; + config.pc_factory = sora_context_->peer_connection_factory(); config.io_context = ioc_.get(); config.role = cc.role; config.sora_client = SORA_CLIENT; @@ -512,10 +465,14 @@ void Sora::DoConnect(const sora_conf::internal::ConnectConfig& cc, } config.forwarding_filter = ff; } - config.network_manager = signaling_thread_->BlockingCall( - [this]() { return connection_context_->default_network_manager(); }); - config.socket_factory = signaling_thread_->BlockingCall( - [this]() { return connection_context_->default_socket_factory(); }); + config.network_manager = + sora_context_->signaling_thread()->BlockingCall([this]() { + return sora_context_->connection_context()->default_network_manager(); + }); + config.socket_factory = + sora_context_->signaling_thread()->BlockingCall([this]() { + return sora_context_->connection_context()->default_socket_factory(); + }); signaling_ = sora::SoraSignaling::Create(std::move(config)); signaling_->Connect(); @@ -583,9 +540,10 @@ void Sora::SwitchCamera(const sora_conf::internal::CameraConfig& cc) { capturer_ = nullptr; auto capturer = CreateVideoCapturer( - cc.capturer_type, (void*)cc.unity_camera_texture, + cc.capturer_type, (void*)cc.unity_camera_texture, false, cc.video_capturer_device, cc.video_width, cc.video_height, cc.video_fps, - on_frame, signaling_thread_.get(), env, android_context); + on_frame, sora_context_->signaling_thread(), env, android_context, + unity_context_); if (!capturer) { RTC_LOG(LS_ERROR) << "Failed to CreateVideoCapturer"; return; @@ -595,7 +553,8 @@ void Sora::SwitchCamera(const sora_conf::internal::CameraConfig& cc) { capturer_type_ = cc.capturer_type; std::string video_track_id = rtc::CreateRandomString(16); - auto video_track = factory_->CreateVideoTrack(video_track_id, capturer.get()); + auto video_track = sora_context_->peer_connection_factory()->CreateVideoTrack( + video_track_id, capturer.get()); // video_track_ をこのスレッド(Unity スレッド)で設定したいので、 // IO スレッドの実行完了を待つ @@ -778,6 +737,7 @@ bool Sora::InitADM(rtc::scoped_refptr adm, rtc::scoped_refptr Sora::CreateVideoCapturer( int capturer_type, void* unity_camera_texture, + bool no_video_device, std::string video_capturer_device, int video_width, int video_height, @@ -785,8 +745,13 @@ rtc::scoped_refptr Sora::CreateVideoCapturer( std::function on_frame, rtc::Thread* signaling_thread, void* jni_env, - void* android_context) { + void* android_context, + UnityContext* unity_context) { if (capturer_type == 0) { + if (no_video_device) { + return nullptr; + } + // 実カメラ（デバイス）を使う sora::CameraDeviceCapturerConfig config; config.width = video_width; @@ -802,7 +767,7 @@ rtc::scoped_refptr Sora::CreateVideoCapturer( } else { // Unity のカメラからの映像を使う UnityCameraCapturerConfig config; - config.context = &UnityContext::Instance(); + config.context = unity_context; config.unity_camera_texture = unity_camera_texture; config.width = video_width; config.height = video_height; @@ -1010,4 +975,18 @@ void Sora::SetVideoEnabled(bool enabled) { video_track_->set_enabled(enabled); } +std::string Sora::GetSelectedSignalingURL() const { + if (signaling_ == nullptr) { + return ""; + } + return signaling_->GetSelectedSignalingURL(); +} + +std::string Sora::GetConnectedSignalingURL() const { + if (signaling_ == nullptr) { + return ""; + } + return signaling_->GetConnectedSignalingURL(); +} + } // namespace sora_unity_sdk diff --git a/src/sora.h b/src/sora.h index 7a72db8..60898e1 100644 --- a/src/sora.h +++ b/src/sora.h @@ -6,6 +6,7 @@ #include // Sora +#include #include // Boost @@ -72,6 +73,9 @@ class Sora : public std::enable_shared_from_this, bool GetVideoEnabled() const; void SetVideoEnabled(bool enabled); + std::string GetSelectedSignalingURL() const; + std::string GetConnectedSignalingURL() const; + private: void* GetAndroidApplicationContext(void* env); static sora_conf::ErrorCode ToErrorCode(sora::SoraSignalingErrorCode ec); @@ -115,6 +119,7 @@ class Sora : public std::enable_shared_from_this, CreateVideoCapturer( int capturer_type, void* unity_camera_texture, + bool no_video_device, std::string video_capturer_device, int video_width, int video_height, @@ -122,7 +127,8 @@ class Sora : public std::enable_shared_from_this, std::function on_frame, rtc::Thread* signaling_thread, void* jni_env, - void* android_context); + void* android_context, + UnityContext* unity_context); void PushEvent(std::function f); @@ -140,7 +146,7 @@ class Sora : public std::enable_shared_from_this, private: std::unique_ptr ioc_; std::shared_ptr signaling_; - UnityContext* context_; + UnityContext* unity_context_; std::unique_ptr renderer_; rtc::scoped_refptr audio_track_; rtc::scoped_refptr video_track_; @@ -156,12 +162,8 @@ class Sora : public std::enable_shared_from_this, std::function on_handle_audio_; std::function on_capturer_frame_; + std::shared_ptr sora_context_; std::unique_ptr io_thread_; - std::unique_ptr network_thread_; - std::unique_ptr worker_thread_; - std::unique_ptr signaling_thread_; - rtc::scoped_refptr factory_; - rtc::scoped_refptr connection_context_; std::mutex event_mutex_; std::deque> event_queue_; diff --git a/src/unity.cpp b/src/unity.cpp index d4d01c0..49e426c 100644 --- a/src/unity.cpp +++ b/src/unity.cpp @@ -253,6 +253,27 @@ void sora_set_video_enabled(void* p, unity_bool_t enabled) { wsora->sora->SetVideoEnabled(enabled); } +// get_*_signaling_url_size() から get_*_signaling_url() までの間に値が変わった場合、 +// 落ちることは無いが、文字列が切り詰められる可能性があるので注意 +int sora_get_selected_signaling_url_size(void* p) { + auto wsora = (SoraWrapper*)p; + return wsora->sora->GetSelectedSignalingURL().size(); +} +int sora_get_connected_signaling_url_size(void* p) { + auto wsora = (SoraWrapper*)p; + return wsora->sora->GetConnectedSignalingURL().size(); +} +void sora_get_selected_signaling_url(void* p, void* buf, int size) { + auto wsora = (SoraWrapper*)p; + std::string str = wsora->sora->GetSelectedSignalingURL(); + std::memcpy(buf, str.c_str(), std::min(size, (int)str.size())); +} +void sora_get_connected_signaling_url(void* p, void* buf, int size) { + auto wsora = (SoraWrapper*)p; + std::string str = wsora->sora->GetConnectedSignalingURL(); + std::memcpy(buf, str.c_str(), std::min(size, (int)str.size())); +} + // iOS の場合は static link で名前が被る可能性があるので、別の名前にしておく void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API #if defined(SORA_UNITY_SDK_IOS) diff --git a/src/unity.h b/src/unity.h index 07fef2b..d7447ee 100644 --- a/src/unity.h +++ b/src/unity.h @@ -107,6 +107,15 @@ UNITY_INTERFACE_EXPORT unity_bool_t sora_get_video_enabled(void* p); UNITY_INTERFACE_EXPORT void sora_set_video_enabled(void* p, unity_bool_t enabled); +UNITY_INTERFACE_EXPORT int sora_get_selected_signaling_url_size(void* p); +UNITY_INTERFACE_EXPORT int sora_get_connected_signaling_url_size(void* p); +UNITY_INTERFACE_EXPORT void sora_get_selected_signaling_url(void* p, + void* buf, + int size); +UNITY_INTERFACE_EXPORT void sora_get_connected_signaling_url(void* p, + void* buf, + int size); + #ifdef __cplusplus } #endif