diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index a26b12771..ae7c35ca3 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -46,6 +46,7 @@ dependencies = [ "reqwest 0.13.4", "rodio", "rubato", + "rustls", "semver", "serde", "serde_json", @@ -72,6 +73,7 @@ dependencies = [ "tempfile", "time", "tokio", + "tokio-tungstenite", "toml 1.1.4+spec-1.1.0", "url", "uuid", @@ -1121,7 +1123,7 @@ dependencies = [ "bitflags 2.13.1", "core-foundation 0.10.1", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1404,6 +1406,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "dbus" version = "0.9.12" @@ -1415,6 +1423,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "dbus-secret-service" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" +dependencies = [ + "dbus", + "openssl", + "zeroize", +] + [[package]] name = "defmt" version = "1.1.1" @@ -2003,6 +2022,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -2010,7 +2038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -2024,6 +2052,12 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -3180,9 +3214,13 @@ version = "3.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" dependencies = [ + "byteorder", + "dbus-secret-service", "log", + "openssl", "security-framework 2.11.1", "security-framework 3.7.0", + "windows-sys 0.60.2", "zeroize", ] @@ -3237,6 +3275,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" dependencies = [ + "cc", "pkg-config", ] @@ -4141,12 +4180,59 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -7097,6 +7183,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -7357,6 +7459,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 2.0.20", + "utf-8", +] + [[package]] name = "typeid" version = "1.0.3" @@ -7566,6 +7687,12 @@ dependencies = [ "url", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8-zero" version = "0.8.1" @@ -8948,6 +9075,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 0e0abf2aa..a85b5f01e 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -42,7 +42,6 @@ hex = "0.4" ignore = "0.4.25" fern = "0.7" infer = "0.19.0" -keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } libc = "0.2" log = "0.4.29" mime_guess = "2" @@ -95,11 +94,14 @@ tauri-plugin-updater = "2" tauri-plugin-window-state = "2" time = { version = "0.3", features = ["formatting"] } tokio = { version = "1.50.0", features = ["full"] } +tokio-tungstenite = { version = "0.27", features = ["rustls-tls-webpki-roots"] } +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } url = "2" uuid = { version = "1", features = ["v4", "serde"] } zip = { version = "2", default-features = false, features = ["deflate"] } [target.'cfg(windows)'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["windows-native"] } windows-sys = { version = "0.59", features = [ "Win32_Foundation", "Win32_Globalization", @@ -112,6 +114,7 @@ windows-sys = { version = "0.59", features = [ [target.'cfg(target_os = "macos")'.dependencies] block2 = "0.6" coreaudio-rs = "0.14.2" +keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } objc2 = "0.6" objc2-app-kit = { version = "0.3.2", features = ["NSApplication", "NSRunningApplication", "NSImage", "NSMenu", "NSMenuItem", "NSAlert", "NSButton", "NSControl", "NSCell", "NSResponder", "NSView"] } objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2"] } @@ -120,6 +123,9 @@ objc2-foundation = { version = "0.3.2", features = ["NSDictionary", "NSError", " objc2-user-notifications = "0.3.2" rodio = { version = "0.22", default-features = false, features = ["playback"] } +[target.'cfg(target_os = "linux")'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["sync-secret-service", "vendored"] } + [features] default = [] diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index d633a372e..8eb4cf1d5 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -24,8 +24,7 @@ use tokio::sync::mpsc as tokio_mpsc; use super::mac_speech; use super::{ - native_input_mute, pocket_voice::parakeet_model_dir, voice_buddy, - voice_capture::VoiceCaptureState, + native_input_mute, pocket_voice::parakeet_model_dir, voice_capture::VoiceCaptureState, }; pub(crate) const EVENT_NAME: &str = "voice-conversation:event"; @@ -42,6 +41,9 @@ const SILENCE_FLUSH_FRAMES: usize = 75; const FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS: u64 = 5; const FINAL_TRANSCRIPT_DELIVERY_TIMEOUT: Duration = Duration::from_secs(FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS); +const OPENAI_NETWORK_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +const OPENAI_FINAL_WRITE_TIMEOUT: Duration = Duration::from_secs(1); +const OPENAI_PRE_ROLL_CHUNKS: usize = 15; const STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS: u64 = mac_speech::RECOGNITION_FINISH_TIMEOUT_SECONDS + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS + 1; @@ -50,6 +52,7 @@ const STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS: u64 = pub enum VoiceInputBackend { Parakeet, Macos, + Openai, } fn active_vad_threshold_for_speech( @@ -604,10 +607,12 @@ impl NativeVoiceState { } // Native input mute is authoritative when installed so a hardware // unmute cannot be masked by a stale renderer fallback latch. - self.microphone_muted.store( - software_microphone_mute(native_microphone_mute_control, muted), - Ordering::SeqCst, - ); + let software_muted = software_microphone_mute(native_microphone_mute_control, muted); + let previous_software_muted = + self.microphone_muted.swap(software_muted, Ordering::SeqCst); + if !native_microphone_mute_control && previous_software_muted != software_muted { + self.input_mute_epoch.fetch_add(1, Ordering::AcqRel); + } owner_window_label }; Ok(Some(owner_window_label)) @@ -828,6 +833,57 @@ impl SttPipeline { )) } + fn new_openai( + api_key: String, + input_muted: Arc, + input_mute_epoch: Arc, + assistant_speaking: Arc, + assistant_vad_threshold: Arc, + speech_vad_threshold: f32, + ) -> Result<(Self, tokio_mpsc::Receiver), String> { + let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); + let (event_tx, event_rx) = tokio_mpsc::channel(64); + let shutdown = Arc::new(AtomicBool::new(false)); + let discard_on_shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_mute_epoch = Arc::new(AtomicU64::new(0)); + let worker_shutdown = Arc::clone(&shutdown); + let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); + let worker_input_muted = Arc::clone(&input_muted); + let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); + let worker_shutdown_mute_epoch = Arc::clone(&shutdown_mute_epoch); + let thread = thread::Builder::new() + .name("berd-openai-stt".into()) + .spawn(move || { + openai_stt_worker( + api_key, + audio_rx, + event_tx, + worker_shutdown, + worker_discard_on_shutdown, + worker_input_muted, + worker_input_mute_epoch, + worker_shutdown_mute_epoch, + assistant_speaking, + assistant_vad_threshold, + speech_vad_threshold, + ) + }) + .map_err(|error| format!("start OpenAI speech recognition: {error}"))?; + Ok(( + Self { + audio_tx, + audio_seen: AtomicBool::new(false), + shutdown, + discard_on_shutdown, + input_muted, + input_mute_epoch, + shutdown_mute_epoch, + thread: Some(thread), + }, + event_rx, + )) + } + fn new_macos( input_muted: Arc, input_mute_epoch: Arc, @@ -1074,7 +1130,10 @@ where } async fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { - let parakeet_available = parakeet_model_dir(app).is_ok(); + let parakeet_available = parakeet_model_dir(app).is_ok() + || app + .state::() + .is_configured(); #[cfg(target_os = "macos")] let macos_status = || async { mac_speech::status_async() @@ -1232,6 +1291,19 @@ pub async fn start_native_voice_conversation( "Download the macOS speech recognition model before starting a call.".to_string(), ); } + let openai_api_key = if input_backend == VoiceInputBackend::Openai { + Some(super::openai_audio::api_key()?) + } else { + None + }; + if openai_api_key.is_some() { + let deadline = tokio::time::Instant::now() + Duration::from_secs(1); + while !webview_window.is_focused().unwrap_or(false) + && tokio::time::Instant::now() < deadline + { + tokio::time::sleep(Duration::from_millis(25)).await; + } + } let window_label = webview_window.label().to_string(); let owner_id = native_owner_id(&session_id); let lifecycle_guard = state @@ -1272,6 +1344,14 @@ pub async fn start_native_voice_conversation( Arc::clone(&state.assistant_vad_threshold), speech_vad_threshold, ), + VoiceInputBackend::Openai => SttPipeline::new_openai( + openai_api_key.expect("OpenAI key resolved for OpenAI input"), + Arc::clone(&state.input_muted), + Arc::clone(&state.input_mute_epoch), + Arc::clone(&state.assistant_speaking), + Arc::clone(&state.assistant_vad_threshold), + speech_vad_threshold, + ), }; let (pipeline, mut events) = match pipeline { Ok(result) => result, @@ -1672,6 +1752,7 @@ pub async fn stop_native_voice_conversation_for_replacement( Ok(status(&app, &state).await) } +#[cfg(test)] fn replacement_caller_matches_target( caller_window_label: &str, target_owner: Option<&str>, @@ -1686,53 +1767,6 @@ fn replacement_caller_matches_target( } } -fn voice_target_window_focus_is_valid( - window_label: &str, - focused: bool, - app_is_active: bool, - main_surface_is_available: bool, - another_window_is_focused: bool, -) -> bool { - focused - || (window_label == "main" - && app_is_active - && main_surface_is_available - && !another_window_is_focused) -} - -fn voice_main_surface_is_available(visible: bool, minimized: bool) -> bool { - visible && !minimized -} - -#[cfg(target_os = "macos")] -fn app_is_active_for_main_window_focus_fallback() -> bool { - use objc2_app_kit::NSRunningApplication; - - // The non-activating floating controls can leave Berd frontmost while - // AppKit reports that none of its ordinary windows are focused. - NSRunningApplication::currentApplication().isActive() -} - -#[cfg(not(target_os = "macos"))] -fn app_is_active_for_main_window_focus_fallback() -> bool { - false -} - -fn another_user_window_is_focused(webview_window: &WebviewWindow) -> Result { - for (label, window) in webview_window.app_handle().webview_windows() { - if label == webview_window.label() || label == voice_buddy::WINDOW_LABEL { - continue; - } - if window - .is_focused() - .map_err(|error| format!("Could not confirm Berd window focus: {error}"))? - { - return Ok(true); - } - } - Ok(false) -} - fn validate_voice_target_session( capture: &VoiceCaptureState, window_sessions: &super::window_session::WindowSessionRegistry, @@ -1740,50 +1774,16 @@ fn validate_voice_target_session( renderer_id: &str, renderer_epoch: u64, target_session_id: &str, - foreground_generation: Option, + _foreground_generation: Option, ) -> Result<(), String> { + capture.activate_renderer(webview_window.label(), renderer_id, renderer_epoch)?; let target_owner = window_sessions.label_for(target_session_id); - let owns_foreground_session = capture.foreground_session_matches_generation( - webview_window.label(), - renderer_id, - renderer_epoch, - target_session_id, - foreground_generation, - )?; - if !replacement_caller_matches_target( - webview_window.label(), - target_owner.as_deref(), - owns_foreground_session, - ) { - return Err("The target session is no longer in the foreground.".to_string()); - } - let focused = webview_window - .is_focused() - .map_err(|error| format!("Could not confirm the target session window focus: {error}"))?; - let app_is_active = !focused - && webview_window.label() == "main" - && app_is_active_for_main_window_focus_fallback(); - let main_surface_is_available = if app_is_active { - let visible = webview_window - .is_visible() - .map_err(|error| format!("Could not confirm the main window visibility: {error}"))?; - let minimized = webview_window - .is_minimized() - .map_err(|error| format!("Could not confirm the main window state: {error}"))?; - voice_main_surface_is_available(visible, minimized) - } else { - false + let caller_owns_target = match target_owner.as_deref() { + Some(owner_window_label) => owner_window_label == webview_window.label(), + None => webview_window.label() == "main", }; - let another_window_is_focused = - main_surface_is_available && another_user_window_is_focused(webview_window)?; - if !voice_target_window_focus_is_valid( - webview_window.label(), - focused, - app_is_active, - main_surface_is_available, - another_window_is_focused, - ) { - return Err("The target session window is no longer focused.".to_string()); + if !caller_owns_target { + return Err("The target session belongs to a different Berd window.".to_string()); } Ok(()) } @@ -2549,6 +2549,556 @@ fn macos_stt_worker( let _ = forward_macos_events(&mut recognition_events, &event_tx, Some(delivery_deadline)); } +#[derive(Debug, PartialEq, Eq)] +struct OpenAiCommittedTurn { + item_id: String, + mute_epoch: u64, +} + +fn block_on_openai_operation( + runtime: &tokio::runtime::Runtime, + shutdown: &AtomicBool, + future: F, + action: &str, +) -> Result, String> +where + F: std::future::Future>, + E: std::fmt::Display, +{ + runtime.block_on(async { + let wait_for_shutdown = async { + while !shutdown.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }; + tokio::select! { + result = tokio::time::timeout(OPENAI_NETWORK_OPERATION_TIMEOUT, future) => { + match result { + Ok(Ok(value)) => Ok(Some(value)), + Ok(Err(error)) => Err(format!("{action}: {error}")), + Err(_) => Err(format!("{action}: operation timed out")), + } + } + () = wait_for_shutdown => Ok(None), + } + }) +} + +fn push_openai_pre_roll(pre_roll: &mut VecDeque>, pcm: Vec) { + pre_roll.push_back(pcm); + while pre_roll.len() > OPENAI_PRE_ROLL_CHUNKS { + pre_roll.pop_front(); + } +} + +fn openai_turn_reached_limit(samples_16k: usize) -> bool { + samples_16k >= MAX_SPEECH_SAMPLES +} + +fn record_openai_transcription_event( + value: &serde_json::Value, + current_mute_epoch: u64, + pending_commit_epochs: &mut VecDeque, + committed: &mut VecDeque, + completed: &mut HashMap, +) -> Result, String> { + let mut newly_committed = None; + match value.get("type").and_then(|value| value.as_str()) { + Some("input_audio_buffer.committed") => { + if let Some(item_id) = value.get("item_id").and_then(|value| value.as_str()) { + let turn = OpenAiCommittedTurn { + item_id: item_id.to_string(), + mute_epoch: pending_commit_epochs + .pop_front() + .unwrap_or(current_mute_epoch), + }; + if turn.mute_epoch == current_mute_epoch { + committed.push_back(OpenAiCommittedTurn { + item_id: turn.item_id.clone(), + mute_epoch: turn.mute_epoch, + }); + } + newly_committed = Some(turn); + } + } + Some("conversation.item.input_audio_transcription.completed") => { + if let (Some(item_id), Some(transcript)) = ( + value.get("item_id").and_then(|value| value.as_str()), + value.get("transcript").and_then(|value| value.as_str()), + ) { + if committed + .iter() + .any(|turn| turn.item_id == item_id && turn.mute_epoch == current_mute_epoch) + { + completed.insert(item_id.to_string(), transcript.trim().to_string()); + } + } + } + Some("conversation.item.input_audio_transcription.failed") | Some("error") => { + return Err(value + .pointer("/error/message") + .and_then(|value| value.as_str()) + .unwrap_or("OpenAI realtime transcription failed.") + .to_string()); + } + _ => {} + } + Ok(newly_committed) +} + +fn deliver_completed_openai_turns( + committed: &mut VecDeque, + completed: &mut HashMap, + event_tx: &tokio_mpsc::Sender, + final_item_id: Option<&str>, + final_delivery: &mut Option>, +) { + while committed + .front() + .is_some_and(|turn| completed.contains_key(&turn.item_id)) + { + let turn = committed.pop_front().expect("checked front"); + let text = completed.remove(&turn.item_id).unwrap_or_default(); + let delivered = (Some(turn.item_id.as_str()) == final_item_id) + .then(|| final_delivery.take()) + .flatten(); + deliver_recognition_result(text, event_tx, delivered); + } +} + +#[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. +fn openai_stt_worker( + key: String, + audio_rx: Receiver, + event_tx: tokio_mpsc::Sender, + shutdown: Arc, + discard_on_shutdown: Arc, + input_muted: Arc, + input_mute_epoch: Arc, + shutdown_mute_epoch: Arc, + assistant_speaking: Arc, + assistant_vad_threshold: Arc, + speech_vad_threshold: f32, +) { + use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; + use futures_util::{SinkExt, StreamExt}; + use rubato::{Fft, FixedSync, Resampler}; + use tokio_tungstenite::tungstenite::{client::IntoClientRequest, Message}; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "Could not initialize OpenAI realtime transcription: {error}" + ))); + return; + } + }; + let model = super::openai_audio::transcription_model(); + let endpoint = match super::openai_audio::realtime_endpoint() { + Ok(endpoint) => endpoint, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + }; + if let Err(existing) = rustls::crypto::aws_lc_rs::default_provider().install_default() { + // Another dependency may have installed the same process-wide provider first. + drop(existing); + } + let mut request = match endpoint.into_client_request() { + Ok(request) => request, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "prepare OpenAI realtime connection: {error}" + ))); + return; + } + }; + let authorization = match format!("Bearer {key}").parse() { + Ok(authorization) => authorization, + Err(_) => { + let _ = event_tx.blocking_send(SttMessage::Failed( + "OpenAI API key is not a valid header value".to_string(), + )); + return; + } + }; + request.headers_mut().insert("Authorization", authorization); + let connection = block_on_openai_operation( + &runtime, + &shutdown, + tokio_tungstenite::connect_async(request), + "connect to OpenAI realtime transcription", + ); + let mut socket = match connection { + Ok(Some((socket, _))) => socket, + Ok(None) => return, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + }; + + macro_rules! send_openai_startup_message { + ($message:expr, $action:literal) => { + match block_on_openai_operation(&runtime, &shutdown, socket.send($message), $action) { + Ok(Some(())) => {} + Ok(None) => return, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + } + }; + } + let session_update = serde_json::json!({ + "type": "session.update", + "session": { + "type": "transcription", + "audio": { "input": { + "format": { "type": "audio/pcm", "rate": 24000 }, + "transcription": { "model": model, "delay": "low" }, + "turn_detection": null + }} + } + }); + send_openai_startup_message!( + Message::Text(session_update.to_string().into()), + "configure OpenAI realtime transcription" + ); + + let mut resampler = match Fft::::new(48_000, 24_000, 960, 2, 1, FixedSync::Input) { + Ok(resampler) => resampler, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "Could not initialize OpenAI audio resampling: {error}" + ))); + return; + } + }; + let chunk_in = resampler.input_frames_next(); + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut input_48k = Vec::new(); + let mut vad_16k = Vec::new(); + let mut silence_frames = 0_usize; + let mut in_speech = false; + let mut turn_has_audio = false; + let mut turn_samples_16k = 0_usize; + let mut pre_roll = VecDeque::>::new(); + let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); + let mut pending_commit_epochs = VecDeque::::new(); + let mut committed_turns = VecDeque::::new(); + let mut completed_turns = HashMap::::new(); + + macro_rules! send_openai_stream_message { + ($message:expr, $action:literal, $on_shutdown:block) => { + match block_on_openai_operation(&runtime, &shutdown, socket.send($message), $action) { + Ok(Some(())) => {} + Ok(None) => $on_shutdown + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + } + }; + } + + 'worker: loop { + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break; + } + if current_mute_epoch != observed_mute_epoch { + observed_mute_epoch = current_mute_epoch; + input_48k.clear(); + vad_16k.clear(); + silence_frames = 0; + turn_has_audio = false; + turn_samples_16k = 0; + pre_roll.clear(); + committed_turns.clear(); + completed_turns.clear(); + if std::mem::take(&mut in_speech) { + let _ = event_tx.blocking_send(SttMessage::Speaking(false)); + } + send_openai_stream_message!( + Message::Text( + serde_json::json!({"type":"input_audio_buffer.clear"}) + .to_string() + .into() + ), + "clear muted OpenAI transcription audio", + { break 'worker } + ); + } + while let Some(event) = runtime.block_on(async { + tokio::time::timeout(Duration::from_millis(1), socket.next()) + .await + .ok() + .flatten() + }) { + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break 'worker; + } + if current_mute_epoch != observed_mute_epoch { + continue 'worker; + } + let message = match event { + Ok(Message::Text(text)) => text, + Ok(Message::Close(_)) => { + let _ = event_tx.blocking_send(SttMessage::Failed( + "OpenAI realtime transcription disconnected.".to_string(), + )); + return; + } + Ok(_) => continue, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "OpenAI realtime transcription failed: {error}" + ))); + return; + } + }; + let Ok(value) = serde_json::from_str::(&message) else { + continue; + }; + if let Err(error) = record_openai_transcription_event( + &value, + observed_mute_epoch, + &mut pending_commit_epochs, + &mut committed_turns, + &mut completed_turns, + ) { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + deliver_completed_openai_turns( + &mut committed_turns, + &mut completed_turns, + &event_tx, + None, + &mut None, + ); + } + + let batch = match audio_rx.recv_timeout(Duration::from_millis(20)) { + Ok(batch) => Some(batch), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + // Stop consuming queued microphone batches as soon as shutdown begins. + // The bounded finalization below commits only audio already accepted by + // this worker, so network backpressure cannot extend the owner timeout. + if shutting_down { + break; + } + if current_mute_epoch != observed_mute_epoch { + continue 'worker; + } + if input_muted.load(Ordering::Acquire) { + continue; + } + let Some(batch) = batch else { continue }; + if batch.mute_epoch != observed_mute_epoch { + continue; + } + input_48k.extend( + batch + .bytes + .chunks_exact(4) + .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])), + ); + while input_48k.len() >= chunk_in { + let chunk: Vec = input_48k.drain(..chunk_in).collect(); + let pcm_24k = resample(&mut resampler, &chunk); + let pcm_bytes: Vec = pcm_24k + .iter() + .flat_map(|sample| { + ((sample.clamp(-1.0, 1.0) * i16::MAX as f32).round() as i16).to_le_bytes() + }) + .collect(); + // Earshot requires 16 kHz; use every third 48 kHz source sample for activity only. + vad_16k.extend(chunk.iter().step_by(3).copied()); + let mut speech_started = false; + let mut should_commit = false; + while vad_16k.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = vad_16k.drain(..VAD_FRAME_SAMPLES).collect(); + let threshold = active_vad_threshold_for_speech( + &assistant_speaking, + &assistant_vad_threshold, + speech_vad_threshold, + ); + if vad.predict_f32(&frame) > threshold { + silence_frames = 0; + if !in_speech { + in_speech = true; + speech_started = true; + let _ = event_tx.blocking_send(SttMessage::Speaking(true)); + } + } else if in_speech { + silence_frames += 1; + if silence_frames >= SILENCE_FLUSH_FRAMES { + should_commit = true; + silence_frames = 0; + in_speech = false; + } + } + } + + if speech_started { + while let Some(pre_roll_bytes) = pre_roll.pop_front() { + send_openai_stream_message!( + Message::Text( + serde_json::json!({ + "type": "input_audio_buffer.append", + "audio": BASE64.encode(pre_roll_bytes) + }) + .to_string() + .into() + ), + "stream OpenAI transcription pre-roll", + { break 'worker } + ); + } + } + + if speech_started || in_speech || should_commit { + send_openai_stream_message!( + Message::Text( + serde_json::json!({ + "type": "input_audio_buffer.append", + "audio": BASE64.encode(pcm_bytes) + }) + .to_string() + .into() + ), + "stream audio to OpenAI realtime transcription", + { break 'worker } + ); + turn_has_audio = true; + turn_samples_16k = turn_samples_16k.saturating_add(chunk.len() / 3); + } else { + push_openai_pre_roll(&mut pre_roll, pcm_bytes); + } + + if should_commit || openai_turn_reached_limit(turn_samples_16k) { + send_openai_stream_message!( + Message::Text( + serde_json::json!({"type":"input_audio_buffer.commit"}) + .to_string() + .into() + ), + "commit OpenAI transcription turn", + { break 'worker } + ); + pending_commit_epochs.push_back(observed_mute_epoch); + turn_has_audio = false; + turn_samples_16k = 0; + silence_frames = 0; + if std::mem::take(&mut in_speech) || should_commit { + let _ = event_tx.blocking_send(SttMessage::Speaking(false)); + } + } + } + } + if discard_on_shutdown.load(Ordering::Acquire) { + return; + } + let mut final_item_id = None::; + let mut final_delivery = None::>; + let mut final_delivered = None::>; + if turn_has_audio { + let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); + let commit = serde_json::json!({"type":"input_audio_buffer.commit"}); + let final_write = runtime.block_on(tokio::time::timeout( + OPENAI_FINAL_WRITE_TIMEOUT, + socket.send(Message::Text(commit.to_string().into())), + )); + if matches!(final_write, Ok(Ok(()))) { + pending_commit_epochs.push_back(observed_mute_epoch); + final_delivery = Some(delivered_tx); + final_delivered = Some(delivered_rx); + } + } + + // Keep receiving until the shutdown commit itself completes. Earlier turns + // are delivered from the queue first, and already committed transcripts are + // drained even when there was no partial turn to commit at shutdown. + let deadline = std::time::Instant::now() + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT; + while std::time::Instant::now() < deadline { + if final_delivered + .as_ref() + .is_some_and(|receiver| receiver.try_recv().is_ok()) + || (final_delivered.is_none() + && committed_turns.is_empty() + && pending_commit_epochs.is_empty()) + { + break; + } + let Some(message) = runtime.block_on(async { + tokio::time::timeout(Duration::from_millis(50), socket.next()) + .await + .ok() + .flatten() + }) else { + continue; + }; + let Ok(Message::Text(text)) = message else { + continue; + }; + let Ok(value) = serde_json::from_str::(&text) else { + continue; + }; + let event_type = value.get("type").and_then(|value| value.as_str()); + match record_openai_transcription_event( + &value, + observed_mute_epoch, + &mut pending_commit_epochs, + &mut committed_turns, + &mut completed_turns, + ) { + Ok(Some(turn)) + if turn.mute_epoch == observed_mute_epoch + && pending_commit_epochs.is_empty() + && final_delivered.is_some() => + { + final_item_id = Some(turn.item_id); + } + Ok(_) => {} + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + break; + } + } + deliver_completed_openai_turns( + &mut committed_turns, + &mut completed_turns, + &event_tx, + final_item_id.as_deref(), + &mut final_delivery, + ); + if event_type == Some("conversation.item.input_audio_transcription.completed") + && final_item_id.as_deref() == value.get("item_id").and_then(|value| value.as_str()) + && final_delivery.is_none() + { + break; + } + } + // Do not await the peer's WebSocket close handshake here. OpenAI may leave + // it pending beyond the bounded voice-stop window; dropping the socket + // closes the connection after final transcript delivery has been drained. + drop(socket); +} + #[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. fn stt_worker( model_dir: PathBuf, @@ -2992,42 +3542,6 @@ mod tests { )); } - #[test] - fn replacement_focus_accepts_an_active_app_only_for_the_main_window() { - assert!(voice_main_surface_is_available(true, false)); - assert!(!voice_main_surface_is_available(false, false)); - assert!(!voice_main_surface_is_available(true, true)); - assert!(voice_target_window_focus_is_valid( - "main", true, false, false, false, - )); - assert!(voice_target_window_focus_is_valid( - "main", false, true, true, false, - )); - assert!(!voice_target_window_focus_is_valid( - "main", false, true, true, true, - )); - assert!(!voice_target_window_focus_is_valid( - "main", false, true, false, false, - )); - assert!(!voice_target_window_focus_is_valid( - "main", false, false, true, false, - )); - assert!(voice_target_window_focus_is_valid( - "session:target", - true, - false, - false, - false, - )); - assert!(!voice_target_window_focus_is_valid( - "session:target", - false, - true, - true, - false, - )); - } - #[test] fn interruption_sensitivity_only_changes_vad_while_assistant_speaks() { let state = NativeVoiceState::default(); @@ -3428,6 +3942,30 @@ mod tests { assert!(state.microphone_is_muted()); } + #[test] + fn software_microphone_mute_advances_the_audio_epoch() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-1".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + runtime.native_microphone_mute_control = false; + } + + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 0); + state + .set_microphone_muted_target("main", "session-1", 4, true) + .expect("mute"); + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 1); + state + .set_microphone_muted_target("main", "session-1", 4, false) + .expect("unmute"); + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 2); + } + #[test] fn owner_stop_authorization_is_lifecycle_bound() { let state = NativeVoiceState::default(); @@ -4091,6 +4629,157 @@ mod tests { assert!(delivered_rx.try_recv().is_ok()); } + #[test] + fn openai_transcripts_are_delivered_in_commit_order() { + let mut pending_epochs = VecDeque::from([7, 7]); + let mut committed = VecDeque::new(); + let mut completed = HashMap::new(); + for item_id in ["first", "second"] { + record_openai_transcription_event( + &serde_json::json!({ + "type": "input_audio_buffer.committed", + "item_id": item_id, + }), + 7, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("commit event"); + } + for (item_id, transcript) in [("second", "two"), ("first", "one")] { + record_openai_transcription_event( + &serde_json::json!({ + "type": "conversation.item.input_audio_transcription.completed", + "item_id": item_id, + "transcript": transcript, + }), + 7, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("completion event"); + } + let (event_tx, mut event_rx) = tokio_mpsc::channel(4); + + deliver_completed_openai_turns(&mut committed, &mut completed, &event_tx, None, &mut None); + + let texts = [event_rx.try_recv(), event_rx.try_recv()].map(|event| match event { + Ok(SttMessage::Final { text, .. }) => text, + _ => panic!("expected finalized transcript"), + }); + assert_eq!(texts, ["one", "two"]); + } + + #[test] + fn openai_transcription_ignores_commits_from_stale_mute_epochs() { + let mut pending_epochs = VecDeque::from([1]); + let mut committed = VecDeque::new(); + let mut completed = HashMap::new(); + + let turn = record_openai_transcription_event( + &serde_json::json!({ + "type": "input_audio_buffer.committed", + "item_id": "stale", + }), + 2, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("commit event") + .expect("recorded commit"); + + assert_eq!(turn.mute_epoch, 1); + assert!(committed.is_empty()); + } + + #[test] + fn openai_transcription_surfaces_protocol_failures() { + let error = record_openai_transcription_event( + &serde_json::json!({ + "type": "conversation.item.input_audio_transcription.failed", + "error": { "message": "bad audio" }, + }), + 0, + &mut VecDeque::new(), + &mut VecDeque::new(), + &mut HashMap::new(), + ) + .expect_err("failure event"); + + assert_eq!(error, "bad audio"); + } + + #[test] + fn empty_openai_final_acknowledges_shutdown_delivery() { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "final".to_string(), + mute_epoch: 3, + }]); + let mut completed = HashMap::from([("final".to_string(), String::new())]); + let (event_tx, mut event_rx) = tokio_mpsc::channel(1); + let (delivered_tx, delivered_rx) = mpsc::sync_channel(1); + let mut final_delivery = Some(delivered_tx); + + deliver_completed_openai_turns( + &mut committed, + &mut completed, + &event_tx, + Some("final"), + &mut final_delivery, + ); + + assert!(delivered_rx.try_recv().is_ok()); + assert!(event_rx.try_recv().is_err()); + } + + #[test] + fn stalled_openai_operation_observes_shutdown() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("runtime"); + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_for_thread = shutdown.clone(); + let signal = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(30)); + shutdown_for_thread.store(true, Ordering::Release); + }); + + let result = block_on_openai_operation( + &runtime, + shutdown.as_ref(), + std::future::pending::>(), + "stalled operation", + ) + .expect("shutdown is not an error"); + + signal.join().expect("shutdown signal"); + assert_eq!(result, None); + } + + #[test] + fn openai_idle_audio_keeps_only_bounded_pre_roll() { + let mut pre_roll = VecDeque::new(); + for index in 0..(OPENAI_PRE_ROLL_CHUNKS * 4) { + push_openai_pre_roll(&mut pre_roll, vec![index as u8]); + } + + assert_eq!(pre_roll.len(), OPENAI_PRE_ROLL_CHUNKS); + assert_eq!( + pre_roll.front(), + Some(&vec![(OPENAI_PRE_ROLL_CHUNKS * 3) as u8]) + ); + } + + #[test] + fn openai_continuous_speech_has_a_turn_limit() { + assert!(!openai_turn_reached_limit(MAX_SPEECH_SAMPLES - 1)); + assert!(openai_turn_reached_limit(MAX_SPEECH_SAMPLES)); + } + #[test] fn native_voice_events_use_renderer_field_names() { let event = NativeVoiceEvent::User { @@ -4137,5 +4826,10 @@ mod tests { .expect("deserialize macOS backend"), VoiceInputBackend::Macos, ); + assert_eq!( + serde_json::from_str::("\"openai\"") + .expect("deserialize OpenAI backend"), + VoiceInputBackend::Openai, + ); } } diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index c69956006..f79a8bd45 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -1,4 +1,4 @@ -//! OpenAI streaming speech playback for voice conversations. +//! OpenAI realtime transcription configuration and streaming speech playback. use std::sync::{ atomic::{AtomicBool, Ordering}, @@ -34,8 +34,8 @@ use super::{ #[cfg(any(test, target_os = "macos"))] use std::time::Instant; -#[cfg(target_os = "macos")] const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; +const DEFAULT_TRANSCRIPTION_MODEL: &str = "gpt-live-transcribe"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; const DEFAULT_TTS_VOICE: &str = "marin"; #[cfg(target_os = "macos")] @@ -56,6 +56,13 @@ const MAX_TTS_INPUT_CHARS: usize = 4096; #[derive(Clone, Debug, Default)] pub struct OpenAiVoiceState { playback: Arc>, + configured: Arc, +} + +impl OpenAiVoiceState { + pub(crate) fn is_configured(&self) -> bool { + self.configured.load(Ordering::Acquire) + } } #[derive(Debug)] @@ -95,6 +102,7 @@ enum OpenAiStreamCommand { #[serde(rename_all = "camelCase")] pub struct OpenAiVoiceStatus { configured: bool, + transcription_model: String, speech_model: String, speech_voice: String, playback_speed: f32, @@ -148,7 +156,6 @@ fn env_trimmed(name: &str) -> Option { .filter(|value| !value.is_empty()) } -#[cfg(target_os = "macos")] fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result, String> { if !path.exists() { return Ok(None); @@ -165,14 +172,11 @@ fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result .map(ToString::to_string)) } -#[cfg(target_os = "macos")] fn goose_openai_api_key() -> Result, String> { if let Some(value) = env_trimmed("OPENAI_API_KEY") { return Ok(Some(value)); } - #[cfg(target_os = "macos")] let mut secure_store_error = None; - #[cfg(target_os = "macos")] { match keyring::Entry::new("goose", "secrets") { Ok(entry) => match entry.get_password() { @@ -215,14 +219,12 @@ fn goose_openai_api_key() -> Result, String> { if let Some(key) = goose_yaml_value(&secrets_path, "OPENAI_API_KEY")? { return Ok(Some(key)); } - #[cfg(target_os = "macos")] if let Some(error) = secure_store_error { return Err(error); } Ok(None) } -#[cfg(target_os = "macos")] pub(crate) fn api_key() -> Result { goose_openai_api_key()?.ok_or_else(|| { "OpenAI voice is not configured. Configure the OpenAI provider in Berd, then try again." @@ -230,7 +232,6 @@ pub(crate) fn api_key() -> Result { }) } -#[cfg(any(test, target_os = "macos"))] fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result { let mut url = reqwest::Url::parse(&raw_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; @@ -252,7 +253,6 @@ fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result Result { if let Some(host) = env_trimmed("OPENAI_HOST") { return normalize_openai_base_url(host, true); @@ -270,6 +270,29 @@ fn base_url() -> Result { Ok(DEFAULT_BASE_URL.to_string()) } +pub(crate) fn realtime_endpoint() -> Result { + let mut url = reqwest::Url::parse(&endpoint("realtime")?) + .map_err(|error| format!("OpenAI realtime endpoint is invalid: {error}"))?; + url.query_pairs_mut().append_pair("intent", "transcription"); + match url.scheme() { + "http" => url.set_scheme("ws").expect("compatible scheme"), + "https" => url.set_scheme("wss").expect("compatible scheme"), + "ws" | "wss" => {} + scheme => { + return Err(format!( + "OpenAI realtime endpoint has unsupported scheme: {scheme}" + )) + } + } + Ok(url.to_string()) +} + +pub(crate) fn transcription_model() -> String { + env_trimmed("OPENAI_TRANSCRIPTION_MODEL") + .or_else(|| env_trimmed("OPENAI_STT_MODEL")) + .unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string()) +} + fn speech_model() -> String { env_trimmed("OPENAI_TTS_MODEL").unwrap_or_else(|| DEFAULT_TTS_MODEL.to_string()) } @@ -278,12 +301,10 @@ fn speech_voice() -> String { env_trimmed("OPENAI_TTS_VOICE").unwrap_or_else(|| DEFAULT_TTS_VOICE.to_string()) } -#[cfg(target_os = "macos")] fn endpoint(path: &str) -> Result { endpoint_for_base_url(&base_url()?, path) } -#[cfg(any(test, target_os = "macos"))] fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { let mut url = reqwest::Url::parse(base_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; @@ -357,6 +378,7 @@ pub fn get_openai_voice_status( // the credential resolver used when a stream starts. let environment_key = env_trimmed("OPENAI_API_KEY"); let configured = openai_voice_configured(provider_configured, environment_key.as_deref()); + state.configured.store(configured, Ordering::Release); let playback_speed = state .playback .lock() @@ -365,6 +387,7 @@ pub fn get_openai_voice_status( let tts_available = cfg!(target_os = "macos"); Ok(OpenAiVoiceStatus { configured, + transcription_model: transcription_model(), speech_model: speech_model(), speech_voice: speech_voice(), playback_speed, @@ -1214,6 +1237,16 @@ mod tests { assert_eq!(drained_at, None); } + #[test] + fn configured_readiness_does_not_require_reading_the_secret() { + let state = OpenAiVoiceState::default(); + assert!(!state.is_configured()); + + state.configured.store(true, Ordering::Release); + + assert!(state.is_configured()); + } + #[cfg(target_os = "macos")] #[test] fn chunks_tts_text_on_char_boundaries() { diff --git a/src-tauri/src/commands/voice_capture.rs b/src-tauri/src/commands/voice_capture.rs index 7745252b5..34d31becf 100644 --- a/src-tauri/src/commands/voice_capture.rs +++ b/src-tauri/src/commands/voice_capture.rs @@ -215,6 +215,7 @@ impl VoiceCaptureState { ) } + #[cfg(test)] pub fn foreground_session_matches_generation( &self, window_label: &str, diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index 15d6d2ba4..ae559725b 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -744,7 +744,9 @@ export function AppShell({ ); const globalVoiceOutput = useVoiceOutputPreference(); const globalOpenAiVoiceSetup = useOpenAiVoiceSetup( - capabilities.voiceConversation && globalVoiceOutput.backend === "openai", + capabilities.voiceConversation && + (globalVoiceInput.backend === "openai" || + globalVoiceOutput.backend === "openai"), ); const globalSiriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && globalVoiceOutput.backend === "siri", diff --git a/src/features/chat/ui/ChatInputToolbar.tsx b/src/features/chat/ui/ChatInputToolbar.tsx index 3050e94f7..0ea5f5094 100644 --- a/src/features/chat/ui/ChatInputToolbar.tsx +++ b/src/features/chat/ui/ChatInputToolbar.tsx @@ -4,6 +4,7 @@ import { MicOff, Phone, PhoneOff, + RotateCcw, ArrowUp, File, FolderOpen, @@ -563,6 +564,8 @@ export function ChatInputToolbar({ > {ownsActiveVoiceConversation ? (