From 2aa2d7034fed710b20d5014a230a339fd3ab75d4 Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Tue, 10 Nov 2026 20:14:26 +0700 Subject: [PATCH] feat: add speech component --- Cargo.lock | 169 ++++++ crates/core/Cargo.toml | 6 + crates/core/src/lib.rs | 3 + crates/core/src/speech.rs | 1065 +++++++++++++++++++++++++++++++++++ crates/core/tests/speech.rs | 282 ++++++++++ crates/icons/assets/mic.svg | 15 + crates/icons/src/lib.rs | 11 + crates/icons/src/segoe.rs | 2 + 8 files changed, 1553 insertions(+) create mode 100644 crates/core/src/speech.rs create mode 100644 crates/core/tests/speech.rs create mode 100644 crates/icons/assets/mic.svg diff --git a/Cargo.lock b/Cargo.lock index d0fda04..be8589f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,6 +119,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -957,6 +979,20 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + [[package]] name = "cosmic-text" version = "0.19.0" @@ -981,6 +1017,36 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpal" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f02e8d0327b42d3e2e4ab2119af397344eb9fc54a34bf0ddeaa1277af8681f1" +dependencies = [ + "alsa", + "block2 0.6.2", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2 0.6.5", + "objc2-audio-toolbox", + "objc2-avf-audio", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "web-sys", + "windows 0.62.2", + "windows-core 0.62.2", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1103,6 +1169,12 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-url" version = "0.3.2" @@ -1397,6 +1469,9 @@ name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +dependencies = [ + "getrandom 0.4.3", +] [[package]] name = "fax" @@ -1460,6 +1535,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" dependencies = [ + "fastrand", "futures-core", "futures-sink", "spin 0.9.9", @@ -1888,6 +1964,8 @@ dependencies = [ name = "gpui-native-core" version = "0.1.0" dependencies = [ + "cpal", + "flume", "gpui", "gpui-native-icons", "gpui-native-theme", @@ -3294,6 +3372,26 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.13.2", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -3501,6 +3599,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "objc-sys" version = "0.3.5" @@ -3556,6 +3676,55 @@ dependencies = [ "objc2-uniform-type-identifiers", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2 0.6.5", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.5", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", +] + [[package]] name = "objc2-core-data" version = "0.2.2" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 8026bf7..3be2ed7 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -11,6 +11,10 @@ gpui-native-theme = { workspace = true } gpui-native-utils = { workspace = true } log = "0.4" unicode-segmentation = "1.12" +flume = { version = "0.12", optional = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +cpal = { version = "0.18", optional = true } [dev-dependencies] gpui = { workspace = true, features = ["test-support"] } @@ -49,6 +53,7 @@ full = [ "empty_state", "alert", "toast", + "speech", ] button = ["spinner"] checkbox = [] @@ -81,3 +86,4 @@ dialog = ["button"] empty_state = [] alert = [] toast = [] +speech = ["spinner", "tooltip", "dep:flume", "dep:cpal"] diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index b5d10a3..10d54e8 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -81,6 +81,9 @@ pub mod separator; #[cfg(feature = "spinner")] pub mod spinner; +#[cfg(feature = "speech")] +pub mod speech; + #[cfg(feature = "field")] pub mod field; diff --git a/crates/core/src/speech.rs b/crates/core/src/speech.rs new file mode 100644 index 0000000..2f9edde --- /dev/null +++ b/crates/core/src/speech.rs @@ -0,0 +1,1065 @@ +//! Dictation: a state that turns speech into text, a button that drives the +//! session, and a waveform that shows what the microphone hears. +//! +//! The application owns where the text goes: subscribe to [`SpeechEvent`] and +//! insert the final transcript where it belongs. Recognition and capture are +//! both replaceable seams; without further setup the state uses [`Microphone`] +//! and has no recognizer, so [`SpeechButton`] renders nothing until the +//! application provides one. + +use std::time::Duration; + +#[cfg(not(target_arch = "wasm32"))] +use std::sync::{Arc, Mutex}; + +use gpui::prelude::FluentBuilder as _; +use gpui::{ + App, Context, ElementId, Entity, EventEmitter, InteractiveElement as _, IntoElement, + ParentElement as _, RenderOnce, Role, SharedString, StatefulInteractiveElement as _, Styled, + Subscription, Task, WeakEntity, Window, div, relative, +}; +use gpui_native_icons::{IconRole, icon}; +use gpui_native_theme::{ActiveTheme as _, BORDER_WIDTH, Size}; +use gpui_native_utils::focus_ring; + +use crate::common::{Sizable, StyledExt as _, h_flex}; +use crate::spinner::{Spinner, SpinnerVariant}; +use crate::tooltip::Tooltip; + +/// One input-level entry per this much of captured audio. +const LEVEL_BUCKET: Duration = Duration::from_millis(80); +/// The level history a waveform draws, one entry per [`LEVEL_BUCKET`]. +const MAX_LEVELS: usize = 128; +/// Amplitude below which background noise reads as silence. +const NOISE_GATE: f32 = 0.03; +/// How much of a full-scale peak maps to a full bar. +const LEVEL_SCALE: f32 = 0.5; +/// How fast a peak falls back before the next bucket's level. +const LEVEL_FALL: f32 = 0.72; +/// How long a session waits for the recognizer's final result once stopped. +const DEFAULT_STOP_TIMEOUT: Duration = Duration::from_secs(3); +/// The bars [`SpeechButton`] shows while capturing. +const BUTTON_BARS: usize = 4; +/// The share of the bar height a silent bar keeps, so silence reads as a dot. +const DOT_HEIGHT: f32 = 0.14; +/// The bar height reduced motion holds at. +const STILL_HEIGHT: f32 = 0.35; + +/// The PCM layout a recognizer consumes: interleaved 16-bit samples. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AudioFormat { + sample_rate: u32, + channels: u8, +} + +impl AudioFormat { + /// Creates a format at the given sample rate and channel count. + pub fn new(sample_rate: u32, channels: u8) -> Self { + Self { + sample_rate, + channels: channels.max(1), + } + } + + /// Samples per second per channel. + pub fn sample_rate(&self) -> u32 { + self.sample_rate + } + + /// The number of interleaved channels. + pub fn channels(&self) -> u8 { + self.channels + } +} + +impl Default for AudioFormat { + /// 16 kHz mono, what most recognizers consume. + fn default() -> Self { + Self::new(16_000, 1) + } +} + +/// Why a dictation session failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SpeechError { + message: SharedString, +} + +impl SpeechError { + /// A recognizer failed to start or to report a result. + pub fn recognizer(error: E) -> Self + where + E: std::fmt::Display, + { + Self::new(format!("Speech recognizer error: {error}")) + } + + /// Audio capture failed to start or to deliver samples. + pub fn input(error: E) -> Self + where + E: std::fmt::Display, + { + Self::new(format!("Audio input error: {error}")) + } + + fn new(message: String) -> Self { + Self { + message: message.into(), + } + } +} + +impl std::fmt::Display for SpeechError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for SpeechError {} + +/// Turns captured audio into text. +pub trait SpeechRecognizer { + /// The PCM layout [`RecognitionSession::push_audio`] delivers. + fn audio_format(&self) -> AudioFormat { + AudioFormat::default() + } + + /// Whether the recognizer can run right now. + fn is_available(&self, _cx: &App) -> bool { + true + } + + /// Opens a session. + fn start( + &self, + sink: SpeechSink, + cx: &mut App, + ) -> Result, SpeechError>; +} + +/// A recognition session opened by [`SpeechRecognizer::start`]. +pub trait RecognitionSession { + /// Delivers interleaved 16-bit PCM in the recognizer's [`SpeechRecognizer::audio_format`]. + fn push_audio(&mut self, samples: &[i16], cx: &mut App); + + /// The user stopped talking. + fn finish(&mut self, cx: &mut App); +} + +/// Reports recognition results for the active session. +#[derive(Clone)] +pub struct SpeechSink { + state: WeakEntity, + generation: u64, +} + +impl SpeechSink { + /// The recognizer accepts audio; the status moves to `Recording`. + pub fn ready(&self, cx: &mut App) { + self.defer(cx, |state, cx| state.mark_ready(cx)); + } + + /// Replaces the hypothesis of the phrase being spoken. + pub fn hypothesis(&self, text: T, cx: &mut App) + where + T: Into + 'static, + { + self.defer(cx, move |state, cx| { + state.report_hypothesis(text.into(), cx) + }); + } + + /// Commits a recognized phrase to the transcript. + pub fn phrase(&self, text: T, cx: &mut App) + where + T: Into + 'static, + { + self.defer(cx, move |state, cx| state.report_phrase(text.into(), cx)); + } + + /// Ends the session normally with what has been recognized so far. + pub fn finish(&self, cx: &mut App) { + self.defer(cx, |state, cx| state.report_finish(cx)); + } + + /// Ends the session with a failure. + pub fn error(&self, error: SpeechError, cx: &mut App) { + self.defer(cx, move |state, cx| state.report_error(error, cx)); + } + + fn defer(&self, cx: &mut App, apply: F) + where + F: FnOnce(&mut SpeechState, &mut Context) + 'static, + { + let state = self.state.clone(); + let generation = self.generation; + cx.defer(move |cx| { + // A report arriving after the state dropped is a normal lifecycle outcome. + if let Some(state) = state.upgrade() { + state.update(cx, |state, cx| state.apply(generation, cx, apply)); + } + }); + } +} + +/// Captures audio for a session. +pub trait AudioInput { + fn start( + &self, + format: AudioFormat, + sink: AudioSink, + cx: &mut App, + ) -> Result; +} + +/// Delivers captured audio to the active session. +#[derive(Clone)] +pub struct AudioSink { + state: WeakEntity, + generation: u64, +} + +impl AudioSink { + /// Pushes interleaved 16-bit PCM in the recognizer's format. + pub fn push(&self, samples: S, cx: &mut App) + where + S: Into> + 'static, + { + let state = self.state.clone(); + let generation = self.generation; + cx.defer(move |cx| { + // Audio arriving after the state dropped is a normal lifecycle outcome. + if let Some(state) = state.upgrade() { + state.update(cx, |state, cx| { + state.receive_audio(generation, samples.into(), cx) + }); + } + }); + } +} + +/// Where a dictation session is in its life. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum SpeechStatus { + /// No session is running. + #[default] + Idle, + /// Audio is captured while the recognizer connects. + Connecting, + /// Audio is captured and recognized. + Recording, + /// Capture stopped; waiting for the final result. + Stopping, +} + +impl SpeechStatus { + /// True for every status but `Idle`. + pub fn is_active(self) -> bool { + self != Self::Idle + } + + /// True while audio is captured. + pub fn is_capturing(self) -> bool { + matches!(self, Self::Connecting | Self::Recording) + } +} + +/// What a session reports. +#[derive(Clone, Debug, PartialEq)] +pub enum SpeechEvent { + /// A session started and audio is being captured. + Started, + /// The transcript changed while the user speaks. + Partial(SharedString), + /// The session ended normally. The text may be empty. + Final(SharedString), + /// The session was cancelled and its transcript discarded. + Cancelled, + /// The session failed and ended. + Error(SpeechError), +} + +/// Owns a dictation session. +pub struct SpeechState { + recognizer: Option>, + input: Option>, + stop_timeout: Duration, + format: AudioFormat, + status: SpeechStatus, + phrases: Vec, + hypothesis: Option, + levels: Vec, + bucket: Vec, + session: Option>, + capture: Option, + generation: u64, + stop_task: Option>, +} + +impl EventEmitter for SpeechState {} + +impl SpeechState { + /// Creates an idle state without a recognizer or input. + pub fn new(_cx: &mut Context) -> Self { + Self { + recognizer: None, + input: None, + stop_timeout: DEFAULT_STOP_TIMEOUT, + format: AudioFormat::default(), + status: SpeechStatus::Idle, + phrases: Vec::new(), + hypothesis: None, + levels: Vec::new(), + bucket: Vec::new(), + session: None, + capture: None, + generation: 0, + stop_task: None, + } + } + + /// Sets the recognizer every session runs with. + pub fn recognizer(mut self, recognizer: R) -> Self + where + R: SpeechRecognizer + 'static, + { + self.recognizer = Some(Box::new(recognizer)); + self + } + + /// Sets the audio source every session captures with. + pub fn input(mut self, input: I) -> Self + where + I: AudioInput + 'static, + { + self.input = Some(Box::new(input)); + self + } + + /// How long `stop` waits for the recognizer's final result. + pub fn stop_timeout(mut self, timeout: Duration) -> Self { + self.stop_timeout = timeout; + self + } + + /// Where the session is. + pub fn status(&self) -> SpeechStatus { + self.status + } + + /// Whether a session can run on this platform. + pub fn has_recognizer(&self) -> bool { + self.recognizer.is_some() && (self.input.is_some() || Self::has_default_input()) + } + + /// Whether the recognizer reports itself ready to run. + pub fn is_available(&self, cx: &App) -> bool { + self.recognizer + .as_ref() + .is_some_and(|recognizer| recognizer.is_available(cx)) + } + + /// The whole transcript so far. + pub fn transcript(&self) -> SharedString { + let mut transcript = String::new(); + + for phrase in &self.phrases { + transcript.push_str(phrase); + } + + if let Some(hypothesis) = &self.hypothesis { + transcript.push_str(hypothesis); + } + + transcript.into() + } + + /// The recent input levels in `0.0..=1.0`, oldest first, one per 80 ms of audio. + pub fn levels(&self) -> &[f32] { + &self.levels + } + + /// Starts a session. Does nothing while one is running. + pub fn start(&mut self, cx: &mut Context) { + if self.status.is_active() { + return; + } + + self.generation = self.generation.wrapping_add(1); + let generation = self.generation; + + let Some(recognizer) = self.recognizer.as_ref() else { + self.fail("No speech recognizer is set up on this platform.", cx); + return; + }; + + if !recognizer.is_available(cx) { + self.fail("The speech recognizer is not available right now.", cx); + return; + } + + self.format = recognizer.audio_format(); + let sink = SpeechSink { + state: cx.entity().downgrade(), + generation, + }; + + let session = match recognizer.start(sink, cx) { + Ok(session) => session, + Err(error) => { + self.fail(error, cx); + return; + } + }; + + let audio_sink = AudioSink { + state: cx.entity().downgrade(), + generation, + }; + + let capture = match self.capture_input(self.format, audio_sink, cx) { + Ok(capture) => capture, + Err(error) => { + self.fail(error, cx); + return; + } + }; + + self.session = Some(session); + self.capture = Some(capture); + self.phrases.clear(); + self.hypothesis = None; + self.levels.clear(); + self.bucket.clear(); + self.status = SpeechStatus::Connecting; + cx.emit(SpeechEvent::Started); + cx.notify(); + } + + /// Ends the session and emits `Final` once the result is in, at the latest after the stop timeout. + pub fn stop(&mut self, cx: &mut Context) { + if !self.status.is_capturing() { + return; + } + self.capture = None; + self.status = SpeechStatus::Stopping; + cx.notify(); + + let generation = self.generation; + let stop_timeout = self.stop_timeout; + + if let Some(mut session) = self.session.take() { + session.finish(cx); + } + + self.stop_task = Some(cx.spawn(async move |this, cx| { + cx.background_executor().timer(stop_timeout).await; + // The state may drop while the timeout is still pending. + this.update(cx, |state, cx| state.apply_stop_timeout(generation, cx)) + .ok(); + })); + } + + /// Cancels the session at once and discards its transcript. + pub fn cancel(&mut self, cx: &mut Context) { + if !self.status.is_active() { + return; + } + self.capture = None; + self.session = None; + self.stop_task = None; + self.reset(); + cx.emit(SpeechEvent::Cancelled); + cx.notify(); + } + + /// `start` when idle, otherwise `stop`. + pub fn toggle(&mut self, cx: &mut Context) { + if self.status == SpeechStatus::Idle { + self.start(cx); + } else { + self.stop(cx); + } + } + + fn has_default_input() -> bool { + cfg!(not(target_arch = "wasm32")) + } + + #[cfg(not(target_arch = "wasm32"))] + fn default_input() -> Option> { + Some(Box::new(Microphone)) + } + + #[cfg(target_arch = "wasm32")] + fn default_input() -> Option> { + None + } + + fn capture_input( + &self, + format: AudioFormat, + sink: AudioSink, + cx: &mut App, + ) -> Result { + if let Some(input) = &self.input { + return input.start(format, sink, cx); + } + match Self::default_input() { + Some(input) => input.start(format, sink, cx), + None => Err(SpeechError::input( + "No audio input is available on this platform; provide one with `SpeechState::input`.", + )), + } + } + + fn reset(&mut self) { + self.generation = self.generation.wrapping_add(1); + self.status = SpeechStatus::Idle; + self.phrases.clear(); + self.hypothesis = None; + self.levels.clear(); + self.bucket.clear(); + } + + /// Ends a failed start without ever emitting `Started`. + fn fail(&mut self, message: M, cx: &mut Context) + where + M: std::fmt::Display, + { + self.capture = None; + self.session = None; + self.reset(); + cx.emit(SpeechEvent::Error(SpeechError::new(message.to_string()))); + cx.notify(); + } + + /// Ends the session normally, emitting `Final` with the transcript so far. + fn finalize(&mut self, cx: &mut Context) { + self.capture = None; + self.session = None; + self.stop_task = None; + let transcript = self.transcript(); + self.reset(); + cx.emit(SpeechEvent::Final(transcript)); + cx.notify(); + } + + fn apply(&mut self, generation: u64, cx: &mut Context, apply: F) + where + F: FnOnce(&mut Self, &mut Context), + { + if self.generation != generation { + return; + } + apply(self, cx); + } + + fn mark_ready(&mut self, cx: &mut Context) { + if self.status == SpeechStatus::Connecting { + self.status = SpeechStatus::Recording; + cx.notify(); + } + } + + fn report_hypothesis(&mut self, text: SharedString, cx: &mut Context) { + if !self.status.is_active() { + return; + } + self.hypothesis = Some(text); + cx.emit(SpeechEvent::Partial(self.transcript())); + cx.notify(); + } + + fn report_phrase(&mut self, text: SharedString, cx: &mut Context) { + if !self.status.is_active() { + return; + } + self.phrases.push(text); + self.hypothesis = None; + cx.emit(SpeechEvent::Partial(self.transcript())); + cx.notify(); + } + + fn report_finish(&mut self, cx: &mut Context) { + if self.status.is_active() { + self.finalize(cx); + } + } + + fn report_error(&mut self, error: SpeechError, cx: &mut Context) { + if !self.status.is_active() { + return; + } + self.capture = None; + self.session = None; + self.stop_task = None; + self.reset(); + cx.emit(SpeechEvent::Error(error)); + cx.notify(); + } + + fn apply_stop_timeout(&mut self, generation: u64, cx: &mut Context) { + if self.generation == generation && self.status == SpeechStatus::Stopping { + self.finalize(cx); + } + } + + fn receive_audio(&mut self, generation: u64, samples: Vec, cx: &mut Context) { + if self.generation != generation || !self.status.is_capturing() { + return; + } + if let Some(session) = self.session.as_mut() { + session.push_audio(&samples, cx); + } + if self.update_levels(&samples) { + cx.notify(); + } + } + + /// Fills one level per bucket; returns whether any level was recorded. + fn update_levels(&mut self, samples: &[i16]) -> bool { + let channels = self.format.channels() as usize; + let bucket_len = ((self.format.sample_rate() as f32 * LEVEL_BUCKET.as_secs_f32()) as usize + * channels) + .max(channels); + self.bucket.extend_from_slice(samples); + + let mut recorded = false; + while self.bucket.len() >= bucket_len { + let peak = self + .bucket + .drain(..bucket_len) + .map(|sample| sample.unsigned_abs() as f32 / i16::MAX as f32) + .fold(0.0f32, f32::max); + let previous = self.levels.last().copied().unwrap_or(0.0); + let level = ((peak - NOISE_GATE).max(0.0) / LEVEL_SCALE) + .min(1.0) + .max(previous * LEVEL_FALL); + self.levels.push(level); + if self.levels.len() > MAX_LEVELS { + self.levels.remove(0); + } + recorded = true; + } + recorded + } +} + +/// Starts and stops a dictation session. +#[derive(IntoElement)] +#[must_use] +pub struct SpeechButton { + speech: Entity, + size: Size, + disabled: bool, + show_when_unsupported: bool, +} + +impl SpeechButton { + /// Creates the button driving the given state. + pub fn new(speech: &Entity) -> Self { + Self { + speech: speech.clone(), + size: Size::default(), + disabled: false, + show_when_unsupported: false, + } + } + + /// Renders the button disabled. + pub fn show_when_unsupported(mut self, show: bool) -> Self { + self.show_when_unsupported = show; + self + } + + /// Disables the button for application reasons, such as a readonly field. + pub fn disabled(mut self, disabled: bool) -> Self { + self.disabled = disabled; + self + } +} + +impl Sizable for SpeechButton { + fn apply_size(&mut self, size: Size) { + self.size = size; + } +} + +impl RenderOnce for SpeechButton { + fn render(self, _window: &mut Window, cx: &mut App) -> impl IntoElement { + let unsupported = !self.speech.read(cx).has_recognizer(); + if unsupported && !self.show_when_unsupported { + return div().into_any_element(); + } + + let (status, unavailable, capturing, reduce_motion, bars) = { + let state = self.speech.read(cx); + let status = state.status(); + let levels = state.levels(); + let tail_start = levels.len().saturating_sub(BUTTON_BARS); + ( + status, + unsupported || !state.is_available(cx), + status.is_capturing(), + cx.theme().accessibility.reduce_motion, + levels[tail_start..].to_vec(), + ) + }; + + let stopping = status == SpeechStatus::Stopping; + let disabled = self.disabled || unavailable || stopping; + + let theme = cx.theme(); + let colors = theme.colors; + let metrics = theme.sizes.resolve(self.size); + let id: ElementId = ("speech-button", self.speech.entity_id()).into(); + + let group = SharedString::from(format!( + "speech-button-{}", + self.speech.entity_id().as_u64() + )); + + let label: SharedString = if unavailable { + "Dictation is unavailable".into() + } else if capturing { + "Stop dictation".into() + } else { + "Start dictation".into() + }; + + let foreground = colors.foreground; + + h_flex() + .id(id.clone()) + .group(group.clone()) + .role(Role::Button) + .justify_center() + .h(metrics.control_height + BORDER_WIDTH * 2.0) + .px(metrics.dense_padding_x) + .gap_0p5() + .rounded(theme.radius.control) + .text_color(foreground) + .when(capturing, |button| button.bg(colors.muted)) + .when(disabled, |button| button.opacity(0.5)) + .aria_label(label.clone()) + .tooltip(Tooltip::text(label)) + .child(if stopping { + Spinner::new(("speech-spinner", self.speech.entity_id())) + .with_size(self.size) + .variant(SpinnerVariant::Icon) + .color(foreground) + .into_any_element() + } else if capturing { + let bar_height = |level: f32| { + if reduce_motion { + STILL_HEIGHT + } else { + DOT_HEIGHT + (1.0 - DOT_HEIGHT) * level + } + }; + h_flex() + .id(("speech-bars", self.speech.entity_id())) + .h(metrics.icon_size) + .items_end() + .gap_0p5() + .children(bars.iter().map(|level| { + div() + .flex_none() + .w(metrics.dense_icon_size / 4.) + .h(relative(bar_height(*level))) + .rounded(theme.radius.full) + .bg(theme.ramps.primary.base) + })) + .group_hover(group.clone(), |style| style.invisible()) + .into_any_element() + } else { + icon(IconRole::Microphone) + .size(metrics.icon_size) + .color(foreground) + .into_any_element() + }) + .when(capturing, |button| { + button.child( + div() + .invisible() + .group_hover(group, |style| style.visible()) + .child( + icon(IconRole::Stop) + .size(metrics.icon_size) + .color(foreground), + ), + ) + }) + .when(!disabled, |button| { + let speech = self.speech.clone(); + button + .cursor_pointer() + .focusable() + .tab_stop(true) + .hover(move |style| style.bg(colors.muted_hover)) + .active(move |style| style.bg(theme.ramps.secondary.pressed)) + .focus_visible(move |style| style.shadow(focus_ring(colors.ring))) + .on_click(move |_, _, cx| speech.update(cx, |state, cx| state.toggle(cx))) + }) + .into_any_element() + } +} + +/// A live waveform of what a dictation session hears. +#[derive(IntoElement)] +#[must_use] +pub struct SpeechWaveform { + speech: Entity, + size: Size, + style: gpui::StyleRefinement, +} + +impl SpeechWaveform { + /// Creates the waveform drawn from the given state. + pub fn new(speech: &Entity) -> Self { + Self { + speech: speech.clone(), + size: Size::default(), + style: gpui::StyleRefinement::default(), + } + } +} + +impl Sizable for SpeechWaveform { + fn apply_size(&mut self, size: Size) { + self.size = size; + } +} + +impl Styled for SpeechWaveform { + fn style(&mut self) -> &mut gpui::StyleRefinement { + &mut self.style + } +} + +impl RenderOnce for SpeechWaveform { + fn render(self, _window: &mut Window, cx: &mut App) -> impl IntoElement { + if !self.speech.read(cx).status().is_capturing() { + return div().into_any_element(); + } + let (levels, reduce_motion) = { + let state = self.speech.read(cx); + ( + state.levels().to_vec(), + cx.theme().accessibility.reduce_motion, + ) + }; + + let theme = cx.theme(); + let colors = theme.colors; + let metrics = theme.sizes.resolve(self.size); + let bar_height = |level: f32| { + if reduce_motion { + STILL_HEIGHT + } else { + DOT_HEIGHT + (1.0 - DOT_HEIGHT) * level + } + }; + + h_flex() + .id(("speech-waveform", self.speech.entity_id())) + .items_end() + .justify_end() + .gap_0p5() + .h(metrics.icon_size) + .w_24() + .overflow_hidden() + .children(levels.iter().map(|level| { + div() + .flex_none() + .w(metrics.dense_icon_size / 4.) + .h(relative(bar_height(*level))) + .rounded(theme.radius.full) + .bg(if *level <= 0.0 { + colors.muted_foreground + } else { + theme.ramps.primary.base + }) + })) + .refine_style(&self.style) + .into_any_element() + } +} + +#[cfg(not(target_arch = "wasm32"))] +/// The built-in input. +pub struct Microphone; + +#[cfg(not(target_arch = "wasm32"))] +impl AudioInput for Microphone { + fn start( + &self, + format: AudioFormat, + sink: AudioSink, + cx: &mut App, + ) -> Result { + use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; + + let host = cpal::default_host(); + let device = host + .default_input_device() + .ok_or_else(|| SpeechError::input("No microphone is connected."))?; + + let config = device.default_input_config().map_err(SpeechError::input)?; + let stream_config = cpal::StreamConfig { + channels: config.channels(), + sample_rate: config.sample_rate(), + buffer_size: cpal::BufferSize::Default, + }; + let device_channels = stream_config.channels.max(1) as usize; + + let (sender, receiver) = flume::unbounded::>(); + let converter = Arc::new(Mutex::new(Resampler::new( + config.sample_rate() as f32 / format.sample_rate().max(1) as f32, + ))); + let error_handler = |error| log::warn!("microphone stream error: {error}"); + + let stream = match config.sample_format() { + cpal::SampleFormat::F32 => device.build_input_stream( + stream_config, + Microphone::callback( + |sample: f32| sample, + device_channels, + format, + converter.clone(), + sender.clone(), + ), + error_handler, + None, + ), + cpal::SampleFormat::I16 => device.build_input_stream( + stream_config, + Microphone::callback( + |sample: i16| sample as f32 / i16::MAX as f32, + device_channels, + format, + converter.clone(), + sender.clone(), + ), + error_handler, + None, + ), + cpal::SampleFormat::U16 => device.build_input_stream( + stream_config, + Microphone::callback( + |sample: u16| (sample as f32 - u16::MAX as f32 / 2.0) / (u16::MAX as f32 / 2.0), + device_channels, + format, + converter.clone(), + sender.clone(), + ), + error_handler, + None, + ), + sample_format => { + return Err(SpeechError::input(format!( + "Unsupported microphone sample format: {sample_format}" + ))); + } + } + .map_err(SpeechError::input)?; + + stream.play().map_err(SpeechError::input)?; + + let task = cx.spawn(async move |cx| { + while let Ok(samples) = receiver.recv_async().await { + cx.update(|cx| sink.push(samples, cx)); + } + }); + + Ok(Subscription::new(move || { + drop(stream); + drop(task); + })) + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl Microphone { + fn callback( + to_sample: fn(T) -> f32, + device_channels: usize, + format: AudioFormat, + converter: Arc>, + sender: flume::Sender>, + ) -> impl FnMut(&[T], &cpal::InputCallbackInfo) + Send + 'static + where + T: Copy + 'static, + { + move |data: &[T], _: &cpal::InputCallbackInfo| { + let Ok(mut converter) = converter.lock() else { + return; + }; + + let mut mono = Vec::with_capacity(data.len() / device_channels + 1); + for frame in data.chunks(device_channels) { + let total: f32 = frame.iter().map(|sample| to_sample(*sample)).sum(); + mono.push(total / frame.len() as f32); + } + + let mut resampled = Vec::with_capacity(mono.len()); + converter.resample(&mono, &mut resampled); + + let channels = format.channels() as usize; + let mut samples = Vec::with_capacity(resampled.len() * channels); + + for value in resampled { + let sample = (value * i16::MAX as f32) + .round() + .clamp(i16::MIN as f32, i16::MAX as f32) as i16; + for _ in 0..channels { + samples.push(sample); + } + } + + if sender.send(samples).is_err() { + log::debug!("speech session closed before its audio arrived"); + } + } + } +} + +#[cfg(not(target_arch = "wasm32"))] +/// A minimal linear resampler that carries its phase across callbacks. +struct Resampler { + step: f32, + phase: f32, + last: Option, +} + +#[cfg(not(target_arch = "wasm32"))] +impl Resampler { + fn new(step: f32) -> Self { + Self { + step, + phase: 0.0, + last: None, + } + } + + fn resample(&mut self, input: &[f32], output: &mut Vec) { + if input.is_empty() { + return; + } + let mut phase = self.phase; + while phase < input.len() as f32 { + let index = (phase as usize).min(input.len() - 1); + let fraction = phase - index as f32; + let previous = if index == 0 { + self.last.unwrap_or(input[0]) + } else { + input[index - 1] + }; + output.push(previous + (input[index] - previous) * fraction); + phase += self.step; + } + self.phase = phase - input.len() as f32; + self.last = Some(input[input.len() - 1]); + } +} diff --git a/crates/core/tests/speech.rs b/crates/core/tests/speech.rs new file mode 100644 index 0000000..008ecda --- /dev/null +++ b/crates/core/tests/speech.rs @@ -0,0 +1,282 @@ +#![cfg(feature = "speech")] + +use std::cell::RefCell; +use std::rc::Rc; +use std::time::Duration; + +use gpui::{App, AppContext as _, Entity, SharedString, Subscription, TestAppContext}; +use gpui_native_core::speech::{ + AudioFormat, AudioInput, AudioSink, RecognitionSession, SpeechError, SpeechEvent, + SpeechRecognizer, SpeechSink, SpeechState, SpeechStatus, +}; + +type EventLog = Rc>>; +type Harness = ( + Entity, + EventLog, + Rc>>, + Rc>, +); + +struct FakeRecognizer { + available: bool, + finish_reports: bool, + sink: Rc>>, + received: Rc>, +} + +impl SpeechRecognizer for FakeRecognizer { + fn is_available(&self, _cx: &App) -> bool { + self.available + } + + fn start( + &self, + sink: SpeechSink, + cx: &mut App, + ) -> Result, SpeechError> { + *self.sink.borrow_mut() = Some(sink.clone()); + sink.ready(cx); + Ok(Box::new(FakeSession { + sink, + finish_reports: self.finish_reports, + received: self.received.clone(), + })) + } +} + +struct FakeSession { + sink: SpeechSink, + finish_reports: bool, + received: Rc>, +} + +impl RecognitionSession for FakeSession { + fn push_audio(&mut self, samples: &[i16], _cx: &mut App) { + *self.received.borrow_mut() += samples.len(); + } + + fn finish(&mut self, cx: &mut App) { + if self.finish_reports { + self.sink.finish(cx); + } + } +} + +/// Delivers one silent second, 16 kHz mono, in 100 ms chunks. +struct FakeInput; + +impl AudioInput for FakeInput { + fn start( + &self, + format: AudioFormat, + sink: AudioSink, + cx: &mut App, + ) -> Result { + let chunk_len = (format.sample_rate() / 10) as usize * format.channels() as usize; + let task = cx.spawn(async move |cx| { + for _ in 0..10 { + cx.background_executor() + .timer(Duration::from_millis(100)) + .await; + cx.update(|cx| sink.push(vec![0; chunk_len], cx)); + } + }); + Ok(Subscription::new(move || drop(task))) + } +} + +fn harness(cx: &mut TestAppContext, available: bool, finish_reports: bool) -> Harness { + let sink_slot: Rc>> = Rc::new(RefCell::new(None)); + let received = Rc::new(RefCell::new(0)); + let log: EventLog = Rc::new(RefCell::new(Vec::new())); + let record = log.clone(); + let speech = cx.update(|cx| { + let speech = cx.new(|cx| { + SpeechState::new(cx) + .recognizer(FakeRecognizer { + available, + finish_reports, + sink: sink_slot.clone(), + received: received.clone(), + }) + .input(FakeInput) + .stop_timeout(Duration::from_secs(2)) + }); + cx.subscribe(&speech, move |_, event, _| { + record.borrow_mut().push(event.clone()) + }) + .detach(); + speech + }); + (speech, log, sink_slot, received) +} + +fn start(speech: &Entity, cx: &mut TestAppContext) { + cx.update(|cx| speech.update(cx, |state, cx| state.start(cx))); + cx.run_until_parked(); +} + +#[test] +fn a_session_starts_reads_and_finalizes() { + let mut cx = TestAppContext::single(); + let (speech, events, sink_slot, _) = harness(&mut cx, true, true); + + start(&speech, &mut cx); + cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Recording)); + assert_eq!(*events.borrow(), vec![SpeechEvent::Started]); + + let sink = sink_slot.borrow().clone().expect("session opened"); + cx.update(|cx| sink.hypothesis("hel", cx)); + cx.run_until_parked(); + cx.update(|cx| { + sink.phrase("hello ", cx); + sink.hypothesis("wor", cx); + }); + cx.run_until_parked(); + cx.update(|cx| { + assert_eq!( + speech.read(cx).transcript(), + SharedString::from("hello wor") + ); + }); + + cx.update(|cx| speech.update(cx, |state, cx| state.stop(cx))); + cx.run_until_parked(); + cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Idle)); + + let events = events.borrow(); + assert!( + matches!(events.last(), Some(SpeechEvent::Final(text)) if text.as_ref() == "hello wor") + ); + assert!(matches!( + events.get(events.len() - 2), + Some(SpeechEvent::Partial(text)) if text.as_ref() == "hello wor" + )); +} + +#[test] +fn starting_twice_runs_a_single_session() { + let mut cx = TestAppContext::single(); + let (speech, events, _, _) = harness(&mut cx, true, true); + + start(&speech, &mut cx); + start(&speech, &mut cx); + + assert_eq!(*events.borrow(), vec![SpeechEvent::Started]); +} + +#[test] +fn cancel_discards_the_transcript_and_ignores_late_reports() { + let mut cx = TestAppContext::single(); + let (speech, events, sink_slot, _) = harness(&mut cx, true, true); + + start(&speech, &mut cx); + let sink = sink_slot.borrow().clone().expect("session opened"); + cx.update(|cx| sink.phrase("hello", cx)); + cx.run_until_parked(); + + cx.update(|cx| speech.update(cx, |state, cx| state.cancel(cx))); + cx.run_until_parked(); + cx.update(|cx| { + let state = speech.read(cx); + assert_eq!(state.status(), SpeechStatus::Idle); + assert_eq!(state.transcript(), SharedString::from("")); + }); + assert!(matches!( + events.borrow().last(), + Some(SpeechEvent::Cancelled) + )); + + cx.update(|cx| sink.finish(cx)); + cx.run_until_parked(); + assert!( + !events + .borrow() + .iter() + .any(|event| matches!(event, SpeechEvent::Final(_))) + ); +} + +#[test] +fn starting_without_a_recognizer_fails() { + let mut cx = TestAppContext::single(); + let log: EventLog = Rc::new(RefCell::new(Vec::new())); + let record = log.clone(); + let speech = cx.update(|cx| { + let speech = cx.new(|cx| SpeechState::new(cx).input(FakeInput)); + cx.subscribe(&speech, move |_, event, _| { + record.borrow_mut().push(event.clone()) + }) + .detach(); + speech + }); + + cx.update(|cx| assert!(!speech.read(cx).has_recognizer())); + start(&speech, &mut cx); + + cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Idle)); + assert!(matches!(events_last(&log), Some(SpeechEvent::Error(_)))); +} + +#[test] +fn starting_an_unavailable_recognizer_fails() { + let mut cx = TestAppContext::single(); + let (speech, events, _, _) = harness(&mut cx, false, true); + + start(&speech, &mut cx); + cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Idle)); + assert!(matches!( + events.borrow().last(), + Some(SpeechEvent::Error(_)) + )); +} + +#[test] +fn the_stop_timeout_finalizes_with_the_transcript_so_far() { + let mut cx = TestAppContext::single(); + let (speech, events, sink_slot, _) = harness(&mut cx, true, false); + + start(&speech, &mut cx); + let sink = sink_slot.borrow().clone().expect("session opened"); + cx.update(|cx| sink.phrase("hello", cx)); + cx.run_until_parked(); + + cx.update(|cx| speech.update(cx, |state, cx| state.stop(cx))); + cx.run_until_parked(); + cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Stopping)); + + cx.executor().advance_clock(Duration::from_secs(2)); + cx.run_until_parked(); + cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Idle)); + assert!( + matches!(events.borrow().last(), Some(SpeechEvent::Final(text)) if text.as_ref() == "hello") + ); +} + +#[test] +fn captured_audio_reaches_the_session_and_the_levels() { + let mut cx = TestAppContext::single(); + let (speech, _, _, received) = harness(&mut cx, true, true); + + start(&speech, &mut cx); + cx.executor().advance_clock(Duration::from_secs(1)); + cx.run_until_parked(); + + cx.update(|cx| { + let state = speech.read(cx); + assert_eq!(*received.borrow(), 16_000); + // One level per 80 ms bucket of the one-second capture. + let levels = state.levels(); + assert_eq!(levels.len(), 12); + assert!(levels.iter().all(|level| *level == 0.0)); + }); + + cx.update(|cx| speech.update(cx, |state, cx| state.stop(cx))); + cx.run_until_parked(); + cx.update(|cx| assert!(speech.read(cx).levels().is_empty())); +} + +fn events_last(log: &EventLog) -> Option { + log.borrow().last().cloned() +} diff --git a/crates/icons/assets/mic.svg b/crates/icons/assets/mic.svg new file mode 100644 index 0000000..351c1b6 --- /dev/null +++ b/crates/icons/assets/mic.svg @@ -0,0 +1,15 @@ + + + + + diff --git a/crates/icons/src/lib.rs b/crates/icons/src/lib.rs index 09688c5..9a2639c 100644 --- a/crates/icons/src/lib.rs +++ b/crates/icons/src/lib.rs @@ -37,6 +37,8 @@ pub enum IconRole { WindowMaximize, WindowRestore, Loader, + Microphone, + Stop, } impl IconRole { @@ -63,6 +65,8 @@ impl IconRole { Self::WindowMaximize, Self::WindowRestore, Self::Loader, + Self::Microphone, + Self::Stop, ] } @@ -89,6 +93,8 @@ impl IconRole { Self::WindowMaximize => "icons/square.svg", Self::WindowRestore => "icons/copy.svg", Self::Loader => "icons/loader.svg", + Self::Microphone => "icons/mic.svg", + Self::Stop => "icons/square.svg", } } @@ -116,6 +122,8 @@ impl IconRole { Self::WindowMaximize => "window-maximize", Self::WindowRestore => "window-restore", Self::Loader => "loader", + Self::Microphone => "microphone", + Self::Stop => "stop", } } @@ -192,6 +200,8 @@ impl IconRole { Self::WindowMaximize => &["window-maximize-symbolic", "window-maximize"], Self::WindowRestore => &["window-restore-symbolic", "window-restore"], Self::Loader => &["process-working-symbolic", "process-working"], + Self::Microphone => &["audio-input-microphone-symbolic", "audio-input-microphone"], + Self::Stop => &["process-stop-symbolic", "process-stop"], } } } @@ -387,6 +397,7 @@ fn asset_bytes(path: &str) -> Option<&'static [u8]> { "icons/panel-left.svg" => Some(include_bytes!("../assets/panel-left.svg")), "icons/panel-right.svg" => Some(include_bytes!("../assets/panel-right.svg")), "icons/loader.svg" => Some(include_bytes!("../assets/loader.svg")), + "icons/mic.svg" => Some(include_bytes!("../assets/mic.svg")), _ => None, } } diff --git a/crates/icons/src/segoe.rs b/crates/icons/src/segoe.rs index a8901b4..a3ad5fd 100644 --- a/crates/icons/src/segoe.rs +++ b/crates/icons/src/segoe.rs @@ -26,6 +26,8 @@ pub(crate) fn glyph(role: IconRole) -> Option<&'static str> { IconRole::WindowMinimize => "\u{E921}", // ChromeMinimize IconRole::WindowMaximize => "\u{E922}", // ChromeMaximize IconRole::WindowRestore => "\u{E923}", // ChromeRestore + IconRole::Microphone => "\u{E720}", // Microphone + IconRole::Stop => "\u{E71A}", // Stop // No Segoe spinner glyph, the icon falls back to the bundled SVG. IconRole::Loader => return None, };