feat: add speech component

2026-11-10 20:14:26 +07:00
parent 67f34ead74
commit 2aa2d7034f
8 changed files with 1553 additions and 0 deletions
@@ -119,6 +119,28 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"

+[[package]]
+name = "alsa"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d"
+dependencies = [
+ "alsa-sys",
+ "bitflags 2.13.2",
+ "cfg-if",
+ "libc",
+]
+
+[[package]]
+name = "alsa-sys"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04"
+dependencies = [
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.6"
@@ -957,6 +979,20 @@ dependencies = [
 "libm",
 ]

+[[package]]
+name = "coreaudio-rs"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58"
+dependencies = [
+ "bitflags 2.13.2",
+ "libc",
+ "objc2-audio-toolbox",
+ "objc2-core-audio",
+ "objc2-core-audio-types",
+ "objc2-core-foundation",
+]
+
 [[package]]
 name = "cosmic-text"
 version = "0.19.0"
@@ -981,6 +1017,36 @@ dependencies = [
 "unicode-segmentation",
 ]

+[[package]]
+name = "cpal"
+version = "0.18.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f02e8d0327b42d3e2e4ab2119af397344eb9fc54a34bf0ddeaa1277af8681f1"
+dependencies = [
+ "alsa",
+ "block2 0.6.2",
+ "coreaudio-rs",
+ "dasp_sample",
+ "jni",
+ "js-sys",
+ "libc",
+ "mach2",
+ "ndk",
+ "ndk-context",
+ "num-derive",
+ "num-traits",
+ "objc2 0.6.5",
+ "objc2-audio-toolbox",
+ "objc2-avf-audio",
+ "objc2-core-audio",
+ "objc2-core-audio-types",
+ "objc2-core-foundation",
+ "objc2-foundation 0.3.2",
+ "web-sys",
+ "windows 0.62.2",
+ "windows-core 0.62.2",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.17"
@@ -1103,6 +1169,12 @@ dependencies = [
 "linktime-proc-macro",
 ]

+[[package]]
+name = "dasp_sample"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
+
 [[package]]
 name = "data-url"
 version = "0.3.2"
@@ -1397,6 +1469,9 @@ name = "fastrand"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
+dependencies = [
+ "getrandom 0.4.3",
+]

 [[package]]
 name = "fax"
@@ -1460,6 +1535,7 @@ version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be"
 dependencies = [
+ "fastrand",
 "futures-core",
 "futures-sink",
 "spin 0.9.9",
@@ -1888,6 +1964,8 @@ dependencies = [
 name = "gpui-native-core"
 version = "0.1.0"
 dependencies = [
+ "cpal",
+ "flume",
 "gpui",
 "gpui-native-icons",
 "gpui-native-theme",
@@ -3294,6 +3372,26 @@ dependencies = [
 "thiserror 2.0.21",
 ]

+[[package]]
+name = "ndk"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4"
+dependencies = [
+ "bitflags 2.13.2",
+ "jni-sys 0.3.1",
+ "log",
+ "ndk-sys",
+ "num_enum",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "ndk-context"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
+
 [[package]]
 name = "ndk-sys"
 version = "0.6.0+11769913"
@@ -3501,6 +3599,28 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "num_enum"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
+dependencies = [
+ "num_enum_derive",
+ "rustversion",
+]
+
+[[package]]
+name = "num_enum_derive"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "objc-sys"
 version = "0.3.5"
@@ -3556,6 +3676,55 @@ dependencies = [
 "objc2-uniform-type-identifiers",
 ]

+[[package]]
+name = "objc2-audio-toolbox"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08"
+dependencies = [
+ "bitflags 2.13.2",
+ "libc",
+ "objc2 0.6.5",
+ "objc2-core-audio",
+ "objc2-core-audio-types",
+ "objc2-core-foundation",
+ "objc2-foundation 0.3.2",
+]
+
+[[package]]
+name = "objc2-avf-audio"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"
+dependencies = [
+ "bitflags 2.13.2",
+ "objc2 0.6.5",
+ "objc2-foundation 0.3.2",
+]
+
+[[package]]
+name = "objc2-core-audio"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2"
+dependencies = [
+ "dispatch2",
+ "objc2 0.6.5",
+ "objc2-core-audio-types",
+ "objc2-core-foundation",
+ "objc2-foundation 0.3.2",
+]
+
+[[package]]
+name = "objc2-core-audio-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c"
+dependencies = [
+ "bitflags 2.13.2",
+ "objc2 0.6.5",
+]
+
 [[package]]
 name = "objc2-core-data"
 version = "0.2.2"
@@ -11,6 +11,10 @@ gpui-native-theme = { workspace = true }
 gpui-native-utils = { workspace = true }
 log = "0.4"
 unicode-segmentation = "1.12"
+flume = { version = "0.12", optional = true }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+cpal = { version = "0.18", optional = true }

 [dev-dependencies]
 gpui = { workspace = true, features = ["test-support"] }
@@ -49,6 +53,7 @@ full = [
    "empty_state",
    "alert",
    "toast",
+    "speech",
 ]
 button = ["spinner"]
 checkbox = []
@@ -81,3 +86,4 @@ dialog = ["button"]
 empty_state = []
 alert = []
 toast = []
+speech = ["spinner", "tooltip", "dep:flume", "dep:cpal"]
@@ -81,6 +81,9 @@ pub mod separator;
 #[cfg(feature = "spinner")]
 pub mod spinner;

+#[cfg(feature = "speech")]
+pub mod speech;
+
 #[cfg(feature = "field")]
 pub mod field;

@@ -0,0 +1,282 @@
+#![cfg(feature = "speech")]
+
+use std::cell::RefCell;
+use std::rc::Rc;
+use std::time::Duration;
+
+use gpui::{App, AppContext as _, Entity, SharedString, Subscription, TestAppContext};
+use gpui_native_core::speech::{
+    AudioFormat, AudioInput, AudioSink, RecognitionSession, SpeechError, SpeechEvent,
+    SpeechRecognizer, SpeechSink, SpeechState, SpeechStatus,
+};
+
+type EventLog = Rc<RefCell<Vec<SpeechEvent>>>;
+type Harness = (
+    Entity<SpeechState>,
+    EventLog,
+    Rc<RefCell<Option<SpeechSink>>>,
+    Rc<RefCell<usize>>,
+);
+
+struct FakeRecognizer {
+    available: bool,
+    finish_reports: bool,
+    sink: Rc<RefCell<Option<SpeechSink>>>,
+    received: Rc<RefCell<usize>>,
+}
+
+impl SpeechRecognizer for FakeRecognizer {
+    fn is_available(&self, _cx: &App) -> bool {
+        self.available
+    }
+
+    fn start(
+        &self,
+        sink: SpeechSink,
+        cx: &mut App,
+    ) -> Result<Box<dyn RecognitionSession>, SpeechError> {
+        *self.sink.borrow_mut() = Some(sink.clone());
+        sink.ready(cx);
+        Ok(Box::new(FakeSession {
+            sink,
+            finish_reports: self.finish_reports,
+            received: self.received.clone(),
+        }))
+    }
+}
+
+struct FakeSession {
+    sink: SpeechSink,
+    finish_reports: bool,
+    received: Rc<RefCell<usize>>,
+}
+
+impl RecognitionSession for FakeSession {
+    fn push_audio(&mut self, samples: &[i16], _cx: &mut App) {
+        *self.received.borrow_mut() += samples.len();
+    }
+
+    fn finish(&mut self, cx: &mut App) {
+        if self.finish_reports {
+            self.sink.finish(cx);
+        }
+    }
+}
+
+/// Delivers one silent second, 16 kHz mono, in 100 ms chunks.
+struct FakeInput;
+
+impl AudioInput for FakeInput {
+    fn start(
+        &self,
+        format: AudioFormat,
+        sink: AudioSink,
+        cx: &mut App,
+    ) -> Result<Subscription, SpeechError> {
+        let chunk_len = (format.sample_rate() / 10) as usize * format.channels() as usize;
+        let task = cx.spawn(async move |cx| {
+            for _ in 0..10 {
+                cx.background_executor()
+                    .timer(Duration::from_millis(100))
+                    .await;
+                cx.update(|cx| sink.push(vec![0; chunk_len], cx));
+            }
+        });
+        Ok(Subscription::new(move || drop(task)))
+    }
+}
+
+fn harness(cx: &mut TestAppContext, available: bool, finish_reports: bool) -> Harness {
+    let sink_slot: Rc<RefCell<Option<SpeechSink>>> = Rc::new(RefCell::new(None));
+    let received = Rc::new(RefCell::new(0));
+    let log: EventLog = Rc::new(RefCell::new(Vec::new()));
+    let record = log.clone();
+    let speech = cx.update(|cx| {
+        let speech = cx.new(|cx| {
+            SpeechState::new(cx)
+                .recognizer(FakeRecognizer {
+                    available,
+                    finish_reports,
+                    sink: sink_slot.clone(),
+                    received: received.clone(),
+                })
+                .input(FakeInput)
+                .stop_timeout(Duration::from_secs(2))
+        });
+        cx.subscribe(&speech, move |_, event, _| {
+            record.borrow_mut().push(event.clone())
+        })
+        .detach();
+        speech
+    });
+    (speech, log, sink_slot, received)
+}
+
+fn start(speech: &Entity<SpeechState>, cx: &mut TestAppContext) {
+    cx.update(|cx| speech.update(cx, |state, cx| state.start(cx)));
+    cx.run_until_parked();
+}
+
+#[test]
+fn a_session_starts_reads_and_finalizes() {
+    let mut cx = TestAppContext::single();
+    let (speech, events, sink_slot, _) = harness(&mut cx, true, true);
+
+    start(&speech, &mut cx);
+    cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Recording));
+    assert_eq!(*events.borrow(), vec![SpeechEvent::Started]);
+
+    let sink = sink_slot.borrow().clone().expect("session opened");
+    cx.update(|cx| sink.hypothesis("hel", cx));
+    cx.run_until_parked();
+    cx.update(|cx| {
+        sink.phrase("hello ", cx);
+        sink.hypothesis("wor", cx);
+    });
+    cx.run_until_parked();
+    cx.update(|cx| {
+        assert_eq!(
+            speech.read(cx).transcript(),
+            SharedString::from("hello wor")
+        );
+    });
+
+    cx.update(|cx| speech.update(cx, |state, cx| state.stop(cx)));
+    cx.run_until_parked();
+    cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Idle));
+
+    let events = events.borrow();
+    assert!(
+        matches!(events.last(), Some(SpeechEvent::Final(text)) if text.as_ref() == "hello wor")
+    );
+    assert!(matches!(
+        events.get(events.len() - 2),
+        Some(SpeechEvent::Partial(text)) if text.as_ref() == "hello wor"
+    ));
+}
+
+#[test]
+fn starting_twice_runs_a_single_session() {
+    let mut cx = TestAppContext::single();
+    let (speech, events, _, _) = harness(&mut cx, true, true);
+
+    start(&speech, &mut cx);
+    start(&speech, &mut cx);
+
+    assert_eq!(*events.borrow(), vec![SpeechEvent::Started]);
+}
+
+#[test]
+fn cancel_discards_the_transcript_and_ignores_late_reports() {
+    let mut cx = TestAppContext::single();
+    let (speech, events, sink_slot, _) = harness(&mut cx, true, true);
+
+    start(&speech, &mut cx);
+    let sink = sink_slot.borrow().clone().expect("session opened");
+    cx.update(|cx| sink.phrase("hello", cx));
+    cx.run_until_parked();
+
+    cx.update(|cx| speech.update(cx, |state, cx| state.cancel(cx)));
+    cx.run_until_parked();
+    cx.update(|cx| {
+        let state = speech.read(cx);
+        assert_eq!(state.status(), SpeechStatus::Idle);
+        assert_eq!(state.transcript(), SharedString::from(""));
+    });
+    assert!(matches!(
+        events.borrow().last(),
+        Some(SpeechEvent::Cancelled)
+    ));
+
+    cx.update(|cx| sink.finish(cx));
+    cx.run_until_parked();
+    assert!(
+        !events
+            .borrow()
+            .iter()
+            .any(|event| matches!(event, SpeechEvent::Final(_)))
+    );
+}
+
+#[test]
+fn starting_without_a_recognizer_fails() {
+    let mut cx = TestAppContext::single();
+    let log: EventLog = Rc::new(RefCell::new(Vec::new()));
+    let record = log.clone();
+    let speech = cx.update(|cx| {
+        let speech = cx.new(|cx| SpeechState::new(cx).input(FakeInput));
+        cx.subscribe(&speech, move |_, event, _| {
+            record.borrow_mut().push(event.clone())
+        })
+        .detach();
+        speech
+    });
+
+    cx.update(|cx| assert!(!speech.read(cx).has_recognizer()));
+    start(&speech, &mut cx);
+
+    cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Idle));
+    assert!(matches!(events_last(&log), Some(SpeechEvent::Error(_))));
+}
+
+#[test]
+fn starting_an_unavailable_recognizer_fails() {
+    let mut cx = TestAppContext::single();
+    let (speech, events, _, _) = harness(&mut cx, false, true);
+
+    start(&speech, &mut cx);
+    cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Idle));
+    assert!(matches!(
+        events.borrow().last(),
+        Some(SpeechEvent::Error(_))
+    ));
+}
+
+#[test]
+fn the_stop_timeout_finalizes_with_the_transcript_so_far() {
+    let mut cx = TestAppContext::single();
+    let (speech, events, sink_slot, _) = harness(&mut cx, true, false);
+
+    start(&speech, &mut cx);
+    let sink = sink_slot.borrow().clone().expect("session opened");
+    cx.update(|cx| sink.phrase("hello", cx));
+    cx.run_until_parked();
+
+    cx.update(|cx| speech.update(cx, |state, cx| state.stop(cx)));
+    cx.run_until_parked();
+    cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Stopping));
+
+    cx.executor().advance_clock(Duration::from_secs(2));
+    cx.run_until_parked();
+    cx.update(|cx| assert_eq!(speech.read(cx).status(), SpeechStatus::Idle));
+    assert!(
+        matches!(events.borrow().last(), Some(SpeechEvent::Final(text)) if text.as_ref() == "hello")
+    );
+}
+
+#[test]
+fn captured_audio_reaches_the_session_and_the_levels() {
+    let mut cx = TestAppContext::single();
+    let (speech, _, _, received) = harness(&mut cx, true, true);
+
+    start(&speech, &mut cx);
+    cx.executor().advance_clock(Duration::from_secs(1));
+    cx.run_until_parked();
+
+    cx.update(|cx| {
+        let state = speech.read(cx);
+        assert_eq!(*received.borrow(), 16_000);
+        // One level per 80 ms bucket of the one-second capture.
+        let levels = state.levels();
+        assert_eq!(levels.len(), 12);
+        assert!(levels.iter().all(|level| *level == 0.0));
+    });
+
+    cx.update(|cx| speech.update(cx, |state, cx| state.stop(cx)));
+    cx.run_until_parked();
+    cx.update(|cx| assert!(speech.read(cx).levels().is_empty()));
+}
+
+fn events_last(log: &EventLog) -> Option<SpeechEvent> {
+    log.borrow().last().cloned()
+}
@@ -0,0 +1,15 @@
+<svg
+  xmlns="http://www.w3.org/2000/svg"
+  width="24"
+  height="24"
+  viewBox="0 0 24 24"
+  fill="none"
+  stroke="currentColor"
+  stroke-width="2"
+  stroke-linecap="round"
+  stroke-linejoin="round"
+>
+  <path d="M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z" />
+  <path d="M19 10v2a7 7 0 0 1-14 0v-2" />
+  <line x1="12" x2="12" y1="19" y2="22" />
+</svg>
@@ -37,6 +37,8 @@ pub enum IconRole {
    WindowMaximize,
    WindowRestore,
    Loader,
+    Microphone,
+    Stop,
 }

 impl IconRole {
@@ -63,6 +65,8 @@ impl IconRole {
            Self::WindowMaximize,
            Self::WindowRestore,
            Self::Loader,
+            Self::Microphone,
+            Self::Stop,
        ]
    }

@@ -89,6 +93,8 @@ impl IconRole {
            Self::WindowMaximize => "icons/square.svg",
            Self::WindowRestore => "icons/copy.svg",
            Self::Loader => "icons/loader.svg",
+            Self::Microphone => "icons/mic.svg",
+            Self::Stop => "icons/square.svg",
        }
    }

@@ -116,6 +122,8 @@ impl IconRole {
            Self::WindowMaximize => "window-maximize",
            Self::WindowRestore => "window-restore",
            Self::Loader => "loader",
+            Self::Microphone => "microphone",
+            Self::Stop => "stop",
        }
    }

@@ -192,6 +200,8 @@ impl IconRole {
            Self::WindowMaximize => &["window-maximize-symbolic", "window-maximize"],
            Self::WindowRestore => &["window-restore-symbolic", "window-restore"],
            Self::Loader => &["process-working-symbolic", "process-working"],
+            Self::Microphone => &["audio-input-microphone-symbolic", "audio-input-microphone"],
+            Self::Stop => &["process-stop-symbolic", "process-stop"],
        }
    }
 }
@@ -387,6 +397,7 @@ fn asset_bytes(path: &str) -> Option<&'static [u8]> {
        "icons/panel-left.svg" => Some(include_bytes!("../assets/panel-left.svg")),
        "icons/panel-right.svg" => Some(include_bytes!("../assets/panel-right.svg")),
        "icons/loader.svg" => Some(include_bytes!("../assets/loader.svg")),
+        "icons/mic.svg" => Some(include_bytes!("../assets/mic.svg")),
        _ => None,
    }
 }
@@ -26,6 +26,8 @@ pub(crate) fn glyph(role: IconRole) -> Option<&'static str> {
        IconRole::WindowMinimize => "\u{E921}", // ChromeMinimize
        IconRole::WindowMaximize => "\u{E922}", // ChromeMaximize
        IconRole::WindowRestore => "\u{E923}",  // ChromeRestore
+        IconRole::Microphone => "\u{E720}",     // Microphone
+        IconRole::Stop => "\u{E71A}",           // Stop
        // No Segoe spinner glyph, the icon falls back to the bundled SVG.
        IconRole::Loader => return None,
    };