feat: add support for speech to text (#68)

Reviewed-on: #68

assets/icons/paperclip.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none">
+<path d="M12.2126 10.7642L7.07919 15.8829C6.51216 16.4483 6.51216 17.365 7.07919 17.9304C7.64621 18.4958 8.56554 18.4958 9.13257 17.9304L19.3995 7.69303C19.9664 7.12769 20.25 6.38673 20.25 5.64575C20.25 4.90465 19.9665 4.16353 19.3995 3.59809C18.8324 3.03271 18.0893 2.75001 17.3461 2.75C16.6029 2.74999 15.8597 3.03269 15.2927 3.59809L5.02581 13.8354C4.17522 14.6836 3.74995 15.7952 3.75 16.9068C3.75005 18.0183 4.17532 19.1298 5.02581 19.9779C5.87635 20.826 6.99113 21.25 8.10591 21.25C9.22067 21.25 10.3354 20.8259 11.1859 19.9779L16.3194 14.8592" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/>
+</svg>

crates/assets/tests/icons.rs

@@ -1,39 +0,0 @@
-use gpui::AssetSource;
-
-#[test]
-fn icons_resolve_from_the_vendored_folder() {
-    // Icons coop's code and the kit components draw, served by the vendored
-    // folder (signed's set, filled with the kit bundle where coop needs more).
-    let vendored = [
-        "icons/search.svg",
-        "icons/close.svg",
-        "icons/panel-left.svg",
-        "icons/window-minimize.svg",
-        "icons/settings-2.svg",
-        "icons/x.svg",
-        "icons/compass.svg",
-        "icons/radio-tower.svg",
-    ];
-    for path in vendored {
-        let own = assets::Assets
-            .load(path)
-            .unwrap_or_else(|e| panic!("{path} errored in own embed: {e}"));
-        assert!(own.is_some(), "own embed is missing {path}");
-
-        let combined = assets::CombinedAssets.load(path).unwrap();
-        assert!(combined.is_some(), "CombinedAssets fails to serve {path}");
-    }
-
-    // Coop's own files still load through the combined source.
-    let own = [
-        "brand/coop.svg",
-        "brand/bg1.jpg",
-        "fonts/Inter/Inter-Medium.ttf",
-    ];
-    for path in own {
-        assert!(
-            matches!(assets::CombinedAssets.load(path), Ok(Some(_))),
-            "CombinedAssets fails to serve coop asset {path}"
-        );
-    }
-}

crates/chat_ui/Cargo.toml

@@ -6,14 +6,16 @@ publish.workspace = true
 
 [dependencies]
 state = { path = "../state" }
+speech = { path = "../speech" }
 ui = { path = "../ui" }
-gpui-kit.workspace = true
+gpui-kit = { workspace = true, features = ["speech"] }
 common = { path = "../common" }
 person = { path = "../person" }
 chat = { path = "../chat" }
 settings = { path = "../settings" }
 
 gpui.workspace = true
+instant.workspace = true
 nostr-sdk.workspace = true
 anyhow.workspace = true
 itertools.workspace = true

crates/chat_ui/src/actions.rs

@@ -1,4 +1,4 @@
-use gpui::Action;
+use gpui::{Action, actions};
 use nostr_sdk::prelude::*;
 use serde::Deserialize;
 use settings::SignerKind;
@@ -15,3 +15,6 @@ pub enum Command {
     Njump(PublicKey),
     Trace(EventId),
 }
+
+// Start or stop dictation in the focused composer
+actions!(chat, [ToggleDictation]);

crates/chat_ui/src/lib.rs

@@ -1,5 +1,6 @@
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::sync::{Arc, LazyLock, RwLock};
+use std::time::Duration;
 
 pub use actions::*;
 use anyhow::Error;
@@ -20,6 +21,7 @@ use gpui_kit::component::input::{Input, InputEvent, InputState, Textarea, Textar
 use gpui_kit::component::menu::DropdownMenu;
 use gpui_kit::component::notification::Notification;
 use gpui_kit::component::scroll::Scrollbar;
+use gpui_kit::component::speech::{SpeechButton, SpeechEvent, SpeechState, SpeechWaveform};
 use gpui_kit::component::tooltip::Tooltip;
 use gpui_kit::component::{ActiveTheme, Disableable, Icon, Sizable, WindowExt, h_flex, v_flex};
 use itertools::Itertools;
@@ -91,6 +93,12 @@ pub struct ChatPanel {
     /// Chat input state
     input: Entity<TextareaState>,
 
+    /// Dictation session for the chat input
+    speech: Entity<SpeechState>,
+
+    /// When the running dictation started, for the elapsed counter
+    speech_started: Option<instant::Instant>,
+
     /// Subject input state
     subject_input: Entity<InputState>,
 
@@ -116,7 +124,7 @@ pub struct ChatPanel {
     tasks: Vec<Task<Result<(), Error>>>,
 
     /// Event subscriptions
-    subscriptions: SmallVec<[Subscription; 3]>,
+    subscriptions: SmallVec<[Subscription; 4]>,
 }
 
 impl ChatPanel {
@@ -160,6 +168,19 @@ impl ChatPanel {
                 .clean_on_escape()
         });
 
+        // Define dictation state
+        let speech = cx.new(|cx| {
+            let state = SpeechState::new(cx)
+                .recognizer(speech::recognizer())
+                // Batch transcription of a long recording can take a while.
+                .stop_timeout(Duration::from_secs(180));
+
+            #[cfg(target_arch = "wasm32")]
+            let state = state.input(speech::input());
+
+            state
+        });
+
         // Define subject input state
         let subject_input = cx.new(|cx| InputState::new(window, cx).placeholder("New subject..."));
         let subject_bar = cx.new(|_cx| false);
@@ -178,15 +199,39 @@ impl ChatPanel {
 
         subscriptions.push(
             // Subscribe the subject input event
-            cx.subscribe_in(
+            cx.subscribe_in(&subject_input, window, move |this, _, event, window, cx| {
-                &subject_input,
+                if let InputEvent::PressEnter { .. } = event {
-                window,
+                    this.change_subject(window, cx);
-                move |this, _input, event, window, cx| {
+                };
-                    if let InputEvent::PressEnter { .. } = event {
+            }),
-                        this.change_subject(window, cx);
+        );
-                    };
+
-                },
+        subscriptions.push(
-            ),
+            // Insert the dictated text into the chat input
+            cx.subscribe_in(&speech, window, move |this, _speech, event, window, cx| {
+                match event {
+                    SpeechEvent::Started => {
+                        this.speech_started = Some(instant::Instant::now());
+                        cx.notify();
+                    }
+                    SpeechEvent::Final(text) => {
+                        this.speech_started = None;
+
+                        if !text.is_empty() {
+                            this.input
+                                .update(cx, |input, cx| input.insert(text.clone(), window, cx));
+                        }
+                    }
+                    SpeechEvent::Error(error) => {
+                        this.speech_started = None;
+                        window.push_notification(Notification::error(error.to_string()), cx);
+                    }
+                    SpeechEvent::Cancelled => {
+                        this.speech_started = None;
+                    }
+                    _ => {}
+                };
+            }),
         );
 
         // Define all functions that will run after the current cycle
@@ -206,6 +251,8 @@ impl ChatPanel {
             directory,
             list_state,
             input,
+            speech,
+            speech_started: None,
             subject_input,
             subject_bar,
             replies_to,
@@ -835,6 +882,17 @@ impl ChatPanel {
         self.directory.read(cx).person(public_key, cx)
     }
 
+    fn toggle_dictation(
+        &mut self,
+        _: &ToggleDictation,
+        _window: &mut Window,
+        cx: &mut Context<Self>,
+    ) {
+        self.speech.update(cx, |speech, cx| {
+            speech.toggle(cx);
+        });
+    }
+
     fn on_command(&mut self, command: &Command, window: &mut Window, cx: &mut Context<Self>) {
         match command {
             Command::Insert(content) => {
@@ -1390,21 +1448,26 @@ impl ChatPanel {
         }
 
         let reply_id = *id;
-        let copy_id = *id;
         let dropdown_id = *id;
         let public_key = *public_key;
 
-        actions
+        let actions = actions.reply(cx.listener(move |this, _event, _window, cx| {
-            .reply(cx.listener(move |this, _event, _window, cx| {
+            this.reply_to(&reply_id, cx);
-                this.reply_to(&reply_id, cx);
+        }));
-            }))
+
-            .copy(cx.listener(move |this, _event, _window, cx| {
+        let actions = if let Some(message) = self.message(id) {
-                this.copy_message(&copy_id, cx);
+            actions.copy(
-            }))
+                SharedString::from(format!("message-copy-{id}")),
-            .dropdown(move |menu, _window, _cx| {
+                message.preview(),
-                menu.menu("Copy author", Box::new(Command::Copy(public_key)))
+            )
-                    .menu("Seen on", Box::new(Command::Trace(dropdown_id)))
+        } else {
-            })
+            actions
+        };
+
+        actions.dropdown(move |menu, _window, _cx| {
+            menu.menu("Copy author", Box::new(Command::Copy(public_key)))
+                .menu("Seen on", Box::new(Command::Trace(dropdown_id)))
+        })
     }
 
     fn render_attachment(&self, url: &Url, cx: &Context<Self>) -> impl IntoElement {
@@ -1669,7 +1732,12 @@ impl ChatPanel {
         items
     }
 
-    fn render_config_menu(&self, _window: &mut Window, cx: &Context<Self>) -> impl IntoElement {
+    fn render_config_menu(
+        &self,
+        disabled: bool,
+        _window: &mut Window,
+        cx: &Context<Self>,
+    ) -> impl IntoElement {
         let (backup, signer_kind) = self
             .room
             .read_with(cx, |this, _cx| {
@@ -1682,6 +1750,7 @@ impl ChatPanel {
             .icon(IconName::Settings2)
             .tooltip("Configuration")
             .ghost()
+            .disabled(disabled)
             .dropdown_menu(move |this, _window, _cx| {
                 let auto = matches!(signer_kind, SignerKind::Auto);
                 let encryption = matches!(signer_kind, SignerKind::Encryption);
@@ -1712,7 +1781,12 @@ impl ChatPanel {
             })
     }
 
-    fn render_emoji_menu(&self, _window: &Window, cx: &Context<Self>) -> impl IntoElement {
+    fn render_emoji_menu(
+        &self,
+        disabled: bool,
+        _window: &Window,
+        cx: &Context<Self>,
+    ) -> impl IntoElement {
         let nostr = NostrRegistry::global(cx);
         let current_user = nostr.read(cx).current_user();
 
@@ -1729,6 +1803,7 @@ impl ChatPanel {
         Button::new("emoji")
             .icon(IconName::Emoji)
             .ghost()
+            .disabled(disabled)
             .dropdown_menu_with_anchor(gpui::Anchor::BottomLeft, move |this, _window, _cx| {
                 let menu = REACTION_EMOJIS.iter().fold(this, |this, emoji| {
                     this.menu(*emoji, Box::new(Command::Insert(emoji.to_string())))
@@ -1802,10 +1877,12 @@ impl Render for ChatPanel {
 
         let is_typing = !self.input.read(cx).value().trim().is_empty();
         let pending_attachments = !self.encrypted_attachments.read(cx).is_empty();
+        let dictating = self.speech.read(cx).status().is_active();
 
         v_flex()
             .image_cache(retain_all(self.id.clone()))
             .on_action(cx.listener(Self::on_command))
+            .on_action(cx.listener(Self::toggle_dictation))
             .size_full()
             .when(*self.subject_bar.read(cx), |this| {
                 this.child(
@@ -1879,26 +1956,55 @@ impl Render for ChatPanel {
                             .items_end()
                             .child(
                                 Button::new("upload")
-                                    .icon(IconName::Plus)
+                                    .icon(IconName::Paperclip)
                                     .tooltip("Upload media")
                                     .loading(self.uploading)
-                                    .disabled(self.uploading)
+                                    .disabled(dictating || self.uploading)
                                     .ghost()
                                     .on_click(cx.listener(move |this, _ev, window, cx| {
                                         this.upload(window, cx);
                                     })),
                             )
-                            .child(Textarea::new(&self.input).appearance(false).flex_1())
+                            .when(dictating, |this| {
+                                this.child(
+                                    h_flex()
+                                        .h_8()
+                                        .flex_1()
+                                        .gap_2()
+                                        .pl_1()
+                                        .child(SpeechWaveform::new(&self.speech).small().flex_1())
+                                        .when_some(self.speech_started, |this, started| {
+                                            this.child(speech::elapsed(started, cx))
+                                        }),
+                                )
+                            })
+                            .when(!dictating, |this| {
+                                this.child(
+                                    Textarea::new(&self.input)
+                                        .appearance(false)
+                                        .small()
+                                        .min_h_8()
+                                        .py_1()
+                                        .flex_1(),
+                                )
+                            })
+                            .child(
+                                h_flex()
+                                    .items_end()
+                                    .gap_1()
+                                    .pl_1()
+                                    .child(SpeechButton::new(&self.speech)),
+                            )
                             .child(
                                 h_flex()
                                     .pl_1()
                                     .gap_1()
-                                    .child(self.render_emoji_menu(window, cx))
+                                    .child(self.render_emoji_menu(dictating, window, cx))
-                                    .child(self.render_config_menu(window, cx))
+                                    .child(self.render_config_menu(dictating, window, cx))
                                     .child(
                                         Button::new("send")
                                             .icon(IconName::PaperPlaneFill)
-                                            .disabled(self.uploading)
+                                            .disabled(dictating || self.uploading)
                                             .ghost()
                                             .on_click(cx.listener(move |this, _ev, window, cx| {
                                                 this.send_text_message(window, cx);

crates/nip29_ui/Cargo.toml

@@ -11,11 +11,13 @@ common = { path = "../common" }
 nip29 = { path = "../nip29" }
 person = { path = "../person" }
 settings = { path = "../settings" }
+speech = { path = "../speech" }
 state = { path = "../state" }
-gpui-kit.workspace = true
+gpui-kit = { workspace = true, features = ["speech"] }
 ui = { path = "../ui" }
 
 gpui.workspace = true
+instant.workspace = true
 nostr-sdk.workspace = true
 anyhow.workspace = true
 log.workspace = true

crates/nip29_ui/src/lib.rs

@@ -1,11 +1,14 @@
 use std::collections::BTreeMap;
+use std::time::Duration;
 
 use anyhow::Result;
+use chat_ui::ToggleDictation;
 use gpui::prelude::FluentBuilder;
 use gpui::{
     Action, AnyElement, App, AppContext, Context, Entity, EventEmitter, FocusHandle, Focusable,
-    FollowMode, IntoElement, ListAlignment, ListScrollEvent, ListState, ParentElement, Render,
+    FollowMode, InteractiveElement, IntoElement, ListAlignment, ListScrollEvent, ListState,
-    SharedString, Styled, Subscription, Task, WeakEntity, Window, div, list, px,
+    ParentElement, Render, SharedString, Styled, Subscription, Task, WeakEntity, Window, div, list,
+    px,
 };
 use gpui_kit::assets::IconName;
 use gpui_kit::base::StyledExt as _;
@@ -15,6 +18,7 @@ use gpui_kit::component::dock::{Panel, PanelEvent};
 use gpui_kit::component::input::{InputEvent, Textarea, TextareaState};
 use gpui_kit::component::notification::Notification;
 use gpui_kit::component::scroll::Scrollbar;
+use gpui_kit::component::speech::{SpeechButton, SpeechEvent, SpeechState, SpeechWaveform};
 use gpui_kit::component::{ActiveTheme, Disableable, Icon, Sizable, WindowExt, h_flex, v_flex};
 use nip29::{Group, GroupEvent, GroupKey, GroupsRegistry, Membership, Pin, Row};
 use nostr_sdk::prelude::*;
@@ -312,6 +316,10 @@ pub struct GroupPanel {
     reply_to: Option<EventId>,
     rendered_texts_by_id: BTreeMap<EventId, RenderedText>,
     input: Entity<TextareaState>,
+    /// Dictation session for the composer input
+    speech: Entity<SpeechState>,
+    /// When the running dictation started, for the elapsed counter
+    speech_started: Option<instant::Instant>,
     /// Uploaded, non-encrypted attachments waiting to be sent.
     attachments: Vec<Url>,
     /// Whether an upload is in flight.
@@ -319,7 +327,7 @@ pub struct GroupPanel {
     /// Whether the user closed the encryption notice this visit
     notice_dismissed: bool,
     tasks: SmallVec<[Task<Result<()>>; 4]>,
-    _subscriptions: SmallVec<[Subscription; 3]>,
+    _subscriptions: SmallVec<[Subscription; 4]>,
 }
 
 impl GroupPanel {
@@ -338,6 +346,18 @@ impl GroupPanel {
                 .clean_on_escape()
         });
 
+        let speech = cx.new(|cx| {
+            let state = SpeechState::new(cx)
+                .recognizer(speech::recognizer())
+                // Batch transcription of a long recording can take a while.
+                .stop_timeout(Duration::from_secs(180));
+
+            #[cfg(target_arch = "wasm32")]
+            let state = state.input(speech::input());
+
+            state
+        });
+
         let mut subscriptions = smallvec![];
 
         subscriptions.push(
@@ -348,6 +368,34 @@ impl GroupPanel {
             }),
         );
 
+        subscriptions.push(
+            cx.subscribe_in(&speech, window, |this, _, event, window, cx| {
+                match event {
+                    SpeechEvent::Started => {
+                        this.speech_started = Some(instant::Instant::now());
+                        cx.notify();
+                    }
+                    SpeechEvent::Final(text) => {
+                        this.speech_started = None;
+
+                        if !text.is_empty() {
+                            this.input.update(cx, |input, cx| {
+                                input.insert(text.clone(), window, cx);
+                            });
+                        }
+                    }
+                    SpeechEvent::Error(error) => {
+                        this.speech_started = None;
+                        window.push_notification(Notification::error(error.to_string()), cx);
+                    }
+                    SpeechEvent::Cancelled => {
+                        this.speech_started = None;
+                    }
+                    _ => {}
+                };
+            }),
+        );
+
         subscriptions.push(cx.subscribe(&group, |this, _group, event, cx| {
             if !matches!(event, GroupEvent::Sent(_)) {
                 this.sync(cx);
@@ -383,6 +431,8 @@ impl GroupPanel {
             notice_dismissed: false,
             rendered_texts_by_id: BTreeMap::new(),
             input,
+            speech,
+            speech_started: None,
             attachments: Vec::new(),
             uploading: false,
             tasks: smallvec![],
@@ -1015,6 +1065,17 @@ impl GroupPanel {
         )
     }
 
+    fn toggle_dictation(
+        &mut self,
+        _ev: &ToggleDictation,
+        _window: &mut Window,
+        cx: &mut Context<Self>,
+    ) {
+        self.speech.update(cx, |speech, cx| {
+            speech.toggle(cx);
+        });
+    }
+
     fn render_composer(&self, cx: &mut Context<Self>) -> AnyElement {
         let notice = self.group.upgrade().and_then(|group| {
             let group = group.read(cx);
@@ -1040,6 +1101,8 @@ impl GroupPanel {
             return self.render_notice(&notice, cx);
         }
 
+        let dictating = self.speech.read(cx).status().is_active();
+
         v_flex()
             .flex_shrink_0()
             .w_full()
@@ -1055,21 +1118,45 @@ impl GroupPanel {
                     .items_end()
                     .child(
                         Button::new("upload")
-                            .icon(IconName::Upload)
+                            .icon(ui::IconName::Paperclip)
-                            .tooltip("Upload file")
+                            .tooltip("Upload media")
                             .loading(self.uploading)
-                            .disabled(self.uploading)
+                            .disabled(dictating || self.uploading)
                             .ghost()
                             .on_click(cx.listener(|this, _event, window, cx| {
                                 this.upload(window, cx);
                             })),
                     )
-                    .child(Textarea::new(&self.input).appearance(false).flex_1())
+                    .when(dictating, |this| {
+                        this.child(
+                            h_flex()
+                                .h_8()
+                                .flex_1()
+                                .gap_2()
+                                .child(SpeechWaveform::new(&self.speech).small().flex_1())
+                                .when_some(self.speech_started, |this, started| {
+                                    this.child(speech::elapsed(started, cx))
+                                }),
+                        )
+                    })
+                    .when(!dictating, |this| {
+                        this.child(
+                            Textarea::new(&self.input)
+                                .appearance(false)
+                                .bg(cx.theme().input)
+                                .small()
+                                .min_h_8()
+                                .py_1()
+                                .flex_1(),
+                        )
+                    })
+                    .child(SpeechButton::new(&self.speech))
                     .child(
                         Button::new("send")
                             .icon(ui::IconName::PaperPlaneFill)
                             .tooltip("Send")
                             .ghost()
+                            .disabled(dictating || self.uploading)
                             .on_click(cx.listener(|this, _event, window, cx| {
                                 this.send(window, cx);
                             })),
@@ -1267,6 +1354,7 @@ impl Render for GroupPanel {
         v_flex()
             .size_full()
             .min_w_0()
+            .on_action(cx.listener(Self::toggle_dictation))
             .child(self.render_encryption_notice(cx))
             .child(self.render_banner(cx))
             .child(self.render_pins(cx))

crates/speech/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "speech"
+version.workspace = true
+edition.workspace = true
+publish.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+log.workspace = true
+gpui.workspace = true
+gpui-kit = { workspace = true, features = ["speech"] }
+flume.workspace = true
+instant.workspace = true
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+common = { path = "../common" }
+parakeet-rs = "=0.3.8"
+ort = "=2.0.0-rc.13"
+reqwest = { version = "0.12", default-features = false, features = [
+    "blocking",
+    "rustls-tls",
+] }
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+js-sys = "0.3"
+wasm-bindgen = "0.2"
+web-sys = { version = "0.3", features = ["Window"] }

crates/speech/src/lib.rs

@@ -0,0 +1,53 @@
+#[cfg(not(target_arch = "wasm32"))]
+pub mod model;
+#[cfg(not(target_arch = "wasm32"))]
+mod parakeet;
+#[cfg(target_arch = "wasm32")]
+mod web;
+
+use std::time::Duration;
+
+use gpui::{
+    Animation, AnimationExt as _, AnyElement, App, IntoElement as _, ParentElement as _,
+    Styled as _, div,
+};
+use gpui_kit::component::ActiveTheme as _;
+use gpui_kit::component::speech::SpeechRecognizer;
+
+/// A ticking `M:SS` counter for the dictation in progress.
+pub fn elapsed(started: instant::Instant, cx: &App) -> AnyElement {
+    div()
+        .text_xs()
+        .text_color(cx.theme().muted_foreground)
+        .with_animation(
+            "dictation-elapsed",
+            Animation::new(Duration::from_secs(1))
+                .repeat()
+                .with_max_fps(1.),
+            move |counter, _| counter.child(format_elapsed(started)),
+        )
+        .into_any_element()
+}
+
+fn format_elapsed(started: instant::Instant) -> String {
+    let seconds = started.elapsed().as_secs();
+    format!("{}:{:02}", seconds / 60, seconds % 60)
+}
+
+/// The recognizer backing the composer's microphone button.
+#[cfg(not(target_arch = "wasm32"))]
+pub fn recognizer() -> impl SpeechRecognizer {
+    parakeet::ParakeetRecognizer
+}
+
+/// The recognizer backing the composer's microphone button.
+#[cfg(target_arch = "wasm32")]
+pub fn recognizer() -> impl SpeechRecognizer {
+    web::WebSpeechRecognizer
+}
+
+/// The audio source backing the composer's microphone button on the web.
+#[cfg(target_arch = "wasm32")]
+pub fn input() -> impl gpui_kit::component::speech::AudioInput {
+    web::SelfCapturedInput
+}

crates/speech/src/model.json

@@ -0,0 +1,23 @@
+{
+  "repository": "istupakov/parakeet-tdt-0.6b-v3-onnx",
+  "revision": "8f23f0c03c8761650bdb5b40aaf3e40d2c15f1ce",
+  "source": "nvidia/parakeet-tdt-0.6b-v3",
+  "license": "CC-BY-4.0",
+  "files": [
+    {
+      "name": "decoder_joint-model.int8.onnx",
+      "size": 18202004,
+      "sha256": "eea7483ee3d1a30375daedc8ed83e3960c91b098812127a0d99d1c8977667a70"
+    },
+    {
+      "name": "encoder-model.int8.onnx",
+      "size": 652183999,
+      "sha256": "6139d2fa7e1b086097b277c7149725edbab89cc7c7ae64b23c741be4055aff09"
+    },
+    {
+      "name": "vocab.txt",
+      "size": 93939,
+      "sha256": "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d"
+    }
+  ]
+}

crates/speech/src/model.rs

@@ -0,0 +1,295 @@
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, OnceLock};
+use std::{fs, io};
+
+use anyhow::{Result, bail};
+use serde::Deserialize;
+
+/// How the model is currently available.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Status {
+    NotInstalled,
+    Downloading { downloaded: u64, total: u64 },
+    Installed,
+}
+
+impl Status {
+    pub fn is_downloading(self) -> bool {
+        matches!(self, Status::Downloading { .. })
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct Manifest {
+    repository: String,
+    revision: String,
+    files: Vec<Artifact>,
+}
+
+#[derive(Debug, Deserialize)]
+struct Artifact {
+    name: String,
+    size: u64,
+    sha256: String,
+}
+
+fn manifest() -> &'static Manifest {
+    static MANIFEST: OnceLock<Manifest> = OnceLock::new();
+
+    MANIFEST.get_or_init(|| {
+        serde_json::from_str(include_str!("model.json")).expect("pinned model manifest")
+    })
+}
+
+/// The directory the model artifacts are downloaded into.
+pub fn dir() -> PathBuf {
+    common::config_dir()
+        .join("speech")
+        .join("parakeet-tdt-0.6b-v3")
+}
+
+/// Total download size in bytes.
+pub fn total_size() -> u64 {
+    manifest().files.iter().map(|file| file.size).sum()
+}
+
+/// The revision of the installed model, if its artifacts are complete.
+fn installed_revision() -> Option<String> {
+    let manifest = manifest();
+    let dir = dir();
+
+    let complete = manifest.files.iter().all(|file| {
+        fs::metadata(dir.join(&file.name)).is_ok_and(|metadata| metadata.len() == file.size)
+    });
+
+    if complete
+        && fs::read_to_string(dir.join("verified"))
+            .is_ok_and(|revision| revision == manifest.revision)
+    {
+        Some(manifest.revision.clone())
+    } else {
+        None
+    }
+}
+
+/// Whether the model is ready to load.
+pub fn installed() -> bool {
+    status() == Status::Installed
+}
+
+/// The current status, reconciling the download state with the disk.
+pub fn status() -> Status {
+    let state = state().lock().unwrap_or_else(|error| error.into_inner());
+
+    if let Download::Running {
+        downloaded, total, ..
+    } = state.download
+    {
+        return Status::Downloading { downloaded, total };
+    }
+
+    if installed_revision().is_some() {
+        Status::Installed
+    } else {
+        Status::NotInstalled
+    }
+}
+
+/// Take the last download failure, if it has not been reported yet.
+pub fn take_error() -> Option<String> {
+    let mut state = state().lock().unwrap_or_else(|error| error.into_inner());
+    state.error.take()
+}
+
+/// Start downloading the model, unless a download is already running.
+pub fn start_download() {
+    let mut state = state().lock().unwrap_or_else(|error| error.into_inner());
+
+    if matches!(state.download, Download::Running { .. }) {
+        return;
+    }
+
+    let cancel = Arc::new(AtomicBool::new(false));
+    state.error = None;
+    state.download = Download::Running {
+        downloaded: 0,
+        total: total_size(),
+        cancel: cancel.clone(),
+    };
+
+    if let Err(error) = std::thread::Builder::new()
+        .name("speech-model-download".into())
+        .spawn(move || download(cancel))
+    {
+        state.download = Download::Idle;
+        state.error = Some(error.to_string());
+    }
+}
+
+/// Request cancellation of the running download.
+pub fn cancel_download() {
+    let state = state().lock().unwrap_or_else(|error| error.into_inner());
+
+    if let Download::Running { cancel, .. } = &state.download {
+        cancel.store(true, Ordering::Release);
+    }
+}
+
+/// Delete the downloaded model and any partial download.
+pub fn delete() {
+    super::parakeet::unload();
+    cancel_download();
+
+    if let Err(error) = fs::remove_dir_all(dir())
+        && error.kind() != io::ErrorKind::NotFound
+    {
+        log::warn!("failed to remove the speech model: {error}");
+    }
+}
+
+fn download(cancel: Arc<AtomicBool>) {
+    let result = download_inner(&cancel);
+
+    let mut state = state().lock().unwrap_or_else(|error| error.into_inner());
+    state.download = Download::Idle;
+
+    // A cancelled download must not report a spurious failure.
+    if !cancelled(&cancel)
+        && let Err(error) = result
+    {
+        state.error = Some(format!("{error:#}"));
+    }
+}
+
+fn download_inner(cancel: &AtomicBool) -> Result<()> {
+    let manifest = manifest();
+    let dir = dir();
+    let client = reqwest::blocking::Client::builder()
+        .connect_timeout(std::time::Duration::from_secs(20))
+        .build()?;
+
+    fs::create_dir_all(&dir)?;
+
+    for file in &manifest.files {
+        if cancelled(cancel) {
+            bail!("Cancelled");
+        }
+
+        let staging = dir.join(format!("{}.part", file.name));
+        let result = download_file(&client, manifest, file, &staging, cancel);
+
+        if result.is_err() || cancelled(cancel) {
+            let _unused = fs::remove_file(&staging);
+            result?;
+            bail!("Cancelled");
+        }
+    }
+
+    fs::write(dir.join("verified"), &manifest.revision)?;
+    Ok(())
+}
+
+fn download_file(
+    client: &reqwest::blocking::Client,
+    manifest: &Manifest,
+    file: &Artifact,
+    staging: &Path,
+    cancel: &AtomicBool,
+) -> Result<()> {
+    use std::io::{Read as _, Write as _};
+
+    use sha2::Digest as _;
+
+    let url = format!(
+        "https://huggingface.co/{}/resolve/{}/{}",
+        manifest.repository, manifest.revision, file.name
+    );
+
+    let mut response = client.get(url).send()?.error_for_status()?;
+    let mut out = fs::File::create(staging)?;
+    let mut hash = sha2::Sha256::new();
+    let mut size = 0u64;
+
+    loop {
+        if cancelled(cancel) {
+            bail!("Cancelled");
+        }
+
+        let mut chunk = [0u8; 64 * 1024];
+        let read = response.read(&mut chunk)?;
+
+        if read == 0 {
+            break;
+        }
+
+        size += read as u64;
+        if size > file.size {
+            bail!("Unexpected model size");
+        }
+
+        hash.update(&chunk[..read]);
+        out.write_all(&chunk[..read])?;
+
+        let mut state = state().lock().unwrap_or_else(|error| error.into_inner());
+        if let Download::Running { downloaded, .. } = &mut state.download {
+            *downloaded += read as u64;
+        }
+    }
+
+    if size != file.size || format!("{:x}", hash.finalize()) != file.sha256 {
+        bail!("Model checksum verification failed");
+    }
+
+    out.sync_all()?;
+    fs::rename(staging, dir().join(&file.name))?;
+    Ok(())
+}
+
+fn cancelled(cancel: &AtomicBool) -> bool {
+    cancel.load(Ordering::Acquire)
+}
+
+enum Download {
+    Idle,
+    Running {
+        downloaded: u64,
+        total: u64,
+        cancel: Arc<AtomicBool>,
+    },
+}
+
+struct State {
+    download: Download,
+    error: Option<String>,
+}
+
+fn state() -> &'static Mutex<State> {
+    static STATE: Mutex<State> = Mutex::new(State {
+        download: Download::Idle,
+        error: None,
+    });
+
+    &STATE
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn manifest_pins_v3_artifacts_with_exact_sizes() {
+        let manifest = manifest();
+
+        assert_eq!(manifest.repository, "istupakov/parakeet-tdt-0.6b-v3-onnx");
+        assert_eq!(manifest.revision.len(), 40);
+        assert_eq!(manifest.files.len(), 3);
+        assert_eq!(total_size(), 670_479_942);
+        assert!(
+            manifest
+                .files
+                .iter()
+                .all(|file| file.sha256.len() == 64 && !file.name.contains('/'))
+        );
+    }
+}

crates/speech/src/parakeet.rs

@@ -0,0 +1,214 @@
+use std::path::PathBuf;
+use std::sync::OnceLock;
+use std::time::Duration;
+
+use anyhow::{Context as _, Result, anyhow};
+use gpui::{App, Task};
+use gpui_kit::component::speech::{
+    AudioFormat, RecognitionSession, SpeechError, SpeechRecognizer, SpeechSink,
+};
+use parakeet_rs::{ParakeetTDT, Transcriber as _};
+
+const MODEL_RATE: u32 = 16_000;
+const MAX_SECONDS: usize = 120;
+
+/// The on-device recognizer.
+pub struct ParakeetRecognizer;
+
+impl SpeechRecognizer for ParakeetRecognizer {
+    fn audio_format(&self) -> AudioFormat {
+        AudioFormat::new(MODEL_RATE, 1)
+    }
+
+    fn is_available(&self, _cx: &App) -> bool {
+        true
+    }
+
+    fn start(
+        &self,
+        sink: SpeechSink,
+        cx: &mut App,
+    ) -> Result<Box<dyn RecognitionSession>, SpeechError> {
+        if super::model::installed() {
+            worker().send_or_log(Command::Warm(super::model::dir()));
+        } else if super::model::status().is_downloading() {
+            return Err(SpeechError::recognizer(anyhow!(
+                "The speech model is still downloading. Try again when it finishes."
+            )));
+        } else {
+            return Err(SpeechError::recognizer(anyhow!(
+                "No speech model is installed. Download it from Settings."
+            )));
+        }
+
+        // The session buffers everything it is pushed,
+        // so audio is accepted right away.
+        sink.ready(cx);
+
+        Ok(Box::new(ParakeetSession {
+            sink,
+            samples: Vec::new(),
+            _result: None,
+        }))
+    }
+}
+
+struct ParakeetSession {
+    sink: SpeechSink,
+    samples: Vec<f32>,
+    _result: Option<Task<()>>,
+}
+
+impl RecognitionSession for ParakeetSession {
+    fn push_audio(&mut self, samples: &[i16], _cx: &mut App) {
+        if self.samples.len() < MAX_SECONDS * MODEL_RATE as usize {
+            self.samples
+                .extend(samples.iter().map(|sample| f32::from(*sample) / 32_768.0));
+        }
+    }
+
+    fn finish(&mut self, cx: &mut App) {
+        let samples = std::mem::take(&mut self.samples);
+        let sink = self.sink.clone();
+
+        // Speech too short to transcribe ends the session at once.
+        let silent = samples.iter().all(|sample| sample.abs() < 0.0001);
+        if samples.len() < MODEL_RATE as usize / 5 || silent {
+            sink.finish(cx);
+            return;
+        }
+
+        let (sender, receiver) = flume::unbounded();
+        worker().send_or_log(Command::Transcribe { samples, sender });
+
+        self._result = Some(cx.spawn(async move |cx| {
+            let result = receiver.recv_async().await;
+
+            cx.update(|cx| match result {
+                Ok(Ok(text)) if !text.is_empty() => {
+                    sink.phrase(text, cx);
+                    sink.finish(cx);
+                }
+                Ok(Ok(_)) => sink.finish(cx),
+                Ok(Err(error)) => sink.error(SpeechError::recognizer(error), cx),
+                Err(_) => sink.error(
+                    SpeechError::recognizer(anyhow!("The speech worker is unavailable")),
+                    cx,
+                ),
+            });
+        }));
+    }
+}
+
+enum Command {
+    Warm(PathBuf),
+    Transcribe {
+        samples: Vec<f32>,
+        sender: flume::Sender<Result<String, anyhow::Error>>,
+    },
+    Unload,
+}
+
+/// A handle to the single inference worker.
+fn worker() -> &'static flume::Sender<Command> {
+    static WORKER: OnceLock<flume::Sender<Command>> = OnceLock::new();
+
+    WORKER.get_or_init(|| {
+        let (sender, receiver) = flume::unbounded::<Command>();
+
+        let spawned = std::thread::Builder::new()
+            .name("speech-parakeet".into())
+            .spawn(move || run_worker(receiver));
+
+        if let Err(error) = spawned {
+            log::error!("failed to spawn the speech worker: {error}");
+        }
+
+        sender
+    })
+}
+
+fn run_worker(receiver: flume::Receiver<Command>) {
+    let mut cached: Option<(PathBuf, ParakeetTDT)> = None;
+
+    while let Ok(command) = receiver.recv_timeout(Duration::from_secs(30)) {
+        match command {
+            Command::Unload => {
+                cached = None;
+            }
+            Command::Warm(dir) => {
+                let loaded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                    load(&mut cached, &dir).map(|_| ())
+                }));
+
+                match flatten_panic(loaded) {
+                    Ok(Ok(())) => {}
+                    Ok(Err(error)) | Err(error) => {
+                        log::warn!("failed to load the speech model: {error:#}")
+                    }
+                }
+            }
+            Command::Transcribe { samples, sender } => {
+                let transcribed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                    transcribe(&mut cached, samples)
+                }));
+
+                let result = match flatten_panic(transcribed) {
+                    Ok(result) => result,
+                    Err(_) => {
+                        cached = None;
+                        Err(anyhow!("Dictation stopped unexpectedly. Try again."))
+                    }
+                };
+
+                if sender.send(result).is_err() {
+                    log::debug!("speech session was cancelled before its result arrived");
+                }
+            }
+        }
+    }
+}
+
+fn load<'a>(
+    cached: &'a mut Option<(PathBuf, ParakeetTDT)>,
+    dir: &PathBuf,
+) -> Result<&'a mut ParakeetTDT> {
+    if cached.as_ref().is_none_or(|(path, _)| path != dir) {
+        let model = ParakeetTDT::from_pretrained(dir, None)
+            .map_err(|error| anyhow!("Could not load the speech model: {error}"))?;
+        *cached = Some((dir.clone(), model));
+    }
+
+    let (_, model) = cached.as_mut().context("speech model was unloaded")?;
+    Ok(model)
+}
+
+fn transcribe(cached: &mut Option<(PathBuf, ParakeetTDT)>, samples: Vec<f32>) -> Result<String> {
+    let model = load(cached, &super::model::dir())?;
+
+    model
+        .transcribe_samples(samples, MODEL_RATE, 1, None)
+        .map(|result| result.text)
+        .map_err(|error| anyhow!("Could not transcribe this recording: {error}"))
+}
+
+trait SendOrLog {
+    fn send_or_log(&self, command: Command);
+}
+
+impl SendOrLog for flume::Sender<Command> {
+    fn send_or_log(&self, command: Command) {
+        if self.send(command).is_err() {
+            log::debug!("speech worker is gone; command dropped");
+        }
+    }
+}
+
+/// Retire cached weights, e.g. after the model was deleted.
+pub(crate) fn unload() {
+    worker().send_or_log(Command::Unload);
+}
+
+fn flatten_panic<T>(result: std::thread::Result<T>) -> Result<T> {
+    result.map_err(|_| anyhow!("The speech worker panicked"))
+}

crates/speech/src/web.rs

@@ -0,0 +1,256 @@
+use std::cell::RefCell;
+
+use anyhow::anyhow;
+use gpui::App;
+use gpui_kit::Subscription;
+use gpui_kit::component::speech::{
+    AudioFormat, AudioInput, AudioSink, RecognitionSession, SpeechError, SpeechRecognizer,
+    SpeechSink,
+};
+use js_sys::{Array, Function, JsString, Reflect};
+use wasm_bindgen::closure::Closure;
+use wasm_bindgen::{JsCast as _, JsValue};
+use web_sys::window;
+
+/// The recognizer backed by `SpeechRecognition` (or its `webkit` prefix).
+pub struct WebSpeechRecognizer;
+
+impl SpeechRecognizer for WebSpeechRecognizer {
+    fn audio_format(&self) -> AudioFormat {
+        AudioFormat::default()
+    }
+
+    fn is_available(&self, _cx: &App) -> bool {
+        true
+    }
+
+    fn start(
+        &self,
+        sink: SpeechSink,
+        cx: &mut App,
+    ) -> Result<Box<dyn RecognitionSession>, SpeechError> {
+        let recognition = Recognition::open(sink, cx).map_err(SpeechError::recognizer)?;
+
+        Ok(Box::new(recognition))
+    }
+}
+
+pub struct SelfCapturedInput;
+
+impl AudioInput for SelfCapturedInput {
+    fn start(
+        &self,
+        _format: AudioFormat,
+        _sink: AudioSink,
+        _cx: &mut App,
+    ) -> Result<Subscription, SpeechError> {
+        Ok(Subscription::new(|| {}))
+    }
+}
+
+enum Event {
+    Hypothesis(String),
+    Phrase(String),
+    Error(String),
+    End,
+}
+
+struct Recognition {
+    recognition: JsValue,
+    on_result: Option<Closure<dyn FnMut(JsValue)>>,
+    on_error: Option<Closure<dyn FnMut(JsValue)>>,
+    on_end: Option<Closure<dyn FnMut(JsValue)>>,
+    _task: gpui::Task<()>,
+}
+
+impl Recognition {
+    fn open(sink: SpeechSink, cx: &mut App) -> anyhow::Result<Self> {
+        let window = window().ok_or_else(|| anyhow!("No browser window"))?;
+
+        let constructor = Reflect::get(&window, &JsString::from("SpeechRecognition"))
+            .ok()
+            .filter(|value| value.is_function())
+            .or_else(|| {
+                Reflect::get(&window, &JsString::from("webkitSpeechRecognition"))
+                    .ok()
+                    .filter(|value| value.is_function())
+            })
+            .ok_or_else(|| anyhow!("This browser does not support speech recognition"))?;
+
+        let recognition =
+            Reflect::construct(constructor.unchecked_ref::<Function>(), &Array::new())
+                .map_err(|error| js_error("constructing the recognizer", error))?;
+
+        let (sender, receiver) = flume::unbounded::<Event>();
+        let error_sender = sender.clone();
+        let end_sender = sender.clone();
+
+        let finalized = RefCell::new(0usize);
+        let on_result = Closure::wrap(Box::new(move |event: JsValue| {
+            let read = |value: &JsValue, property: &str| -> Option<JsValue> {
+                Reflect::get(value, &JsString::from(property)).ok()
+            };
+
+            let Some(results) = read(&event, "results") else {
+                return;
+            };
+            let length = read(&results, "length")
+                .and_then(|length| length.as_f64())
+                .unwrap_or(0.0) as usize;
+
+            let mut finalized = finalized.borrow_mut();
+            let mut interim = String::new();
+
+            for index in *finalized..length {
+                let Some(result) = read(&results, &index.to_string()) else {
+                    continue;
+                };
+
+                let Some(alternative) = read(&result, "0") else {
+                    continue;
+                };
+
+                let Some(text) = read(&alternative, "transcript").and_then(|text| text.as_string())
+                else {
+                    continue;
+                };
+
+                let is_final = read(&result, "isFinal")
+                    .and_then(|final_value| final_value.as_bool())
+                    .unwrap_or(false);
+
+                if is_final {
+                    *finalized += 1;
+                    if sender.send(Event::Phrase(text)).is_err() {
+                        log::debug!("speech session was cancelled mid-utterance");
+                    }
+                } else {
+                    interim.push_str(&text);
+                }
+            }
+
+            if !interim.is_empty() && sender.send(Event::Hypothesis(interim)).is_err() {
+                log::debug!("speech session was cancelled mid-utterance");
+            }
+        }) as Box<dyn FnMut(JsValue)>);
+
+        let on_error = Closure::wrap(Box::new(move |event: JsValue| {
+            let message = Reflect::get(&event, &JsString::from("error"))
+                .ok()
+                .and_then(|error| error.as_string())
+                .unwrap_or_else(|| "unknown speech error".into());
+
+            if error_sender.send(Event::Error(message)).is_err() {
+                log::debug!("speech session was cancelled before its error arrived");
+            }
+        }) as Box<dyn FnMut(JsValue)>);
+
+        let on_end = Closure::wrap(Box::new(move |_event: JsValue| {
+            if end_sender.send(Event::End).is_err() {
+                log::debug!("speech session was cancelled before its end arrived");
+            }
+        }) as Box<dyn FnMut(JsValue)>);
+
+        set_callback(&recognition, "onresult", on_result.as_ref().unchecked_ref())?;
+        set_callback(&recognition, "onerror", on_error.as_ref().unchecked_ref())?;
+        set_callback(&recognition, "onend", on_end.as_ref().unchecked_ref())?;
+        set_bool(&recognition, "continuous", true)?;
+        set_bool(&recognition, "interimResults", true)?;
+
+        call_method(&recognition, "start")?;
+
+        // The browser consumes audio as soon as it has permission
+        sink.ready(cx);
+
+        let task = cx.spawn(async move |cx| {
+            while let Ok(event) = receiver.recv_async().await {
+                let done = matches!(event, Event::End);
+
+                // The speech state ignores sink calls once its session ended,
+                // so a late event after a cancel is harmless.
+                cx.update(|cx| match event {
+                    Event::Hypothesis(text) => sink.hypothesis(text, cx),
+                    Event::Phrase(text) => sink.phrase(text, cx),
+                    Event::Error(message) => {
+                        sink.error(SpeechError::recognizer(anyhow!("{message}")), cx)
+                    }
+                    Event::End => sink.finish(cx),
+                });
+
+                if done {
+                    break;
+                }
+            }
+        });
+
+        Ok(Self {
+            recognition,
+            on_result: Some(on_result),
+            on_error: Some(on_error),
+            on_end: Some(on_end),
+            _task: task,
+        })
+    }
+
+    fn stop(&mut self) {
+        self.on_result = None;
+        self.on_error = None;
+        self.on_end = None;
+
+        let _ignored = call_method(&self.recognition, "abort");
+    }
+}
+
+impl Drop for Recognition {
+    fn drop(&mut self) {
+        self.stop();
+    }
+}
+
+impl RecognitionSession for Recognition {
+    fn push_audio(&mut self, _samples: &[i16], _cx: &mut App) {
+        // The browser recognizer captures the microphone itself.
+        // the state still pushes audio through its input, which is a no-op here.
+    }
+
+    fn finish(&mut self, _cx: &mut App) {
+        // The end handler reports completion, so no result is fabricated here.
+        let _ignored = call_method(&self.recognition, "stop");
+    }
+}
+
+fn set_callback(target: &JsValue, property: &str, callback: &Function) -> anyhow::Result<()> {
+    Reflect::set(target, &JsString::from(property), callback)
+        .map_err(|error| js_error(property, error))?;
+    Ok(())
+}
+
+fn set_bool(target: &JsValue, property: &str, value: bool) -> anyhow::Result<()> {
+    Reflect::set(target, &JsString::from(property), &JsValue::from(value))
+        .map_err(|error| js_error(property, error))?;
+    Ok(())
+}
+
+fn call_method(target: &JsValue, method: &str) -> anyhow::Result<()> {
+    let function = Reflect::get(target, &JsString::from(method))
+        .map_err(|error| js_error(method, error))?
+        .unchecked_into::<Function>();
+
+    function
+        .call0(target)
+        .map(|_| ())
+        .map_err(|error| js_error(method, error))
+}
+
+fn js_error(what: &str, error: JsValue) -> anyhow::Error {
+    let description = error
+        .as_string()
+        .or_else(|| {
+            Reflect::get(&error, &JsString::from("message"))
+                .ok()
+                .and_then(|message| message.as_string())
+        })
+        .unwrap_or_else(|| "unknown error".into());
+
+    anyhow!("{what} failed: {description}")
+}

crates/ui/src/icon.rs

@@ -20,6 +20,7 @@ pub enum IconName {
     Link,
     Lock,
     Message,
+    Paperclip,
     PaperPlaneFill,
     Plus,
     PlusChat,
@@ -51,6 +52,7 @@ impl IconNamed for IconName {
             Self::Link => "icons/link.svg",
             Self::Lock => "icons/lock.svg",
             Self::Message => "icons/message.svg",
+            Self::Paperclip => "icons/paperclip.svg",
             Self::PaperPlaneFill => "icons/paper-plane-fill.svg",
             Self::Plus => "icons/plus.svg",
             Self::PlusChat => "icons/plus-chat.svg",

crates/ui/src/message.rs

@@ -10,6 +10,7 @@ use gpui::{
 };
 use gpui_kit::assets::IconName;
 use gpui_kit::component::button::{Button, ButtonVariants as _};
+use gpui_kit::component::clipboard::Clipboard;
 use gpui_kit::component::menu::{DropdownMenu as _, PopupMenu, PopupMenuItem};
 use gpui_kit::component::{
     ActiveTheme, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex, v_flex,
@@ -602,7 +603,7 @@ pub struct MessageActions {
     style: StyleRefinement,
     reactions: SmallVec<[(&'static str, SharedClickListener); 4]>,
     reply: Option<SharedClickListener>,
-    copy: Option<SharedClickListener>,
+    copy: Option<(ElementId, SharedString)>,
     dropdown: Option<DropdownBuilder>,
 }
 
@@ -634,9 +635,10 @@ impl MessageActions {
         self
     }
 
-    /// Add the copy button.
+    /// Add the copy button for the given value.
-    pub fn copy(mut self, handler: impl Fn(&ClickEvent, &mut Window, &mut App) + 'static) -> Self {
+    /// The id must be unique per bar, the clipboard component keys its copied state on it.
-        self.copy = Some(Rc::new(handler));
+    pub fn copy(mut self, id: impl Into<ElementId>, value: impl Into<SharedString>) -> Self {
+        self.copy = Some((id.into(), value.into()));
         self
     }
 
@@ -707,15 +709,8 @@ impl RenderOnce for MessageActions {
                         .on_click(move |event, window, cx| reply(event, window, cx)),
                 )
             })
-            .when_some(copy, |this, copy| {
+            .when_some(copy, |this, (id, value)| {
-                this.child(
+                this.child(Clipboard::new(id).value(value).tooltip("Copy").small())
-                    Button::new("copy")
-                        .icon(IconName::Copy)
-                        .tooltip("Copy")
-                        .small()
-                        .ghost()
-                        .on_click(move |event, window, cx| copy(event, window, cx)),
-                )
             })
             .when_some(dropdown, |this, dropdown| {
                 this.child(div().flex_shrink_0().h_4().w_px().bg(cx.theme().border))

crates/workspace/Cargo.toml

@@ -15,6 +15,7 @@ chat_ui = { path = "../chat_ui" }
 nip29 = { path = "../nip29" }
 nip29_ui = { path = "../nip29_ui" }
 settings = { path = "../settings" }
+speech = { path = "../speech" }
 person = { path = "../person" }
 auto_update = { path = "../auto_update" }
 

crates/workspace/src/dialogs/new_chat.rs

@@ -10,7 +10,6 @@ use gpui::{
     Window, div, px, uniform_list,
 };
 use gpui_kit::assets::IconName;
-use gpui_kit::base::StyledExt as _;
 use gpui_kit::component::button::{Button, ButtonVariants};
 use gpui_kit::component::dock::{DockArea, DockPlacement, PanelHandle};
 use gpui_kit::component::input::{Input, InputEvent, InputState};
@@ -205,6 +204,7 @@ impl NewChat {
 
                 Ok(())
             }));
+
             return;
         }
 
@@ -369,15 +369,13 @@ impl NewChat {
                     .p_0p5()
                     .rounded_full()
                     .bg(cx.theme().secondary)
-                    .text_xs()
-                    .font_medium()
                     .hover(|this| this.bg(cx.theme().primary))
                     .child(
                         Avatar::new(profile.avatar())
                             .seed(profile.avatar_seed())
                             .xsmall(),
                     )
-                    .child(div().max_w_32().truncate().child(profile.name()))
+                    .child(div().max_w_32().truncate().text_xs().child(profile.name()))
                     .on_click(cx.listener(move |this, _event, _window, cx| {
                         this.toggle_member(&public_key, cx);
                     }))

crates/workspace/src/dialogs/profile.rs

@@ -4,17 +4,17 @@ use anyhow::Error;
 use chat::{ChatRegistry, Room, RoomKind};
 use gpui::prelude::FluentBuilder;
 use gpui::{
-    App, AppContext, ClipboardItem, Context, Div, Entity, IntoElement, ParentElement, Render,
+    App, AppContext, Context, Div, Entity, IntoElement, ParentElement, Render, SharedString,
-    SharedString, Styled, Task, Window, div, retain_all,
+    Styled, Task, Window, div, retain_all,
 };
 use gpui_kit::assets::IconName;
 use gpui_kit::base::StyledExt as _;
 use gpui_kit::component::button::{Button, ButtonVariants};
+use gpui_kit::component::clipboard::Clipboard;
 use gpui_kit::component::dock::DockPlacement;
 use gpui_kit::component::input::{Input, InputState, Textarea, TextareaState};
 use gpui_kit::component::notification::Notification;
 use gpui_kit::component::{ActiveTheme, Disableable, Sizable, WindowExt, h_flex, v_flex};
-use instant::Duration;
 use nostr_sdk::prelude::*;
 use person::{Directory, Person, PersonRegistry, shorten_pubkey};
 use settings::AppSettings;
@@ -49,8 +49,6 @@ struct ProfileDialog {
     website_input: Entity<InputState>,
     /// Uploading state
     uploading: bool,
-    /// Copied states
-    copied: bool,
     /// Updating state
     updating: bool,
     /// Tasks
@@ -92,7 +90,6 @@ impl ProfileDialog {
             bio_input,
             website_input,
             uploading: false,
-            copied: false,
             updating: false,
             tasks: vec![],
         }
@@ -141,35 +138,6 @@ impl ProfileDialog {
         cx.notify();
     }
 
-    fn copy(&mut self, value: String, window: &mut Window, cx: &mut Context<Self>) {
-        let item = ClipboardItem::new_string(value);
-        cx.write_to_clipboard(item);
-
-        self.set_copied(true, window, cx);
-    }
-
-    fn set_copied(&mut self, status: bool, window: &mut Window, cx: &mut Context<Self>) {
-        self.copied = status;
-        cx.notify();
-
-        if status {
-            self.tasks.push(cx.spawn_in(window, async move |this, cx| {
-                cx.background_executor().timer(Duration::from_secs(2)).await;
-
-                // Reset the copied state after a delay
-                cx.update(|window, cx| {
-                    this.update(cx, |this, cx| {
-                        this.set_copied(false, window, cx);
-                    })
-                    .ok();
-                })
-                .ok();
-
-                Ok(())
-            }));
-        }
-    }
-
     fn set_uploading(&mut self, status: bool, cx: &mut Context<Self>) {
         self.uploading = status;
         cx.notify();
@@ -368,7 +336,8 @@ impl Render for ProfileDialog {
                     .w_full()
                     .items_center()
                     .justify_center()
-                    .bg(cx.theme().sidebar)
+                    .rounded(cx.theme().radius_lg)
+                    .bg(cx.theme().muted)
                     .child(Avatar::new(avatar).seed(profile.avatar_seed()).large())
                     .when(self.editing, |this| {
                         this.child(
@@ -393,25 +362,13 @@ impl Render for ProfileDialog {
                         .justify_center()
                         .gap_3()
                         .rounded(cx.theme().radius)
-                        .bg(cx.theme().sidebar)
+                        .bg(cx.theme().muted)
                         .text_xs()
                         .child(shorten_pkey)
-                        .child(
+                        .child(Clipboard::new("copy").value_fn({
-                            Button::new("copy")
+                            let public_key = self.public_key;
-                                .icon({
+                            move |_, _| public_key.to_bech32().unwrap_or_default().into()
-                                    if self.copied {
+                        })),
-                                        IconName::CircleCheck
-                                    } else {
-                                        IconName::Copy
-                                    }
-                                })
-                                .xsmall()
-                                .secondary()
-                                .on_click(cx.listener(move |this, _ev, window, cx| {
-                                    let pk = this.public_key.to_bech32().unwrap();
-                                    this.copy(pk, window, cx)
-                                })),
-                        ),
                 ),
             )
             .when(self.editing, |this| {
@@ -544,7 +501,7 @@ fn field(
     };
 
     v_flex()
-        .gap_1p5()
+        .gap_1()
         .child(label(text, cx))
-        .child(div().text_xs().text_color(color).child(content))
+        .child(div().text_sm().text_color(color).child(content))
 }

crates/workspace/src/dialogs/relays.rs

@@ -264,7 +264,7 @@ impl RelayManager {
 
     fn render_tab_bar(&self, cx: &mut Context<Self>) -> impl IntoElement {
         TabBar::new("relays-tabs")
-            .segmented()
+            .pill()
             .selected_index(self.active_tab)
             .on_click(cx.listener(|this, ix: &usize, _window, cx| {
                 this.active_tab = *ix;

crates/workspace/src/dialogs/settings.rs

@@ -1,19 +1,30 @@
 #[cfg(not(target_arch = "wasm32"))]
+use std::time::Duration;
+
+use chat_ui::ToggleDictation;
+#[cfg(not(target_arch = "wasm32"))]
 use gpui::PathPromptOptions;
+#[cfg(not(target_arch = "wasm32"))]
+use gpui::Task;
 use gpui::http_client::Url;
 use gpui::{
-    App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, Subscription,
+    AnyElement, App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled,
-    Window, div,
+    Subscription, Window, div,
 };
 use gpui_kit::assets::IconName;
 use gpui_kit::component::button::{Button, ButtonVariants};
 use gpui_kit::component::input::{Input, InputEvent, InputState, NumberInput};
+use gpui_kit::component::kbd::Kbd;
 use gpui_kit::component::menu::{DropdownMenu, PopupMenuItem};
 use gpui_kit::component::notification::Notification;
+#[cfg(not(target_arch = "wasm32"))]
+use gpui_kit::component::progress::Progress;
 use gpui_kit::component::separator::Separator;
 use gpui_kit::component::switch::Switch;
 use gpui_kit::component::{Disableable, Sizable, WindowExt, h_flex, v_flex};
 use settings::{AppSettings, AppearanceMode};
+#[cfg(not(target_arch = "wasm32"))]
+use speech::model::{self as speech_model};
 use ui::{settings_block, settings_row};
 
 /// Bounds for the theme number inputs
@@ -29,6 +40,8 @@ pub struct Preferences {
     font_size_input: Entity<InputState>,
     radius_input: Entity<InputState>,
     radius_lg_input: Entity<InputState>,
+    #[cfg(not(target_arch = "wasm32"))]
+    _speech_poll: Task<()>,
     /// Keeps the control subscriptions alive while the dialog is open
     _subscriptions: Vec<Subscription>,
 }
@@ -94,11 +107,40 @@ impl Preferences {
             }),
         ];
 
+        #[cfg(not(target_arch = "wasm32"))]
+        let _speech_poll = cx.spawn_in(window, async move |this, cx| {
+            let timer = Duration::from_millis(500);
+            let mut last = speech_model::status();
+
+            loop {
+                cx.background_executor().timer(timer).await;
+
+                let status = speech_model::status();
+                let error = speech_model::take_error();
+
+                if status == last {
+                    continue;
+                }
+
+                last = status;
+
+                this.update_in(cx, |_, window, cx| {
+                    if let Some(error) = error {
+                        window.push_notification(Notification::error(error).autohide(false), cx);
+                    }
+                    cx.notify();
+                })
+                .ok();
+            }
+        });
+
         Self {
             file_input,
             font_size_input,
             radius_input,
             radius_lg_input,
+            #[cfg(not(target_arch = "wasm32"))]
+            _speech_poll,
             _subscriptions: subscriptions,
         }
     }
@@ -188,10 +230,102 @@ impl Preferences {
             .detach();
         }
     }
+
+    fn render_speech(
+        &mut self,
+        #[cfg_attr(target_arch = "wasm32", allow(unused_variables))] cx: &mut Context<Self>,
+    ) -> AnyElement {
+        #[cfg(target_arch = "wasm32")]
+        {
+            return div().into_any_element();
+        }
+
+        #[cfg(not(target_arch = "wasm32"))]
+        {
+            let size = speech_model::total_size() / 1_000_000;
+            let status = speech_model::status();
+
+            let (description, control) = match &status {
+                speech_model::Status::NotInstalled => (
+                    format!("{size} MB download"),
+                    Button::new("download-model")
+                        .label("Download")
+                        .secondary()
+                        .small()
+                        .on_click(|_, _window, cx| {
+                            speech_model::start_download();
+                            cx.refresh_windows();
+                        })
+                        .into_any_element(),
+                ),
+                speech_model::Status::Downloading { downloaded, total } => (
+                    format!(
+                        "Downloading · {} of {} MB",
+                        downloaded / 1_000_000,
+                        total / 1_000_000
+                    ),
+                    Button::new("cancel-download")
+                        .label("Cancel")
+                        .ghost()
+                        .small()
+                        .on_click(|_, _window, cx| {
+                            speech_model::cancel_download();
+                            cx.refresh_windows();
+                        })
+                        .into_any_element(),
+                ),
+                speech_model::Status::Installed => (
+                    format!("Parakeet v3 · {size} MB"),
+                    Button::new("remove-model")
+                        .label("Remove")
+                        .secondary()
+                        .small()
+                        .on_click(|_, _window, cx| {
+                            speech_model::delete();
+                            cx.refresh_windows();
+                        })
+                        .into_any_element(),
+                ),
+            };
+
+            let mut block = v_flex().w_full().gap_3().child(settings_row(
+                "Speech model",
+                description,
+                control,
+                cx,
+            ));
+
+            if let speech_model::Status::Downloading { downloaded, total } = &status {
+                let (downloaded, total) = (*downloaded, *total);
+
+                block = block.child(
+                    Progress::new("speech-model-download")
+                        .value(downloaded as f32 / total.max(1) as f32 * 100.)
+                        .small(),
+                );
+            }
+
+            block.into_any_element()
+        }
+    }
+
+    /// Shows the keystroke actually bound to the dictation action at startup
+    fn render_dictation_shortcut(
+        &self,
+        window: &mut Window,
+        cx: &mut Context<Self>,
+    ) -> impl IntoElement {
+        let control = match Kbd::global_binding_for_action(&ToggleDictation, window) {
+            Some(kbd) => kbd.into_any_element(),
+            None => div().into_any_element(),
+        };
+
+        settings_row("Shortcut", "Press to start or stop dictation", control, cx)
+    }
 }
 
 impl Render for Preferences {
-    fn render(&mut self, _window: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
+    fn render(&mut self, window: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
         const SCREENING: &str = "Show an screening dialog to verify the unknown sender.";
         const WOT: &str = "Treat people followed by your contacts as known senders.";
         const AVATAR: &str = "Hide all avatar pictures to improve performance.";
@@ -374,6 +508,9 @@ impl Render for Preferences {
                 cx,
             ))
             .child(Separator::horizontal())
+            .child(self.render_speech(cx))
+            .child(self.render_dictation_shortcut(window, cx))
+            .child(Separator::horizontal())
             .child(settings_block(
                 "Media Upload Service",
                 MEDIA,

crates/workspace/src/panels/backup.rs

@@ -1,16 +1,14 @@
 use anyhow::Error;
 use gpui::{
-    App, AppContext, ClipboardItem, Context, Entity, EventEmitter, FocusHandle, Focusable,
+    App, AppContext, Context, Entity, EventEmitter, FocusHandle, Focusable, IntoElement,
-    IntoElement, ParentElement, Render, SharedString, Styled, Task, Window, div,
+    ParentElement, Render, SharedString, Styled, Task, Window, div,
 };
-use gpui_kit::assets::IconName;
 use gpui_kit::base::StyledExt as _;
-use gpui_kit::component::button::{Button, ButtonVariants};
+use gpui_kit::component::clipboard::Clipboard;
 use gpui_kit::component::dock::{Panel, PanelEvent};
 use gpui_kit::component::input::{Input, InputState};
 use gpui_kit::component::separator::Separator;
-use gpui_kit::component::{ActiveTheme, Sizable, v_flex};
+use gpui_kit::component::{ActiveTheme, v_flex};
-use instant::Duration;
 use nostr_sdk::prelude::*;
 use state::USER_KEYRING;
 
@@ -32,9 +30,6 @@ pub struct BackupPanel {
     /// Secret key input
     nsec_input: Entity<InputState>,
 
-    /// Copied status
-    copied: bool,
-
     /// Background tasks
     tasks: Vec<Task<Result<(), Error>>>,
 }
@@ -54,7 +49,6 @@ impl BackupPanel {
             focus_handle: cx.focus_handle(),
             npub_input,
             nsec_input,
-            copied: false,
             tasks: vec![],
         }
     }
@@ -81,31 +75,6 @@ impl BackupPanel {
             Ok(())
         }));
     }
-
-    fn copy_secret(&mut self, cx: &mut Context<Self>) {
-        let value = self.nsec_input.read(cx).value();
-        let item = ClipboardItem::new_string(value.to_string());
-        cx.write_to_clipboard(item);
-
-        // Set the copied status to true
-        self.set_copied(true, cx);
-    }
-
-    fn set_copied(&mut self, status: bool, cx: &mut Context<Self>) {
-        self.copied = status;
-        cx.notify();
-
-        self.tasks.push(cx.spawn(async move |this, cx| {
-            cx.background_executor().timer(Duration::from_secs(2)).await;
-
-            // Clear the error message after a delay
-            this.update(cx, |this, cx| {
-                this.set_copied(false, cx);
-            })?;
-
-            Ok(())
-        }));
-    }
 }
 
 impl gpui_kit::component::dock::BasePanel for BackupPanel {
@@ -174,20 +143,12 @@ impl Render for BackupPanel {
                             .child(Input::new(&self.nsec_input).disabled(true)),
                     )
                     .child(
-                        Button::new("copy")
+                        Clipboard::new("copy-secret")
-                            .icon(IconName::Copy)
+                            .accessibility_label("Copy secret key")
-                            .label({
+                            .value_fn({
-                                if self.copied {
+                                let nsec_input = self.nsec_input.clone();
-                                    "Copied"
+                                move |_, cx| nsec_input.read(cx).value()
-                                } else {
+                            }),
-                                    "Copy secret key"
-                                }
-                            })
-                            .primary()
-                            .small()
-                            .on_click(cx.listener(move |this, _ev, _window, cx| {
-                                this.copy_secret(cx);
-                            })),
                     ),
             )
     }

crates/workspace/src/sidebar/mod.rs

@@ -340,7 +340,7 @@ impl Sidebar {
             )
             .child(
                 div()
-                    .text_sm()
+                    .text_xs()
                     .text_color(cx.theme().sidebar_foreground)
                     .child(label),
             )

desktop/Cargo.toml

@@ -26,6 +26,11 @@ icons = [
     "resources/icon.ico",
 ]
 
+# Merged over the generated Info.plist; required for the speech input to
+# request microphone and speech recognition access on macOS.
+[package.metadata.packager.macos]
+info_plist_path = "resources/Info.plist"
+
 [dependencies]
 assets = { path = "../crates/assets" }
 workspace = { path = "../crates/workspace" }
@@ -33,6 +38,7 @@ ui = { path = "../crates/ui" }
 state = { path = "../crates/state" }
 nip4e = { path = "../crates/nip4e" }
 chat = { path = "../crates/chat" }
+chat_ui = { path = "../crates/chat_ui" }
 nip29 = { path = "../crates/nip29" }
 settings = { path = "../crates/settings" }
 auto_update = { path = "../crates/auto_update" }

desktop/resources/Info.plist

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>NSMicrophoneUsageDescription</key>
+    <string>Dictate text into messages.</string>
+    <key>NSSpeechRecognitionUsageDescription</key>
+    <string>Turn your speech into text.</string>
+  </dict>
+</plist>

desktop/src/main.rs

@@ -1,5 +1,6 @@
 use std::sync::{Arc, Mutex};
 
+use chat_ui::ToggleDictation;
 use gpui::{
     App, Bounds, Global, KeyBinding, Menu, MenuItem, SharedString, TitlebarOptions,
     WindowBackgroundAppearance, WindowBounds, WindowDecorations, WindowKind, WindowOptions,
@@ -45,13 +46,19 @@ fn main() {
             // Register the `quit` function
             cx.on_action(quit);
 
-            // Register the `quit` function with CMD+Q (macOS)
+            // Register the `quit` function with CMD+Q (macOS) and dictation with CMD+D
             #[cfg(target_os = "macos")]
-            cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]);
+            cx.bind_keys([
+                KeyBinding::new("cmd-q", Quit, None),
+                KeyBinding::new("cmd-d", ToggleDictation, None),
+            ]);
 
-            // Register the `quit` function with Super+Q (others)
+            // Register the `quit` function with Super+Q (others) and dictation with Ctrl+D
             #[cfg(not(target_os = "macos"))]
-            cx.bind_keys([KeyBinding::new("super-q", Quit, None)]);
+            cx.bind_keys([
+                KeyBinding::new("super-q", Quit, None),
+                KeyBinding::new("ctrl-d", ToggleDictation, None),
+            ]);
 
             // Set menu items
             cx.set_menus(vec![Menu {

web/Cargo.toml

@@ -14,6 +14,7 @@ ui = { path = "../crates/ui" }
 state = { path = "../crates/state" }
 nip4e = { path = "../crates/nip4e" }
 chat = { path = "../crates/chat" }
+chat_ui = { path = "../crates/chat_ui" }
 nip29 = { path = "../crates/nip29" }
 settings = { path = "../crates/settings" }
 person = { path = "../crates/person" }

web/src/lib.rs

@@ -78,6 +78,11 @@ pub fn run() -> Result<(), JsValue> {
 
     APPLICATION.with(|application| {
         *application.borrow_mut() = Some(app.run_embedded(|cx| {
+            cx.bind_keys([
+                KeyBinding::new("cmd-d", chat_ui::ToggleDictation, None),
+                KeyBinding::new("ctrl-d", chat_ui::ToggleDictation, None),
+            ]);
+
             gpui_kit::init(cx);
 
             load_embedded_fonts(cx);