From 6fdbedac147d81cf6f2106e148130e137da4b9bb Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Tue, 27 Oct 2026 09:17:11 +0700 Subject: [PATCH 1/5] clean up --- assets/icons/paperclip.svg | 3 ++ crates/assets/tests/icons.rs | 39 -------------- crates/chat_ui/src/lib.rs | 40 +++++++++----- crates/nip29_ui/src/lib.rs | 15 ++++-- crates/ui/src/icon.rs | 2 + crates/ui/src/message.rs | 21 +++----- crates/workspace/src/dialogs/new_chat.rs | 6 +-- crates/workspace/src/dialogs/profile.rs | 67 +++++------------------- crates/workspace/src/dialogs/relays.rs | 2 +- crates/workspace/src/panels/backup.rs | 59 ++++----------------- crates/workspace/src/sidebar/mod.rs | 2 +- 11 files changed, 77 insertions(+), 179 deletions(-) create mode 100644 assets/icons/paperclip.svg delete mode 100644 crates/assets/tests/icons.rs diff --git a/assets/icons/paperclip.svg b/assets/icons/paperclip.svg new file mode 100644 index 00000000..b1e60469 --- /dev/null +++ b/assets/icons/paperclip.svg @@ -0,0 +1,3 @@ + + + diff --git a/crates/assets/tests/icons.rs b/crates/assets/tests/icons.rs deleted file mode 100644 index f9be2375..00000000 --- a/crates/assets/tests/icons.rs +++ /dev/null @@ -1,39 +0,0 @@ -use gpui::AssetSource; - -#[test] -fn icons_resolve_from_the_vendored_folder() { - // Icons coop's code and the kit components draw, served by the vendored - // folder (signed's set, filled with the kit bundle where coop needs more). - let vendored = [ - "icons/search.svg", - "icons/close.svg", - "icons/panel-left.svg", - "icons/window-minimize.svg", - "icons/settings-2.svg", - "icons/x.svg", - "icons/compass.svg", - "icons/radio-tower.svg", - ]; - for path in vendored { - let own = assets::Assets - .load(path) - .unwrap_or_else(|e| panic!("{path} errored in own embed: {e}")); - assert!(own.is_some(), "own embed is missing {path}"); - - let combined = assets::CombinedAssets.load(path).unwrap(); - assert!(combined.is_some(), "CombinedAssets fails to serve {path}"); - } - - // Coop's own files still load through the combined source. - let own = [ - "brand/coop.svg", - "brand/bg1.jpg", - "fonts/Inter/Inter-Medium.ttf", - ]; - for path in own { - assert!( - matches!(assets::CombinedAssets.load(path), Ok(Some(_))), - "CombinedAssets fails to serve coop asset {path}" - ); - } -} diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index 28e0f739..e339002b 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -1390,21 +1390,26 @@ impl ChatPanel { } let reply_id = *id; - let copy_id = *id; let dropdown_id = *id; let public_key = *public_key; - actions - .reply(cx.listener(move |this, _event, _window, cx| { - this.reply_to(&reply_id, cx); - })) - .copy(cx.listener(move |this, _event, _window, cx| { - this.copy_message(©_id, cx); - })) - .dropdown(move |menu, _window, _cx| { - menu.menu("Copy author", Box::new(Command::Copy(public_key))) - .menu("Seen on", Box::new(Command::Trace(dropdown_id))) - }) + let actions = actions.reply(cx.listener(move |this, _event, _window, cx| { + this.reply_to(&reply_id, cx); + })); + + let actions = if let Some(message) = self.message(id) { + actions.copy( + SharedString::from(format!("message-copy-{id}")), + message.preview(), + ) + } else { + actions + }; + + actions.dropdown(move |menu, _window, _cx| { + menu.menu("Copy author", Box::new(Command::Copy(public_key))) + .menu("Seen on", Box::new(Command::Trace(dropdown_id))) + }) } fn render_attachment(&self, url: &Url, cx: &Context) -> impl IntoElement { @@ -1879,7 +1884,7 @@ impl Render for ChatPanel { .items_end() .child( Button::new("upload") - .icon(IconName::Plus) + .icon(IconName::Paperclip) .tooltip("Upload media") .loading(self.uploading) .disabled(self.uploading) @@ -1888,7 +1893,14 @@ impl Render for ChatPanel { this.upload(window, cx); })), ) - .child(Textarea::new(&self.input).appearance(false).flex_1()) + .child( + Textarea::new(&self.input) + .appearance(false) + .small() + .min_h_8() + .py_1() + .flex_1(), + ) .child( h_flex() .pl_1() diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 2d2b187a..cb2dbbd4 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -1055,8 +1055,8 @@ impl GroupPanel { .items_end() .child( Button::new("upload") - .icon(IconName::Upload) - .tooltip("Upload file") + .icon(ui::IconName::Paperclip) + .tooltip("Upload media") .loading(self.uploading) .disabled(self.uploading) .ghost() @@ -1064,12 +1064,21 @@ impl GroupPanel { this.upload(window, cx); })), ) - .child(Textarea::new(&self.input).appearance(false).flex_1()) + .child( + Textarea::new(&self.input) + .appearance(false) + .bg(cx.theme().input) + .small() + .min_h_8() + .py_1() + .flex_1(), + ) .child( Button::new("send") .icon(ui::IconName::PaperPlaneFill) .tooltip("Send") .ghost() + .disabled(self.uploading) .on_click(cx.listener(|this, _event, window, cx| { this.send(window, cx); })), diff --git a/crates/ui/src/icon.rs b/crates/ui/src/icon.rs index 20b9effb..2d723053 100644 --- a/crates/ui/src/icon.rs +++ b/crates/ui/src/icon.rs @@ -20,6 +20,7 @@ pub enum IconName { Link, Lock, Message, + Paperclip, PaperPlaneFill, Plus, PlusChat, @@ -51,6 +52,7 @@ impl IconNamed for IconName { Self::Link => "icons/link.svg", Self::Lock => "icons/lock.svg", Self::Message => "icons/message.svg", + Self::Paperclip => "icons/paperclip.svg", Self::PaperPlaneFill => "icons/paper-plane-fill.svg", Self::Plus => "icons/plus.svg", Self::PlusChat => "icons/plus-chat.svg", diff --git a/crates/ui/src/message.rs b/crates/ui/src/message.rs index 6c61bc98..745f5108 100644 --- a/crates/ui/src/message.rs +++ b/crates/ui/src/message.rs @@ -10,6 +10,7 @@ use gpui::{ }; use gpui_kit::assets::IconName; use gpui_kit::component::button::{Button, ButtonVariants as _}; +use gpui_kit::component::clipboard::Clipboard; use gpui_kit::component::menu::{DropdownMenu as _, PopupMenu, PopupMenuItem}; use gpui_kit::component::{ ActiveTheme, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex, v_flex, @@ -602,7 +603,7 @@ pub struct MessageActions { style: StyleRefinement, reactions: SmallVec<[(&'static str, SharedClickListener); 4]>, reply: Option, - copy: Option, + copy: Option<(ElementId, SharedString)>, dropdown: Option, } @@ -634,9 +635,10 @@ impl MessageActions { self } - /// Add the copy button. - pub fn copy(mut self, handler: impl Fn(&ClickEvent, &mut Window, &mut App) + 'static) -> Self { - self.copy = Some(Rc::new(handler)); + /// Add the copy button for the given value. + /// The id must be unique per bar, the clipboard component keys its copied state on it. + pub fn copy(mut self, id: impl Into, value: impl Into) -> Self { + self.copy = Some((id.into(), value.into())); self } @@ -707,15 +709,8 @@ impl RenderOnce for MessageActions { .on_click(move |event, window, cx| reply(event, window, cx)), ) }) - .when_some(copy, |this, copy| { - this.child( - Button::new("copy") - .icon(IconName::Copy) - .tooltip("Copy") - .small() - .ghost() - .on_click(move |event, window, cx| copy(event, window, cx)), - ) + .when_some(copy, |this, (id, value)| { + this.child(Clipboard::new(id).value(value).tooltip("Copy").small()) }) .when_some(dropdown, |this, dropdown| { this.child(div().flex_shrink_0().h_4().w_px().bg(cx.theme().border)) diff --git a/crates/workspace/src/dialogs/new_chat.rs b/crates/workspace/src/dialogs/new_chat.rs index e46bbf52..81d92ad0 100644 --- a/crates/workspace/src/dialogs/new_chat.rs +++ b/crates/workspace/src/dialogs/new_chat.rs @@ -10,7 +10,6 @@ use gpui::{ Window, div, px, uniform_list, }; use gpui_kit::assets::IconName; -use gpui_kit::base::StyledExt as _; use gpui_kit::component::button::{Button, ButtonVariants}; use gpui_kit::component::dock::{DockArea, DockPlacement, PanelHandle}; use gpui_kit::component::input::{Input, InputEvent, InputState}; @@ -205,6 +204,7 @@ impl NewChat { Ok(()) })); + return; } @@ -369,15 +369,13 @@ impl NewChat { .p_0p5() .rounded_full() .bg(cx.theme().secondary) - .text_xs() - .font_medium() .hover(|this| this.bg(cx.theme().primary)) .child( Avatar::new(profile.avatar()) .seed(profile.avatar_seed()) .xsmall(), ) - .child(div().max_w_32().truncate().child(profile.name())) + .child(div().max_w_32().truncate().text_xs().child(profile.name())) .on_click(cx.listener(move |this, _event, _window, cx| { this.toggle_member(&public_key, cx); })) diff --git a/crates/workspace/src/dialogs/profile.rs b/crates/workspace/src/dialogs/profile.rs index 82a67fd8..b3c83eb2 100644 --- a/crates/workspace/src/dialogs/profile.rs +++ b/crates/workspace/src/dialogs/profile.rs @@ -4,17 +4,17 @@ use anyhow::Error; use chat::{ChatRegistry, Room, RoomKind}; use gpui::prelude::FluentBuilder; use gpui::{ - App, AppContext, ClipboardItem, Context, Div, Entity, IntoElement, ParentElement, Render, - SharedString, Styled, Task, Window, div, retain_all, + App, AppContext, Context, Div, Entity, IntoElement, ParentElement, Render, SharedString, + Styled, Task, Window, div, retain_all, }; use gpui_kit::assets::IconName; use gpui_kit::base::StyledExt as _; use gpui_kit::component::button::{Button, ButtonVariants}; +use gpui_kit::component::clipboard::Clipboard; use gpui_kit::component::dock::DockPlacement; use gpui_kit::component::input::{Input, InputState, Textarea, TextareaState}; use gpui_kit::component::notification::Notification; use gpui_kit::component::{ActiveTheme, Disableable, Sizable, WindowExt, h_flex, v_flex}; -use instant::Duration; use nostr_sdk::prelude::*; use person::{Directory, Person, PersonRegistry, shorten_pubkey}; use settings::AppSettings; @@ -49,8 +49,6 @@ struct ProfileDialog { website_input: Entity, /// Uploading state uploading: bool, - /// Copied states - copied: bool, /// Updating state updating: bool, /// Tasks @@ -92,7 +90,6 @@ impl ProfileDialog { bio_input, website_input, uploading: false, - copied: false, updating: false, tasks: vec![], } @@ -141,35 +138,6 @@ impl ProfileDialog { cx.notify(); } - fn copy(&mut self, value: String, window: &mut Window, cx: &mut Context) { - let item = ClipboardItem::new_string(value); - cx.write_to_clipboard(item); - - self.set_copied(true, window, cx); - } - - fn set_copied(&mut self, status: bool, window: &mut Window, cx: &mut Context) { - self.copied = status; - cx.notify(); - - if status { - self.tasks.push(cx.spawn_in(window, async move |this, cx| { - cx.background_executor().timer(Duration::from_secs(2)).await; - - // Reset the copied state after a delay - cx.update(|window, cx| { - this.update(cx, |this, cx| { - this.set_copied(false, window, cx); - }) - .ok(); - }) - .ok(); - - Ok(()) - })); - } - } - fn set_uploading(&mut self, status: bool, cx: &mut Context) { self.uploading = status; cx.notify(); @@ -368,7 +336,8 @@ impl Render for ProfileDialog { .w_full() .items_center() .justify_center() - .bg(cx.theme().sidebar) + .rounded(cx.theme().radius_lg) + .bg(cx.theme().muted) .child(Avatar::new(avatar).seed(profile.avatar_seed()).large()) .when(self.editing, |this| { this.child( @@ -393,25 +362,13 @@ impl Render for ProfileDialog { .justify_center() .gap_3() .rounded(cx.theme().radius) - .bg(cx.theme().sidebar) + .bg(cx.theme().muted) .text_xs() .child(shorten_pkey) - .child( - Button::new("copy") - .icon({ - if self.copied { - IconName::CircleCheck - } else { - IconName::Copy - } - }) - .xsmall() - .secondary() - .on_click(cx.listener(move |this, _ev, window, cx| { - let pk = this.public_key.to_bech32().unwrap(); - this.copy(pk, window, cx) - })), - ), + .child(Clipboard::new("copy").value_fn({ + let public_key = self.public_key; + move |_, _| public_key.to_bech32().unwrap_or_default().into() + })), ), ) .when(self.editing, |this| { @@ -544,7 +501,7 @@ fn field( }; v_flex() - .gap_1p5() + .gap_1() .child(label(text, cx)) - .child(div().text_xs().text_color(color).child(content)) + .child(div().text_sm().text_color(color).child(content)) } diff --git a/crates/workspace/src/dialogs/relays.rs b/crates/workspace/src/dialogs/relays.rs index 4ec18ec2..620a3b94 100644 --- a/crates/workspace/src/dialogs/relays.rs +++ b/crates/workspace/src/dialogs/relays.rs @@ -264,7 +264,7 @@ impl RelayManager { fn render_tab_bar(&self, cx: &mut Context) -> impl IntoElement { TabBar::new("relays-tabs") - .segmented() + .pill() .selected_index(self.active_tab) .on_click(cx.listener(|this, ix: &usize, _window, cx| { this.active_tab = *ix; diff --git a/crates/workspace/src/panels/backup.rs b/crates/workspace/src/panels/backup.rs index 8a0bbcc5..1e20353a 100644 --- a/crates/workspace/src/panels/backup.rs +++ b/crates/workspace/src/panels/backup.rs @@ -1,16 +1,14 @@ use anyhow::Error; use gpui::{ - App, AppContext, ClipboardItem, Context, Entity, EventEmitter, FocusHandle, Focusable, - IntoElement, ParentElement, Render, SharedString, Styled, Task, Window, div, + App, AppContext, Context, Entity, EventEmitter, FocusHandle, Focusable, IntoElement, + ParentElement, Render, SharedString, Styled, Task, Window, div, }; -use gpui_kit::assets::IconName; use gpui_kit::base::StyledExt as _; -use gpui_kit::component::button::{Button, ButtonVariants}; +use gpui_kit::component::clipboard::Clipboard; use gpui_kit::component::dock::{Panel, PanelEvent}; use gpui_kit::component::input::{Input, InputState}; use gpui_kit::component::separator::Separator; -use gpui_kit::component::{ActiveTheme, Sizable, v_flex}; -use instant::Duration; +use gpui_kit::component::{ActiveTheme, v_flex}; use nostr_sdk::prelude::*; use state::USER_KEYRING; @@ -32,9 +30,6 @@ pub struct BackupPanel { /// Secret key input nsec_input: Entity, - /// Copied status - copied: bool, - /// Background tasks tasks: Vec>>, } @@ -54,7 +49,6 @@ impl BackupPanel { focus_handle: cx.focus_handle(), npub_input, nsec_input, - copied: false, tasks: vec![], } } @@ -81,31 +75,6 @@ impl BackupPanel { Ok(()) })); } - - fn copy_secret(&mut self, cx: &mut Context) { - let value = self.nsec_input.read(cx).value(); - let item = ClipboardItem::new_string(value.to_string()); - cx.write_to_clipboard(item); - - // Set the copied status to true - self.set_copied(true, cx); - } - - fn set_copied(&mut self, status: bool, cx: &mut Context) { - self.copied = status; - cx.notify(); - - self.tasks.push(cx.spawn(async move |this, cx| { - cx.background_executor().timer(Duration::from_secs(2)).await; - - // Clear the error message after a delay - this.update(cx, |this, cx| { - this.set_copied(false, cx); - })?; - - Ok(()) - })); - } } impl gpui_kit::component::dock::BasePanel for BackupPanel { @@ -174,20 +143,12 @@ impl Render for BackupPanel { .child(Input::new(&self.nsec_input).disabled(true)), ) .child( - Button::new("copy") - .icon(IconName::Copy) - .label({ - if self.copied { - "Copied" - } else { - "Copy secret key" - } - }) - .primary() - .small() - .on_click(cx.listener(move |this, _ev, _window, cx| { - this.copy_secret(cx); - })), + Clipboard::new("copy-secret") + .accessibility_label("Copy secret key") + .value_fn({ + let nsec_input = self.nsec_input.clone(); + move |_, cx| nsec_input.read(cx).value() + }), ), ) } diff --git a/crates/workspace/src/sidebar/mod.rs b/crates/workspace/src/sidebar/mod.rs index 640008f7..83b21fe5 100644 --- a/crates/workspace/src/sidebar/mod.rs +++ b/crates/workspace/src/sidebar/mod.rs @@ -340,7 +340,7 @@ impl Sidebar { ) .child( div() - .text_sm() + .text_xs() .text_color(cx.theme().sidebar_foreground) .child(label), ) -- 2.54.0 From 9fa4f4f2ff4827c73318e37974df51b1fc2e3934 Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Tue, 27 Oct 2026 09:52:28 +0700 Subject: [PATCH 2/5] add speech component --- Cargo.lock | 366 ++++++++++++++++++++++++++++++++++- crates/chat_ui/Cargo.toml | 2 +- crates/chat_ui/src/lib.rs | 54 +++++- crates/nip29_ui/Cargo.toml | 2 +- crates/nip29_ui/src/lib.rs | 28 ++- desktop/Cargo.toml | 5 + desktop/resources/Info.plist | 10 + 7 files changed, 448 insertions(+), 19 deletions(-) create mode 100644 desktop/resources/Info.plist diff --git a/Cargo.lock b/Cargo.lock index 0c20ab12..f61fef15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,6 +191,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -600,7 +622,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 8.0.0", "num-rational", "v_frame", ] @@ -676,6 +698,24 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efbd3e1070bbdf4cd88a75264e18e8a26f7cb5c6949eadf0ceb85fb159cf08f8" +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.11.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.3", + "shlex 1.3.0", + "syn 2.0.119", +] + [[package]] name = "bip39" version = "2.2.2" @@ -1001,7 +1041,22 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", ] [[package]] @@ -1130,6 +1185,17 @@ dependencies = [ "inout 0.2.5", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "cmake" version = "0.1.58" @@ -1521,6 +1587,26 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953" +dependencies = [ + "bindgen", +] + [[package]] name = "cosmic-text" version = "0.19.0" @@ -1545,6 +1631,29 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2 0.4.3", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1694,6 +1803,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.11.1" @@ -2774,8 +2889,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a363235cd4062ef5f0953fc096fc764ed6f53dfbd9f409f669f93ea3b1ff9" dependencies = [ "anyhow", + "block2 0.6.2", "chrono", "core-text", + "cpal", "enum-iterator", "gpui-base", "gpui-component-macros", @@ -2792,7 +2909,10 @@ dependencies = [ "num-traits", "objc2 0.6.5", "objc2-app-kit 0.3.2", + "objc2-av-foundation", + "objc2-avf-audio", "objc2-foundation 0.3.2", + "objc2-speech", "once_cell", "paste", "raw-window-handle", @@ -2946,7 +3066,7 @@ dependencies = [ "image", "libc", "log", - "mach2", + "mach2 0.6.0", "metal", "objc2 0.6.5", "objc2-core-foundation", @@ -4206,6 +4326,22 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -4580,6 +4716,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "mach2" version = "0.6.0" @@ -4802,6 +4947,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minisign-verify" version = "0.2.5" @@ -4935,12 +5086,35 @@ dependencies = [ "native-theme", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.13.2", + "jni-sys 0.3.1", + "log", + "ndk-sys 0.5.0+25.2.9519653", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -5034,6 +5208,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -5392,6 +5576,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "objc" version = "0.2.7" @@ -5475,6 +5681,27 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -5696,6 +5923,18 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-speech" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9eb609f9d2a25e0f8b76954f3acaa58348ce2a91285fe867643b2224ac4d263" +dependencies = [ + "block2 0.6.2", + "objc2 0.6.5", + "objc2-avf-audio", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-user-notifications" version = "0.3.2" @@ -5735,6 +5974,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni 0.21.1", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -7024,7 +7286,7 @@ checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.22.4", "log", "once_cell", "rustls", @@ -7516,6 +7778,12 @@ dependencies = [ "dirs 6.0.0", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -9109,7 +9377,7 @@ version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2c8ddfbbf66e501434657400268cefe51b95140a0266236415cda7627732df7" dependencies = [ - "jni", + "jni 0.22.4", "log", "ndk-context", "objc2 0.6.5", @@ -9278,7 +9546,7 @@ dependencies = [ "libloading", "log", "naga", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "objc2 0.6.5", "objc2-core-foundation", "objc2-foundation 0.3.2", @@ -9370,6 +9638,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.57.0" @@ -9446,6 +9724,16 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -9712,6 +10000,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -9739,6 +10036,21 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -9814,6 +10126,12 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -9832,6 +10150,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -9850,6 +10174,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -9880,6 +10210,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -9898,6 +10234,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -9916,6 +10258,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -9934,6 +10282,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/crates/chat_ui/Cargo.toml b/crates/chat_ui/Cargo.toml index 810ba59e..7c8b97c4 100644 --- a/crates/chat_ui/Cargo.toml +++ b/crates/chat_ui/Cargo.toml @@ -7,7 +7,7 @@ publish.workspace = true [dependencies] state = { path = "../state" } ui = { path = "../ui" } -gpui-kit.workspace = true +gpui-kit = { workspace = true, features = ["speech"] } common = { path = "../common" } person = { path = "../person" } chat = { path = "../chat" } diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index e339002b..4a338f32 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -20,6 +20,7 @@ use gpui_kit::component::input::{Input, InputEvent, InputState, Textarea, Textar use gpui_kit::component::menu::DropdownMenu; use gpui_kit::component::notification::Notification; use gpui_kit::component::scroll::Scrollbar; +use gpui_kit::component::speech::{SpeechButton, SpeechEvent, SpeechState, SpeechWaveform}; use gpui_kit::component::tooltip::Tooltip; use gpui_kit::component::{ActiveTheme, Disableable, Icon, Sizable, WindowExt, h_flex, v_flex}; use itertools::Itertools; @@ -91,6 +92,9 @@ pub struct ChatPanel { /// Chat input state input: Entity, + /// Dictation session for the chat input + speech: Entity, + /// Subject input state subject_input: Entity, @@ -116,7 +120,7 @@ pub struct ChatPanel { tasks: Vec>>, /// Event subscriptions - subscriptions: SmallVec<[Subscription; 3]>, + subscriptions: SmallVec<[Subscription; 4]>, } impl ChatPanel { @@ -160,6 +164,9 @@ impl ChatPanel { .clean_on_escape() }); + // Define dictation state + let speech = cx.new(SpeechState::new); + // Define subject input state let subject_input = cx.new(|cx| InputState::new(window, cx).placeholder("New subject...")); let subject_bar = cx.new(|_cx| false); @@ -178,15 +185,28 @@ impl ChatPanel { subscriptions.push( // Subscribe the subject input event - cx.subscribe_in( - &subject_input, - window, - move |this, _input, event, window, cx| { - if let InputEvent::PressEnter { .. } = event { - this.change_subject(window, cx); - }; - }, - ), + cx.subscribe_in(&subject_input, window, move |this, _, event, window, cx| { + if let InputEvent::PressEnter { .. } = event { + this.change_subject(window, cx); + }; + }), + ); + + subscriptions.push( + // Insert the dictated text into the chat input + cx.subscribe_in(&speech, window, move |this, _speech, event, window, cx| { + match event { + SpeechEvent::Final(text) if !text.is_empty() => { + this.input.update(cx, |input, cx| { + input.insert(text.clone(), window, cx); + }); + } + SpeechEvent::Error(error) => { + window.push_notification(Notification::error(error.to_string()), cx); + } + _ => {} + }; + }), ); // Define all functions that will run after the current cycle @@ -206,6 +226,7 @@ impl ChatPanel { directory, list_state, input, + speech, subject_input, subject_bar, replies_to, @@ -1807,6 +1828,7 @@ impl Render for ChatPanel { let is_typing = !self.input.read(cx).value().trim().is_empty(); let pending_attachments = !self.encrypted_attachments.read(cx).is_empty(); + let capturing = self.speech.read(cx).status().is_capturing(); v_flex() .image_cache(retain_all(self.id.clone())) @@ -1901,6 +1923,18 @@ impl Render for ChatPanel { .py_1() .flex_1(), ) + .child( + h_flex() + .items_end() + .gap_1() + .pl_1() + .when(capturing, |this| { + this.child( + SpeechWaveform::new(&self.speech).xsmall().w_12(), + ) + }) + .child(SpeechButton::new(&self.speech)), + ) .child( h_flex() .pl_1() diff --git a/crates/nip29_ui/Cargo.toml b/crates/nip29_ui/Cargo.toml index 7827e6ce..0d0783fc 100644 --- a/crates/nip29_ui/Cargo.toml +++ b/crates/nip29_ui/Cargo.toml @@ -12,7 +12,7 @@ nip29 = { path = "../nip29" } person = { path = "../person" } settings = { path = "../settings" } state = { path = "../state" } -gpui-kit.workspace = true +gpui-kit = { workspace = true, features = ["speech"] } ui = { path = "../ui" } gpui.workspace = true diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index cb2dbbd4..c89cd8e5 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -15,6 +15,7 @@ use gpui_kit::component::dock::{Panel, PanelEvent}; use gpui_kit::component::input::{InputEvent, Textarea, TextareaState}; use gpui_kit::component::notification::Notification; use gpui_kit::component::scroll::Scrollbar; +use gpui_kit::component::speech::{SpeechButton, SpeechEvent, SpeechState, SpeechWaveform}; use gpui_kit::component::{ActiveTheme, Disableable, Icon, Sizable, WindowExt, h_flex, v_flex}; use nip29::{Group, GroupEvent, GroupKey, GroupsRegistry, Membership, Pin, Row}; use nostr_sdk::prelude::*; @@ -312,6 +313,8 @@ pub struct GroupPanel { reply_to: Option, rendered_texts_by_id: BTreeMap, input: Entity, + /// Dictation session for the composer input + speech: Entity, /// Uploaded, non-encrypted attachments waiting to be sent. attachments: Vec, /// Whether an upload is in flight. @@ -319,7 +322,7 @@ pub struct GroupPanel { /// Whether the user closed the encryption notice this visit notice_dismissed: bool, tasks: SmallVec<[Task>; 4]>, - _subscriptions: SmallVec<[Subscription; 3]>, + _subscriptions: SmallVec<[Subscription; 4]>, } impl GroupPanel { @@ -338,6 +341,8 @@ impl GroupPanel { .clean_on_escape() }); + let speech = cx.new(SpeechState::new); + let mut subscriptions = smallvec![]; subscriptions.push( @@ -348,6 +353,22 @@ impl GroupPanel { }), ); + subscriptions.push( + cx.subscribe_in(&speech, window, |this, _, event, window, cx| { + match event { + SpeechEvent::Final(text) if !text.is_empty() => { + this.input.update(cx, |input, cx| { + input.insert(text.clone(), window, cx); + }); + } + SpeechEvent::Error(error) => { + window.push_notification(Notification::error(error.to_string()), cx); + } + _ => {} + }; + }), + ); + subscriptions.push(cx.subscribe(&group, |this, _group, event, cx| { if !matches!(event, GroupEvent::Sent(_)) { this.sync(cx); @@ -383,6 +404,7 @@ impl GroupPanel { notice_dismissed: false, rendered_texts_by_id: BTreeMap::new(), input, + speech, attachments: Vec::new(), uploading: false, tasks: smallvec![], @@ -1073,6 +1095,10 @@ impl GroupPanel { .py_1() .flex_1(), ) + .when(self.speech.read(cx).status().is_capturing(), |this| { + this.child(SpeechWaveform::new(&self.speech).xsmall().w_12()) + }) + .child(SpeechButton::new(&self.speech)) .child( Button::new("send") .icon(ui::IconName::PaperPlaneFill) diff --git a/desktop/Cargo.toml b/desktop/Cargo.toml index adf1d030..781e8b11 100644 --- a/desktop/Cargo.toml +++ b/desktop/Cargo.toml @@ -26,6 +26,11 @@ icons = [ "resources/icon.ico", ] +# Merged over the generated Info.plist; required for the speech input to +# request microphone and speech recognition access on macOS. +[package.metadata.packager.macos] +info_plist_path = "resources/Info.plist" + [dependencies] assets = { path = "../crates/assets" } workspace = { path = "../crates/workspace" } diff --git a/desktop/resources/Info.plist b/desktop/resources/Info.plist new file mode 100644 index 00000000..c0bfc62a --- /dev/null +++ b/desktop/resources/Info.plist @@ -0,0 +1,10 @@ + + + + + NSMicrophoneUsageDescription + Dictate text into messages. + NSSpeechRecognitionUsageDescription + Turn your speech into text. + + -- 2.54.0 From 589c43b9825969817cfb3ebbd2bf4e4b17c29b99 Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Tue, 27 Oct 2026 10:55:05 +0700 Subject: [PATCH 3/5] add parakeet --- Cargo.lock | 565 ++++++++++++++++++++++- crates/chat_ui/Cargo.toml | 1 + crates/chat_ui/src/lib.rs | 13 +- crates/nip29_ui/Cargo.toml | 1 + crates/nip29_ui/src/lib.rs | 13 +- crates/speech/Cargo.toml | 29 ++ crates/speech/src/lib.rs | 26 ++ crates/speech/src/model.json | 23 + crates/speech/src/model.rs | 295 ++++++++++++ crates/speech/src/parakeet.rs | 214 +++++++++ crates/speech/src/web.rs | 256 ++++++++++ crates/workspace/Cargo.toml | 1 + crates/workspace/src/dialogs/settings.rs | 111 ++++- 13 files changed, 1527 insertions(+), 21 deletions(-) create mode 100644 crates/speech/Cargo.toml create mode 100644 crates/speech/src/lib.rs create mode 100644 crates/speech/src/model.json create mode 100644 crates/speech/src/model.rs create mode 100644 crates/speech/src/parakeet.rs create mode 100644 crates/speech/src/web.rs diff --git a/Cargo.lock b/Cargo.lock index f61fef15..7a6bc562 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,7 +153,9 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", + "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -606,7 +608,7 @@ dependencies = [ "log", "num-rational", "num-traits", - "pastey", + "pastey 0.1.1", "rayon", "thiserror 2.0.21", "v_frame", @@ -680,6 +682,12 @@ version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3a647dce9631f152a99ec16cfc09267ae55fe280d5dc288964027d458076323" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -692,6 +700,12 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bech32" version = "0.12.0" @@ -707,7 +721,7 @@ dependencies = [ "bitflags 2.13.2", "cexpr", "clang-sys", - "itertools 0.11.0", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -1005,6 +1019,15 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -1145,6 +1168,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] @@ -1222,7 +1246,7 @@ dependencies = [ "cocoa-foundation 0.1.2", "core-foundation 0.9.4", "core-graphics 0.23.2", - "foreign-types", + "foreign-types 0.5.0", "libc", "objc", ] @@ -1238,7 +1262,7 @@ dependencies = [ "cocoa-foundation 0.2.1", "core-foundation 0.10.1", "core-graphics 0.24.0", - "foreign-types", + "foreign-types 0.5.0", "libc", "objc", ] @@ -1314,6 +1338,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1508,7 +1547,7 @@ dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", "core-graphics-types 0.1.3", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1521,7 +1560,7 @@ dependencies = [ "bitflags 2.13.2", "core-foundation 0.10.1", "core-graphics-types 0.2.0", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1534,7 +1573,7 @@ dependencies = [ "bitflags 2.13.2", "core-foundation 0.9.4", "core-graphics-types 0.1.3", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1568,7 +1607,7 @@ checksum = "a593227b66cbd4007b2a050dfdd9e1d1318311409c8d600dc82ba1b15ca9c130" dependencies = [ "core-foundation 0.10.1", "core-graphics 0.24.0", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1769,14 +1808,44 @@ dependencies = [ "cmov", ] +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -1792,17 +1861,37 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core", + "darling_core 0.24.1", "quote", "syn 3.0.9", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -1875,6 +1964,16 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "der" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.9" @@ -1884,6 +1983,37 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -2200,6 +2330,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etagere" version = "0.2.15" @@ -2256,6 +2392,17 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "eyre" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" +dependencies = [ + "autocfg", + "indenter", + "once_cell", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -2463,6 +2610,15 @@ dependencies = [ "ttf-parser", ] +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -2470,7 +2626,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -2484,6 +2640,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -3058,7 +3220,7 @@ dependencies = [ "derive_more", "dispatch2", "etagere", - "foreign-types", + "foreign-types 0.5.0", "futures", "gpui-pre", "gpui-pre-collections", @@ -3197,7 +3359,7 @@ dependencies = [ "core-graphics 0.24.0", "ctor", "dispatch2", - "foreign-types", + "foreign-types 0.5.0", "futures", "gpui-pre", "gpui-pre-apple", @@ -3750,6 +3912,18 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "hmac-sha256" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad320b3b96fb2a455a0726d16efe0a5afdbd34b71dea5bc53b05ea057714d4e" + +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "html5ever" version = "0.27.0" @@ -4086,6 +4260,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + [[package]] name = "indexed_db_futures" version = "0.6.4" @@ -4696,6 +4876,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + [[package]] name = "mac" version = "0.1.1" @@ -4731,6 +4917,22 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey 0.2.3", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "macroific" version = "2.0.0" @@ -4872,6 +5074,16 @@ dependencies = [ "xml5ever", ] +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -4925,7 +5137,7 @@ dependencies = [ "bitflags 2.13.2", "block", "core-graphics-types 0.2.0", - "foreign-types", + "foreign-types 0.5.0", "log", "objc", "paste", @@ -4991,6 +5203,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -5086,6 +5320,38 @@ dependencies = [ "native-theme", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.8.0" @@ -5171,6 +5437,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] @@ -6003,6 +6270,28 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags 2.13.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "oo7" version = "0.6.0" @@ -6066,12 +6355,49 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -6097,6 +6423,30 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "page_size" version = "0.6.0" @@ -6107,6 +6457,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "parakeet-rs" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f54545a861dca6b43c8d44e3dd932764ca196d84bcf0d1b48ea212b43b29666" +dependencies = [ + "eyre", + "hound", + "ndarray", + "ort", + "realfft", + "serde", + "serde_json", + "tokenizers", +] + [[package]] name = "parking" version = "2.2.1" @@ -6148,6 +6514,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pathfinder_geometry" version = "0.5.1" @@ -6187,6 +6559,15 @@ dependencies = [ "hmac 0.13.0", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -6488,6 +6869,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -6829,6 +7219,12 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -6839,6 +7235,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -6871,6 +7278,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "realfft" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "538895ccf2006b473aafbe8d72770e21038961ab67aae34f3c577485fc703a7d" +dependencies = [ + "rustfft", +] + [[package]] name = "reborrow" version = "0.5.5" @@ -6970,7 +7386,9 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "futures-channel", "futures-core", + "futures-util", "h2", "http", "http-body", @@ -7218,6 +7636,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.5" @@ -7698,7 +8130,7 @@ version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ - "darling", + "darling 0.24.1", "proc-macro2", "quote", "syn 3.0.9", @@ -7933,6 +8365,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "speech" +version = "2.0.0-alpha2" +dependencies = [ + "anyhow", + "common", + "flume 0.11.1", + "gpui-kit", + "gpui-pre", + "js-sys", + "log", + "ort", + "parakeet-rs", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "spin" version = "0.9.9" @@ -7960,6 +8413,18 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "sqlite-wasm-rs" version = "0.5.5" @@ -8044,6 +8509,12 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" @@ -8455,6 +8926,39 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tokenizers" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.21", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.54.1" @@ -8760,6 +9264,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "triomphe" version = "0.1.17" @@ -8889,6 +9403,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -8925,6 +9448,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "universal-hash" version = "0.5.1" @@ -8960,7 +9489,9 @@ checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" dependencies = [ "base64 0.23.1", "cookie_store", + "der", "log", + "native-tls", "percent-encoding", "rustls", "rustls-pki-types", @@ -8969,6 +9500,7 @@ dependencies = [ "serde_json", "ureq-proto", "utf8-zero", + "webpki-root-certs", "webpki-roots 1.0.11", ] @@ -10373,6 +10905,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] diff --git a/crates/chat_ui/Cargo.toml b/crates/chat_ui/Cargo.toml index 7c8b97c4..7f5b25a0 100644 --- a/crates/chat_ui/Cargo.toml +++ b/crates/chat_ui/Cargo.toml @@ -6,6 +6,7 @@ publish.workspace = true [dependencies] state = { path = "../state" } +speech = { path = "../speech" } ui = { path = "../ui" } gpui-kit = { workspace = true, features = ["speech"] } common = { path = "../common" } diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index 4a338f32..73ce1548 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, LazyLock, RwLock}; +use std::time::Duration; pub use actions::*; use anyhow::Error; @@ -165,7 +166,17 @@ impl ChatPanel { }); // Define dictation state - let speech = cx.new(SpeechState::new); + let speech = cx.new(|cx| { + let state = SpeechState::new(cx) + .recognizer(speech::recognizer()) + // Batch transcription of a long recording can take a while. + .stop_timeout(Duration::from_secs(180)); + + #[cfg(target_arch = "wasm32")] + let state = state.input(speech::input()); + + state + }); // Define subject input state let subject_input = cx.new(|cx| InputState::new(window, cx).placeholder("New subject...")); diff --git a/crates/nip29_ui/Cargo.toml b/crates/nip29_ui/Cargo.toml index 0d0783fc..451c1d50 100644 --- a/crates/nip29_ui/Cargo.toml +++ b/crates/nip29_ui/Cargo.toml @@ -11,6 +11,7 @@ common = { path = "../common" } nip29 = { path = "../nip29" } person = { path = "../person" } settings = { path = "../settings" } +speech = { path = "../speech" } state = { path = "../state" } gpui-kit = { workspace = true, features = ["speech"] } ui = { path = "../ui" } diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index c89cd8e5..331c2805 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::time::Duration; use anyhow::Result; use gpui::prelude::FluentBuilder; @@ -341,7 +342,17 @@ impl GroupPanel { .clean_on_escape() }); - let speech = cx.new(SpeechState::new); + let speech = cx.new(|cx| { + let state = SpeechState::new(cx) + .recognizer(speech::recognizer()) + // Batch transcription of a long recording can take a while. + .stop_timeout(Duration::from_secs(180)); + + #[cfg(target_arch = "wasm32")] + let state = state.input(speech::input()); + + state + }); let mut subscriptions = smallvec![]; diff --git a/crates/speech/Cargo.toml b/crates/speech/Cargo.toml new file mode 100644 index 00000000..444cc638 --- /dev/null +++ b/crates/speech/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "speech" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +log.workspace = true +gpui.workspace = true +gpui-kit = { workspace = true, features = ["speech"] } +flume.workspace = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +common = { path = "../common" } +parakeet-rs = "=0.3.8" +ort = "=2.0.0-rc.13" +reqwest = { version = "0.12", default-features = false, features = [ + "blocking", + "rustls-tls", +] } +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true + +[target.'cfg(target_arch = "wasm32")'.dependencies] +js-sys = "0.3" +wasm-bindgen = "0.2" +web-sys = { version = "0.3", features = ["Window"] } diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs new file mode 100644 index 00000000..99014d77 --- /dev/null +++ b/crates/speech/src/lib.rs @@ -0,0 +1,26 @@ +#[cfg(not(target_arch = "wasm32"))] +pub mod model; +#[cfg(not(target_arch = "wasm32"))] +mod parakeet; +#[cfg(target_arch = "wasm32")] +mod web; + +use gpui_kit::component::speech::SpeechRecognizer; + +/// The recognizer backing the composer's microphone button. +#[cfg(not(target_arch = "wasm32"))] +pub fn recognizer() -> impl SpeechRecognizer { + parakeet::ParakeetRecognizer +} + +/// The recognizer backing the composer's microphone button. +#[cfg(target_arch = "wasm32")] +pub fn recognizer() -> impl SpeechRecognizer { + web::WebSpeechRecognizer +} + +/// The audio source backing the composer's microphone button on the web. +#[cfg(target_arch = "wasm32")] +pub fn input() -> impl gpui_kit::component::speech::AudioInput { + web::SelfCapturedInput +} diff --git a/crates/speech/src/model.json b/crates/speech/src/model.json new file mode 100644 index 00000000..50779f63 --- /dev/null +++ b/crates/speech/src/model.json @@ -0,0 +1,23 @@ +{ + "repository": "istupakov/parakeet-tdt-0.6b-v3-onnx", + "revision": "8f23f0c03c8761650bdb5b40aaf3e40d2c15f1ce", + "source": "nvidia/parakeet-tdt-0.6b-v3", + "license": "CC-BY-4.0", + "files": [ + { + "name": "decoder_joint-model.int8.onnx", + "size": 18202004, + "sha256": "eea7483ee3d1a30375daedc8ed83e3960c91b098812127a0d99d1c8977667a70" + }, + { + "name": "encoder-model.int8.onnx", + "size": 652183999, + "sha256": "6139d2fa7e1b086097b277c7149725edbab89cc7c7ae64b23c741be4055aff09" + }, + { + "name": "vocab.txt", + "size": 93939, + "sha256": "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d" + } + ] +} diff --git a/crates/speech/src/model.rs b/crates/speech/src/model.rs new file mode 100644 index 00000000..2abfbbc3 --- /dev/null +++ b/crates/speech/src/model.rs @@ -0,0 +1,295 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::{fs, io}; + +use anyhow::{Result, bail}; +use serde::Deserialize; + +/// How the model is currently available. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + NotInstalled, + Downloading { downloaded: u64, total: u64 }, + Installed, +} + +impl Status { + pub fn is_downloading(self) -> bool { + matches!(self, Status::Downloading { .. }) + } +} + +#[derive(Debug, Deserialize)] +struct Manifest { + repository: String, + revision: String, + files: Vec, +} + +#[derive(Debug, Deserialize)] +struct Artifact { + name: String, + size: u64, + sha256: String, +} + +fn manifest() -> &'static Manifest { + static MANIFEST: OnceLock = OnceLock::new(); + + MANIFEST.get_or_init(|| { + serde_json::from_str(include_str!("model.json")).expect("pinned model manifest") + }) +} + +/// The directory the model artifacts are downloaded into. +pub fn dir() -> PathBuf { + common::config_dir() + .join("speech") + .join("parakeet-tdt-0.6b-v3") +} + +/// Total download size in bytes. +pub fn total_size() -> u64 { + manifest().files.iter().map(|file| file.size).sum() +} + +/// The revision of the installed model, if its artifacts are complete. +fn installed_revision() -> Option { + let manifest = manifest(); + let dir = dir(); + + let complete = manifest.files.iter().all(|file| { + fs::metadata(dir.join(&file.name)).is_ok_and(|metadata| metadata.len() == file.size) + }); + + if complete + && fs::read_to_string(dir.join("verified")) + .is_ok_and(|revision| revision == manifest.revision) + { + Some(manifest.revision.clone()) + } else { + None + } +} + +/// Whether the model is ready to load. +pub fn installed() -> bool { + status() == Status::Installed +} + +/// The current status, reconciling the download state with the disk. +pub fn status() -> Status { + let state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if let Download::Running { + downloaded, total, .. + } = state.download + { + return Status::Downloading { downloaded, total }; + } + + if installed_revision().is_some() { + Status::Installed + } else { + Status::NotInstalled + } +} + +/// Take the last download failure, if it has not been reported yet. +pub fn take_error() -> Option { + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + state.error.take() +} + +/// Start downloading the model, unless a download is already running. +pub fn start_download() { + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if matches!(state.download, Download::Running { .. }) { + return; + } + + let cancel = Arc::new(AtomicBool::new(false)); + state.error = None; + state.download = Download::Running { + downloaded: 0, + total: total_size(), + cancel: cancel.clone(), + }; + + if let Err(error) = std::thread::Builder::new() + .name("speech-model-download".into()) + .spawn(move || download(cancel)) + { + state.download = Download::Idle; + state.error = Some(error.to_string()); + } +} + +/// Request cancellation of the running download. +pub fn cancel_download() { + let state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if let Download::Running { cancel, .. } = &state.download { + cancel.store(true, Ordering::Release); + } +} + +/// Delete the downloaded model and any partial download. +pub fn delete() { + super::parakeet::unload(); + cancel_download(); + + if let Err(error) = fs::remove_dir_all(dir()) + && error.kind() != io::ErrorKind::NotFound + { + log::warn!("failed to remove the speech model: {error}"); + } +} + +fn download(cancel: Arc) { + let result = download_inner(&cancel); + + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + state.download = Download::Idle; + + // A cancelled download must not report a spurious failure. + if !cancelled(&cancel) + && let Err(error) = result + { + state.error = Some(format!("{error:#}")); + } +} + +fn download_inner(cancel: &AtomicBool) -> Result<()> { + let manifest = manifest(); + let dir = dir(); + let client = reqwest::blocking::Client::builder() + .connect_timeout(std::time::Duration::from_secs(20)) + .build()?; + + fs::create_dir_all(&dir)?; + + for file in &manifest.files { + if cancelled(cancel) { + bail!("Cancelled"); + } + + let staging = dir.join(format!("{}.part", file.name)); + let result = download_file(&client, manifest, file, &staging, cancel); + + if result.is_err() || cancelled(cancel) { + let _unused = fs::remove_file(&staging); + result?; + bail!("Cancelled"); + } + } + + fs::write(dir.join("verified"), &manifest.revision)?; + Ok(()) +} + +fn download_file( + client: &reqwest::blocking::Client, + manifest: &Manifest, + file: &Artifact, + staging: &Path, + cancel: &AtomicBool, +) -> Result<()> { + use std::io::{Read as _, Write as _}; + + use sha2::Digest as _; + + let url = format!( + "https://huggingface.co/{}/resolve/{}/{}", + manifest.repository, manifest.revision, file.name + ); + + let mut response = client.get(url).send()?.error_for_status()?; + let mut out = fs::File::create(staging)?; + let mut hash = sha2::Sha256::new(); + let mut size = 0u64; + + loop { + if cancelled(cancel) { + bail!("Cancelled"); + } + + let mut chunk = [0u8; 64 * 1024]; + let read = response.read(&mut chunk)?; + + if read == 0 { + break; + } + + size += read as u64; + if size > file.size { + bail!("Unexpected model size"); + } + + hash.update(&chunk[..read]); + out.write_all(&chunk[..read])?; + + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + if let Download::Running { downloaded, .. } = &mut state.download { + *downloaded += read as u64; + } + } + + if size != file.size || format!("{:x}", hash.finalize()) != file.sha256 { + bail!("Model checksum verification failed"); + } + + out.sync_all()?; + fs::rename(staging, dir().join(&file.name))?; + Ok(()) +} + +fn cancelled(cancel: &AtomicBool) -> bool { + cancel.load(Ordering::Acquire) +} + +enum Download { + Idle, + Running { + downloaded: u64, + total: u64, + cancel: Arc, + }, +} + +struct State { + download: Download, + error: Option, +} + +fn state() -> &'static Mutex { + static STATE: Mutex = Mutex::new(State { + download: Download::Idle, + error: None, + }); + + &STATE +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manifest_pins_v3_artifacts_with_exact_sizes() { + let manifest = manifest(); + + assert_eq!(manifest.repository, "istupakov/parakeet-tdt-0.6b-v3-onnx"); + assert_eq!(manifest.revision.len(), 40); + assert_eq!(manifest.files.len(), 3); + assert_eq!(total_size(), 670_479_942); + assert!( + manifest + .files + .iter() + .all(|file| file.sha256.len() == 64 && !file.name.contains('/')) + ); + } +} diff --git a/crates/speech/src/parakeet.rs b/crates/speech/src/parakeet.rs new file mode 100644 index 00000000..6c19229b --- /dev/null +++ b/crates/speech/src/parakeet.rs @@ -0,0 +1,214 @@ +use std::path::PathBuf; +use std::sync::OnceLock; +use std::time::Duration; + +use anyhow::{Context as _, Result, anyhow}; +use gpui::{App, Task}; +use gpui_kit::component::speech::{ + AudioFormat, RecognitionSession, SpeechError, SpeechRecognizer, SpeechSink, +}; +use parakeet_rs::{ParakeetTDT, Transcriber as _}; + +const MODEL_RATE: u32 = 16_000; +const MAX_SECONDS: usize = 120; + +/// The on-device recognizer. +pub struct ParakeetRecognizer; + +impl SpeechRecognizer for ParakeetRecognizer { + fn audio_format(&self) -> AudioFormat { + AudioFormat::new(MODEL_RATE, 1) + } + + fn is_available(&self, _cx: &App) -> bool { + true + } + + fn start( + &self, + sink: SpeechSink, + cx: &mut App, + ) -> Result, SpeechError> { + if super::model::installed() { + worker().send_or_log(Command::Warm(super::model::dir())); + } else if super::model::status().is_downloading() { + return Err(SpeechError::recognizer(anyhow!( + "The speech model is still downloading. Try again when it finishes." + ))); + } else { + return Err(SpeechError::recognizer(anyhow!( + "No speech model is installed. Download it from Settings." + ))); + } + + // The session buffers everything it is pushed, + // so audio is accepted right away. + sink.ready(cx); + + Ok(Box::new(ParakeetSession { + sink, + samples: Vec::new(), + _result: None, + })) + } +} + +struct ParakeetSession { + sink: SpeechSink, + samples: Vec, + _result: Option>, +} + +impl RecognitionSession for ParakeetSession { + fn push_audio(&mut self, samples: &[i16], _cx: &mut App) { + if self.samples.len() < MAX_SECONDS * MODEL_RATE as usize { + self.samples + .extend(samples.iter().map(|sample| f32::from(*sample) / 32_768.0)); + } + } + + fn finish(&mut self, cx: &mut App) { + let samples = std::mem::take(&mut self.samples); + let sink = self.sink.clone(); + + // Speech too short to transcribe ends the session at once. + let silent = samples.iter().all(|sample| sample.abs() < 0.0001); + if samples.len() < MODEL_RATE as usize / 5 || silent { + sink.finish(cx); + return; + } + + let (sender, receiver) = flume::unbounded(); + worker().send_or_log(Command::Transcribe { samples, sender }); + + self._result = Some(cx.spawn(async move |cx| { + let result = receiver.recv_async().await; + + cx.update(|cx| match result { + Ok(Ok(text)) if !text.is_empty() => { + sink.phrase(text, cx); + sink.finish(cx); + } + Ok(Ok(_)) => sink.finish(cx), + Ok(Err(error)) => sink.error(SpeechError::recognizer(error), cx), + Err(_) => sink.error( + SpeechError::recognizer(anyhow!("The speech worker is unavailable")), + cx, + ), + }); + })); + } +} + +enum Command { + Warm(PathBuf), + Transcribe { + samples: Vec, + sender: flume::Sender>, + }, + Unload, +} + +/// A handle to the single inference worker. +fn worker() -> &'static flume::Sender { + static WORKER: OnceLock> = OnceLock::new(); + + WORKER.get_or_init(|| { + let (sender, receiver) = flume::unbounded::(); + + let spawned = std::thread::Builder::new() + .name("speech-parakeet".into()) + .spawn(move || run_worker(receiver)); + + if let Err(error) = spawned { + log::error!("failed to spawn the speech worker: {error}"); + } + + sender + }) +} + +fn run_worker(receiver: flume::Receiver) { + let mut cached: Option<(PathBuf, ParakeetTDT)> = None; + + while let Ok(command) = receiver.recv_timeout(Duration::from_secs(30)) { + match command { + Command::Unload => { + cached = None; + } + Command::Warm(dir) => { + let loaded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + load(&mut cached, &dir).map(|_| ()) + })); + + match flatten_panic(loaded) { + Ok(Ok(())) => {} + Ok(Err(error)) | Err(error) => { + log::warn!("failed to load the speech model: {error:#}") + } + } + } + Command::Transcribe { samples, sender } => { + let transcribed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + transcribe(&mut cached, samples) + })); + + let result = match flatten_panic(transcribed) { + Ok(result) => result, + Err(_) => { + cached = None; + Err(anyhow!("Dictation stopped unexpectedly. Try again.")) + } + }; + + if sender.send(result).is_err() { + log::debug!("speech session was cancelled before its result arrived"); + } + } + } + } +} + +fn load<'a>( + cached: &'a mut Option<(PathBuf, ParakeetTDT)>, + dir: &PathBuf, +) -> Result<&'a mut ParakeetTDT> { + if cached.as_ref().is_none_or(|(path, _)| path != dir) { + let model = ParakeetTDT::from_pretrained(dir, None) + .map_err(|error| anyhow!("Could not load the speech model: {error}"))?; + *cached = Some((dir.clone(), model)); + } + + let (_, model) = cached.as_mut().context("speech model was unloaded")?; + Ok(model) +} + +fn transcribe(cached: &mut Option<(PathBuf, ParakeetTDT)>, samples: Vec) -> Result { + let model = load(cached, &super::model::dir())?; + + model + .transcribe_samples(samples, MODEL_RATE, 1, None) + .map(|result| result.text) + .map_err(|error| anyhow!("Could not transcribe this recording: {error}")) +} + +trait SendOrLog { + fn send_or_log(&self, command: Command); +} + +impl SendOrLog for flume::Sender { + fn send_or_log(&self, command: Command) { + if self.send(command).is_err() { + log::debug!("speech worker is gone; command dropped"); + } + } +} + +/// Retire cached weights, e.g. after the model was deleted. +pub(crate) fn unload() { + worker().send_or_log(Command::Unload); +} + +fn flatten_panic(result: std::thread::Result) -> Result { + result.map_err(|_| anyhow!("The speech worker panicked")) +} diff --git a/crates/speech/src/web.rs b/crates/speech/src/web.rs new file mode 100644 index 00000000..09303dae --- /dev/null +++ b/crates/speech/src/web.rs @@ -0,0 +1,256 @@ +use std::cell::RefCell; + +use anyhow::anyhow; +use gpui::App; +use gpui_kit::Subscription; +use gpui_kit::component::speech::{ + AudioFormat, AudioInput, AudioSink, RecognitionSession, SpeechError, SpeechRecognizer, + SpeechSink, +}; +use js_sys::{Array, Function, JsString, Reflect}; +use wasm_bindgen::closure::Closure; +use wasm_bindgen::{JsCast as _, JsValue}; +use web_sys::window; + +/// The recognizer backed by `SpeechRecognition` (or its `webkit` prefix). +pub struct WebSpeechRecognizer; + +impl SpeechRecognizer for WebSpeechRecognizer { + fn audio_format(&self) -> AudioFormat { + AudioFormat::default() + } + + fn is_available(&self, _cx: &App) -> bool { + true + } + + fn start( + &self, + sink: SpeechSink, + cx: &mut App, + ) -> Result, SpeechError> { + let recognition = Recognition::open(sink, cx).map_err(SpeechError::recognizer)?; + + Ok(Box::new(recognition)) + } +} + +pub struct SelfCapturedInput; + +impl AudioInput for SelfCapturedInput { + fn start( + &self, + _format: AudioFormat, + _sink: AudioSink, + _cx: &mut App, + ) -> Result { + Ok(Subscription::new(|| {})) + } +} + +enum Event { + Hypothesis(String), + Phrase(String), + Error(String), + End, +} + +struct Recognition { + recognition: JsValue, + on_result: Option>, + on_error: Option>, + on_end: Option>, + _task: gpui::Task<()>, +} + +impl Recognition { + fn open(sink: SpeechSink, cx: &mut App) -> anyhow::Result { + let window = window().ok_or_else(|| anyhow!("No browser window"))?; + + let constructor = Reflect::get(&window, &JsString::from("SpeechRecognition")) + .ok() + .filter(|value| value.is_function()) + .or_else(|| { + Reflect::get(&window, &JsString::from("webkitSpeechRecognition")) + .ok() + .filter(|value| value.is_function()) + }) + .ok_or_else(|| anyhow!("This browser does not support speech recognition"))?; + + let recognition = + Reflect::construct(constructor.unchecked_ref::(), &Array::new()) + .map_err(|error| js_error("constructing the recognizer", error))?; + + let (sender, receiver) = flume::unbounded::(); + let error_sender = sender.clone(); + let end_sender = sender.clone(); + + let finalized = RefCell::new(0usize); + let on_result = Closure::wrap(Box::new(move |event: JsValue| { + let read = |value: &JsValue, property: &str| -> Option { + Reflect::get(value, &JsString::from(property)).ok() + }; + + let Some(results) = read(&event, "results") else { + return; + }; + let length = read(&results, "length") + .and_then(|length| length.as_f64()) + .unwrap_or(0.0) as usize; + + let mut finalized = finalized.borrow_mut(); + let mut interim = String::new(); + + for index in *finalized..length { + let Some(result) = read(&results, &index.to_string()) else { + continue; + }; + + let Some(alternative) = read(&result, "0") else { + continue; + }; + + let Some(text) = read(&alternative, "transcript").and_then(|text| text.as_string()) + else { + continue; + }; + + let is_final = read(&result, "isFinal") + .and_then(|final_value| final_value.as_bool()) + .unwrap_or(false); + + if is_final { + *finalized += 1; + if sender.send(Event::Phrase(text)).is_err() { + log::debug!("speech session was cancelled mid-utterance"); + } + } else { + interim.push_str(&text); + } + } + + if !interim.is_empty() && sender.send(Event::Hypothesis(interim)).is_err() { + log::debug!("speech session was cancelled mid-utterance"); + } + }) as Box); + + let on_error = Closure::wrap(Box::new(move |event: JsValue| { + let message = Reflect::get(&event, &JsString::from("error")) + .ok() + .and_then(|error| error.as_string()) + .unwrap_or_else(|| "unknown speech error".into()); + + if error_sender.send(Event::Error(message)).is_err() { + log::debug!("speech session was cancelled before its error arrived"); + } + }) as Box); + + let on_end = Closure::wrap(Box::new(move |_event: JsValue| { + if end_sender.send(Event::End).is_err() { + log::debug!("speech session was cancelled before its end arrived"); + } + }) as Box); + + set_callback(&recognition, "onresult", on_result.as_ref().unchecked_ref())?; + set_callback(&recognition, "onerror", on_error.as_ref().unchecked_ref())?; + set_callback(&recognition, "onend", on_end.as_ref().unchecked_ref())?; + set_bool(&recognition, "continuous", true)?; + set_bool(&recognition, "interimResults", true)?; + + call_method(&recognition, "start")?; + + // The browser consumes audio as soon as it has permission + sink.ready(cx); + + let task = cx.spawn(async move |cx| { + while let Ok(event) = receiver.recv_async().await { + let done = matches!(event, Event::End); + + // The speech state ignores sink calls once its session ended, + // so a late event after a cancel is harmless. + cx.update(|cx| match event { + Event::Hypothesis(text) => sink.hypothesis(text, cx), + Event::Phrase(text) => sink.phrase(text, cx), + Event::Error(message) => { + sink.error(SpeechError::recognizer(anyhow!("{message}")), cx) + } + Event::End => sink.finish(cx), + }); + + if done { + break; + } + } + }); + + Ok(Self { + recognition, + on_result: Some(on_result), + on_error: Some(on_error), + on_end: Some(on_end), + _task: task, + }) + } + + fn stop(&mut self) { + self.on_result = None; + self.on_error = None; + self.on_end = None; + + let _ignored = call_method(&self.recognition, "abort"); + } +} + +impl Drop for Recognition { + fn drop(&mut self) { + self.stop(); + } +} + +impl RecognitionSession for Recognition { + fn push_audio(&mut self, _samples: &[i16], _cx: &mut App) { + // The browser recognizer captures the microphone itself. + // the state still pushes audio through its input, which is a no-op here. + } + + fn finish(&mut self, _cx: &mut App) { + // The end handler reports completion, so no result is fabricated here. + let _ignored = call_method(&self.recognition, "stop"); + } +} + +fn set_callback(target: &JsValue, property: &str, callback: &Function) -> anyhow::Result<()> { + Reflect::set(target, &JsString::from(property), callback) + .map_err(|error| js_error(property, error))?; + Ok(()) +} + +fn set_bool(target: &JsValue, property: &str, value: bool) -> anyhow::Result<()> { + Reflect::set(target, &JsString::from(property), &JsValue::from(value)) + .map_err(|error| js_error(property, error))?; + Ok(()) +} + +fn call_method(target: &JsValue, method: &str) -> anyhow::Result<()> { + let function = Reflect::get(target, &JsString::from(method)) + .map_err(|error| js_error(method, error))? + .unchecked_into::(); + + function + .call0(target) + .map(|_| ()) + .map_err(|error| js_error(method, error)) +} + +fn js_error(what: &str, error: JsValue) -> anyhow::Error { + let description = error + .as_string() + .or_else(|| { + Reflect::get(&error, &JsString::from("message")) + .ok() + .and_then(|message| message.as_string()) + }) + .unwrap_or_else(|| "unknown error".into()); + + anyhow!("{what} failed: {description}") +} diff --git a/crates/workspace/Cargo.toml b/crates/workspace/Cargo.toml index 9fdc7c30..1972576c 100644 --- a/crates/workspace/Cargo.toml +++ b/crates/workspace/Cargo.toml @@ -15,6 +15,7 @@ chat_ui = { path = "../chat_ui" } nip29 = { path = "../nip29" } nip29_ui = { path = "../nip29_ui" } settings = { path = "../settings" } +speech = { path = "../speech" } person = { path = "../person" } auto_update = { path = "../auto_update" } diff --git a/crates/workspace/src/dialogs/settings.rs b/crates/workspace/src/dialogs/settings.rs index eae40c36..40cb0af1 100644 --- a/crates/workspace/src/dialogs/settings.rs +++ b/crates/workspace/src/dialogs/settings.rs @@ -1,9 +1,12 @@ +#[cfg(not(target_arch = "wasm32"))] +use std::time::Duration; + #[cfg(not(target_arch = "wasm32"))] use gpui::PathPromptOptions; use gpui::http_client::Url; use gpui::{ - App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, Subscription, - Window, div, + AnyElement, App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, + Subscription, Task, Window, div, }; use gpui_kit::assets::IconName; use gpui_kit::component::button::{Button, ButtonVariants}; @@ -12,8 +15,10 @@ use gpui_kit::component::menu::{DropdownMenu, PopupMenuItem}; use gpui_kit::component::notification::Notification; use gpui_kit::component::separator::Separator; use gpui_kit::component::switch::Switch; -use gpui_kit::component::{Disableable, Sizable, WindowExt, h_flex, v_flex}; +use gpui_kit::component::{ActiveTheme, Disableable, Sizable, WindowExt, h_flex, v_flex}; use settings::{AppSettings, AppearanceMode}; +#[cfg(not(target_arch = "wasm32"))] +use speech::model::{self as speech_model}; use ui::{settings_block, settings_row}; /// Bounds for the theme number inputs @@ -29,6 +34,8 @@ pub struct Preferences { font_size_input: Entity, radius_input: Entity, radius_lg_input: Entity, + #[cfg(not(target_arch = "wasm32"))] + _speech_poll: Task<()>, /// Keeps the control subscriptions alive while the dialog is open _subscriptions: Vec, } @@ -94,11 +101,40 @@ impl Preferences { }), ]; + #[cfg(not(target_arch = "wasm32"))] + let _speech_poll = cx.spawn_in(window, async move |this, cx| { + let timer = Duration::from_millis(500); + let mut last = speech_model::status(); + + loop { + cx.background_executor().timer(timer).await; + + let status = speech_model::status(); + let error = speech_model::take_error(); + + if status == last { + continue; + } + + last = status; + + this.update_in(cx, |_, window, cx| { + if let Some(error) = error { + window.push_notification(Notification::error(error).autohide(false), cx); + } + cx.notify(); + }) + .ok(); + } + }); + Self { file_input, font_size_input, radius_input, radius_lg_input, + #[cfg(not(target_arch = "wasm32"))] + _speech_poll, _subscriptions: subscriptions, } } @@ -188,6 +224,74 @@ impl Preferences { .detach(); } } + + fn render_speech( + &mut self, + #[cfg_attr(target_arch = "wasm32", allow(unused_variables))] cx: &mut Context, + ) -> AnyElement { + #[cfg(target_arch = "wasm32")] + { + return div().into_any_element(); + } + + #[cfg(not(target_arch = "wasm32"))] + { + const SPEECH: &str = "On-device speech model used by the speech to text feature. Audio never leaves the device."; + + let control = match speech_model::status() { + speech_model::Status::NotInstalled => { + let size = speech_model::total_size() / 1024 / 1024; + + h_flex().gap_1().child( + Button::new("download-model") + .label(format!("Download model ({size} MB)")) + .small() + .on_click(|_, _window, cx| { + speech_model::start_download(); + cx.refresh_windows(); + }), + ) + } + speech_model::Status::Downloading { downloaded, total } => { + let percent = downloaded * 100 / total.max(1); + let downloaded = downloaded / 1024 / 1024; + let total = total / 1024 / 1024; + + h_flex() + .gap_2() + .child( + div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child(format!("Downloading {percent}% · {downloaded}/{total} MB")), + ) + .child( + Button::new("cancel-download") + .label("Cancel") + .small() + .on_click(|_, _window, cx| { + speech_model::cancel_download(); + cx.refresh_windows(); + }), + ) + } + speech_model::Status::Installed => h_flex() + .gap_2() + .child(div().text_sm().child("Installed")) + .child( + Button::new("delete-model") + .label("Delete") + .small() + .on_click(|_, _window, cx| { + speech_model::delete(); + cx.refresh_windows(); + }), + ), + }; + + settings_block("Speech Recognition", SPEECH, control, cx).into_any_element() + } + } } impl Render for Preferences { @@ -391,5 +495,6 @@ impl Render for Preferences { ), cx, )) + .child(self.render_speech(cx)) } } -- 2.54.0 From f4860750991cde4170a972d4aa305b868f4e724c Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Tue, 27 Oct 2026 12:36:45 +0700 Subject: [PATCH 4/5] update ui --- Cargo.lock | 3 ++ crates/chat_ui/Cargo.toml | 1 + crates/chat_ui/src/lib.rs | 85 +++++++++++++++++++++++++++----------- crates/nip29_ui/Cargo.toml | 1 + crates/nip29_ui/src/lib.rs | 62 +++++++++++++++++++-------- crates/speech/Cargo.toml | 1 + crates/speech/src/lib.rs | 27 ++++++++++++ 7 files changed, 139 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a6bc562..38ef4737 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1161,6 +1161,7 @@ dependencies = [ "common", "gpui-kit", "gpui-pre", + "instant", "itertools 0.13.0", "nostr-sdk", "person", @@ -5430,6 +5431,7 @@ dependencies = [ "common", "gpui-kit", "gpui-pre", + "instant", "log", "nip29", "nostr-sdk", @@ -8374,6 +8376,7 @@ dependencies = [ "flume 0.11.1", "gpui-kit", "gpui-pre", + "instant", "js-sys", "log", "ort", diff --git a/crates/chat_ui/Cargo.toml b/crates/chat_ui/Cargo.toml index 7f5b25a0..d14400cf 100644 --- a/crates/chat_ui/Cargo.toml +++ b/crates/chat_ui/Cargo.toml @@ -15,6 +15,7 @@ chat = { path = "../chat" } settings = { path = "../settings" } gpui.workspace = true +instant.workspace = true nostr-sdk.workspace = true anyhow.workspace = true itertools.workspace = true diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index 73ce1548..d23359ec 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -96,6 +96,9 @@ pub struct ChatPanel { /// Dictation session for the chat input speech: Entity, + /// When the running dictation started, for the elapsed counter + speech_started: Option, + /// Subject input state subject_input: Entity, @@ -207,14 +210,25 @@ impl ChatPanel { // Insert the dictated text into the chat input cx.subscribe_in(&speech, window, move |this, _speech, event, window, cx| { match event { - SpeechEvent::Final(text) if !text.is_empty() => { - this.input.update(cx, |input, cx| { - input.insert(text.clone(), window, cx); - }); + SpeechEvent::Started => { + this.speech_started = Some(instant::Instant::now()); + cx.notify(); + } + SpeechEvent::Final(text) => { + this.speech_started = None; + + if !text.is_empty() { + this.input + .update(cx, |input, cx| input.insert(text.clone(), window, cx)); + } } SpeechEvent::Error(error) => { + this.speech_started = None; window.push_notification(Notification::error(error.to_string()), cx); } + SpeechEvent::Cancelled => { + this.speech_started = None; + } _ => {} }; }), @@ -238,6 +252,7 @@ impl ChatPanel { list_state, input, speech, + speech_started: None, subject_input, subject_bar, replies_to, @@ -1706,7 +1721,12 @@ impl ChatPanel { items } - fn render_config_menu(&self, _window: &mut Window, cx: &Context) -> impl IntoElement { + fn render_config_menu( + &self, + disabled: bool, + _window: &mut Window, + cx: &Context, + ) -> impl IntoElement { let (backup, signer_kind) = self .room .read_with(cx, |this, _cx| { @@ -1719,6 +1739,7 @@ impl ChatPanel { .icon(IconName::Settings2) .tooltip("Configuration") .ghost() + .disabled(disabled) .dropdown_menu(move |this, _window, _cx| { let auto = matches!(signer_kind, SignerKind::Auto); let encryption = matches!(signer_kind, SignerKind::Encryption); @@ -1749,7 +1770,12 @@ impl ChatPanel { }) } - fn render_emoji_menu(&self, _window: &Window, cx: &Context) -> impl IntoElement { + fn render_emoji_menu( + &self, + disabled: bool, + _window: &Window, + cx: &Context, + ) -> impl IntoElement { let nostr = NostrRegistry::global(cx); let current_user = nostr.read(cx).current_user(); @@ -1766,6 +1792,7 @@ impl ChatPanel { Button::new("emoji") .icon(IconName::Emoji) .ghost() + .disabled(disabled) .dropdown_menu_with_anchor(gpui::Anchor::BottomLeft, move |this, _window, _cx| { let menu = REACTION_EMOJIS.iter().fold(this, |this, emoji| { this.menu(*emoji, Box::new(Command::Insert(emoji.to_string()))) @@ -1839,7 +1866,7 @@ impl Render for ChatPanel { let is_typing = !self.input.read(cx).value().trim().is_empty(); let pending_attachments = !self.encrypted_attachments.read(cx).is_empty(); - let capturing = self.speech.read(cx).status().is_capturing(); + let dictating = self.speech.read(cx).status().is_active(); v_flex() .image_cache(retain_all(self.id.clone())) @@ -1920,42 +1947,52 @@ impl Render for ChatPanel { .icon(IconName::Paperclip) .tooltip("Upload media") .loading(self.uploading) - .disabled(self.uploading) + .disabled(dictating || self.uploading) .ghost() .on_click(cx.listener(move |this, _ev, window, cx| { this.upload(window, cx); })), ) - .child( - Textarea::new(&self.input) - .appearance(false) - .small() - .min_h_8() - .py_1() - .flex_1(), - ) + .when(dictating, |this| { + this.child( + h_flex() + .h_8() + .flex_1() + .gap_2() + .pl_1() + .child(SpeechWaveform::new(&self.speech).small().flex_1()) + .when_some(self.speech_started, |this, started| { + this.child(speech::elapsed(started, cx)) + }), + ) + }) + .when(!dictating, |this| { + this.child( + Textarea::new(&self.input) + .appearance(false) + .small() + .min_h_8() + .py_1() + .flex_1(), + ) + }) .child( h_flex() .items_end() .gap_1() .pl_1() - .when(capturing, |this| { - this.child( - SpeechWaveform::new(&self.speech).xsmall().w_12(), - ) - }) .child(SpeechButton::new(&self.speech)), ) .child( h_flex() .pl_1() .gap_1() - .child(self.render_emoji_menu(window, cx)) - .child(self.render_config_menu(window, cx)) + .child(self.render_emoji_menu(dictating, window, cx)) + .child(self.render_config_menu(dictating, window, cx)) .child( Button::new("send") .icon(IconName::PaperPlaneFill) - .disabled(self.uploading) + .disabled(dictating || self.uploading) .ghost() .on_click(cx.listener(move |this, _ev, window, cx| { this.send_text_message(window, cx); diff --git a/crates/nip29_ui/Cargo.toml b/crates/nip29_ui/Cargo.toml index 451c1d50..c6fa175d 100644 --- a/crates/nip29_ui/Cargo.toml +++ b/crates/nip29_ui/Cargo.toml @@ -17,6 +17,7 @@ gpui-kit = { workspace = true, features = ["speech"] } ui = { path = "../ui" } gpui.workspace = true +instant.workspace = true nostr-sdk.workspace = true anyhow.workspace = true log.workspace = true diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 331c2805..73161f4a 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -316,6 +316,8 @@ pub struct GroupPanel { input: Entity, /// Dictation session for the composer input speech: Entity, + /// When the running dictation started, for the elapsed counter + speech_started: Option, /// Uploaded, non-encrypted attachments waiting to be sent. attachments: Vec, /// Whether an upload is in flight. @@ -367,14 +369,26 @@ impl GroupPanel { subscriptions.push( cx.subscribe_in(&speech, window, |this, _, event, window, cx| { match event { - SpeechEvent::Final(text) if !text.is_empty() => { - this.input.update(cx, |input, cx| { - input.insert(text.clone(), window, cx); - }); + SpeechEvent::Started => { + this.speech_started = Some(instant::Instant::now()); + cx.notify(); + } + SpeechEvent::Final(text) => { + this.speech_started = None; + + if !text.is_empty() { + this.input.update(cx, |input, cx| { + input.insert(text.clone(), window, cx); + }); + } } SpeechEvent::Error(error) => { + this.speech_started = None; window.push_notification(Notification::error(error.to_string()), cx); } + SpeechEvent::Cancelled => { + this.speech_started = None; + } _ => {} }; }), @@ -416,6 +430,7 @@ impl GroupPanel { rendered_texts_by_id: BTreeMap::new(), input, speech, + speech_started: None, attachments: Vec::new(), uploading: false, tasks: smallvec![], @@ -1073,6 +1088,8 @@ impl GroupPanel { return self.render_notice(¬ice, cx); } + let dictating = self.speech.read(cx).status().is_active(); + v_flex() .flex_shrink_0() .w_full() @@ -1091,23 +1108,34 @@ impl GroupPanel { .icon(ui::IconName::Paperclip) .tooltip("Upload media") .loading(self.uploading) - .disabled(self.uploading) + .disabled(dictating || self.uploading) .ghost() .on_click(cx.listener(|this, _event, window, cx| { this.upload(window, cx); })), ) - .child( - Textarea::new(&self.input) - .appearance(false) - .bg(cx.theme().input) - .small() - .min_h_8() - .py_1() - .flex_1(), - ) - .when(self.speech.read(cx).status().is_capturing(), |this| { - this.child(SpeechWaveform::new(&self.speech).xsmall().w_12()) + .when(dictating, |this| { + this.child( + h_flex() + .h_8() + .flex_1() + .gap_2() + .child(SpeechWaveform::new(&self.speech).small().flex_1()) + .when_some(self.speech_started, |this, started| { + this.child(speech::elapsed(started, cx)) + }), + ) + }) + .when(!dictating, |this| { + this.child( + Textarea::new(&self.input) + .appearance(false) + .bg(cx.theme().input) + .small() + .min_h_8() + .py_1() + .flex_1(), + ) }) .child(SpeechButton::new(&self.speech)) .child( @@ -1115,7 +1143,7 @@ impl GroupPanel { .icon(ui::IconName::PaperPlaneFill) .tooltip("Send") .ghost() - .disabled(self.uploading) + .disabled(dictating || self.uploading) .on_click(cx.listener(|this, _event, window, cx| { this.send(window, cx); })), diff --git a/crates/speech/Cargo.toml b/crates/speech/Cargo.toml index 444cc638..724ebf75 100644 --- a/crates/speech/Cargo.toml +++ b/crates/speech/Cargo.toml @@ -10,6 +10,7 @@ log.workspace = true gpui.workspace = true gpui-kit = { workspace = true, features = ["speech"] } flume.workspace = true +instant.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] common = { path = "../common" } diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs index 99014d77..d4cfcdcb 100644 --- a/crates/speech/src/lib.rs +++ b/crates/speech/src/lib.rs @@ -5,8 +5,35 @@ mod parakeet; #[cfg(target_arch = "wasm32")] mod web; +use std::time::Duration; + +use gpui::{ + Animation, AnimationExt as _, AnyElement, App, IntoElement as _, ParentElement as _, + Styled as _, div, +}; +use gpui_kit::component::ActiveTheme as _; use gpui_kit::component::speech::SpeechRecognizer; +/// A ticking `M:SS` counter for the dictation in progress. +pub fn elapsed(started: instant::Instant, cx: &App) -> AnyElement { + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .with_animation( + "dictation-elapsed", + Animation::new(Duration::from_secs(1)) + .repeat() + .with_max_fps(1.), + move |counter, _| counter.child(format_elapsed(started)), + ) + .into_any_element() +} + +fn format_elapsed(started: instant::Instant) -> String { + let seconds = started.elapsed().as_secs(); + format!("{}:{:02}", seconds / 60, seconds % 60) +} + /// The recognizer backing the composer's microphone button. #[cfg(not(target_arch = "wasm32"))] pub fn recognizer() -> impl SpeechRecognizer { -- 2.54.0 From 47feaa1e23162109ada6122c2d72960b8365e6d1 Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Tue, 27 Oct 2026 13:24:25 +0700 Subject: [PATCH 5/5] Add a global ToggleDictation keybinding --- Cargo.lock | 2 + crates/chat_ui/src/actions.rs | 5 +- crates/chat_ui/src/lib.rs | 12 ++ crates/nip29_ui/src/lib.rs | 18 ++- crates/workspace/src/dialogs/settings.rs | 140 ++++++++++++++--------- desktop/Cargo.toml | 1 + desktop/src/main.rs | 15 ++- web/Cargo.toml | 1 + web/src/lib.rs | 5 + 9 files changed, 138 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38ef4737..001b2d59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1460,6 +1460,7 @@ dependencies = [ "assets", "auto_update", "chat", + "chat_ui", "flume 0.11.1", "gpui-kit", "gpui-pre", @@ -1488,6 +1489,7 @@ version = "2.0.0-alpha2" dependencies = [ "assets", "chat", + "chat_ui", "console_error_panic_hook", "console_log", "getrandom 0.2.17", diff --git a/crates/chat_ui/src/actions.rs b/crates/chat_ui/src/actions.rs index b1c47642..2ef1f3f0 100644 --- a/crates/chat_ui/src/actions.rs +++ b/crates/chat_ui/src/actions.rs @@ -1,4 +1,4 @@ -use gpui::Action; +use gpui::{Action, actions}; use nostr_sdk::prelude::*; use serde::Deserialize; use settings::SignerKind; @@ -15,3 +15,6 @@ pub enum Command { Njump(PublicKey), Trace(EventId), } + +// Start or stop dictation in the focused composer +actions!(chat, [ToggleDictation]); diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index d23359ec..f50dcb44 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -882,6 +882,17 @@ impl ChatPanel { self.directory.read(cx).person(public_key, cx) } + fn toggle_dictation( + &mut self, + _: &ToggleDictation, + _window: &mut Window, + cx: &mut Context, + ) { + self.speech.update(cx, |speech, cx| { + speech.toggle(cx); + }); + } + fn on_command(&mut self, command: &Command, window: &mut Window, cx: &mut Context) { match command { Command::Insert(content) => { @@ -1871,6 +1882,7 @@ impl Render for ChatPanel { v_flex() .image_cache(retain_all(self.id.clone())) .on_action(cx.listener(Self::on_command)) + .on_action(cx.listener(Self::toggle_dictation)) .size_full() .when(*self.subject_bar.read(cx), |this| { this.child( diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 73161f4a..21a61b96 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -2,11 +2,13 @@ use std::collections::BTreeMap; use std::time::Duration; use anyhow::Result; +use chat_ui::ToggleDictation; use gpui::prelude::FluentBuilder; use gpui::{ Action, AnyElement, App, AppContext, Context, Entity, EventEmitter, FocusHandle, Focusable, - FollowMode, IntoElement, ListAlignment, ListScrollEvent, ListState, ParentElement, Render, - SharedString, Styled, Subscription, Task, WeakEntity, Window, div, list, px, + FollowMode, InteractiveElement, IntoElement, ListAlignment, ListScrollEvent, ListState, + ParentElement, Render, SharedString, Styled, Subscription, Task, WeakEntity, Window, div, list, + px, }; use gpui_kit::assets::IconName; use gpui_kit::base::StyledExt as _; @@ -1063,6 +1065,17 @@ impl GroupPanel { ) } + fn toggle_dictation( + &mut self, + _ev: &ToggleDictation, + _window: &mut Window, + cx: &mut Context, + ) { + self.speech.update(cx, |speech, cx| { + speech.toggle(cx); + }); + } + fn render_composer(&self, cx: &mut Context) -> AnyElement { let notice = self.group.upgrade().and_then(|group| { let group = group.read(cx); @@ -1341,6 +1354,7 @@ impl Render for GroupPanel { v_flex() .size_full() .min_w_0() + .on_action(cx.listener(Self::toggle_dictation)) .child(self.render_encryption_notice(cx)) .child(self.render_banner(cx)) .child(self.render_pins(cx)) diff --git a/crates/workspace/src/dialogs/settings.rs b/crates/workspace/src/dialogs/settings.rs index 40cb0af1..53a79811 100644 --- a/crates/workspace/src/dialogs/settings.rs +++ b/crates/workspace/src/dialogs/settings.rs @@ -1,21 +1,27 @@ #[cfg(not(target_arch = "wasm32"))] use std::time::Duration; +use chat_ui::ToggleDictation; #[cfg(not(target_arch = "wasm32"))] use gpui::PathPromptOptions; +#[cfg(not(target_arch = "wasm32"))] +use gpui::Task; use gpui::http_client::Url; use gpui::{ AnyElement, App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, - Subscription, Task, Window, div, + Subscription, Window, div, }; use gpui_kit::assets::IconName; use gpui_kit::component::button::{Button, ButtonVariants}; use gpui_kit::component::input::{Input, InputEvent, InputState, NumberInput}; +use gpui_kit::component::kbd::Kbd; use gpui_kit::component::menu::{DropdownMenu, PopupMenuItem}; use gpui_kit::component::notification::Notification; +#[cfg(not(target_arch = "wasm32"))] +use gpui_kit::component::progress::Progress; use gpui_kit::component::separator::Separator; use gpui_kit::component::switch::Switch; -use gpui_kit::component::{ActiveTheme, Disableable, Sizable, WindowExt, h_flex, v_flex}; +use gpui_kit::component::{Disableable, Sizable, WindowExt, h_flex, v_flex}; use settings::{AppSettings, AppearanceMode}; #[cfg(not(target_arch = "wasm32"))] use speech::model::{self as speech_model}; @@ -236,66 +242,90 @@ impl Preferences { #[cfg(not(target_arch = "wasm32"))] { - const SPEECH: &str = "On-device speech model used by the speech to text feature. Audio never leaves the device."; + let size = speech_model::total_size() / 1_000_000; + let status = speech_model::status(); - let control = match speech_model::status() { - speech_model::Status::NotInstalled => { - let size = speech_model::total_size() / 1024 / 1024; - - h_flex().gap_1().child( - Button::new("download-model") - .label(format!("Download model ({size} MB)")) - .small() - .on_click(|_, _window, cx| { - speech_model::start_download(); - cx.refresh_windows(); - }), - ) - } - speech_model::Status::Downloading { downloaded, total } => { - let percent = downloaded * 100 / total.max(1); - let downloaded = downloaded / 1024 / 1024; - let total = total / 1024 / 1024; - - h_flex() - .gap_2() - .child( - div() - .text_sm() - .text_color(cx.theme().muted_foreground) - .child(format!("Downloading {percent}% · {downloaded}/{total} MB")), - ) - .child( - Button::new("cancel-download") - .label("Cancel") - .small() - .on_click(|_, _window, cx| { - speech_model::cancel_download(); - cx.refresh_windows(); - }), - ) - } - speech_model::Status::Installed => h_flex() - .gap_2() - .child(div().text_sm().child("Installed")) - .child( - Button::new("delete-model") - .label("Delete") - .small() - .on_click(|_, _window, cx| { - speech_model::delete(); - cx.refresh_windows(); - }), + let (description, control) = match &status { + speech_model::Status::NotInstalled => ( + format!("{size} MB download"), + Button::new("download-model") + .label("Download") + .secondary() + .small() + .on_click(|_, _window, cx| { + speech_model::start_download(); + cx.refresh_windows(); + }) + .into_any_element(), + ), + speech_model::Status::Downloading { downloaded, total } => ( + format!( + "Downloading · {} of {} MB", + downloaded / 1_000_000, + total / 1_000_000 ), + Button::new("cancel-download") + .label("Cancel") + .ghost() + .small() + .on_click(|_, _window, cx| { + speech_model::cancel_download(); + cx.refresh_windows(); + }) + .into_any_element(), + ), + speech_model::Status::Installed => ( + format!("Parakeet v3 · {size} MB"), + Button::new("remove-model") + .label("Remove") + .secondary() + .small() + .on_click(|_, _window, cx| { + speech_model::delete(); + cx.refresh_windows(); + }) + .into_any_element(), + ), }; - settings_block("Speech Recognition", SPEECH, control, cx).into_any_element() + let mut block = v_flex().w_full().gap_3().child(settings_row( + "Speech model", + description, + control, + cx, + )); + + if let speech_model::Status::Downloading { downloaded, total } = &status { + let (downloaded, total) = (*downloaded, *total); + + block = block.child( + Progress::new("speech-model-download") + .value(downloaded as f32 / total.max(1) as f32 * 100.) + .small(), + ); + } + + block.into_any_element() } } + + /// Shows the keystroke actually bound to the dictation action at startup + fn render_dictation_shortcut( + &self, + window: &mut Window, + cx: &mut Context, + ) -> impl IntoElement { + let control = match Kbd::global_binding_for_action(&ToggleDictation, window) { + Some(kbd) => kbd.into_any_element(), + None => div().into_any_element(), + }; + + settings_row("Shortcut", "Press to start or stop dictation", control, cx) + } } impl Render for Preferences { - fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { + fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { const SCREENING: &str = "Show an screening dialog to verify the unknown sender."; const WOT: &str = "Treat people followed by your contacts as known senders."; const AVATAR: &str = "Hide all avatar pictures to improve performance."; @@ -478,6 +508,9 @@ impl Render for Preferences { cx, )) .child(Separator::horizontal()) + .child(self.render_speech(cx)) + .child(self.render_dictation_shortcut(window, cx)) + .child(Separator::horizontal()) .child(settings_block( "Media Upload Service", MEDIA, @@ -495,6 +528,5 @@ impl Render for Preferences { ), cx, )) - .child(self.render_speech(cx)) } } diff --git a/desktop/Cargo.toml b/desktop/Cargo.toml index 781e8b11..2db31079 100644 --- a/desktop/Cargo.toml +++ b/desktop/Cargo.toml @@ -38,6 +38,7 @@ ui = { path = "../crates/ui" } state = { path = "../crates/state" } nip4e = { path = "../crates/nip4e" } chat = { path = "../crates/chat" } +chat_ui = { path = "../crates/chat_ui" } nip29 = { path = "../crates/nip29" } settings = { path = "../crates/settings" } auto_update = { path = "../crates/auto_update" } diff --git a/desktop/src/main.rs b/desktop/src/main.rs index a4c34176..84c0b6de 100644 --- a/desktop/src/main.rs +++ b/desktop/src/main.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, Mutex}; +use chat_ui::ToggleDictation; use gpui::{ App, Bounds, Global, KeyBinding, Menu, MenuItem, SharedString, TitlebarOptions, WindowBackgroundAppearance, WindowBounds, WindowDecorations, WindowKind, WindowOptions, @@ -45,13 +46,19 @@ fn main() { // Register the `quit` function cx.on_action(quit); - // Register the `quit` function with CMD+Q (macOS) + // Register the `quit` function with CMD+Q (macOS) and dictation with CMD+D #[cfg(target_os = "macos")] - cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]); + cx.bind_keys([ + KeyBinding::new("cmd-q", Quit, None), + KeyBinding::new("cmd-d", ToggleDictation, None), + ]); - // Register the `quit` function with Super+Q (others) + // Register the `quit` function with Super+Q (others) and dictation with Ctrl+D #[cfg(not(target_os = "macos"))] - cx.bind_keys([KeyBinding::new("super-q", Quit, None)]); + cx.bind_keys([ + KeyBinding::new("super-q", Quit, None), + KeyBinding::new("ctrl-d", ToggleDictation, None), + ]); // Set menu items cx.set_menus(vec![Menu { diff --git a/web/Cargo.toml b/web/Cargo.toml index 8658a426..6251560b 100644 --- a/web/Cargo.toml +++ b/web/Cargo.toml @@ -14,6 +14,7 @@ ui = { path = "../crates/ui" } state = { path = "../crates/state" } nip4e = { path = "../crates/nip4e" } chat = { path = "../crates/chat" } +chat_ui = { path = "../crates/chat_ui" } nip29 = { path = "../crates/nip29" } settings = { path = "../crates/settings" } person = { path = "../crates/person" } diff --git a/web/src/lib.rs b/web/src/lib.rs index 5afcbbf4..abc91bac 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -78,6 +78,11 @@ pub fn run() -> Result<(), JsValue> { APPLICATION.with(|application| { *application.borrow_mut() = Some(app.run_embedded(|cx| { + cx.bind_keys([ + KeyBinding::new("cmd-d", chat_ui::ToggleDictation, None), + KeyBinding::new("ctrl-d", chat_ui::ToggleDictation, None), + ]); + gpui_kit::init(cx); load_embedded_fonts(cx); -- 2.54.0