From 6ce20927eec6908522fe6068e445b7d6a53412e9 Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Fri, 30 Oct 2026 10:37:29 +0700 Subject: [PATCH 1/4] wip --- Cargo.lock | 75 ++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + crates/chat_ui/src/actions.rs | 2 + crates/chat_ui/src/lib.rs | 53 +++++++++++++----------- crates/nip29_ui/src/lib.rs | 14 ++++--- crates/nip29_ui/src/message.rs | 43 +++++++++++++++++-- crates/speech/Cargo.toml | 1 + crates/speech/src/lib.rs | 1 + crates/speech/src/speaker.rs | 47 +++++++++++++++++++++ 9 files changed, 205 insertions(+), 32 deletions(-) create mode 100644 crates/speech/src/speaker.rs diff --git a/Cargo.lock b/Cargo.lock index ef3f2843..b132544c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2155,6 +2155,27 @@ dependencies = [ "wio", ] +[[package]] +name = "dyn-clonable" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a36efbb9bfd58e1723780aa04b61aba95ace6a05d9ffabfdb0b43672552f0805" +dependencies = [ + "dyn-clonable-impl", + "dyn-clone", +] + +[[package]] +name = "dyn-clonable-impl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8671d54058979a37a26f3511fbf8d198ba1aa35ffb202c42587d918d77213a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -5939,6 +5960,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", "objc2 0.6.5", "objc2-foundation 0.3.2", ] @@ -6436,6 +6459,15 @@ dependencies = [ "ureq", ] +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + [[package]] name = "page_size" version = "0.6.0" @@ -8372,6 +8404,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "tts", "wasm-bindgen", "web-sys", ] @@ -8443,6 +8476,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ssip" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b81f5ada7d4745722e62f8be4aca56abcf0e0e46e409612ac94a4a3a2b471d39" + +[[package]] +name = "ssip-client-async" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfbcb0ed774006ab9f14d4595fc02668cd7f5e662a87ff82264e6506238f47f" +dependencies = [ + "ssip", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -9289,6 +9337,33 @@ dependencies = [ "core_maths", ] +[[package]] +name = "tts" +version = "0.27.0" +source = "git+https://github.com/ndarilek/tts-rs#8fbcb720cb86c5166a9ed46272e3777f742da18e" +dependencies = [ + "block2 0.6.2", + "dyn-clonable", + "hound", + "jni 0.22.4", + "libloading", + "ndk-context", + "objc2 0.6.5", + "objc2-avf-audio", + "objc2-foundation 0.3.2", + "oxilangtag", + "parking_lot", + "ssip", + "ssip-client-async", + "tempfile", + "thiserror 2.0.21", + "tracing", + "wasm-bindgen", + "web-sys", + "windows 0.62.2", + "zbus", +] + [[package]] name = "tungstenite" version = "0.28.0" diff --git a/Cargo.toml b/Cargo.toml index 3c31e66e..48581112 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ rand = { version = "0.10", default-features = false, features = [ "std", "sys_rn # Others anyhow = "1.0.44" +tts = { git = "https://github.com/ndarilek/tts-rs" } chrono = { version = "0.4.38", features = ["wasmbind"] } futures = "0.3" itertools = "0.13.0" diff --git a/crates/chat_ui/src/actions.rs b/crates/chat_ui/src/actions.rs index 2ef1f3f0..5b312ae9 100644 --- a/crates/chat_ui/src/actions.rs +++ b/crates/chat_ui/src/actions.rs @@ -14,6 +14,8 @@ pub enum Command { Relays(PublicKey), Njump(PublicKey), Trace(EventId), + Speak(EventId), + StopSpeaking, } // Start or stop dictation in the focused composer diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index f50dcb44..a674d126 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -30,6 +30,7 @@ use person::{Directory, Person}; use regex::Regex; use settings::{AppSettings, SignerKind}; use smallvec::{SmallVec, smallvec}; +use speech::speaker::Speaker; use state::{ FileAttachment, LocalFile, NostrRegistry, PickedFile, download_and_decrypt_file, pick_file, }; @@ -65,65 +66,45 @@ pub fn init(room: WeakEntity, window: &mut Window, cx: &mut App) -> Entity pub struct ChatPanel { id: SharedString, focus_handle: FocusHandle, - /// Chat Room room: WeakEntity, - /// Person directory shared with the room directory: Entity, - /// Message list state list_state: ListState, - /// All messages (sorted by created_at) messages: Vec, - /// O(1) message lookup by EventId message_index: HashMap, - /// All reactions reactions: BTreeMap, PublicKey)>>, - /// Mapping message ids to their rendered texts rendered_texts_by_id: BTreeMap, - /// Mapping message (rumor event) ids to their reports reports_by_id: Arc>>>, - /// Chat input state input: Entity, - /// Dictation session for the chat input speech: Entity, - /// When the running dictation started, for the elapsed counter speech_started: Option, - + /// Reads messages aloud + speaker: Speaker, /// Subject input state subject_input: Entity, - /// Subject bar visibility subject_bar: Entity, - /// Replies to replies_to: Entity>, - /// Media Attachment attachments: Entity>, - /// Uploaded, encrypted file attachments which are not sent yet encrypted_attachments: Entity>, - /// Decrypted attachments of file messages, by message id decrypted_files: HashMap, - /// Upload state uploading: bool, - - /// Async operations tasks: Vec>>, - - /// Event subscriptions subscriptions: SmallVec<[Subscription; 4]>, } @@ -253,6 +234,7 @@ impl ChatPanel { input, speech, speech_started: None, + speaker: Speaker::new(), subject_input, subject_bar, replies_to, @@ -955,6 +937,20 @@ impl ChatPanel { Command::Trace(id) => { self.open_trace(id, window, cx); } + Command::Speak(id) => { + let text = self.message(id).map(|message| message.preview()); + + if let Some(text) = text + && let Err(e) = self.speaker.speak(&text) + { + window.push_notification(Notification::error(e.to_string()), cx); + } + } + Command::StopSpeaking => { + if let Err(error) = self.speaker.stop() { + window.push_notification(Notification::error(error.to_string()), cx); + } + } } } @@ -1464,9 +1460,20 @@ impl ChatPanel { actions }; + let speaking = self.speaker.speaking(); + actions.dropdown(move |menu, _window, _cx| { - menu.menu("Copy author", Box::new(Command::Copy(public_key))) + let menu = menu + .menu("Copy author", Box::new(Command::Copy(public_key))) .menu("Seen on", Box::new(Command::Trace(dropdown_id))) + .separator() + .menu("Read aloud", Box::new(Command::Speak(dropdown_id))); + + if speaking { + menu.menu("Stop speaking", Box::new(Command::StopSpeaking)) + } else { + menu + } }) } diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 21a61b96..b1aaf717 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -26,6 +26,7 @@ use person::Directory; use serde::Deserialize; use settings::AppSettings; use smallvec::{SmallVec, smallvec}; +use speech::speaker::Speaker; use state::{NostrRegistry, pick_file, upload_file}; use ui::{Avatar, RenderedText, WelcomeMessage, rendered_text}; use util::{attachment_name, display_name, opens_run, pin_label, report}; @@ -320,6 +321,8 @@ pub struct GroupPanel { speech: Entity, /// When the running dictation started, for the elapsed counter speech_started: Option, + /// Reads messages aloud + speaker: Speaker, /// Uploaded, non-encrypted attachments waiting to be sent. attachments: Vec, /// Whether an upload is in flight. @@ -419,9 +422,10 @@ impl GroupPanel { } }, )); + this.track_participants(cx); }); - let mut this = Self { + Self { focus_handle: cx.focus_handle(), group: group.downgrade(), directory: Directory::new(cx), @@ -433,15 +437,12 @@ impl GroupPanel { input, speech, speech_started: None, + speaker: Speaker::new(), attachments: Vec::new(), uploading: false, tasks: smallvec![], _subscriptions: subscriptions, - }; - - this.track_participants(cx); - - this + } } /// Track the keys rendered by this panel: members, message authors, and @@ -1059,6 +1060,7 @@ impl GroupPanel { panel: cx.weak_entity(), admin, pinned, + speaking: self.speaker.speaking(), persons: self.directory.clone(), }, cx, diff --git a/crates/nip29_ui/src/message.rs b/crates/nip29_ui/src/message.rs index 6c670673..552383af 100644 --- a/crates/nip29_ui/src/message.rs +++ b/crates/nip29_ui/src/message.rs @@ -8,7 +8,8 @@ use gpui::{ use gpui_kit::assets::IconName; use gpui_kit::base::StyledExt as _; use gpui_kit::component::menu::{ContextMenuExt as _, PopupMenuItem}; -use gpui_kit::component::{ActiveTheme, Icon, Sizable, h_flex}; +use gpui_kit::component::notification::Notification; +use gpui_kit::component::{ActiveTheme, Icon, Sizable, WindowExt, h_flex}; use nip29::{Activity, Group, Reaction}; use nostr_sdk::prelude::{EventId, Kind}; use person::Directory; @@ -26,6 +27,8 @@ pub(crate) struct Actions { pub panel: WeakEntity, pub admin: bool, pub pinned: bool, + /// Whether the panel's speaker is currently saying something + pub speaking: bool, pub persons: Entity, } @@ -88,9 +91,9 @@ pub(crate) fn render( let pinned = actions.pinned; let group = actions.group; let panel = actions.panel; + let speaking = actions.speaking; + let text = message.preview(); - // Reading the panel here would re-enter its ongoing update, so resolve - // the current user's custom emojis through the shared person directory. let custom = nostr .read(cx) .current_user() @@ -139,6 +142,40 @@ pub(crate) fn render( } }); + let menu = menu.separator().item({ + let panel = panel.clone(); + let text = text.clone(); + + PopupMenuItem::new("Read aloud").on_click(move |_event, _window, cx| { + if let Err(error) = panel.update_in(cx, |panel, window, cx| { + if let Err(error) = panel.speaker.speak(&text) { + window.push_notification(Notification::error(error.to_string()), cx); + } + }) { + log::warn!("nip29: reading a message aloud failed: {error}"); + } + }) + }); + + let menu = if speaking { + menu.item({ + let panel = panel.clone(); + + PopupMenuItem::new("Stop speaking").on_click(move |_event, _window, cx| { + if let Err(error) = panel.update_in(cx, |panel, window, cx| { + if let Err(error) = panel.speaker.stop() { + window + .push_notification(Notification::error(error.to_string()), cx); + } + }) { + log::warn!("nip29: stopping the speaker failed: {error}"); + } + }) + }) + } else { + menu + }; + if !admin { return menu; } diff --git a/crates/speech/Cargo.toml b/crates/speech/Cargo.toml index 724ebf75..37c949f3 100644 --- a/crates/speech/Cargo.toml +++ b/crates/speech/Cargo.toml @@ -11,6 +11,7 @@ gpui.workspace = true gpui-kit = { workspace = true, features = ["speech"] } flume.workspace = true instant.workspace = true +tts.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] common = { path = "../common" } diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs index d4cfcdcb..7db0f630 100644 --- a/crates/speech/src/lib.rs +++ b/crates/speech/src/lib.rs @@ -2,6 +2,7 @@ pub mod model; #[cfg(not(target_arch = "wasm32"))] mod parakeet; +pub mod speaker; #[cfg(target_arch = "wasm32")] mod web; diff --git a/crates/speech/src/speaker.rs b/crates/speech/src/speaker.rs new file mode 100644 index 00000000..b5e00751 --- /dev/null +++ b/crates/speech/src/speaker.rs @@ -0,0 +1,47 @@ +use tts::Tts; + +/// Reads text aloud through the platform's speech backend, created on first use. +pub struct Speaker { + tts: Option, +} + +impl Default for Speaker { + fn default() -> Self { + Self::new() + } +} + +impl Speaker { + pub fn new() -> Self { + Self { tts: None } + } + + /// Speak the text, interrupting what this speaker is already saying. + pub fn speak(&mut self, text: &str) -> Result<(), tts::Error> { + let tts = match self.tts.as_ref() { + Some(tts) => tts, + None => self.tts.insert(Tts::default()?), + }; + + tts.speak(text, true)?; + Ok(()) + } + + /// Stop speaking; a speaker that never spoke is already silent. + pub fn stop(&mut self) -> Result<(), tts::Error> { + match self.tts.as_ref() { + Some(tts) => tts.stop()?, + None => return Ok(()), + } + + Ok(()) + } + + /// Best-effort speaking state; a speaker that never spoke is silent. + pub fn speaking(&self) -> bool { + self.tts + .as_ref() + .and_then(|tts| tts.is_speaking().ok()) + .unwrap_or(false) + } +} -- 2.54.0 From 905a339e3e9b6082f9521a9ccf13b18f8f84a2c2 Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Fri, 30 Oct 2026 12:51:05 +0700 Subject: [PATCH 2/4] update ui --- assets/icons/speech.svg | 3 ++ crates/chat_ui/src/actions.rs | 2 -- crates/chat_ui/src/lib.rs | 36 +++++++---------------- crates/nip29_ui/src/lib.rs | 4 +-- crates/nip29_ui/src/message.rs | 54 ++++++++-------------------------- crates/ui/src/icon.rs | 2 ++ crates/ui/src/message.rs | 28 ++++++++++++++++-- 7 files changed, 56 insertions(+), 73 deletions(-) create mode 100644 assets/icons/speech.svg diff --git a/assets/icons/speech.svg b/assets/icons/speech.svg new file mode 100644 index 00000000..6b057003 --- /dev/null +++ b/assets/icons/speech.svg @@ -0,0 +1,3 @@ + + + diff --git a/crates/chat_ui/src/actions.rs b/crates/chat_ui/src/actions.rs index 5b312ae9..2ef1f3f0 100644 --- a/crates/chat_ui/src/actions.rs +++ b/crates/chat_ui/src/actions.rs @@ -14,8 +14,6 @@ pub enum Command { Relays(PublicKey), Njump(PublicKey), Trace(EventId), - Speak(EventId), - StopSpeaking, } // Start or stop dictation in the focused composer diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index a674d126..aaddc47c 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -937,20 +937,6 @@ impl ChatPanel { Command::Trace(id) => { self.open_trace(id, window, cx); } - Command::Speak(id) => { - let text = self.message(id).map(|message| message.preview()); - - if let Some(text) = text - && let Err(e) = self.speaker.speak(&text) - { - window.push_notification(Notification::error(e.to_string()), cx); - } - } - Command::StopSpeaking => { - if let Err(error) = self.speaker.stop() { - window.push_notification(Notification::error(error.to_string()), cx); - } - } } } @@ -1445,6 +1431,7 @@ impl ChatPanel { let reply_id = *id; let dropdown_id = *id; + let speech_id = *id; let public_key = *public_key; let actions = actions.reply(cx.listener(move |this, _event, _window, cx| { @@ -1460,20 +1447,19 @@ impl ChatPanel { actions }; - let speaking = self.speaker.speaking(); + let actions = actions.speech(cx.listener(move |this, _event, window, cx| { + let text = this.message(&speech_id).map(|message| message.preview()); + + if let Some(text) = text + && let Err(error) = this.speaker.speak(&text) + { + window.push_notification(Notification::error(error.to_string()), cx); + } + })); actions.dropdown(move |menu, _window, _cx| { - let menu = menu - .menu("Copy author", Box::new(Command::Copy(public_key))) + menu.menu("Copy author", Box::new(Command::Copy(public_key))) .menu("Seen on", Box::new(Command::Trace(dropdown_id))) - .separator() - .menu("Read aloud", Box::new(Command::Speak(dropdown_id))); - - if speaking { - menu.menu("Stop speaking", Box::new(Command::StopSpeaking)) - } else { - menu - } }) } diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index b1aaf717..093dc9b9 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -445,8 +445,7 @@ impl GroupPanel { } } - /// Track the keys rendered by this panel: members, message authors, and - /// activity subjects + /// Track the keys rendered by this panel. fn track_participants(&mut self, cx: &mut Context) { let Some(group) = self.group.upgrade() else { return; @@ -1060,7 +1059,6 @@ impl GroupPanel { panel: cx.weak_entity(), admin, pinned, - speaking: self.speaker.speaking(), persons: self.directory.clone(), }, cx, diff --git a/crates/nip29_ui/src/message.rs b/crates/nip29_ui/src/message.rs index 552383af..8b356f0a 100644 --- a/crates/nip29_ui/src/message.rs +++ b/crates/nip29_ui/src/message.rs @@ -16,8 +16,8 @@ use person::Directory; use settings::AppSettings; use state::NostrRegistry; use ui::{ - Avatar, MessageMedia, MessageRow, REACTION_EMOJIS, ReactionBar, ReplyPreview, group_reactions, - react_menu, + Avatar, MessageActions, MessageMedia, MessageRow, REACTION_EMOJIS, ReactionBar, ReplyPreview, + group_reactions, react_menu, }; use crate::GroupPanel; @@ -27,8 +27,6 @@ pub(crate) struct Actions { pub panel: WeakEntity, pub admin: bool, pub pinned: bool, - /// Whether the panel's speaker is currently saying something - pub speaking: bool, pub persons: Entity, } @@ -90,9 +88,17 @@ pub(crate) fn render( let admin = actions.admin; let pinned = actions.pinned; let group = actions.group; - let panel = actions.panel; - let speaking = actions.speaking; - let text = message.preview(); + let speech_panel = actions.panel; + let panel = speech_panel.clone(); + let speech_text = message.preview(); + + let row = row.overlay(MessageActions::new().speech(move |_event, _window, cx| { + _ = speech_panel.update_in(cx, |panel, window, cx| { + if let Err(error) = panel.speaker.speak(&speech_text) { + window.push_notification(Notification::error(error.to_string()), cx); + } + }); + })); let custom = nostr .read(cx) @@ -142,40 +148,6 @@ pub(crate) fn render( } }); - let menu = menu.separator().item({ - let panel = panel.clone(); - let text = text.clone(); - - PopupMenuItem::new("Read aloud").on_click(move |_event, _window, cx| { - if let Err(error) = panel.update_in(cx, |panel, window, cx| { - if let Err(error) = panel.speaker.speak(&text) { - window.push_notification(Notification::error(error.to_string()), cx); - } - }) { - log::warn!("nip29: reading a message aloud failed: {error}"); - } - }) - }); - - let menu = if speaking { - menu.item({ - let panel = panel.clone(); - - PopupMenuItem::new("Stop speaking").on_click(move |_event, _window, cx| { - if let Err(error) = panel.update_in(cx, |panel, window, cx| { - if let Err(error) = panel.speaker.stop() { - window - .push_notification(Notification::error(error.to_string()), cx); - } - }) { - log::warn!("nip29: stopping the speaker failed: {error}"); - } - }) - }) - } else { - menu - }; - if !admin { return menu; } diff --git a/crates/ui/src/icon.rs b/crates/ui/src/icon.rs index 2d723053..473bb275 100644 --- a/crates/ui/src/icon.rs +++ b/crates/ui/src/icon.rs @@ -26,6 +26,7 @@ pub enum IconName { PlusChat, Profile, Relay, + Speech, Search, Settings, Settings2, @@ -58,6 +59,7 @@ impl IconNamed for IconName { Self::PlusChat => "icons/plus-chat.svg", Self::Profile => "icons/profile.svg", Self::Relay => "icons/relay.svg", + Self::Speech => "icons/speech.svg", Self::Search => "icons/search.svg", Self::Settings => "icons/settings.svg", Self::Settings2 => "icons/settings2.svg", diff --git a/crates/ui/src/message.rs b/crates/ui/src/message.rs index 745f5108..b50305cf 100644 --- a/crates/ui/src/message.rs +++ b/crates/ui/src/message.rs @@ -597,13 +597,14 @@ pub fn react_menu( }) } -/// A hovered message's action bar: quick reactions, reply, copy, and an overflow menu. +/// A hovered message's action bar: quick reactions, reply, copy, read aloud, and an overflow menu. #[derive(IntoElement)] pub struct MessageActions { style: StyleRefinement, reactions: SmallVec<[(&'static str, SharedClickListener); 4]>, reply: Option, copy: Option<(ElementId, SharedString)>, + speech: Option, dropdown: Option, } @@ -615,6 +616,7 @@ impl MessageActions { reactions: SmallVec::new(), reply: None, copy: None, + speech: None, dropdown: None, } } @@ -642,6 +644,15 @@ impl MessageActions { self } + /// Add the read-aloud button. + pub fn speech( + mut self, + handler: impl Fn(&ClickEvent, &mut Window, &mut App) + 'static, + ) -> Self { + self.speech = Some(Rc::new(handler)); + self + } + /// Fill the bar's overflow menu. pub fn dropdown( mut self, @@ -671,10 +682,13 @@ impl RenderOnce for MessageActions { reactions, reply, copy, + speech, dropdown, } = self; - let has_center = reply.is_some() || copy.is_some(); + // The leading separator only separates reactions from the center buttons + let has_center = + !reactions.is_empty() && (reply.is_some() || copy.is_some() || speech.is_some()); h_flex() .p_0p5() @@ -712,6 +726,16 @@ impl RenderOnce for MessageActions { .when_some(copy, |this, (id, value)| { this.child(Clipboard::new(id).value(value).tooltip("Copy").small()) }) + .when_some(speech, |this, speech| { + this.child( + Button::new("speech") + .icon(crate::IconName::Speech) + .tooltip("Read aloud") + .small() + .ghost() + .on_click(move |event, window, cx| speech(event, window, cx)), + ) + }) .when_some(dropdown, |this, dropdown| { this.child(div().flex_shrink_0().h_4().w_px().bg(cx.theme().border)) .child( -- 2.54.0 From 41837e487b8a628c05b6e6ef4e252531ef2f33dd Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Fri, 30 Oct 2026 14:03:14 +0700 Subject: [PATCH 3/4] add reveal text --- crates/chat_ui/src/lib.rs | 53 ++++++++++++++++++++++------ crates/nip29_ui/src/lib.rs | 62 +++++++++++++++++++++++---------- crates/nip29_ui/src/message.rs | 9 +++-- crates/speech/src/speaker.rs | 63 +++++++++++++++++++++++++++------- crates/ui/src/message.rs | 50 ++++++++++++++++++++++++--- 5 files changed, 189 insertions(+), 48 deletions(-) diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index aaddc47c..3b88734f 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -36,8 +36,8 @@ use state::{ }; use ui::{ Avatar, COMPACT_REACTION_EMOJIS, IconName, MessageActions, MessageMedia, MessageRow, - REACTION_EMOJIS, ReactionBar, RenderedText, ReplyPreview, WelcomeMessage, group_reactions, - opens_run, rendered_text, + REACTION_EMOJIS, ReactionBar, RenderedText, ReplyPreview, RevealText, WelcomeMessage, + group_reactions, opens_run, rendered_text, }; use crate::file::*; @@ -90,6 +90,9 @@ pub struct ChatPanel { speech_started: Option, /// Reads messages aloud speaker: Speaker, + + /// The message being revealed in step with speech, with its estimated duration + speech_reveal: Option<(EventId, Duration)>, /// Subject input state subject_input: Entity, /// Subject bar visibility @@ -222,6 +225,20 @@ impl ChatPanel { this.get_messages(window, cx); }); + // End the speech reveal when the utterance finishes ahead of its estimate + let speaker = Speaker::new(); + let end_events = speaker.end_events(); + let reveal_end_task = cx.spawn(async move |this, cx| { + while end_events.recv_async().await.is_ok() { + this.update(cx, |this, cx| { + this.speech_reveal = None; + cx.notify(); + })?; + } + + Ok(()) + }); + Self { focus_handle: cx.focus_handle(), id, @@ -234,7 +251,8 @@ impl ChatPanel { input, speech, speech_started: None, - speaker: Speaker::new(), + speaker, + speech_reveal: None, subject_input, subject_bar, replies_to, @@ -245,7 +263,7 @@ impl ChatPanel { reports_by_id, uploading: false, subscriptions, - tasks: vec![], + tasks: vec![reveal_end_task], } } @@ -1174,6 +1192,15 @@ impl ChatPanel { nostr.read(cx).current_user(), )); + let body = match self + .speech_reveal + .as_ref() + .filter(|(reveal_id, _)| *reveal_id == id) + { + Some((_, duration)) => RevealText::new(message.preview(), *duration).into_any_element(), + None => rendered_text, + }; + MessageRow::new(ix) .show_author(show_author) .hide_avatar(hide_avatar) @@ -1198,7 +1225,7 @@ impl ChatPanel { .when(has_replies, |this| { this.children(self.render_message_replies(replies, cx)) }) - .when(message.file.is_none(), |this| this.child(rendered_text)) + .when(message.file.is_none(), |this| this.child(body)) .child(MessageMedia::new(&message.media)) .when_some(message.file.as_ref(), |this, file| { this.child(self.render_message_file(&id, file, cx)) @@ -1431,7 +1458,6 @@ impl ChatPanel { let reply_id = *id; let dropdown_id = *id; - let speech_id = *id; let public_key = *public_key; let actions = actions.reply(cx.listener(move |this, _event, _window, cx| { @@ -1447,13 +1473,20 @@ impl ChatPanel { actions }; + let speech_id = *id; let actions = actions.speech(cx.listener(move |this, _event, window, cx| { let text = this.message(&speech_id).map(|message| message.preview()); - if let Some(text) = text - && let Err(error) = this.speaker.speak(&text) - { - window.push_notification(Notification::error(error.to_string()), cx); + if let Some(text) = text { + match this.speaker.speak(&text) { + Ok(duration) => { + this.speech_reveal = Some((speech_id, duration)); + cx.notify(); + } + Err(error) => { + window.push_notification(Notification::error(error.to_string()), cx) + } + } } })); diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 093dc9b9..02d34bde 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -28,7 +28,7 @@ use settings::AppSettings; use smallvec::{SmallVec, smallvec}; use speech::speaker::Speaker; use state::{NostrRegistry, pick_file, upload_file}; -use ui::{Avatar, RenderedText, WelcomeMessage, rendered_text}; +use ui::{Avatar, RenderedText, RevealText, WelcomeMessage, rendered_text}; use util::{attachment_name, display_name, opens_run, pin_label, report}; mod details; @@ -323,6 +323,8 @@ pub struct GroupPanel { speech_started: Option, /// Reads messages aloud speaker: Speaker, + /// The message being revealed in step with speech, with its estimated duration + speech_reveal: Option<(EventId, Duration)>, /// Uploaded, non-encrypted attachments waiting to be sent. attachments: Vec, /// Whether an upload is in flight. @@ -425,6 +427,20 @@ impl GroupPanel { this.track_participants(cx); }); + // End the speech reveal when the utterance finishes ahead of its estimate + let speaker = Speaker::new(); + let end_events = speaker.end_events(); + let reveal_end_task = cx.spawn(async move |this, cx| { + while end_events.recv_async().await.is_ok() { + this.update(cx, |this, cx| { + this.speech_reveal = None; + cx.notify(); + })?; + } + + Ok(()) + }); + Self { focus_handle: cx.focus_handle(), group: group.downgrade(), @@ -437,10 +453,11 @@ impl GroupPanel { input, speech, speech_started: None, - speaker: Speaker::new(), + speaker, + speech_reveal: None, attachments: Vec::new(), uploading: false, - tasks: smallvec![], + tasks: smallvec![reveal_end_task], _subscriptions: subscriptions, } } @@ -1023,22 +1040,29 @@ impl GroupPanel { let nostr = NostrRegistry::global(cx); let directory = self.directory.clone(); - let content = self - .rendered_texts_by_id - .entry(message.id) - .or_insert_with(|| { - rendered_text( - &message.content, - message - .mentions - .iter() - .map(|mention| (mention.range.clone(), mention.public_key)), - &message.emojis, - &directory, - cx, - ) - }) - .element(ix.into(), window, cx); + let content = match self + .speech_reveal + .as_ref() + .filter(|(reveal_id, _)| *reveal_id == message.id) + { + Some((_, duration)) => RevealText::new(message.preview(), *duration).into_any_element(), + None => self + .rendered_texts_by_id + .entry(message.id) + .or_insert_with(|| { + rendered_text( + &message.content, + message + .mentions + .iter() + .map(|mention| (mention.range.clone(), mention.public_key)), + &message.emojis, + &directory, + cx, + ) + }) + .element(ix.into(), window, cx), + }; let admin = nostr .read(cx) diff --git a/crates/nip29_ui/src/message.rs b/crates/nip29_ui/src/message.rs index 8b356f0a..ef5d89c1 100644 --- a/crates/nip29_ui/src/message.rs +++ b/crates/nip29_ui/src/message.rs @@ -91,11 +91,16 @@ pub(crate) fn render( let speech_panel = actions.panel; let panel = speech_panel.clone(); let speech_text = message.preview(); + let speech_id = message.id; let row = row.overlay(MessageActions::new().speech(move |_event, _window, cx| { _ = speech_panel.update_in(cx, |panel, window, cx| { - if let Err(error) = panel.speaker.speak(&speech_text) { - window.push_notification(Notification::error(error.to_string()), cx); + match panel.speaker.speak(&speech_text) { + Ok(duration) => { + panel.speech_reveal = Some((speech_id, duration)); + cx.notify(); + } + Err(error) => window.push_notification(Notification::error(error.to_string()), cx), } }); })); diff --git a/crates/speech/src/speaker.rs b/crates/speech/src/speaker.rs index b5e00751..ed5138bc 100644 --- a/crates/speech/src/speaker.rs +++ b/crates/speech/src/speaker.rs @@ -1,33 +1,54 @@ +use std::time::Duration; + use tts::Tts; +/// Words per minute of an engine at its normal rate, for estimating speech duration. +const NORMAL_WPM: f32 = 160.0; + /// Reads text aloud through the platform's speech backend, created on first use. pub struct Speaker { tts: Option, -} - -impl Default for Speaker { - fn default() -> Self { - Self::new() - } + end_tx: flume::Sender<()>, + end_rx: flume::Receiver<()>, } impl Speaker { pub fn new() -> Self { - Self { tts: None } + let (end_tx, end_rx) = flume::unbounded(); + + Self { + tts: None, + end_tx, + end_rx, + } + } + + /// Cloneable stream of utterance-finish events. + pub fn end_events(&self) -> flume::Receiver<()> { + self.end_rx.clone() } /// Speak the text, interrupting what this speaker is already saying. - pub fn speak(&mut self, text: &str) -> Result<(), tts::Error> { + pub fn speak(&mut self, text: &str) -> Result { + let end_tx = self.end_tx.clone(); let tts = match self.tts.as_ref() { Some(tts) => tts, - None => self.tts.insert(Tts::default()?), + None => self.tts.insert({ + let tts = Tts::default()?; + if let Err(error) = tts.on_utterance_end(move |_| { + // A send error only means the panel dropped its event loop. + let _ = end_tx.send(()); + }) { + log::warn!("utterance end events unavailable ({error})"); + } + tts + }), }; - tts.speak(text, true)?; - Ok(()) + Ok(Self::estimate(tts, text)) } - /// Stop speaking; a speaker that never spoke is already silent. + /// Stop speaking, a speaker that never spoke is already silent. pub fn stop(&mut self) -> Result<(), tts::Error> { match self.tts.as_ref() { Some(tts) => tts.stop()?, @@ -37,11 +58,27 @@ impl Speaker { Ok(()) } - /// Best-effort speaking state; a speaker that never spoke is silent. + /// Best-effort speaking state, a speaker that never spoke is silent. pub fn speaking(&self) -> bool { self.tts .as_ref() .and_then(|tts| tts.is_speaking().ok()) .unwrap_or(false) } + + /// How long speaking the text is expected to take at the engine's current speed. + fn estimate(tts: &Tts, text: &str) -> Duration { + let words = text.split_whitespace().count().max(1) as f32; + let speed = match (tts.get_rate(), tts.normal_rate()) { + (Ok(rate), Ok(normal)) if normal > 0. => (rate / normal).clamp(0.2, 4.), + _ => 1., + }; + Duration::from_secs_f32(words / NORMAL_WPM * 60. / speed) + } +} + +impl Default for Speaker { + fn default() -> Self { + Self::new() + } } diff --git a/crates/ui/src/message.rs b/crates/ui/src/message.rs index b50305cf..2f28fb3f 100644 --- a/crates/ui/src/message.rs +++ b/crates/ui/src/message.rs @@ -1,12 +1,13 @@ use std::collections::BTreeMap; use std::rc::Rc; +use std::time::Duration; use gpui::prelude::FluentBuilder as _; use gpui::{ - AnyElement, App, ClickEvent, Context, ElementId, InteractiveElement as _, IntoElement, - MouseButton, MouseDownEvent, ObjectFit, ParentElement as _, RenderOnce, SharedString, - SharedUri, StatefulInteractiveElement as _, StyleRefinement, Styled, StyledImage, Window, div, - img, px, + Animation, AnimationExt as _, AnyElement, App, ClickEvent, Context, ElementId, + InteractiveElement as _, IntoElement, MouseButton, MouseDownEvent, ObjectFit, + ParentElement as _, RenderOnce, SharedString, SharedUri, StatefulInteractiveElement as _, + StyleRefinement, Styled, StyledImage, Window, div, img, px, }; use gpui_kit::assets::IconName; use gpui_kit::component::button::{Button, ButtonVariants as _}; @@ -750,3 +751,44 @@ impl RenderOnce for MessageActions { .refine_style(&style) } } + +/// A message's text revealed word by word, paced by the estimated speech duration. +#[derive(IntoElement)] +pub struct RevealText { + text: SharedString, + duration: Duration, +} + +impl RevealText { + pub fn new(text: impl Into, duration: Duration) -> Self { + Self { + text: text.into(), + duration, + } + } +} + +impl RenderOnce for RevealText { + fn render(self, _window: &mut Window, _cx: &mut App) -> impl IntoElement { + let words: Vec = self + .text + .split_whitespace() + .map(SharedString::from) + .collect(); + let total = words.len().max(1) as f32; + + div().flex().flex_wrap().gap_x_1().with_animation( + "speech-reveal", + Animation::new(self.duration).with_max_fps(30.), + move |this, progress| { + let visible = ((progress * total).ceil() as usize).min(words.len()); + + this.children(words.iter().enumerate().map(|(ix, word)| { + div() + .when(ix >= visible, |this| this.invisible()) + .child(word.clone()) + })) + }, + ) + } +} -- 2.54.0 From 348504c33a9994ba9849f69b9075e2a6be71251e Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Fri, 30 Oct 2026 14:15:14 +0700 Subject: [PATCH 4/4] . --- crates/chat_ui/src/lib.rs | 53 ++++++++++++++++++++++++---------- crates/nip29_ui/src/lib.rs | 4 +++ crates/nip29_ui/src/message.rs | 44 +++++++++++++++++++++------- crates/speech/src/speaker.rs | 4 +++ crates/ui/src/message.rs | 13 +++++---- 5 files changed, 87 insertions(+), 31 deletions(-) diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index 3b88734f..e9719a86 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -30,7 +30,7 @@ use person::{Directory, Person}; use regex::Regex; use settings::{AppSettings, SignerKind}; use smallvec::{SmallVec, smallvec}; -use speech::speaker::Speaker; +use speech::speaker::{END_GRACE, Speaker}; use state::{ FileAttachment, LocalFile, NostrRegistry, PickedFile, download_and_decrypt_file, pick_file, }; @@ -46,8 +46,7 @@ use crate::file::*; const PRIVATE_NOTICE: &str = "This conversation is private. Only members can see each other's messages."; -/// Regex matching strings that consist entirely of emoji characters, -/// zero-width joiners, variation selectors, and keycap combiners. +/// Regex matching strings that consist entirely of emoji characters. static EMOJI_RE: LazyLock = LazyLock::new(|| Regex::new(r"^[\p{Emoji}\u{200D}\u{FE0F}\u{20E3}]+$").unwrap()); @@ -90,9 +89,10 @@ pub struct ChatPanel { speech_started: Option, /// Reads messages aloud speaker: Speaker, - /// The message being revealed in step with speech, with its estimated duration speech_reveal: Option<(EventId, Duration)>, + /// Fallback reveal end for backends that cannot report utterance end + reveal_timeout: Option>>, /// Subject input state subject_input: Entity, /// Subject bar visibility @@ -253,6 +253,7 @@ impl ChatPanel { speech_started: None, speaker, speech_reveal: None, + reveal_timeout: None, subject_input, subject_bar, replies_to, @@ -1473,22 +1474,42 @@ impl ChatPanel { actions }; + let speaking = self.speech_reveal.is_some(); let speech_id = *id; - let actions = actions.speech(cx.listener(move |this, _event, window, cx| { - let text = this.message(&speech_id).map(|message| message.preview()); - if let Some(text) = text { - match this.speaker.speak(&text) { - Ok(duration) => { - this.speech_reveal = Some((speech_id, duration)); - cx.notify(); - } - Err(error) => { - window.push_notification(Notification::error(error.to_string()), cx) + let actions = actions.speech( + speaking, + cx.listener(move |this, _event, window, cx| { + let text = this.message(&speech_id).map(|message| message.preview()); + + if let Some(text) = text { + match this.speaker.speak(&text) { + Ok(duration) => { + this.speech_reveal = Some((speech_id, duration)); + + // Re-enable speaking when no end event ever arrives + let session = (speech_id, duration); + let timer = cx.background_executor().timer(duration + END_GRACE); + this.reveal_timeout = Some(cx.spawn(async move |this, cx| { + timer.await; + this.update(cx, |this, cx| { + if this.speech_reveal.as_ref() == Some(&session) { + this.speech_reveal = None; + cx.notify(); + } + })?; + + Ok(()) + })); + cx.notify(); + } + Err(error) => { + window.push_notification(Notification::error(error.to_string()), cx) + } } } - } - })); + }), + ); actions.dropdown(move |menu, _window, _cx| { menu.menu("Copy author", Box::new(Command::Copy(public_key))) diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 02d34bde..715afbb8 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -325,6 +325,8 @@ pub struct GroupPanel { speaker: Speaker, /// The message being revealed in step with speech, with its estimated duration speech_reveal: Option<(EventId, Duration)>, + /// Fallback reveal end for backends that cannot report utterance end + reveal_timeout: Option>>, /// Uploaded, non-encrypted attachments waiting to be sent. attachments: Vec, /// Whether an upload is in flight. @@ -455,6 +457,7 @@ impl GroupPanel { speech_started: None, speaker, speech_reveal: None, + reveal_timeout: None, attachments: Vec::new(), uploading: false, tasks: smallvec![reveal_end_task], @@ -1083,6 +1086,7 @@ impl GroupPanel { panel: cx.weak_entity(), admin, pinned, + speaking: self.speech_reveal.is_some(), persons: self.directory.clone(), }, cx, diff --git a/crates/nip29_ui/src/message.rs b/crates/nip29_ui/src/message.rs index ef5d89c1..0eda88d7 100644 --- a/crates/nip29_ui/src/message.rs +++ b/crates/nip29_ui/src/message.rs @@ -14,6 +14,7 @@ use nip29::{Activity, Group, Reaction}; use nostr_sdk::prelude::{EventId, Kind}; use person::Directory; use settings::AppSettings; +use speech::speaker::END_GRACE; use state::NostrRegistry; use ui::{ Avatar, MessageActions, MessageMedia, MessageRow, REACTION_EMOJIS, ReactionBar, ReplyPreview, @@ -27,6 +28,8 @@ pub(crate) struct Actions { pub panel: WeakEntity, pub admin: bool, pub pinned: bool, + /// Whether a message is currently being spoken, disabling the speech buttons + pub speaking: bool, pub persons: Entity, } @@ -92,18 +95,39 @@ pub(crate) fn render( let panel = speech_panel.clone(); let speech_text = message.preview(); let speech_id = message.id; + let speaking = actions.speaking; - let row = row.overlay(MessageActions::new().speech(move |_event, _window, cx| { - _ = speech_panel.update_in(cx, |panel, window, cx| { - match panel.speaker.speak(&speech_text) { - Ok(duration) => { - panel.speech_reveal = Some((speech_id, duration)); - cx.notify(); + let row = row.overlay( + MessageActions::new().speech(speaking, move |_event, _window, cx| { + _ = speech_panel.update_in(cx, |panel, window, cx| { + match panel.speaker.speak(&speech_text) { + Ok(duration) => { + panel.speech_reveal = Some((speech_id, duration)); + + let session = (speech_id, duration); + let timer = cx.background_executor().timer(duration + END_GRACE); + + panel.reveal_timeout = Some(cx.spawn(async move |this, cx| { + timer.await; + this.update(cx, |this, cx| { + if this.speech_reveal.as_ref() == Some(&session) { + this.speech_reveal = None; + cx.notify(); + } + })?; + + Ok(()) + })); + + cx.notify(); + } + Err(error) => { + window.push_notification(Notification::error(error.to_string()), cx) + } } - Err(error) => window.push_notification(Notification::error(error.to_string()), cx), - } - }); - })); + }); + }), + ); let custom = nostr .read(cx) diff --git a/crates/speech/src/speaker.rs b/crates/speech/src/speaker.rs index ed5138bc..e690fdbe 100644 --- a/crates/speech/src/speaker.rs +++ b/crates/speech/src/speaker.rs @@ -5,6 +5,10 @@ use tts::Tts; /// Words per minute of an engine at its normal rate, for estimating speech duration. const NORMAL_WPM: f32 = 160.0; +/// Extra time past the estimate before speech is considered finished, for backends that +/// cannot report utterance end. +pub const END_GRACE: Duration = Duration::from_secs(5); + /// Reads text aloud through the platform's speech backend, created on first use. pub struct Speaker { tts: Option, diff --git a/crates/ui/src/message.rs b/crates/ui/src/message.rs index 2f28fb3f..89b124ce 100644 --- a/crates/ui/src/message.rs +++ b/crates/ui/src/message.rs @@ -14,7 +14,8 @@ use gpui_kit::component::button::{Button, ButtonVariants as _}; use gpui_kit::component::clipboard::Clipboard; use gpui_kit::component::menu::{DropdownMenu as _, PopupMenu, PopupMenuItem}; use gpui_kit::component::{ - ActiveTheme, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex, v_flex, + ActiveTheme, Disableable, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex, + v_flex, }; use smallvec::SmallVec; @@ -605,7 +606,7 @@ pub struct MessageActions { reactions: SmallVec<[(&'static str, SharedClickListener); 4]>, reply: Option, copy: Option<(ElementId, SharedString)>, - speech: Option, + speech: Option<(bool, SharedClickListener)>, dropdown: Option, } @@ -645,12 +646,13 @@ impl MessageActions { self } - /// Add the read-aloud button. + /// Add the read-aloud button, disabled while a message is already being spoken. pub fn speech( mut self, + disabled: bool, handler: impl Fn(&ClickEvent, &mut Window, &mut App) + 'static, ) -> Self { - self.speech = Some(Rc::new(handler)); + self.speech = Some((disabled, Rc::new(handler))); self } @@ -727,13 +729,14 @@ impl RenderOnce for MessageActions { .when_some(copy, |this, (id, value)| { this.child(Clipboard::new(id).value(value).tooltip("Copy").small()) }) - .when_some(speech, |this, speech| { + .when_some(speech, |this, (disabled, speech)| { this.child( Button::new("speech") .icon(crate::IconName::Speech) .tooltip("Read aloud") .small() .ghost() + .disabled(disabled) .on_click(move |event, window, cx| speech(event, window, cx)), ) }) -- 2.54.0