diff --git a/Cargo.lock b/Cargo.lock index ef3f2843..b132544c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2155,6 +2155,27 @@ dependencies = [ "wio", ] +[[package]] +name = "dyn-clonable" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a36efbb9bfd58e1723780aa04b61aba95ace6a05d9ffabfdb0b43672552f0805" +dependencies = [ + "dyn-clonable-impl", + "dyn-clone", +] + +[[package]] +name = "dyn-clonable-impl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8671d54058979a37a26f3511fbf8d198ba1aa35ffb202c42587d918d77213a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -5939,6 +5960,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", "objc2 0.6.5", "objc2-foundation 0.3.2", ] @@ -6436,6 +6459,15 @@ dependencies = [ "ureq", ] +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + [[package]] name = "page_size" version = "0.6.0" @@ -8372,6 +8404,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "tts", "wasm-bindgen", "web-sys", ] @@ -8443,6 +8476,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ssip" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b81f5ada7d4745722e62f8be4aca56abcf0e0e46e409612ac94a4a3a2b471d39" + +[[package]] +name = "ssip-client-async" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfbcb0ed774006ab9f14d4595fc02668cd7f5e662a87ff82264e6506238f47f" +dependencies = [ + "ssip", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -9289,6 +9337,33 @@ dependencies = [ "core_maths", ] +[[package]] +name = "tts" +version = "0.27.0" +source = "git+https://github.com/ndarilek/tts-rs#8fbcb720cb86c5166a9ed46272e3777f742da18e" +dependencies = [ + "block2 0.6.2", + "dyn-clonable", + "hound", + "jni 0.22.4", + "libloading", + "ndk-context", + "objc2 0.6.5", + "objc2-avf-audio", + "objc2-foundation 0.3.2", + "oxilangtag", + "parking_lot", + "ssip", + "ssip-client-async", + "tempfile", + "thiserror 2.0.21", + "tracing", + "wasm-bindgen", + "web-sys", + "windows 0.62.2", + "zbus", +] + [[package]] name = "tungstenite" version = "0.28.0" diff --git a/Cargo.toml b/Cargo.toml index 3c31e66e..48581112 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ rand = { version = "0.10", default-features = false, features = [ "std", "sys_rn # Others anyhow = "1.0.44" +tts = { git = "https://github.com/ndarilek/tts-rs" } chrono = { version = "0.4.38", features = ["wasmbind"] } futures = "0.3" itertools = "0.13.0" diff --git a/crates/chat_ui/src/actions.rs b/crates/chat_ui/src/actions.rs index 2ef1f3f0..5b312ae9 100644 --- a/crates/chat_ui/src/actions.rs +++ b/crates/chat_ui/src/actions.rs @@ -14,6 +14,8 @@ pub enum Command { Relays(PublicKey), Njump(PublicKey), Trace(EventId), + Speak(EventId), + StopSpeaking, } // Start or stop dictation in the focused composer diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index f50dcb44..a674d126 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -30,6 +30,7 @@ use person::{Directory, Person}; use regex::Regex; use settings::{AppSettings, SignerKind}; use smallvec::{SmallVec, smallvec}; +use speech::speaker::Speaker; use state::{ FileAttachment, LocalFile, NostrRegistry, PickedFile, download_and_decrypt_file, pick_file, }; @@ -65,65 +66,45 @@ pub fn init(room: WeakEntity, window: &mut Window, cx: &mut App) -> Entity pub struct ChatPanel { id: SharedString, focus_handle: FocusHandle, - /// Chat Room room: WeakEntity, - /// Person directory shared with the room directory: Entity, - /// Message list state list_state: ListState, - /// All messages (sorted by created_at) messages: Vec, - /// O(1) message lookup by EventId message_index: HashMap, - /// All reactions reactions: BTreeMap, PublicKey)>>, - /// Mapping message ids to their rendered texts rendered_texts_by_id: BTreeMap, - /// Mapping message (rumor event) ids to their reports reports_by_id: Arc>>>, - /// Chat input state input: Entity, - /// Dictation session for the chat input speech: Entity, - /// When the running dictation started, for the elapsed counter speech_started: Option, - + /// Reads messages aloud + speaker: Speaker, /// Subject input state subject_input: Entity, - /// Subject bar visibility subject_bar: Entity, - /// Replies to replies_to: Entity>, - /// Media Attachment attachments: Entity>, - /// Uploaded, encrypted file attachments which are not sent yet encrypted_attachments: Entity>, - /// Decrypted attachments of file messages, by message id decrypted_files: HashMap, - /// Upload state uploading: bool, - - /// Async operations tasks: Vec>>, - - /// Event subscriptions subscriptions: SmallVec<[Subscription; 4]>, } @@ -253,6 +234,7 @@ impl ChatPanel { input, speech, speech_started: None, + speaker: Speaker::new(), subject_input, subject_bar, replies_to, @@ -955,6 +937,20 @@ impl ChatPanel { Command::Trace(id) => { self.open_trace(id, window, cx); } + Command::Speak(id) => { + let text = self.message(id).map(|message| message.preview()); + + if let Some(text) = text + && let Err(e) = self.speaker.speak(&text) + { + window.push_notification(Notification::error(e.to_string()), cx); + } + } + Command::StopSpeaking => { + if let Err(error) = self.speaker.stop() { + window.push_notification(Notification::error(error.to_string()), cx); + } + } } } @@ -1464,9 +1460,20 @@ impl ChatPanel { actions }; + let speaking = self.speaker.speaking(); + actions.dropdown(move |menu, _window, _cx| { - menu.menu("Copy author", Box::new(Command::Copy(public_key))) + let menu = menu + .menu("Copy author", Box::new(Command::Copy(public_key))) .menu("Seen on", Box::new(Command::Trace(dropdown_id))) + .separator() + .menu("Read aloud", Box::new(Command::Speak(dropdown_id))); + + if speaking { + menu.menu("Stop speaking", Box::new(Command::StopSpeaking)) + } else { + menu + } }) } diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 21a61b96..b1aaf717 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -26,6 +26,7 @@ use person::Directory; use serde::Deserialize; use settings::AppSettings; use smallvec::{SmallVec, smallvec}; +use speech::speaker::Speaker; use state::{NostrRegistry, pick_file, upload_file}; use ui::{Avatar, RenderedText, WelcomeMessage, rendered_text}; use util::{attachment_name, display_name, opens_run, pin_label, report}; @@ -320,6 +321,8 @@ pub struct GroupPanel { speech: Entity, /// When the running dictation started, for the elapsed counter speech_started: Option, + /// Reads messages aloud + speaker: Speaker, /// Uploaded, non-encrypted attachments waiting to be sent. attachments: Vec, /// Whether an upload is in flight. @@ -419,9 +422,10 @@ impl GroupPanel { } }, )); + this.track_participants(cx); }); - let mut this = Self { + Self { focus_handle: cx.focus_handle(), group: group.downgrade(), directory: Directory::new(cx), @@ -433,15 +437,12 @@ impl GroupPanel { input, speech, speech_started: None, + speaker: Speaker::new(), attachments: Vec::new(), uploading: false, tasks: smallvec![], _subscriptions: subscriptions, - }; - - this.track_participants(cx); - - this + } } /// Track the keys rendered by this panel: members, message authors, and @@ -1059,6 +1060,7 @@ impl GroupPanel { panel: cx.weak_entity(), admin, pinned, + speaking: self.speaker.speaking(), persons: self.directory.clone(), }, cx, diff --git a/crates/nip29_ui/src/message.rs b/crates/nip29_ui/src/message.rs index 6c670673..552383af 100644 --- a/crates/nip29_ui/src/message.rs +++ b/crates/nip29_ui/src/message.rs @@ -8,7 +8,8 @@ use gpui::{ use gpui_kit::assets::IconName; use gpui_kit::base::StyledExt as _; use gpui_kit::component::menu::{ContextMenuExt as _, PopupMenuItem}; -use gpui_kit::component::{ActiveTheme, Icon, Sizable, h_flex}; +use gpui_kit::component::notification::Notification; +use gpui_kit::component::{ActiveTheme, Icon, Sizable, WindowExt, h_flex}; use nip29::{Activity, Group, Reaction}; use nostr_sdk::prelude::{EventId, Kind}; use person::Directory; @@ -26,6 +27,8 @@ pub(crate) struct Actions { pub panel: WeakEntity, pub admin: bool, pub pinned: bool, + /// Whether the panel's speaker is currently saying something + pub speaking: bool, pub persons: Entity, } @@ -88,9 +91,9 @@ pub(crate) fn render( let pinned = actions.pinned; let group = actions.group; let panel = actions.panel; + let speaking = actions.speaking; + let text = message.preview(); - // Reading the panel here would re-enter its ongoing update, so resolve - // the current user's custom emojis through the shared person directory. let custom = nostr .read(cx) .current_user() @@ -139,6 +142,40 @@ pub(crate) fn render( } }); + let menu = menu.separator().item({ + let panel = panel.clone(); + let text = text.clone(); + + PopupMenuItem::new("Read aloud").on_click(move |_event, _window, cx| { + if let Err(error) = panel.update_in(cx, |panel, window, cx| { + if let Err(error) = panel.speaker.speak(&text) { + window.push_notification(Notification::error(error.to_string()), cx); + } + }) { + log::warn!("nip29: reading a message aloud failed: {error}"); + } + }) + }); + + let menu = if speaking { + menu.item({ + let panel = panel.clone(); + + PopupMenuItem::new("Stop speaking").on_click(move |_event, _window, cx| { + if let Err(error) = panel.update_in(cx, |panel, window, cx| { + if let Err(error) = panel.speaker.stop() { + window + .push_notification(Notification::error(error.to_string()), cx); + } + }) { + log::warn!("nip29: stopping the speaker failed: {error}"); + } + }) + }) + } else { + menu + }; + if !admin { return menu; } diff --git a/crates/speech/Cargo.toml b/crates/speech/Cargo.toml index 724ebf75..37c949f3 100644 --- a/crates/speech/Cargo.toml +++ b/crates/speech/Cargo.toml @@ -11,6 +11,7 @@ gpui.workspace = true gpui-kit = { workspace = true, features = ["speech"] } flume.workspace = true instant.workspace = true +tts.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] common = { path = "../common" } diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs index d4cfcdcb..7db0f630 100644 --- a/crates/speech/src/lib.rs +++ b/crates/speech/src/lib.rs @@ -2,6 +2,7 @@ pub mod model; #[cfg(not(target_arch = "wasm32"))] mod parakeet; +pub mod speaker; #[cfg(target_arch = "wasm32")] mod web; diff --git a/crates/speech/src/speaker.rs b/crates/speech/src/speaker.rs new file mode 100644 index 00000000..b5e00751 --- /dev/null +++ b/crates/speech/src/speaker.rs @@ -0,0 +1,47 @@ +use tts::Tts; + +/// Reads text aloud through the platform's speech backend, created on first use. +pub struct Speaker { + tts: Option, +} + +impl Default for Speaker { + fn default() -> Self { + Self::new() + } +} + +impl Speaker { + pub fn new() -> Self { + Self { tts: None } + } + + /// Speak the text, interrupting what this speaker is already saying. + pub fn speak(&mut self, text: &str) -> Result<(), tts::Error> { + let tts = match self.tts.as_ref() { + Some(tts) => tts, + None => self.tts.insert(Tts::default()?), + }; + + tts.speak(text, true)?; + Ok(()) + } + + /// Stop speaking; a speaker that never spoke is already silent. + pub fn stop(&mut self) -> Result<(), tts::Error> { + match self.tts.as_ref() { + Some(tts) => tts.stop()?, + None => return Ok(()), + } + + Ok(()) + } + + /// Best-effort speaking state; a speaker that never spoke is silent. + pub fn speaking(&self) -> bool { + self.tts + .as_ref() + .and_then(|tts| tts.is_speaking().ok()) + .unwrap_or(false) + } +}