wip

Cargo.lock

@@ -2155,6 +2155,27 @@ dependencies = [
  "wio",
 ]
 
+[[package]]
+name = "dyn-clonable"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a36efbb9bfd58e1723780aa04b61aba95ace6a05d9ffabfdb0b43672552f0805"
+dependencies = [
+ "dyn-clonable-impl",
+ "dyn-clone",
+]
+
+[[package]]
+name = "dyn-clonable-impl"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e8671d54058979a37a26f3511fbf8d198ba1aa35ffb202c42587d918d77213a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "dyn-clone"
 version = "1.0.20"
@@ -5939,6 +5960,8 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"
 dependencies = [
+ "bitflags 2.13.2",
+ "block2 0.6.2",
  "objc2 0.6.5",
  "objc2-foundation 0.3.2",
 ]
@@ -6436,6 +6459,15 @@ dependencies = [
  "ureq",
 ]
 
+[[package]]
+name = "oxilangtag"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "page_size"
 version = "0.6.0"
@@ -8372,6 +8404,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2 0.10.9",
+ "tts",
  "wasm-bindgen",
  "web-sys",
 ]
@@ -8443,6 +8476,21 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "ssip"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b81f5ada7d4745722e62f8be4aca56abcf0e0e46e409612ac94a4a3a2b471d39"
+
+[[package]]
+name = "ssip-client-async"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cfbcb0ed774006ab9f14d4595fc02668cd7f5e662a87ff82264e6506238f47f"
+dependencies = [
+ "ssip",
+]
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.1"
@@ -9289,6 +9337,33 @@ dependencies = [
  "core_maths",
 ]
 
+[[package]]
+name = "tts"
+version = "0.27.0"
+source = "git+https://github.com/ndarilek/tts-rs#8fbcb720cb86c5166a9ed46272e3777f742da18e"
+dependencies = [
+ "block2 0.6.2",
+ "dyn-clonable",
+ "hound",
+ "jni 0.22.4",
+ "libloading",
+ "ndk-context",
+ "objc2 0.6.5",
+ "objc2-avf-audio",
+ "objc2-foundation 0.3.2",
+ "oxilangtag",
+ "parking_lot",
+ "ssip",
+ "ssip-client-async",
+ "tempfile",
+ "thiserror 2.0.21",
+ "tracing",
+ "wasm-bindgen",
+ "web-sys",
+ "windows 0.62.2",
+ "zbus",
+]
+
 [[package]]
 name = "tungstenite"
 version = "0.28.0"

Cargo.toml

@@ -50,6 +50,7 @@ rand = { version = "0.10", default-features = false, features = [ "std", "sys_rn
 
 # Others
 anyhow = "1.0.44"
+tts = { git = "https://github.com/ndarilek/tts-rs" }
 chrono = { version = "0.4.38", features = ["wasmbind"] }
 futures = "0.3"
 itertools = "0.13.0"

crates/chat_ui/src/actions.rs

@@ -14,6 +14,8 @@ pub enum Command {
     Relays(PublicKey),
     Njump(PublicKey),
     Trace(EventId),
+    Speak(EventId),
+    StopSpeaking,
 }
 
 // Start or stop dictation in the focused composer

crates/chat_ui/src/lib.rs

@@ -30,6 +30,7 @@ use person::{Directory, Person};
 use regex::Regex;
 use settings::{AppSettings, SignerKind};
 use smallvec::{SmallVec, smallvec};
+use speech::speaker::Speaker;
 use state::{
     FileAttachment, LocalFile, NostrRegistry, PickedFile, download_and_decrypt_file, pick_file,
 };
@@ -65,65 +66,45 @@ pub fn init(room: WeakEntity<Room>, window: &mut Window, cx: &mut App) -> Entity
 pub struct ChatPanel {
     id: SharedString,
     focus_handle: FocusHandle,
-
     /// Chat Room
     room: WeakEntity<Room>,
-
     /// Person directory shared with the room
     directory: Entity<Directory>,
-
     /// Message list state
     list_state: ListState,
-
     /// All messages (sorted by created_at)
     messages: Vec<Message>,
-
     /// O(1) message lookup by EventId
     message_index: HashMap<EventId, usize>,
-
     /// All reactions
     reactions: BTreeMap<EventId, Vec<(SharedString, Option<SharedUri>, PublicKey)>>,
-
     /// Mapping message ids to their rendered texts
     rendered_texts_by_id: BTreeMap<EventId, RenderedText>,
-
     /// Mapping message (rumor event) ids to their reports
     reports_by_id: Arc<RwLock<BTreeMap<EventId, Vec<SendReport>>>>,
-
     /// Chat input state
     input: Entity<TextareaState>,
-
     /// Dictation session for the chat input
     speech: Entity<SpeechState>,
-
     /// When the running dictation started, for the elapsed counter
     speech_started: Option<instant::Instant>,
-
+    /// Reads messages aloud
+    speaker: Speaker,
     /// Subject input state
     subject_input: Entity<InputState>,
-
     /// Subject bar visibility
     subject_bar: Entity<bool>,
-
     /// Replies to
     replies_to: Entity<HashSet<EventId>>,
-
     /// Media Attachment
     attachments: Entity<Vec<Url>>,
-
     /// Uploaded, encrypted file attachments which are not sent yet
     encrypted_attachments: Entity<Vec<PendingFile>>,
-
     /// Decrypted attachments of file messages, by message id
     decrypted_files: HashMap<EventId, DecryptedFile>,
-
     /// Upload state
     uploading: bool,
-
-    /// Async operations
     tasks: Vec<Task<Result<(), Error>>>,
-
-    /// Event subscriptions
     subscriptions: SmallVec<[Subscription; 4]>,
 }
 
@@ -253,6 +234,7 @@ impl ChatPanel {
             input,
             speech,
             speech_started: None,
+            speaker: Speaker::new(),
             subject_input,
             subject_bar,
             replies_to,
@@ -955,6 +937,20 @@ impl ChatPanel {
             Command::Trace(id) => {
                 self.open_trace(id, window, cx);
             }
+            Command::Speak(id) => {
+                let text = self.message(id).map(|message| message.preview());
+
+                if let Some(text) = text
+                    && let Err(e) = self.speaker.speak(&text)
+                {
+                    window.push_notification(Notification::error(e.to_string()), cx);
+                }
+            }
+            Command::StopSpeaking => {
+                if let Err(error) = self.speaker.stop() {
+                    window.push_notification(Notification::error(error.to_string()), cx);
+                }
+            }
         }
     }
 
@@ -1464,9 +1460,20 @@ impl ChatPanel {
             actions
         };
 
+        let speaking = self.speaker.speaking();
+
         actions.dropdown(move |menu, _window, _cx| {
-            menu.menu("Copy author", Box::new(Command::Copy(public_key)))
+            let menu = menu
+                .menu("Copy author", Box::new(Command::Copy(public_key)))
                 .menu("Seen on", Box::new(Command::Trace(dropdown_id)))
+                .separator()
+                .menu("Read aloud", Box::new(Command::Speak(dropdown_id)));
+
+            if speaking {
+                menu.menu("Stop speaking", Box::new(Command::StopSpeaking))
+            } else {
+                menu
+            }
         })
     }
 

crates/nip29_ui/src/lib.rs

@@ -26,6 +26,7 @@ use person::Directory;
 use serde::Deserialize;
 use settings::AppSettings;
 use smallvec::{SmallVec, smallvec};
+use speech::speaker::Speaker;
 use state::{NostrRegistry, pick_file, upload_file};
 use ui::{Avatar, RenderedText, WelcomeMessage, rendered_text};
 use util::{attachment_name, display_name, opens_run, pin_label, report};
@@ -320,6 +321,8 @@ pub struct GroupPanel {
     speech: Entity<SpeechState>,
     /// When the running dictation started, for the elapsed counter
     speech_started: Option<instant::Instant>,
+    /// Reads messages aloud
+    speaker: Speaker,
     /// Uploaded, non-encrypted attachments waiting to be sent.
     attachments: Vec<Url>,
     /// Whether an upload is in flight.
@@ -419,9 +422,10 @@ impl GroupPanel {
                     }
                 },
             ));
+            this.track_participants(cx);
         });
 
-        let mut this = Self {
+        Self {
             focus_handle: cx.focus_handle(),
             group: group.downgrade(),
             directory: Directory::new(cx),
@@ -433,15 +437,12 @@ impl GroupPanel {
             input,
             speech,
             speech_started: None,
+            speaker: Speaker::new(),
             attachments: Vec::new(),
             uploading: false,
             tasks: smallvec![],
             _subscriptions: subscriptions,
-        };
+        }
-
-        this.track_participants(cx);
-
-        this
     }
 
     /// Track the keys rendered by this panel: members, message authors, and
@@ -1059,6 +1060,7 @@ impl GroupPanel {
                 panel: cx.weak_entity(),
                 admin,
                 pinned,
+                speaking: self.speaker.speaking(),
                 persons: self.directory.clone(),
             },
             cx,

crates/nip29_ui/src/message.rs

@@ -8,7 +8,8 @@ use gpui::{
 use gpui_kit::assets::IconName;
 use gpui_kit::base::StyledExt as _;
 use gpui_kit::component::menu::{ContextMenuExt as _, PopupMenuItem};
-use gpui_kit::component::{ActiveTheme, Icon, Sizable, h_flex};
+use gpui_kit::component::notification::Notification;
+use gpui_kit::component::{ActiveTheme, Icon, Sizable, WindowExt, h_flex};
 use nip29::{Activity, Group, Reaction};
 use nostr_sdk::prelude::{EventId, Kind};
 use person::Directory;
@@ -26,6 +27,8 @@ pub(crate) struct Actions {
     pub panel: WeakEntity<GroupPanel>,
     pub admin: bool,
     pub pinned: bool,
+    /// Whether the panel's speaker is currently saying something
+    pub speaking: bool,
     pub persons: Entity<Directory>,
 }
 
@@ -88,9 +91,9 @@ pub(crate) fn render(
     let pinned = actions.pinned;
     let group = actions.group;
     let panel = actions.panel;
+    let speaking = actions.speaking;
+    let text = message.preview();
 
-    // Reading the panel here would re-enter its ongoing update, so resolve
-    // the current user's custom emojis through the shared person directory.
     let custom = nostr
         .read(cx)
         .current_user()
@@ -139,6 +142,40 @@ pub(crate) fn render(
                     }
                 });
 
+            let menu = menu.separator().item({
+                let panel = panel.clone();
+                let text = text.clone();
+
+                PopupMenuItem::new("Read aloud").on_click(move |_event, _window, cx| {
+                    if let Err(error) = panel.update_in(cx, |panel, window, cx| {
+                        if let Err(error) = panel.speaker.speak(&text) {
+                            window.push_notification(Notification::error(error.to_string()), cx);
+                        }
+                    }) {
+                        log::warn!("nip29: reading a message aloud failed: {error}");
+                    }
+                })
+            });
+
+            let menu = if speaking {
+                menu.item({
+                    let panel = panel.clone();
+
+                    PopupMenuItem::new("Stop speaking").on_click(move |_event, _window, cx| {
+                        if let Err(error) = panel.update_in(cx, |panel, window, cx| {
+                            if let Err(error) = panel.speaker.stop() {
+                                window
+                                    .push_notification(Notification::error(error.to_string()), cx);
+                            }
+                        }) {
+                            log::warn!("nip29: stopping the speaker failed: {error}");
+                        }
+                    })
+                })
+            } else {
+                menu
+            };
+
             if !admin {
                 return menu;
             }

crates/speech/Cargo.toml

@@ -11,6 +11,7 @@ gpui.workspace = true
 gpui-kit = { workspace = true, features = ["speech"] }
 flume.workspace = true
 instant.workspace = true
+tts.workspace = true
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 common = { path = "../common" }

crates/speech/src/lib.rs

@@ -2,6 +2,7 @@
 pub mod model;
 #[cfg(not(target_arch = "wasm32"))]
 mod parakeet;
+pub mod speaker;
 #[cfg(target_arch = "wasm32")]
 mod web;
 

crates/speech/src/speaker.rs

@@ -0,0 +1,47 @@
+use tts::Tts;
+
+/// Reads text aloud through the platform's speech backend, created on first use.
+pub struct Speaker {
+    tts: Option<Tts>,
+}
+
+impl Default for Speaker {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Speaker {
+    pub fn new() -> Self {
+        Self { tts: None }
+    }
+
+    /// Speak the text, interrupting what this speaker is already saying.
+    pub fn speak(&mut self, text: &str) -> Result<(), tts::Error> {
+        let tts = match self.tts.as_ref() {
+            Some(tts) => tts,
+            None => self.tts.insert(Tts::default()?),
+        };
+
+        tts.speak(text, true)?;
+        Ok(())
+    }
+
+    /// Stop speaking; a speaker that never spoke is already silent.
+    pub fn stop(&mut self) -> Result<(), tts::Error> {
+        match self.tts.as_ref() {
+            Some(tts) => tts.stop()?,
+            None => return Ok(()),
+        }
+
+        Ok(())
+    }
+
+    /// Best-effort speaking state; a speaker that never spoke is silent.
+    pub fn speaking(&self) -> bool {
+        self.tts
+            .as_ref()
+            .and_then(|tts| tts.is_speaking().ok())
+            .unwrap_or(false)
+    }
+}