reya/coop
1
0
Fork 1
Code Issues Pull Requests Releases 5 Activity

feat: add text to speech (#69)

Browse Source 
Reviewed-on: #69
This commit was merged in pull request #69.
This commit is contained in:
reya
2026-10-30 07:16:46 +00:00
parent af2a276b1b
commit 23d7017f25
11 changed files with 420 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chat_ui/src/lib.rs
+75 -28
View File
@@ -30,13 +30,14 @@ use person::{Directory, Person};
use regex::Regex;
use settings::{AppSettings, SignerKind};
use smallvec::{SmallVec, smallvec};
use speech::speaker::{END_GRACE, Speaker};
use state::{
FileAttachment, LocalFile, NostrRegistry, PickedFile, download_and_decrypt_file, pick_file,
};
use ui::{
Avatar, COMPACT_REACTION_EMOJIS, IconName, MessageActions, MessageMedia, MessageRow,
REACTION_EMOJIS, ReactionBar, RenderedText, ReplyPreview, WelcomeMessage, group_reactions,
opens_run, rendered_text,
REACTION_EMOJIS, ReactionBar, RenderedText, ReplyPreview, RevealText, WelcomeMessage,
group_reactions, opens_run, rendered_text,
};
use crate::file::*;
@@ -45,8 +46,7 @@ use crate::file::*;
const PRIVATE_NOTICE: &str =
"This conversation is private. Only members can see each other's messages.";
/// Regex matching strings that consist entirely of emoji characters,
/// zero-width joiners, variation selectors, and keycap combiners.
/// Regex matching strings that consist entirely of emoji characters.
static EMOJI_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^[\p{Emoji}\u{200D}\u{FE0F}\u{20E3}]+$").unwrap());
@@ -65,65 +65,49 @@ pub fn init(room: WeakEntity<Room>, window: &mut Window, cx: &mut App) -> Entity
pub struct ChatPanel {
id: SharedString,
focus_handle: FocusHandle,
/// Chat Room
room: WeakEntity<Room>,
/// Person directory shared with the room
directory: Entity<Directory>,
/// Message list state
list_state: ListState,
/// All messages (sorted by created_at)
messages: Vec<Message>,
/// O(1) message lookup by EventId
message_index: HashMap<EventId, usize>,
/// All reactions
reactions: BTreeMap<EventId, Vec<(SharedString, Option<SharedUri>, PublicKey)>>,
/// Mapping message ids to their rendered texts
rendered_texts_by_id: BTreeMap<EventId, RenderedText>,
/// Mapping message (rumor event) ids to their reports
reports_by_id: Arc<RwLock<BTreeMap<EventId, Vec<SendReport>>>>,
/// Chat input state
input: Entity<TextareaState>,
/// Dictation session for the chat input
speech: Entity<SpeechState>,
/// When the running dictation started, for the elapsed counter
speech_started: Option<instant::Instant>,
/// Reads messages aloud
speaker: Speaker,
/// The message being revealed in step with speech, with its estimated duration
speech_reveal: Option<(EventId, Duration)>,
/// Fallback reveal end for backends that cannot report utterance end
reveal_timeout: Option<Task<Result<(), Error>>>,
/// Subject input state
subject_input: Entity<InputState>,
/// Subject bar visibility
subject_bar: Entity<bool>,
/// Replies to
replies_to: Entity<HashSet<EventId>>,
/// Media Attachment
attachments: Entity<Vec<Url>>,
/// Uploaded, encrypted file attachments which are not sent yet
encrypted_attachments: Entity<Vec<PendingFile>>,
/// Decrypted attachments of file messages, by message id
decrypted_files: HashMap<EventId, DecryptedFile>,
/// Upload state
uploading: bool,
/// Async operations
tasks: Vec<Task<Result<(), Error>>>,
/// Event subscriptions
subscriptions: SmallVec<[Subscription; 4]>,
}
@@ -241,6 +225,20 @@ impl ChatPanel {
this.get_messages(window, cx);
});
// End the speech reveal when the utterance finishes ahead of its estimate
let speaker = Speaker::new();
let end_events = speaker.end_events();
let reveal_end_task = cx.spawn(async move |this, cx| {
while end_events.recv_async().await.is_ok() {
this.update(cx, |this, cx| {
this.speech_reveal = None;
cx.notify();
})?;
}
Ok(())
});
Self {
focus_handle: cx.focus_handle(),
id,
@@ -253,6 +251,9 @@ impl ChatPanel {
input,
speech,
speech_started: None,
speaker,
speech_reveal: None,
reveal_timeout: None,
subject_input,
subject_bar,
replies_to,
@@ -263,7 +264,7 @@ impl ChatPanel {
reports_by_id,
uploading: false,
subscriptions,
tasks: vec![],
tasks: vec![reveal_end_task],
}
}
@@ -1192,6 +1193,15 @@ impl ChatPanel {
nostr.read(cx).current_user(),
));
let body = match self
.speech_reveal
.as_ref()
.filter(|(reveal_id, _)| *reveal_id == id)
{
Some((_, duration)) => RevealText::new(message.preview(), *duration).into_any_element(),
None => rendered_text,
};
MessageRow::new(ix)
.show_author(show_author)
.hide_avatar(hide_avatar)
@@ -1216,7 +1226,7 @@ impl ChatPanel {
.when(has_replies, |this| {
this.children(self.render_message_replies(replies, cx))
})
.when(message.file.is_none(), |this| this.child(rendered_text))
.when(message.file.is_none(), |this| this.child(body))
.child(MessageMedia::new(&message.media))
.when_some(message.file.as_ref(), |this, file| {
this.child(self.render_message_file(&id, file, cx))
@@ -1464,6 +1474,43 @@ impl ChatPanel {
actions
};
let speaking = self.speech_reveal.is_some();
let speech_id = *id;
let actions = actions.speech(
speaking,
cx.listener(move |this, _event, window, cx| {
let text = this.message(&speech_id).map(|message| message.preview());
if let Some(text) = text {
match this.speaker.speak(&text) {
Ok(duration) => {
this.speech_reveal = Some((speech_id, duration));
// Re-enable speaking when no end event ever arrives
let session = (speech_id, duration);
let timer = cx.background_executor().timer(duration + END_GRACE);
this.reveal_timeout = Some(cx.spawn(async move |this, cx| {
timer.await;
this.update(cx, |this, cx| {
if this.speech_reveal.as_ref() == Some(&session) {
this.speech_reveal = None;
cx.notify();
}
})?;
Ok(())
}));
cx.notify();
}
Err(error) => {
window.push_notification(Notification::error(error.to_string()), cx)
}
}
}
}),
);
actions.dropdown(move |menu, _window, _cx| {
menu.menu("Copy author", Box::new(Command::Copy(public_key)))
.menu("Seen on", Box::new(Command::Trace(dropdown_id)))
crates/nip29_ui/src/lib.rs
+54 -26
View File
@@ -26,8 +26,9 @@ use person::Directory;
use serde::Deserialize;
use settings::AppSettings;
use smallvec::{SmallVec, smallvec};
use speech::speaker::Speaker;
use state::{NostrRegistry, pick_file, upload_file};
use ui::{Avatar, RenderedText, WelcomeMessage, rendered_text};
use ui::{Avatar, RenderedText, RevealText, WelcomeMessage, rendered_text};
use util::{attachment_name, display_name, opens_run, pin_label, report};
mod details;
@@ -320,6 +321,12 @@ pub struct GroupPanel {
speech: Entity<SpeechState>,
/// When the running dictation started, for the elapsed counter
speech_started: Option<instant::Instant>,
/// Reads messages aloud
speaker: Speaker,
/// The message being revealed in step with speech, with its estimated duration
speech_reveal: Option<(EventId, Duration)>,
/// Fallback reveal end for backends that cannot report utterance end
reveal_timeout: Option<Task<Result<()>>>,
/// Uploaded, non-encrypted attachments waiting to be sent.
attachments: Vec<Url>,
/// Whether an upload is in flight.
@@ -419,9 +426,24 @@ impl GroupPanel {
}
},
));
this.track_participants(cx);
});
let mut this = Self {
// End the speech reveal when the utterance finishes ahead of its estimate
let speaker = Speaker::new();
let end_events = speaker.end_events();
let reveal_end_task = cx.spawn(async move |this, cx| {
while end_events.recv_async().await.is_ok() {
this.update(cx, |this, cx| {
this.speech_reveal = None;
cx.notify();
})?;
}
Ok(())
});
Self {
focus_handle: cx.focus_handle(),
group: group.downgrade(),
directory: Directory::new(cx),
@@ -433,19 +455,17 @@ impl GroupPanel {
input,
speech,
speech_started: None,
speaker,
speech_reveal: None,
reveal_timeout: None,
attachments: Vec::new(),
uploading: false,
tasks: smallvec![],
tasks: smallvec![reveal_end_task],
_subscriptions: subscriptions,
};
this.track_participants(cx);
this
}
}
/// Track the keys rendered by this panel: members, message authors, and
/// activity subjects
/// Track the keys rendered by this panel.
fn track_participants(&mut self, cx: &mut Context<Self>) {
let Some(group) = self.group.upgrade() else {
return;
@@ -1023,22 +1043,29 @@ impl GroupPanel {
let nostr = NostrRegistry::global(cx);
let directory = self.directory.clone();
let content = self
.rendered_texts_by_id
.entry(message.id)
.or_insert_with(|| {
rendered_text(
&message.content,
message
.mentions
.iter()
.map(|mention| (mention.range.clone(), mention.public_key)),
&message.emojis,
&directory,
cx,
)
})
.element(ix.into(), window, cx);
let content = match self
.speech_reveal
.as_ref()
.filter(|(reveal_id, _)| *reveal_id == message.id)
{
Some((_, duration)) => RevealText::new(message.preview(), *duration).into_any_element(),
None => self
.rendered_texts_by_id
.entry(message.id)
.or_insert_with(|| {
rendered_text(
&message.content,
message
.mentions
.iter()
.map(|mention| (mention.range.clone(), mention.public_key)),
&message.emojis,
&directory,
cx,
)
})
.element(ix.into(), window, cx),
};
let admin = nostr
.read(cx)
@@ -1059,6 +1086,7 @@ impl GroupPanel {
panel: cx.weak_entity(),
admin,
pinned,
speaking: self.speech_reveal.is_some(),
persons: self.directory.clone(),
},
cx,
crates/nip29_ui/src/message.rs
+44 -6
View File
@@ -8,15 +8,17 @@ use gpui::{
use gpui_kit::assets::IconName;
use gpui_kit::base::StyledExt as _;
use gpui_kit::component::menu::{ContextMenuExt as _, PopupMenuItem};
use gpui_kit::component::{ActiveTheme, Icon, Sizable, h_flex};
use gpui_kit::component::notification::Notification;
use gpui_kit::component::{ActiveTheme, Icon, Sizable, WindowExt, h_flex};
use nip29::{Activity, Group, Reaction};
use nostr_sdk::prelude::{EventId, Kind};
use person::Directory;
use settings::AppSettings;
use speech::speaker::END_GRACE;
use state::NostrRegistry;
use ui::{
Avatar, MessageMedia, MessageRow, REACTION_EMOJIS, ReactionBar, ReplyPreview, group_reactions,
react_menu,
Avatar, MessageActions, MessageMedia, MessageRow, REACTION_EMOJIS, ReactionBar, ReplyPreview,
group_reactions, react_menu,
};
use crate::GroupPanel;
@@ -26,6 +28,8 @@ pub(crate) struct Actions {
pub panel: WeakEntity<GroupPanel>,
pub admin: bool,
pub pinned: bool,
/// Whether a message is currently being spoken, disabling the speech buttons
pub speaking: bool,
pub persons: Entity<Directory>,
}
@@ -87,10 +91,44 @@ pub(crate) fn render(
let admin = actions.admin;
let pinned = actions.pinned;
let group = actions.group;
let panel = actions.panel;
let speech_panel = actions.panel;
let panel = speech_panel.clone();
let speech_text = message.preview();
let speech_id = message.id;
let speaking = actions.speaking;
let row = row.overlay(
MessageActions::new().speech(speaking, move |_event, _window, cx| {
_ = speech_panel.update_in(cx, |panel, window, cx| {
match panel.speaker.speak(&speech_text) {
Ok(duration) => {
panel.speech_reveal = Some((speech_id, duration));
let session = (speech_id, duration);
let timer = cx.background_executor().timer(duration + END_GRACE);
panel.reveal_timeout = Some(cx.spawn(async move |this, cx| {
timer.await;
this.update(cx, |this, cx| {
if this.speech_reveal.as_ref() == Some(&session) {
this.speech_reveal = None;
cx.notify();
}
})?;
Ok(())
}));
cx.notify();
}
Err(error) => {
window.push_notification(Notification::error(error.to_string()), cx)
}
}
});
}),
);
// Reading the panel here would re-enter its ongoing update, so resolve
// the current user's custom emojis through the shared person directory.
let custom = nostr
.read(cx)
.current_user()
crates/speech/Cargo.toml
+1
View File
@@ -11,6 +11,7 @@ gpui.workspace = true
gpui-kit = { workspace = true, features = ["speech"] }
flume.workspace = true
instant.workspace = true
tts.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
common = { path = "../common" }
crates/speech/src/lib.rs
+1
View File
@@ -2,6 +2,7 @@
pub mod model;
#[cfg(not(target_arch = "wasm32"))]
mod parakeet;
pub mod speaker;
#[cfg(target_arch = "wasm32")]
mod web;
crates/speech/src/speaker.rs
+88
View File
@@ -0,0 +1,88 @@
use std::time::Duration;
use tts::Tts;
/// Words per minute of an engine at its normal rate, for estimating speech duration.
const NORMAL_WPM: f32 = 160.0;
/// Extra time past the estimate before speech is considered finished, for backends that
/// cannot report utterance end.
pub const END_GRACE: Duration = Duration::from_secs(5);
/// Reads text aloud through the platform's speech backend, created on first use.
pub struct Speaker {
tts: Option<Tts>,
end_tx: flume::Sender<()>,
end_rx: flume::Receiver<()>,
}
impl Speaker {
pub fn new() -> Self {
let (end_tx, end_rx) = flume::unbounded();
Self {
tts: None,
end_tx,
end_rx,
}
}
/// Cloneable stream of utterance-finish events.
pub fn end_events(&self) -> flume::Receiver<()> {
self.end_rx.clone()
}
/// Speak the text, interrupting what this speaker is already saying.
pub fn speak(&mut self, text: &str) -> Result<Duration, tts::Error> {
let end_tx = self.end_tx.clone();
let tts = match self.tts.as_ref() {
Some(tts) => tts,
None => self.tts.insert({
let tts = Tts::default()?;
if let Err(error) = tts.on_utterance_end(move |_| {
// A send error only means the panel dropped its event loop.
let _ = end_tx.send(());
}) {
log::warn!("utterance end events unavailable ({error})");
}
tts
}),
};
tts.speak(text, true)?;
Ok(Self::estimate(tts, text))
}
/// Stop speaking, a speaker that never spoke is already silent.
pub fn stop(&mut self) -> Result<(), tts::Error> {
match self.tts.as_ref() {
Some(tts) => tts.stop()?,
None => return Ok(()),
}
Ok(())
}
/// Best-effort speaking state, a speaker that never spoke is silent.
pub fn speaking(&self) -> bool {
self.tts
.as_ref()
.and_then(|tts| tts.is_speaking().ok())
.unwrap_or(false)
}
/// How long speaking the text is expected to take at the engine's current speed.
fn estimate(tts: &Tts, text: &str) -> Duration {
let words = text.split_whitespace().count().max(1) as f32;
let speed = match (tts.get_rate(), tts.normal_rate()) {
(Ok(rate), Ok(normal)) if normal > 0. => (rate / normal).clamp(0.2, 4.),
_ => 1.,
};
Duration::from_secs_f32(words / NORMAL_WPM * 60. / speed)
}
}
impl Default for Speaker {
fn default() -> Self {
Self::new()
}
}
crates/ui/src/icon.rs
+2
View File
@@ -26,6 +26,7 @@ pub enum IconName {
PlusChat,
Profile,
Relay,
Speech,
Search,
Settings,
Settings2,
@@ -58,6 +59,7 @@ impl IconNamed for IconName {
Self::PlusChat => "icons/plus-chat.svg",
Self::Profile => "icons/profile.svg",
Self::Relay => "icons/relay.svg",
Self::Speech => "icons/speech.svg",
Self::Search => "icons/search.svg",
Self::Settings => "icons/settings.svg",
Self::Settings2 => "icons/settings2.svg",
crates/ui/src/message.rs
+76 -7
View File
@@ -1,19 +1,21 @@
use std::collections::BTreeMap;
use std::rc::Rc;
use std::time::Duration;
use gpui::prelude::FluentBuilder as _;
use gpui::{
AnyElement, App, ClickEvent, Context, ElementId, InteractiveElement as _, IntoElement,
MouseButton, MouseDownEvent, ObjectFit, ParentElement as _, RenderOnce, SharedString,
SharedUri, StatefulInteractiveElement as _, StyleRefinement, Styled, StyledImage, Window, div,
img, px,
Animation, AnimationExt as _, AnyElement, App, ClickEvent, Context, ElementId,
InteractiveElement as _, IntoElement, MouseButton, MouseDownEvent, ObjectFit,
ParentElement as _, RenderOnce, SharedString, SharedUri, StatefulInteractiveElement as _,
StyleRefinement, Styled, StyledImage, Window, div, img, px,
};
use gpui_kit::assets::IconName;
use gpui_kit::component::button::{Button, ButtonVariants as _};
use gpui_kit::component::clipboard::Clipboard;
use gpui_kit::component::menu::{DropdownMenu as _, PopupMenu, PopupMenuItem};
use gpui_kit::component::{
ActiveTheme, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex, v_flex,
ActiveTheme, Disableable, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex,
v_flex,
};
use smallvec::SmallVec;
@@ -597,13 +599,14 @@ pub fn react_menu(
})
}
/// A hovered message's action bar: quick reactions, reply, copy, and an overflow menu.
/// A hovered message's action bar: quick reactions, reply, copy, read aloud, and an overflow menu.
#[derive(IntoElement)]
pub struct MessageActions {
style: StyleRefinement,
reactions: SmallVec<[(&'static str, SharedClickListener); 4]>,
reply: Option<SharedClickListener>,
copy: Option<(ElementId, SharedString)>,
speech: Option<(bool, SharedClickListener)>,
dropdown: Option<DropdownBuilder>,
}
@@ -615,6 +618,7 @@ impl MessageActions {
reactions: SmallVec::new(),
reply: None,
copy: None,
speech: None,
dropdown: None,
}
}
@@ -642,6 +646,16 @@ impl MessageActions {
self
}
/// Add the read-aloud button, disabled while a message is already being spoken.
pub fn speech(
mut self,
disabled: bool,
handler: impl Fn(&ClickEvent, &mut Window, &mut App) + 'static,
) -> Self {
self.speech = Some((disabled, Rc::new(handler)));
self
}
/// Fill the bar's overflow menu.
pub fn dropdown(
mut self,
@@ -671,10 +685,13 @@ impl RenderOnce for MessageActions {
reactions,
reply,
copy,
speech,
dropdown,
} = self;
let has_center = reply.is_some() || copy.is_some();
// The leading separator only separates reactions from the center buttons
let has_center =
!reactions.is_empty() && (reply.is_some() || copy.is_some() || speech.is_some());
h_flex()
.p_0p5()
@@ -712,6 +729,17 @@ impl RenderOnce for MessageActions {
.when_some(copy, |this, (id, value)| {
this.child(Clipboard::new(id).value(value).tooltip("Copy").small())
})
.when_some(speech, |this, (disabled, speech)| {
this.child(
Button::new("speech")
.icon(crate::IconName::Speech)
.tooltip("Read aloud")
.small()
.ghost()
.disabled(disabled)
.on_click(move |event, window, cx| speech(event, window, cx)),
)
})
.when_some(dropdown, |this, dropdown| {
this.child(div().flex_shrink_0().h_4().w_px().bg(cx.theme().border))
.child(
@@ -726,3 +754,44 @@ impl RenderOnce for MessageActions {
.refine_style(&style)
}
}
/// A message's text revealed word by word, paced by the estimated speech duration.
#[derive(IntoElement)]
pub struct RevealText {
text: SharedString,
duration: Duration,
}
impl RevealText {
pub fn new(text: impl Into<SharedString>, duration: Duration) -> Self {
Self {
text: text.into(),
duration,
}
}
}
impl RenderOnce for RevealText {
fn render(self, _window: &mut Window, _cx: &mut App) -> impl IntoElement {
let words: Vec<SharedString> = self
.text
.split_whitespace()
.map(SharedString::from)
.collect();
let total = words.len().max(1) as f32;
div().flex().flex_wrap().gap_x_1().with_animation(
"speech-reveal",
Animation::new(self.duration).with_max_fps(30.),
move |this, progress| {
let visible = ((progress * total).ceil() as usize).min(words.len());
this.children(words.iter().enumerate().map(|(ix, word)| {
div()
.when(ix >= visible, |this| this.invisible())
.child(word.clone())
}))
},
)
}
}