From 41837e487b8a628c05b6e6ef4e252531ef2f33dd Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Fri, 30 Oct 2026 14:03:14 +0700 Subject: [PATCH] add reveal text --- crates/chat_ui/src/lib.rs | 53 ++++++++++++++++++++++------ crates/nip29_ui/src/lib.rs | 62 +++++++++++++++++++++++---------- crates/nip29_ui/src/message.rs | 9 +++-- crates/speech/src/speaker.rs | 63 +++++++++++++++++++++++++++------- crates/ui/src/message.rs | 50 ++++++++++++++++++++++++--- 5 files changed, 189 insertions(+), 48 deletions(-) diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index aaddc47c..3b88734f 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -36,8 +36,8 @@ use state::{ }; use ui::{ Avatar, COMPACT_REACTION_EMOJIS, IconName, MessageActions, MessageMedia, MessageRow, - REACTION_EMOJIS, ReactionBar, RenderedText, ReplyPreview, WelcomeMessage, group_reactions, - opens_run, rendered_text, + REACTION_EMOJIS, ReactionBar, RenderedText, ReplyPreview, RevealText, WelcomeMessage, + group_reactions, opens_run, rendered_text, }; use crate::file::*; @@ -90,6 +90,9 @@ pub struct ChatPanel { speech_started: Option, /// Reads messages aloud speaker: Speaker, + + /// The message being revealed in step with speech, with its estimated duration + speech_reveal: Option<(EventId, Duration)>, /// Subject input state subject_input: Entity, /// Subject bar visibility @@ -222,6 +225,20 @@ impl ChatPanel { this.get_messages(window, cx); }); + // End the speech reveal when the utterance finishes ahead of its estimate + let speaker = Speaker::new(); + let end_events = speaker.end_events(); + let reveal_end_task = cx.spawn(async move |this, cx| { + while end_events.recv_async().await.is_ok() { + this.update(cx, |this, cx| { + this.speech_reveal = None; + cx.notify(); + })?; + } + + Ok(()) + }); + Self { focus_handle: cx.focus_handle(), id, @@ -234,7 +251,8 @@ impl ChatPanel { input, speech, speech_started: None, - speaker: Speaker::new(), + speaker, + speech_reveal: None, subject_input, subject_bar, replies_to, @@ -245,7 +263,7 @@ impl ChatPanel { reports_by_id, uploading: false, subscriptions, - tasks: vec![], + tasks: vec![reveal_end_task], } } @@ -1174,6 +1192,15 @@ impl ChatPanel { nostr.read(cx).current_user(), )); + let body = match self + .speech_reveal + .as_ref() + .filter(|(reveal_id, _)| *reveal_id == id) + { + Some((_, duration)) => RevealText::new(message.preview(), *duration).into_any_element(), + None => rendered_text, + }; + MessageRow::new(ix) .show_author(show_author) .hide_avatar(hide_avatar) @@ -1198,7 +1225,7 @@ impl ChatPanel { .when(has_replies, |this| { this.children(self.render_message_replies(replies, cx)) }) - .when(message.file.is_none(), |this| this.child(rendered_text)) + .when(message.file.is_none(), |this| this.child(body)) .child(MessageMedia::new(&message.media)) .when_some(message.file.as_ref(), |this, file| { this.child(self.render_message_file(&id, file, cx)) @@ -1431,7 +1458,6 @@ impl ChatPanel { let reply_id = *id; let dropdown_id = *id; - let speech_id = *id; let public_key = *public_key; let actions = actions.reply(cx.listener(move |this, _event, _window, cx| { @@ -1447,13 +1473,20 @@ impl ChatPanel { actions }; + let speech_id = *id; let actions = actions.speech(cx.listener(move |this, _event, window, cx| { let text = this.message(&speech_id).map(|message| message.preview()); - if let Some(text) = text - && let Err(error) = this.speaker.speak(&text) - { - window.push_notification(Notification::error(error.to_string()), cx); + if let Some(text) = text { + match this.speaker.speak(&text) { + Ok(duration) => { + this.speech_reveal = Some((speech_id, duration)); + cx.notify(); + } + Err(error) => { + window.push_notification(Notification::error(error.to_string()), cx) + } + } } })); diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 093dc9b9..02d34bde 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -28,7 +28,7 @@ use settings::AppSettings; use smallvec::{SmallVec, smallvec}; use speech::speaker::Speaker; use state::{NostrRegistry, pick_file, upload_file}; -use ui::{Avatar, RenderedText, WelcomeMessage, rendered_text}; +use ui::{Avatar, RenderedText, RevealText, WelcomeMessage, rendered_text}; use util::{attachment_name, display_name, opens_run, pin_label, report}; mod details; @@ -323,6 +323,8 @@ pub struct GroupPanel { speech_started: Option, /// Reads messages aloud speaker: Speaker, + /// The message being revealed in step with speech, with its estimated duration + speech_reveal: Option<(EventId, Duration)>, /// Uploaded, non-encrypted attachments waiting to be sent. attachments: Vec, /// Whether an upload is in flight. @@ -425,6 +427,20 @@ impl GroupPanel { this.track_participants(cx); }); + // End the speech reveal when the utterance finishes ahead of its estimate + let speaker = Speaker::new(); + let end_events = speaker.end_events(); + let reveal_end_task = cx.spawn(async move |this, cx| { + while end_events.recv_async().await.is_ok() { + this.update(cx, |this, cx| { + this.speech_reveal = None; + cx.notify(); + })?; + } + + Ok(()) + }); + Self { focus_handle: cx.focus_handle(), group: group.downgrade(), @@ -437,10 +453,11 @@ impl GroupPanel { input, speech, speech_started: None, - speaker: Speaker::new(), + speaker, + speech_reveal: None, attachments: Vec::new(), uploading: false, - tasks: smallvec![], + tasks: smallvec![reveal_end_task], _subscriptions: subscriptions, } } @@ -1023,22 +1040,29 @@ impl GroupPanel { let nostr = NostrRegistry::global(cx); let directory = self.directory.clone(); - let content = self - .rendered_texts_by_id - .entry(message.id) - .or_insert_with(|| { - rendered_text( - &message.content, - message - .mentions - .iter() - .map(|mention| (mention.range.clone(), mention.public_key)), - &message.emojis, - &directory, - cx, - ) - }) - .element(ix.into(), window, cx); + let content = match self + .speech_reveal + .as_ref() + .filter(|(reveal_id, _)| *reveal_id == message.id) + { + Some((_, duration)) => RevealText::new(message.preview(), *duration).into_any_element(), + None => self + .rendered_texts_by_id + .entry(message.id) + .or_insert_with(|| { + rendered_text( + &message.content, + message + .mentions + .iter() + .map(|mention| (mention.range.clone(), mention.public_key)), + &message.emojis, + &directory, + cx, + ) + }) + .element(ix.into(), window, cx), + }; let admin = nostr .read(cx) diff --git a/crates/nip29_ui/src/message.rs b/crates/nip29_ui/src/message.rs index 8b356f0a..ef5d89c1 100644 --- a/crates/nip29_ui/src/message.rs +++ b/crates/nip29_ui/src/message.rs @@ -91,11 +91,16 @@ pub(crate) fn render( let speech_panel = actions.panel; let panel = speech_panel.clone(); let speech_text = message.preview(); + let speech_id = message.id; let row = row.overlay(MessageActions::new().speech(move |_event, _window, cx| { _ = speech_panel.update_in(cx, |panel, window, cx| { - if let Err(error) = panel.speaker.speak(&speech_text) { - window.push_notification(Notification::error(error.to_string()), cx); + match panel.speaker.speak(&speech_text) { + Ok(duration) => { + panel.speech_reveal = Some((speech_id, duration)); + cx.notify(); + } + Err(error) => window.push_notification(Notification::error(error.to_string()), cx), } }); })); diff --git a/crates/speech/src/speaker.rs b/crates/speech/src/speaker.rs index b5e00751..ed5138bc 100644 --- a/crates/speech/src/speaker.rs +++ b/crates/speech/src/speaker.rs @@ -1,33 +1,54 @@ +use std::time::Duration; + use tts::Tts; +/// Words per minute of an engine at its normal rate, for estimating speech duration. +const NORMAL_WPM: f32 = 160.0; + /// Reads text aloud through the platform's speech backend, created on first use. pub struct Speaker { tts: Option, -} - -impl Default for Speaker { - fn default() -> Self { - Self::new() - } + end_tx: flume::Sender<()>, + end_rx: flume::Receiver<()>, } impl Speaker { pub fn new() -> Self { - Self { tts: None } + let (end_tx, end_rx) = flume::unbounded(); + + Self { + tts: None, + end_tx, + end_rx, + } + } + + /// Cloneable stream of utterance-finish events. + pub fn end_events(&self) -> flume::Receiver<()> { + self.end_rx.clone() } /// Speak the text, interrupting what this speaker is already saying. - pub fn speak(&mut self, text: &str) -> Result<(), tts::Error> { + pub fn speak(&mut self, text: &str) -> Result { + let end_tx = self.end_tx.clone(); let tts = match self.tts.as_ref() { Some(tts) => tts, - None => self.tts.insert(Tts::default()?), + None => self.tts.insert({ + let tts = Tts::default()?; + if let Err(error) = tts.on_utterance_end(move |_| { + // A send error only means the panel dropped its event loop. + let _ = end_tx.send(()); + }) { + log::warn!("utterance end events unavailable ({error})"); + } + tts + }), }; - tts.speak(text, true)?; - Ok(()) + Ok(Self::estimate(tts, text)) } - /// Stop speaking; a speaker that never spoke is already silent. + /// Stop speaking, a speaker that never spoke is already silent. pub fn stop(&mut self) -> Result<(), tts::Error> { match self.tts.as_ref() { Some(tts) => tts.stop()?, @@ -37,11 +58,27 @@ impl Speaker { Ok(()) } - /// Best-effort speaking state; a speaker that never spoke is silent. + /// Best-effort speaking state, a speaker that never spoke is silent. pub fn speaking(&self) -> bool { self.tts .as_ref() .and_then(|tts| tts.is_speaking().ok()) .unwrap_or(false) } + + /// How long speaking the text is expected to take at the engine's current speed. + fn estimate(tts: &Tts, text: &str) -> Duration { + let words = text.split_whitespace().count().max(1) as f32; + let speed = match (tts.get_rate(), tts.normal_rate()) { + (Ok(rate), Ok(normal)) if normal > 0. => (rate / normal).clamp(0.2, 4.), + _ => 1., + }; + Duration::from_secs_f32(words / NORMAL_WPM * 60. / speed) + } +} + +impl Default for Speaker { + fn default() -> Self { + Self::new() + } } diff --git a/crates/ui/src/message.rs b/crates/ui/src/message.rs index b50305cf..2f28fb3f 100644 --- a/crates/ui/src/message.rs +++ b/crates/ui/src/message.rs @@ -1,12 +1,13 @@ use std::collections::BTreeMap; use std::rc::Rc; +use std::time::Duration; use gpui::prelude::FluentBuilder as _; use gpui::{ - AnyElement, App, ClickEvent, Context, ElementId, InteractiveElement as _, IntoElement, - MouseButton, MouseDownEvent, ObjectFit, ParentElement as _, RenderOnce, SharedString, - SharedUri, StatefulInteractiveElement as _, StyleRefinement, Styled, StyledImage, Window, div, - img, px, + Animation, AnimationExt as _, AnyElement, App, ClickEvent, Context, ElementId, + InteractiveElement as _, IntoElement, MouseButton, MouseDownEvent, ObjectFit, + ParentElement as _, RenderOnce, SharedString, SharedUri, StatefulInteractiveElement as _, + StyleRefinement, Styled, StyledImage, Window, div, img, px, }; use gpui_kit::assets::IconName; use gpui_kit::component::button::{Button, ButtonVariants as _}; @@ -750,3 +751,44 @@ impl RenderOnce for MessageActions { .refine_style(&style) } } + +/// A message's text revealed word by word, paced by the estimated speech duration. +#[derive(IntoElement)] +pub struct RevealText { + text: SharedString, + duration: Duration, +} + +impl RevealText { + pub fn new(text: impl Into, duration: Duration) -> Self { + Self { + text: text.into(), + duration, + } + } +} + +impl RenderOnce for RevealText { + fn render(self, _window: &mut Window, _cx: &mut App) -> impl IntoElement { + let words: Vec = self + .text + .split_whitespace() + .map(SharedString::from) + .collect(); + let total = words.len().max(1) as f32; + + div().flex().flex_wrap().gap_x_1().with_animation( + "speech-reveal", + Animation::new(self.duration).with_max_fps(30.), + move |this, progress| { + let visible = ((progress * total).ceil() as usize).min(words.len()); + + this.children(words.iter().enumerate().map(|(ix, word)| { + div() + .when(ix >= visible, |this| this.invisible()) + .child(word.clone()) + })) + }, + ) + } +}