feat: add text to speech (#69)
Reviewed-on: #69
This commit was merged in pull request #69.
This commit is contained in:
Generated
+75
@@ -2155,6 +2155,27 @@ dependencies = [
|
|||||||
"wio",
|
"wio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dyn-clonable"
|
||||||
|
version = "0.9.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a36efbb9bfd58e1723780aa04b61aba95ace6a05d9ffabfdb0b43672552f0805"
|
||||||
|
dependencies = [
|
||||||
|
"dyn-clonable-impl",
|
||||||
|
"dyn-clone",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dyn-clonable-impl"
|
||||||
|
version = "0.9.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7e8671d54058979a37a26f3511fbf8d198ba1aa35ffb202c42587d918d77213a"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.119",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dyn-clone"
|
name = "dyn-clone"
|
||||||
version = "1.0.20"
|
version = "1.0.20"
|
||||||
@@ -5939,6 +5960,8 @@ version = "0.3.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"
|
checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"bitflags 2.13.2",
|
||||||
|
"block2 0.6.2",
|
||||||
"objc2 0.6.5",
|
"objc2 0.6.5",
|
||||||
"objc2-foundation 0.3.2",
|
"objc2-foundation 0.3.2",
|
||||||
]
|
]
|
||||||
@@ -6436,6 +6459,15 @@ dependencies = [
|
|||||||
"ureq",
|
"ureq",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "oxilangtag"
|
||||||
|
version = "0.1.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "page_size"
|
name = "page_size"
|
||||||
version = "0.6.0"
|
version = "0.6.0"
|
||||||
@@ -8372,6 +8404,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2 0.10.9",
|
"sha2 0.10.9",
|
||||||
|
"tts",
|
||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
"web-sys",
|
"web-sys",
|
||||||
]
|
]
|
||||||
@@ -8443,6 +8476,21 @@ dependencies = [
|
|||||||
"web-sys",
|
"web-sys",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ssip"
|
||||||
|
version = "0.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b81f5ada7d4745722e62f8be4aca56abcf0e0e46e409612ac94a4a3a2b471d39"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ssip-client-async"
|
||||||
|
version = "0.19.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2cfbcb0ed774006ab9f14d4595fc02668cd7f5e662a87ff82264e6506238f47f"
|
||||||
|
dependencies = [
|
||||||
|
"ssip",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "stable_deref_trait"
|
name = "stable_deref_trait"
|
||||||
version = "1.2.1"
|
version = "1.2.1"
|
||||||
@@ -9289,6 +9337,33 @@ dependencies = [
|
|||||||
"core_maths",
|
"core_maths",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tts"
|
||||||
|
version = "0.27.0"
|
||||||
|
source = "git+https://github.com/ndarilek/tts-rs#8fbcb720cb86c5166a9ed46272e3777f742da18e"
|
||||||
|
dependencies = [
|
||||||
|
"block2 0.6.2",
|
||||||
|
"dyn-clonable",
|
||||||
|
"hound",
|
||||||
|
"jni 0.22.4",
|
||||||
|
"libloading",
|
||||||
|
"ndk-context",
|
||||||
|
"objc2 0.6.5",
|
||||||
|
"objc2-avf-audio",
|
||||||
|
"objc2-foundation 0.3.2",
|
||||||
|
"oxilangtag",
|
||||||
|
"parking_lot",
|
||||||
|
"ssip",
|
||||||
|
"ssip-client-async",
|
||||||
|
"tempfile",
|
||||||
|
"thiserror 2.0.21",
|
||||||
|
"tracing",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"web-sys",
|
||||||
|
"windows 0.62.2",
|
||||||
|
"zbus",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tungstenite"
|
name = "tungstenite"
|
||||||
version = "0.28.0"
|
version = "0.28.0"
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ rand = { version = "0.10", default-features = false, features = [ "std", "sys_rn
|
|||||||
|
|
||||||
# Others
|
# Others
|
||||||
anyhow = "1.0.44"
|
anyhow = "1.0.44"
|
||||||
|
tts = { git = "https://github.com/ndarilek/tts-rs" }
|
||||||
chrono = { version = "0.4.38", features = ["wasmbind"] }
|
chrono = { version = "0.4.38", features = ["wasmbind"] }
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
itertools = "0.13.0"
|
itertools = "0.13.0"
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none">
|
||||||
|
<path d="M10.75 14.75V17.25" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M14.25 7.75V20.25" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M17.75 10.75V17.25" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M21.25 12.75V15.25" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/><path d="M2.75 5.25V3.75H7M11.25 5.25V3.75H7M7 3.75V12.25M7 12.25H5.25M7 12.25H8.75" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 726 B |
+75
-28
@@ -30,13 +30,14 @@ use person::{Directory, Person};
|
|||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use settings::{AppSettings, SignerKind};
|
use settings::{AppSettings, SignerKind};
|
||||||
use smallvec::{SmallVec, smallvec};
|
use smallvec::{SmallVec, smallvec};
|
||||||
|
use speech::speaker::{END_GRACE, Speaker};
|
||||||
use state::{
|
use state::{
|
||||||
FileAttachment, LocalFile, NostrRegistry, PickedFile, download_and_decrypt_file, pick_file,
|
FileAttachment, LocalFile, NostrRegistry, PickedFile, download_and_decrypt_file, pick_file,
|
||||||
};
|
};
|
||||||
use ui::{
|
use ui::{
|
||||||
Avatar, COMPACT_REACTION_EMOJIS, IconName, MessageActions, MessageMedia, MessageRow,
|
Avatar, COMPACT_REACTION_EMOJIS, IconName, MessageActions, MessageMedia, MessageRow,
|
||||||
REACTION_EMOJIS, ReactionBar, RenderedText, ReplyPreview, WelcomeMessage, group_reactions,
|
REACTION_EMOJIS, ReactionBar, RenderedText, ReplyPreview, RevealText, WelcomeMessage,
|
||||||
opens_run, rendered_text,
|
group_reactions, opens_run, rendered_text,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::file::*;
|
use crate::file::*;
|
||||||
@@ -45,8 +46,7 @@ use crate::file::*;
|
|||||||
const PRIVATE_NOTICE: &str =
|
const PRIVATE_NOTICE: &str =
|
||||||
"This conversation is private. Only members can see each other's messages.";
|
"This conversation is private. Only members can see each other's messages.";
|
||||||
|
|
||||||
/// Regex matching strings that consist entirely of emoji characters,
|
/// Regex matching strings that consist entirely of emoji characters.
|
||||||
/// zero-width joiners, variation selectors, and keycap combiners.
|
|
||||||
static EMOJI_RE: LazyLock<Regex> =
|
static EMOJI_RE: LazyLock<Regex> =
|
||||||
LazyLock::new(|| Regex::new(r"^[\p{Emoji}\u{200D}\u{FE0F}\u{20E3}]+$").unwrap());
|
LazyLock::new(|| Regex::new(r"^[\p{Emoji}\u{200D}\u{FE0F}\u{20E3}]+$").unwrap());
|
||||||
|
|
||||||
@@ -65,65 +65,49 @@ pub fn init(room: WeakEntity<Room>, window: &mut Window, cx: &mut App) -> Entity
|
|||||||
pub struct ChatPanel {
|
pub struct ChatPanel {
|
||||||
id: SharedString,
|
id: SharedString,
|
||||||
focus_handle: FocusHandle,
|
focus_handle: FocusHandle,
|
||||||
|
|
||||||
/// Chat Room
|
/// Chat Room
|
||||||
room: WeakEntity<Room>,
|
room: WeakEntity<Room>,
|
||||||
|
|
||||||
/// Person directory shared with the room
|
/// Person directory shared with the room
|
||||||
directory: Entity<Directory>,
|
directory: Entity<Directory>,
|
||||||
|
|
||||||
/// Message list state
|
/// Message list state
|
||||||
list_state: ListState,
|
list_state: ListState,
|
||||||
|
|
||||||
/// All messages (sorted by created_at)
|
/// All messages (sorted by created_at)
|
||||||
messages: Vec<Message>,
|
messages: Vec<Message>,
|
||||||
|
|
||||||
/// O(1) message lookup by EventId
|
/// O(1) message lookup by EventId
|
||||||
message_index: HashMap<EventId, usize>,
|
message_index: HashMap<EventId, usize>,
|
||||||
|
|
||||||
/// All reactions
|
/// All reactions
|
||||||
reactions: BTreeMap<EventId, Vec<(SharedString, Option<SharedUri>, PublicKey)>>,
|
reactions: BTreeMap<EventId, Vec<(SharedString, Option<SharedUri>, PublicKey)>>,
|
||||||
|
|
||||||
/// Mapping message ids to their rendered texts
|
/// Mapping message ids to their rendered texts
|
||||||
rendered_texts_by_id: BTreeMap<EventId, RenderedText>,
|
rendered_texts_by_id: BTreeMap<EventId, RenderedText>,
|
||||||
|
|
||||||
/// Mapping message (rumor event) ids to their reports
|
/// Mapping message (rumor event) ids to their reports
|
||||||
reports_by_id: Arc<RwLock<BTreeMap<EventId, Vec<SendReport>>>>,
|
reports_by_id: Arc<RwLock<BTreeMap<EventId, Vec<SendReport>>>>,
|
||||||
|
|
||||||
/// Chat input state
|
/// Chat input state
|
||||||
input: Entity<TextareaState>,
|
input: Entity<TextareaState>,
|
||||||
|
|
||||||
/// Dictation session for the chat input
|
/// Dictation session for the chat input
|
||||||
speech: Entity<SpeechState>,
|
speech: Entity<SpeechState>,
|
||||||
|
|
||||||
/// When the running dictation started, for the elapsed counter
|
/// When the running dictation started, for the elapsed counter
|
||||||
speech_started: Option<instant::Instant>,
|
speech_started: Option<instant::Instant>,
|
||||||
|
/// Reads messages aloud
|
||||||
|
speaker: Speaker,
|
||||||
|
/// The message being revealed in step with speech, with its estimated duration
|
||||||
|
speech_reveal: Option<(EventId, Duration)>,
|
||||||
|
/// Fallback reveal end for backends that cannot report utterance end
|
||||||
|
reveal_timeout: Option<Task<Result<(), Error>>>,
|
||||||
/// Subject input state
|
/// Subject input state
|
||||||
subject_input: Entity<InputState>,
|
subject_input: Entity<InputState>,
|
||||||
|
|
||||||
/// Subject bar visibility
|
/// Subject bar visibility
|
||||||
subject_bar: Entity<bool>,
|
subject_bar: Entity<bool>,
|
||||||
|
|
||||||
/// Replies to
|
/// Replies to
|
||||||
replies_to: Entity<HashSet<EventId>>,
|
replies_to: Entity<HashSet<EventId>>,
|
||||||
|
|
||||||
/// Media Attachment
|
/// Media Attachment
|
||||||
attachments: Entity<Vec<Url>>,
|
attachments: Entity<Vec<Url>>,
|
||||||
|
|
||||||
/// Uploaded, encrypted file attachments which are not sent yet
|
/// Uploaded, encrypted file attachments which are not sent yet
|
||||||
encrypted_attachments: Entity<Vec<PendingFile>>,
|
encrypted_attachments: Entity<Vec<PendingFile>>,
|
||||||
|
|
||||||
/// Decrypted attachments of file messages, by message id
|
/// Decrypted attachments of file messages, by message id
|
||||||
decrypted_files: HashMap<EventId, DecryptedFile>,
|
decrypted_files: HashMap<EventId, DecryptedFile>,
|
||||||
|
|
||||||
/// Upload state
|
/// Upload state
|
||||||
uploading: bool,
|
uploading: bool,
|
||||||
|
|
||||||
/// Async operations
|
|
||||||
tasks: Vec<Task<Result<(), Error>>>,
|
tasks: Vec<Task<Result<(), Error>>>,
|
||||||
|
|
||||||
/// Event subscriptions
|
|
||||||
subscriptions: SmallVec<[Subscription; 4]>,
|
subscriptions: SmallVec<[Subscription; 4]>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -241,6 +225,20 @@ impl ChatPanel {
|
|||||||
this.get_messages(window, cx);
|
this.get_messages(window, cx);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// End the speech reveal when the utterance finishes ahead of its estimate
|
||||||
|
let speaker = Speaker::new();
|
||||||
|
let end_events = speaker.end_events();
|
||||||
|
let reveal_end_task = cx.spawn(async move |this, cx| {
|
||||||
|
while end_events.recv_async().await.is_ok() {
|
||||||
|
this.update(cx, |this, cx| {
|
||||||
|
this.speech_reveal = None;
|
||||||
|
cx.notify();
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
focus_handle: cx.focus_handle(),
|
focus_handle: cx.focus_handle(),
|
||||||
id,
|
id,
|
||||||
@@ -253,6 +251,9 @@ impl ChatPanel {
|
|||||||
input,
|
input,
|
||||||
speech,
|
speech,
|
||||||
speech_started: None,
|
speech_started: None,
|
||||||
|
speaker,
|
||||||
|
speech_reveal: None,
|
||||||
|
reveal_timeout: None,
|
||||||
subject_input,
|
subject_input,
|
||||||
subject_bar,
|
subject_bar,
|
||||||
replies_to,
|
replies_to,
|
||||||
@@ -263,7 +264,7 @@ impl ChatPanel {
|
|||||||
reports_by_id,
|
reports_by_id,
|
||||||
uploading: false,
|
uploading: false,
|
||||||
subscriptions,
|
subscriptions,
|
||||||
tasks: vec![],
|
tasks: vec![reveal_end_task],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1192,6 +1193,15 @@ impl ChatPanel {
|
|||||||
nostr.read(cx).current_user(),
|
nostr.read(cx).current_user(),
|
||||||
));
|
));
|
||||||
|
|
||||||
|
let body = match self
|
||||||
|
.speech_reveal
|
||||||
|
.as_ref()
|
||||||
|
.filter(|(reveal_id, _)| *reveal_id == id)
|
||||||
|
{
|
||||||
|
Some((_, duration)) => RevealText::new(message.preview(), *duration).into_any_element(),
|
||||||
|
None => rendered_text,
|
||||||
|
};
|
||||||
|
|
||||||
MessageRow::new(ix)
|
MessageRow::new(ix)
|
||||||
.show_author(show_author)
|
.show_author(show_author)
|
||||||
.hide_avatar(hide_avatar)
|
.hide_avatar(hide_avatar)
|
||||||
@@ -1216,7 +1226,7 @@ impl ChatPanel {
|
|||||||
.when(has_replies, |this| {
|
.when(has_replies, |this| {
|
||||||
this.children(self.render_message_replies(replies, cx))
|
this.children(self.render_message_replies(replies, cx))
|
||||||
})
|
})
|
||||||
.when(message.file.is_none(), |this| this.child(rendered_text))
|
.when(message.file.is_none(), |this| this.child(body))
|
||||||
.child(MessageMedia::new(&message.media))
|
.child(MessageMedia::new(&message.media))
|
||||||
.when_some(message.file.as_ref(), |this, file| {
|
.when_some(message.file.as_ref(), |this, file| {
|
||||||
this.child(self.render_message_file(&id, file, cx))
|
this.child(self.render_message_file(&id, file, cx))
|
||||||
@@ -1464,6 +1474,43 @@ impl ChatPanel {
|
|||||||
actions
|
actions
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let speaking = self.speech_reveal.is_some();
|
||||||
|
let speech_id = *id;
|
||||||
|
|
||||||
|
let actions = actions.speech(
|
||||||
|
speaking,
|
||||||
|
cx.listener(move |this, _event, window, cx| {
|
||||||
|
let text = this.message(&speech_id).map(|message| message.preview());
|
||||||
|
|
||||||
|
if let Some(text) = text {
|
||||||
|
match this.speaker.speak(&text) {
|
||||||
|
Ok(duration) => {
|
||||||
|
this.speech_reveal = Some((speech_id, duration));
|
||||||
|
|
||||||
|
// Re-enable speaking when no end event ever arrives
|
||||||
|
let session = (speech_id, duration);
|
||||||
|
let timer = cx.background_executor().timer(duration + END_GRACE);
|
||||||
|
this.reveal_timeout = Some(cx.spawn(async move |this, cx| {
|
||||||
|
timer.await;
|
||||||
|
this.update(cx, |this, cx| {
|
||||||
|
if this.speech_reveal.as_ref() == Some(&session) {
|
||||||
|
this.speech_reveal = None;
|
||||||
|
cx.notify();
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}));
|
||||||
|
cx.notify();
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
window.push_notification(Notification::error(error.to_string()), cx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
actions.dropdown(move |menu, _window, _cx| {
|
actions.dropdown(move |menu, _window, _cx| {
|
||||||
menu.menu("Copy author", Box::new(Command::Copy(public_key)))
|
menu.menu("Copy author", Box::new(Command::Copy(public_key)))
|
||||||
.menu("Seen on", Box::new(Command::Trace(dropdown_id)))
|
.menu("Seen on", Box::new(Command::Trace(dropdown_id)))
|
||||||
|
|||||||
+40
-12
@@ -26,8 +26,9 @@ use person::Directory;
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use settings::AppSettings;
|
use settings::AppSettings;
|
||||||
use smallvec::{SmallVec, smallvec};
|
use smallvec::{SmallVec, smallvec};
|
||||||
|
use speech::speaker::Speaker;
|
||||||
use state::{NostrRegistry, pick_file, upload_file};
|
use state::{NostrRegistry, pick_file, upload_file};
|
||||||
use ui::{Avatar, RenderedText, WelcomeMessage, rendered_text};
|
use ui::{Avatar, RenderedText, RevealText, WelcomeMessage, rendered_text};
|
||||||
use util::{attachment_name, display_name, opens_run, pin_label, report};
|
use util::{attachment_name, display_name, opens_run, pin_label, report};
|
||||||
|
|
||||||
mod details;
|
mod details;
|
||||||
@@ -320,6 +321,12 @@ pub struct GroupPanel {
|
|||||||
speech: Entity<SpeechState>,
|
speech: Entity<SpeechState>,
|
||||||
/// When the running dictation started, for the elapsed counter
|
/// When the running dictation started, for the elapsed counter
|
||||||
speech_started: Option<instant::Instant>,
|
speech_started: Option<instant::Instant>,
|
||||||
|
/// Reads messages aloud
|
||||||
|
speaker: Speaker,
|
||||||
|
/// The message being revealed in step with speech, with its estimated duration
|
||||||
|
speech_reveal: Option<(EventId, Duration)>,
|
||||||
|
/// Fallback reveal end for backends that cannot report utterance end
|
||||||
|
reveal_timeout: Option<Task<Result<()>>>,
|
||||||
/// Uploaded, non-encrypted attachments waiting to be sent.
|
/// Uploaded, non-encrypted attachments waiting to be sent.
|
||||||
attachments: Vec<Url>,
|
attachments: Vec<Url>,
|
||||||
/// Whether an upload is in flight.
|
/// Whether an upload is in flight.
|
||||||
@@ -419,9 +426,24 @@ impl GroupPanel {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
));
|
));
|
||||||
|
this.track_participants(cx);
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut this = Self {
|
// End the speech reveal when the utterance finishes ahead of its estimate
|
||||||
|
let speaker = Speaker::new();
|
||||||
|
let end_events = speaker.end_events();
|
||||||
|
let reveal_end_task = cx.spawn(async move |this, cx| {
|
||||||
|
while end_events.recv_async().await.is_ok() {
|
||||||
|
this.update(cx, |this, cx| {
|
||||||
|
this.speech_reveal = None;
|
||||||
|
cx.notify();
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
|
||||||
|
Self {
|
||||||
focus_handle: cx.focus_handle(),
|
focus_handle: cx.focus_handle(),
|
||||||
group: group.downgrade(),
|
group: group.downgrade(),
|
||||||
directory: Directory::new(cx),
|
directory: Directory::new(cx),
|
||||||
@@ -433,19 +455,17 @@ impl GroupPanel {
|
|||||||
input,
|
input,
|
||||||
speech,
|
speech,
|
||||||
speech_started: None,
|
speech_started: None,
|
||||||
|
speaker,
|
||||||
|
speech_reveal: None,
|
||||||
|
reveal_timeout: None,
|
||||||
attachments: Vec::new(),
|
attachments: Vec::new(),
|
||||||
uploading: false,
|
uploading: false,
|
||||||
tasks: smallvec![],
|
tasks: smallvec![reveal_end_task],
|
||||||
_subscriptions: subscriptions,
|
_subscriptions: subscriptions,
|
||||||
};
|
}
|
||||||
|
|
||||||
this.track_participants(cx);
|
|
||||||
|
|
||||||
this
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Track the keys rendered by this panel: members, message authors, and
|
/// Track the keys rendered by this panel.
|
||||||
/// activity subjects
|
|
||||||
fn track_participants(&mut self, cx: &mut Context<Self>) {
|
fn track_participants(&mut self, cx: &mut Context<Self>) {
|
||||||
let Some(group) = self.group.upgrade() else {
|
let Some(group) = self.group.upgrade() else {
|
||||||
return;
|
return;
|
||||||
@@ -1023,7 +1043,13 @@ impl GroupPanel {
|
|||||||
let nostr = NostrRegistry::global(cx);
|
let nostr = NostrRegistry::global(cx);
|
||||||
let directory = self.directory.clone();
|
let directory = self.directory.clone();
|
||||||
|
|
||||||
let content = self
|
let content = match self
|
||||||
|
.speech_reveal
|
||||||
|
.as_ref()
|
||||||
|
.filter(|(reveal_id, _)| *reveal_id == message.id)
|
||||||
|
{
|
||||||
|
Some((_, duration)) => RevealText::new(message.preview(), *duration).into_any_element(),
|
||||||
|
None => self
|
||||||
.rendered_texts_by_id
|
.rendered_texts_by_id
|
||||||
.entry(message.id)
|
.entry(message.id)
|
||||||
.or_insert_with(|| {
|
.or_insert_with(|| {
|
||||||
@@ -1038,7 +1064,8 @@ impl GroupPanel {
|
|||||||
cx,
|
cx,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.element(ix.into(), window, cx);
|
.element(ix.into(), window, cx),
|
||||||
|
};
|
||||||
|
|
||||||
let admin = nostr
|
let admin = nostr
|
||||||
.read(cx)
|
.read(cx)
|
||||||
@@ -1059,6 +1086,7 @@ impl GroupPanel {
|
|||||||
panel: cx.weak_entity(),
|
panel: cx.weak_entity(),
|
||||||
admin,
|
admin,
|
||||||
pinned,
|
pinned,
|
||||||
|
speaking: self.speech_reveal.is_some(),
|
||||||
persons: self.directory.clone(),
|
persons: self.directory.clone(),
|
||||||
},
|
},
|
||||||
cx,
|
cx,
|
||||||
|
|||||||
@@ -8,15 +8,17 @@ use gpui::{
|
|||||||
use gpui_kit::assets::IconName;
|
use gpui_kit::assets::IconName;
|
||||||
use gpui_kit::base::StyledExt as _;
|
use gpui_kit::base::StyledExt as _;
|
||||||
use gpui_kit::component::menu::{ContextMenuExt as _, PopupMenuItem};
|
use gpui_kit::component::menu::{ContextMenuExt as _, PopupMenuItem};
|
||||||
use gpui_kit::component::{ActiveTheme, Icon, Sizable, h_flex};
|
use gpui_kit::component::notification::Notification;
|
||||||
|
use gpui_kit::component::{ActiveTheme, Icon, Sizable, WindowExt, h_flex};
|
||||||
use nip29::{Activity, Group, Reaction};
|
use nip29::{Activity, Group, Reaction};
|
||||||
use nostr_sdk::prelude::{EventId, Kind};
|
use nostr_sdk::prelude::{EventId, Kind};
|
||||||
use person::Directory;
|
use person::Directory;
|
||||||
use settings::AppSettings;
|
use settings::AppSettings;
|
||||||
|
use speech::speaker::END_GRACE;
|
||||||
use state::NostrRegistry;
|
use state::NostrRegistry;
|
||||||
use ui::{
|
use ui::{
|
||||||
Avatar, MessageMedia, MessageRow, REACTION_EMOJIS, ReactionBar, ReplyPreview, group_reactions,
|
Avatar, MessageActions, MessageMedia, MessageRow, REACTION_EMOJIS, ReactionBar, ReplyPreview,
|
||||||
react_menu,
|
group_reactions, react_menu,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::GroupPanel;
|
use crate::GroupPanel;
|
||||||
@@ -26,6 +28,8 @@ pub(crate) struct Actions {
|
|||||||
pub panel: WeakEntity<GroupPanel>,
|
pub panel: WeakEntity<GroupPanel>,
|
||||||
pub admin: bool,
|
pub admin: bool,
|
||||||
pub pinned: bool,
|
pub pinned: bool,
|
||||||
|
/// Whether a message is currently being spoken, disabling the speech buttons
|
||||||
|
pub speaking: bool,
|
||||||
pub persons: Entity<Directory>,
|
pub persons: Entity<Directory>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -87,10 +91,44 @@ pub(crate) fn render(
|
|||||||
let admin = actions.admin;
|
let admin = actions.admin;
|
||||||
let pinned = actions.pinned;
|
let pinned = actions.pinned;
|
||||||
let group = actions.group;
|
let group = actions.group;
|
||||||
let panel = actions.panel;
|
let speech_panel = actions.panel;
|
||||||
|
let panel = speech_panel.clone();
|
||||||
|
let speech_text = message.preview();
|
||||||
|
let speech_id = message.id;
|
||||||
|
let speaking = actions.speaking;
|
||||||
|
|
||||||
|
let row = row.overlay(
|
||||||
|
MessageActions::new().speech(speaking, move |_event, _window, cx| {
|
||||||
|
_ = speech_panel.update_in(cx, |panel, window, cx| {
|
||||||
|
match panel.speaker.speak(&speech_text) {
|
||||||
|
Ok(duration) => {
|
||||||
|
panel.speech_reveal = Some((speech_id, duration));
|
||||||
|
|
||||||
|
let session = (speech_id, duration);
|
||||||
|
let timer = cx.background_executor().timer(duration + END_GRACE);
|
||||||
|
|
||||||
|
panel.reveal_timeout = Some(cx.spawn(async move |this, cx| {
|
||||||
|
timer.await;
|
||||||
|
this.update(cx, |this, cx| {
|
||||||
|
if this.speech_reveal.as_ref() == Some(&session) {
|
||||||
|
this.speech_reveal = None;
|
||||||
|
cx.notify();
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}));
|
||||||
|
|
||||||
|
cx.notify();
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
window.push_notification(Notification::error(error.to_string()), cx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
// Reading the panel here would re-enter its ongoing update, so resolve
|
|
||||||
// the current user's custom emojis through the shared person directory.
|
|
||||||
let custom = nostr
|
let custom = nostr
|
||||||
.read(cx)
|
.read(cx)
|
||||||
.current_user()
|
.current_user()
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ gpui.workspace = true
|
|||||||
gpui-kit = { workspace = true, features = ["speech"] }
|
gpui-kit = { workspace = true, features = ["speech"] }
|
||||||
flume.workspace = true
|
flume.workspace = true
|
||||||
instant.workspace = true
|
instant.workspace = true
|
||||||
|
tts.workspace = true
|
||||||
|
|
||||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||||
common = { path = "../common" }
|
common = { path = "../common" }
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
pub mod model;
|
pub mod model;
|
||||||
#[cfg(not(target_arch = "wasm32"))]
|
#[cfg(not(target_arch = "wasm32"))]
|
||||||
mod parakeet;
|
mod parakeet;
|
||||||
|
pub mod speaker;
|
||||||
#[cfg(target_arch = "wasm32")]
|
#[cfg(target_arch = "wasm32")]
|
||||||
mod web;
|
mod web;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use tts::Tts;
|
||||||
|
|
||||||
|
/// Words per minute of an engine at its normal rate, for estimating speech duration.
|
||||||
|
const NORMAL_WPM: f32 = 160.0;
|
||||||
|
|
||||||
|
/// Extra time past the estimate before speech is considered finished, for backends that
|
||||||
|
/// cannot report utterance end.
|
||||||
|
pub const END_GRACE: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
|
/// Reads text aloud through the platform's speech backend, created on first use.
|
||||||
|
pub struct Speaker {
|
||||||
|
tts: Option<Tts>,
|
||||||
|
end_tx: flume::Sender<()>,
|
||||||
|
end_rx: flume::Receiver<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Speaker {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
let (end_tx, end_rx) = flume::unbounded();
|
||||||
|
|
||||||
|
Self {
|
||||||
|
tts: None,
|
||||||
|
end_tx,
|
||||||
|
end_rx,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cloneable stream of utterance-finish events.
|
||||||
|
pub fn end_events(&self) -> flume::Receiver<()> {
|
||||||
|
self.end_rx.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Speak the text, interrupting what this speaker is already saying.
|
||||||
|
pub fn speak(&mut self, text: &str) -> Result<Duration, tts::Error> {
|
||||||
|
let end_tx = self.end_tx.clone();
|
||||||
|
let tts = match self.tts.as_ref() {
|
||||||
|
Some(tts) => tts,
|
||||||
|
None => self.tts.insert({
|
||||||
|
let tts = Tts::default()?;
|
||||||
|
if let Err(error) = tts.on_utterance_end(move |_| {
|
||||||
|
// A send error only means the panel dropped its event loop.
|
||||||
|
let _ = end_tx.send(());
|
||||||
|
}) {
|
||||||
|
log::warn!("utterance end events unavailable ({error})");
|
||||||
|
}
|
||||||
|
tts
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
tts.speak(text, true)?;
|
||||||
|
Ok(Self::estimate(tts, text))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop speaking, a speaker that never spoke is already silent.
|
||||||
|
pub fn stop(&mut self) -> Result<(), tts::Error> {
|
||||||
|
match self.tts.as_ref() {
|
||||||
|
Some(tts) => tts.stop()?,
|
||||||
|
None => return Ok(()),
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Best-effort speaking state, a speaker that never spoke is silent.
|
||||||
|
pub fn speaking(&self) -> bool {
|
||||||
|
self.tts
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|tts| tts.is_speaking().ok())
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How long speaking the text is expected to take at the engine's current speed.
|
||||||
|
fn estimate(tts: &Tts, text: &str) -> Duration {
|
||||||
|
let words = text.split_whitespace().count().max(1) as f32;
|
||||||
|
let speed = match (tts.get_rate(), tts.normal_rate()) {
|
||||||
|
(Ok(rate), Ok(normal)) if normal > 0. => (rate / normal).clamp(0.2, 4.),
|
||||||
|
_ => 1.,
|
||||||
|
};
|
||||||
|
Duration::from_secs_f32(words / NORMAL_WPM * 60. / speed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for Speaker {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,6 +26,7 @@ pub enum IconName {
|
|||||||
PlusChat,
|
PlusChat,
|
||||||
Profile,
|
Profile,
|
||||||
Relay,
|
Relay,
|
||||||
|
Speech,
|
||||||
Search,
|
Search,
|
||||||
Settings,
|
Settings,
|
||||||
Settings2,
|
Settings2,
|
||||||
@@ -58,6 +59,7 @@ impl IconNamed for IconName {
|
|||||||
Self::PlusChat => "icons/plus-chat.svg",
|
Self::PlusChat => "icons/plus-chat.svg",
|
||||||
Self::Profile => "icons/profile.svg",
|
Self::Profile => "icons/profile.svg",
|
||||||
Self::Relay => "icons/relay.svg",
|
Self::Relay => "icons/relay.svg",
|
||||||
|
Self::Speech => "icons/speech.svg",
|
||||||
Self::Search => "icons/search.svg",
|
Self::Search => "icons/search.svg",
|
||||||
Self::Settings => "icons/settings.svg",
|
Self::Settings => "icons/settings.svg",
|
||||||
Self::Settings2 => "icons/settings2.svg",
|
Self::Settings2 => "icons/settings2.svg",
|
||||||
|
|||||||
@@ -1,19 +1,21 @@
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use gpui::prelude::FluentBuilder as _;
|
use gpui::prelude::FluentBuilder as _;
|
||||||
use gpui::{
|
use gpui::{
|
||||||
AnyElement, App, ClickEvent, Context, ElementId, InteractiveElement as _, IntoElement,
|
Animation, AnimationExt as _, AnyElement, App, ClickEvent, Context, ElementId,
|
||||||
MouseButton, MouseDownEvent, ObjectFit, ParentElement as _, RenderOnce, SharedString,
|
InteractiveElement as _, IntoElement, MouseButton, MouseDownEvent, ObjectFit,
|
||||||
SharedUri, StatefulInteractiveElement as _, StyleRefinement, Styled, StyledImage, Window, div,
|
ParentElement as _, RenderOnce, SharedString, SharedUri, StatefulInteractiveElement as _,
|
||||||
img, px,
|
StyleRefinement, Styled, StyledImage, Window, div, img, px,
|
||||||
};
|
};
|
||||||
use gpui_kit::assets::IconName;
|
use gpui_kit::assets::IconName;
|
||||||
use gpui_kit::component::button::{Button, ButtonVariants as _};
|
use gpui_kit::component::button::{Button, ButtonVariants as _};
|
||||||
use gpui_kit::component::clipboard::Clipboard;
|
use gpui_kit::component::clipboard::Clipboard;
|
||||||
use gpui_kit::component::menu::{DropdownMenu as _, PopupMenu, PopupMenuItem};
|
use gpui_kit::component::menu::{DropdownMenu as _, PopupMenu, PopupMenuItem};
|
||||||
use gpui_kit::component::{
|
use gpui_kit::component::{
|
||||||
ActiveTheme, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex, v_flex,
|
ActiveTheme, Disableable, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex,
|
||||||
|
v_flex,
|
||||||
};
|
};
|
||||||
use smallvec::SmallVec;
|
use smallvec::SmallVec;
|
||||||
|
|
||||||
@@ -597,13 +599,14 @@ pub fn react_menu(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A hovered message's action bar: quick reactions, reply, copy, and an overflow menu.
|
/// A hovered message's action bar: quick reactions, reply, copy, read aloud, and an overflow menu.
|
||||||
#[derive(IntoElement)]
|
#[derive(IntoElement)]
|
||||||
pub struct MessageActions {
|
pub struct MessageActions {
|
||||||
style: StyleRefinement,
|
style: StyleRefinement,
|
||||||
reactions: SmallVec<[(&'static str, SharedClickListener); 4]>,
|
reactions: SmallVec<[(&'static str, SharedClickListener); 4]>,
|
||||||
reply: Option<SharedClickListener>,
|
reply: Option<SharedClickListener>,
|
||||||
copy: Option<(ElementId, SharedString)>,
|
copy: Option<(ElementId, SharedString)>,
|
||||||
|
speech: Option<(bool, SharedClickListener)>,
|
||||||
dropdown: Option<DropdownBuilder>,
|
dropdown: Option<DropdownBuilder>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -615,6 +618,7 @@ impl MessageActions {
|
|||||||
reactions: SmallVec::new(),
|
reactions: SmallVec::new(),
|
||||||
reply: None,
|
reply: None,
|
||||||
copy: None,
|
copy: None,
|
||||||
|
speech: None,
|
||||||
dropdown: None,
|
dropdown: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -642,6 +646,16 @@ impl MessageActions {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Add the read-aloud button, disabled while a message is already being spoken.
|
||||||
|
pub fn speech(
|
||||||
|
mut self,
|
||||||
|
disabled: bool,
|
||||||
|
handler: impl Fn(&ClickEvent, &mut Window, &mut App) + 'static,
|
||||||
|
) -> Self {
|
||||||
|
self.speech = Some((disabled, Rc::new(handler)));
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Fill the bar's overflow menu.
|
/// Fill the bar's overflow menu.
|
||||||
pub fn dropdown(
|
pub fn dropdown(
|
||||||
mut self,
|
mut self,
|
||||||
@@ -671,10 +685,13 @@ impl RenderOnce for MessageActions {
|
|||||||
reactions,
|
reactions,
|
||||||
reply,
|
reply,
|
||||||
copy,
|
copy,
|
||||||
|
speech,
|
||||||
dropdown,
|
dropdown,
|
||||||
} = self;
|
} = self;
|
||||||
|
|
||||||
let has_center = reply.is_some() || copy.is_some();
|
// The leading separator only separates reactions from the center buttons
|
||||||
|
let has_center =
|
||||||
|
!reactions.is_empty() && (reply.is_some() || copy.is_some() || speech.is_some());
|
||||||
|
|
||||||
h_flex()
|
h_flex()
|
||||||
.p_0p5()
|
.p_0p5()
|
||||||
@@ -712,6 +729,17 @@ impl RenderOnce for MessageActions {
|
|||||||
.when_some(copy, |this, (id, value)| {
|
.when_some(copy, |this, (id, value)| {
|
||||||
this.child(Clipboard::new(id).value(value).tooltip("Copy").small())
|
this.child(Clipboard::new(id).value(value).tooltip("Copy").small())
|
||||||
})
|
})
|
||||||
|
.when_some(speech, |this, (disabled, speech)| {
|
||||||
|
this.child(
|
||||||
|
Button::new("speech")
|
||||||
|
.icon(crate::IconName::Speech)
|
||||||
|
.tooltip("Read aloud")
|
||||||
|
.small()
|
||||||
|
.ghost()
|
||||||
|
.disabled(disabled)
|
||||||
|
.on_click(move |event, window, cx| speech(event, window, cx)),
|
||||||
|
)
|
||||||
|
})
|
||||||
.when_some(dropdown, |this, dropdown| {
|
.when_some(dropdown, |this, dropdown| {
|
||||||
this.child(div().flex_shrink_0().h_4().w_px().bg(cx.theme().border))
|
this.child(div().flex_shrink_0().h_4().w_px().bg(cx.theme().border))
|
||||||
.child(
|
.child(
|
||||||
@@ -726,3 +754,44 @@ impl RenderOnce for MessageActions {
|
|||||||
.refine_style(&style)
|
.refine_style(&style)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A message's text revealed word by word, paced by the estimated speech duration.
|
||||||
|
#[derive(IntoElement)]
|
||||||
|
pub struct RevealText {
|
||||||
|
text: SharedString,
|
||||||
|
duration: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RevealText {
|
||||||
|
pub fn new(text: impl Into<SharedString>, duration: Duration) -> Self {
|
||||||
|
Self {
|
||||||
|
text: text.into(),
|
||||||
|
duration,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RenderOnce for RevealText {
|
||||||
|
fn render(self, _window: &mut Window, _cx: &mut App) -> impl IntoElement {
|
||||||
|
let words: Vec<SharedString> = self
|
||||||
|
.text
|
||||||
|
.split_whitespace()
|
||||||
|
.map(SharedString::from)
|
||||||
|
.collect();
|
||||||
|
let total = words.len().max(1) as f32;
|
||||||
|
|
||||||
|
div().flex().flex_wrap().gap_x_1().with_animation(
|
||||||
|
"speech-reveal",
|
||||||
|
Animation::new(self.duration).with_max_fps(30.),
|
||||||
|
move |this, progress| {
|
||||||
|
let visible = ((progress * total).ceil() as usize).min(words.len());
|
||||||
|
|
||||||
|
this.children(words.iter().enumerate().map(|(ix, word)| {
|
||||||
|
div()
|
||||||
|
.when(ix >= visible, |this| this.invisible())
|
||||||
|
.child(word.clone())
|
||||||
|
}))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user