feat: add support for speech to text #68
Generated
+3
@@ -1161,6 +1161,7 @@ dependencies = [
|
|||||||
"common",
|
"common",
|
||||||
"gpui-kit",
|
"gpui-kit",
|
||||||
"gpui-pre",
|
"gpui-pre",
|
||||||
|
"instant",
|
||||||
"itertools 0.13.0",
|
"itertools 0.13.0",
|
||||||
"nostr-sdk",
|
"nostr-sdk",
|
||||||
"person",
|
"person",
|
||||||
@@ -5430,6 +5431,7 @@ dependencies = [
|
|||||||
"common",
|
"common",
|
||||||
"gpui-kit",
|
"gpui-kit",
|
||||||
"gpui-pre",
|
"gpui-pre",
|
||||||
|
"instant",
|
||||||
"log",
|
"log",
|
||||||
"nip29",
|
"nip29",
|
||||||
"nostr-sdk",
|
"nostr-sdk",
|
||||||
@@ -8374,6 +8376,7 @@ dependencies = [
|
|||||||
"flume 0.11.1",
|
"flume 0.11.1",
|
||||||
"gpui-kit",
|
"gpui-kit",
|
||||||
"gpui-pre",
|
"gpui-pre",
|
||||||
|
"instant",
|
||||||
"js-sys",
|
"js-sys",
|
||||||
"log",
|
"log",
|
||||||
"ort",
|
"ort",
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ chat = { path = "../chat" }
|
|||||||
settings = { path = "../settings" }
|
settings = { path = "../settings" }
|
||||||
|
|
||||||
gpui.workspace = true
|
gpui.workspace = true
|
||||||
|
instant.workspace = true
|
||||||
nostr-sdk.workspace = true
|
nostr-sdk.workspace = true
|
||||||
anyhow.workspace = true
|
anyhow.workspace = true
|
||||||
itertools.workspace = true
|
itertools.workspace = true
|
||||||
|
|||||||
+54
-17
@@ -96,6 +96,9 @@ pub struct ChatPanel {
|
|||||||
/// Dictation session for the chat input
|
/// Dictation session for the chat input
|
||||||
speech: Entity<SpeechState>,
|
speech: Entity<SpeechState>,
|
||||||
|
|
||||||
|
/// When the running dictation started, for the elapsed counter
|
||||||
|
speech_started: Option<instant::Instant>,
|
||||||
|
|
||||||
/// Subject input state
|
/// Subject input state
|
||||||
subject_input: Entity<InputState>,
|
subject_input: Entity<InputState>,
|
||||||
|
|
||||||
@@ -207,14 +210,25 @@ impl ChatPanel {
|
|||||||
// Insert the dictated text into the chat input
|
// Insert the dictated text into the chat input
|
||||||
cx.subscribe_in(&speech, window, move |this, _speech, event, window, cx| {
|
cx.subscribe_in(&speech, window, move |this, _speech, event, window, cx| {
|
||||||
match event {
|
match event {
|
||||||
SpeechEvent::Final(text) if !text.is_empty() => {
|
SpeechEvent::Started => {
|
||||||
this.input.update(cx, |input, cx| {
|
this.speech_started = Some(instant::Instant::now());
|
||||||
input.insert(text.clone(), window, cx);
|
cx.notify();
|
||||||
});
|
}
|
||||||
|
SpeechEvent::Final(text) => {
|
||||||
|
this.speech_started = None;
|
||||||
|
|
||||||
|
if !text.is_empty() {
|
||||||
|
this.input
|
||||||
|
.update(cx, |input, cx| input.insert(text.clone(), window, cx));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
SpeechEvent::Error(error) => {
|
SpeechEvent::Error(error) => {
|
||||||
|
this.speech_started = None;
|
||||||
window.push_notification(Notification::error(error.to_string()), cx);
|
window.push_notification(Notification::error(error.to_string()), cx);
|
||||||
}
|
}
|
||||||
|
SpeechEvent::Cancelled => {
|
||||||
|
this.speech_started = None;
|
||||||
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
};
|
};
|
||||||
}),
|
}),
|
||||||
@@ -238,6 +252,7 @@ impl ChatPanel {
|
|||||||
list_state,
|
list_state,
|
||||||
input,
|
input,
|
||||||
speech,
|
speech,
|
||||||
|
speech_started: None,
|
||||||
subject_input,
|
subject_input,
|
||||||
subject_bar,
|
subject_bar,
|
||||||
replies_to,
|
replies_to,
|
||||||
@@ -1706,7 +1721,12 @@ impl ChatPanel {
|
|||||||
items
|
items
|
||||||
}
|
}
|
||||||
|
|
||||||
fn render_config_menu(&self, _window: &mut Window, cx: &Context<Self>) -> impl IntoElement {
|
fn render_config_menu(
|
||||||
|
&self,
|
||||||
|
disabled: bool,
|
||||||
|
_window: &mut Window,
|
||||||
|
cx: &Context<Self>,
|
||||||
|
) -> impl IntoElement {
|
||||||
let (backup, signer_kind) = self
|
let (backup, signer_kind) = self
|
||||||
.room
|
.room
|
||||||
.read_with(cx, |this, _cx| {
|
.read_with(cx, |this, _cx| {
|
||||||
@@ -1719,6 +1739,7 @@ impl ChatPanel {
|
|||||||
.icon(IconName::Settings2)
|
.icon(IconName::Settings2)
|
||||||
.tooltip("Configuration")
|
.tooltip("Configuration")
|
||||||
.ghost()
|
.ghost()
|
||||||
|
.disabled(disabled)
|
||||||
.dropdown_menu(move |this, _window, _cx| {
|
.dropdown_menu(move |this, _window, _cx| {
|
||||||
let auto = matches!(signer_kind, SignerKind::Auto);
|
let auto = matches!(signer_kind, SignerKind::Auto);
|
||||||
let encryption = matches!(signer_kind, SignerKind::Encryption);
|
let encryption = matches!(signer_kind, SignerKind::Encryption);
|
||||||
@@ -1749,7 +1770,12 @@ impl ChatPanel {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn render_emoji_menu(&self, _window: &Window, cx: &Context<Self>) -> impl IntoElement {
|
fn render_emoji_menu(
|
||||||
|
&self,
|
||||||
|
disabled: bool,
|
||||||
|
_window: &Window,
|
||||||
|
cx: &Context<Self>,
|
||||||
|
) -> impl IntoElement {
|
||||||
let nostr = NostrRegistry::global(cx);
|
let nostr = NostrRegistry::global(cx);
|
||||||
let current_user = nostr.read(cx).current_user();
|
let current_user = nostr.read(cx).current_user();
|
||||||
|
|
||||||
@@ -1766,6 +1792,7 @@ impl ChatPanel {
|
|||||||
Button::new("emoji")
|
Button::new("emoji")
|
||||||
.icon(IconName::Emoji)
|
.icon(IconName::Emoji)
|
||||||
.ghost()
|
.ghost()
|
||||||
|
.disabled(disabled)
|
||||||
.dropdown_menu_with_anchor(gpui::Anchor::BottomLeft, move |this, _window, _cx| {
|
.dropdown_menu_with_anchor(gpui::Anchor::BottomLeft, move |this, _window, _cx| {
|
||||||
let menu = REACTION_EMOJIS.iter().fold(this, |this, emoji| {
|
let menu = REACTION_EMOJIS.iter().fold(this, |this, emoji| {
|
||||||
this.menu(*emoji, Box::new(Command::Insert(emoji.to_string())))
|
this.menu(*emoji, Box::new(Command::Insert(emoji.to_string())))
|
||||||
@@ -1839,7 +1866,7 @@ impl Render for ChatPanel {
|
|||||||
|
|
||||||
let is_typing = !self.input.read(cx).value().trim().is_empty();
|
let is_typing = !self.input.read(cx).value().trim().is_empty();
|
||||||
let pending_attachments = !self.encrypted_attachments.read(cx).is_empty();
|
let pending_attachments = !self.encrypted_attachments.read(cx).is_empty();
|
||||||
let capturing = self.speech.read(cx).status().is_capturing();
|
let dictating = self.speech.read(cx).status().is_active();
|
||||||
|
|
||||||
v_flex()
|
v_flex()
|
||||||
.image_cache(retain_all(self.id.clone()))
|
.image_cache(retain_all(self.id.clone()))
|
||||||
@@ -1920,13 +1947,27 @@ impl Render for ChatPanel {
|
|||||||
.icon(IconName::Paperclip)
|
.icon(IconName::Paperclip)
|
||||||
.tooltip("Upload media")
|
.tooltip("Upload media")
|
||||||
.loading(self.uploading)
|
.loading(self.uploading)
|
||||||
.disabled(self.uploading)
|
.disabled(dictating || self.uploading)
|
||||||
.ghost()
|
.ghost()
|
||||||
.on_click(cx.listener(move |this, _ev, window, cx| {
|
.on_click(cx.listener(move |this, _ev, window, cx| {
|
||||||
this.upload(window, cx);
|
this.upload(window, cx);
|
||||||
})),
|
})),
|
||||||
)
|
)
|
||||||
.child(
|
.when(dictating, |this| {
|
||||||
|
this.child(
|
||||||
|
h_flex()
|
||||||
|
.h_8()
|
||||||
|
.flex_1()
|
||||||
|
.gap_2()
|
||||||
|
.pl_1()
|
||||||
|
.child(SpeechWaveform::new(&self.speech).small().flex_1())
|
||||||
|
.when_some(self.speech_started, |this, started| {
|
||||||
|
this.child(speech::elapsed(started, cx))
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.when(!dictating, |this| {
|
||||||
|
this.child(
|
||||||
Textarea::new(&self.input)
|
Textarea::new(&self.input)
|
||||||
.appearance(false)
|
.appearance(false)
|
||||||
.small()
|
.small()
|
||||||
@@ -1934,28 +1975,24 @@ impl Render for ChatPanel {
|
|||||||
.py_1()
|
.py_1()
|
||||||
.flex_1(),
|
.flex_1(),
|
||||||
)
|
)
|
||||||
|
})
|
||||||
.child(
|
.child(
|
||||||
h_flex()
|
h_flex()
|
||||||
.items_end()
|
.items_end()
|
||||||
.gap_1()
|
.gap_1()
|
||||||
.pl_1()
|
.pl_1()
|
||||||
.when(capturing, |this| {
|
|
||||||
this.child(
|
|
||||||
SpeechWaveform::new(&self.speech).xsmall().w_12(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.child(SpeechButton::new(&self.speech)),
|
.child(SpeechButton::new(&self.speech)),
|
||||||
)
|
)
|
||||||
.child(
|
.child(
|
||||||
h_flex()
|
h_flex()
|
||||||
.pl_1()
|
.pl_1()
|
||||||
.gap_1()
|
.gap_1()
|
||||||
.child(self.render_emoji_menu(window, cx))
|
.child(self.render_emoji_menu(dictating, window, cx))
|
||||||
.child(self.render_config_menu(window, cx))
|
.child(self.render_config_menu(dictating, window, cx))
|
||||||
.child(
|
.child(
|
||||||
Button::new("send")
|
Button::new("send")
|
||||||
.icon(IconName::PaperPlaneFill)
|
.icon(IconName::PaperPlaneFill)
|
||||||
.disabled(self.uploading)
|
.disabled(dictating || self.uploading)
|
||||||
.ghost()
|
.ghost()
|
||||||
.on_click(cx.listener(move |this, _ev, window, cx| {
|
.on_click(cx.listener(move |this, _ev, window, cx| {
|
||||||
this.send_text_message(window, cx);
|
this.send_text_message(window, cx);
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ gpui-kit = { workspace = true, features = ["speech"] }
|
|||||||
ui = { path = "../ui" }
|
ui = { path = "../ui" }
|
||||||
|
|
||||||
gpui.workspace = true
|
gpui.workspace = true
|
||||||
|
instant.workspace = true
|
||||||
nostr-sdk.workspace = true
|
nostr-sdk.workspace = true
|
||||||
anyhow.workspace = true
|
anyhow.workspace = true
|
||||||
log.workspace = true
|
log.workspace = true
|
||||||
|
|||||||
@@ -316,6 +316,8 @@ pub struct GroupPanel {
|
|||||||
input: Entity<TextareaState>,
|
input: Entity<TextareaState>,
|
||||||
/// Dictation session for the composer input
|
/// Dictation session for the composer input
|
||||||
speech: Entity<SpeechState>,
|
speech: Entity<SpeechState>,
|
||||||
|
/// When the running dictation started, for the elapsed counter
|
||||||
|
speech_started: Option<instant::Instant>,
|
||||||
/// Uploaded, non-encrypted attachments waiting to be sent.
|
/// Uploaded, non-encrypted attachments waiting to be sent.
|
||||||
attachments: Vec<Url>,
|
attachments: Vec<Url>,
|
||||||
/// Whether an upload is in flight.
|
/// Whether an upload is in flight.
|
||||||
@@ -367,14 +369,26 @@ impl GroupPanel {
|
|||||||
subscriptions.push(
|
subscriptions.push(
|
||||||
cx.subscribe_in(&speech, window, |this, _, event, window, cx| {
|
cx.subscribe_in(&speech, window, |this, _, event, window, cx| {
|
||||||
match event {
|
match event {
|
||||||
SpeechEvent::Final(text) if !text.is_empty() => {
|
SpeechEvent::Started => {
|
||||||
|
this.speech_started = Some(instant::Instant::now());
|
||||||
|
cx.notify();
|
||||||
|
}
|
||||||
|
SpeechEvent::Final(text) => {
|
||||||
|
this.speech_started = None;
|
||||||
|
|
||||||
|
if !text.is_empty() {
|
||||||
this.input.update(cx, |input, cx| {
|
this.input.update(cx, |input, cx| {
|
||||||
input.insert(text.clone(), window, cx);
|
input.insert(text.clone(), window, cx);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
}
|
||||||
SpeechEvent::Error(error) => {
|
SpeechEvent::Error(error) => {
|
||||||
|
this.speech_started = None;
|
||||||
window.push_notification(Notification::error(error.to_string()), cx);
|
window.push_notification(Notification::error(error.to_string()), cx);
|
||||||
}
|
}
|
||||||
|
SpeechEvent::Cancelled => {
|
||||||
|
this.speech_started = None;
|
||||||
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
};
|
};
|
||||||
}),
|
}),
|
||||||
@@ -416,6 +430,7 @@ impl GroupPanel {
|
|||||||
rendered_texts_by_id: BTreeMap::new(),
|
rendered_texts_by_id: BTreeMap::new(),
|
||||||
input,
|
input,
|
||||||
speech,
|
speech,
|
||||||
|
speech_started: None,
|
||||||
attachments: Vec::new(),
|
attachments: Vec::new(),
|
||||||
uploading: false,
|
uploading: false,
|
||||||
tasks: smallvec![],
|
tasks: smallvec![],
|
||||||
@@ -1073,6 +1088,8 @@ impl GroupPanel {
|
|||||||
return self.render_notice(¬ice, cx);
|
return self.render_notice(¬ice, cx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let dictating = self.speech.read(cx).status().is_active();
|
||||||
|
|
||||||
v_flex()
|
v_flex()
|
||||||
.flex_shrink_0()
|
.flex_shrink_0()
|
||||||
.w_full()
|
.w_full()
|
||||||
@@ -1091,13 +1108,26 @@ impl GroupPanel {
|
|||||||
.icon(ui::IconName::Paperclip)
|
.icon(ui::IconName::Paperclip)
|
||||||
.tooltip("Upload media")
|
.tooltip("Upload media")
|
||||||
.loading(self.uploading)
|
.loading(self.uploading)
|
||||||
.disabled(self.uploading)
|
.disabled(dictating || self.uploading)
|
||||||
.ghost()
|
.ghost()
|
||||||
.on_click(cx.listener(|this, _event, window, cx| {
|
.on_click(cx.listener(|this, _event, window, cx| {
|
||||||
this.upload(window, cx);
|
this.upload(window, cx);
|
||||||
})),
|
})),
|
||||||
)
|
)
|
||||||
.child(
|
.when(dictating, |this| {
|
||||||
|
this.child(
|
||||||
|
h_flex()
|
||||||
|
.h_8()
|
||||||
|
.flex_1()
|
||||||
|
.gap_2()
|
||||||
|
.child(SpeechWaveform::new(&self.speech).small().flex_1())
|
||||||
|
.when_some(self.speech_started, |this, started| {
|
||||||
|
this.child(speech::elapsed(started, cx))
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.when(!dictating, |this| {
|
||||||
|
this.child(
|
||||||
Textarea::new(&self.input)
|
Textarea::new(&self.input)
|
||||||
.appearance(false)
|
.appearance(false)
|
||||||
.bg(cx.theme().input)
|
.bg(cx.theme().input)
|
||||||
@@ -1106,8 +1136,6 @@ impl GroupPanel {
|
|||||||
.py_1()
|
.py_1()
|
||||||
.flex_1(),
|
.flex_1(),
|
||||||
)
|
)
|
||||||
.when(self.speech.read(cx).status().is_capturing(), |this| {
|
|
||||||
this.child(SpeechWaveform::new(&self.speech).xsmall().w_12())
|
|
||||||
})
|
})
|
||||||
.child(SpeechButton::new(&self.speech))
|
.child(SpeechButton::new(&self.speech))
|
||||||
.child(
|
.child(
|
||||||
@@ -1115,7 +1143,7 @@ impl GroupPanel {
|
|||||||
.icon(ui::IconName::PaperPlaneFill)
|
.icon(ui::IconName::PaperPlaneFill)
|
||||||
.tooltip("Send")
|
.tooltip("Send")
|
||||||
.ghost()
|
.ghost()
|
||||||
.disabled(self.uploading)
|
.disabled(dictating || self.uploading)
|
||||||
.on_click(cx.listener(|this, _event, window, cx| {
|
.on_click(cx.listener(|this, _event, window, cx| {
|
||||||
this.send(window, cx);
|
this.send(window, cx);
|
||||||
})),
|
})),
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ log.workspace = true
|
|||||||
gpui.workspace = true
|
gpui.workspace = true
|
||||||
gpui-kit = { workspace = true, features = ["speech"] }
|
gpui-kit = { workspace = true, features = ["speech"] }
|
||||||
flume.workspace = true
|
flume.workspace = true
|
||||||
|
instant.workspace = true
|
||||||
|
|
||||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||||
common = { path = "../common" }
|
common = { path = "../common" }
|
||||||
|
|||||||
@@ -5,8 +5,35 @@ mod parakeet;
|
|||||||
#[cfg(target_arch = "wasm32")]
|
#[cfg(target_arch = "wasm32")]
|
||||||
mod web;
|
mod web;
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use gpui::{
|
||||||
|
Animation, AnimationExt as _, AnyElement, App, IntoElement as _, ParentElement as _,
|
||||||
|
Styled as _, div,
|
||||||
|
};
|
||||||
|
use gpui_kit::component::ActiveTheme as _;
|
||||||
use gpui_kit::component::speech::SpeechRecognizer;
|
use gpui_kit::component::speech::SpeechRecognizer;
|
||||||
|
|
||||||
|
/// A ticking `M:SS` counter for the dictation in progress.
|
||||||
|
pub fn elapsed(started: instant::Instant, cx: &App) -> AnyElement {
|
||||||
|
div()
|
||||||
|
.text_xs()
|
||||||
|
.text_color(cx.theme().muted_foreground)
|
||||||
|
.with_animation(
|
||||||
|
"dictation-elapsed",
|
||||||
|
Animation::new(Duration::from_secs(1))
|
||||||
|
.repeat()
|
||||||
|
.with_max_fps(1.),
|
||||||
|
move |counter, _| counter.child(format_elapsed(started)),
|
||||||
|
)
|
||||||
|
.into_any_element()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_elapsed(started: instant::Instant) -> String {
|
||||||
|
let seconds = started.elapsed().as_secs();
|
||||||
|
format!("{}:{:02}", seconds / 60, seconds % 60)
|
||||||
|
}
|
||||||
|
|
||||||
/// The recognizer backing the composer's microphone button.
|
/// The recognizer backing the composer's microphone button.
|
||||||
#[cfg(not(target_arch = "wasm32"))]
|
#[cfg(not(target_arch = "wasm32"))]
|
||||||
pub fn recognizer() -> impl SpeechRecognizer {
|
pub fn recognizer() -> impl SpeechRecognizer {
|
||||||
|
|||||||
Reference in New Issue
Block a user