Author SHA1 Message Date
reya 9fa4f4f2ff add speech component 2026-10-27 09:52:28 +07:00
reya 6fdbedac14 clean up 2026-10-27 09:17:11 +07:00
22 changed files with 125 additions and 1802 deletions
+3
View File
@@ -20,6 +20,9 @@ jobs:
- platform: windows-arm64
os: windows-11-arm
target: aarch64-pc-windows-msvc
- platform: macos-x64
os: macos-15-intel
target: x86_64-apple-darwin
- platform: macos-arm64
os: macos-latest
target: aarch64-apple-darwin
Generated
+63 -601
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -4,7 +4,7 @@ members = ["crates/*", "desktop", "web"]
default-members = ["desktop"]
[workspace.package]
version = "2.0.0-alpha3"
version = "2.0.0-alpha2"
edition = "2024"
publish = false
+3 -3
View File
@@ -37,8 +37,9 @@ pub fn asset_filter_for(os: &str, arch: &str) -> Option<AssetFilter> {
};
let arch = match (os, arch) {
// cargo-packager names the disk images `aarch64`.
// cargo-packager names the disk images `aarch64`/`x64`.
("macos", "aarch64") => "aarch64",
("macos", "x86_64") => "x64",
// `script/bundle-linux` names the tarballs `aarch64`/`x86_64`.
("linux", "aarch64") => "aarch64",
("linux", "x86_64") => "x86_64",
@@ -242,7 +243,7 @@ mod tests {
#[test]
fn picks_the_artifact_matching_os_and_architecture() {
assert_eq!(selected("macos", "aarch64"), Some("Coop_1.0.1_aarch64.dmg"));
assert_eq!(selected("macos", "x86_64"), None);
assert_eq!(selected("macos", "x86_64"), Some("Coop_1.0.1_x64.dmg"));
assert_eq!(
selected("linux", "aarch64"),
Some("coop-linux-aarch64.tar.gz")
@@ -257,7 +258,6 @@ mod tests {
fn has_no_target_for_windows_or_unknown_platforms() {
assert_eq!(asset_filter_for("windows", "x86_64"), None);
assert_eq!(asset_filter_for("freebsd", "x86_64"), None);
assert_eq!(asset_filter_for("macos", "x86_64"), None);
assert_eq!(asset_filter_for("macos", "riscv64"), None);
}
-2
View File
@@ -6,7 +6,6 @@ publish.workspace = true
[dependencies]
state = { path = "../state" }
speech = { path = "../speech" }
ui = { path = "../ui" }
gpui-kit = { workspace = true, features = ["speech"] }
common = { path = "../common" }
@@ -15,7 +14,6 @@ chat = { path = "../chat" }
settings = { path = "../settings" }
gpui.workspace = true
instant.workspace = true
nostr-sdk.workspace = true
anyhow.workspace = true
itertools.workspace = true
+1 -4
View File
@@ -1,4 +1,4 @@
use gpui::{Action, actions};
use gpui::Action;
use nostr_sdk::prelude::*;
use serde::Deserialize;
use settings::SignerKind;
@@ -15,6 +15,3 @@ pub enum Command {
Njump(PublicKey),
Trace(EventId),
}
// Start or stop dictation in the focused composer
actions!(chat, [ToggleDictation]);
+25 -85
View File
@@ -1,6 +1,5 @@
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::{Arc, LazyLock, RwLock};
use std::time::Duration;
pub use actions::*;
use anyhow::Error;
@@ -96,9 +95,6 @@ pub struct ChatPanel {
/// Dictation session for the chat input
speech: Entity<SpeechState>,
/// When the running dictation started, for the elapsed counter
speech_started: Option<instant::Instant>,
/// Subject input state
subject_input: Entity<InputState>,
@@ -169,17 +165,7 @@ impl ChatPanel {
});
// Define dictation state
let speech = cx.new(|cx| {
let state = SpeechState::new(cx)
.recognizer(speech::recognizer())
// Batch transcription of a long recording can take a while.
.stop_timeout(Duration::from_secs(180));
#[cfg(target_arch = "wasm32")]
let state = state.input(speech::input());
state
});
let speech = cx.new(SpeechState::new);
// Define subject input state
let subject_input = cx.new(|cx| InputState::new(window, cx).placeholder("New subject..."));
@@ -210,25 +196,14 @@ impl ChatPanel {
// Insert the dictated text into the chat input
cx.subscribe_in(&speech, window, move |this, _speech, event, window, cx| {
match event {
SpeechEvent::Started => {
this.speech_started = Some(instant::Instant::now());
cx.notify();
}
SpeechEvent::Final(text) => {
this.speech_started = None;
if !text.is_empty() {
this.input
.update(cx, |input, cx| input.insert(text.clone(), window, cx));
}
SpeechEvent::Final(text) if !text.is_empty() => {
this.input.update(cx, |input, cx| {
input.insert(text.clone(), window, cx);
});
}
SpeechEvent::Error(error) => {
this.speech_started = None;
window.push_notification(Notification::error(error.to_string()), cx);
}
SpeechEvent::Cancelled => {
this.speech_started = None;
}
_ => {}
};
}),
@@ -252,7 +227,6 @@ impl ChatPanel {
list_state,
input,
speech,
speech_started: None,
subject_input,
subject_bar,
replies_to,
@@ -882,17 +856,6 @@ impl ChatPanel {
self.directory.read(cx).person(public_key, cx)
}
fn toggle_dictation(
&mut self,
_: &ToggleDictation,
_window: &mut Window,
cx: &mut Context<Self>,
) {
self.speech.update(cx, |speech, cx| {
speech.toggle(cx);
});
}
fn on_command(&mut self, command: &Command, window: &mut Window, cx: &mut Context<Self>) {
match command {
Command::Insert(content) => {
@@ -1732,12 +1695,7 @@ impl ChatPanel {
items
}
fn render_config_menu(
&self,
disabled: bool,
_window: &mut Window,
cx: &Context<Self>,
) -> impl IntoElement {
fn render_config_menu(&self, _window: &mut Window, cx: &Context<Self>) -> impl IntoElement {
let (backup, signer_kind) = self
.room
.read_with(cx, |this, _cx| {
@@ -1750,7 +1708,6 @@ impl ChatPanel {
.icon(IconName::Settings2)
.tooltip("Configuration")
.ghost()
.disabled(disabled)
.dropdown_menu(move |this, _window, _cx| {
let auto = matches!(signer_kind, SignerKind::Auto);
let encryption = matches!(signer_kind, SignerKind::Encryption);
@@ -1781,12 +1738,7 @@ impl ChatPanel {
})
}
fn render_emoji_menu(
&self,
disabled: bool,
_window: &Window,
cx: &Context<Self>,
) -> impl IntoElement {
fn render_emoji_menu(&self, _window: &Window, cx: &Context<Self>) -> impl IntoElement {
let nostr = NostrRegistry::global(cx);
let current_user = nostr.read(cx).current_user();
@@ -1803,7 +1755,6 @@ impl ChatPanel {
Button::new("emoji")
.icon(IconName::Emoji)
.ghost()
.disabled(disabled)
.dropdown_menu_with_anchor(gpui::Anchor::BottomLeft, move |this, _window, _cx| {
let menu = REACTION_EMOJIS.iter().fold(this, |this, emoji| {
this.menu(*emoji, Box::new(Command::Insert(emoji.to_string())))
@@ -1877,12 +1828,11 @@ impl Render for ChatPanel {
let is_typing = !self.input.read(cx).value().trim().is_empty();
let pending_attachments = !self.encrypted_attachments.read(cx).is_empty();
let dictating = self.speech.read(cx).status().is_active();
let capturing = self.speech.read(cx).status().is_capturing();
v_flex()
.image_cache(retain_all(self.id.clone()))
.on_action(cx.listener(Self::on_command))
.on_action(cx.listener(Self::toggle_dictation))
.size_full()
.when(*self.subject_bar.read(cx), |this| {
this.child(
@@ -1959,52 +1909,42 @@ impl Render for ChatPanel {
.icon(IconName::Paperclip)
.tooltip("Upload media")
.loading(self.uploading)
.disabled(dictating || self.uploading)
.disabled(self.uploading)
.ghost()
.on_click(cx.listener(move |this, _ev, window, cx| {
this.upload(window, cx);
})),
)
.when(dictating, |this| {
this.child(
h_flex()
.h_8()
.flex_1()
.gap_2()
.pl_1()
.child(SpeechWaveform::new(&self.speech).small().flex_1())
.when_some(self.speech_started, |this, started| {
this.child(speech::elapsed(started, cx))
}),
)
})
.when(!dictating, |this| {
this.child(
Textarea::new(&self.input)
.appearance(false)
.small()
.min_h_8()
.py_1()
.flex_1(),
)
})
.child(
Textarea::new(&self.input)
.appearance(false)
.small()
.min_h_8()
.py_1()
.flex_1(),
)
.child(
h_flex()
.items_end()
.gap_1()
.pl_1()
.when(capturing, |this| {
this.child(
SpeechWaveform::new(&self.speech).xsmall().w_12(),
)
})
.child(SpeechButton::new(&self.speech)),
)
.child(
h_flex()
.pl_1()
.gap_1()
.child(self.render_emoji_menu(dictating, window, cx))
.child(self.render_config_menu(dictating, window, cx))
.child(self.render_emoji_menu(window, cx))
.child(self.render_config_menu(window, cx))
.child(
Button::new("send")
.icon(IconName::PaperPlaneFill)
.disabled(dictating || self.uploading)
.disabled(self.uploading)
.ghost()
.on_click(cx.listener(move |this, _ev, window, cx| {
this.send_text_message(window, cx);
-2
View File
@@ -11,13 +11,11 @@ common = { path = "../common" }
nip29 = { path = "../nip29" }
person = { path = "../person" }
settings = { path = "../settings" }
speech = { path = "../speech" }
state = { path = "../state" }
gpui-kit = { workspace = true, features = ["speech"] }
ui = { path = "../ui" }
gpui.workspace = true
instant.workspace = true
nostr-sdk.workspace = true
anyhow.workspace = true
log.workspace = true
+20 -73
View File
@@ -1,14 +1,11 @@
use std::collections::BTreeMap;
use std::time::Duration;
use anyhow::Result;
use chat_ui::ToggleDictation;
use gpui::prelude::FluentBuilder;
use gpui::{
Action, AnyElement, App, AppContext, Context, Entity, EventEmitter, FocusHandle, Focusable,
FollowMode, InteractiveElement, IntoElement, ListAlignment, ListScrollEvent, ListState,
ParentElement, Render, SharedString, Styled, Subscription, Task, WeakEntity, Window, div, list,
px,
FollowMode, IntoElement, ListAlignment, ListScrollEvent, ListState, ParentElement, Render,
SharedString, Styled, Subscription, Task, WeakEntity, Window, div, list, px,
};
use gpui_kit::assets::IconName;
use gpui_kit::base::StyledExt as _;
@@ -318,8 +315,6 @@ pub struct GroupPanel {
input: Entity<TextareaState>,
/// Dictation session for the composer input
speech: Entity<SpeechState>,
/// When the running dictation started, for the elapsed counter
speech_started: Option<instant::Instant>,
/// Uploaded, non-encrypted attachments waiting to be sent.
attachments: Vec<Url>,
/// Whether an upload is in flight.
@@ -346,17 +341,7 @@ impl GroupPanel {
.clean_on_escape()
});
let speech = cx.new(|cx| {
let state = SpeechState::new(cx)
.recognizer(speech::recognizer())
// Batch transcription of a long recording can take a while.
.stop_timeout(Duration::from_secs(180));
#[cfg(target_arch = "wasm32")]
let state = state.input(speech::input());
state
});
let speech = cx.new(SpeechState::new);
let mut subscriptions = smallvec![];
@@ -371,26 +356,14 @@ impl GroupPanel {
subscriptions.push(
cx.subscribe_in(&speech, window, |this, _, event, window, cx| {
match event {
SpeechEvent::Started => {
this.speech_started = Some(instant::Instant::now());
cx.notify();
}
SpeechEvent::Final(text) => {
this.speech_started = None;
if !text.is_empty() {
this.input.update(cx, |input, cx| {
input.insert(text.clone(), window, cx);
});
}
SpeechEvent::Final(text) if !text.is_empty() => {
this.input.update(cx, |input, cx| {
input.insert(text.clone(), window, cx);
});
}
SpeechEvent::Error(error) => {
this.speech_started = None;
window.push_notification(Notification::error(error.to_string()), cx);
}
SpeechEvent::Cancelled => {
this.speech_started = None;
}
_ => {}
};
}),
@@ -432,7 +405,6 @@ impl GroupPanel {
rendered_texts_by_id: BTreeMap::new(),
input,
speech,
speech_started: None,
attachments: Vec::new(),
uploading: false,
tasks: smallvec![],
@@ -1065,17 +1037,6 @@ impl GroupPanel {
)
}
fn toggle_dictation(
&mut self,
_ev: &ToggleDictation,
_window: &mut Window,
cx: &mut Context<Self>,
) {
self.speech.update(cx, |speech, cx| {
speech.toggle(cx);
});
}
fn render_composer(&self, cx: &mut Context<Self>) -> AnyElement {
let notice = self.group.upgrade().and_then(|group| {
let group = group.read(cx);
@@ -1101,8 +1062,6 @@ impl GroupPanel {
return self.render_notice(&notice, cx);
}
let dictating = self.speech.read(cx).status().is_active();
v_flex()
.flex_shrink_0()
.w_full()
@@ -1121,34 +1080,23 @@ impl GroupPanel {
.icon(ui::IconName::Paperclip)
.tooltip("Upload media")
.loading(self.uploading)
.disabled(dictating || self.uploading)
.disabled(self.uploading)
.ghost()
.on_click(cx.listener(|this, _event, window, cx| {
this.upload(window, cx);
})),
)
.when(dictating, |this| {
this.child(
h_flex()
.h_8()
.flex_1()
.gap_2()
.child(SpeechWaveform::new(&self.speech).small().flex_1())
.when_some(self.speech_started, |this, started| {
this.child(speech::elapsed(started, cx))
}),
)
})
.when(!dictating, |this| {
this.child(
Textarea::new(&self.input)
.appearance(false)
.bg(cx.theme().input)
.small()
.min_h_8()
.py_1()
.flex_1(),
)
.child(
Textarea::new(&self.input)
.appearance(false)
.bg(cx.theme().input)
.small()
.min_h_8()
.py_1()
.flex_1(),
)
.when(self.speech.read(cx).status().is_capturing(), |this| {
this.child(SpeechWaveform::new(&self.speech).xsmall().w_12())
})
.child(SpeechButton::new(&self.speech))
.child(
@@ -1156,7 +1104,7 @@ impl GroupPanel {
.icon(ui::IconName::PaperPlaneFill)
.tooltip("Send")
.ghost()
.disabled(dictating || self.uploading)
.disabled(self.uploading)
.on_click(cx.listener(|this, _event, window, cx| {
this.send(window, cx);
})),
@@ -1354,7 +1302,6 @@ impl Render for GroupPanel {
v_flex()
.size_full()
.min_w_0()
.on_action(cx.listener(Self::toggle_dictation))
.child(self.render_encryption_notice(cx))
.child(self.render_banner(cx))
.child(self.render_pins(cx))
-30
View File
@@ -1,30 +0,0 @@
[package]
name = "speech"
version.workspace = true
edition.workspace = true
publish.workspace = true
[dependencies]
anyhow.workspace = true
log.workspace = true
gpui.workspace = true
gpui-kit = { workspace = true, features = ["speech"] }
flume.workspace = true
instant.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
common = { path = "../common" }
parakeet-rs = "=0.3.8"
ort = "=2.0.0-rc.13"
reqwest = { version = "0.12", default-features = false, features = [
"blocking",
"rustls-tls",
] }
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
[target.'cfg(target_arch = "wasm32")'.dependencies]
js-sys = "0.3"
wasm-bindgen = "0.2"
web-sys = { version = "0.3", features = ["Window"] }
-53
View File
@@ -1,53 +0,0 @@
#[cfg(not(target_arch = "wasm32"))]
pub mod model;
#[cfg(not(target_arch = "wasm32"))]
mod parakeet;
#[cfg(target_arch = "wasm32")]
mod web;
use std::time::Duration;
use gpui::{
Animation, AnimationExt as _, AnyElement, App, IntoElement as _, ParentElement as _,
Styled as _, div,
};
use gpui_kit::component::ActiveTheme as _;
use gpui_kit::component::speech::SpeechRecognizer;
/// A ticking `M:SS` counter for the dictation in progress.
pub fn elapsed(started: instant::Instant, cx: &App) -> AnyElement {
div()
.text_xs()
.text_color(cx.theme().muted_foreground)
.with_animation(
"dictation-elapsed",
Animation::new(Duration::from_secs(1))
.repeat()
.with_max_fps(1.),
move |counter, _| counter.child(format_elapsed(started)),
)
.into_any_element()
}
fn format_elapsed(started: instant::Instant) -> String {
let seconds = started.elapsed().as_secs();
format!("{}:{:02}", seconds / 60, seconds % 60)
}
/// The recognizer backing the composer's microphone button.
#[cfg(not(target_arch = "wasm32"))]
pub fn recognizer() -> impl SpeechRecognizer {
parakeet::ParakeetRecognizer
}
/// The recognizer backing the composer's microphone button.
#[cfg(target_arch = "wasm32")]
pub fn recognizer() -> impl SpeechRecognizer {
web::WebSpeechRecognizer
}
/// The audio source backing the composer's microphone button on the web.
#[cfg(target_arch = "wasm32")]
pub fn input() -> impl gpui_kit::component::speech::AudioInput {
web::SelfCapturedInput
}
-23
View File
@@ -1,23 +0,0 @@
{
"repository": "istupakov/parakeet-tdt-0.6b-v3-onnx",
"revision": "8f23f0c03c8761650bdb5b40aaf3e40d2c15f1ce",
"source": "nvidia/parakeet-tdt-0.6b-v3",
"license": "CC-BY-4.0",
"files": [
{
"name": "decoder_joint-model.int8.onnx",
"size": 18202004,
"sha256": "eea7483ee3d1a30375daedc8ed83e3960c91b098812127a0d99d1c8977667a70"
},
{
"name": "encoder-model.int8.onnx",
"size": 652183999,
"sha256": "6139d2fa7e1b086097b277c7149725edbab89cc7c7ae64b23c741be4055aff09"
},
{
"name": "vocab.txt",
"size": 93939,
"sha256": "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d"
}
]
}
-295
View File
@@ -1,295 +0,0 @@
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::{fs, io};
use anyhow::{Result, bail};
use serde::Deserialize;
/// How the model is currently available.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Status {
NotInstalled,
Downloading { downloaded: u64, total: u64 },
Installed,
}
impl Status {
pub fn is_downloading(self) -> bool {
matches!(self, Status::Downloading { .. })
}
}
#[derive(Debug, Deserialize)]
struct Manifest {
repository: String,
revision: String,
files: Vec<Artifact>,
}
#[derive(Debug, Deserialize)]
struct Artifact {
name: String,
size: u64,
sha256: String,
}
fn manifest() -> &'static Manifest {
static MANIFEST: OnceLock<Manifest> = OnceLock::new();
MANIFEST.get_or_init(|| {
serde_json::from_str(include_str!("model.json")).expect("pinned model manifest")
})
}
/// The directory the model artifacts are downloaded into.
pub fn dir() -> PathBuf {
common::config_dir()
.join("speech")
.join("parakeet-tdt-0.6b-v3")
}
/// Total download size in bytes.
pub fn total_size() -> u64 {
manifest().files.iter().map(|file| file.size).sum()
}
/// The revision of the installed model, if its artifacts are complete.
fn installed_revision() -> Option<String> {
let manifest = manifest();
let dir = dir();
let complete = manifest.files.iter().all(|file| {
fs::metadata(dir.join(&file.name)).is_ok_and(|metadata| metadata.len() == file.size)
});
if complete
&& fs::read_to_string(dir.join("verified"))
.is_ok_and(|revision| revision == manifest.revision)
{
Some(manifest.revision.clone())
} else {
None
}
}
/// Whether the model is ready to load.
pub fn installed() -> bool {
status() == Status::Installed
}
/// The current status, reconciling the download state with the disk.
pub fn status() -> Status {
let state = state().lock().unwrap_or_else(|error| error.into_inner());
if let Download::Running {
downloaded, total, ..
} = state.download
{
return Status::Downloading { downloaded, total };
}
if installed_revision().is_some() {
Status::Installed
} else {
Status::NotInstalled
}
}
/// Take the last download failure, if it has not been reported yet.
pub fn take_error() -> Option<String> {
let mut state = state().lock().unwrap_or_else(|error| error.into_inner());
state.error.take()
}
/// Start downloading the model, unless a download is already running.
pub fn start_download() {
let mut state = state().lock().unwrap_or_else(|error| error.into_inner());
if matches!(state.download, Download::Running { .. }) {
return;
}
let cancel = Arc::new(AtomicBool::new(false));
state.error = None;
state.download = Download::Running {
downloaded: 0,
total: total_size(),
cancel: cancel.clone(),
};
if let Err(error) = std::thread::Builder::new()
.name("speech-model-download".into())
.spawn(move || download(cancel))
{
state.download = Download::Idle;
state.error = Some(error.to_string());
}
}
/// Request cancellation of the running download.
pub fn cancel_download() {
let state = state().lock().unwrap_or_else(|error| error.into_inner());
if let Download::Running { cancel, .. } = &state.download {
cancel.store(true, Ordering::Release);
}
}
/// Delete the downloaded model and any partial download.
pub fn delete() {
super::parakeet::unload();
cancel_download();
if let Err(error) = fs::remove_dir_all(dir())
&& error.kind() != io::ErrorKind::NotFound
{
log::warn!("failed to remove the speech model: {error}");
}
}
fn download(cancel: Arc<AtomicBool>) {
let result = download_inner(&cancel);
let mut state = state().lock().unwrap_or_else(|error| error.into_inner());
state.download = Download::Idle;
// A cancelled download must not report a spurious failure.
if !cancelled(&cancel)
&& let Err(error) = result
{
state.error = Some(format!("{error:#}"));
}
}
fn download_inner(cancel: &AtomicBool) -> Result<()> {
let manifest = manifest();
let dir = dir();
let client = reqwest::blocking::Client::builder()
.connect_timeout(std::time::Duration::from_secs(20))
.build()?;
fs::create_dir_all(&dir)?;
for file in &manifest.files {
if cancelled(cancel) {
bail!("Cancelled");
}
let staging = dir.join(format!("{}.part", file.name));
let result = download_file(&client, manifest, file, &staging, cancel);
if result.is_err() || cancelled(cancel) {
let _unused = fs::remove_file(&staging);
result?;
bail!("Cancelled");
}
}
fs::write(dir.join("verified"), &manifest.revision)?;
Ok(())
}
fn download_file(
client: &reqwest::blocking::Client,
manifest: &Manifest,
file: &Artifact,
staging: &Path,
cancel: &AtomicBool,
) -> Result<()> {
use std::io::{Read as _, Write as _};
use sha2::Digest as _;
let url = format!(
"https://huggingface.co/{}/resolve/{}/{}",
manifest.repository, manifest.revision, file.name
);
let mut response = client.get(url).send()?.error_for_status()?;
let mut out = fs::File::create(staging)?;
let mut hash = sha2::Sha256::new();
let mut size = 0u64;
loop {
if cancelled(cancel) {
bail!("Cancelled");
}
let mut chunk = [0u8; 64 * 1024];
let read = response.read(&mut chunk)?;
if read == 0 {
break;
}
size += read as u64;
if size > file.size {
bail!("Unexpected model size");
}
hash.update(&chunk[..read]);
out.write_all(&chunk[..read])?;
let mut state = state().lock().unwrap_or_else(|error| error.into_inner());
if let Download::Running { downloaded, .. } = &mut state.download {
*downloaded += read as u64;
}
}
if size != file.size || format!("{:x}", hash.finalize()) != file.sha256 {
bail!("Model checksum verification failed");
}
out.sync_all()?;
fs::rename(staging, dir().join(&file.name))?;
Ok(())
}
fn cancelled(cancel: &AtomicBool) -> bool {
cancel.load(Ordering::Acquire)
}
enum Download {
Idle,
Running {
downloaded: u64,
total: u64,
cancel: Arc<AtomicBool>,
},
}
struct State {
download: Download,
error: Option<String>,
}
fn state() -> &'static Mutex<State> {
static STATE: Mutex<State> = Mutex::new(State {
download: Download::Idle,
error: None,
});
&STATE
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn manifest_pins_v3_artifacts_with_exact_sizes() {
let manifest = manifest();
assert_eq!(manifest.repository, "istupakov/parakeet-tdt-0.6b-v3-onnx");
assert_eq!(manifest.revision.len(), 40);
assert_eq!(manifest.files.len(), 3);
assert_eq!(total_size(), 670_479_942);
assert!(
manifest
.files
.iter()
.all(|file| file.sha256.len() == 64 && !file.name.contains('/'))
);
}
}
-214
View File
@@ -1,214 +0,0 @@
use std::path::PathBuf;
use std::sync::OnceLock;
use std::time::Duration;
use anyhow::{Context as _, Result, anyhow};
use gpui::{App, Task};
use gpui_kit::component::speech::{
AudioFormat, RecognitionSession, SpeechError, SpeechRecognizer, SpeechSink,
};
use parakeet_rs::{ParakeetTDT, Transcriber as _};
const MODEL_RATE: u32 = 16_000;
const MAX_SECONDS: usize = 120;
/// The on-device recognizer.
pub struct ParakeetRecognizer;
impl SpeechRecognizer for ParakeetRecognizer {
fn audio_format(&self) -> AudioFormat {
AudioFormat::new(MODEL_RATE, 1)
}
fn is_available(&self, _cx: &App) -> bool {
true
}
fn start(
&self,
sink: SpeechSink,
cx: &mut App,
) -> Result<Box<dyn RecognitionSession>, SpeechError> {
if super::model::installed() {
worker().send_or_log(Command::Warm(super::model::dir()));
} else if super::model::status().is_downloading() {
return Err(SpeechError::recognizer(anyhow!(
"The speech model is still downloading. Try again when it finishes."
)));
} else {
return Err(SpeechError::recognizer(anyhow!(
"No speech model is installed. Download it from Settings."
)));
}
// The session buffers everything it is pushed,
// so audio is accepted right away.
sink.ready(cx);
Ok(Box::new(ParakeetSession {
sink,
samples: Vec::new(),
_result: None,
}))
}
}
struct ParakeetSession {
sink: SpeechSink,
samples: Vec<f32>,
_result: Option<Task<()>>,
}
impl RecognitionSession for ParakeetSession {
fn push_audio(&mut self, samples: &[i16], _cx: &mut App) {
if self.samples.len() < MAX_SECONDS * MODEL_RATE as usize {
self.samples
.extend(samples.iter().map(|sample| f32::from(*sample) / 32_768.0));
}
}
fn finish(&mut self, cx: &mut App) {
let samples = std::mem::take(&mut self.samples);
let sink = self.sink.clone();
// Speech too short to transcribe ends the session at once.
let silent = samples.iter().all(|sample| sample.abs() < 0.0001);
if samples.len() < MODEL_RATE as usize / 5 || silent {
sink.finish(cx);
return;
}
let (sender, receiver) = flume::unbounded();
worker().send_or_log(Command::Transcribe { samples, sender });
self._result = Some(cx.spawn(async move |cx| {
let result = receiver.recv_async().await;
cx.update(|cx| match result {
Ok(Ok(text)) if !text.is_empty() => {
sink.phrase(text, cx);
sink.finish(cx);
}
Ok(Ok(_)) => sink.finish(cx),
Ok(Err(error)) => sink.error(SpeechError::recognizer(error), cx),
Err(_) => sink.error(
SpeechError::recognizer(anyhow!("The speech worker is unavailable")),
cx,
),
});
}));
}
}
enum Command {
Warm(PathBuf),
Transcribe {
samples: Vec<f32>,
sender: flume::Sender<Result<String, anyhow::Error>>,
},
Unload,
}
/// A handle to the single inference worker.
fn worker() -> &'static flume::Sender<Command> {
static WORKER: OnceLock<flume::Sender<Command>> = OnceLock::new();
WORKER.get_or_init(|| {
let (sender, receiver) = flume::unbounded::<Command>();
let spawned = std::thread::Builder::new()
.name("speech-parakeet".into())
.spawn(move || run_worker(receiver));
if let Err(error) = spawned {
log::error!("failed to spawn the speech worker: {error}");
}
sender
})
}
fn run_worker(receiver: flume::Receiver<Command>) {
let mut cached: Option<(PathBuf, ParakeetTDT)> = None;
while let Ok(command) = receiver.recv_timeout(Duration::from_secs(30)) {
match command {
Command::Unload => {
cached = None;
}
Command::Warm(dir) => {
let loaded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
load(&mut cached, &dir).map(|_| ())
}));
match flatten_panic(loaded) {
Ok(Ok(())) => {}
Ok(Err(error)) | Err(error) => {
log::warn!("failed to load the speech model: {error:#}")
}
}
}
Command::Transcribe { samples, sender } => {
let transcribed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
transcribe(&mut cached, samples)
}));
let result = match flatten_panic(transcribed) {
Ok(result) => result,
Err(_) => {
cached = None;
Err(anyhow!("Dictation stopped unexpectedly. Try again."))
}
};
if sender.send(result).is_err() {
log::debug!("speech session was cancelled before its result arrived");
}
}
}
}
}
fn load<'a>(
cached: &'a mut Option<(PathBuf, ParakeetTDT)>,
dir: &PathBuf,
) -> Result<&'a mut ParakeetTDT> {
if cached.as_ref().is_none_or(|(path, _)| path != dir) {
let model = ParakeetTDT::from_pretrained(dir, None)
.map_err(|error| anyhow!("Could not load the speech model: {error}"))?;
*cached = Some((dir.clone(), model));
}
let (_, model) = cached.as_mut().context("speech model was unloaded")?;
Ok(model)
}
fn transcribe(cached: &mut Option<(PathBuf, ParakeetTDT)>, samples: Vec<f32>) -> Result<String> {
let model = load(cached, &super::model::dir())?;
model
.transcribe_samples(samples, MODEL_RATE, 1, None)
.map(|result| result.text)
.map_err(|error| anyhow!("Could not transcribe this recording: {error}"))
}
trait SendOrLog {
fn send_or_log(&self, command: Command);
}
impl SendOrLog for flume::Sender<Command> {
fn send_or_log(&self, command: Command) {
if self.send(command).is_err() {
log::debug!("speech worker is gone; command dropped");
}
}
}
/// Retire cached weights, e.g. after the model was deleted.
pub(crate) fn unload() {
worker().send_or_log(Command::Unload);
}
fn flatten_panic<T>(result: std::thread::Result<T>) -> Result<T> {
result.map_err(|_| anyhow!("The speech worker panicked"))
}
-256
View File
@@ -1,256 +0,0 @@
use std::cell::RefCell;
use anyhow::anyhow;
use gpui::App;
use gpui_kit::Subscription;
use gpui_kit::component::speech::{
AudioFormat, AudioInput, AudioSink, RecognitionSession, SpeechError, SpeechRecognizer,
SpeechSink,
};
use js_sys::{Array, Function, JsString, Reflect};
use wasm_bindgen::closure::Closure;
use wasm_bindgen::{JsCast as _, JsValue};
use web_sys::window;
/// The recognizer backed by `SpeechRecognition` (or its `webkit` prefix).
pub struct WebSpeechRecognizer;
impl SpeechRecognizer for WebSpeechRecognizer {
fn audio_format(&self) -> AudioFormat {
AudioFormat::default()
}
fn is_available(&self, _cx: &App) -> bool {
true
}
fn start(
&self,
sink: SpeechSink,
cx: &mut App,
) -> Result<Box<dyn RecognitionSession>, SpeechError> {
let recognition = Recognition::open(sink, cx).map_err(SpeechError::recognizer)?;
Ok(Box::new(recognition))
}
}
pub struct SelfCapturedInput;
impl AudioInput for SelfCapturedInput {
fn start(
&self,
_format: AudioFormat,
_sink: AudioSink,
_cx: &mut App,
) -> Result<Subscription, SpeechError> {
Ok(Subscription::new(|| {}))
}
}
enum Event {
Hypothesis(String),
Phrase(String),
Error(String),
End,
}
struct Recognition {
recognition: JsValue,
on_result: Option<Closure<dyn FnMut(JsValue)>>,
on_error: Option<Closure<dyn FnMut(JsValue)>>,
on_end: Option<Closure<dyn FnMut(JsValue)>>,
_task: gpui::Task<()>,
}
impl Recognition {
fn open(sink: SpeechSink, cx: &mut App) -> anyhow::Result<Self> {
let window = window().ok_or_else(|| anyhow!("No browser window"))?;
let constructor = Reflect::get(&window, &JsString::from("SpeechRecognition"))
.ok()
.filter(|value| value.is_function())
.or_else(|| {
Reflect::get(&window, &JsString::from("webkitSpeechRecognition"))
.ok()
.filter(|value| value.is_function())
})
.ok_or_else(|| anyhow!("This browser does not support speech recognition"))?;
let recognition =
Reflect::construct(constructor.unchecked_ref::<Function>(), &Array::new())
.map_err(|error| js_error("constructing the recognizer", error))?;
let (sender, receiver) = flume::unbounded::<Event>();
let error_sender = sender.clone();
let end_sender = sender.clone();
let finalized = RefCell::new(0usize);
let on_result = Closure::wrap(Box::new(move |event: JsValue| {
let read = |value: &JsValue, property: &str| -> Option<JsValue> {
Reflect::get(value, &JsString::from(property)).ok()
};
let Some(results) = read(&event, "results") else {
return;
};
let length = read(&results, "length")
.and_then(|length| length.as_f64())
.unwrap_or(0.0) as usize;
let mut finalized = finalized.borrow_mut();
let mut interim = String::new();
for index in *finalized..length {
let Some(result) = read(&results, &index.to_string()) else {
continue;
};
let Some(alternative) = read(&result, "0") else {
continue;
};
let Some(text) = read(&alternative, "transcript").and_then(|text| text.as_string())
else {
continue;
};
let is_final = read(&result, "isFinal")
.and_then(|final_value| final_value.as_bool())
.unwrap_or(false);
if is_final {
*finalized += 1;
if sender.send(Event::Phrase(text)).is_err() {
log::debug!("speech session was cancelled mid-utterance");
}
} else {
interim.push_str(&text);
}
}
if !interim.is_empty() && sender.send(Event::Hypothesis(interim)).is_err() {
log::debug!("speech session was cancelled mid-utterance");
}
}) as Box<dyn FnMut(JsValue)>);
let on_error = Closure::wrap(Box::new(move |event: JsValue| {
let message = Reflect::get(&event, &JsString::from("error"))
.ok()
.and_then(|error| error.as_string())
.unwrap_or_else(|| "unknown speech error".into());
if error_sender.send(Event::Error(message)).is_err() {
log::debug!("speech session was cancelled before its error arrived");
}
}) as Box<dyn FnMut(JsValue)>);
let on_end = Closure::wrap(Box::new(move |_event: JsValue| {
if end_sender.send(Event::End).is_err() {
log::debug!("speech session was cancelled before its end arrived");
}
}) as Box<dyn FnMut(JsValue)>);
set_callback(&recognition, "onresult", on_result.as_ref().unchecked_ref())?;
set_callback(&recognition, "onerror", on_error.as_ref().unchecked_ref())?;
set_callback(&recognition, "onend", on_end.as_ref().unchecked_ref())?;
set_bool(&recognition, "continuous", true)?;
set_bool(&recognition, "interimResults", true)?;
call_method(&recognition, "start")?;
// The browser consumes audio as soon as it has permission
sink.ready(cx);
let task = cx.spawn(async move |cx| {
while let Ok(event) = receiver.recv_async().await {
let done = matches!(event, Event::End);
// The speech state ignores sink calls once its session ended,
// so a late event after a cancel is harmless.
cx.update(|cx| match event {
Event::Hypothesis(text) => sink.hypothesis(text, cx),
Event::Phrase(text) => sink.phrase(text, cx),
Event::Error(message) => {
sink.error(SpeechError::recognizer(anyhow!("{message}")), cx)
}
Event::End => sink.finish(cx),
});
if done {
break;
}
}
});
Ok(Self {
recognition,
on_result: Some(on_result),
on_error: Some(on_error),
on_end: Some(on_end),
_task: task,
})
}
fn stop(&mut self) {
self.on_result = None;
self.on_error = None;
self.on_end = None;
let _ignored = call_method(&self.recognition, "abort");
}
}
impl Drop for Recognition {
fn drop(&mut self) {
self.stop();
}
}
impl RecognitionSession for Recognition {
fn push_audio(&mut self, _samples: &[i16], _cx: &mut App) {
// The browser recognizer captures the microphone itself.
// the state still pushes audio through its input, which is a no-op here.
}
fn finish(&mut self, _cx: &mut App) {
// The end handler reports completion, so no result is fabricated here.
let _ignored = call_method(&self.recognition, "stop");
}
}
fn set_callback(target: &JsValue, property: &str, callback: &Function) -> anyhow::Result<()> {
Reflect::set(target, &JsString::from(property), callback)
.map_err(|error| js_error(property, error))?;
Ok(())
}
fn set_bool(target: &JsValue, property: &str, value: bool) -> anyhow::Result<()> {
Reflect::set(target, &JsString::from(property), &JsValue::from(value))
.map_err(|error| js_error(property, error))?;
Ok(())
}
fn call_method(target: &JsValue, method: &str) -> anyhow::Result<()> {
let function = Reflect::get(target, &JsString::from(method))
.map_err(|error| js_error(method, error))?
.unchecked_into::<Function>();
function
.call0(target)
.map(|_| ())
.map_err(|error| js_error(method, error))
}
fn js_error(what: &str, error: JsValue) -> anyhow::Error {
let description = error
.as_string()
.or_else(|| {
Reflect::get(&error, &JsString::from("message"))
.ok()
.and_then(|message| message.as_string())
})
.unwrap_or_else(|| "unknown error".into());
anyhow!("{what} failed: {description}")
}
-1
View File
@@ -15,7 +15,6 @@ chat_ui = { path = "../chat_ui" }
nip29 = { path = "../nip29" }
nip29_ui = { path = "../nip29_ui" }
settings = { path = "../settings" }
speech = { path = "../speech" }
person = { path = "../person" }
auto_update = { path = "../auto_update" }
+3 -140
View File
@@ -1,30 +1,19 @@
#[cfg(not(target_arch = "wasm32"))]
use std::time::Duration;
use chat_ui::ToggleDictation;
#[cfg(not(target_arch = "wasm32"))]
use gpui::PathPromptOptions;
#[cfg(not(target_arch = "wasm32"))]
use gpui::Task;
use gpui::http_client::Url;
use gpui::{
AnyElement, App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled,
Subscription, Window, div,
App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, Subscription,
Window, div,
};
use gpui_kit::assets::IconName;
use gpui_kit::component::button::{Button, ButtonVariants};
use gpui_kit::component::input::{Input, InputEvent, InputState, NumberInput};
use gpui_kit::component::kbd::Kbd;
use gpui_kit::component::menu::{DropdownMenu, PopupMenuItem};
use gpui_kit::component::notification::Notification;
#[cfg(not(target_arch = "wasm32"))]
use gpui_kit::component::progress::Progress;
use gpui_kit::component::separator::Separator;
use gpui_kit::component::switch::Switch;
use gpui_kit::component::{Disableable, Sizable, WindowExt, h_flex, v_flex};
use settings::{AppSettings, AppearanceMode};
#[cfg(not(target_arch = "wasm32"))]
use speech::model::{self as speech_model};
use ui::{settings_block, settings_row};
/// Bounds for the theme number inputs
@@ -40,8 +29,6 @@ pub struct Preferences {
font_size_input: Entity<InputState>,
radius_input: Entity<InputState>,
radius_lg_input: Entity<InputState>,
#[cfg(not(target_arch = "wasm32"))]
_speech_poll: Task<()>,
/// Keeps the control subscriptions alive while the dialog is open
_subscriptions: Vec<Subscription>,
}
@@ -107,40 +94,11 @@ impl Preferences {
}),
];
#[cfg(not(target_arch = "wasm32"))]
let _speech_poll = cx.spawn_in(window, async move |this, cx| {
let timer = Duration::from_millis(500);
let mut last = speech_model::status();
loop {
cx.background_executor().timer(timer).await;
let status = speech_model::status();
let error = speech_model::take_error();
if status == last {
continue;
}
last = status;
this.update_in(cx, |_, window, cx| {
if let Some(error) = error {
window.push_notification(Notification::error(error).autohide(false), cx);
}
cx.notify();
})
.ok();
}
});
Self {
file_input,
font_size_input,
radius_input,
radius_lg_input,
#[cfg(not(target_arch = "wasm32"))]
_speech_poll,
_subscriptions: subscriptions,
}
}
@@ -230,102 +188,10 @@ impl Preferences {
.detach();
}
}
fn render_speech(
&mut self,
#[cfg_attr(target_arch = "wasm32", allow(unused_variables))] cx: &mut Context<Self>,
) -> AnyElement {
#[cfg(target_arch = "wasm32")]
{
return div().into_any_element();
}
#[cfg(not(target_arch = "wasm32"))]
{
let size = speech_model::total_size() / 1_000_000;
let status = speech_model::status();
let (description, control) = match &status {
speech_model::Status::NotInstalled => (
format!("{size} MB download"),
Button::new("download-model")
.label("Download")
.secondary()
.small()
.on_click(|_, _window, cx| {
speech_model::start_download();
cx.refresh_windows();
})
.into_any_element(),
),
speech_model::Status::Downloading { downloaded, total } => (
format!(
"Downloading · {} of {} MB",
downloaded / 1_000_000,
total / 1_000_000
),
Button::new("cancel-download")
.label("Cancel")
.ghost()
.small()
.on_click(|_, _window, cx| {
speech_model::cancel_download();
cx.refresh_windows();
})
.into_any_element(),
),
speech_model::Status::Installed => (
format!("Parakeet v3 · {size} MB"),
Button::new("remove-model")
.label("Remove")
.secondary()
.small()
.on_click(|_, _window, cx| {
speech_model::delete();
cx.refresh_windows();
})
.into_any_element(),
),
};
let mut block = v_flex().w_full().gap_3().child(settings_row(
"Speech model",
description,
control,
cx,
));
if let speech_model::Status::Downloading { downloaded, total } = &status {
let (downloaded, total) = (*downloaded, *total);
block = block.child(
Progress::new("speech-model-download")
.value(downloaded as f32 / total.max(1) as f32 * 100.)
.small(),
);
}
block.into_any_element()
}
}
/// Shows the keystroke actually bound to the dictation action at startup
fn render_dictation_shortcut(
&self,
window: &mut Window,
cx: &mut Context<Self>,
) -> impl IntoElement {
let control = match Kbd::global_binding_for_action(&ToggleDictation, window) {
Some(kbd) => kbd.into_any_element(),
None => div().into_any_element(),
};
settings_row("Shortcut", "Press to start or stop dictation", control, cx)
}
}
impl Render for Preferences {
fn render(&mut self, window: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
fn render(&mut self, _window: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
const SCREENING: &str = "Show an screening dialog to verify the unknown sender.";
const WOT: &str = "Treat people followed by your contacts as known senders.";
const AVATAR: &str = "Hide all avatar pictures to improve performance.";
@@ -508,9 +374,6 @@ impl Render for Preferences {
cx,
))
.child(Separator::horizontal())
.child(self.render_speech(cx))
.child(self.render_dictation_shortcut(window, cx))
.child(Separator::horizontal())
.child(settings_block(
"Media Upload Service",
MEDIA,
+1 -2
View File
@@ -14,7 +14,7 @@ product-name = "Coop"
description = "Chat Freely, Stay Private on Nostr"
identifier = "su.reya.coop"
category = "SocialNetworking"
version = "2.0.0-alpha3"
version = "2.0.0-alpha2"
out-dir = "../dist"
before-packaging-command = "cargo build --release"
resources = ["Cargo.toml", "src"]
@@ -38,7 +38,6 @@ ui = { path = "../crates/ui" }
state = { path = "../crates/state" }
nip4e = { path = "../crates/nip4e" }
chat = { path = "../crates/chat" }
chat_ui = { path = "../crates/chat_ui" }
nip29 = { path = "../crates/nip29" }
settings = { path = "../crates/settings" }
auto_update = { path = "../crates/auto_update" }
+4 -11
View File
@@ -1,6 +1,5 @@
use std::sync::{Arc, Mutex};
use chat_ui::ToggleDictation;
use gpui::{
App, Bounds, Global, KeyBinding, Menu, MenuItem, SharedString, TitlebarOptions,
WindowBackgroundAppearance, WindowBounds, WindowDecorations, WindowKind, WindowOptions,
@@ -46,19 +45,13 @@ fn main() {
// Register the `quit` function
cx.on_action(quit);
// Register the `quit` function with CMD+Q (macOS) and dictation with CMD+D
// Register the `quit` function with CMD+Q (macOS)
#[cfg(target_os = "macos")]
cx.bind_keys([
KeyBinding::new("cmd-q", Quit, None),
KeyBinding::new("cmd-d", ToggleDictation, None),
]);
cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]);
// Register the `quit` function with Super+Q (others) and dictation with Ctrl+D
// Register the `quit` function with Super+Q (others)
#[cfg(not(target_os = "macos"))]
cx.bind_keys([
KeyBinding::new("super-q", Quit, None),
KeyBinding::new("ctrl-d", ToggleDictation, None),
]);
cx.bind_keys([KeyBinding::new("super-q", Quit, None)]);
// Set menu items
cx.set_menus(vec![Menu {
+1
View File
@@ -3,6 +3,7 @@ channel = "1.98"
profile = "minimal"
components = ["rustfmt", "clippy"]
targets = [
"x86_64-apple-darwin",
"aarch64-apple-darwin",
"x86_64-unknown-linux-gnu",
"x86_64-pc-windows-msvc",
-1
View File
@@ -14,7 +14,6 @@ ui = { path = "../crates/ui" }
state = { path = "../crates/state" }
nip4e = { path = "../crates/nip4e" }
chat = { path = "../crates/chat" }
chat_ui = { path = "../crates/chat_ui" }
nip29 = { path = "../crates/nip29" }
settings = { path = "../crates/settings" }
person = { path = "../crates/person" }
-5
View File
@@ -78,11 +78,6 @@ pub fn run() -> Result<(), JsValue> {
APPLICATION.with(|application| {
*application.borrow_mut() = Some(app.run_embedded(|cx| {
cx.bind_keys([
KeyBinding::new("cmd-d", chat_ui::ToggleDictation, None),
KeyBinding::new("ctrl-d", chat_ui::ToggleDictation, None),
]);
gpui_kit::init(cx);
load_embedded_fonts(cx);