From 77b2a5f3cded7ca6bb35d7dab7d994ce310a2da0 Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Tue, 27 Oct 2026 06:25:19 +0000 Subject: [PATCH] feat: add support for speech to text (#68) Reviewed-on: https://git.reya.info/reya/coop/pulls/68 --- Cargo.lock | 934 ++++++++++++++++++++++- assets/icons/paperclip.svg | 3 + crates/assets/tests/icons.rs | 39 - crates/chat_ui/Cargo.toml | 4 +- crates/chat_ui/src/actions.rs | 5 +- crates/chat_ui/src/lib.rs | 166 +++- crates/nip29_ui/Cargo.toml | 4 +- crates/nip29_ui/src/lib.rs | 102 ++- crates/speech/Cargo.toml | 30 + crates/speech/src/lib.rs | 53 ++ crates/speech/src/model.json | 23 + crates/speech/src/model.rs | 295 +++++++ crates/speech/src/parakeet.rs | 214 ++++++ crates/speech/src/web.rs | 256 +++++++ crates/ui/src/icon.rs | 2 + crates/ui/src/message.rs | 21 +- crates/workspace/Cargo.toml | 1 + crates/workspace/src/dialogs/new_chat.rs | 6 +- crates/workspace/src/dialogs/profile.rs | 67 +- crates/workspace/src/dialogs/relays.rs | 2 +- crates/workspace/src/dialogs/settings.rs | 143 +++- crates/workspace/src/panels/backup.rs | 59 +- crates/workspace/src/sidebar/mod.rs | 2 +- desktop/Cargo.toml | 6 + desktop/resources/Info.plist | 10 + desktop/src/main.rs | 15 +- web/Cargo.toml | 1 + web/src/lib.rs | 5 + 28 files changed, 2238 insertions(+), 230 deletions(-) create mode 100644 assets/icons/paperclip.svg delete mode 100644 crates/assets/tests/icons.rs create mode 100644 crates/speech/Cargo.toml create mode 100644 crates/speech/src/lib.rs create mode 100644 crates/speech/src/model.json create mode 100644 crates/speech/src/model.rs create mode 100644 crates/speech/src/parakeet.rs create mode 100644 crates/speech/src/web.rs create mode 100644 desktop/resources/Info.plist diff --git a/Cargo.lock b/Cargo.lock index 0c20ab12..001b2d59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,7 +153,9 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", + "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -191,6 +193,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -584,7 +608,7 @@ dependencies = [ "log", "num-rational", "num-traits", - "pastey", + "pastey 0.1.1", "rayon", "thiserror 2.0.21", "v_frame", @@ -600,7 +624,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 8.0.0", "num-rational", "v_frame", ] @@ -658,6 +682,12 @@ version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3a647dce9631f152a99ec16cfc09267ae55fe280d5dc288964027d458076323" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -670,12 +700,36 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bech32" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efbd3e1070bbdf4cd88a75264e18e8a26f7cb5c6949eadf0ceb85fb159cf08f8" +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.3", + "shlex 1.3.0", + "syn 2.0.119", +] + [[package]] name = "bip39" version = "2.2.2" @@ -965,6 +1019,15 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -1001,7 +1064,22 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", ] [[package]] @@ -1083,6 +1161,7 @@ dependencies = [ "common", "gpui-kit", "gpui-pre", + "instant", "itertools 0.13.0", "nostr-sdk", "person", @@ -1090,6 +1169,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] @@ -1130,6 +1210,17 @@ dependencies = [ "inout 0.2.5", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "cmake" version = "0.1.58" @@ -1156,7 +1247,7 @@ dependencies = [ "cocoa-foundation 0.1.2", "core-foundation 0.9.4", "core-graphics 0.23.2", - "foreign-types", + "foreign-types 0.5.0", "libc", "objc", ] @@ -1172,7 +1263,7 @@ dependencies = [ "cocoa-foundation 0.2.1", "core-foundation 0.10.1", "core-graphics 0.24.0", - "foreign-types", + "foreign-types 0.5.0", "libc", "objc", ] @@ -1248,6 +1339,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1354,6 +1460,7 @@ dependencies = [ "assets", "auto_update", "chat", + "chat_ui", "flume 0.11.1", "gpui-kit", "gpui-pre", @@ -1382,6 +1489,7 @@ version = "2.0.0-alpha2" dependencies = [ "assets", "chat", + "chat_ui", "console_error_panic_hook", "console_log", "getrandom 0.2.17", @@ -1442,7 +1550,7 @@ dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", "core-graphics-types 0.1.3", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1455,7 +1563,7 @@ dependencies = [ "bitflags 2.13.2", "core-foundation 0.10.1", "core-graphics-types 0.2.0", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1468,7 +1576,7 @@ dependencies = [ "bitflags 2.13.2", "core-foundation 0.9.4", "core-graphics-types 0.1.3", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1502,7 +1610,7 @@ checksum = "a593227b66cbd4007b2a050dfdd9e1d1318311409c8d600dc82ba1b15ca9c130" dependencies = [ "core-foundation 0.10.1", "core-graphics 0.24.0", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1521,6 +1629,26 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953" +dependencies = [ + "bindgen", +] + [[package]] name = "cosmic-text" version = "0.19.0" @@ -1545,6 +1673,29 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2 0.4.3", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1660,14 +1811,44 @@ dependencies = [ "cmov", ] +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -1683,17 +1864,43 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core", + "darling_core 0.24.1", "quote", "syn 3.0.9", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.11.1" @@ -1760,6 +1967,16 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "der" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.9" @@ -1769,6 +1986,37 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -2085,6 +2333,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etagere" version = "0.2.15" @@ -2141,6 +2395,17 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "eyre" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" +dependencies = [ + "autocfg", + "indenter", + "once_cell", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -2348,6 +2613,15 @@ dependencies = [ "ttf-parser", ] +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -2355,7 +2629,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -2369,6 +2643,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -2774,8 +3054,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a363235cd4062ef5f0953fc096fc764ed6f53dfbd9f409f669f93ea3b1ff9" dependencies = [ "anyhow", + "block2 0.6.2", "chrono", "core-text", + "cpal", "enum-iterator", "gpui-base", "gpui-component-macros", @@ -2792,7 +3074,10 @@ dependencies = [ "num-traits", "objc2 0.6.5", "objc2-app-kit 0.3.2", + "objc2-av-foundation", + "objc2-avf-audio", "objc2-foundation 0.3.2", + "objc2-speech", "once_cell", "paste", "raw-window-handle", @@ -2938,7 +3223,7 @@ dependencies = [ "derive_more", "dispatch2", "etagere", - "foreign-types", + "foreign-types 0.5.0", "futures", "gpui-pre", "gpui-pre-collections", @@ -2946,7 +3231,7 @@ dependencies = [ "image", "libc", "log", - "mach2", + "mach2 0.6.0", "metal", "objc2 0.6.5", "objc2-core-foundation", @@ -3077,7 +3362,7 @@ dependencies = [ "core-graphics 0.24.0", "ctor", "dispatch2", - "foreign-types", + "foreign-types 0.5.0", "futures", "gpui-pre", "gpui-pre-apple", @@ -3630,6 +3915,18 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "hmac-sha256" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad320b3b96fb2a455a0726d16efe0a5afdbd34b71dea5bc53b05ea057714d4e" + +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "html5ever" version = "0.27.0" @@ -3966,6 +4263,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + [[package]] name = "indexed_db_futures" version = "0.6.4" @@ -4206,6 +4509,22 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -4560,6 +4879,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + [[package]] name = "mac" version = "0.1.1" @@ -4580,12 +4905,37 @@ dependencies = [ "uuid", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "mach2" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey 0.2.3", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "macroific" version = "2.0.0" @@ -4727,6 +5077,16 @@ dependencies = [ "xml5ever", ] +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -4780,7 +5140,7 @@ dependencies = [ "bitflags 2.13.2", "block", "core-graphics-types 0.2.0", - "foreign-types", + "foreign-types 0.5.0", "log", "objc", "paste", @@ -4802,6 +5162,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minisign-verify" version = "0.2.5" @@ -4840,6 +5206,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -4935,12 +5323,67 @@ dependencies = [ "native-theme", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.13.2", + "jni-sys 0.3.1", + "log", + "ndk-sys 0.5.0+25.2.9519653", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -4990,6 +5433,7 @@ dependencies = [ "common", "gpui-kit", "gpui-pre", + "instant", "log", "nip29", "nostr-sdk", @@ -4997,6 +5441,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] @@ -5034,6 +5479,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -5392,6 +5847,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "objc" version = "0.2.7" @@ -5475,6 +5952,27 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2 0.6.5", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -5696,6 +6194,18 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-speech" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9eb609f9d2a25e0f8b76954f3acaa58348ce2a91285fe867643b2224ac4d263" +dependencies = [ + "block2 0.6.2", + "objc2 0.6.5", + "objc2-avf-audio", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-user-notifications" version = "0.3.2" @@ -5735,12 +6245,57 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni 0.21.1", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags 2.13.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "oo7" version = "0.6.0" @@ -5804,12 +6359,49 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -5835,6 +6427,30 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "page_size" version = "0.6.0" @@ -5845,6 +6461,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "parakeet-rs" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f54545a861dca6b43c8d44e3dd932764ca196d84bcf0d1b48ea212b43b29666" +dependencies = [ + "eyre", + "hound", + "ndarray", + "ort", + "realfft", + "serde", + "serde_json", + "tokenizers", +] + [[package]] name = "parking" version = "2.2.1" @@ -5886,6 +6518,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pathfinder_geometry" version = "0.5.1" @@ -5925,6 +6563,15 @@ dependencies = [ "hmac 0.13.0", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -6226,6 +6873,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -6567,6 +7223,12 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -6577,6 +7239,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -6609,6 +7282,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "realfft" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "538895ccf2006b473aafbe8d72770e21038961ab67aae34f3c577485fc703a7d" +dependencies = [ + "rustfft", +] + [[package]] name = "reborrow" version = "0.5.5" @@ -6708,7 +7390,9 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "futures-channel", "futures-core", + "futures-util", "h2", "http", "http-body", @@ -6956,6 +7640,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.5" @@ -7024,7 +7722,7 @@ checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.22.4", "log", "once_cell", "rustls", @@ -7436,7 +8134,7 @@ version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ - "darling", + "darling 0.24.1", "proc-macro2", "quote", "syn 3.0.9", @@ -7516,6 +8214,12 @@ dependencies = [ "dirs 6.0.0", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -7665,6 +8369,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "speech" +version = "2.0.0-alpha2" +dependencies = [ + "anyhow", + "common", + "flume 0.11.1", + "gpui-kit", + "gpui-pre", + "instant", + "js-sys", + "log", + "ort", + "parakeet-rs", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "spin" version = "0.9.9" @@ -7692,6 +8418,18 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "sqlite-wasm-rs" version = "0.5.5" @@ -7776,6 +8514,12 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" @@ -8187,6 +8931,39 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tokenizers" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.21", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.54.1" @@ -8492,6 +9269,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "triomphe" version = "0.1.17" @@ -8621,6 +9408,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -8657,6 +9453,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "universal-hash" version = "0.5.1" @@ -8692,7 +9494,9 @@ checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" dependencies = [ "base64 0.23.1", "cookie_store", + "der", "log", + "native-tls", "percent-encoding", "rustls", "rustls-pki-types", @@ -8701,6 +9505,7 @@ dependencies = [ "serde_json", "ureq-proto", "utf8-zero", + "webpki-root-certs", "webpki-roots 1.0.11", ] @@ -9109,7 +9914,7 @@ version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2c8ddfbbf66e501434657400268cefe51b95140a0266236415cda7627732df7" dependencies = [ - "jni", + "jni 0.22.4", "log", "ndk-context", "objc2 0.6.5", @@ -9278,7 +10083,7 @@ dependencies = [ "libloading", "log", "naga", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "objc2 0.6.5", "objc2-core-foundation", "objc2-foundation 0.3.2", @@ -9370,6 +10175,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.57.0" @@ -9446,6 +10261,16 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -9712,6 +10537,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -9739,6 +10573,21 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -9814,6 +10663,12 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -9832,6 +10687,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -9850,6 +10711,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -9880,6 +10747,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -9898,6 +10771,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -9916,6 +10795,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -9934,6 +10819,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -10019,6 +10910,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] diff --git a/assets/icons/paperclip.svg b/assets/icons/paperclip.svg new file mode 100644 index 00000000..b1e60469 --- /dev/null +++ b/assets/icons/paperclip.svg @@ -0,0 +1,3 @@ + + + diff --git a/crates/assets/tests/icons.rs b/crates/assets/tests/icons.rs deleted file mode 100644 index f9be2375..00000000 --- a/crates/assets/tests/icons.rs +++ /dev/null @@ -1,39 +0,0 @@ -use gpui::AssetSource; - -#[test] -fn icons_resolve_from_the_vendored_folder() { - // Icons coop's code and the kit components draw, served by the vendored - // folder (signed's set, filled with the kit bundle where coop needs more). - let vendored = [ - "icons/search.svg", - "icons/close.svg", - "icons/panel-left.svg", - "icons/window-minimize.svg", - "icons/settings-2.svg", - "icons/x.svg", - "icons/compass.svg", - "icons/radio-tower.svg", - ]; - for path in vendored { - let own = assets::Assets - .load(path) - .unwrap_or_else(|e| panic!("{path} errored in own embed: {e}")); - assert!(own.is_some(), "own embed is missing {path}"); - - let combined = assets::CombinedAssets.load(path).unwrap(); - assert!(combined.is_some(), "CombinedAssets fails to serve {path}"); - } - - // Coop's own files still load through the combined source. - let own = [ - "brand/coop.svg", - "brand/bg1.jpg", - "fonts/Inter/Inter-Medium.ttf", - ]; - for path in own { - assert!( - matches!(assets::CombinedAssets.load(path), Ok(Some(_))), - "CombinedAssets fails to serve coop asset {path}" - ); - } -} diff --git a/crates/chat_ui/Cargo.toml b/crates/chat_ui/Cargo.toml index 810ba59e..d14400cf 100644 --- a/crates/chat_ui/Cargo.toml +++ b/crates/chat_ui/Cargo.toml @@ -6,14 +6,16 @@ publish.workspace = true [dependencies] state = { path = "../state" } +speech = { path = "../speech" } ui = { path = "../ui" } -gpui-kit.workspace = true +gpui-kit = { workspace = true, features = ["speech"] } common = { path = "../common" } person = { path = "../person" } chat = { path = "../chat" } settings = { path = "../settings" } gpui.workspace = true +instant.workspace = true nostr-sdk.workspace = true anyhow.workspace = true itertools.workspace = true diff --git a/crates/chat_ui/src/actions.rs b/crates/chat_ui/src/actions.rs index b1c47642..2ef1f3f0 100644 --- a/crates/chat_ui/src/actions.rs +++ b/crates/chat_ui/src/actions.rs @@ -1,4 +1,4 @@ -use gpui::Action; +use gpui::{Action, actions}; use nostr_sdk::prelude::*; use serde::Deserialize; use settings::SignerKind; @@ -15,3 +15,6 @@ pub enum Command { Njump(PublicKey), Trace(EventId), } + +// Start or stop dictation in the focused composer +actions!(chat, [ToggleDictation]); diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index 28e0f739..f50dcb44 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, LazyLock, RwLock}; +use std::time::Duration; pub use actions::*; use anyhow::Error; @@ -20,6 +21,7 @@ use gpui_kit::component::input::{Input, InputEvent, InputState, Textarea, Textar use gpui_kit::component::menu::DropdownMenu; use gpui_kit::component::notification::Notification; use gpui_kit::component::scroll::Scrollbar; +use gpui_kit::component::speech::{SpeechButton, SpeechEvent, SpeechState, SpeechWaveform}; use gpui_kit::component::tooltip::Tooltip; use gpui_kit::component::{ActiveTheme, Disableable, Icon, Sizable, WindowExt, h_flex, v_flex}; use itertools::Itertools; @@ -91,6 +93,12 @@ pub struct ChatPanel { /// Chat input state input: Entity, + /// Dictation session for the chat input + speech: Entity, + + /// When the running dictation started, for the elapsed counter + speech_started: Option, + /// Subject input state subject_input: Entity, @@ -116,7 +124,7 @@ pub struct ChatPanel { tasks: Vec>>, /// Event subscriptions - subscriptions: SmallVec<[Subscription; 3]>, + subscriptions: SmallVec<[Subscription; 4]>, } impl ChatPanel { @@ -160,6 +168,19 @@ impl ChatPanel { .clean_on_escape() }); + // Define dictation state + let speech = cx.new(|cx| { + let state = SpeechState::new(cx) + .recognizer(speech::recognizer()) + // Batch transcription of a long recording can take a while. + .stop_timeout(Duration::from_secs(180)); + + #[cfg(target_arch = "wasm32")] + let state = state.input(speech::input()); + + state + }); + // Define subject input state let subject_input = cx.new(|cx| InputState::new(window, cx).placeholder("New subject...")); let subject_bar = cx.new(|_cx| false); @@ -178,15 +199,39 @@ impl ChatPanel { subscriptions.push( // Subscribe the subject input event - cx.subscribe_in( - &subject_input, - window, - move |this, _input, event, window, cx| { - if let InputEvent::PressEnter { .. } = event { - this.change_subject(window, cx); - }; - }, - ), + cx.subscribe_in(&subject_input, window, move |this, _, event, window, cx| { + if let InputEvent::PressEnter { .. } = event { + this.change_subject(window, cx); + }; + }), + ); + + subscriptions.push( + // Insert the dictated text into the chat input + cx.subscribe_in(&speech, window, move |this, _speech, event, window, cx| { + match event { + SpeechEvent::Started => { + this.speech_started = Some(instant::Instant::now()); + cx.notify(); + } + SpeechEvent::Final(text) => { + this.speech_started = None; + + if !text.is_empty() { + this.input + .update(cx, |input, cx| input.insert(text.clone(), window, cx)); + } + } + SpeechEvent::Error(error) => { + this.speech_started = None; + window.push_notification(Notification::error(error.to_string()), cx); + } + SpeechEvent::Cancelled => { + this.speech_started = None; + } + _ => {} + }; + }), ); // Define all functions that will run after the current cycle @@ -206,6 +251,8 @@ impl ChatPanel { directory, list_state, input, + speech, + speech_started: None, subject_input, subject_bar, replies_to, @@ -835,6 +882,17 @@ impl ChatPanel { self.directory.read(cx).person(public_key, cx) } + fn toggle_dictation( + &mut self, + _: &ToggleDictation, + _window: &mut Window, + cx: &mut Context, + ) { + self.speech.update(cx, |speech, cx| { + speech.toggle(cx); + }); + } + fn on_command(&mut self, command: &Command, window: &mut Window, cx: &mut Context) { match command { Command::Insert(content) => { @@ -1390,21 +1448,26 @@ impl ChatPanel { } let reply_id = *id; - let copy_id = *id; let dropdown_id = *id; let public_key = *public_key; - actions - .reply(cx.listener(move |this, _event, _window, cx| { - this.reply_to(&reply_id, cx); - })) - .copy(cx.listener(move |this, _event, _window, cx| { - this.copy_message(©_id, cx); - })) - .dropdown(move |menu, _window, _cx| { - menu.menu("Copy author", Box::new(Command::Copy(public_key))) - .menu("Seen on", Box::new(Command::Trace(dropdown_id))) - }) + let actions = actions.reply(cx.listener(move |this, _event, _window, cx| { + this.reply_to(&reply_id, cx); + })); + + let actions = if let Some(message) = self.message(id) { + actions.copy( + SharedString::from(format!("message-copy-{id}")), + message.preview(), + ) + } else { + actions + }; + + actions.dropdown(move |menu, _window, _cx| { + menu.menu("Copy author", Box::new(Command::Copy(public_key))) + .menu("Seen on", Box::new(Command::Trace(dropdown_id))) + }) } fn render_attachment(&self, url: &Url, cx: &Context) -> impl IntoElement { @@ -1669,7 +1732,12 @@ impl ChatPanel { items } - fn render_config_menu(&self, _window: &mut Window, cx: &Context) -> impl IntoElement { + fn render_config_menu( + &self, + disabled: bool, + _window: &mut Window, + cx: &Context, + ) -> impl IntoElement { let (backup, signer_kind) = self .room .read_with(cx, |this, _cx| { @@ -1682,6 +1750,7 @@ impl ChatPanel { .icon(IconName::Settings2) .tooltip("Configuration") .ghost() + .disabled(disabled) .dropdown_menu(move |this, _window, _cx| { let auto = matches!(signer_kind, SignerKind::Auto); let encryption = matches!(signer_kind, SignerKind::Encryption); @@ -1712,7 +1781,12 @@ impl ChatPanel { }) } - fn render_emoji_menu(&self, _window: &Window, cx: &Context) -> impl IntoElement { + fn render_emoji_menu( + &self, + disabled: bool, + _window: &Window, + cx: &Context, + ) -> impl IntoElement { let nostr = NostrRegistry::global(cx); let current_user = nostr.read(cx).current_user(); @@ -1729,6 +1803,7 @@ impl ChatPanel { Button::new("emoji") .icon(IconName::Emoji) .ghost() + .disabled(disabled) .dropdown_menu_with_anchor(gpui::Anchor::BottomLeft, move |this, _window, _cx| { let menu = REACTION_EMOJIS.iter().fold(this, |this, emoji| { this.menu(*emoji, Box::new(Command::Insert(emoji.to_string()))) @@ -1802,10 +1877,12 @@ impl Render for ChatPanel { let is_typing = !self.input.read(cx).value().trim().is_empty(); let pending_attachments = !self.encrypted_attachments.read(cx).is_empty(); + let dictating = self.speech.read(cx).status().is_active(); v_flex() .image_cache(retain_all(self.id.clone())) .on_action(cx.listener(Self::on_command)) + .on_action(cx.listener(Self::toggle_dictation)) .size_full() .when(*self.subject_bar.read(cx), |this| { this.child( @@ -1879,26 +1956,55 @@ impl Render for ChatPanel { .items_end() .child( Button::new("upload") - .icon(IconName::Plus) + .icon(IconName::Paperclip) .tooltip("Upload media") .loading(self.uploading) - .disabled(self.uploading) + .disabled(dictating || self.uploading) .ghost() .on_click(cx.listener(move |this, _ev, window, cx| { this.upload(window, cx); })), ) - .child(Textarea::new(&self.input).appearance(false).flex_1()) + .when(dictating, |this| { + this.child( + h_flex() + .h_8() + .flex_1() + .gap_2() + .pl_1() + .child(SpeechWaveform::new(&self.speech).small().flex_1()) + .when_some(self.speech_started, |this, started| { + this.child(speech::elapsed(started, cx)) + }), + ) + }) + .when(!dictating, |this| { + this.child( + Textarea::new(&self.input) + .appearance(false) + .small() + .min_h_8() + .py_1() + .flex_1(), + ) + }) + .child( + h_flex() + .items_end() + .gap_1() + .pl_1() + .child(SpeechButton::new(&self.speech)), + ) .child( h_flex() .pl_1() .gap_1() - .child(self.render_emoji_menu(window, cx)) - .child(self.render_config_menu(window, cx)) + .child(self.render_emoji_menu(dictating, window, cx)) + .child(self.render_config_menu(dictating, window, cx)) .child( Button::new("send") .icon(IconName::PaperPlaneFill) - .disabled(self.uploading) + .disabled(dictating || self.uploading) .ghost() .on_click(cx.listener(move |this, _ev, window, cx| { this.send_text_message(window, cx); diff --git a/crates/nip29_ui/Cargo.toml b/crates/nip29_ui/Cargo.toml index 7827e6ce..c6fa175d 100644 --- a/crates/nip29_ui/Cargo.toml +++ b/crates/nip29_ui/Cargo.toml @@ -11,11 +11,13 @@ common = { path = "../common" } nip29 = { path = "../nip29" } person = { path = "../person" } settings = { path = "../settings" } +speech = { path = "../speech" } state = { path = "../state" } -gpui-kit.workspace = true +gpui-kit = { workspace = true, features = ["speech"] } ui = { path = "../ui" } gpui.workspace = true +instant.workspace = true nostr-sdk.workspace = true anyhow.workspace = true log.workspace = true diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index 2d2b187a..21a61b96 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -1,11 +1,14 @@ use std::collections::BTreeMap; +use std::time::Duration; use anyhow::Result; +use chat_ui::ToggleDictation; use gpui::prelude::FluentBuilder; use gpui::{ Action, AnyElement, App, AppContext, Context, Entity, EventEmitter, FocusHandle, Focusable, - FollowMode, IntoElement, ListAlignment, ListScrollEvent, ListState, ParentElement, Render, - SharedString, Styled, Subscription, Task, WeakEntity, Window, div, list, px, + FollowMode, InteractiveElement, IntoElement, ListAlignment, ListScrollEvent, ListState, + ParentElement, Render, SharedString, Styled, Subscription, Task, WeakEntity, Window, div, list, + px, }; use gpui_kit::assets::IconName; use gpui_kit::base::StyledExt as _; @@ -15,6 +18,7 @@ use gpui_kit::component::dock::{Panel, PanelEvent}; use gpui_kit::component::input::{InputEvent, Textarea, TextareaState}; use gpui_kit::component::notification::Notification; use gpui_kit::component::scroll::Scrollbar; +use gpui_kit::component::speech::{SpeechButton, SpeechEvent, SpeechState, SpeechWaveform}; use gpui_kit::component::{ActiveTheme, Disableable, Icon, Sizable, WindowExt, h_flex, v_flex}; use nip29::{Group, GroupEvent, GroupKey, GroupsRegistry, Membership, Pin, Row}; use nostr_sdk::prelude::*; @@ -312,6 +316,10 @@ pub struct GroupPanel { reply_to: Option, rendered_texts_by_id: BTreeMap, input: Entity, + /// Dictation session for the composer input + speech: Entity, + /// When the running dictation started, for the elapsed counter + speech_started: Option, /// Uploaded, non-encrypted attachments waiting to be sent. attachments: Vec, /// Whether an upload is in flight. @@ -319,7 +327,7 @@ pub struct GroupPanel { /// Whether the user closed the encryption notice this visit notice_dismissed: bool, tasks: SmallVec<[Task>; 4]>, - _subscriptions: SmallVec<[Subscription; 3]>, + _subscriptions: SmallVec<[Subscription; 4]>, } impl GroupPanel { @@ -338,6 +346,18 @@ impl GroupPanel { .clean_on_escape() }); + let speech = cx.new(|cx| { + let state = SpeechState::new(cx) + .recognizer(speech::recognizer()) + // Batch transcription of a long recording can take a while. + .stop_timeout(Duration::from_secs(180)); + + #[cfg(target_arch = "wasm32")] + let state = state.input(speech::input()); + + state + }); + let mut subscriptions = smallvec![]; subscriptions.push( @@ -348,6 +368,34 @@ impl GroupPanel { }), ); + subscriptions.push( + cx.subscribe_in(&speech, window, |this, _, event, window, cx| { + match event { + SpeechEvent::Started => { + this.speech_started = Some(instant::Instant::now()); + cx.notify(); + } + SpeechEvent::Final(text) => { + this.speech_started = None; + + if !text.is_empty() { + this.input.update(cx, |input, cx| { + input.insert(text.clone(), window, cx); + }); + } + } + SpeechEvent::Error(error) => { + this.speech_started = None; + window.push_notification(Notification::error(error.to_string()), cx); + } + SpeechEvent::Cancelled => { + this.speech_started = None; + } + _ => {} + }; + }), + ); + subscriptions.push(cx.subscribe(&group, |this, _group, event, cx| { if !matches!(event, GroupEvent::Sent(_)) { this.sync(cx); @@ -383,6 +431,8 @@ impl GroupPanel { notice_dismissed: false, rendered_texts_by_id: BTreeMap::new(), input, + speech, + speech_started: None, attachments: Vec::new(), uploading: false, tasks: smallvec![], @@ -1015,6 +1065,17 @@ impl GroupPanel { ) } + fn toggle_dictation( + &mut self, + _ev: &ToggleDictation, + _window: &mut Window, + cx: &mut Context, + ) { + self.speech.update(cx, |speech, cx| { + speech.toggle(cx); + }); + } + fn render_composer(&self, cx: &mut Context) -> AnyElement { let notice = self.group.upgrade().and_then(|group| { let group = group.read(cx); @@ -1040,6 +1101,8 @@ impl GroupPanel { return self.render_notice(¬ice, cx); } + let dictating = self.speech.read(cx).status().is_active(); + v_flex() .flex_shrink_0() .w_full() @@ -1055,21 +1118,45 @@ impl GroupPanel { .items_end() .child( Button::new("upload") - .icon(IconName::Upload) - .tooltip("Upload file") + .icon(ui::IconName::Paperclip) + .tooltip("Upload media") .loading(self.uploading) - .disabled(self.uploading) + .disabled(dictating || self.uploading) .ghost() .on_click(cx.listener(|this, _event, window, cx| { this.upload(window, cx); })), ) - .child(Textarea::new(&self.input).appearance(false).flex_1()) + .when(dictating, |this| { + this.child( + h_flex() + .h_8() + .flex_1() + .gap_2() + .child(SpeechWaveform::new(&self.speech).small().flex_1()) + .when_some(self.speech_started, |this, started| { + this.child(speech::elapsed(started, cx)) + }), + ) + }) + .when(!dictating, |this| { + this.child( + Textarea::new(&self.input) + .appearance(false) + .bg(cx.theme().input) + .small() + .min_h_8() + .py_1() + .flex_1(), + ) + }) + .child(SpeechButton::new(&self.speech)) .child( Button::new("send") .icon(ui::IconName::PaperPlaneFill) .tooltip("Send") .ghost() + .disabled(dictating || self.uploading) .on_click(cx.listener(|this, _event, window, cx| { this.send(window, cx); })), @@ -1267,6 +1354,7 @@ impl Render for GroupPanel { v_flex() .size_full() .min_w_0() + .on_action(cx.listener(Self::toggle_dictation)) .child(self.render_encryption_notice(cx)) .child(self.render_banner(cx)) .child(self.render_pins(cx)) diff --git a/crates/speech/Cargo.toml b/crates/speech/Cargo.toml new file mode 100644 index 00000000..724ebf75 --- /dev/null +++ b/crates/speech/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "speech" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +log.workspace = true +gpui.workspace = true +gpui-kit = { workspace = true, features = ["speech"] } +flume.workspace = true +instant.workspace = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +common = { path = "../common" } +parakeet-rs = "=0.3.8" +ort = "=2.0.0-rc.13" +reqwest = { version = "0.12", default-features = false, features = [ + "blocking", + "rustls-tls", +] } +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true + +[target.'cfg(target_arch = "wasm32")'.dependencies] +js-sys = "0.3" +wasm-bindgen = "0.2" +web-sys = { version = "0.3", features = ["Window"] } diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs new file mode 100644 index 00000000..d4cfcdcb --- /dev/null +++ b/crates/speech/src/lib.rs @@ -0,0 +1,53 @@ +#[cfg(not(target_arch = "wasm32"))] +pub mod model; +#[cfg(not(target_arch = "wasm32"))] +mod parakeet; +#[cfg(target_arch = "wasm32")] +mod web; + +use std::time::Duration; + +use gpui::{ + Animation, AnimationExt as _, AnyElement, App, IntoElement as _, ParentElement as _, + Styled as _, div, +}; +use gpui_kit::component::ActiveTheme as _; +use gpui_kit::component::speech::SpeechRecognizer; + +/// A ticking `M:SS` counter for the dictation in progress. +pub fn elapsed(started: instant::Instant, cx: &App) -> AnyElement { + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .with_animation( + "dictation-elapsed", + Animation::new(Duration::from_secs(1)) + .repeat() + .with_max_fps(1.), + move |counter, _| counter.child(format_elapsed(started)), + ) + .into_any_element() +} + +fn format_elapsed(started: instant::Instant) -> String { + let seconds = started.elapsed().as_secs(); + format!("{}:{:02}", seconds / 60, seconds % 60) +} + +/// The recognizer backing the composer's microphone button. +#[cfg(not(target_arch = "wasm32"))] +pub fn recognizer() -> impl SpeechRecognizer { + parakeet::ParakeetRecognizer +} + +/// The recognizer backing the composer's microphone button. +#[cfg(target_arch = "wasm32")] +pub fn recognizer() -> impl SpeechRecognizer { + web::WebSpeechRecognizer +} + +/// The audio source backing the composer's microphone button on the web. +#[cfg(target_arch = "wasm32")] +pub fn input() -> impl gpui_kit::component::speech::AudioInput { + web::SelfCapturedInput +} diff --git a/crates/speech/src/model.json b/crates/speech/src/model.json new file mode 100644 index 00000000..50779f63 --- /dev/null +++ b/crates/speech/src/model.json @@ -0,0 +1,23 @@ +{ + "repository": "istupakov/parakeet-tdt-0.6b-v3-onnx", + "revision": "8f23f0c03c8761650bdb5b40aaf3e40d2c15f1ce", + "source": "nvidia/parakeet-tdt-0.6b-v3", + "license": "CC-BY-4.0", + "files": [ + { + "name": "decoder_joint-model.int8.onnx", + "size": 18202004, + "sha256": "eea7483ee3d1a30375daedc8ed83e3960c91b098812127a0d99d1c8977667a70" + }, + { + "name": "encoder-model.int8.onnx", + "size": 652183999, + "sha256": "6139d2fa7e1b086097b277c7149725edbab89cc7c7ae64b23c741be4055aff09" + }, + { + "name": "vocab.txt", + "size": 93939, + "sha256": "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d" + } + ] +} diff --git a/crates/speech/src/model.rs b/crates/speech/src/model.rs new file mode 100644 index 00000000..2abfbbc3 --- /dev/null +++ b/crates/speech/src/model.rs @@ -0,0 +1,295 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::{fs, io}; + +use anyhow::{Result, bail}; +use serde::Deserialize; + +/// How the model is currently available. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + NotInstalled, + Downloading { downloaded: u64, total: u64 }, + Installed, +} + +impl Status { + pub fn is_downloading(self) -> bool { + matches!(self, Status::Downloading { .. }) + } +} + +#[derive(Debug, Deserialize)] +struct Manifest { + repository: String, + revision: String, + files: Vec, +} + +#[derive(Debug, Deserialize)] +struct Artifact { + name: String, + size: u64, + sha256: String, +} + +fn manifest() -> &'static Manifest { + static MANIFEST: OnceLock = OnceLock::new(); + + MANIFEST.get_or_init(|| { + serde_json::from_str(include_str!("model.json")).expect("pinned model manifest") + }) +} + +/// The directory the model artifacts are downloaded into. +pub fn dir() -> PathBuf { + common::config_dir() + .join("speech") + .join("parakeet-tdt-0.6b-v3") +} + +/// Total download size in bytes. +pub fn total_size() -> u64 { + manifest().files.iter().map(|file| file.size).sum() +} + +/// The revision of the installed model, if its artifacts are complete. +fn installed_revision() -> Option { + let manifest = manifest(); + let dir = dir(); + + let complete = manifest.files.iter().all(|file| { + fs::metadata(dir.join(&file.name)).is_ok_and(|metadata| metadata.len() == file.size) + }); + + if complete + && fs::read_to_string(dir.join("verified")) + .is_ok_and(|revision| revision == manifest.revision) + { + Some(manifest.revision.clone()) + } else { + None + } +} + +/// Whether the model is ready to load. +pub fn installed() -> bool { + status() == Status::Installed +} + +/// The current status, reconciling the download state with the disk. +pub fn status() -> Status { + let state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if let Download::Running { + downloaded, total, .. + } = state.download + { + return Status::Downloading { downloaded, total }; + } + + if installed_revision().is_some() { + Status::Installed + } else { + Status::NotInstalled + } +} + +/// Take the last download failure, if it has not been reported yet. +pub fn take_error() -> Option { + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + state.error.take() +} + +/// Start downloading the model, unless a download is already running. +pub fn start_download() { + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if matches!(state.download, Download::Running { .. }) { + return; + } + + let cancel = Arc::new(AtomicBool::new(false)); + state.error = None; + state.download = Download::Running { + downloaded: 0, + total: total_size(), + cancel: cancel.clone(), + }; + + if let Err(error) = std::thread::Builder::new() + .name("speech-model-download".into()) + .spawn(move || download(cancel)) + { + state.download = Download::Idle; + state.error = Some(error.to_string()); + } +} + +/// Request cancellation of the running download. +pub fn cancel_download() { + let state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if let Download::Running { cancel, .. } = &state.download { + cancel.store(true, Ordering::Release); + } +} + +/// Delete the downloaded model and any partial download. +pub fn delete() { + super::parakeet::unload(); + cancel_download(); + + if let Err(error) = fs::remove_dir_all(dir()) + && error.kind() != io::ErrorKind::NotFound + { + log::warn!("failed to remove the speech model: {error}"); + } +} + +fn download(cancel: Arc) { + let result = download_inner(&cancel); + + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + state.download = Download::Idle; + + // A cancelled download must not report a spurious failure. + if !cancelled(&cancel) + && let Err(error) = result + { + state.error = Some(format!("{error:#}")); + } +} + +fn download_inner(cancel: &AtomicBool) -> Result<()> { + let manifest = manifest(); + let dir = dir(); + let client = reqwest::blocking::Client::builder() + .connect_timeout(std::time::Duration::from_secs(20)) + .build()?; + + fs::create_dir_all(&dir)?; + + for file in &manifest.files { + if cancelled(cancel) { + bail!("Cancelled"); + } + + let staging = dir.join(format!("{}.part", file.name)); + let result = download_file(&client, manifest, file, &staging, cancel); + + if result.is_err() || cancelled(cancel) { + let _unused = fs::remove_file(&staging); + result?; + bail!("Cancelled"); + } + } + + fs::write(dir.join("verified"), &manifest.revision)?; + Ok(()) +} + +fn download_file( + client: &reqwest::blocking::Client, + manifest: &Manifest, + file: &Artifact, + staging: &Path, + cancel: &AtomicBool, +) -> Result<()> { + use std::io::{Read as _, Write as _}; + + use sha2::Digest as _; + + let url = format!( + "https://huggingface.co/{}/resolve/{}/{}", + manifest.repository, manifest.revision, file.name + ); + + let mut response = client.get(url).send()?.error_for_status()?; + let mut out = fs::File::create(staging)?; + let mut hash = sha2::Sha256::new(); + let mut size = 0u64; + + loop { + if cancelled(cancel) { + bail!("Cancelled"); + } + + let mut chunk = [0u8; 64 * 1024]; + let read = response.read(&mut chunk)?; + + if read == 0 { + break; + } + + size += read as u64; + if size > file.size { + bail!("Unexpected model size"); + } + + hash.update(&chunk[..read]); + out.write_all(&chunk[..read])?; + + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + if let Download::Running { downloaded, .. } = &mut state.download { + *downloaded += read as u64; + } + } + + if size != file.size || format!("{:x}", hash.finalize()) != file.sha256 { + bail!("Model checksum verification failed"); + } + + out.sync_all()?; + fs::rename(staging, dir().join(&file.name))?; + Ok(()) +} + +fn cancelled(cancel: &AtomicBool) -> bool { + cancel.load(Ordering::Acquire) +} + +enum Download { + Idle, + Running { + downloaded: u64, + total: u64, + cancel: Arc, + }, +} + +struct State { + download: Download, + error: Option, +} + +fn state() -> &'static Mutex { + static STATE: Mutex = Mutex::new(State { + download: Download::Idle, + error: None, + }); + + &STATE +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manifest_pins_v3_artifacts_with_exact_sizes() { + let manifest = manifest(); + + assert_eq!(manifest.repository, "istupakov/parakeet-tdt-0.6b-v3-onnx"); + assert_eq!(manifest.revision.len(), 40); + assert_eq!(manifest.files.len(), 3); + assert_eq!(total_size(), 670_479_942); + assert!( + manifest + .files + .iter() + .all(|file| file.sha256.len() == 64 && !file.name.contains('/')) + ); + } +} diff --git a/crates/speech/src/parakeet.rs b/crates/speech/src/parakeet.rs new file mode 100644 index 00000000..6c19229b --- /dev/null +++ b/crates/speech/src/parakeet.rs @@ -0,0 +1,214 @@ +use std::path::PathBuf; +use std::sync::OnceLock; +use std::time::Duration; + +use anyhow::{Context as _, Result, anyhow}; +use gpui::{App, Task}; +use gpui_kit::component::speech::{ + AudioFormat, RecognitionSession, SpeechError, SpeechRecognizer, SpeechSink, +}; +use parakeet_rs::{ParakeetTDT, Transcriber as _}; + +const MODEL_RATE: u32 = 16_000; +const MAX_SECONDS: usize = 120; + +/// The on-device recognizer. +pub struct ParakeetRecognizer; + +impl SpeechRecognizer for ParakeetRecognizer { + fn audio_format(&self) -> AudioFormat { + AudioFormat::new(MODEL_RATE, 1) + } + + fn is_available(&self, _cx: &App) -> bool { + true + } + + fn start( + &self, + sink: SpeechSink, + cx: &mut App, + ) -> Result, SpeechError> { + if super::model::installed() { + worker().send_or_log(Command::Warm(super::model::dir())); + } else if super::model::status().is_downloading() { + return Err(SpeechError::recognizer(anyhow!( + "The speech model is still downloading. Try again when it finishes." + ))); + } else { + return Err(SpeechError::recognizer(anyhow!( + "No speech model is installed. Download it from Settings." + ))); + } + + // The session buffers everything it is pushed, + // so audio is accepted right away. + sink.ready(cx); + + Ok(Box::new(ParakeetSession { + sink, + samples: Vec::new(), + _result: None, + })) + } +} + +struct ParakeetSession { + sink: SpeechSink, + samples: Vec, + _result: Option>, +} + +impl RecognitionSession for ParakeetSession { + fn push_audio(&mut self, samples: &[i16], _cx: &mut App) { + if self.samples.len() < MAX_SECONDS * MODEL_RATE as usize { + self.samples + .extend(samples.iter().map(|sample| f32::from(*sample) / 32_768.0)); + } + } + + fn finish(&mut self, cx: &mut App) { + let samples = std::mem::take(&mut self.samples); + let sink = self.sink.clone(); + + // Speech too short to transcribe ends the session at once. + let silent = samples.iter().all(|sample| sample.abs() < 0.0001); + if samples.len() < MODEL_RATE as usize / 5 || silent { + sink.finish(cx); + return; + } + + let (sender, receiver) = flume::unbounded(); + worker().send_or_log(Command::Transcribe { samples, sender }); + + self._result = Some(cx.spawn(async move |cx| { + let result = receiver.recv_async().await; + + cx.update(|cx| match result { + Ok(Ok(text)) if !text.is_empty() => { + sink.phrase(text, cx); + sink.finish(cx); + } + Ok(Ok(_)) => sink.finish(cx), + Ok(Err(error)) => sink.error(SpeechError::recognizer(error), cx), + Err(_) => sink.error( + SpeechError::recognizer(anyhow!("The speech worker is unavailable")), + cx, + ), + }); + })); + } +} + +enum Command { + Warm(PathBuf), + Transcribe { + samples: Vec, + sender: flume::Sender>, + }, + Unload, +} + +/// A handle to the single inference worker. +fn worker() -> &'static flume::Sender { + static WORKER: OnceLock> = OnceLock::new(); + + WORKER.get_or_init(|| { + let (sender, receiver) = flume::unbounded::(); + + let spawned = std::thread::Builder::new() + .name("speech-parakeet".into()) + .spawn(move || run_worker(receiver)); + + if let Err(error) = spawned { + log::error!("failed to spawn the speech worker: {error}"); + } + + sender + }) +} + +fn run_worker(receiver: flume::Receiver) { + let mut cached: Option<(PathBuf, ParakeetTDT)> = None; + + while let Ok(command) = receiver.recv_timeout(Duration::from_secs(30)) { + match command { + Command::Unload => { + cached = None; + } + Command::Warm(dir) => { + let loaded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + load(&mut cached, &dir).map(|_| ()) + })); + + match flatten_panic(loaded) { + Ok(Ok(())) => {} + Ok(Err(error)) | Err(error) => { + log::warn!("failed to load the speech model: {error:#}") + } + } + } + Command::Transcribe { samples, sender } => { + let transcribed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + transcribe(&mut cached, samples) + })); + + let result = match flatten_panic(transcribed) { + Ok(result) => result, + Err(_) => { + cached = None; + Err(anyhow!("Dictation stopped unexpectedly. Try again.")) + } + }; + + if sender.send(result).is_err() { + log::debug!("speech session was cancelled before its result arrived"); + } + } + } + } +} + +fn load<'a>( + cached: &'a mut Option<(PathBuf, ParakeetTDT)>, + dir: &PathBuf, +) -> Result<&'a mut ParakeetTDT> { + if cached.as_ref().is_none_or(|(path, _)| path != dir) { + let model = ParakeetTDT::from_pretrained(dir, None) + .map_err(|error| anyhow!("Could not load the speech model: {error}"))?; + *cached = Some((dir.clone(), model)); + } + + let (_, model) = cached.as_mut().context("speech model was unloaded")?; + Ok(model) +} + +fn transcribe(cached: &mut Option<(PathBuf, ParakeetTDT)>, samples: Vec) -> Result { + let model = load(cached, &super::model::dir())?; + + model + .transcribe_samples(samples, MODEL_RATE, 1, None) + .map(|result| result.text) + .map_err(|error| anyhow!("Could not transcribe this recording: {error}")) +} + +trait SendOrLog { + fn send_or_log(&self, command: Command); +} + +impl SendOrLog for flume::Sender { + fn send_or_log(&self, command: Command) { + if self.send(command).is_err() { + log::debug!("speech worker is gone; command dropped"); + } + } +} + +/// Retire cached weights, e.g. after the model was deleted. +pub(crate) fn unload() { + worker().send_or_log(Command::Unload); +} + +fn flatten_panic(result: std::thread::Result) -> Result { + result.map_err(|_| anyhow!("The speech worker panicked")) +} diff --git a/crates/speech/src/web.rs b/crates/speech/src/web.rs new file mode 100644 index 00000000..09303dae --- /dev/null +++ b/crates/speech/src/web.rs @@ -0,0 +1,256 @@ +use std::cell::RefCell; + +use anyhow::anyhow; +use gpui::App; +use gpui_kit::Subscription; +use gpui_kit::component::speech::{ + AudioFormat, AudioInput, AudioSink, RecognitionSession, SpeechError, SpeechRecognizer, + SpeechSink, +}; +use js_sys::{Array, Function, JsString, Reflect}; +use wasm_bindgen::closure::Closure; +use wasm_bindgen::{JsCast as _, JsValue}; +use web_sys::window; + +/// The recognizer backed by `SpeechRecognition` (or its `webkit` prefix). +pub struct WebSpeechRecognizer; + +impl SpeechRecognizer for WebSpeechRecognizer { + fn audio_format(&self) -> AudioFormat { + AudioFormat::default() + } + + fn is_available(&self, _cx: &App) -> bool { + true + } + + fn start( + &self, + sink: SpeechSink, + cx: &mut App, + ) -> Result, SpeechError> { + let recognition = Recognition::open(sink, cx).map_err(SpeechError::recognizer)?; + + Ok(Box::new(recognition)) + } +} + +pub struct SelfCapturedInput; + +impl AudioInput for SelfCapturedInput { + fn start( + &self, + _format: AudioFormat, + _sink: AudioSink, + _cx: &mut App, + ) -> Result { + Ok(Subscription::new(|| {})) + } +} + +enum Event { + Hypothesis(String), + Phrase(String), + Error(String), + End, +} + +struct Recognition { + recognition: JsValue, + on_result: Option>, + on_error: Option>, + on_end: Option>, + _task: gpui::Task<()>, +} + +impl Recognition { + fn open(sink: SpeechSink, cx: &mut App) -> anyhow::Result { + let window = window().ok_or_else(|| anyhow!("No browser window"))?; + + let constructor = Reflect::get(&window, &JsString::from("SpeechRecognition")) + .ok() + .filter(|value| value.is_function()) + .or_else(|| { + Reflect::get(&window, &JsString::from("webkitSpeechRecognition")) + .ok() + .filter(|value| value.is_function()) + }) + .ok_or_else(|| anyhow!("This browser does not support speech recognition"))?; + + let recognition = + Reflect::construct(constructor.unchecked_ref::(), &Array::new()) + .map_err(|error| js_error("constructing the recognizer", error))?; + + let (sender, receiver) = flume::unbounded::(); + let error_sender = sender.clone(); + let end_sender = sender.clone(); + + let finalized = RefCell::new(0usize); + let on_result = Closure::wrap(Box::new(move |event: JsValue| { + let read = |value: &JsValue, property: &str| -> Option { + Reflect::get(value, &JsString::from(property)).ok() + }; + + let Some(results) = read(&event, "results") else { + return; + }; + let length = read(&results, "length") + .and_then(|length| length.as_f64()) + .unwrap_or(0.0) as usize; + + let mut finalized = finalized.borrow_mut(); + let mut interim = String::new(); + + for index in *finalized..length { + let Some(result) = read(&results, &index.to_string()) else { + continue; + }; + + let Some(alternative) = read(&result, "0") else { + continue; + }; + + let Some(text) = read(&alternative, "transcript").and_then(|text| text.as_string()) + else { + continue; + }; + + let is_final = read(&result, "isFinal") + .and_then(|final_value| final_value.as_bool()) + .unwrap_or(false); + + if is_final { + *finalized += 1; + if sender.send(Event::Phrase(text)).is_err() { + log::debug!("speech session was cancelled mid-utterance"); + } + } else { + interim.push_str(&text); + } + } + + if !interim.is_empty() && sender.send(Event::Hypothesis(interim)).is_err() { + log::debug!("speech session was cancelled mid-utterance"); + } + }) as Box); + + let on_error = Closure::wrap(Box::new(move |event: JsValue| { + let message = Reflect::get(&event, &JsString::from("error")) + .ok() + .and_then(|error| error.as_string()) + .unwrap_or_else(|| "unknown speech error".into()); + + if error_sender.send(Event::Error(message)).is_err() { + log::debug!("speech session was cancelled before its error arrived"); + } + }) as Box); + + let on_end = Closure::wrap(Box::new(move |_event: JsValue| { + if end_sender.send(Event::End).is_err() { + log::debug!("speech session was cancelled before its end arrived"); + } + }) as Box); + + set_callback(&recognition, "onresult", on_result.as_ref().unchecked_ref())?; + set_callback(&recognition, "onerror", on_error.as_ref().unchecked_ref())?; + set_callback(&recognition, "onend", on_end.as_ref().unchecked_ref())?; + set_bool(&recognition, "continuous", true)?; + set_bool(&recognition, "interimResults", true)?; + + call_method(&recognition, "start")?; + + // The browser consumes audio as soon as it has permission + sink.ready(cx); + + let task = cx.spawn(async move |cx| { + while let Ok(event) = receiver.recv_async().await { + let done = matches!(event, Event::End); + + // The speech state ignores sink calls once its session ended, + // so a late event after a cancel is harmless. + cx.update(|cx| match event { + Event::Hypothesis(text) => sink.hypothesis(text, cx), + Event::Phrase(text) => sink.phrase(text, cx), + Event::Error(message) => { + sink.error(SpeechError::recognizer(anyhow!("{message}")), cx) + } + Event::End => sink.finish(cx), + }); + + if done { + break; + } + } + }); + + Ok(Self { + recognition, + on_result: Some(on_result), + on_error: Some(on_error), + on_end: Some(on_end), + _task: task, + }) + } + + fn stop(&mut self) { + self.on_result = None; + self.on_error = None; + self.on_end = None; + + let _ignored = call_method(&self.recognition, "abort"); + } +} + +impl Drop for Recognition { + fn drop(&mut self) { + self.stop(); + } +} + +impl RecognitionSession for Recognition { + fn push_audio(&mut self, _samples: &[i16], _cx: &mut App) { + // The browser recognizer captures the microphone itself. + // the state still pushes audio through its input, which is a no-op here. + } + + fn finish(&mut self, _cx: &mut App) { + // The end handler reports completion, so no result is fabricated here. + let _ignored = call_method(&self.recognition, "stop"); + } +} + +fn set_callback(target: &JsValue, property: &str, callback: &Function) -> anyhow::Result<()> { + Reflect::set(target, &JsString::from(property), callback) + .map_err(|error| js_error(property, error))?; + Ok(()) +} + +fn set_bool(target: &JsValue, property: &str, value: bool) -> anyhow::Result<()> { + Reflect::set(target, &JsString::from(property), &JsValue::from(value)) + .map_err(|error| js_error(property, error))?; + Ok(()) +} + +fn call_method(target: &JsValue, method: &str) -> anyhow::Result<()> { + let function = Reflect::get(target, &JsString::from(method)) + .map_err(|error| js_error(method, error))? + .unchecked_into::(); + + function + .call0(target) + .map(|_| ()) + .map_err(|error| js_error(method, error)) +} + +fn js_error(what: &str, error: JsValue) -> anyhow::Error { + let description = error + .as_string() + .or_else(|| { + Reflect::get(&error, &JsString::from("message")) + .ok() + .and_then(|message| message.as_string()) + }) + .unwrap_or_else(|| "unknown error".into()); + + anyhow!("{what} failed: {description}") +} diff --git a/crates/ui/src/icon.rs b/crates/ui/src/icon.rs index 20b9effb..2d723053 100644 --- a/crates/ui/src/icon.rs +++ b/crates/ui/src/icon.rs @@ -20,6 +20,7 @@ pub enum IconName { Link, Lock, Message, + Paperclip, PaperPlaneFill, Plus, PlusChat, @@ -51,6 +52,7 @@ impl IconNamed for IconName { Self::Link => "icons/link.svg", Self::Lock => "icons/lock.svg", Self::Message => "icons/message.svg", + Self::Paperclip => "icons/paperclip.svg", Self::PaperPlaneFill => "icons/paper-plane-fill.svg", Self::Plus => "icons/plus.svg", Self::PlusChat => "icons/plus-chat.svg", diff --git a/crates/ui/src/message.rs b/crates/ui/src/message.rs index 6c61bc98..745f5108 100644 --- a/crates/ui/src/message.rs +++ b/crates/ui/src/message.rs @@ -10,6 +10,7 @@ use gpui::{ }; use gpui_kit::assets::IconName; use gpui_kit::component::button::{Button, ButtonVariants as _}; +use gpui_kit::component::clipboard::Clipboard; use gpui_kit::component::menu::{DropdownMenu as _, PopupMenu, PopupMenuItem}; use gpui_kit::component::{ ActiveTheme, InteractiveElementExt as _, Sizable as _, StyledExt as _, h_flex, v_flex, @@ -602,7 +603,7 @@ pub struct MessageActions { style: StyleRefinement, reactions: SmallVec<[(&'static str, SharedClickListener); 4]>, reply: Option, - copy: Option, + copy: Option<(ElementId, SharedString)>, dropdown: Option, } @@ -634,9 +635,10 @@ impl MessageActions { self } - /// Add the copy button. - pub fn copy(mut self, handler: impl Fn(&ClickEvent, &mut Window, &mut App) + 'static) -> Self { - self.copy = Some(Rc::new(handler)); + /// Add the copy button for the given value. + /// The id must be unique per bar, the clipboard component keys its copied state on it. + pub fn copy(mut self, id: impl Into, value: impl Into) -> Self { + self.copy = Some((id.into(), value.into())); self } @@ -707,15 +709,8 @@ impl RenderOnce for MessageActions { .on_click(move |event, window, cx| reply(event, window, cx)), ) }) - .when_some(copy, |this, copy| { - this.child( - Button::new("copy") - .icon(IconName::Copy) - .tooltip("Copy") - .small() - .ghost() - .on_click(move |event, window, cx| copy(event, window, cx)), - ) + .when_some(copy, |this, (id, value)| { + this.child(Clipboard::new(id).value(value).tooltip("Copy").small()) }) .when_some(dropdown, |this, dropdown| { this.child(div().flex_shrink_0().h_4().w_px().bg(cx.theme().border)) diff --git a/crates/workspace/Cargo.toml b/crates/workspace/Cargo.toml index 9fdc7c30..1972576c 100644 --- a/crates/workspace/Cargo.toml +++ b/crates/workspace/Cargo.toml @@ -15,6 +15,7 @@ chat_ui = { path = "../chat_ui" } nip29 = { path = "../nip29" } nip29_ui = { path = "../nip29_ui" } settings = { path = "../settings" } +speech = { path = "../speech" } person = { path = "../person" } auto_update = { path = "../auto_update" } diff --git a/crates/workspace/src/dialogs/new_chat.rs b/crates/workspace/src/dialogs/new_chat.rs index e46bbf52..81d92ad0 100644 --- a/crates/workspace/src/dialogs/new_chat.rs +++ b/crates/workspace/src/dialogs/new_chat.rs @@ -10,7 +10,6 @@ use gpui::{ Window, div, px, uniform_list, }; use gpui_kit::assets::IconName; -use gpui_kit::base::StyledExt as _; use gpui_kit::component::button::{Button, ButtonVariants}; use gpui_kit::component::dock::{DockArea, DockPlacement, PanelHandle}; use gpui_kit::component::input::{Input, InputEvent, InputState}; @@ -205,6 +204,7 @@ impl NewChat { Ok(()) })); + return; } @@ -369,15 +369,13 @@ impl NewChat { .p_0p5() .rounded_full() .bg(cx.theme().secondary) - .text_xs() - .font_medium() .hover(|this| this.bg(cx.theme().primary)) .child( Avatar::new(profile.avatar()) .seed(profile.avatar_seed()) .xsmall(), ) - .child(div().max_w_32().truncate().child(profile.name())) + .child(div().max_w_32().truncate().text_xs().child(profile.name())) .on_click(cx.listener(move |this, _event, _window, cx| { this.toggle_member(&public_key, cx); })) diff --git a/crates/workspace/src/dialogs/profile.rs b/crates/workspace/src/dialogs/profile.rs index 82a67fd8..b3c83eb2 100644 --- a/crates/workspace/src/dialogs/profile.rs +++ b/crates/workspace/src/dialogs/profile.rs @@ -4,17 +4,17 @@ use anyhow::Error; use chat::{ChatRegistry, Room, RoomKind}; use gpui::prelude::FluentBuilder; use gpui::{ - App, AppContext, ClipboardItem, Context, Div, Entity, IntoElement, ParentElement, Render, - SharedString, Styled, Task, Window, div, retain_all, + App, AppContext, Context, Div, Entity, IntoElement, ParentElement, Render, SharedString, + Styled, Task, Window, div, retain_all, }; use gpui_kit::assets::IconName; use gpui_kit::base::StyledExt as _; use gpui_kit::component::button::{Button, ButtonVariants}; +use gpui_kit::component::clipboard::Clipboard; use gpui_kit::component::dock::DockPlacement; use gpui_kit::component::input::{Input, InputState, Textarea, TextareaState}; use gpui_kit::component::notification::Notification; use gpui_kit::component::{ActiveTheme, Disableable, Sizable, WindowExt, h_flex, v_flex}; -use instant::Duration; use nostr_sdk::prelude::*; use person::{Directory, Person, PersonRegistry, shorten_pubkey}; use settings::AppSettings; @@ -49,8 +49,6 @@ struct ProfileDialog { website_input: Entity, /// Uploading state uploading: bool, - /// Copied states - copied: bool, /// Updating state updating: bool, /// Tasks @@ -92,7 +90,6 @@ impl ProfileDialog { bio_input, website_input, uploading: false, - copied: false, updating: false, tasks: vec![], } @@ -141,35 +138,6 @@ impl ProfileDialog { cx.notify(); } - fn copy(&mut self, value: String, window: &mut Window, cx: &mut Context) { - let item = ClipboardItem::new_string(value); - cx.write_to_clipboard(item); - - self.set_copied(true, window, cx); - } - - fn set_copied(&mut self, status: bool, window: &mut Window, cx: &mut Context) { - self.copied = status; - cx.notify(); - - if status { - self.tasks.push(cx.spawn_in(window, async move |this, cx| { - cx.background_executor().timer(Duration::from_secs(2)).await; - - // Reset the copied state after a delay - cx.update(|window, cx| { - this.update(cx, |this, cx| { - this.set_copied(false, window, cx); - }) - .ok(); - }) - .ok(); - - Ok(()) - })); - } - } - fn set_uploading(&mut self, status: bool, cx: &mut Context) { self.uploading = status; cx.notify(); @@ -368,7 +336,8 @@ impl Render for ProfileDialog { .w_full() .items_center() .justify_center() - .bg(cx.theme().sidebar) + .rounded(cx.theme().radius_lg) + .bg(cx.theme().muted) .child(Avatar::new(avatar).seed(profile.avatar_seed()).large()) .when(self.editing, |this| { this.child( @@ -393,25 +362,13 @@ impl Render for ProfileDialog { .justify_center() .gap_3() .rounded(cx.theme().radius) - .bg(cx.theme().sidebar) + .bg(cx.theme().muted) .text_xs() .child(shorten_pkey) - .child( - Button::new("copy") - .icon({ - if self.copied { - IconName::CircleCheck - } else { - IconName::Copy - } - }) - .xsmall() - .secondary() - .on_click(cx.listener(move |this, _ev, window, cx| { - let pk = this.public_key.to_bech32().unwrap(); - this.copy(pk, window, cx) - })), - ), + .child(Clipboard::new("copy").value_fn({ + let public_key = self.public_key; + move |_, _| public_key.to_bech32().unwrap_or_default().into() + })), ), ) .when(self.editing, |this| { @@ -544,7 +501,7 @@ fn field( }; v_flex() - .gap_1p5() + .gap_1() .child(label(text, cx)) - .child(div().text_xs().text_color(color).child(content)) + .child(div().text_sm().text_color(color).child(content)) } diff --git a/crates/workspace/src/dialogs/relays.rs b/crates/workspace/src/dialogs/relays.rs index 4ec18ec2..620a3b94 100644 --- a/crates/workspace/src/dialogs/relays.rs +++ b/crates/workspace/src/dialogs/relays.rs @@ -264,7 +264,7 @@ impl RelayManager { fn render_tab_bar(&self, cx: &mut Context) -> impl IntoElement { TabBar::new("relays-tabs") - .segmented() + .pill() .selected_index(self.active_tab) .on_click(cx.listener(|this, ix: &usize, _window, cx| { this.active_tab = *ix; diff --git a/crates/workspace/src/dialogs/settings.rs b/crates/workspace/src/dialogs/settings.rs index eae40c36..53a79811 100644 --- a/crates/workspace/src/dialogs/settings.rs +++ b/crates/workspace/src/dialogs/settings.rs @@ -1,19 +1,30 @@ #[cfg(not(target_arch = "wasm32"))] +use std::time::Duration; + +use chat_ui::ToggleDictation; +#[cfg(not(target_arch = "wasm32"))] use gpui::PathPromptOptions; +#[cfg(not(target_arch = "wasm32"))] +use gpui::Task; use gpui::http_client::Url; use gpui::{ - App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, Subscription, - Window, div, + AnyElement, App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, + Subscription, Window, div, }; use gpui_kit::assets::IconName; use gpui_kit::component::button::{Button, ButtonVariants}; use gpui_kit::component::input::{Input, InputEvent, InputState, NumberInput}; +use gpui_kit::component::kbd::Kbd; use gpui_kit::component::menu::{DropdownMenu, PopupMenuItem}; use gpui_kit::component::notification::Notification; +#[cfg(not(target_arch = "wasm32"))] +use gpui_kit::component::progress::Progress; use gpui_kit::component::separator::Separator; use gpui_kit::component::switch::Switch; use gpui_kit::component::{Disableable, Sizable, WindowExt, h_flex, v_flex}; use settings::{AppSettings, AppearanceMode}; +#[cfg(not(target_arch = "wasm32"))] +use speech::model::{self as speech_model}; use ui::{settings_block, settings_row}; /// Bounds for the theme number inputs @@ -29,6 +40,8 @@ pub struct Preferences { font_size_input: Entity, radius_input: Entity, radius_lg_input: Entity, + #[cfg(not(target_arch = "wasm32"))] + _speech_poll: Task<()>, /// Keeps the control subscriptions alive while the dialog is open _subscriptions: Vec, } @@ -94,11 +107,40 @@ impl Preferences { }), ]; + #[cfg(not(target_arch = "wasm32"))] + let _speech_poll = cx.spawn_in(window, async move |this, cx| { + let timer = Duration::from_millis(500); + let mut last = speech_model::status(); + + loop { + cx.background_executor().timer(timer).await; + + let status = speech_model::status(); + let error = speech_model::take_error(); + + if status == last { + continue; + } + + last = status; + + this.update_in(cx, |_, window, cx| { + if let Some(error) = error { + window.push_notification(Notification::error(error).autohide(false), cx); + } + cx.notify(); + }) + .ok(); + } + }); + Self { file_input, font_size_input, radius_input, radius_lg_input, + #[cfg(not(target_arch = "wasm32"))] + _speech_poll, _subscriptions: subscriptions, } } @@ -188,10 +230,102 @@ impl Preferences { .detach(); } } + + fn render_speech( + &mut self, + #[cfg_attr(target_arch = "wasm32", allow(unused_variables))] cx: &mut Context, + ) -> AnyElement { + #[cfg(target_arch = "wasm32")] + { + return div().into_any_element(); + } + + #[cfg(not(target_arch = "wasm32"))] + { + let size = speech_model::total_size() / 1_000_000; + let status = speech_model::status(); + + let (description, control) = match &status { + speech_model::Status::NotInstalled => ( + format!("{size} MB download"), + Button::new("download-model") + .label("Download") + .secondary() + .small() + .on_click(|_, _window, cx| { + speech_model::start_download(); + cx.refresh_windows(); + }) + .into_any_element(), + ), + speech_model::Status::Downloading { downloaded, total } => ( + format!( + "Downloading · {} of {} MB", + downloaded / 1_000_000, + total / 1_000_000 + ), + Button::new("cancel-download") + .label("Cancel") + .ghost() + .small() + .on_click(|_, _window, cx| { + speech_model::cancel_download(); + cx.refresh_windows(); + }) + .into_any_element(), + ), + speech_model::Status::Installed => ( + format!("Parakeet v3 · {size} MB"), + Button::new("remove-model") + .label("Remove") + .secondary() + .small() + .on_click(|_, _window, cx| { + speech_model::delete(); + cx.refresh_windows(); + }) + .into_any_element(), + ), + }; + + let mut block = v_flex().w_full().gap_3().child(settings_row( + "Speech model", + description, + control, + cx, + )); + + if let speech_model::Status::Downloading { downloaded, total } = &status { + let (downloaded, total) = (*downloaded, *total); + + block = block.child( + Progress::new("speech-model-download") + .value(downloaded as f32 / total.max(1) as f32 * 100.) + .small(), + ); + } + + block.into_any_element() + } + } + + /// Shows the keystroke actually bound to the dictation action at startup + fn render_dictation_shortcut( + &self, + window: &mut Window, + cx: &mut Context, + ) -> impl IntoElement { + let control = match Kbd::global_binding_for_action(&ToggleDictation, window) { + Some(kbd) => kbd.into_any_element(), + None => div().into_any_element(), + }; + + settings_row("Shortcut", "Press to start or stop dictation", control, cx) + } } impl Render for Preferences { - fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { + fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { const SCREENING: &str = "Show an screening dialog to verify the unknown sender."; const WOT: &str = "Treat people followed by your contacts as known senders."; const AVATAR: &str = "Hide all avatar pictures to improve performance."; @@ -374,6 +508,9 @@ impl Render for Preferences { cx, )) .child(Separator::horizontal()) + .child(self.render_speech(cx)) + .child(self.render_dictation_shortcut(window, cx)) + .child(Separator::horizontal()) .child(settings_block( "Media Upload Service", MEDIA, diff --git a/crates/workspace/src/panels/backup.rs b/crates/workspace/src/panels/backup.rs index 8a0bbcc5..1e20353a 100644 --- a/crates/workspace/src/panels/backup.rs +++ b/crates/workspace/src/panels/backup.rs @@ -1,16 +1,14 @@ use anyhow::Error; use gpui::{ - App, AppContext, ClipboardItem, Context, Entity, EventEmitter, FocusHandle, Focusable, - IntoElement, ParentElement, Render, SharedString, Styled, Task, Window, div, + App, AppContext, Context, Entity, EventEmitter, FocusHandle, Focusable, IntoElement, + ParentElement, Render, SharedString, Styled, Task, Window, div, }; -use gpui_kit::assets::IconName; use gpui_kit::base::StyledExt as _; -use gpui_kit::component::button::{Button, ButtonVariants}; +use gpui_kit::component::clipboard::Clipboard; use gpui_kit::component::dock::{Panel, PanelEvent}; use gpui_kit::component::input::{Input, InputState}; use gpui_kit::component::separator::Separator; -use gpui_kit::component::{ActiveTheme, Sizable, v_flex}; -use instant::Duration; +use gpui_kit::component::{ActiveTheme, v_flex}; use nostr_sdk::prelude::*; use state::USER_KEYRING; @@ -32,9 +30,6 @@ pub struct BackupPanel { /// Secret key input nsec_input: Entity, - /// Copied status - copied: bool, - /// Background tasks tasks: Vec>>, } @@ -54,7 +49,6 @@ impl BackupPanel { focus_handle: cx.focus_handle(), npub_input, nsec_input, - copied: false, tasks: vec![], } } @@ -81,31 +75,6 @@ impl BackupPanel { Ok(()) })); } - - fn copy_secret(&mut self, cx: &mut Context) { - let value = self.nsec_input.read(cx).value(); - let item = ClipboardItem::new_string(value.to_string()); - cx.write_to_clipboard(item); - - // Set the copied status to true - self.set_copied(true, cx); - } - - fn set_copied(&mut self, status: bool, cx: &mut Context) { - self.copied = status; - cx.notify(); - - self.tasks.push(cx.spawn(async move |this, cx| { - cx.background_executor().timer(Duration::from_secs(2)).await; - - // Clear the error message after a delay - this.update(cx, |this, cx| { - this.set_copied(false, cx); - })?; - - Ok(()) - })); - } } impl gpui_kit::component::dock::BasePanel for BackupPanel { @@ -174,20 +143,12 @@ impl Render for BackupPanel { .child(Input::new(&self.nsec_input).disabled(true)), ) .child( - Button::new("copy") - .icon(IconName::Copy) - .label({ - if self.copied { - "Copied" - } else { - "Copy secret key" - } - }) - .primary() - .small() - .on_click(cx.listener(move |this, _ev, _window, cx| { - this.copy_secret(cx); - })), + Clipboard::new("copy-secret") + .accessibility_label("Copy secret key") + .value_fn({ + let nsec_input = self.nsec_input.clone(); + move |_, cx| nsec_input.read(cx).value() + }), ), ) } diff --git a/crates/workspace/src/sidebar/mod.rs b/crates/workspace/src/sidebar/mod.rs index 640008f7..83b21fe5 100644 --- a/crates/workspace/src/sidebar/mod.rs +++ b/crates/workspace/src/sidebar/mod.rs @@ -340,7 +340,7 @@ impl Sidebar { ) .child( div() - .text_sm() + .text_xs() .text_color(cx.theme().sidebar_foreground) .child(label), ) diff --git a/desktop/Cargo.toml b/desktop/Cargo.toml index adf1d030..2db31079 100644 --- a/desktop/Cargo.toml +++ b/desktop/Cargo.toml @@ -26,6 +26,11 @@ icons = [ "resources/icon.ico", ] +# Merged over the generated Info.plist; required for the speech input to +# request microphone and speech recognition access on macOS. +[package.metadata.packager.macos] +info_plist_path = "resources/Info.plist" + [dependencies] assets = { path = "../crates/assets" } workspace = { path = "../crates/workspace" } @@ -33,6 +38,7 @@ ui = { path = "../crates/ui" } state = { path = "../crates/state" } nip4e = { path = "../crates/nip4e" } chat = { path = "../crates/chat" } +chat_ui = { path = "../crates/chat_ui" } nip29 = { path = "../crates/nip29" } settings = { path = "../crates/settings" } auto_update = { path = "../crates/auto_update" } diff --git a/desktop/resources/Info.plist b/desktop/resources/Info.plist new file mode 100644 index 00000000..c0bfc62a --- /dev/null +++ b/desktop/resources/Info.plist @@ -0,0 +1,10 @@ + + + + + NSMicrophoneUsageDescription + Dictate text into messages. + NSSpeechRecognitionUsageDescription + Turn your speech into text. + + diff --git a/desktop/src/main.rs b/desktop/src/main.rs index a4c34176..84c0b6de 100644 --- a/desktop/src/main.rs +++ b/desktop/src/main.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, Mutex}; +use chat_ui::ToggleDictation; use gpui::{ App, Bounds, Global, KeyBinding, Menu, MenuItem, SharedString, TitlebarOptions, WindowBackgroundAppearance, WindowBounds, WindowDecorations, WindowKind, WindowOptions, @@ -45,13 +46,19 @@ fn main() { // Register the `quit` function cx.on_action(quit); - // Register the `quit` function with CMD+Q (macOS) + // Register the `quit` function with CMD+Q (macOS) and dictation with CMD+D #[cfg(target_os = "macos")] - cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]); + cx.bind_keys([ + KeyBinding::new("cmd-q", Quit, None), + KeyBinding::new("cmd-d", ToggleDictation, None), + ]); - // Register the `quit` function with Super+Q (others) + // Register the `quit` function with Super+Q (others) and dictation with Ctrl+D #[cfg(not(target_os = "macos"))] - cx.bind_keys([KeyBinding::new("super-q", Quit, None)]); + cx.bind_keys([ + KeyBinding::new("super-q", Quit, None), + KeyBinding::new("ctrl-d", ToggleDictation, None), + ]); // Set menu items cx.set_menus(vec![Menu { diff --git a/web/Cargo.toml b/web/Cargo.toml index 8658a426..6251560b 100644 --- a/web/Cargo.toml +++ b/web/Cargo.toml @@ -14,6 +14,7 @@ ui = { path = "../crates/ui" } state = { path = "../crates/state" } nip4e = { path = "../crates/nip4e" } chat = { path = "../crates/chat" } +chat_ui = { path = "../crates/chat_ui" } nip29 = { path = "../crates/nip29" } settings = { path = "../crates/settings" } person = { path = "../crates/person" } diff --git a/web/src/lib.rs b/web/src/lib.rs index 5afcbbf4..abc91bac 100644 --- a/web/src/lib.rs +++ b/web/src/lib.rs @@ -78,6 +78,11 @@ pub fn run() -> Result<(), JsValue> { APPLICATION.with(|application| { *application.borrow_mut() = Some(app.run_embedded(|cx| { + cx.bind_keys([ + KeyBinding::new("cmd-d", chat_ui::ToggleDictation, None), + KeyBinding::new("ctrl-d", chat_ui::ToggleDictation, None), + ]); + gpui_kit::init(cx); load_embedded_fonts(cx);