From 589c43b9825969817cfb3ebbd2bf4e4b17c29b99 Mon Sep 17 00:00:00 2001 From: Ren Amamiya Date: Tue, 27 Oct 2026 10:55:05 +0700 Subject: [PATCH] add parakeet --- Cargo.lock | 565 ++++++++++++++++++++++- crates/chat_ui/Cargo.toml | 1 + crates/chat_ui/src/lib.rs | 13 +- crates/nip29_ui/Cargo.toml | 1 + crates/nip29_ui/src/lib.rs | 13 +- crates/speech/Cargo.toml | 29 ++ crates/speech/src/lib.rs | 26 ++ crates/speech/src/model.json | 23 + crates/speech/src/model.rs | 295 ++++++++++++ crates/speech/src/parakeet.rs | 214 +++++++++ crates/speech/src/web.rs | 256 ++++++++++ crates/workspace/Cargo.toml | 1 + crates/workspace/src/dialogs/settings.rs | 111 ++++- 13 files changed, 1527 insertions(+), 21 deletions(-) create mode 100644 crates/speech/Cargo.toml create mode 100644 crates/speech/src/lib.rs create mode 100644 crates/speech/src/model.json create mode 100644 crates/speech/src/model.rs create mode 100644 crates/speech/src/parakeet.rs create mode 100644 crates/speech/src/web.rs diff --git a/Cargo.lock b/Cargo.lock index f61fef15..7a6bc562 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,7 +153,9 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", + "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -606,7 +608,7 @@ dependencies = [ "log", "num-rational", "num-traits", - "pastey", + "pastey 0.1.1", "rayon", "thiserror 2.0.21", "v_frame", @@ -680,6 +682,12 @@ version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3a647dce9631f152a99ec16cfc09267ae55fe280d5dc288964027d458076323" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -692,6 +700,12 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bech32" version = "0.12.0" @@ -707,7 +721,7 @@ dependencies = [ "bitflags 2.13.2", "cexpr", "clang-sys", - "itertools 0.11.0", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -1005,6 +1019,15 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -1145,6 +1168,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] @@ -1222,7 +1246,7 @@ dependencies = [ "cocoa-foundation 0.1.2", "core-foundation 0.9.4", "core-graphics 0.23.2", - "foreign-types", + "foreign-types 0.5.0", "libc", "objc", ] @@ -1238,7 +1262,7 @@ dependencies = [ "cocoa-foundation 0.2.1", "core-foundation 0.10.1", "core-graphics 0.24.0", - "foreign-types", + "foreign-types 0.5.0", "libc", "objc", ] @@ -1314,6 +1338,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1508,7 +1547,7 @@ dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", "core-graphics-types 0.1.3", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1521,7 +1560,7 @@ dependencies = [ "bitflags 2.13.2", "core-foundation 0.10.1", "core-graphics-types 0.2.0", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1534,7 +1573,7 @@ dependencies = [ "bitflags 2.13.2", "core-foundation 0.9.4", "core-graphics-types 0.1.3", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1568,7 +1607,7 @@ checksum = "a593227b66cbd4007b2a050dfdd9e1d1318311409c8d600dc82ba1b15ca9c130" dependencies = [ "core-foundation 0.10.1", "core-graphics 0.24.0", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1769,14 +1808,44 @@ dependencies = [ "cmov", ] +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -1792,17 +1861,37 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core", + "darling_core 0.24.1", "quote", "syn 3.0.9", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -1875,6 +1964,16 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "der" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.9" @@ -1884,6 +1983,37 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -2200,6 +2330,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etagere" version = "0.2.15" @@ -2256,6 +2392,17 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "eyre" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" +dependencies = [ + "autocfg", + "indenter", + "once_cell", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -2463,6 +2610,15 @@ dependencies = [ "ttf-parser", ] +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -2470,7 +2626,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -2484,6 +2640,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -3058,7 +3220,7 @@ dependencies = [ "derive_more", "dispatch2", "etagere", - "foreign-types", + "foreign-types 0.5.0", "futures", "gpui-pre", "gpui-pre-collections", @@ -3197,7 +3359,7 @@ dependencies = [ "core-graphics 0.24.0", "ctor", "dispatch2", - "foreign-types", + "foreign-types 0.5.0", "futures", "gpui-pre", "gpui-pre-apple", @@ -3750,6 +3912,18 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "hmac-sha256" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad320b3b96fb2a455a0726d16efe0a5afdbd34b71dea5bc53b05ea057714d4e" + +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "html5ever" version = "0.27.0" @@ -4086,6 +4260,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + [[package]] name = "indexed_db_futures" version = "0.6.4" @@ -4696,6 +4876,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + [[package]] name = "mac" version = "0.1.1" @@ -4731,6 +4917,22 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey 0.2.3", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "macroific" version = "2.0.0" @@ -4872,6 +5074,16 @@ dependencies = [ "xml5ever", ] +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -4925,7 +5137,7 @@ dependencies = [ "bitflags 2.13.2", "block", "core-graphics-types 0.2.0", - "foreign-types", + "foreign-types 0.5.0", "log", "objc", "paste", @@ -4991,6 +5203,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -5086,6 +5320,38 @@ dependencies = [ "native-theme", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.8.0" @@ -5171,6 +5437,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] @@ -6003,6 +6270,28 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags 2.13.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "oo7" version = "0.6.0" @@ -6066,12 +6355,49 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -6097,6 +6423,30 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "page_size" version = "0.6.0" @@ -6107,6 +6457,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "parakeet-rs" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f54545a861dca6b43c8d44e3dd932764ca196d84bcf0d1b48ea212b43b29666" +dependencies = [ + "eyre", + "hound", + "ndarray", + "ort", + "realfft", + "serde", + "serde_json", + "tokenizers", +] + [[package]] name = "parking" version = "2.2.1" @@ -6148,6 +6514,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pathfinder_geometry" version = "0.5.1" @@ -6187,6 +6559,15 @@ dependencies = [ "hmac 0.13.0", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -6488,6 +6869,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -6829,6 +7219,12 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -6839,6 +7235,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -6871,6 +7278,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "realfft" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "538895ccf2006b473aafbe8d72770e21038961ab67aae34f3c577485fc703a7d" +dependencies = [ + "rustfft", +] + [[package]] name = "reborrow" version = "0.5.5" @@ -6970,7 +7386,9 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "futures-channel", "futures-core", + "futures-util", "h2", "http", "http-body", @@ -7218,6 +7636,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.5" @@ -7698,7 +8130,7 @@ version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ - "darling", + "darling 0.24.1", "proc-macro2", "quote", "syn 3.0.9", @@ -7933,6 +8365,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "speech" +version = "2.0.0-alpha2" +dependencies = [ + "anyhow", + "common", + "flume 0.11.1", + "gpui-kit", + "gpui-pre", + "js-sys", + "log", + "ort", + "parakeet-rs", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "spin" version = "0.9.9" @@ -7960,6 +8413,18 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "sqlite-wasm-rs" version = "0.5.5" @@ -8044,6 +8509,12 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" @@ -8455,6 +8926,39 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tokenizers" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.21", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.54.1" @@ -8760,6 +9264,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "triomphe" version = "0.1.17" @@ -8889,6 +9403,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" @@ -8925,6 +9448,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "universal-hash" version = "0.5.1" @@ -8960,7 +9489,9 @@ checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" dependencies = [ "base64 0.23.1", "cookie_store", + "der", "log", + "native-tls", "percent-encoding", "rustls", "rustls-pki-types", @@ -8969,6 +9500,7 @@ dependencies = [ "serde_json", "ureq-proto", "utf8-zero", + "webpki-root-certs", "webpki-roots 1.0.11", ] @@ -10373,6 +10905,7 @@ dependencies = [ "serde", "settings", "smallvec", + "speech", "state", "ui", ] diff --git a/crates/chat_ui/Cargo.toml b/crates/chat_ui/Cargo.toml index 7c8b97c4..7f5b25a0 100644 --- a/crates/chat_ui/Cargo.toml +++ b/crates/chat_ui/Cargo.toml @@ -6,6 +6,7 @@ publish.workspace = true [dependencies] state = { path = "../state" } +speech = { path = "../speech" } ui = { path = "../ui" } gpui-kit = { workspace = true, features = ["speech"] } common = { path = "../common" } diff --git a/crates/chat_ui/src/lib.rs b/crates/chat_ui/src/lib.rs index 4a338f32..73ce1548 100644 --- a/crates/chat_ui/src/lib.rs +++ b/crates/chat_ui/src/lib.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, LazyLock, RwLock}; +use std::time::Duration; pub use actions::*; use anyhow::Error; @@ -165,7 +166,17 @@ impl ChatPanel { }); // Define dictation state - let speech = cx.new(SpeechState::new); + let speech = cx.new(|cx| { + let state = SpeechState::new(cx) + .recognizer(speech::recognizer()) + // Batch transcription of a long recording can take a while. + .stop_timeout(Duration::from_secs(180)); + + #[cfg(target_arch = "wasm32")] + let state = state.input(speech::input()); + + state + }); // Define subject input state let subject_input = cx.new(|cx| InputState::new(window, cx).placeholder("New subject...")); diff --git a/crates/nip29_ui/Cargo.toml b/crates/nip29_ui/Cargo.toml index 0d0783fc..451c1d50 100644 --- a/crates/nip29_ui/Cargo.toml +++ b/crates/nip29_ui/Cargo.toml @@ -11,6 +11,7 @@ common = { path = "../common" } nip29 = { path = "../nip29" } person = { path = "../person" } settings = { path = "../settings" } +speech = { path = "../speech" } state = { path = "../state" } gpui-kit = { workspace = true, features = ["speech"] } ui = { path = "../ui" } diff --git a/crates/nip29_ui/src/lib.rs b/crates/nip29_ui/src/lib.rs index c89cd8e5..331c2805 100644 --- a/crates/nip29_ui/src/lib.rs +++ b/crates/nip29_ui/src/lib.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::time::Duration; use anyhow::Result; use gpui::prelude::FluentBuilder; @@ -341,7 +342,17 @@ impl GroupPanel { .clean_on_escape() }); - let speech = cx.new(SpeechState::new); + let speech = cx.new(|cx| { + let state = SpeechState::new(cx) + .recognizer(speech::recognizer()) + // Batch transcription of a long recording can take a while. + .stop_timeout(Duration::from_secs(180)); + + #[cfg(target_arch = "wasm32")] + let state = state.input(speech::input()); + + state + }); let mut subscriptions = smallvec![]; diff --git a/crates/speech/Cargo.toml b/crates/speech/Cargo.toml new file mode 100644 index 00000000..444cc638 --- /dev/null +++ b/crates/speech/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "speech" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +log.workspace = true +gpui.workspace = true +gpui-kit = { workspace = true, features = ["speech"] } +flume.workspace = true + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +common = { path = "../common" } +parakeet-rs = "=0.3.8" +ort = "=2.0.0-rc.13" +reqwest = { version = "0.12", default-features = false, features = [ + "blocking", + "rustls-tls", +] } +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true + +[target.'cfg(target_arch = "wasm32")'.dependencies] +js-sys = "0.3" +wasm-bindgen = "0.2" +web-sys = { version = "0.3", features = ["Window"] } diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs new file mode 100644 index 00000000..99014d77 --- /dev/null +++ b/crates/speech/src/lib.rs @@ -0,0 +1,26 @@ +#[cfg(not(target_arch = "wasm32"))] +pub mod model; +#[cfg(not(target_arch = "wasm32"))] +mod parakeet; +#[cfg(target_arch = "wasm32")] +mod web; + +use gpui_kit::component::speech::SpeechRecognizer; + +/// The recognizer backing the composer's microphone button. +#[cfg(not(target_arch = "wasm32"))] +pub fn recognizer() -> impl SpeechRecognizer { + parakeet::ParakeetRecognizer +} + +/// The recognizer backing the composer's microphone button. +#[cfg(target_arch = "wasm32")] +pub fn recognizer() -> impl SpeechRecognizer { + web::WebSpeechRecognizer +} + +/// The audio source backing the composer's microphone button on the web. +#[cfg(target_arch = "wasm32")] +pub fn input() -> impl gpui_kit::component::speech::AudioInput { + web::SelfCapturedInput +} diff --git a/crates/speech/src/model.json b/crates/speech/src/model.json new file mode 100644 index 00000000..50779f63 --- /dev/null +++ b/crates/speech/src/model.json @@ -0,0 +1,23 @@ +{ + "repository": "istupakov/parakeet-tdt-0.6b-v3-onnx", + "revision": "8f23f0c03c8761650bdb5b40aaf3e40d2c15f1ce", + "source": "nvidia/parakeet-tdt-0.6b-v3", + "license": "CC-BY-4.0", + "files": [ + { + "name": "decoder_joint-model.int8.onnx", + "size": 18202004, + "sha256": "eea7483ee3d1a30375daedc8ed83e3960c91b098812127a0d99d1c8977667a70" + }, + { + "name": "encoder-model.int8.onnx", + "size": 652183999, + "sha256": "6139d2fa7e1b086097b277c7149725edbab89cc7c7ae64b23c741be4055aff09" + }, + { + "name": "vocab.txt", + "size": 93939, + "sha256": "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d" + } + ] +} diff --git a/crates/speech/src/model.rs b/crates/speech/src/model.rs new file mode 100644 index 00000000..2abfbbc3 --- /dev/null +++ b/crates/speech/src/model.rs @@ -0,0 +1,295 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::{fs, io}; + +use anyhow::{Result, bail}; +use serde::Deserialize; + +/// How the model is currently available. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + NotInstalled, + Downloading { downloaded: u64, total: u64 }, + Installed, +} + +impl Status { + pub fn is_downloading(self) -> bool { + matches!(self, Status::Downloading { .. }) + } +} + +#[derive(Debug, Deserialize)] +struct Manifest { + repository: String, + revision: String, + files: Vec, +} + +#[derive(Debug, Deserialize)] +struct Artifact { + name: String, + size: u64, + sha256: String, +} + +fn manifest() -> &'static Manifest { + static MANIFEST: OnceLock = OnceLock::new(); + + MANIFEST.get_or_init(|| { + serde_json::from_str(include_str!("model.json")).expect("pinned model manifest") + }) +} + +/// The directory the model artifacts are downloaded into. +pub fn dir() -> PathBuf { + common::config_dir() + .join("speech") + .join("parakeet-tdt-0.6b-v3") +} + +/// Total download size in bytes. +pub fn total_size() -> u64 { + manifest().files.iter().map(|file| file.size).sum() +} + +/// The revision of the installed model, if its artifacts are complete. +fn installed_revision() -> Option { + let manifest = manifest(); + let dir = dir(); + + let complete = manifest.files.iter().all(|file| { + fs::metadata(dir.join(&file.name)).is_ok_and(|metadata| metadata.len() == file.size) + }); + + if complete + && fs::read_to_string(dir.join("verified")) + .is_ok_and(|revision| revision == manifest.revision) + { + Some(manifest.revision.clone()) + } else { + None + } +} + +/// Whether the model is ready to load. +pub fn installed() -> bool { + status() == Status::Installed +} + +/// The current status, reconciling the download state with the disk. +pub fn status() -> Status { + let state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if let Download::Running { + downloaded, total, .. + } = state.download + { + return Status::Downloading { downloaded, total }; + } + + if installed_revision().is_some() { + Status::Installed + } else { + Status::NotInstalled + } +} + +/// Take the last download failure, if it has not been reported yet. +pub fn take_error() -> Option { + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + state.error.take() +} + +/// Start downloading the model, unless a download is already running. +pub fn start_download() { + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if matches!(state.download, Download::Running { .. }) { + return; + } + + let cancel = Arc::new(AtomicBool::new(false)); + state.error = None; + state.download = Download::Running { + downloaded: 0, + total: total_size(), + cancel: cancel.clone(), + }; + + if let Err(error) = std::thread::Builder::new() + .name("speech-model-download".into()) + .spawn(move || download(cancel)) + { + state.download = Download::Idle; + state.error = Some(error.to_string()); + } +} + +/// Request cancellation of the running download. +pub fn cancel_download() { + let state = state().lock().unwrap_or_else(|error| error.into_inner()); + + if let Download::Running { cancel, .. } = &state.download { + cancel.store(true, Ordering::Release); + } +} + +/// Delete the downloaded model and any partial download. +pub fn delete() { + super::parakeet::unload(); + cancel_download(); + + if let Err(error) = fs::remove_dir_all(dir()) + && error.kind() != io::ErrorKind::NotFound + { + log::warn!("failed to remove the speech model: {error}"); + } +} + +fn download(cancel: Arc) { + let result = download_inner(&cancel); + + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + state.download = Download::Idle; + + // A cancelled download must not report a spurious failure. + if !cancelled(&cancel) + && let Err(error) = result + { + state.error = Some(format!("{error:#}")); + } +} + +fn download_inner(cancel: &AtomicBool) -> Result<()> { + let manifest = manifest(); + let dir = dir(); + let client = reqwest::blocking::Client::builder() + .connect_timeout(std::time::Duration::from_secs(20)) + .build()?; + + fs::create_dir_all(&dir)?; + + for file in &manifest.files { + if cancelled(cancel) { + bail!("Cancelled"); + } + + let staging = dir.join(format!("{}.part", file.name)); + let result = download_file(&client, manifest, file, &staging, cancel); + + if result.is_err() || cancelled(cancel) { + let _unused = fs::remove_file(&staging); + result?; + bail!("Cancelled"); + } + } + + fs::write(dir.join("verified"), &manifest.revision)?; + Ok(()) +} + +fn download_file( + client: &reqwest::blocking::Client, + manifest: &Manifest, + file: &Artifact, + staging: &Path, + cancel: &AtomicBool, +) -> Result<()> { + use std::io::{Read as _, Write as _}; + + use sha2::Digest as _; + + let url = format!( + "https://huggingface.co/{}/resolve/{}/{}", + manifest.repository, manifest.revision, file.name + ); + + let mut response = client.get(url).send()?.error_for_status()?; + let mut out = fs::File::create(staging)?; + let mut hash = sha2::Sha256::new(); + let mut size = 0u64; + + loop { + if cancelled(cancel) { + bail!("Cancelled"); + } + + let mut chunk = [0u8; 64 * 1024]; + let read = response.read(&mut chunk)?; + + if read == 0 { + break; + } + + size += read as u64; + if size > file.size { + bail!("Unexpected model size"); + } + + hash.update(&chunk[..read]); + out.write_all(&chunk[..read])?; + + let mut state = state().lock().unwrap_or_else(|error| error.into_inner()); + if let Download::Running { downloaded, .. } = &mut state.download { + *downloaded += read as u64; + } + } + + if size != file.size || format!("{:x}", hash.finalize()) != file.sha256 { + bail!("Model checksum verification failed"); + } + + out.sync_all()?; + fs::rename(staging, dir().join(&file.name))?; + Ok(()) +} + +fn cancelled(cancel: &AtomicBool) -> bool { + cancel.load(Ordering::Acquire) +} + +enum Download { + Idle, + Running { + downloaded: u64, + total: u64, + cancel: Arc, + }, +} + +struct State { + download: Download, + error: Option, +} + +fn state() -> &'static Mutex { + static STATE: Mutex = Mutex::new(State { + download: Download::Idle, + error: None, + }); + + &STATE +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manifest_pins_v3_artifacts_with_exact_sizes() { + let manifest = manifest(); + + assert_eq!(manifest.repository, "istupakov/parakeet-tdt-0.6b-v3-onnx"); + assert_eq!(manifest.revision.len(), 40); + assert_eq!(manifest.files.len(), 3); + assert_eq!(total_size(), 670_479_942); + assert!( + manifest + .files + .iter() + .all(|file| file.sha256.len() == 64 && !file.name.contains('/')) + ); + } +} diff --git a/crates/speech/src/parakeet.rs b/crates/speech/src/parakeet.rs new file mode 100644 index 00000000..6c19229b --- /dev/null +++ b/crates/speech/src/parakeet.rs @@ -0,0 +1,214 @@ +use std::path::PathBuf; +use std::sync::OnceLock; +use std::time::Duration; + +use anyhow::{Context as _, Result, anyhow}; +use gpui::{App, Task}; +use gpui_kit::component::speech::{ + AudioFormat, RecognitionSession, SpeechError, SpeechRecognizer, SpeechSink, +}; +use parakeet_rs::{ParakeetTDT, Transcriber as _}; + +const MODEL_RATE: u32 = 16_000; +const MAX_SECONDS: usize = 120; + +/// The on-device recognizer. +pub struct ParakeetRecognizer; + +impl SpeechRecognizer for ParakeetRecognizer { + fn audio_format(&self) -> AudioFormat { + AudioFormat::new(MODEL_RATE, 1) + } + + fn is_available(&self, _cx: &App) -> bool { + true + } + + fn start( + &self, + sink: SpeechSink, + cx: &mut App, + ) -> Result, SpeechError> { + if super::model::installed() { + worker().send_or_log(Command::Warm(super::model::dir())); + } else if super::model::status().is_downloading() { + return Err(SpeechError::recognizer(anyhow!( + "The speech model is still downloading. Try again when it finishes." + ))); + } else { + return Err(SpeechError::recognizer(anyhow!( + "No speech model is installed. Download it from Settings." + ))); + } + + // The session buffers everything it is pushed, + // so audio is accepted right away. + sink.ready(cx); + + Ok(Box::new(ParakeetSession { + sink, + samples: Vec::new(), + _result: None, + })) + } +} + +struct ParakeetSession { + sink: SpeechSink, + samples: Vec, + _result: Option>, +} + +impl RecognitionSession for ParakeetSession { + fn push_audio(&mut self, samples: &[i16], _cx: &mut App) { + if self.samples.len() < MAX_SECONDS * MODEL_RATE as usize { + self.samples + .extend(samples.iter().map(|sample| f32::from(*sample) / 32_768.0)); + } + } + + fn finish(&mut self, cx: &mut App) { + let samples = std::mem::take(&mut self.samples); + let sink = self.sink.clone(); + + // Speech too short to transcribe ends the session at once. + let silent = samples.iter().all(|sample| sample.abs() < 0.0001); + if samples.len() < MODEL_RATE as usize / 5 || silent { + sink.finish(cx); + return; + } + + let (sender, receiver) = flume::unbounded(); + worker().send_or_log(Command::Transcribe { samples, sender }); + + self._result = Some(cx.spawn(async move |cx| { + let result = receiver.recv_async().await; + + cx.update(|cx| match result { + Ok(Ok(text)) if !text.is_empty() => { + sink.phrase(text, cx); + sink.finish(cx); + } + Ok(Ok(_)) => sink.finish(cx), + Ok(Err(error)) => sink.error(SpeechError::recognizer(error), cx), + Err(_) => sink.error( + SpeechError::recognizer(anyhow!("The speech worker is unavailable")), + cx, + ), + }); + })); + } +} + +enum Command { + Warm(PathBuf), + Transcribe { + samples: Vec, + sender: flume::Sender>, + }, + Unload, +} + +/// A handle to the single inference worker. +fn worker() -> &'static flume::Sender { + static WORKER: OnceLock> = OnceLock::new(); + + WORKER.get_or_init(|| { + let (sender, receiver) = flume::unbounded::(); + + let spawned = std::thread::Builder::new() + .name("speech-parakeet".into()) + .spawn(move || run_worker(receiver)); + + if let Err(error) = spawned { + log::error!("failed to spawn the speech worker: {error}"); + } + + sender + }) +} + +fn run_worker(receiver: flume::Receiver) { + let mut cached: Option<(PathBuf, ParakeetTDT)> = None; + + while let Ok(command) = receiver.recv_timeout(Duration::from_secs(30)) { + match command { + Command::Unload => { + cached = None; + } + Command::Warm(dir) => { + let loaded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + load(&mut cached, &dir).map(|_| ()) + })); + + match flatten_panic(loaded) { + Ok(Ok(())) => {} + Ok(Err(error)) | Err(error) => { + log::warn!("failed to load the speech model: {error:#}") + } + } + } + Command::Transcribe { samples, sender } => { + let transcribed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + transcribe(&mut cached, samples) + })); + + let result = match flatten_panic(transcribed) { + Ok(result) => result, + Err(_) => { + cached = None; + Err(anyhow!("Dictation stopped unexpectedly. Try again.")) + } + }; + + if sender.send(result).is_err() { + log::debug!("speech session was cancelled before its result arrived"); + } + } + } + } +} + +fn load<'a>( + cached: &'a mut Option<(PathBuf, ParakeetTDT)>, + dir: &PathBuf, +) -> Result<&'a mut ParakeetTDT> { + if cached.as_ref().is_none_or(|(path, _)| path != dir) { + let model = ParakeetTDT::from_pretrained(dir, None) + .map_err(|error| anyhow!("Could not load the speech model: {error}"))?; + *cached = Some((dir.clone(), model)); + } + + let (_, model) = cached.as_mut().context("speech model was unloaded")?; + Ok(model) +} + +fn transcribe(cached: &mut Option<(PathBuf, ParakeetTDT)>, samples: Vec) -> Result { + let model = load(cached, &super::model::dir())?; + + model + .transcribe_samples(samples, MODEL_RATE, 1, None) + .map(|result| result.text) + .map_err(|error| anyhow!("Could not transcribe this recording: {error}")) +} + +trait SendOrLog { + fn send_or_log(&self, command: Command); +} + +impl SendOrLog for flume::Sender { + fn send_or_log(&self, command: Command) { + if self.send(command).is_err() { + log::debug!("speech worker is gone; command dropped"); + } + } +} + +/// Retire cached weights, e.g. after the model was deleted. +pub(crate) fn unload() { + worker().send_or_log(Command::Unload); +} + +fn flatten_panic(result: std::thread::Result) -> Result { + result.map_err(|_| anyhow!("The speech worker panicked")) +} diff --git a/crates/speech/src/web.rs b/crates/speech/src/web.rs new file mode 100644 index 00000000..09303dae --- /dev/null +++ b/crates/speech/src/web.rs @@ -0,0 +1,256 @@ +use std::cell::RefCell; + +use anyhow::anyhow; +use gpui::App; +use gpui_kit::Subscription; +use gpui_kit::component::speech::{ + AudioFormat, AudioInput, AudioSink, RecognitionSession, SpeechError, SpeechRecognizer, + SpeechSink, +}; +use js_sys::{Array, Function, JsString, Reflect}; +use wasm_bindgen::closure::Closure; +use wasm_bindgen::{JsCast as _, JsValue}; +use web_sys::window; + +/// The recognizer backed by `SpeechRecognition` (or its `webkit` prefix). +pub struct WebSpeechRecognizer; + +impl SpeechRecognizer for WebSpeechRecognizer { + fn audio_format(&self) -> AudioFormat { + AudioFormat::default() + } + + fn is_available(&self, _cx: &App) -> bool { + true + } + + fn start( + &self, + sink: SpeechSink, + cx: &mut App, + ) -> Result, SpeechError> { + let recognition = Recognition::open(sink, cx).map_err(SpeechError::recognizer)?; + + Ok(Box::new(recognition)) + } +} + +pub struct SelfCapturedInput; + +impl AudioInput for SelfCapturedInput { + fn start( + &self, + _format: AudioFormat, + _sink: AudioSink, + _cx: &mut App, + ) -> Result { + Ok(Subscription::new(|| {})) + } +} + +enum Event { + Hypothesis(String), + Phrase(String), + Error(String), + End, +} + +struct Recognition { + recognition: JsValue, + on_result: Option>, + on_error: Option>, + on_end: Option>, + _task: gpui::Task<()>, +} + +impl Recognition { + fn open(sink: SpeechSink, cx: &mut App) -> anyhow::Result { + let window = window().ok_or_else(|| anyhow!("No browser window"))?; + + let constructor = Reflect::get(&window, &JsString::from("SpeechRecognition")) + .ok() + .filter(|value| value.is_function()) + .or_else(|| { + Reflect::get(&window, &JsString::from("webkitSpeechRecognition")) + .ok() + .filter(|value| value.is_function()) + }) + .ok_or_else(|| anyhow!("This browser does not support speech recognition"))?; + + let recognition = + Reflect::construct(constructor.unchecked_ref::(), &Array::new()) + .map_err(|error| js_error("constructing the recognizer", error))?; + + let (sender, receiver) = flume::unbounded::(); + let error_sender = sender.clone(); + let end_sender = sender.clone(); + + let finalized = RefCell::new(0usize); + let on_result = Closure::wrap(Box::new(move |event: JsValue| { + let read = |value: &JsValue, property: &str| -> Option { + Reflect::get(value, &JsString::from(property)).ok() + }; + + let Some(results) = read(&event, "results") else { + return; + }; + let length = read(&results, "length") + .and_then(|length| length.as_f64()) + .unwrap_or(0.0) as usize; + + let mut finalized = finalized.borrow_mut(); + let mut interim = String::new(); + + for index in *finalized..length { + let Some(result) = read(&results, &index.to_string()) else { + continue; + }; + + let Some(alternative) = read(&result, "0") else { + continue; + }; + + let Some(text) = read(&alternative, "transcript").and_then(|text| text.as_string()) + else { + continue; + }; + + let is_final = read(&result, "isFinal") + .and_then(|final_value| final_value.as_bool()) + .unwrap_or(false); + + if is_final { + *finalized += 1; + if sender.send(Event::Phrase(text)).is_err() { + log::debug!("speech session was cancelled mid-utterance"); + } + } else { + interim.push_str(&text); + } + } + + if !interim.is_empty() && sender.send(Event::Hypothesis(interim)).is_err() { + log::debug!("speech session was cancelled mid-utterance"); + } + }) as Box); + + let on_error = Closure::wrap(Box::new(move |event: JsValue| { + let message = Reflect::get(&event, &JsString::from("error")) + .ok() + .and_then(|error| error.as_string()) + .unwrap_or_else(|| "unknown speech error".into()); + + if error_sender.send(Event::Error(message)).is_err() { + log::debug!("speech session was cancelled before its error arrived"); + } + }) as Box); + + let on_end = Closure::wrap(Box::new(move |_event: JsValue| { + if end_sender.send(Event::End).is_err() { + log::debug!("speech session was cancelled before its end arrived"); + } + }) as Box); + + set_callback(&recognition, "onresult", on_result.as_ref().unchecked_ref())?; + set_callback(&recognition, "onerror", on_error.as_ref().unchecked_ref())?; + set_callback(&recognition, "onend", on_end.as_ref().unchecked_ref())?; + set_bool(&recognition, "continuous", true)?; + set_bool(&recognition, "interimResults", true)?; + + call_method(&recognition, "start")?; + + // The browser consumes audio as soon as it has permission + sink.ready(cx); + + let task = cx.spawn(async move |cx| { + while let Ok(event) = receiver.recv_async().await { + let done = matches!(event, Event::End); + + // The speech state ignores sink calls once its session ended, + // so a late event after a cancel is harmless. + cx.update(|cx| match event { + Event::Hypothesis(text) => sink.hypothesis(text, cx), + Event::Phrase(text) => sink.phrase(text, cx), + Event::Error(message) => { + sink.error(SpeechError::recognizer(anyhow!("{message}")), cx) + } + Event::End => sink.finish(cx), + }); + + if done { + break; + } + } + }); + + Ok(Self { + recognition, + on_result: Some(on_result), + on_error: Some(on_error), + on_end: Some(on_end), + _task: task, + }) + } + + fn stop(&mut self) { + self.on_result = None; + self.on_error = None; + self.on_end = None; + + let _ignored = call_method(&self.recognition, "abort"); + } +} + +impl Drop for Recognition { + fn drop(&mut self) { + self.stop(); + } +} + +impl RecognitionSession for Recognition { + fn push_audio(&mut self, _samples: &[i16], _cx: &mut App) { + // The browser recognizer captures the microphone itself. + // the state still pushes audio through its input, which is a no-op here. + } + + fn finish(&mut self, _cx: &mut App) { + // The end handler reports completion, so no result is fabricated here. + let _ignored = call_method(&self.recognition, "stop"); + } +} + +fn set_callback(target: &JsValue, property: &str, callback: &Function) -> anyhow::Result<()> { + Reflect::set(target, &JsString::from(property), callback) + .map_err(|error| js_error(property, error))?; + Ok(()) +} + +fn set_bool(target: &JsValue, property: &str, value: bool) -> anyhow::Result<()> { + Reflect::set(target, &JsString::from(property), &JsValue::from(value)) + .map_err(|error| js_error(property, error))?; + Ok(()) +} + +fn call_method(target: &JsValue, method: &str) -> anyhow::Result<()> { + let function = Reflect::get(target, &JsString::from(method)) + .map_err(|error| js_error(method, error))? + .unchecked_into::(); + + function + .call0(target) + .map(|_| ()) + .map_err(|error| js_error(method, error)) +} + +fn js_error(what: &str, error: JsValue) -> anyhow::Error { + let description = error + .as_string() + .or_else(|| { + Reflect::get(&error, &JsString::from("message")) + .ok() + .and_then(|message| message.as_string()) + }) + .unwrap_or_else(|| "unknown error".into()); + + anyhow!("{what} failed: {description}") +} diff --git a/crates/workspace/Cargo.toml b/crates/workspace/Cargo.toml index 9fdc7c30..1972576c 100644 --- a/crates/workspace/Cargo.toml +++ b/crates/workspace/Cargo.toml @@ -15,6 +15,7 @@ chat_ui = { path = "../chat_ui" } nip29 = { path = "../nip29" } nip29_ui = { path = "../nip29_ui" } settings = { path = "../settings" } +speech = { path = "../speech" } person = { path = "../person" } auto_update = { path = "../auto_update" } diff --git a/crates/workspace/src/dialogs/settings.rs b/crates/workspace/src/dialogs/settings.rs index eae40c36..40cb0af1 100644 --- a/crates/workspace/src/dialogs/settings.rs +++ b/crates/workspace/src/dialogs/settings.rs @@ -1,9 +1,12 @@ +#[cfg(not(target_arch = "wasm32"))] +use std::time::Duration; + #[cfg(not(target_arch = "wasm32"))] use gpui::PathPromptOptions; use gpui::http_client::Url; use gpui::{ - App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, Subscription, - Window, div, + AnyElement, App, AppContext, Context, Entity, IntoElement, ParentElement, Render, Styled, + Subscription, Task, Window, div, }; use gpui_kit::assets::IconName; use gpui_kit::component::button::{Button, ButtonVariants}; @@ -12,8 +15,10 @@ use gpui_kit::component::menu::{DropdownMenu, PopupMenuItem}; use gpui_kit::component::notification::Notification; use gpui_kit::component::separator::Separator; use gpui_kit::component::switch::Switch; -use gpui_kit::component::{Disableable, Sizable, WindowExt, h_flex, v_flex}; +use gpui_kit::component::{ActiveTheme, Disableable, Sizable, WindowExt, h_flex, v_flex}; use settings::{AppSettings, AppearanceMode}; +#[cfg(not(target_arch = "wasm32"))] +use speech::model::{self as speech_model}; use ui::{settings_block, settings_row}; /// Bounds for the theme number inputs @@ -29,6 +34,8 @@ pub struct Preferences { font_size_input: Entity, radius_input: Entity, radius_lg_input: Entity, + #[cfg(not(target_arch = "wasm32"))] + _speech_poll: Task<()>, /// Keeps the control subscriptions alive while the dialog is open _subscriptions: Vec, } @@ -94,11 +101,40 @@ impl Preferences { }), ]; + #[cfg(not(target_arch = "wasm32"))] + let _speech_poll = cx.spawn_in(window, async move |this, cx| { + let timer = Duration::from_millis(500); + let mut last = speech_model::status(); + + loop { + cx.background_executor().timer(timer).await; + + let status = speech_model::status(); + let error = speech_model::take_error(); + + if status == last { + continue; + } + + last = status; + + this.update_in(cx, |_, window, cx| { + if let Some(error) = error { + window.push_notification(Notification::error(error).autohide(false), cx); + } + cx.notify(); + }) + .ok(); + } + }); + Self { file_input, font_size_input, radius_input, radius_lg_input, + #[cfg(not(target_arch = "wasm32"))] + _speech_poll, _subscriptions: subscriptions, } } @@ -188,6 +224,74 @@ impl Preferences { .detach(); } } + + fn render_speech( + &mut self, + #[cfg_attr(target_arch = "wasm32", allow(unused_variables))] cx: &mut Context, + ) -> AnyElement { + #[cfg(target_arch = "wasm32")] + { + return div().into_any_element(); + } + + #[cfg(not(target_arch = "wasm32"))] + { + const SPEECH: &str = "On-device speech model used by the speech to text feature. Audio never leaves the device."; + + let control = match speech_model::status() { + speech_model::Status::NotInstalled => { + let size = speech_model::total_size() / 1024 / 1024; + + h_flex().gap_1().child( + Button::new("download-model") + .label(format!("Download model ({size} MB)")) + .small() + .on_click(|_, _window, cx| { + speech_model::start_download(); + cx.refresh_windows(); + }), + ) + } + speech_model::Status::Downloading { downloaded, total } => { + let percent = downloaded * 100 / total.max(1); + let downloaded = downloaded / 1024 / 1024; + let total = total / 1024 / 1024; + + h_flex() + .gap_2() + .child( + div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child(format!("Downloading {percent}% ยท {downloaded}/{total} MB")), + ) + .child( + Button::new("cancel-download") + .label("Cancel") + .small() + .on_click(|_, _window, cx| { + speech_model::cancel_download(); + cx.refresh_windows(); + }), + ) + } + speech_model::Status::Installed => h_flex() + .gap_2() + .child(div().text_sm().child("Installed")) + .child( + Button::new("delete-model") + .label("Delete") + .small() + .on_click(|_, _window, cx| { + speech_model::delete(); + cx.refresh_windows(); + }), + ), + }; + + settings_block("Speech Recognition", SPEECH, control, cx).into_any_element() + } + } } impl Render for Preferences { @@ -391,5 +495,6 @@ impl Render for Preferences { ), cx, )) + .child(self.render_speech(cx)) } }