@@ -0,0 +1,256 @@
|
||||
use std::cell::RefCell;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use gpui::App;
|
||||
use gpui_kit::Subscription;
|
||||
use gpui_kit::component::speech::{
|
||||
AudioFormat, AudioInput, AudioSink, RecognitionSession, SpeechError, SpeechRecognizer,
|
||||
SpeechSink,
|
||||
};
|
||||
use js_sys::{Array, Function, JsString, Reflect};
|
||||
use wasm_bindgen::closure::Closure;
|
||||
use wasm_bindgen::{JsCast as _, JsValue};
|
||||
use web_sys::window;
|
||||
|
||||
/// The recognizer backed by `SpeechRecognition` (or its `webkit` prefix).
|
||||
pub struct WebSpeechRecognizer;
|
||||
|
||||
impl SpeechRecognizer for WebSpeechRecognizer {
|
||||
fn audio_format(&self) -> AudioFormat {
|
||||
AudioFormat::default()
|
||||
}
|
||||
|
||||
fn is_available(&self, _cx: &App) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn start(
|
||||
&self,
|
||||
sink: SpeechSink,
|
||||
cx: &mut App,
|
||||
) -> Result<Box<dyn RecognitionSession>, SpeechError> {
|
||||
let recognition = Recognition::open(sink, cx).map_err(SpeechError::recognizer)?;
|
||||
|
||||
Ok(Box::new(recognition))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SelfCapturedInput;
|
||||
|
||||
impl AudioInput for SelfCapturedInput {
|
||||
fn start(
|
||||
&self,
|
||||
_format: AudioFormat,
|
||||
_sink: AudioSink,
|
||||
_cx: &mut App,
|
||||
) -> Result<Subscription, SpeechError> {
|
||||
Ok(Subscription::new(|| {}))
|
||||
}
|
||||
}
|
||||
|
||||
enum Event {
|
||||
Hypothesis(String),
|
||||
Phrase(String),
|
||||
Error(String),
|
||||
End,
|
||||
}
|
||||
|
||||
struct Recognition {
|
||||
recognition: JsValue,
|
||||
on_result: Option<Closure<dyn FnMut(JsValue)>>,
|
||||
on_error: Option<Closure<dyn FnMut(JsValue)>>,
|
||||
on_end: Option<Closure<dyn FnMut(JsValue)>>,
|
||||
_task: gpui::Task<()>,
|
||||
}
|
||||
|
||||
impl Recognition {
|
||||
fn open(sink: SpeechSink, cx: &mut App) -> anyhow::Result<Self> {
|
||||
let window = window().ok_or_else(|| anyhow!("No browser window"))?;
|
||||
|
||||
let constructor = Reflect::get(&window, &JsString::from("SpeechRecognition"))
|
||||
.ok()
|
||||
.filter(|value| value.is_function())
|
||||
.or_else(|| {
|
||||
Reflect::get(&window, &JsString::from("webkitSpeechRecognition"))
|
||||
.ok()
|
||||
.filter(|value| value.is_function())
|
||||
})
|
||||
.ok_or_else(|| anyhow!("This browser does not support speech recognition"))?;
|
||||
|
||||
let recognition =
|
||||
Reflect::construct(constructor.unchecked_ref::<Function>(), &Array::new())
|
||||
.map_err(|error| js_error("constructing the recognizer", error))?;
|
||||
|
||||
let (sender, receiver) = flume::unbounded::<Event>();
|
||||
let error_sender = sender.clone();
|
||||
let end_sender = sender.clone();
|
||||
|
||||
let finalized = RefCell::new(0usize);
|
||||
let on_result = Closure::wrap(Box::new(move |event: JsValue| {
|
||||
let read = |value: &JsValue, property: &str| -> Option<JsValue> {
|
||||
Reflect::get(value, &JsString::from(property)).ok()
|
||||
};
|
||||
|
||||
let Some(results) = read(&event, "results") else {
|
||||
return;
|
||||
};
|
||||
let length = read(&results, "length")
|
||||
.and_then(|length| length.as_f64())
|
||||
.unwrap_or(0.0) as usize;
|
||||
|
||||
let mut finalized = finalized.borrow_mut();
|
||||
let mut interim = String::new();
|
||||
|
||||
for index in *finalized..length {
|
||||
let Some(result) = read(&results, &index.to_string()) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let Some(alternative) = read(&result, "0") else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let Some(text) = read(&alternative, "transcript").and_then(|text| text.as_string())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let is_final = read(&result, "isFinal")
|
||||
.and_then(|final_value| final_value.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
if is_final {
|
||||
*finalized += 1;
|
||||
if sender.send(Event::Phrase(text)).is_err() {
|
||||
log::debug!("speech session was cancelled mid-utterance");
|
||||
}
|
||||
} else {
|
||||
interim.push_str(&text);
|
||||
}
|
||||
}
|
||||
|
||||
if !interim.is_empty() && sender.send(Event::Hypothesis(interim)).is_err() {
|
||||
log::debug!("speech session was cancelled mid-utterance");
|
||||
}
|
||||
}) as Box<dyn FnMut(JsValue)>);
|
||||
|
||||
let on_error = Closure::wrap(Box::new(move |event: JsValue| {
|
||||
let message = Reflect::get(&event, &JsString::from("error"))
|
||||
.ok()
|
||||
.and_then(|error| error.as_string())
|
||||
.unwrap_or_else(|| "unknown speech error".into());
|
||||
|
||||
if error_sender.send(Event::Error(message)).is_err() {
|
||||
log::debug!("speech session was cancelled before its error arrived");
|
||||
}
|
||||
}) as Box<dyn FnMut(JsValue)>);
|
||||
|
||||
let on_end = Closure::wrap(Box::new(move |_event: JsValue| {
|
||||
if end_sender.send(Event::End).is_err() {
|
||||
log::debug!("speech session was cancelled before its end arrived");
|
||||
}
|
||||
}) as Box<dyn FnMut(JsValue)>);
|
||||
|
||||
set_callback(&recognition, "onresult", on_result.as_ref().unchecked_ref())?;
|
||||
set_callback(&recognition, "onerror", on_error.as_ref().unchecked_ref())?;
|
||||
set_callback(&recognition, "onend", on_end.as_ref().unchecked_ref())?;
|
||||
set_bool(&recognition, "continuous", true)?;
|
||||
set_bool(&recognition, "interimResults", true)?;
|
||||
|
||||
call_method(&recognition, "start")?;
|
||||
|
||||
// The browser consumes audio as soon as it has permission
|
||||
sink.ready(cx);
|
||||
|
||||
let task = cx.spawn(async move |cx| {
|
||||
while let Ok(event) = receiver.recv_async().await {
|
||||
let done = matches!(event, Event::End);
|
||||
|
||||
// The speech state ignores sink calls once its session ended,
|
||||
// so a late event after a cancel is harmless.
|
||||
cx.update(|cx| match event {
|
||||
Event::Hypothesis(text) => sink.hypothesis(text, cx),
|
||||
Event::Phrase(text) => sink.phrase(text, cx),
|
||||
Event::Error(message) => {
|
||||
sink.error(SpeechError::recognizer(anyhow!("{message}")), cx)
|
||||
}
|
||||
Event::End => sink.finish(cx),
|
||||
});
|
||||
|
||||
if done {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
recognition,
|
||||
on_result: Some(on_result),
|
||||
on_error: Some(on_error),
|
||||
on_end: Some(on_end),
|
||||
_task: task,
|
||||
})
|
||||
}
|
||||
|
||||
fn stop(&mut self) {
|
||||
self.on_result = None;
|
||||
self.on_error = None;
|
||||
self.on_end = None;
|
||||
|
||||
let _ignored = call_method(&self.recognition, "abort");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Recognition {
|
||||
fn drop(&mut self) {
|
||||
self.stop();
|
||||
}
|
||||
}
|
||||
|
||||
impl RecognitionSession for Recognition {
|
||||
fn push_audio(&mut self, _samples: &[i16], _cx: &mut App) {
|
||||
// The browser recognizer captures the microphone itself.
|
||||
// the state still pushes audio through its input, which is a no-op here.
|
||||
}
|
||||
|
||||
fn finish(&mut self, _cx: &mut App) {
|
||||
// The end handler reports completion, so no result is fabricated here.
|
||||
let _ignored = call_method(&self.recognition, "stop");
|
||||
}
|
||||
}
|
||||
|
||||
fn set_callback(target: &JsValue, property: &str, callback: &Function) -> anyhow::Result<()> {
|
||||
Reflect::set(target, &JsString::from(property), callback)
|
||||
.map_err(|error| js_error(property, error))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_bool(target: &JsValue, property: &str, value: bool) -> anyhow::Result<()> {
|
||||
Reflect::set(target, &JsString::from(property), &JsValue::from(value))
|
||||
.map_err(|error| js_error(property, error))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn call_method(target: &JsValue, method: &str) -> anyhow::Result<()> {
|
||||
let function = Reflect::get(target, &JsString::from(method))
|
||||
.map_err(|error| js_error(method, error))?
|
||||
.unchecked_into::<Function>();
|
||||
|
||||
function
|
||||
.call0(target)
|
||||
.map(|_| ())
|
||||
.map_err(|error| js_error(method, error))
|
||||
}
|
||||
|
||||
fn js_error(what: &str, error: JsValue) -> anyhow::Error {
|
||||
let description = error
|
||||
.as_string()
|
||||
.or_else(|| {
|
||||
Reflect::get(&error, &JsString::from("message"))
|
||||
.ok()
|
||||
.and_then(|message| message.as_string())
|
||||
})
|
||||
.unwrap_or_else(|| "unknown error".into());
|
||||
|
||||
anyhow!("{what} failed: {description}")
|
||||
}
|
||||
Reference in New Issue
Block a user