From 2fd9fe63b4d576f1778c89f2a886edbb9c0772fa Mon Sep 17 00:00:00 2001 From: leifiyoo <250759920+leifiyoo@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:22:32 +0100 Subject: [PATCH] Add the narrator crate: GameNarrator, SAPI speech and screen narration state horizon-narrator ports vanilla's narrator core: - GameNarrator's status rules (NarratorStatus, queued vs. interrupting messages, the status announcement and its toast keys) - com.mojang.text2speech's NarratorWindows on SAPI's SpVoice, on a speech thread of its own so the game never waits on it - ScreenNarrationCollector and Screen's narration timing (delays, the suppression after init, findNarratableWidget) HORIZON_NO_AUDIO gives an active but silent narrator for scripted runs. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 8 + Cargo.toml | 1 + crates/horizon-narrator/Cargo.toml | 17 + crates/horizon-narrator/examples/say.rs | 28 ++ crates/horizon-narrator/src/lib.rs | 421 ++++++++++++++++++ crates/horizon-narrator/src/screen.rs | 417 +++++++++++++++++ crates/horizon-narrator/src/speech.rs | 120 +++++ crates/horizon-narrator/src/speech/linux.rs | 14 + crates/horizon-narrator/src/speech/mac.rs | 14 + crates/horizon-narrator/src/speech/windows.rs | 83 ++++ 10 files changed, 1123 insertions(+) create mode 100644 crates/horizon-narrator/Cargo.toml create mode 100644 crates/horizon-narrator/examples/say.rs create mode 100644 crates/horizon-narrator/src/lib.rs create mode 100644 crates/horizon-narrator/src/screen.rs create mode 100644 crates/horizon-narrator/src/speech.rs create mode 100644 crates/horizon-narrator/src/speech/linux.rs create mode 100644 crates/horizon-narrator/src/speech/mac.rs create mode 100644 crates/horizon-narrator/src/speech/windows.rs diff --git a/Cargo.lock b/Cargo.lock index 9413fb1a6a..296337a781 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1293,6 +1293,14 @@ dependencies = [ "zip", ] +[[package]] +name = "horizon-narrator" +version = "0.1.0" +dependencies = [ + "log", + "windows 0.62.2", +] + [[package]] name = "horizon-port-pack" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 788c0e564f..2bb1cfc3ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ exclude = ["launcher", "crates/horizon-telemetry-server", "crates/horizon-mod-re members = [ "crates/horizon-core", "crates/horizon-dualsense", + "crates/horizon-narrator", "crates/horizon-threads", "crates/horizon-hdr", "crates/horizon-world", diff --git a/crates/horizon-narrator/Cargo.toml b/crates/horizon-narrator/Cargo.toml new file mode 100644 index 0000000000..d4871af332 --- /dev/null +++ b/crates/horizon-narrator/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "horizon-narrator" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version = "1.85" +description = "The narrator: vanilla's text-to-speech (GameNarrator, com.mojang.text2speech) and screen narration state" + +[dependencies] +log = "0.4" + +# NarratorWindows: SAPI's SpVoice over COM, the voice the Windows narrator settings pick. +[target.'cfg(windows)'.dependencies] +windows = { version = "0.62.2", features = ["Win32_Foundation", "Win32_Media_Speech", "Win32_System_Com"] } + +[lints] +workspace = true diff --git a/crates/horizon-narrator/examples/say.rs b/crates/horizon-narrator/examples/say.rs new file mode 100644 index 0000000000..2d76ce2390 --- /dev/null +++ b/crates/horizon-narrator/examples/say.rs @@ -0,0 +1,28 @@ +//! Says its arguments with the platform's narrator, as the game would: +//! `cargo run -p horizon-narrator --example say -- [--volume 0.5] Hello there`. +fn main() { + let mut args: Vec = std::env::args().skip(1).collect(); + let mut volume = 1.0; + if args.first().is_some_and(|a| a == "--volume") && args.len() > 1 { + volume = args[1].parse().unwrap_or(1.0); + args.drain(..2); + } + horizon_narrator::init(); + horizon_narrator::set_status(horizon_narrator::Status::All); + horizon_narrator::set_volume(volume); + if !horizon_narrator::is_active() { + eprintln!("The narrator is not available on this system"); + std::process::exit(1); + } + let message = if args.is_empty() { + "Narrator test".to_owned() + } else { + args.join(" ") + }; + horizon_narrator::say_system_queued(&message); + // The speech thread speaks asynchronously; give it time to finish. + std::thread::sleep(std::time::Duration::from_millis( + 400 + 90 * message.len() as u64, + )); + println!("said: {message}"); +} diff --git a/crates/horizon-narrator/src/lib.rs b/crates/horizon-narrator/src/lib.rs new file mode 100644 index 0000000000..3e66cf3b5b --- /dev/null +++ b/crates/horizon-narrator/src/lib.rs @@ -0,0 +1,421 @@ +//! The narrator: vanilla's text-to-speech (`GameNarrator` over +//! `com.mojang.text2speech.Narrator` 1.19.12) and the screen narration state +//! (`ScreenNarrationCollector` and `Screen`'s narration timing, [`screen`]). +//! +//! One narrator per process, as `Minecraft.narrator`: [`init`] opens the +//! platform's speech on its own thread (SAPI on Windows, NSSpeechSynthesizer +//! on macOS, flite on Linux, as text2speech does), and the free functions are +//! `GameNarrator`'s methods. Speaking never blocks the caller: the speech +//! thread takes the calls in order. Until [`init`] (the dedicated server, +//! tests) the narrator is inactive, as text2speech's `Narrator.EMPTY`. +//! +//! The game keeps the narrator's [`Status`] and voice volume in step with its +//! options ([`set_status`], [`set_volume`]); the narrator reads them where +//! `GameNarrator` reads `options.narrator()` and +//! `getFinalSoundSourceVolume(VOICE)`. + +pub mod screen; +mod speech; + +use std::collections::VecDeque; +use std::sync::{Mutex, MutexGuard}; + +/// `NarratorStatus`, by its `options.txt` id. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Status { + #[default] + Off, + All, + Chat, + System, +} + +impl Status { + pub const ALL: [Status; 4] = [Status::Off, Status::All, Status::Chat, Status::System]; + + /// `byId`: `ByIdMap.continuous` with `OutOfBoundsStrategy.WRAP`. + pub fn by_id(id: i32) -> Status { + Self::ALL[id.rem_euclid(4) as usize] + } + pub fn id(self) -> i32 { + self as i32 + } + /// The status the narrator hotkey moves to (`byId(getId() + 1)`). + pub fn next(self) -> Status { + Self::by_id(self.id() + 1) + } + /// The translation key of `getName`. + pub fn key(self) -> &'static str { + match self { + Status::Off => "options.narrator.off", + Status::All => "options.narrator.all", + Status::Chat => "options.narrator.chat", + Status::System => "options.narrator.system", + } + } + pub fn should_narrate_chat(self) -> bool { + matches!(self, Status::All | Status::Chat) + } + pub fn should_narrate_system(self) -> bool { + matches!(self, Status::All | Status::System) + } + pub fn should_narrate_system_or_chat(self) -> bool { + matches!(self, Status::All | Status::System | Status::Chat) + } +} + +/// What `GameNarrator` hands its `Narrator`: the speech thread, or a test's +/// recorder. +pub trait Sink: Send { + fn say(&mut self, message: &str, interrupt: bool, volume: f32); + fn clear(&mut self); + /// `Narrator.active`: false for `Narrator.EMPTY`, which a platform without + /// speech (or whose speech failed to open) gets. + fn active(&mut self) -> bool; +} + +/// How many narrated messages [`recent`] keeps. +const RECENT: usize = 32; + +/// `GameNarrator`, with its options read from [`Status`] and the volume the +/// game sets. +pub struct GameNarrator { + sink: Option>, + status: Status, + volume: f32, + /// The messages the narrator was asked to say, oldest first (vanilla only + /// logs them in its IDE): the control socket and tests read them. + recent: VecDeque, +} + +impl GameNarrator { + pub const fn new() -> Self { + Self { + sink: None, + status: Status::Off, + volume: 1.0, + recent: VecDeque::new(), + } + } + pub fn with_sink(sink: Box) -> Self { + Self { + sink: Some(sink), + ..Self::new() + } + } + pub fn set_sink(&mut self, sink: Box) { + self.sink = Some(sink); + } + pub fn set_status(&mut self, status: Status) { + self.status = status; + } + pub fn status(&self) -> Status { + self.status + } + /// `getFinalSoundSourceVolume(SoundSource.VOICE)`: master times voice. + pub fn set_volume(&mut self, volume: f32) { + self.volume = volume; + } + + pub fn say_chat_queued(&mut self, message: &str) { + if self.status.should_narrate_chat() { + self.narrate_not_interrupting(message); + } + } + pub fn say_system_chat_queued(&mut self, message: &str) { + if self.status.should_narrate_system_or_chat() { + self.narrate_not_interrupting(message); + } + } + pub fn say_system_queued(&mut self, message: &str) { + if self.status.should_narrate_system() { + self.narrate_not_interrupting(message); + } + } + fn narrate_not_interrupting(&mut self, message: &str) { + if !message.is_empty() { + self.log(message); + self.narrate(message, false); + } + } + /// `saySystemNow`: what is said now cuts off whatever is being said. + pub fn say_system_now(&mut self, message: &str) { + if self.status.should_narrate_system() && !message.is_empty() { + self.log(message); + if self.is_active() { + if let Some(sink) = &mut self.sink { + sink.clear(); + } + self.narrate(message, true); + } + } + } + fn narrate(&mut self, message: &str, interrupt: bool) { + let volume = self.volume; + if let Some(sink) = &mut self.sink { + sink.say(message, interrupt, volume); + } + } + fn log(&mut self, message: &str) { + if self.recent.len() == RECENT { + self.recent.pop_front(); + } + self.recent.push_back(message.to_owned()); + } + + /// `updateNarratorStatus`, once the option holds `status`: clears, then + /// says `announcement` (`options.narrator` + " : " + the status's name) + /// whatever the status, `Off` included. The toast that goes with it is + /// the caller's ([`toast`]). + pub fn update_status(&mut self, status: Status, announcement: &str) { + self.status = status; + self.clear(); + self.log(announcement); + self.narrate(announcement, true); + } + + pub fn is_active(&mut self) -> bool { + self.sink.as_mut().is_some_and(|sink| sink.active()) + } + pub fn clear(&mut self) { + if self.status != Status::Off && self.is_active() { + if let Some(sink) = &mut self.sink { + sink.clear(); + } + } + } + pub fn recent(&self) -> Vec { + self.recent.iter().cloned().collect() + } +} + +impl Default for GameNarrator { + fn default() -> Self { + Self::new() + } +} + +/// The toast `updateNarratorStatus` shows (`SystemToastId.NARRATOR_TOGGLE`): +/// translation keys of its title and of its message, if it has one. +pub fn toast(status: Status, active: bool) -> (&'static str, Option<&'static str>) { + if !active { + ( + "narrator.toast.disabled", + Some("options.narrator.notavailable"), + ) + } else if status == Status::Off { + ("narrator.toast.disabled", None) + } else { + ("narrator.toast.enabled", Some(status.key())) + } +} + +static NARRATOR: Mutex = Mutex::new(GameNarrator::new()); + +fn narrator() -> MutexGuard<'static, GameNarrator> { + NARRATOR.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// `new GameNarrator`: opens the platform's speech on its own thread. With +/// `HORIZON_NO_AUDIO` set the narrator is active but silent, so scripted runs +/// still narrate into [`recent`]. Calling it again does nothing. +pub fn init() { + let mut narrator = narrator(); + if narrator.sink.is_none() { + narrator.sink = Some(Box::new(speech::Thread::spawn( + std::env::var_os("HORIZON_NO_AUDIO").is_some(), + ))); + } +} + +/// `destroy`: stops speaking and closes the speech. +pub fn destroy() { + if let Some(mut sink) = narrator().sink.take() { + sink.clear(); + } +} + +pub fn set_status(status: Status) { + narrator().set_status(status); +} +pub fn status() -> Status { + narrator().status() +} +pub fn set_volume(volume: f32) { + narrator().set_volume(volume); +} +pub fn is_active() -> bool { + narrator().is_active() +} +pub fn say_chat_queued(message: &str) { + narrator().say_chat_queued(message); +} +pub fn say_system_chat_queued(message: &str) { + narrator().say_system_chat_queued(message); +} +pub fn say_system_queued(message: &str) { + narrator().say_system_queued(message); +} +pub fn say_system_now(message: &str) { + narrator().say_system_now(message); +} +pub fn update_status(status: Status, announcement: &str) { + narrator().update_status(status, announcement); +} +pub fn clear() { + narrator().clear(); +} +/// The last messages the narrator was asked to say, oldest first. +pub fn recent() -> Vec { + narrator().recent() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + #[derive(Debug, PartialEq)] + enum Call { + Say(String, bool, f32), + Clear, + } + + #[derive(Clone, Default)] + struct Recorder { + calls: Arc>>, + inactive: bool, + } + impl Recorder { + fn take(&self) -> Vec { + std::mem::take(&mut self.calls.lock().unwrap()) + } + } + impl Sink for Recorder { + fn say(&mut self, message: &str, interrupt: bool, volume: f32) { + self.calls + .lock() + .unwrap() + .push(Call::Say(message.into(), interrupt, volume)); + } + fn clear(&mut self) { + self.calls.lock().unwrap().push(Call::Clear); + } + fn active(&mut self) -> bool { + !self.inactive + } + } + + fn narrator(status: Status) -> (GameNarrator, Recorder) { + let recorder = Recorder::default(); + let mut n = GameNarrator::with_sink(Box::new(recorder.clone())); + n.set_status(status); + (n, recorder) + } + + #[test] + fn statuses_wrap_through_their_ids_as_the_hotkey_cycles_them() { + assert_eq!(Status::by_id(0), Status::Off); + assert_eq!(Status::by_id(3), Status::System); + assert_eq!(Status::by_id(4), Status::Off); + assert_eq!(Status::by_id(-1), Status::System); + assert_eq!(Status::Off.next(), Status::All); + assert_eq!(Status::System.next(), Status::Off); + } + + #[test] + fn each_status_narrates_its_own_kinds_of_message() { + for (status, chat, system, either) in [ + (Status::Off, false, false, false), + (Status::All, true, true, true), + (Status::Chat, true, false, true), + (Status::System, false, true, true), + ] { + assert_eq!(status.should_narrate_chat(), chat); + assert_eq!(status.should_narrate_system(), system); + assert_eq!(status.should_narrate_system_or_chat(), either); + } + } + + #[test] + fn queued_messages_wait_their_turn_and_empty_ones_are_skipped() { + let (mut n, r) = narrator(Status::Chat); + n.set_volume(0.5); + n.say_chat_queued(" hi"); + n.say_system_queued("not in chat mode"); + n.say_system_chat_queued("Steve joined the game"); + n.say_chat_queued(""); + assert_eq!( + r.take(), + vec![ + Call::Say(" hi".into(), false, 0.5), + Call::Say("Steve joined the game".into(), false, 0.5), + ] + ); + assert_eq!(n.recent(), [" hi", "Steve joined the game"]); + } + + #[test] + fn saying_now_clears_and_interrupts_only_for_system_narration() { + let (mut n, r) = narrator(Status::System); + n.say_system_now("Options"); + assert_eq!( + r.take(), + vec![Call::Clear, Call::Say("Options".into(), true, 1.0)] + ); + n.set_status(Status::Chat); + n.say_system_now("Options"); + assert!(r.take().is_empty()); + } + + #[test] + fn an_inactive_narrator_logs_but_says_nothing() { + let recorder = Recorder { + inactive: true, + ..Default::default() + }; + let mut n = GameNarrator::with_sink(Box::new(recorder.clone())); + n.set_status(Status::All); + n.say_system_now("Options"); + n.clear(); + assert!(recorder.take().is_empty()); + assert_eq!(n.recent(), ["Options"]); + assert!(!n.is_active()); + assert!(!GameNarrator::new().is_active()); + } + + #[test] + fn a_status_change_is_announced_even_when_turning_it_off() { + let (mut n, r) = narrator(Status::All); + n.update_status(Status::Off, "Narrator : OFF"); + // clear() reads the new status: off, so nothing is cleared first. + assert_eq!( + r.take(), + vec![Call::Say("Narrator : OFF".into(), true, 1.0)] + ); + n.update_status(Status::All, "Narrator : Narrates All"); + assert_eq!( + r.take(), + vec![ + Call::Clear, + Call::Say("Narrator : Narrates All".into(), true, 1.0) + ] + ); + } + + #[test] + fn the_toggle_toast_names_the_status_or_why_there_is_none() { + assert_eq!( + toast(Status::All, false), + ( + "narrator.toast.disabled", + Some("options.narrator.notavailable") + ) + ); + assert_eq!(toast(Status::Off, true), ("narrator.toast.disabled", None)); + assert_eq!( + toast(Status::Chat, true), + ("narrator.toast.enabled", Some("options.narrator.chat")) + ); + } +} diff --git a/crates/horizon-narrator/src/screen.rs b/crates/horizon-narrator/src/screen.rs new file mode 100644 index 0000000000..748e0a88c1 --- /dev/null +++ b/crates/horizon-narrator/src/screen.rs @@ -0,0 +1,417 @@ +//! A screen's narration: `ScreenNarrationCollector` (what to say, and what +//! was said already) and `Screen`'s timing (`scheduleNarration`, +//! `handleDelayedNarration`, the suppression after `init`). +//! +//! The game builds a screen's narration each time it runs ([`Output`]): the +//! screen's title, then the narrated widget's own elements one level deeper +//! ([`Output::nest`]). Only what changed since the last run is said, unless a +//! run is forced; elements are read title first, then positions, hints and +//! usage, each outer before inner, joined with ". ". +use std::collections::BTreeMap; + +/// `NarratedElementType`, in reading order. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ElementType { + Title, + Position, + Hint, + Usage, +} + +/// `NarrationTrigger`: what scheduled the narration. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Trigger { + Mouse, + Keyboard, + #[default] + System, +} + +/// `NarratableEntry.NarrationPriority`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub enum Priority { + #[default] + None, + Hovered, + Focused, +} + +impl Priority { + pub fn is_terminal(self) -> bool { + self == Priority::Focused + } +} + +/// `ScreenNarrationCollector.NarrationEntry`. `contents` is the thunk's text: +/// `None` is `NarrationThunk.EMPTY`, which equals no text, not even an empty +/// one. +#[derive(Debug, Default)] +struct Entry { + contents: Option>, + generation: i64, + already_narrated: bool, +} + +/// `ScreenNarrationCollector`. +#[derive(Debug, Default)] +pub struct Collector { + generation: i64, + entries: BTreeMap<(ElementType, u32), Entry>, +} + +impl Collector { + /// `update`: a new generation, filled by `updater`. + pub fn update(&mut self, trigger: Trigger, updater: impl FnOnce(&mut Output<'_>)) { + self.generation += 1; + updater(&mut Output { + collector: self, + depth: 0, + trigger, + }); + } + + /// `collectNarrationText`: this generation's elements not narrated yet + /// (all of them when `force`), marked narrated. + pub fn collect(&mut self, force: bool) -> String { + let mut result = String::new(); + let mut first = true; + for entry in self.entries.values_mut() { + if entry.generation == self.generation && (force || !entry.already_narrated) { + for line in entry.contents.iter().flatten() { + if !first { + result.push_str(". "); + } + first = false; + result.push_str(line); + } + entry.already_narrated = true; + } + } + result + } + + fn add(&mut self, key: (ElementType, u32), contents: Vec) { + let generation = self.generation; + let entry = self.entries.entry(key).or_insert_with(|| Entry { + generation: -1, + ..Entry::default() + }); + // NarrationEntry.update: new text is narrated again, and so is text + // that was missing from the generation before. + if entry.contents.as_ref() != Some(&contents) { + entry.contents = Some(contents); + entry.already_narrated = false; + } else if entry.generation + 1 != generation { + entry.already_narrated = false; + } + entry.generation = generation; + } +} + +/// `NarrationElementOutput` at a depth. +pub struct Output<'a> { + collector: &'a mut Collector, + depth: u32, + trigger: Trigger, +} + +impl Output<'_> { + /// `add(type, NarrationThunk.from(text))`. + pub fn add(&mut self, kind: ElementType, text: impl Into) { + self.collector.add((kind, self.depth), vec![text.into()]); + } + /// `add(type, NarrationThunk.from(lines))`: one element, each line read + /// as its own sentence. + pub fn add_lines(&mut self, kind: ElementType, lines: Vec) { + self.collector.add((kind, self.depth), lines); + } + /// `nest`: the narrated element's own elements, read after the screen's + /// of the same type. + pub fn nest(&mut self) -> Output<'_> { + Output { + collector: self.collector, + depth: self.depth + 1, + trigger: self.trigger, + } + } + pub fn trigger(&self) -> Trigger { + self.trigger + } +} + +/// `Screen.NARRATE_SUPPRESS_AFTER_INIT_TIME`, and the delay after the narrator +/// is turned on. +pub const SUPPRESS_AFTER_INIT_MS: i64 = 2000; +pub const DELAY_MOUSE_MOVE_MS: i64 = 750; +pub const DELAY_MOUSE_ACTION_MS: i64 = 200; +pub const DELAY_KEYBOARD_ACTION_MS: i64 = 200; + +/// A screen's narration state (`Screen`'s `narrationState`, +/// `narrationSuppressTime`, `nextNarrationTime`, `nextNarrationTriggeredBy` +/// and `lastNarratable`). A new screen starts a new one. Times are +/// milliseconds on any steady clock (`Util.getMillis`). +#[derive(Debug)] +pub struct ScreenNarration { + pub collector: Collector, + suppress_time: i64, + next_time: i64, + next_trigger: Trigger, + /// The widget narrated last, by the caller's identity for it. + pub last_narratable: Option, +} + +impl Default for ScreenNarration { + fn default() -> Self { + Self { + collector: Collector::default(), + suppress_time: i64::MIN, + next_time: i64::MAX, + next_trigger: Trigger::System, + last_narratable: None, + } + } +} + +impl ScreenNarration { + /// The end of `init` (after its `triggerImmediateNarration(false)`): + /// screens opened from the keyboard wait for the next key before + /// narrating again; the others stay quiet for two seconds. + pub fn after_init(&mut self, now: i64, keyboard: bool) { + self.suppress_time = if keyboard { + i64::MAX + } else { + now.saturating_add(SUPPRESS_AFTER_INIT_MS) + }; + } + + /// `scheduleNarration(delay, ignoreSuppression, trigger)`. + pub fn schedule(&mut self, now: i64, delay: i64, ignore_suppression: bool, trigger: Trigger) { + self.next_time = now.saturating_add(delay); + self.next_trigger = trigger; + if ignore_suppression { + self.suppress_time = i64::MIN; + } + } + /// `scheduleNarration()`: as soon as the screen next ticks. + pub fn schedule_now(&mut self, now: i64) { + self.schedule(now, 0, true, Trigger::System); + } + pub fn after_mouse_move(&mut self, now: i64) { + self.schedule(now, DELAY_MOUSE_MOVE_MS, false, Trigger::Mouse); + } + pub fn after_mouse_action(&mut self, now: i64) { + self.schedule(now, DELAY_MOUSE_ACTION_MS, true, Trigger::Mouse); + } + pub fn after_keyboard_action(&mut self, now: i64) { + self.schedule(now, DELAY_KEYBOARD_ACTION_MS, true, Trigger::Keyboard); + } + /// `updateNarratorStatus`'s scheduling: once the narrator is turned on, + /// the screen is narrated two seconds later. + pub fn after_status_change(&mut self, now: i64, was_disabled: bool, trigger: Trigger) { + if was_disabled { + self.schedule(now, SUPPRESS_AFTER_INIT_MS, false, trigger); + } + } + + /// `handleDelayedNarration`'s test: the trigger of a narration due now + /// (only changed elements are said), which is then no longer pending. + pub fn due(&mut self, now: i64) -> Option { + if now > self.next_time && now > self.suppress_time { + self.next_time = i64::MAX; + Some(self.next_trigger) + } else { + None + } + } + + /// `runNarration`: rebuilds the narration and returns what to say now + /// (`saySystemNow`), empty for nothing. + pub fn run( + &mut self, + only_changed: bool, + trigger: Trigger, + updater: impl FnOnce(&mut Output<'_>), + ) -> String { + self.collector.update(trigger, updater); + self.collector.collect(!only_changed) + } +} + +/// `Screen.findNarratableWidget` over the screen's active narratables, in +/// order, each with the caller's identity for it and its priority, and the +/// identity of the one narrated last: the first focused one that isn't the +/// last narrated, else the highest priority one (the first of equals), else +/// the last narrated focused one. The index and priority of the one found; +/// `None` when no narratable is focused or hovered. +pub fn find_narratable( + entries: &[(u64, Priority)], + last: Option, +) -> Option<(usize, Priority)> { + let mut result: Option<(usize, Priority)> = None; + let mut low_priority = None; + for (i, &(id, priority)) in entries.iter().enumerate() { + if priority.is_terminal() { + if Some(id) != last { + return Some((i, priority)); + } + low_priority = Some((i, priority)); + } else if priority > result.map_or(Priority::None, |(_, p)| p) { + result = Some((i, priority)); + } + } + result.or(low_priority) +} + +#[cfg(test)] +mod tests { + use super::*; + use ElementType::*; + + fn run( + n: &mut ScreenNarration, + only_changed: bool, + items: &[(ElementType, u32, &str)], + ) -> String { + n.run(only_changed, Trigger::System, |out| { + for &(kind, depth, text) in items { + let mut out = Output { + collector: out.collector, + depth, + trigger: Trigger::System, + }; + out.add(kind, text); + } + }) + } + + #[test] + fn elements_are_read_by_type_then_depth() { + let mut n = ScreenNarration::default(); + let said = n.run(false, Trigger::System, |out| { + out.add(Title, "Options"); + out.add(Position, "Screen element 2 out of 9"); + out.add(Usage, "Use Tab to navigate"); + let mut widget = out.nest(); + widget.add(Usage, "Press Enter to activate"); + widget.add(Title, "Button: Done"); + }); + assert_eq!( + said, + "Options. Button: Done. Screen element 2 out of 9. Use Tab to navigate. Press Enter to activate" + ); + } + + #[test] + fn only_what_changed_is_said_again() { + let mut n = ScreenNarration::default(); + let first = [(Title, 0, "Options"), (Title, 1, "Button: Done")]; + assert_eq!(run(&mut n, true, &first), "Options. Button: Done"); + assert_eq!(run(&mut n, true, &first), ""); + let moved = [ + (Title, 0, "Options"), + (Title, 1, "Button: Video Settings..."), + ]; + assert_eq!(run(&mut n, true, &moved), "Button: Video Settings..."); + // Forced: everything of this generation. + assert_eq!( + run(&mut n, false, &moved), + "Options. Button: Video Settings..." + ); + } + + #[test] + fn an_element_missing_for_a_generation_is_said_again_when_it_returns() { + let mut n = ScreenNarration::default(); + assert_eq!( + run(&mut n, true, &[(Title, 0, "Options"), (Usage, 0, "Tab")]), + "Options. Tab" + ); + assert_eq!(run(&mut n, true, &[(Title, 0, "Options")]), ""); + assert_eq!( + run(&mut n, true, &[(Title, 0, "Options"), (Usage, 0, "Tab")]), + "Tab" + ); + } + + #[test] + fn lines_are_one_element_read_as_sentences() { + let mut n = ScreenNarration::default(); + let said = n.run(false, Trigger::System, |out| { + out.add_lines(Title, vec!["First".into(), "".into(), "Third".into()]); + }); + assert_eq!(said, "First. . Third"); + } + + #[test] + fn narration_waits_for_its_delay_and_the_suppression() { + let mut n = ScreenNarration::default(); + assert_eq!(n.due(0), None); + n.after_init(1000, false); + n.after_mouse_move(1000); + // Due at 1750 but suppressed until 3000. + assert_eq!(n.due(2000), None); + assert_eq!(n.due(3001), Some(Trigger::Mouse)); + assert_eq!(n.due(4000), None); + // A click lifts the suppression. + n.after_init(5000, false); + n.after_mouse_action(5000); + assert_eq!(n.due(5200), None); + assert_eq!(n.due(5201), Some(Trigger::Mouse)); + } + + #[test] + fn a_screen_opened_by_keyboard_waits_for_the_next_key() { + let mut n = ScreenNarration::default(); + n.after_init(0, true); + n.after_mouse_move(0); + assert_eq!(n.due(1_000_000), None); + n.after_keyboard_action(10); + assert_eq!(n.due(211), Some(Trigger::Keyboard)); + } + + #[test] + fn turning_the_narrator_on_narrates_the_screen_two_seconds_later() { + let mut n = ScreenNarration::default(); + n.after_status_change(0, false, Trigger::Keyboard); + assert_eq!(n.due(10_000), None); + n.after_status_change(0, true, Trigger::Keyboard); + assert_eq!(n.due(2000), None); + assert_eq!(n.due(2001), Some(Trigger::Keyboard)); + } + + #[test] + fn the_focused_widget_wins_unless_it_was_just_narrated() { + use Priority::*; + let ids = |p: &[Priority]| { + p.iter() + .enumerate() + .map(|(i, &p)| (i as u64 + 10, p)) + .collect::>() + }; + assert_eq!( + find_narratable(&ids(&[None, None]), Option::None), + Option::None + ); + assert_eq!( + find_narratable(&ids(&[None, Hovered, Hovered]), Option::None), + Some((1, Hovered)) + ); + assert_eq!( + find_narratable(&ids(&[Hovered, Focused]), Option::None), + Some((1, Focused)) + ); + // The focused one was narrated last: a hovered one goes first. + assert_eq!( + find_narratable(&ids(&[Hovered, Focused]), Some(11)), + Some((0, Hovered)) + ); + assert_eq!( + find_narratable(&ids(&[None, Focused]), Some(11)), + Some((1, Focused)) + ); + assert_eq!( + find_narratable(&ids(&[Focused, Focused]), Some(10)), + Some((1, Focused)) + ); + } +} diff --git a/crates/horizon-narrator/src/speech.rs b/crates/horizon-narrator/src/speech.rs new file mode 100644 index 0000000000..409b79058d --- /dev/null +++ b/crates/horizon-narrator/src/speech.rs @@ -0,0 +1,120 @@ +//! `com.mojang.text2speech.Narrator`: the platform's speech, opened and driven +//! on a thread of its own so the game never waits on it. +use std::sync::mpsc::{self, Receiver, Sender}; + +#[cfg(target_os = "linux")] +mod linux; +#[cfg(target_os = "macos")] +mod mac; +#[cfg(windows)] +mod windows; + +/// A platform's speech (`NarratorWindows`, `NarratorMac`, `NarratorLinux`), +/// used only on the thread that opened it. +pub trait Speech { + /// `say`: `interrupt` drops what is queued and cuts off what is being said. + fn say(&mut self, message: &str, interrupt: bool, volume: f32); + /// `clear`: drops what is queued and stops speaking. + fn clear(&mut self); + /// Called while the thread waits for work (every 50 ms), for speech that + /// keeps its own queue. + fn idle(&mut self) {} +} + +/// `Narrator.getNarrator`: the platform's speech, or why there is none +/// (`Narrator.EMPTY`). +fn open() -> Result, String> { + #[cfg(windows)] + return windows::Sapi::open().map(|s| Box::new(s) as Box); + #[cfg(target_os = "macos")] + return mac::Synthesizer::open().map(|s| Box::new(s) as Box); + #[cfg(target_os = "linux")] + return linux::Flite::open().map(|s| Box::new(s) as Box); + #[allow(unreachable_code)] + Err(format!("Unsupported platform {}", std::env::consts::OS)) +} + +/// Speech that is there but makes no sound (`HORIZON_NO_AUDIO`). +struct Silent; +impl Speech for Silent { + fn say(&mut self, _: &str, _: bool, _: f32) {} + fn clear(&mut self) {} +} + +enum Command { + Say(String, bool, f32), + Clear, +} + +/// The game's handle on the speech thread. +pub(crate) struct Thread { + commands: Sender, + /// Whether the speech opened, until [`super::Sink::active`] first asks. + opened: Option>, + active: bool, +} + +impl Thread { + pub(crate) fn spawn(silent: bool) -> Thread { + let (commands, receive) = mpsc::channel(); + let (opened, ready) = mpsc::channel(); + let spawned = std::thread::Builder::new() + .name("Narrator".into()) + .spawn(move || run(silent, &opened, &receive)); + if let Err(e) = spawned { + log::error!("Error while loading the narrator: {e}"); + } + Thread { + commands, + opened: Some(ready), + active: false, + } + } +} + +fn run(silent: bool, opened: &Sender, commands: &Receiver) { + let mut speech: Box = if silent { + Box::new(Silent) + } else { + match open() { + Ok(speech) => speech, + Err(e) => { + log::error!("Error while loading the narrator: {e}"); + let _ = opened.send(false); + return; + } + } + }; + let _ = opened.send(true); + loop { + match commands.recv_timeout(std::time::Duration::from_millis(50)) { + Ok(Command::Say(message, interrupt, volume)) => speech.say(&message, interrupt, volume), + Ok(Command::Clear) => speech.clear(), + Err(mpsc::RecvTimeoutError::Timeout) => speech.idle(), + // `destroy`: the game dropped its handle. + Err(mpsc::RecvTimeoutError::Disconnected) => { + speech.clear(); + return; + } + } + } +} + +impl super::Sink for Thread { + fn say(&mut self, message: &str, interrupt: bool, volume: f32) { + let _ = self + .commands + .send(Command::Say(message.to_owned(), interrupt, volume)); + } + fn clear(&mut self) { + let _ = self.commands.send(Command::Clear); + } + /// Waits for the speech to open the first time it is asked (vanilla opens + /// it in `Minecraft`'s constructor, before anything asks). + fn active(&mut self) -> bool { + if let Some(opened) = self.opened.take() { + self.active = opened.recv().unwrap_or(false); + } + self.active + } +} diff --git a/crates/horizon-narrator/src/speech/linux.rs b/crates/horizon-narrator/src/speech/linux.rs new file mode 100644 index 0000000000..868b775d9a --- /dev/null +++ b/crates/horizon-narrator/src/speech/linux.rs @@ -0,0 +1,14 @@ +//! `NarratorLinux`: flite's `cmu_us_kal16` voice. + +pub(super) struct Flite; + +impl Flite { + pub(super) fn open() -> Result { + Err("flite is not ported yet".into()) + } +} + +impl super::Speech for Flite { + fn say(&mut self, _: &str, _: bool, _: f32) {} + fn clear(&mut self) {} +} diff --git a/crates/horizon-narrator/src/speech/mac.rs b/crates/horizon-narrator/src/speech/mac.rs new file mode 100644 index 0000000000..75ccbb81b1 --- /dev/null +++ b/crates/horizon-narrator/src/speech/mac.rs @@ -0,0 +1,14 @@ +//! `NarratorMac`: AppKit's `NSSpeechSynthesizer`. + +pub(super) struct Synthesizer; + +impl Synthesizer { + pub(super) fn open() -> Result { + Err("NSSpeechSynthesizer is not ported yet".into()) + } +} + +impl super::Speech for Synthesizer { + fn say(&mut self, _: &str, _: bool, _: f32) {} + fn clear(&mut self) {} +} diff --git a/crates/horizon-narrator/src/speech/windows.rs b/crates/horizon-narrator/src/speech/windows.rs new file mode 100644 index 0000000000..f40058ca00 --- /dev/null +++ b/crates/horizon-narrator/src/speech/windows.rs @@ -0,0 +1,83 @@ +//! `NarratorWindows`: SAPI's `SpVoice`, the voice and rate chosen in Windows' +//! speech settings. +use windows::Win32::Media::Speech::{ + ISpVoice, SPF_ASYNC, SPF_IS_NOT_XML, SPF_PURGEBEFORESPEAK, SpVoice, +}; +use windows::Win32::System::Com::{ + CLSCTX_INPROC_HANDLER, CLSCTX_INPROC_SERVER, CLSCTX_LOCAL_SERVER, COINIT_MULTITHREADED, + CoCreateInstance, CoInitializeEx, CoUninitialize, +}; +use windows::core::{HSTRING, w}; + +use std::mem::ManuallyDrop; + +pub(super) struct Sapi { + /// Released before `CoUninitialize` (`drop`). + voice: ManuallyDrop, +} + +impl Sapi { + /// `initSAPI`: `CoCreateInstance(SP_VOICE)` with the in-process and local + /// server contexts (7). The speech thread is its own multithreaded + /// apartment: no window messages reach it, and asynchronous speech needs + /// none. + pub(super) fn open() -> Result { + // SAFETY: COM is initialised on this thread before any COM call, and + // released in `drop` on the same thread. + unsafe { + CoInitializeEx(None, COINIT_MULTITHREADED) + .ok() + .map_err(|e| format!("CoInitializeEx returned {e}"))?; + match CoCreateInstance( + &SpVoice, + None, + CLSCTX_INPROC_SERVER | CLSCTX_INPROC_HANDLER | CLSCTX_LOCAL_SERVER, + ) { + Ok(voice) => Ok(Sapi { + voice: ManuallyDrop::new(voice), + }), + Err(e) => { + CoUninitialize(); + Err(format!("SP_VOICE returned code {e}")) + } + } + } + } +} + +impl super::Speech for Sapi { + /// `Speak` asynchronously as plain text, purging what is queued when + /// interrupting; the volume is set first, as `(short)(volume * 100.0F)`. + fn say(&mut self, message: &str, interrupt: bool, volume: f32) { + let mut flags = (SPF_ASYNC.0 | SPF_IS_NOT_XML.0) as u32; + if interrupt { + flags |= SPF_PURGEBEFORESPEAK.0 as u32; + } + // SAFETY: `voice` is a live SpVoice of this thread's apartment. + unsafe { + let _ = self.voice.SetVolume((volume * 100.0) as i16 as u16); + if let Err(e) = self.voice.Speak(&HSTRING::from(message), flags, None) { + log::warn!("Narrator: Speak failed: {e}"); + } + } + } + /// `Skip("Sentence", Integer.MAX_VALUE)`: skips what is left of the + /// queue. + fn clear(&mut self) { + let mut skipped = 0; + // SAFETY: as in `say`; `skipped` outlives the call. + unsafe { + let _ = self.voice.Skip(w!("Sentence"), i32::MAX, &mut skipped); + } + } +} + +impl Drop for Sapi { + fn drop(&mut self) { + // SAFETY: the voice is released before COM is, on its own thread. + unsafe { + ManuallyDrop::drop(&mut self.voice); + CoUninitialize(); + } + } +}