//! Starting and stopping the services, and the dashboard snapshot (`app/src/admin.spec.md`, //! "Service control" and "Dashboard data"). use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use async_trait::async_trait; use serde_json::{json, Value}; use tokio::sync::mpsc; use crate::dashboard::Dashboard; use crate::discord::CoreEvent; use crate::system::Usage; #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] #[serde(rename_all = "lowercase")] pub enum State { Stopped, Starting, Running, Stopping, Failed, } /// The running services as the controller sees them. #[async_trait] pub trait Running: Send + Sync + 'static { /// Stop everything; an error means something may still run (the LLM's server still answers). async fn stop(&self) -> Result<(), String>; /// Worker, voice channel members and missed requests, for the snapshot. fn status(&self, now: Instant) -> Value; } /// What starts the services (the real ones, or a fake in tests). #[async_trait] pub trait Host: Send + Sync + 'static { async fn start(&self) -> Result<(Arc, mpsc::UnboundedReceiver), String>; /// Stored secrets, masked in error messages. fn secrets(&self) -> Vec; /// Called when a start is accepted (state `starting`, under the controller's lock): a fresh /// cancel, so a stop from now on cancels this start and an earlier stop does not. fn prepare_start(&self) {} /// Stop a start in progress (the LLM's first start downloads the model); `start` then returns /// `Err("cancelled")` if it had not finished. fn cancel_start(&self) {} /// The LLM's state for the dashboard, also while starting. fn llm(&self) -> Value { Value::Null } /// After a failed run (a start or a stop that could not clear everything): stop what may be left. async fn cleanup(&self) -> Result<(), String> { Ok(()) } } struct Inner { state: State, error: Option, running: Option>, /// The running services' dashboard and the task feeding it their events. Each start gets a new /// one, so an event of an earlier run can never reach a later dashboard. dashboard: Arc>, feeder: Option>, /// Stop was asked for while starting. stop_asked: bool, } pub struct Controller { host: Box, inner: Mutex, usage: tokio::sync::Mutex>, } impl Controller { pub fn new(host: impl Host) -> Arc { Arc::new(Self { host: Box::new(host), inner: Mutex::new(Inner { state: State::Stopped, error: None, running: None, dashboard: Arc::default(), feeder: None, stop_asked: false }), usage: tokio::sync::Mutex::new(None) }) } pub fn state(&self) -> State { self.inner.lock().unwrap().state } /// Start from stopped or failed; a failed start leaves nothing running. pub async fn start(&self) -> Result<(), String> { self.begin_start()?; self.run_start().await } /// The same in the background: the state is `starting` when this returns, so a stop right after /// it cancels the start and a second start is refused. pub fn start_in_background(self: &Arc) -> Result<(), String> { self.begin_start()?; let me = self.clone(); tokio::spawn(async move { let _ = me.run_start().await; }); Ok(()) } fn begin_start(&self) -> Result<(), String> { let mut i = self.inner.lock().unwrap(); if !matches!(i.state, State::Stopped | State::Failed) { return Err(format!("cannot start while {:?}", i.state).to_lowercase()); } i.state = State::Starting; i.error = None; i.stop_asked = false; self.host.prepare_start(); Ok(()) } async fn run_start(&self) -> Result<(), String> { crate::info_log!("app", "starting the services"); let started = self.host.start().await; // The stop flag is read under the same lock that makes the state Running, so a stop asked // for while starting is never lost. let started = { let mut i = self.inner.lock().unwrap(); if std::mem::take(&mut i.stop_asked) { i.state = State::Stopping; started } else { return match started { Ok((running, mut events)) => { let dash: Arc> = Arc::default(); let d2 = dash.clone(); let feeder = tokio::spawn(async move { while let Some(ev) = events.recv().await { d2.lock().unwrap().on_event(&ev, Instant::now()); } }); i.state = State::Running; i.running = Some(running); i.dashboard = dash; i.feeder = Some(feeder); Ok(()) } Err(e) => { let e = crate::logs::redact(&e, &self.host.secrets()); crate::error_log!("app", "the services did not start: {e}"); i.state = State::Failed; i.error = Some(e.clone()); Err(e) } }; } }; // Stopped while starting: whatever came up goes down again. If that fails, or the cancelled // start could not clear the LLM's server, the state says so. let left = match started { Ok((running, _)) => running.stop().await.err(), Err(e) if self.host.llm()["state"] == "failed" => Some(e), Err(_) => None, }; let mut i = self.inner.lock().unwrap(); if let Some(e) = left { let e = crate::logs::redact(&e, &self.host.secrets()); crate::error_log!("app", "start cancelled, but: {e}"); i.state = State::Failed; i.error = Some(e); } else { i.state = State::Stopped; crate::info_log!("app", "start cancelled"); } Err("cancelled".into()) } /// Stop from running, or cancel a start; what was said is forgotten. pub async fn stop(&self) -> Result<(), String> { let running = { let mut i = self.inner.lock().unwrap(); if i.state == State::Starting { i.stop_asked = true; drop(i); self.host.cancel_start(); return Ok(()); } if i.state != State::Running { return Err(format!("cannot stop while {:?}", i.state).to_lowercase()); } i.state = State::Stopping; i.running.take() }; let stopped = match running { Some(r) => r.stop().await, None => Ok(()), }; // Stop feeding before forgetting: nothing still queued may reach the screen afterwards. let feeder = self.inner.lock().unwrap().feeder.take(); if let Some(f) = feeder { f.abort(); let _ = f.await; } let mut i = self.inner.lock().unwrap(); i.dashboard = Arc::default(); match stopped { Ok(()) => { i.state = State::Stopped; drop(i); crate::info_log!("app", "services stopped"); Ok(()) } // Something may still run: shown as failed with the reason; a start (Strata's // `--restart`) or another stop attempt can follow. Err(e) => { let e = crate::logs::redact(&e, &self.host.secrets()); i.state = State::Failed; i.error = Some(e.clone()); drop(i); crate::error_log!("app", "the services did not stop cleanly: {e}"); Err(e) } } } /// The app is closing: stop what runs (a start in progress is cancelled and waited for, up to /// 150 s), then, if the state is failed, clear what may be left. Ok means nothing is left running. pub async fn shutdown(&self) -> Result<(), String> { if matches!(self.state(), State::Running | State::Starting) { let _ = self.stop().await; } let until = Instant::now() + Duration::from_secs(150); while matches!(self.state(), State::Starting | State::Stopping) { if Instant::now() >= until { return Err("서비스가 아직 멈추지 않았습니다".into()); } tokio::time::sleep(Duration::from_millis(100)).await; } if self.state() != State::Failed { return Ok(()); } match self.host.cleanup().await { Ok(()) => { let mut i = self.inner.lock().unwrap(); i.state = State::Stopped; i.error = None; drop(i); crate::info_log!("app", "cleared what the failed run left"); Ok(()) } Err(e) => { let e = crate::logs::redact(&e, &self.host.secrets()); crate::error_log!("app", "could not clear what the failed run left: {e}"); self.inner.lock().unwrap().error = Some(e.clone()); Err(e) } } } pub async fn restart(&self) -> Result<(), String> { if self.state() == State::Running { self.stop().await?; } self.start().await } /// Everything the dashboard shows. pub async fn snapshot(&self) -> Value { let now = Instant::now(); let (state, error, running, dashboard) = { let i = self.inner.lock().unwrap(); (i.state, i.error.clone(), i.running.clone(), i.dashboard.clone()) }; let status = running.map(|r| r.status(now)).unwrap_or_else(|| json!({"worker": null, "voiceMembers": [], "missed": []})); let mut view = dashboard.lock().unwrap().view(&[], now); view["missed"] = status["missed"].clone(); view["worker"] = status["worker"].clone(); view["voiceMembers"] = status["voiceMembers"].clone(); view["browser"] = status.get("browser").cloned().unwrap_or_else(|| json!({"state": "off", "note": null})); view["state"] = json!(state); view["error"] = json!(error); view["system"] = json!(self.system().await); view["llm"] = self.host.llm(); view["game"] = status.get("game").cloned().unwrap_or(Value::Null); view["performance"] = Value::Null; // the performance stages come next in M7 view } /// Memory and GPU use, sampled at most once a second. async fn system(&self) -> Usage { let mut cache = self.usage.lock().await; if let Some((at, u)) = cache.as_ref() { if at.elapsed() < Duration::from_secs(1) { return u.clone(); } } let u = crate::system::usage().await; *cache = Some((Instant::now(), u.clone())); u } } /// The real services from a data folder. pub struct ServicesHost { data_dir: PathBuf, llm: Arc, } impl ServicesHost { pub fn new(data_dir: PathBuf) -> Self { let llm = Arc::new(crate::strata::Llm::new(&data_dir)); Self { data_dir, llm } } } #[async_trait] impl Host for ServicesHost { async fn start(&self) -> Result<(Arc, mpsc::UnboundedReceiver), String> { let mut s = crate::services::start_services(&self.data_dir, self.llm.clone()).await.map_err(|e| e.to_string())?; let (tx, rx) = mpsc::unbounded_channel(); let mut events = std::mem::replace(&mut s.events, mpsc::unbounded_channel().1); tokio::spawn(async move { while let Some(e) = events.recv().await { if tx.send(e).is_err() { break; } } }); Ok((Arc::new(s), rx)) } fn secrets(&self) -> Vec { crate::secrets::Secrets::new(&self.data_dir.join("config").join("secrets.dat")).values() } fn prepare_start(&self) { self.llm.arm(); } fn cancel_start(&self) { self.llm.cancel(); } fn llm(&self) -> Value { json!(self.llm.status()) } async fn cleanup(&self) -> Result<(), String> { if self.llm.status().state == crate::strata::LlmState::Failed { self.llm.stop().await } else { Ok(()) } } } #[async_trait] impl Running for crate::services::Services { async fn stop(&self) -> Result<(), String> { crate::services::Services::stop(self).await } fn status(&self, now: Instant) -> Value { let hello = self.worker.hello(); let missed: Vec<_> = self.missed.lock().unwrap().iter().cloned().collect(); json!({ "worker": {"connected": hello.is_some(), "sttReady": hello.as_ref().map(|h| h.stt_ready), "ttsReady": hello.as_ref().map(|h| h.tts_ready), "restarts": self.worker_process.restarts()}, "voiceMembers": self.core.voice_members().into_iter() .map(|(id, name)| json!({"id": id.to_string(), "name": name})).collect::>(), "missed": crate::dashboard::missed_view(&missed, now), "browser": self.browser.view(), "game": self.game.view(), }) } } #[cfg(test)] mod tests { use super::*; use std::sync::atomic::{AtomicU32, Ordering}; #[derive(Default)] struct Counts { starts: AtomicU32, stops: AtomicU32, stop_fails: std::sync::atomic::AtomicBool, cleanups: AtomicU32, cleanup_fails: std::sync::atomic::AtomicBool, } struct FakeRunning(Arc); #[async_trait] impl Running for FakeRunning { async fn stop(&self) -> Result<(), String> { self.0.stops.fetch_add(1, Ordering::SeqCst); if self.0.stop_fails.load(Ordering::SeqCst) { Err("Strata를 끄지 못했습니다: still answering".into()) } else { Ok(()) } } fn status(&self, _: Instant) -> Value { json!({"worker": {"connected": true}, "voiceMembers": [{"id": "11", "name": "민수"}], "missed": []}) } } struct FakeHost { counts: Arc, fail_first: AtomicU32, events: Mutex>>, } #[async_trait] impl Host for FakeHost { async fn start(&self) -> Result<(Arc, mpsc::UnboundedReceiver), String> { self.counts.starts.fetch_add(1, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(20)).await; if self.fail_first.load(Ordering::SeqCst) > 0 { self.fail_first.fetch_sub(1, Ordering::SeqCst); return Err("login failed for token Mt0kenSecretValue".into()); } let (tx, rx) = mpsc::unbounded_channel(); *self.events.lock().unwrap() = Some(tx); Ok((Arc::new(FakeRunning(self.counts.clone())), rx)) } fn secrets(&self) -> Vec { vec!["Mt0kenSecretValue".into()] } } fn controller(fail_first: u32) -> (Arc, Arc, Arc) { let counts = Arc::new(Counts::default()); let host = Arc::new(FakeHost { counts: counts.clone(), fail_first: AtomicU32::new(fail_first), events: Mutex::new(None) }); struct Shared(Arc); #[async_trait] impl Host for Shared { async fn start(&self) -> Result<(Arc, mpsc::UnboundedReceiver), String> { self.0.start().await } fn secrets(&self) -> Vec { self.0.secrets() } async fn cleanup(&self) -> Result<(), String> { self.0.counts.cleanups.fetch_add(1, Ordering::SeqCst); if self.0.counts.cleanup_fails.load(Ordering::SeqCst) { Err("Strata를 끄지 못했습니다".into()) } else { Ok(()) } } } (Controller::new(Shared(host.clone())), counts, host) } /// A start that takes a while (the LLM's first start) and honours a cancel or ignores it. struct SlowHost { counts: Arc, honours_cancel: bool, cancelled: Arc, } #[async_trait] impl Host for SlowHost { async fn start(&self) -> Result<(Arc, mpsc::UnboundedReceiver), String> { self.counts.starts.fetch_add(1, Ordering::SeqCst); for _ in 0..50 { if self.honours_cancel && self.cancelled.load(Ordering::SeqCst) { return Err("cancelled".into()); } tokio::time::sleep(Duration::from_millis(10)).await; } Ok((Arc::new(FakeRunning(self.counts.clone())), mpsc::unbounded_channel().1)) } fn secrets(&self) -> Vec { vec![] } fn prepare_start(&self) { self.cancelled.store(false, Ordering::SeqCst); // as the LLM does: armed when a start is accepted } fn cancel_start(&self) { self.cancelled.store(true, Ordering::SeqCst); } fn llm(&self) -> Value { json!({"state": "starting", "percent": 12.5}) } } #[tokio::test] async fn a_stop_while_starting_cancels_the_start_or_stops_what_came_up() { for honours_cancel in [true, false] { let counts = Arc::new(Counts::default()); let c = Controller::new(SlowHost { counts: counts.clone(), honours_cancel, cancelled: Arc::default() }); let c2 = c.clone(); let start = tokio::spawn(async move { c2.start().await }); tokio::time::sleep(Duration::from_millis(50)).await; assert_eq!(c.snapshot().await["llm"]["percent"], json!(12.5), "progress shows while starting"); c.stop().await.unwrap(); assert_eq!(start.await.unwrap().unwrap_err(), "cancelled"); let s = c.snapshot().await; assert_eq!((s["state"].clone(), s["error"].clone()), (json!("stopped"), Value::Null)); // What came up anyway was stopped; a cancelled start left nothing to stop. assert_eq!(counts.stops.load(Ordering::SeqCst), if honours_cancel { 0 } else { 1 }); c.start().await.unwrap(); assert_eq!(c.state(), State::Running, "it can start again"); } } #[tokio::test] async fn a_background_start_is_starting_at_once_so_an_immediate_stop_or_second_start_is_handled() { let counts = Arc::new(Counts::default()); let c = Controller::new(SlowHost { counts: counts.clone(), honours_cancel: true, cancelled: Arc::default() }); c.start_in_background().unwrap(); assert_eq!(c.state(), State::Starting, "starting before anything else runs"); assert!(c.start_in_background().unwrap_err().contains("starting"), "no second start"); c.stop().await.unwrap(); for _ in 0..100 { if c.state() == State::Stopped { break; } tokio::time::sleep(Duration::from_millis(10)).await; } assert_eq!(c.state(), State::Stopped); assert_eq!(counts.starts.load(Ordering::SeqCst), 1); } #[tokio::test] async fn a_stop_that_fails_is_reported_and_a_new_start_is_allowed() { let (c, counts, _) = controller(0); c.start().await.unwrap(); counts.stop_fails.store(true, Ordering::SeqCst); let e = c.stop().await.unwrap_err(); assert!(e.contains("Strata"), "{e}"); let s = c.snapshot().await; assert_eq!((s["state"].clone(), s["error"].clone()), (json!("failed"), json!(e))); counts.stop_fails.store(false, Ordering::SeqCst); c.start().await.unwrap(); assert_eq!(c.state(), State::Running); } #[tokio::test] async fn closing_the_app_after_a_failed_stop_clears_what_is_left_or_says_it_could_not() { let (c, counts, _) = controller(0); c.shutdown().await.unwrap(); // nothing ran: nothing to do assert_eq!(counts.cleanups.load(Ordering::SeqCst), 0); c.start().await.unwrap(); counts.stop_fails.store(true, Ordering::SeqCst); counts.cleanup_fails.store(true, Ordering::SeqCst); assert!(c.shutdown().await.is_err(), "the stop and the clean-up both failed"); assert_eq!(c.state(), State::Failed); counts.cleanup_fails.store(false, Ordering::SeqCst); c.shutdown().await.unwrap(); let s = c.snapshot().await; assert_eq!((s["state"].clone(), s["error"].clone()), (json!("stopped"), Value::Null)); assert_eq!(counts.cleanups.load(Ordering::SeqCst), 2); } #[tokio::test] async fn start_and_stop_follow_the_state_rules() { let (c, counts, _) = controller(0); assert!(c.stop().await.is_err(), "nothing to stop"); let c2 = c.clone(); let first = tokio::spawn(async move { c2.start().await }); tokio::time::sleep(Duration::from_millis(5)).await; assert_eq!(c.state(), State::Starting); assert!(c.start().await.unwrap_err().contains("starting"), "one start at a time"); first.await.unwrap().unwrap(); assert_eq!(c.state(), State::Running); assert!(c.start().await.is_err()); c.restart().await.unwrap(); assert_eq!((counts.starts.load(Ordering::SeqCst), counts.stops.load(Ordering::SeqCst)), (2, 1)); c.stop().await.unwrap(); assert_eq!(c.state(), State::Stopped); } #[tokio::test] async fn a_failed_start_is_reported_masked_and_can_be_retried() { let (c, counts, _) = controller(1); let e = c.start().await.unwrap_err(); assert_eq!(e, "login failed for token [비밀값]"); let snap = c.snapshot().await; assert_eq!((snap["state"].clone(), snap["error"].clone()), (json!("failed"), json!(e))); c.start().await.unwrap(); assert_eq!(c.state(), State::Running); assert_eq!(counts.stops.load(Ordering::SeqCst), 0, "a failed start leaves nothing to stop"); } #[tokio::test] async fn events_queued_before_a_stop_or_sent_by_an_earlier_run_never_reach_the_screen() { let (c, _, host) = controller(0); c.start().await.unwrap(); let old = host.events.lock().unwrap().clone().unwrap(); // Queued right before the stop, never handled before it. old.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap(); old.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) }, "자비스".into())).unwrap(); c.stop().await.unwrap(); tokio::time::sleep(Duration::from_millis(30)).await; let s = c.snapshot().await; assert_eq!((s["state"].clone(), s["voiceChannel"].clone(), s["captions"].clone()), (json!("stopped"), Value::Null, json!([]))); // After a new start, the earlier run's sender still reaches nothing. c.start().await.unwrap(); let _ = old.send(CoreEvent::Joined { guild_id: 1, channel_id: 78, session: 2 }); tokio::time::sleep(Duration::from_millis(30)).await; assert_eq!(c.snapshot().await["voiceChannel"], Value::Null); } #[tokio::test] async fn the_snapshot_carries_the_events_and_stopping_forgets_what_was_said() { let (c, _, host) = controller(0); c.start().await.unwrap(); let tx = host.events.lock().unwrap().clone().unwrap(); tx.send(CoreEvent::Ready { user_id: 1, tag: "Jarvis#0001".into() }).unwrap(); tx.send(CoreEvent::Joined { guild_id: 1, channel_id: 77, session: 1 }).unwrap(); tx.send(CoreEvent::Partial(crate::discord::Heard { speaker_id: 11, speaker_name: "민수".into(), utt: 1, session: Some(1) }, "자비스 안녕".into())).unwrap(); tokio::time::sleep(Duration::from_millis(50)).await; let s = c.snapshot().await; assert_eq!(s["state"], json!("running")); assert_eq!(s["discord"], json!("Jarvis#0001")); assert_eq!(s["voiceChannel"], json!("77")); assert_eq!(s["captions"][0]["text"], json!("자비스 안녕")); assert_eq!(s["voiceMembers"][0]["name"], json!("민수")); assert_eq!(s["worker"]["connected"], json!(true)); assert!(s["system"]["ramTotalMb"].is_u64() || !cfg!(any(target_os = "linux", windows))); assert_eq!(s["performance"], Value::Null); c.stop().await.unwrap(); let s = c.snapshot().await; assert_eq!(s["captions"], json!([])); assert_eq!(s["worker"], Value::Null); } }