- the normal stage reads other programs' memory afresh (the card after the LLM stopped, less the STT worker's model) and keeps it; the game stage counts from the latest normal reading, so the game is not counted against its own reserve - a restart that fails brings back the stage that ran before and tells the brain ready; the failed stage is tried again after 60 s, three times at most; when nothing comes back the brain is told paused so requests fail at once - the worker client keeps the latest llm state and sends it first on every connection - switches are numbered: llm switching carries an id, llm_idle names it, and a late idle of an earlier switch no longer lets a new one through - a request that waited 180 s for a switch fails without calling the brain - the dashboard shows the stage that runs also on a fallback placement without experts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
436 lines
20 KiB
Rust
436 lines
20 KiB
Rust
//! Starts everything the app runs: the LLM (`strata.rs`), the voice worker process, the connection
|
|
//! to it, and the Discord core. Used by the app binary (and the Tauri shell from M5).
|
|
|
|
use std::path::Path;
|
|
use std::sync::Arc;
|
|
|
|
use serde_json::Value;
|
|
use tokio::sync::mpsc;
|
|
|
|
use crate::conversation::{self, Conversation};
|
|
use crate::discord::{CoreEvent, DiscordCore};
|
|
use crate::settings::load_settings;
|
|
use crate::strata::Llm;
|
|
use crate::worker_client::{WorkerClient, WorkerClientOptions};
|
|
use crate::worker_process::{ProcessEvent, WorkerProcess, WorkerProcessOptions};
|
|
|
|
pub struct Services {
|
|
pub core: DiscordCore,
|
|
pub events: mpsc::UnboundedReceiver<CoreEvent>,
|
|
pub worker: WorkerClient,
|
|
pub worker_process: WorkerProcess,
|
|
/// The conversation's missed requests, for the dashboard.
|
|
pub missed: conversation::MissedLog,
|
|
pub llm: Arc<Llm>,
|
|
pub browser: BrowserRun,
|
|
/// Game detection while the services run (admin.spec.md, "Game detection").
|
|
pub game: Arc<crate::game::GameWatch>,
|
|
/// Moves the LLM between the performance stages as games come and go.
|
|
pub perf: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
|
}
|
|
|
|
/// The bot browser of a run (admin.spec.md, "Bot browser").
|
|
pub enum BrowserRun {
|
|
/// Not started: the feature is off, or (with a note) it is not installed.
|
|
Off(Option<String>),
|
|
Failed(String),
|
|
/// Running; the note says why it runs without the virtual monitor that was asked for.
|
|
Running(Arc<crate::browser::BotBrowser>, Option<String>),
|
|
}
|
|
|
|
impl BrowserRun {
|
|
/// `{state, note}` for the dashboard.
|
|
pub fn view(&self) -> Value {
|
|
let (state, note) = match self {
|
|
BrowserRun::Off(note) => ("off", note.clone()),
|
|
BrowserRun::Failed(e) => ("failed", Some(e.clone())),
|
|
BrowserRun::Running(b, note) if b.running() => ("ready", note.clone()),
|
|
BrowserRun::Running(_, note) => ("exited", note.clone()),
|
|
};
|
|
serde_json::json!({"state": state, "note": note})
|
|
}
|
|
|
|
/// Stop the browser, then take the virtual monitor off the desktop (its window first, so
|
|
/// Windows never moves it onto a real monitor).
|
|
pub async fn stop(&self) {
|
|
if let BrowserRun::Running(b, _) = self {
|
|
b.stop().await;
|
|
crate::vdisplay::detach();
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Services {
|
|
/// Stop everything; an error means the LLM's server may still run.
|
|
pub async fn stop(&self) -> Result<(), String> {
|
|
let perf = self.perf.lock().unwrap().take();
|
|
if let Some(p) = perf {
|
|
p.abort();
|
|
let _ = p.await;
|
|
}
|
|
self.game.stop();
|
|
self.core.close().await;
|
|
self.worker.close();
|
|
self.worker_process.stop().await;
|
|
self.browser.stop().await;
|
|
self.llm.stop().await
|
|
}
|
|
}
|
|
|
|
/// Start the bot browser when its feature is on and it is installed, on the virtual monitor when
|
|
/// sharing is on; a failure leaves the services running without it (or without the monitor).
|
|
async fn start_browser(data_dir: &Path, settings: &crate::settings::Settings) -> BrowserRun {
|
|
// A monitor an earlier run left on goes off first.
|
|
crate::vdisplay::detach();
|
|
if !settings.features.browser {
|
|
return BrowserRun::Off(None);
|
|
}
|
|
let layout = crate::install::Layout { data: data_dir.to_path_buf(), src: std::path::PathBuf::new() };
|
|
if !crate::install::Installer::new(layout.clone(), crate::install::steps()).done("browser") {
|
|
crate::warn_log!("browser", "the bot browser is not installed");
|
|
return BrowserRun::Off(Some("설치되지 않음".into()));
|
|
}
|
|
let (exe, profile) = (layout.chrome_exe(), crate::browser::profile_dir(data_dir));
|
|
let mut note = None;
|
|
if settings.features.browser_share {
|
|
let started = match crate::vdisplay::attach() {
|
|
Ok(r) => crate::browser::BotBrowser::start(&exe, &profile, Some(r)).await,
|
|
Err(e) => Err(e),
|
|
};
|
|
match started {
|
|
Ok(b) => return BrowserRun::Running(Arc::new(b), None),
|
|
Err(e) => {
|
|
crate::vdisplay::detach();
|
|
crate::warn_log!("browser", "Warning: no virtual monitor for the bot browser: {e}");
|
|
note = Some(format!("가상 모니터 없이 실행: {e}"));
|
|
}
|
|
}
|
|
}
|
|
match crate::browser::BotBrowser::start(&exe, &profile, None).await {
|
|
Ok(b) => BrowserRun::Running(Arc::new(b), note),
|
|
Err(e) => {
|
|
crate::warn_log!("browser", "Warning: the bot browser did not start: {e}");
|
|
BrowserRun::Failed(e)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Start the services; the LLM first (its first start downloads the model), so nothing answers
|
|
/// before the brain can. A failure stops what already started.
|
|
pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services, Box<dyn std::error::Error + Send + Sync>> {
|
|
let settings = load_settings(&data_dir.join("config").join("settings.json"))?;
|
|
let game_settings = settings.game.clone();
|
|
let perf_settings = settings.clone();
|
|
// The stored token (the setup wizard keeps it encrypted, admin.spec.md), else the environment.
|
|
let secrets = crate::secrets::Secrets::new(&data_dir.join("config").join("secrets.dat"));
|
|
let stored = secrets.get(crate::secrets::DISCORD_TOKEN)?;
|
|
// The MCP servers' environment values stay encrypted on disk: they reach the brain only in the
|
|
// worker's environment (admin.spec.md, "Features and MCP servers").
|
|
let mcp_env = crate::features::mcp_env(&secrets, &settings.mcp_servers)?;
|
|
let token = match stored {
|
|
Some(t) => t,
|
|
None => std::env::var("JARVIS_DISCORD_TOKEN").map_err(|_| "no bot token: set it in the app or JARVIS_DISCORD_TOKEN")?,
|
|
};
|
|
|
|
// What these services are given stays masked in the logs until the next start, also if it is
|
|
// replaced or removed in the store meanwhile (admin.spec.md, "Logs").
|
|
if let Some(hub) = crate::logs::installed() {
|
|
hub.set_secrets_in_use(secrets_in_use(&token, &mcp_env));
|
|
}
|
|
|
|
let (base_url, model) = llm.start(&settings).await?;
|
|
let brain_config = match crate::strata::write_brain_config(data_dir, &base_url, &model, &settings) {
|
|
Ok(f) => f,
|
|
Err(e) => return Err(after_llm_stop(&llm, format!("could not write the brain settings: {e}")).await.into()),
|
|
};
|
|
let browser = start_browser(data_dir, &settings).await;
|
|
|
|
let w = &settings.voice_worker;
|
|
let mut wopts = WorkerProcessOptions::new(&w.python, &w.cwd, settings.worker_args());
|
|
// With the installed runtime, the worker's models, caches and temp files stay in the data folder.
|
|
let layout = crate::install::Layout { data: data_dir.to_path_buf(), src: w.cwd.clone() };
|
|
if crate::install::Installer::new(layout.clone(), crate::install::steps()).done("settings") {
|
|
wopts.env.extend(layout.env());
|
|
}
|
|
wopts.env.insert("JARVIS_CONFIG_PATH".into(), brain_config.to_string_lossy().into_owned());
|
|
wopts.env.insert(crate::features::MCP_ENV_VAR.into(), mcp_env.to_string());
|
|
if let BrowserRun::Running(b, _) = &browser {
|
|
wopts.env.insert("JARVIS_BROWSER_CDP".into(), b.cdp_url());
|
|
}
|
|
// Background collection (admin.spec.md): the enabled targets, and the keep days in any case.
|
|
match crate::collect::write_config(data_dir, &settings) {
|
|
Ok(f) => { wopts.env.insert(crate::collect::COLLECT_ENV_VAR.into(), f.to_string_lossy().into_owned()); }
|
|
Err(e) => return Err(after_stop(&llm, &browser, format!("could not write the collection settings: {e}")).await.into()),
|
|
}
|
|
let worker_process = match WorkerProcess::new(wopts) {
|
|
Ok(p) => p,
|
|
Err(e) => return Err(after_stop(&llm, &browser, e.to_string()).await.into()),
|
|
};
|
|
let mut process_events = worker_process.subscribe();
|
|
worker_process.start();
|
|
let mut opts = WorkerClientOptions::new(worker_process.url(), worker_process.token.clone());
|
|
opts.stt = settings.voice.stt.clone();
|
|
opts.stt.insert("hotwords".into(), Value::from(settings.voice.wake_words.clone()));
|
|
opts.memory = settings.memory.clone();
|
|
let (worker, worker_events) = WorkerClient::new(opts);
|
|
// The worker loads its models before it listens: connect once it says READY (after restarts the
|
|
// client keeps retrying on its own).
|
|
let w2 = worker.clone();
|
|
tokio::spawn(async move {
|
|
loop {
|
|
match process_events.recv().await {
|
|
Ok(ProcessEvent::Ready) => return w2.start(),
|
|
Ok(_) => {}
|
|
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {}
|
|
Err(_) => return,
|
|
}
|
|
}
|
|
});
|
|
|
|
let (calls, response) = (settings.call.clone(), settings.response.clone());
|
|
let chat_calls = conversation::ChatCalls::from_settings(&settings);
|
|
let (core, core_events) = DiscordCore::new(settings, token, worker.clone(), worker_events);
|
|
// Call detection and the response policy answer what is heard; every event still reaches `events`.
|
|
let (out, events) = mpsc::unbounded_channel();
|
|
let conv = Conversation::new(core.clone(), calls, response).with_chat(chat_calls);
|
|
let missed = conv.missed_handle();
|
|
tokio::spawn(conversation::run(conv, core_events, out));
|
|
if let Err(e) = core.start().await {
|
|
worker.close();
|
|
worker_process.stop().await;
|
|
return Err(after_stop(&llm, &browser, e.to_string()).await.into());
|
|
}
|
|
crate::info_log!("app", "services started");
|
|
let game = Arc::new(crate::game::GameWatch::start(game_settings));
|
|
let on = { let g = game.clone(); Arc::new(move || g.on()) };
|
|
let perf = std::sync::Mutex::new(Some(tokio::spawn(perf_loop(perf_settings, llm.clone(), Arc::new(worker.clone()), on, RETRY_AFTER))));
|
|
Ok(Services { core, events, worker, worker_process, missed, llm, browser, game, perf })
|
|
}
|
|
|
|
/// The brain as the performance stages see it: told the LLM's state, and asked to finish what it is
|
|
/// answering (the worker client; a fake in tests).
|
|
#[async_trait::async_trait]
|
|
pub trait LlmBrain: Send + Sync {
|
|
/// Announce a switch; its number for `wait_llm_idle`.
|
|
fn llm_switching(&self, notice: Option<&str>) -> u64;
|
|
fn llm_state(&self, state: &str) -> bool;
|
|
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool;
|
|
/// The STT worker's model is loaded (on the graphics card).
|
|
fn stt_on_card(&self) -> bool;
|
|
}
|
|
|
|
#[async_trait::async_trait]
|
|
impl LlmBrain for WorkerClient {
|
|
fn llm_switching(&self, notice: Option<&str>) -> u64 {
|
|
WorkerClient::llm_switching(self, notice)
|
|
}
|
|
fn llm_state(&self, state: &str) -> bool {
|
|
WorkerClient::llm_state(self, state)
|
|
}
|
|
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool {
|
|
WorkerClient::wait_llm_idle(self, id, t).await
|
|
}
|
|
fn stt_on_card(&self) -> bool {
|
|
self.hello().is_some_and(|h| h.stt_ready)
|
|
}
|
|
}
|
|
|
|
/// What a request that waits for a switch hears once (voice_worker.spec.md, "LLM switches").
|
|
fn switch_notice(stage: crate::perf::Stage) -> Option<&'static str> {
|
|
match stage {
|
|
crate::perf::Stage::Gaming => Some("잠깐만요, 게임 모드로 바꾸는 중이에요"),
|
|
crate::perf::Stage::Normal => Some("잠깐만요, 평소 모드로 돌아가는 중이에요"),
|
|
crate::perf::Stage::Paused => None,
|
|
}
|
|
}
|
|
|
|
/// How long after a failed switch the stage is tried again (`perf::Retry`).
|
|
const RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(60);
|
|
|
|
/// The performance stages (admin.spec.md, "Performance stages"): every second the stage wanted
|
|
/// from game mode (`perf.rs`); a change tells the brain the LLM switches, waits until the answers
|
|
/// in progress are written (at most 2 minutes), restarts the LLM in that stage (or stops it to
|
|
/// pause) and tells the brain it is ready (or paused). A restart that fails brings back the stage
|
|
/// that ran before; the failed one is tried again after `retry_after`, three times at most. With
|
|
/// `llm.url` nothing moves.
|
|
async fn perf_loop(settings: crate::settings::Settings, llm: Arc<Llm>, brain: Arc<dyn LlmBrain>,
|
|
game: Arc<dyn Fn() -> bool + Send + Sync>, retry_after: std::time::Duration) {
|
|
use crate::perf::{Perf, Retry, Stage};
|
|
if settings.llm.url.is_some() {
|
|
return;
|
|
}
|
|
let mut perf = Perf::new(std::time::Duration::from_secs(settings.game.revert_seconds.into()), settings.game.pause_in_game);
|
|
let mut retry = Retry::new(retry_after);
|
|
// The stage that runs (a game stage that had to pause counts as the game stage, so it is not
|
|
// tried again while the game runs).
|
|
let mut actual = Stage::Normal;
|
|
loop {
|
|
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
|
let now = std::time::Instant::now();
|
|
let want = perf.target(game(), now);
|
|
if want == actual || !retry.allowed(want, now) {
|
|
continue;
|
|
}
|
|
crate::info_log!("perf", "performance stage: {want:?}");
|
|
let id = brain.llm_switching(switch_notice(want));
|
|
if !brain.wait_llm_idle(id, std::time::Duration::from_secs(120)).await {
|
|
crate::warn_log!("perf", "Warning: the brain did not finish its answers in 2 minutes; switching anyway");
|
|
}
|
|
let ended = match llm.switch(&settings, want, brain.stt_on_card()).await {
|
|
Ok(ended) => {
|
|
retry.succeeded();
|
|
actual = want;
|
|
ended
|
|
}
|
|
Err(e) => {
|
|
retry.failed(want, now);
|
|
crate::error_log!("perf", "the LLM did not start in the {want:?} stage: {e}; going back to {actual:?}");
|
|
match llm.switch(&settings, actual, brain.stt_on_card()).await {
|
|
Ok(ended) => ended,
|
|
Err(e) => {
|
|
crate::error_log!("perf", "the LLM did not come back in the {actual:?} stage either: {e}");
|
|
// Nothing answers: requests fail at once instead of waiting.
|
|
actual = Stage::Paused;
|
|
Stage::Paused
|
|
}
|
|
}
|
|
}
|
|
};
|
|
brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" });
|
|
}
|
|
}
|
|
|
|
|
|
/// The secret values the services get: the bot token and the MCP servers' environment values.
|
|
pub fn secrets_in_use(token: &str, mcp_env: &Value) -> Vec<String> {
|
|
let mut v = vec![token.to_owned()];
|
|
for vars in mcp_env.as_object().into_iter().flat_map(|m| m.values()) {
|
|
v.extend(vars.as_object().into_iter().flat_map(|m| m.values()).filter_map(Value::as_str).map(str::to_owned));
|
|
}
|
|
v
|
|
}
|
|
|
|
/// Stop the bot browser and the LLM after a failed start.
|
|
async fn after_stop(llm: &Llm, browser: &BrowserRun, error: String) -> String {
|
|
browser.stop().await;
|
|
after_llm_stop(llm, error).await
|
|
}
|
|
|
|
/// Stop the LLM after a failed start; the error also says if its server may still run.
|
|
async fn after_llm_stop(llm: &Llm, error: String) -> String {
|
|
match llm.stop().await {
|
|
Ok(()) => error,
|
|
Err(stop) => format!("{error} ({stop})"),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod perf_tests {
|
|
use super::*;
|
|
use crate::strata::tests::{card, fake_placed, launches, layout, server_after_launch};
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::sync::Mutex;
|
|
use std::time::Duration;
|
|
|
|
#[derive(Default)]
|
|
struct FakeBrain(Mutex<Vec<String>>);
|
|
|
|
#[async_trait::async_trait]
|
|
impl LlmBrain for FakeBrain {
|
|
fn llm_switching(&self, notice: Option<&str>) -> u64 {
|
|
self.0.lock().unwrap().push(format!("switching{}", notice.map(|n| format!(": {n}")).unwrap_or_default()));
|
|
1
|
|
}
|
|
fn llm_state(&self, state: &str) -> bool {
|
|
self.0.lock().unwrap().push(state.into());
|
|
true
|
|
}
|
|
async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool {
|
|
self.0.lock().unwrap().push("waited".into());
|
|
true
|
|
}
|
|
fn stt_on_card(&self) -> bool {
|
|
false
|
|
}
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_game_moves_the_llm_to_the_game_stage_and_its_end_moves_it_back() {
|
|
let l = layout("perf-loop");
|
|
let port = server_after_launch(&l, "list").await;
|
|
let llm = Arc::new(Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)));
|
|
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}, "game": {"revertSeconds": 0}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
let brain = Arc::new(FakeBrain::default());
|
|
let game = Arc::new(AtomicBool::new(false));
|
|
let g = game.clone();
|
|
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)), RETRY_AFTER));
|
|
std::fs::remove_file(l.data.join("calls")).unwrap();
|
|
game.store(true, Ordering::SeqCst);
|
|
let until = |want: &'static str| {
|
|
let brain = brain.clone();
|
|
async move {
|
|
for _ in 0..200 {
|
|
if brain.0.lock().unwrap().iter().filter(|m| m.as_str() == want).count() > 0 {
|
|
return;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
}
|
|
panic!("never {want}: {:?}", brain.0.lock().unwrap());
|
|
}
|
|
};
|
|
until("ready").await;
|
|
assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready"]);
|
|
assert!(launches(&l).last().unwrap().contains("--context 16384"), "{:?}", launches(&l));
|
|
assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming"));
|
|
// The game ends (no wait set): back to the normal stage.
|
|
brain.0.lock().unwrap().clear();
|
|
game.store(false, Ordering::SeqCst);
|
|
until("ready").await;
|
|
assert_eq!(brain.0.lock().unwrap()[0], "switching: 잠깐만요, 평소 모드로 돌아가는 중이에요");
|
|
assert!(launches(&l).last().unwrap().contains("--context 32768"));
|
|
assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal"));
|
|
task.abort();
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
async fn a_game_stage_that_fails_to_start_goes_back_and_is_tried_three_times() {
|
|
let l = layout("perf-loop-fail");
|
|
let port = server_after_launch(&l, "list").await;
|
|
// This Strata cannot start at the game's context.
|
|
let log = l.strata_home().join("x.engine.log");
|
|
let exe = crate::strata::tests::fake_strata(&l, &format!(
|
|
"case \"$*\" in *'--context 16384'*) echo 'out of memory' >&2; exit 4;; esac; rm -f '{}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{}'; echo 'Strata is serving.'",
|
|
l.data.join("gone").display(), log.display()));
|
|
let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0)));
|
|
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
|
|
llm.start(&s).await.unwrap();
|
|
let brain = Arc::new(FakeBrain::default());
|
|
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_millis(300)));
|
|
tokio::time::sleep(Duration::from_secs(8)).await;
|
|
let got = brain.0.lock().unwrap().clone();
|
|
// Each try: switching, waited, the game stage fails, the normal stage comes back, ready.
|
|
assert_eq!(got.iter().filter(|m| m.starts_with("switching")).count(), 3, "{got:?}");
|
|
assert_eq!(got.iter().filter(|m| m.as_str() == "ready").count(), 3, "{got:?}");
|
|
assert!(!got.iter().any(|m| m == "paused"));
|
|
assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal"));
|
|
assert_eq!(llm.status().state, crate::strata::LlmState::Ready, "the normal stage runs again");
|
|
task.abort();
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn the_services_secrets_are_the_token_and_every_mcp_value() {
|
|
let env = serde_json::json!({"brave": {"BRAVE_API_KEY": "k1"}, "other": {"A": "k2", "B": "k3"}});
|
|
let mut v = secrets_in_use("tok", &env);
|
|
v.sort();
|
|
assert_eq!(v, vec!["k1", "k2", "k3", "tok"]);
|
|
}
|
|
}
|