Files
javis_bot/app/src/services.rs
javis-bot aa6d950af6 fix(app): performance stage switches survive failures, reconnects and late signals
- the normal stage reads other programs' memory afresh (the card after the LLM stopped, less the
  STT worker's model) and keeps it; the game stage counts from the latest normal reading, so the
  game is not counted against its own reserve
- a restart that fails brings back the stage that ran before and tells the brain ready; the
  failed stage is tried again after 60 s, three times at most; when nothing comes back the brain
  is told paused so requests fail at once
- the worker client keeps the latest llm state and sends it first on every connection
- switches are numbered: llm switching carries an id, llm_idle names it, and a late idle of an
  earlier switch no longer lets a new one through
- a request that waited 180 s for a switch fails without calling the brain
- the dashboard shows the stage that runs also on a fallback placement without experts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 10:34:24 +09:00

436 lines
20 KiB
Rust

//! Starts everything the app runs: the LLM (`strata.rs`), the voice worker process, the connection
//! to it, and the Discord core. Used by the app binary (and the Tauri shell from M5).
use std::path::Path;
use std::sync::Arc;
use serde_json::Value;
use tokio::sync::mpsc;
use crate::conversation::{self, Conversation};
use crate::discord::{CoreEvent, DiscordCore};
use crate::settings::load_settings;
use crate::strata::Llm;
use crate::worker_client::{WorkerClient, WorkerClientOptions};
use crate::worker_process::{ProcessEvent, WorkerProcess, WorkerProcessOptions};
pub struct Services {
pub core: DiscordCore,
pub events: mpsc::UnboundedReceiver<CoreEvent>,
pub worker: WorkerClient,
pub worker_process: WorkerProcess,
/// The conversation's missed requests, for the dashboard.
pub missed: conversation::MissedLog,
pub llm: Arc<Llm>,
pub browser: BrowserRun,
/// Game detection while the services run (admin.spec.md, "Game detection").
pub game: Arc<crate::game::GameWatch>,
/// Moves the LLM between the performance stages as games come and go.
pub perf: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
}
/// The bot browser of a run (admin.spec.md, "Bot browser").
pub enum BrowserRun {
/// Not started: the feature is off, or (with a note) it is not installed.
Off(Option<String>),
Failed(String),
/// Running; the note says why it runs without the virtual monitor that was asked for.
Running(Arc<crate::browser::BotBrowser>, Option<String>),
}
impl BrowserRun {
/// `{state, note}` for the dashboard.
pub fn view(&self) -> Value {
let (state, note) = match self {
BrowserRun::Off(note) => ("off", note.clone()),
BrowserRun::Failed(e) => ("failed", Some(e.clone())),
BrowserRun::Running(b, note) if b.running() => ("ready", note.clone()),
BrowserRun::Running(_, note) => ("exited", note.clone()),
};
serde_json::json!({"state": state, "note": note})
}
/// Stop the browser, then take the virtual monitor off the desktop (its window first, so
/// Windows never moves it onto a real monitor).
pub async fn stop(&self) {
if let BrowserRun::Running(b, _) = self {
b.stop().await;
crate::vdisplay::detach();
}
}
}
impl Services {
/// Stop everything; an error means the LLM's server may still run.
pub async fn stop(&self) -> Result<(), String> {
let perf = self.perf.lock().unwrap().take();
if let Some(p) = perf {
p.abort();
let _ = p.await;
}
self.game.stop();
self.core.close().await;
self.worker.close();
self.worker_process.stop().await;
self.browser.stop().await;
self.llm.stop().await
}
}
/// Start the bot browser when its feature is on and it is installed, on the virtual monitor when
/// sharing is on; a failure leaves the services running without it (or without the monitor).
async fn start_browser(data_dir: &Path, settings: &crate::settings::Settings) -> BrowserRun {
// A monitor an earlier run left on goes off first.
crate::vdisplay::detach();
if !settings.features.browser {
return BrowserRun::Off(None);
}
let layout = crate::install::Layout { data: data_dir.to_path_buf(), src: std::path::PathBuf::new() };
if !crate::install::Installer::new(layout.clone(), crate::install::steps()).done("browser") {
crate::warn_log!("browser", "the bot browser is not installed");
return BrowserRun::Off(Some("설치되지 않음".into()));
}
let (exe, profile) = (layout.chrome_exe(), crate::browser::profile_dir(data_dir));
let mut note = None;
if settings.features.browser_share {
let started = match crate::vdisplay::attach() {
Ok(r) => crate::browser::BotBrowser::start(&exe, &profile, Some(r)).await,
Err(e) => Err(e),
};
match started {
Ok(b) => return BrowserRun::Running(Arc::new(b), None),
Err(e) => {
crate::vdisplay::detach();
crate::warn_log!("browser", "Warning: no virtual monitor for the bot browser: {e}");
note = Some(format!("가상 모니터 없이 실행: {e}"));
}
}
}
match crate::browser::BotBrowser::start(&exe, &profile, None).await {
Ok(b) => BrowserRun::Running(Arc::new(b), note),
Err(e) => {
crate::warn_log!("browser", "Warning: the bot browser did not start: {e}");
BrowserRun::Failed(e)
}
}
}
/// Start the services; the LLM first (its first start downloads the model), so nothing answers
/// before the brain can. A failure stops what already started.
pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services, Box<dyn std::error::Error + Send + Sync>> {
let settings = load_settings(&data_dir.join("config").join("settings.json"))?;
let game_settings = settings.game.clone();
let perf_settings = settings.clone();
// The stored token (the setup wizard keeps it encrypted, admin.spec.md), else the environment.
let secrets = crate::secrets::Secrets::new(&data_dir.join("config").join("secrets.dat"));
let stored = secrets.get(crate::secrets::DISCORD_TOKEN)?;
// The MCP servers' environment values stay encrypted on disk: they reach the brain only in the
// worker's environment (admin.spec.md, "Features and MCP servers").
let mcp_env = crate::features::mcp_env(&secrets, &settings.mcp_servers)?;
let token = match stored {
Some(t) => t,
None => std::env::var("JARVIS_DISCORD_TOKEN").map_err(|_| "no bot token: set it in the app or JARVIS_DISCORD_TOKEN")?,
};
// What these services are given stays masked in the logs until the next start, also if it is
// replaced or removed in the store meanwhile (admin.spec.md, "Logs").
if let Some(hub) = crate::logs::installed() {
hub.set_secrets_in_use(secrets_in_use(&token, &mcp_env));
}
let (base_url, model) = llm.start(&settings).await?;
let brain_config = match crate::strata::write_brain_config(data_dir, &base_url, &model, &settings) {
Ok(f) => f,
Err(e) => return Err(after_llm_stop(&llm, format!("could not write the brain settings: {e}")).await.into()),
};
let browser = start_browser(data_dir, &settings).await;
let w = &settings.voice_worker;
let mut wopts = WorkerProcessOptions::new(&w.python, &w.cwd, settings.worker_args());
// With the installed runtime, the worker's models, caches and temp files stay in the data folder.
let layout = crate::install::Layout { data: data_dir.to_path_buf(), src: w.cwd.clone() };
if crate::install::Installer::new(layout.clone(), crate::install::steps()).done("settings") {
wopts.env.extend(layout.env());
}
wopts.env.insert("JARVIS_CONFIG_PATH".into(), brain_config.to_string_lossy().into_owned());
wopts.env.insert(crate::features::MCP_ENV_VAR.into(), mcp_env.to_string());
if let BrowserRun::Running(b, _) = &browser {
wopts.env.insert("JARVIS_BROWSER_CDP".into(), b.cdp_url());
}
// Background collection (admin.spec.md): the enabled targets, and the keep days in any case.
match crate::collect::write_config(data_dir, &settings) {
Ok(f) => { wopts.env.insert(crate::collect::COLLECT_ENV_VAR.into(), f.to_string_lossy().into_owned()); }
Err(e) => return Err(after_stop(&llm, &browser, format!("could not write the collection settings: {e}")).await.into()),
}
let worker_process = match WorkerProcess::new(wopts) {
Ok(p) => p,
Err(e) => return Err(after_stop(&llm, &browser, e.to_string()).await.into()),
};
let mut process_events = worker_process.subscribe();
worker_process.start();
let mut opts = WorkerClientOptions::new(worker_process.url(), worker_process.token.clone());
opts.stt = settings.voice.stt.clone();
opts.stt.insert("hotwords".into(), Value::from(settings.voice.wake_words.clone()));
opts.memory = settings.memory.clone();
let (worker, worker_events) = WorkerClient::new(opts);
// The worker loads its models before it listens: connect once it says READY (after restarts the
// client keeps retrying on its own).
let w2 = worker.clone();
tokio::spawn(async move {
loop {
match process_events.recv().await {
Ok(ProcessEvent::Ready) => return w2.start(),
Ok(_) => {}
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {}
Err(_) => return,
}
}
});
let (calls, response) = (settings.call.clone(), settings.response.clone());
let chat_calls = conversation::ChatCalls::from_settings(&settings);
let (core, core_events) = DiscordCore::new(settings, token, worker.clone(), worker_events);
// Call detection and the response policy answer what is heard; every event still reaches `events`.
let (out, events) = mpsc::unbounded_channel();
let conv = Conversation::new(core.clone(), calls, response).with_chat(chat_calls);
let missed = conv.missed_handle();
tokio::spawn(conversation::run(conv, core_events, out));
if let Err(e) = core.start().await {
worker.close();
worker_process.stop().await;
return Err(after_stop(&llm, &browser, e.to_string()).await.into());
}
crate::info_log!("app", "services started");
let game = Arc::new(crate::game::GameWatch::start(game_settings));
let on = { let g = game.clone(); Arc::new(move || g.on()) };
let perf = std::sync::Mutex::new(Some(tokio::spawn(perf_loop(perf_settings, llm.clone(), Arc::new(worker.clone()), on, RETRY_AFTER))));
Ok(Services { core, events, worker, worker_process, missed, llm, browser, game, perf })
}
/// The brain as the performance stages see it: told the LLM's state, and asked to finish what it is
/// answering (the worker client; a fake in tests).
#[async_trait::async_trait]
pub trait LlmBrain: Send + Sync {
/// Announce a switch; its number for `wait_llm_idle`.
fn llm_switching(&self, notice: Option<&str>) -> u64;
fn llm_state(&self, state: &str) -> bool;
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool;
/// The STT worker's model is loaded (on the graphics card).
fn stt_on_card(&self) -> bool;
}
#[async_trait::async_trait]
impl LlmBrain for WorkerClient {
fn llm_switching(&self, notice: Option<&str>) -> u64 {
WorkerClient::llm_switching(self, notice)
}
fn llm_state(&self, state: &str) -> bool {
WorkerClient::llm_state(self, state)
}
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool {
WorkerClient::wait_llm_idle(self, id, t).await
}
fn stt_on_card(&self) -> bool {
self.hello().is_some_and(|h| h.stt_ready)
}
}
/// What a request that waits for a switch hears once (voice_worker.spec.md, "LLM switches").
fn switch_notice(stage: crate::perf::Stage) -> Option<&'static str> {
match stage {
crate::perf::Stage::Gaming => Some("잠깐만요, 게임 모드로 바꾸는 중이에요"),
crate::perf::Stage::Normal => Some("잠깐만요, 평소 모드로 돌아가는 중이에요"),
crate::perf::Stage::Paused => None,
}
}
/// How long after a failed switch the stage is tried again (`perf::Retry`).
const RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(60);
/// The performance stages (admin.spec.md, "Performance stages"): every second the stage wanted
/// from game mode (`perf.rs`); a change tells the brain the LLM switches, waits until the answers
/// in progress are written (at most 2 minutes), restarts the LLM in that stage (or stops it to
/// pause) and tells the brain it is ready (or paused). A restart that fails brings back the stage
/// that ran before; the failed one is tried again after `retry_after`, three times at most. With
/// `llm.url` nothing moves.
async fn perf_loop(settings: crate::settings::Settings, llm: Arc<Llm>, brain: Arc<dyn LlmBrain>,
game: Arc<dyn Fn() -> bool + Send + Sync>, retry_after: std::time::Duration) {
use crate::perf::{Perf, Retry, Stage};
if settings.llm.url.is_some() {
return;
}
let mut perf = Perf::new(std::time::Duration::from_secs(settings.game.revert_seconds.into()), settings.game.pause_in_game);
let mut retry = Retry::new(retry_after);
// The stage that runs (a game stage that had to pause counts as the game stage, so it is not
// tried again while the game runs).
let mut actual = Stage::Normal;
loop {
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
let now = std::time::Instant::now();
let want = perf.target(game(), now);
if want == actual || !retry.allowed(want, now) {
continue;
}
crate::info_log!("perf", "performance stage: {want:?}");
let id = brain.llm_switching(switch_notice(want));
if !brain.wait_llm_idle(id, std::time::Duration::from_secs(120)).await {
crate::warn_log!("perf", "Warning: the brain did not finish its answers in 2 minutes; switching anyway");
}
let ended = match llm.switch(&settings, want, brain.stt_on_card()).await {
Ok(ended) => {
retry.succeeded();
actual = want;
ended
}
Err(e) => {
retry.failed(want, now);
crate::error_log!("perf", "the LLM did not start in the {want:?} stage: {e}; going back to {actual:?}");
match llm.switch(&settings, actual, brain.stt_on_card()).await {
Ok(ended) => ended,
Err(e) => {
crate::error_log!("perf", "the LLM did not come back in the {actual:?} stage either: {e}");
// Nothing answers: requests fail at once instead of waiting.
actual = Stage::Paused;
Stage::Paused
}
}
}
};
brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" });
}
}
/// The secret values the services get: the bot token and the MCP servers' environment values.
pub fn secrets_in_use(token: &str, mcp_env: &Value) -> Vec<String> {
let mut v = vec![token.to_owned()];
for vars in mcp_env.as_object().into_iter().flat_map(|m| m.values()) {
v.extend(vars.as_object().into_iter().flat_map(|m| m.values()).filter_map(Value::as_str).map(str::to_owned));
}
v
}
/// Stop the bot browser and the LLM after a failed start.
async fn after_stop(llm: &Llm, browser: &BrowserRun, error: String) -> String {
browser.stop().await;
after_llm_stop(llm, error).await
}
/// Stop the LLM after a failed start; the error also says if its server may still run.
async fn after_llm_stop(llm: &Llm, error: String) -> String {
match llm.stop().await {
Ok(()) => error,
Err(stop) => format!("{error} ({stop})"),
}
}
#[cfg(test)]
mod perf_tests {
use super::*;
use crate::strata::tests::{card, fake_placed, launches, layout, server_after_launch};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Mutex;
use std::time::Duration;
#[derive(Default)]
struct FakeBrain(Mutex<Vec<String>>);
#[async_trait::async_trait]
impl LlmBrain for FakeBrain {
fn llm_switching(&self, notice: Option<&str>) -> u64 {
self.0.lock().unwrap().push(format!("switching{}", notice.map(|n| format!(": {n}")).unwrap_or_default()));
1
}
fn llm_state(&self, state: &str) -> bool {
self.0.lock().unwrap().push(state.into());
true
}
async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool {
self.0.lock().unwrap().push("waited".into());
true
}
fn stt_on_card(&self) -> bool {
false
}
}
#[cfg(unix)]
#[tokio::test]
async fn a_game_moves_the_llm_to_the_game_stage_and_its_end_moves_it_back() {
let l = layout("perf-loop");
let port = server_after_launch(&l, "list").await;
let llm = Arc::new(Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)));
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}, "game": {"revertSeconds": 0}})).unwrap();
llm.start(&s).await.unwrap();
let brain = Arc::new(FakeBrain::default());
let game = Arc::new(AtomicBool::new(false));
let g = game.clone();
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)), RETRY_AFTER));
std::fs::remove_file(l.data.join("calls")).unwrap();
game.store(true, Ordering::SeqCst);
let until = |want: &'static str| {
let brain = brain.clone();
async move {
for _ in 0..200 {
if brain.0.lock().unwrap().iter().filter(|m| m.as_str() == want).count() > 0 {
return;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
panic!("never {want}: {:?}", brain.0.lock().unwrap());
}
};
until("ready").await;
assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready"]);
assert!(launches(&l).last().unwrap().contains("--context 16384"), "{:?}", launches(&l));
assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming"));
// The game ends (no wait set): back to the normal stage.
brain.0.lock().unwrap().clear();
game.store(false, Ordering::SeqCst);
until("ready").await;
assert_eq!(brain.0.lock().unwrap()[0], "switching: 잠깐만요, 평소 모드로 돌아가는 중이에요");
assert!(launches(&l).last().unwrap().contains("--context 32768"));
assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal"));
task.abort();
}
#[cfg(unix)]
#[tokio::test]
async fn a_game_stage_that_fails_to_start_goes_back_and_is_tried_three_times() {
let l = layout("perf-loop-fail");
let port = server_after_launch(&l, "list").await;
// This Strata cannot start at the game's context.
let log = l.strata_home().join("x.engine.log");
let exe = crate::strata::tests::fake_strata(&l, &format!(
"case \"$*\" in *'--context 16384'*) echo 'out of memory' >&2; exit 4;; esac; rm -f '{}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{}'; echo 'Strata is serving.'",
l.data.join("gone").display(), log.display()));
let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0)));
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
let brain = Arc::new(FakeBrain::default());
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_millis(300)));
tokio::time::sleep(Duration::from_secs(8)).await;
let got = brain.0.lock().unwrap().clone();
// Each try: switching, waited, the game stage fails, the normal stage comes back, ready.
assert_eq!(got.iter().filter(|m| m.starts_with("switching")).count(), 3, "{got:?}");
assert_eq!(got.iter().filter(|m| m.as_str() == "ready").count(), 3, "{got:?}");
assert!(!got.iter().any(|m| m == "paused"));
assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal"));
assert_eq!(llm.status().state, crate::strata::LlmState::Ready, "the normal stage runs again");
task.abort();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_services_secrets_are_the_token_and_every_mcp_value() {
let env = serde_json::json!({"brave": {"BRAVE_API_KEY": "k1"}, "other": {"A": "k2", "B": "k3"}});
let mut v = secrets_in_use("tok", &env);
v.sort();
assert_eq!(v, vec!["k1", "k2", "k3", "tok"]);
}
}