Files
javis_bot/app/src/services.rs
javis-bot 39ac20682e feat(app): PC control through agent-desktop, with the emergency stop and the anti-cheat block
app/src/pc.rs runs every pc_action from the worker (admin.spec.md, "PC
control"): one agent-desktop 1.2.0 run per command with a 15 s limit and
its state in the data folder, never --force or raw input, typed text
only through stdin. Each action is decided by permissions::decide for
its asker (not registered, never, ask without consent), then refused
during the emergency stop, while an anti-cheat game runs (Vanguard,
EAC, BattlEye, FACEIT, GameGuard, XIGNCODE3 and the person's list), on
an administrator window and in a password field. Results carry the
window and its elements read again (200 at most, cut to 120 chars),
agent-desktop's error codes become Korean reasons, and each action is
recorded as pc.<action> with the program's file name only.

The emergency stop is a global hotkey (pcControl.stopHotkey, default
Ctrl+Shift+F12) on its own message loop; it kills the running command
and blocks everything but reading until pressed again or lifted on the
PC control screen. features.pcControl (on by default, nothing happens
until someone is registered) and the pinned exe hash decide whether
the brain gets the tool. The installer ships agent-desktop.exe from the
pinned release archive, both hashes checked.

Tested with a fake agent-desktop (342 lib tests pass) and cargo check
for x86_64-pc-windows-gnu; not yet run against real agent-desktop.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:42:48 +09:00

633 lines
31 KiB
Rust

//! Starts everything the app runs: the LLM (`strata.rs`), the voice worker process, the connection
//! to it, and the Discord core. Used by the app binary (and the Tauri shell from M5).
use std::path::Path;
use std::sync::Arc;
use serde_json::Value;
use tokio::sync::mpsc;
use crate::conversation::{self, Conversation};
use crate::discord::{CoreEvent, DiscordCore};
use crate::settings::load_settings;
use crate::strata::Llm;
use crate::worker_client::{WorkerClient, WorkerClientOptions};
use crate::worker_process::{ProcessEvent, WorkerProcess, WorkerProcessOptions};
pub struct Services {
pub core: DiscordCore,
pub events: mpsc::UnboundedReceiver<CoreEvent>,
pub worker: WorkerClient,
pub worker_process: WorkerProcess,
/// The conversation's missed requests, for the dashboard.
pub missed: conversation::MissedLog,
pub llm: Arc<Llm>,
pub browser: BrowserRun,
/// Game detection while the services run (admin.spec.md, "Game detection").
pub game: Arc<crate::game::GameWatch>,
/// Moves the LLM between the performance stages as games come and go.
pub perf: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
/// Fires the reminders and scheduled tasks (admin.spec.md, "Schedules").
pub schedules: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
/// Announces new collected items (admin.spec.md, "News notifications").
pub news: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
/// PC control's emergency-stop hotkey while PC control is available (admin.spec.md, "PC control").
pub pc_hotkey: std::sync::Mutex<Option<crate::pc::Hotkey>>,
}
/// The bot browser of a run (admin.spec.md, "Bot browser").
pub enum BrowserRun {
/// Not started: the feature is off, or (with a note) it is not installed.
Off(Option<String>),
Failed(String),
/// Running; the note says why it runs without the virtual monitor that was asked for.
Running(Arc<crate::browser::BotBrowser>, Option<String>),
}
impl BrowserRun {
/// `{state, note}` for the dashboard.
pub fn view(&self) -> Value {
let (state, note) = match self {
BrowserRun::Off(note) => ("off", note.clone()),
BrowserRun::Failed(e) => ("failed", Some(e.clone())),
BrowserRun::Running(b, note) if b.running() => ("ready", note.clone()),
BrowserRun::Running(_, note) => ("exited", note.clone()),
};
serde_json::json!({"state": state, "note": note})
}
/// Stop the browser, then take the virtual monitor off the desktop (its window first, so
/// Windows never moves it onto a real monitor).
pub async fn stop(&self) {
if let BrowserRun::Running(b, _) = self {
b.stop().await;
crate::vdisplay::detach();
}
}
}
impl Services {
/// Stop everything; an error means the LLM's server may still run.
pub async fn stop(&self) -> Result<(), String> {
let perf = self.perf.lock().unwrap().take();
if let Some(p) = perf {
p.abort();
let _ = p.await;
}
let schedules = self.schedules.lock().unwrap().take();
if let Some(s) = schedules {
s.abort();
let _ = s.await;
}
let news = self.news.lock().unwrap().take();
if let Some(n) = news {
n.abort();
let _ = n.await;
}
if let Some(h) = self.pc_hotkey.lock().unwrap().take() {
h.end();
}
self.game.stop();
self.core.close().await;
self.worker.close();
self.worker_process.stop().await;
self.browser.stop().await;
self.llm.stop().await
}
}
/// Start the bot browser when its feature is on and it is installed, on the virtual monitor when
/// sharing is on; a failure leaves the services running without it (or without the monitor).
async fn start_browser(data_dir: &Path, settings: &crate::settings::Settings) -> BrowserRun {
// A monitor an earlier run left on goes off first.
crate::vdisplay::detach();
if !settings.features.browser {
return BrowserRun::Off(None);
}
let layout = crate::install::Layout { data: data_dir.to_path_buf(), src: std::path::PathBuf::new() };
if !crate::install::Installer::new(layout.clone(), crate::install::steps()).done("browser") {
crate::warn_log!("browser", "the bot browser is not installed");
return BrowserRun::Off(Some("설치되지 않음".into()));
}
let (exe, profile) = (layout.chrome_exe(), crate::browser::profile_dir(data_dir));
let mut note = None;
if settings.features.browser_share {
let started = match crate::vdisplay::attach() {
Ok(r) => crate::browser::BotBrowser::start(&exe, &profile, Some(r)).await,
Err(e) => Err(e),
};
match started {
Ok(b) => return BrowserRun::Running(Arc::new(b), None),
Err(e) => {
crate::vdisplay::detach();
crate::warn_log!("browser", "Warning: no virtual monitor for the bot browser: {e}");
note = Some(format!("가상 모니터 없이 실행: {e}"));
}
}
}
match crate::browser::BotBrowser::start(&exe, &profile, None).await {
Ok(b) => BrowserRun::Running(Arc::new(b), note),
Err(e) => {
crate::warn_log!("browser", "Warning: the bot browser did not start: {e}");
BrowserRun::Failed(e)
}
}
}
/// Start the services; the LLM first (its first start downloads the model), so nothing answers
/// before the brain can. A failure stops what already started.
pub async fn start_services(data_dir: &Path, llm: Arc<Llm>) -> Result<Services, Box<dyn std::error::Error + Send + Sync>> {
let settings = load_settings(&data_dir.join("config").join("settings.json"))?;
// Game detection first: a game already running is not taken for other programs' memory.
let game = Arc::new(crate::game::GameWatch::start(settings.game.clone()));
llm.game_at_start(game.on() || game.in_background_at_start());
let perf_settings = settings.clone();
// The stored token (the setup wizard keeps it encrypted, admin.spec.md), else the environment.
let secrets = crate::secrets::Secrets::new(&data_dir.join("config").join("secrets.dat"));
let stored = secrets.get(crate::secrets::DISCORD_TOKEN)?;
// The MCP servers' environment values stay encrypted on disk: they reach the brain only in the
// worker's environment (admin.spec.md, "Features and MCP servers").
let mcp_env = crate::features::mcp_env(&secrets, &settings.mcp_servers)?;
let kma_key = secrets.get(crate::secrets::KMA_KEY)?;
let token = match stored {
Some(t) => t,
None => std::env::var("JARVIS_DISCORD_TOKEN").map_err(|_| "no bot token: set it in the app or JARVIS_DISCORD_TOKEN")?,
};
// What these services are given stays masked in the logs until the next start, also if it is
// replaced or removed in the store meanwhile (admin.spec.md, "Logs").
if let Some(hub) = crate::logs::installed() {
hub.set_secrets_in_use(secrets_in_use(&token, &mcp_env, kma_key.as_deref()));
}
// PC control: agent-desktop's hash is checked once per start (admin.spec.md, "PC control").
let pc_exe = crate::pc::check(&settings);
let (base_url, model) = llm.start(&settings).await?;
let brain_config = match crate::strata::write_brain_config(data_dir, &base_url, &model, &settings, pc_exe.is_some()) {
Ok(f) => f,
Err(e) => return Err(after_llm_stop(&llm, format!("could not write the brain settings: {e}")).await.into()),
};
let browser = start_browser(data_dir, &settings).await;
let w = &settings.voice_worker;
let mut wopts = WorkerProcessOptions::new(&w.python, &w.cwd, settings.worker_args());
// With the installed runtime, the worker's models, caches and temp files stay in the data folder.
let layout = crate::install::Layout { data: data_dir.to_path_buf(), src: w.cwd.clone() };
if crate::install::Installer::new(layout.clone(), crate::install::steps()).done("settings") {
wopts.env.extend(layout.env());
}
wopts.env.insert("JARVIS_CONFIG_PATH".into(), brain_config.to_string_lossy().into_owned());
wopts.env.insert(crate::features::MCP_ENV_VAR.into(), mcp_env.to_string());
if let Some(k) = &kma_key {
wopts.env.insert(KMA_ENV_VAR.into(), k.clone());
}
if let BrowserRun::Running(b, _) = &browser {
wopts.env.insert("JARVIS_BROWSER_CDP".into(), b.cdp_url());
}
if let Some(dir) = recordings_dir(data_dir, &settings) {
wopts.env.insert(RECORDINGS_ENV_VAR.into(), dir.to_string_lossy().into_owned());
}
// Background collection (admin.spec.md): the enabled targets, and the keep days in any case.
match crate::collect::write_config(data_dir, &settings) {
Ok(f) => { wopts.env.insert(crate::collect::COLLECT_ENV_VAR.into(), f.to_string_lossy().into_owned()); }
Err(e) => return Err(after_stop(&llm, &browser, format!("could not write the collection settings: {e}")).await.into()),
}
let worker_process = match WorkerProcess::new(wopts) {
Ok(p) => p,
Err(e) => return Err(after_stop(&llm, &browser, e.to_string()).await.into()),
};
let mut process_events = worker_process.subscribe();
worker_process.start();
let mut opts = WorkerClientOptions::new(worker_process.url(), worker_process.token.clone());
opts.stt = settings.voice.stt.clone();
opts.stt.insert("hotwords".into(), Value::from(settings.voice.wake_words.clone()));
opts.memory = settings.memory.clone();
let (worker, worker_events) = WorkerClient::new(opts);
// The worker loads its models before it listens: connect once it says READY (after restarts the
// client keeps retrying on its own).
let w2 = worker.clone();
tokio::spawn(async move {
loop {
match process_events.recv().await {
Ok(ProcessEvent::Ready) => return w2.start(),
Ok(_) => {}
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {}
Err(_) => return,
}
}
});
let (calls, response) = (settings.call.clone(), settings.response.clone());
let chat_calls = conversation::ChatCalls::from_settings(&settings);
let schedule_channel = settings.schedule_channel;
let news_settings = settings.clone();
let pc = pc_exe.as_ref().map(|exe| {
let g = game.clone();
crate::pc::Pc::new(settings.clone(), Box::new(crate::pc::Cli::new(exe, data_dir)), crate::pc::emergency(),
Box::new(move || g.name()))
});
let stop_hotkey = settings.pc_control.stop_hotkey.clone();
let (core, core_events) = DiscordCore::new(settings, token, worker.clone(), worker_events);
let pc_hotkey = std::sync::Mutex::new(None);
if let Some(pc) = pc {
core.set_pc(Arc::new(pc));
*pc_hotkey.lock().unwrap() = crate::pc::Hotkey::start(&stop_hotkey, crate::pc::emergency());
crate::info_log!("conversation", "PC control available (agent-desktop {})", crate::pc::AGENT_DESKTOP_VERSION);
}
let schedule_store = Arc::new(crate::schedule::Store::new(data_dir));
core.set_schedules(schedule_store.clone());
let activity = Arc::new(crate::activity::Activity::new(data_dir));
core.set_activity(activity.clone());
core.set_notes(Arc::new(crate::notes::Notes::new(data_dir)));
// Call detection and the response policy answer what is heard; every event still reaches `events`.
let (out, events) = mpsc::unbounded_channel();
let conv = Conversation::new(core.clone(), calls, response).with_chat(chat_calls);
let missed = conv.missed_handle();
tokio::spawn(conversation::run(conv, core_events, out));
if let Err(e) = core.start().await {
worker.close();
worker_process.stop().await;
return Err(after_stop(&llm, &browser, e.to_string()).await.into());
}
crate::info_log!("app", "services started");
let schedules = std::sync::Mutex::new(Some(tokio::spawn(crate::schedule::run(schedule_store, activity.clone(),
Arc::new(core.clone()), schedule_channel))));
let news = std::sync::Mutex::new(Some(tokio::spawn(crate::collect_notify::run(data_dir.to_path_buf(), news_settings,
Arc::new(core.clone()), activity))));
let on = { let g = game.clone(); Arc::new(move || g.on()) };
let perf = std::sync::Mutex::new(Some(tokio::spawn(perf_loop(perf_settings, llm.clone(), Arc::new(worker.clone()), on, RETRY_AFTER))));
Ok(Services { core, events, worker, worker_process, missed, llm, browser, game, perf, schedules, news, pc_hotkey })
}
/// The voice worker writes its recordings where this names (voice_worker.spec.md, "Recordings").
pub const RECORDINGS_ENV_VAR: &str = "JARVIS_RECORDINGS_DIR";
/// The stored KMA key reaches the brain's weather tool only in the worker's environment.
pub const KMA_ENV_VAR: &str = "JARVIS_KMA_API_KEY";
/// Where the voice worker keeps its recordings, when `voice.record` is on.
pub fn recordings_dir(data_dir: &Path, settings: &crate::settings::Settings) -> Option<std::path::PathBuf> {
settings.voice.record.then(|| data_dir.join("recordings"))
}
/// The brain as the performance stages see it: told the LLM's state, and asked to finish what it is
/// answering (the worker client; a fake in tests).
#[async_trait::async_trait]
pub trait LlmBrain: Send + Sync {
/// Announce a switch; its number for `wait_llm_idle`.
fn llm_switching(&self, notice: Option<&str>) -> u64;
/// `ready` (with whether a game runs) or `paused`.
fn llm_state(&self, state: &str, game: bool) -> bool;
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool;
/// What the worker's loaded models (STT, TTS) hold on the graphics card, in MiB.
fn worker_models_mib(&self) -> i64;
}
#[async_trait::async_trait]
impl LlmBrain for WorkerClient {
fn llm_switching(&self, notice: Option<&str>) -> u64 {
WorkerClient::llm_switching(self, notice)
}
fn llm_state(&self, state: &str, game: bool) -> bool {
if state == "ready" { WorkerClient::llm_ready(self, game) } else { WorkerClient::llm_state(self, state) }
}
async fn wait_llm_idle(&self, id: u64, t: std::time::Duration) -> bool {
WorkerClient::wait_llm_idle(self, id, t).await
}
fn worker_models_mib(&self) -> i64 {
self.hello().map_or(0, |h| crate::placement::worker_models_mib(h.stt_ready, h.tts_ready))
}
}
/// What a request that waits for a switch hears once (voice_worker.spec.md, "LLM switches").
fn switch_notice(stage: crate::perf::Stage) -> Option<&'static str> {
match stage {
crate::perf::Stage::Gaming => Some("잠깐만요, 게임 모드로 바꾸는 중이에요"),
crate::perf::Stage::Normal => Some("잠깐만요, 평소 모드로 돌아가는 중이에요"),
crate::perf::Stage::Paused => None,
}
}
/// How long after a failed switch the stage is tried again (`perf::Retry`).
const RETRY_AFTER: std::time::Duration = std::time::Duration::from_secs(60);
/// The performance stages (admin.spec.md, "Performance stages"): every second the stage wanted
/// from game mode (`perf.rs`); a change tells the brain the LLM switches, waits until the answers
/// in progress are written (at most 2 minutes), restarts the LLM in that stage (or stops it to
/// pause) and tells the brain it is ready (or paused). A restart that fails brings back the stage
/// that ran before; the failed one is tried again after `retry_after`, three times at most. With
/// `llm.url` nothing moves.
async fn perf_loop(settings: crate::settings::Settings, llm: Arc<Llm>, brain: Arc<dyn LlmBrain>,
game: Arc<dyn Fn() -> bool + Send + Sync>, retry_after: std::time::Duration) {
use crate::perf::{Perf, Retry, Stage};
if settings.llm.url.is_some() {
return;
}
let mut perf = Perf::new(std::time::Duration::from_secs(settings.game.revert_seconds.into()), settings.game.pause_in_game);
let mut retry = Retry::new(retry_after);
// The stage that runs (a game stage that had to pause counts as the game stage, so it is not
// tried again while the game runs).
let mut actual = Stage::Normal;
let mut last_want = Stage::Normal;
loop {
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
let now = std::time::Instant::now();
let want = perf.target(game(), now);
// Another stage wanted (also back to the one that runs): earlier failures no longer count.
if want != last_want {
retry.succeeded();
last_want = want;
}
if want == actual || !retry.allowed(want, now) {
continue;
}
crate::info_log!("perf", "performance stage: {want:?}");
let id = brain.llm_switching(switch_notice(want));
if !brain.wait_llm_idle(id, std::time::Duration::from_secs(120)).await {
crate::warn_log!("perf", "Warning: the brain did not finish its answers in 2 minutes; switching anyway");
}
let ended = match llm.switch(&settings, want, brain.worker_models_mib(), game()).await {
Ok(ended) => {
retry.succeeded();
actual = want;
ended
}
Err(e) => {
crate::error_log!("perf", "the LLM did not start in the {want:?} stage: {e}; going back to {actual:?}");
let back = match llm.switch(&settings, actual, brain.worker_models_mib(), game()).await {
Ok(ended) => ended,
Err(e) => {
crate::error_log!("perf", "the LLM did not come back in the {actual:?} stage either: {e}");
// Nothing answers: requests fail at once instead of waiting.
actual = Stage::Paused;
Stage::Paused
}
};
// The wait counts from when that is over, not from when the switch began.
retry.failed(want, std::time::Instant::now());
back
}
};
// In the game stage partial passes wait while an answer is written (M0 gamemode_tune).
brain.llm_state(if ended == Stage::Paused { "paused" } else { "ready" }, ended == Stage::Gaming);
}
}
/// The secret values the services get: the bot token and the MCP servers' environment values.
pub fn secrets_in_use(token: &str, mcp_env: &Value, kma_key: Option<&str>) -> Vec<String> {
let mut v = vec![token.to_owned()];
v.extend(kma_key.map(str::to_owned));
for vars in mcp_env.as_object().into_iter().flat_map(|m| m.values()) {
v.extend(vars.as_object().into_iter().flat_map(|m| m.values()).filter_map(Value::as_str).map(str::to_owned));
}
v
}
/// Stop the bot browser and the LLM after a failed start.
async fn after_stop(llm: &Llm, browser: &BrowserRun, error: String) -> String {
browser.stop().await;
after_llm_stop(llm, error).await
}
/// Stop the LLM after a failed start; the error also says if its server may still run.
async fn after_llm_stop(llm: &Llm, error: String) -> String {
match llm.stop().await {
Ok(()) => error,
Err(stop) => format!("{error} ({stop})"),
}
}
#[cfg(test)]
mod perf_tests {
use super::*;
use crate::strata::tests::{card, fake_placed, launches, layout, server_after_launch};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Mutex;
use std::time::Duration;
#[derive(Default)]
struct FakeBrain(Mutex<Vec<String>>);
#[async_trait::async_trait]
impl LlmBrain for FakeBrain {
fn llm_switching(&self, notice: Option<&str>) -> u64 {
self.0.lock().unwrap().push(format!("switching{}", notice.map(|n| format!(": {n}")).unwrap_or_default()));
1
}
fn llm_state(&self, state: &str, game: bool) -> bool {
self.0.lock().unwrap().push(if game { format!("{state} (game)") } else { state.into() });
true
}
async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool {
self.0.lock().unwrap().push("waited".into());
true
}
fn worker_models_mib(&self) -> i64 {
0
}
}
#[cfg(unix)]
#[tokio::test]
async fn a_game_moves_the_llm_to_the_game_stage_and_its_end_moves_it_back() {
let l = layout("perf-loop");
let port = server_after_launch(&l, "list").await;
let llm = Arc::new(Llm::with_exe(l.clone(), fake_placed(&l)).with_gpu(card(&l, "GPU-test", 0)));
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}, "game": {"revertSeconds": 0}})).unwrap();
llm.start(&s).await.unwrap();
let brain = Arc::new(FakeBrain::default());
let game = Arc::new(AtomicBool::new(false));
let g = game.clone();
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)), RETRY_AFTER));
std::fs::remove_file(l.data.join("calls")).unwrap();
game.store(true, Ordering::SeqCst);
let until = |want: &'static str| {
let brain = brain.clone();
async move {
for _ in 0..200 {
if brain.0.lock().unwrap().iter().filter(|m| m.as_str() == want).count() > 0 {
return;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
panic!("never {want}: {:?}", brain.0.lock().unwrap());
}
};
until("ready (game)").await;
assert_eq!(*brain.0.lock().unwrap(), ["switching: 잠깐만요, 게임 모드로 바꾸는 중이에요", "waited", "ready (game)"]);
assert!(launches(&l).last().unwrap().contains("--context 16384"), "{:?}", launches(&l));
assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming"));
// The game ends (no wait set): back to the normal stage.
brain.0.lock().unwrap().clear();
game.store(false, Ordering::SeqCst);
until("ready").await;
assert_eq!(brain.0.lock().unwrap()[0], "switching: 잠깐만요, 평소 모드로 돌아가는 중이에요");
assert!(launches(&l).last().unwrap().contains("--context 32768"));
assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal"));
task.abort();
}
/// A Strata that cannot start at the game's context (taking `fail_secs` to fail).
#[cfg(unix)]
fn strata_failing_in_games(l: &crate::install::Layout, fail_secs: f64) -> std::path::PathBuf {
let log = l.strata_home().join("x.engine.log");
crate::strata::tests::fake_strata(l, &format!(
"case \"$*\" in *'--context 16384'*) sleep {fail_secs}; echo 'out of memory' >&2; exit 4;; esac; rm -f '{}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{}'; echo 'Strata is serving.'",
l.data.join("gone").display(), log.display()))
}
#[cfg(unix)]
#[tokio::test]
async fn a_new_game_tries_the_game_stage_again_after_three_failures_with_the_last_one() {
let l = layout("perf-loop-new-game");
let port = server_after_launch(&l, "list").await;
let llm = Arc::new(Llm::with_exe(l.clone(), strata_failing_in_games(&l, 0.0)).with_gpu(card(&l, "GPU-test", 0)));
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}, "game": {"revertSeconds": 0}})).unwrap();
llm.start(&s).await.unwrap();
let brain = Arc::new(FakeBrain::default());
let game = Arc::new(AtomicBool::new(true));
let g = game.clone();
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(move || g.load(Ordering::SeqCst)), Duration::from_millis(200)));
let switchings = |b: &FakeBrain| b.0.lock().unwrap().iter().filter(|m| m.starts_with("switching")).count();
tokio::time::sleep(Duration::from_secs(6)).await;
assert_eq!(switchings(&brain), 3, "three tries");
// The game ends (the normal stage runs already) and a new one starts: tried again.
game.store(false, Ordering::SeqCst);
tokio::time::sleep(Duration::from_secs(1)).await;
game.store(true, Ordering::SeqCst);
tokio::time::sleep(Duration::from_secs(2)).await;
assert!(switchings(&brain) > 3, "{:?}", brain.0.lock().unwrap());
task.abort();
}
#[cfg(unix)]
#[tokio::test]
async fn the_wait_before_the_next_try_counts_from_when_the_failed_switch_is_over() {
// The failing start takes 1.5 s, longer than the 1 s wait: the next try still waits 1 s
// after the normal stage came back.
let l = layout("perf-loop-wait");
let port = server_after_launch(&l, "list").await;
let llm = Arc::new(Llm::with_exe(l.clone(), strata_failing_in_games(&l, 1.5)).with_gpu(card(&l, "GPU-test", 0)));
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
let brain = Arc::new(TimedBrain::default());
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_secs(1)));
tokio::time::sleep(Duration::from_secs(7)).await;
task.abort();
let got = brain.0.lock().unwrap().clone();
let at = |what: &str, n: usize| got.iter().filter(|(m, _)| m.starts_with(what)).nth(n).map(|(_, t)| *t).expect(what);
let gap = at("switching", 1).duration_since(at("ready", 0));
assert!(gap >= Duration::from_secs(1), "the next try came {gap:?} after the normal stage was back");
}
#[cfg(unix)]
#[tokio::test]
async fn going_back_to_normal_while_the_game_runs_keeps_the_reading_from_before_the_game() {
// 24 GB card; other programs `OTHERS` at the start. A game (2500 MiB) starts; the first
// game start fails and the normal stage comes back while the game still runs; the second
// try works. It must count other programs from the reading before the game, not one that
// has the game in it.
let l = layout("perf-loop-keep-reading");
let port = server_after_launch(&l, "list").await;
let log = l.strata_home().join("x.engine.log");
let failed_once = l.data.join("failed-once");
let exe = crate::strata::tests::fake_strata(&l, &format!(
"case \"$*\" in *'--context 16384'*'expert-vram=0'*) ;; *'--context 16384'*) if [ ! -e '{fo}' ]; then touch '{fo}'; exit 4; fi;; esac; rm -f '{g}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{log}'; echo 'Strata is serving.'",
fo = failed_once.display(), g = l.data.join("gone").display(), log = log.display()));
let others = Arc::new(std::sync::atomic::AtomicI64::new(crate::strata::tests::OTHERS));
let calls = l.data.join("calls");
let o2 = others.clone();
let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(Arc::new(move || {
let c = std::fs::read_to_string(&calls).unwrap_or_default();
let running = c.lines().last().is_some_and(|x| x.starts_with("launch --family"));
Some(crate::strata::tests::gpu("GPU-test", o2.load(Ordering::SeqCst) + if running { 9874 } else { 0 }, 24576))
})));
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
others.store(crate::strata::tests::OTHERS + 2500, Ordering::SeqCst);
let brain = Arc::new(FakeBrain::default());
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_millis(300)));
for _ in 0..100 {
if llm.stage_view()["stage"] == serde_json::json!("gaming") { break; }
tokio::time::sleep(Duration::from_millis(100)).await;
}
task.abort();
assert!(failed_once.exists(), "the first game start failed");
assert_eq!(llm.stage_view()["stage"], serde_json::json!("gaming"));
let probe16 = crate::placement::Probe { context: 16384, parallel: 1, gpu: "GPU-test".into(), model_mib: 9874, staging_gib: 1.1 };
let right = crate::placement::game(24576, crate::strata::tests::OTHERS, &probe16, 8192).expert_gib;
let wrong = crate::placement::game(24576, crate::strata::tests::OTHERS + 2500, &probe16, 8192).expert_gib;
assert_ne!(right, wrong);
assert!(launches(&l).last().unwrap().contains(&format!("--set expert-vram={right} ")), "{:?}", launches(&l));
}
#[derive(Default)]
struct TimedBrain(Mutex<Vec<(String, std::time::Instant)>>);
#[async_trait::async_trait]
impl LlmBrain for TimedBrain {
fn llm_switching(&self, _notice: Option<&str>) -> u64 {
self.0.lock().unwrap().push(("switching".into(), std::time::Instant::now()));
1
}
fn llm_state(&self, state: &str, _game: bool) -> bool {
self.0.lock().unwrap().push((state.into(), std::time::Instant::now()));
true
}
async fn wait_llm_idle(&self, _id: u64, _t: Duration) -> bool {
true
}
fn worker_models_mib(&self) -> i64 {
0
}
}
#[cfg(unix)]
#[tokio::test]
async fn a_game_stage_that_fails_to_start_goes_back_and_is_tried_three_times() {
let l = layout("perf-loop-fail");
let port = server_after_launch(&l, "list").await;
// This Strata cannot start at the game's context.
let log = l.strata_home().join("x.engine.log");
let exe = crate::strata::tests::fake_strata(&l, &format!(
"case \"$*\" in *'--context 16384'*) echo 'out of memory' >&2; exit 4;; esac; rm -f '{}'; echo 'strata-q36: (1.10 GiB staging) on 16 CPU threads' >> '{}'; echo 'Strata is serving.'",
l.data.join("gone").display(), log.display()));
let llm = Arc::new(Llm::with_exe(l.clone(), exe).with_gpu(card(&l, "GPU-test", 0)));
let s = crate::settings::parse_settings(&serde_json::json!({"llm": {"port": port}})).unwrap();
llm.start(&s).await.unwrap();
let brain = Arc::new(FakeBrain::default());
let task = tokio::spawn(perf_loop(s, llm.clone(), brain.clone(), Arc::new(|| true), Duration::from_millis(300)));
tokio::time::sleep(Duration::from_secs(8)).await;
let got = brain.0.lock().unwrap().clone();
// Each try: switching, waited, the game stage fails, the normal stage comes back, ready.
assert_eq!(got.iter().filter(|m| m.starts_with("switching")).count(), 3, "{got:?}");
assert_eq!(got.iter().filter(|m| m.as_str() == "ready").count(), 3, "{got:?}");
assert!(!got.iter().any(|m| m == "paused"));
assert_eq!(llm.stage_view()["stage"], serde_json::json!("normal"));
assert_eq!(llm.status().state, crate::strata::LlmState::Ready, "the normal stage runs again");
task.abort();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_services_secrets_are_the_token_every_mcp_value_and_the_kma_key() {
let env = serde_json::json!({"brave": {"BRAVE_API_KEY": "k1"}, "other": {"A": "k2", "B": "k3"}});
let mut v = secrets_in_use("tok", &env, Some("kma"));
v.sort();
assert_eq!(v, vec!["k1", "k2", "k3", "kma", "tok"]);
assert_eq!(secrets_in_use("tok", &env, None).len(), 4);
}
#[test]
fn the_worker_gets_a_recordings_folder_in_the_data_folder_only_with_voice_record_on() {
let off = crate::settings::parse_settings(&serde_json::json!({})).unwrap();
assert_eq!(recordings_dir(Path::new("/d"), &off), None);
let on = crate::settings::parse_settings(&serde_json::json!({"voice": {"record": true}})).unwrap();
assert_eq!(recordings_dir(Path::new("/d"), &on), Some(Path::new("/d").join("recordings")));
}
}