Files
javis_bot/app/src/perf.rs
javis-bot aa6d950af6 fix(app): performance stage switches survive failures, reconnects and late signals
- the normal stage reads other programs' memory afresh (the card after the LLM stopped, less the
  STT worker's model) and keeps it; the game stage counts from the latest normal reading, so the
  game is not counted against its own reserve
- a restart that fails brings back the stage that ran before and tells the brain ready; the
  failed stage is tried again after 60 s, three times at most; when nothing comes back the brain
  is told paused so requests fail at once
- the worker client keeps the latest llm state and sends it first on every connection
- switches are numbered: llm switching carries an id, llm_idle names it, and a late idle of an
  earlier switch no longer lets a new one through
- a request that waited 180 s for a switch fails without calling the brain
- the dashboard shows the stage that runs also on a fallback placement without experts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 10:34:24 +09:00

137 lines
5.0 KiB
Rust

//! The performance stages (`docs/v2/pc-features.md`, "자동 성능 조절" and "전환 규칙"; admin.spec.md,
//! "Performance stages"): which placement the LLM should run in, from game mode and time. The
//! switching itself (waiting for the answer in progress, restarting the LLM) is `services.rs`.
use std::time::{Duration, Instant};
use serde::Serialize;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Stage {
Normal,
Gaming,
/// The LLM is stopped while a game runs (chosen, or no placement leaves the game room).
Paused,
}
/// Picks the stage: a game means the game stage (or pause when the person chose it) at once; after
/// the game ends the normal stage comes back only once `revert_after` has passed, so quitting and
/// starting a game again does not move the model back and forth.
#[derive(Debug)]
pub struct Perf {
revert_after: Duration,
pause_in_game: bool,
game_since_off: Option<Instant>,
current: Stage,
}
impl Perf {
pub fn new(revert_after: Duration, pause_in_game: bool) -> Self {
Self { revert_after, pause_in_game, game_since_off: None, current: Stage::Normal }
}
/// The stage wanted now, given whether a game runs.
pub fn target(&mut self, game: bool, now: Instant) -> Stage {
if game {
self.game_since_off = None;
self.current = if self.pause_in_game { Stage::Paused } else { Stage::Gaming };
} else if self.current != Stage::Normal {
let since = *self.game_since_off.get_or_insert(now);
if now.duration_since(since) >= self.revert_after {
self.current = Stage::Normal;
self.game_since_off = None;
}
}
self.current
}
}
/// How often a stage that failed to start is tried again (then not until the wanted stage changes).
pub const MAX_TRIES: u32 = 3;
/// The limited retry of a stage whose restart failed: after a failure it waits `after`, and after
/// `MAX_TRIES` failures of the same stage it stops until another stage is wanted.
#[derive(Debug)]
pub struct Retry {
after: Duration,
stage: Option<Stage>,
tries: u32,
next: Option<Instant>,
}
impl Retry {
pub fn new(after: Duration) -> Self {
Self { after, stage: None, tries: 0, next: None }
}
/// Whether switching to `want` may be tried now.
pub fn allowed(&self, want: Stage, now: Instant) -> bool {
self.stage != Some(want) || (self.tries < MAX_TRIES && self.next.is_none_or(|n| now >= n))
}
pub fn failed(&mut self, want: Stage, now: Instant) {
if self.stage == Some(want) {
self.tries += 1;
} else {
(self.stage, self.tries) = (Some(want), 1);
}
self.next = Some(now + self.after);
}
pub fn succeeded(&mut self) {
(self.stage, self.tries, self.next) = (None, 0, None);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_game_switches_at_once_and_its_end_only_after_the_wait() {
let t = Instant::now();
let s = |n: u64| t + Duration::from_secs(n);
let mut p = Perf::new(Duration::from_secs(30), false);
assert_eq!(p.target(false, s(0)), Stage::Normal);
assert_eq!(p.target(true, s(1)), Stage::Gaming);
// The game ends: still the game stage for 30 s.
assert_eq!(p.target(false, s(2)), Stage::Gaming);
assert_eq!(p.target(false, s(31)), Stage::Gaming);
// Started again within the wait: nothing moves, and the wait starts over at the next end.
assert_eq!(p.target(true, s(32)), Stage::Gaming);
assert_eq!(p.target(false, s(40)), Stage::Gaming);
assert_eq!(p.target(false, s(69)), Stage::Gaming);
assert_eq!(p.target(false, s(70)), Stage::Normal);
assert_eq!(p.target(false, s(500)), Stage::Normal);
}
#[test]
fn a_failed_stage_is_tried_again_after_a_wait_three_times_at_most() {
let t = Instant::now();
let s = |n: u64| t + Duration::from_secs(n);
let mut r = Retry::new(Duration::from_secs(60));
assert!(r.allowed(Stage::Gaming, s(0)));
r.failed(Stage::Gaming, s(0));
assert!(!r.allowed(Stage::Gaming, s(59)), "waits");
assert!(r.allowed(Stage::Normal, s(1)), "another stage may be tried at once");
assert!(r.allowed(Stage::Gaming, s(60)));
r.failed(Stage::Gaming, s(60));
r.failed(Stage::Gaming, s(120));
assert!(!r.allowed(Stage::Gaming, s(1000)), "three failures: no more tries");
// Another stage wanted (and failing) starts its own count; a success clears it.
r.failed(Stage::Normal, s(1000));
assert!(r.allowed(Stage::Gaming, s(1000)));
r.succeeded();
assert!(r.allowed(Stage::Normal, s(1000)));
}
#[test]
fn with_pause_chosen_a_game_pauses_the_llm() {
let t = Instant::now();
let mut p = Perf::new(Duration::ZERO, true);
assert_eq!(p.target(true, t), Stage::Paused);
assert_eq!(p.target(false, t), Stage::Normal);
}
}