Files
javis_bot/app/tests/worker_process.rs
javis-bot 71f9b0cd54 fix(app): a worker run's late lines keep its run's secrets masked
Each worker run's log sink keeps the values masked when it started (its secrets in use), so a
line still being logged after a stop gave up waiting, and after the next start replaced the values
in use, is still masked. The debug echo on stderr is masked the same way.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 10:19:15 +09:00

201 lines
8.2 KiB
Rust

//! The worker process with a stand-in `voice_worker` package run by a real Python, so the spawn line
//! is the one the app uses. JARVIS_TEST_PYTHON overrides the Python (default python3 / python).
use std::path::PathBuf;
use std::time::Duration;
use jarvis_app::worker_process::{ProcessEvent, WorkerProcess, WorkerProcessOptions};
use serde_json::Value;
const FAKE_MAIN: &str = r#"
import json, os, sys, time
out = os.environ["FAKE_OUT"]
runs = len(os.listdir(out))
with open(os.path.join(out, f"run{runs}.json"), "w") as f:
json.dump({"argv": sys.argv[1:], "token": os.environ.get("JARVIS_VOICE_TOKEN")}, f)
if os.environ.get("FAKE_CRASH_FIRST") == "1" and runs == 0:
sys.exit(3)
print("loading models...", flush=True)
print("READY 127.0.0.1:" + sys.argv[sys.argv.index("--port") + 1], flush=True)
if os.environ.get("FAKE_LAST_LINES") == "1":
for i in range(3):
print(f"line {i}", file=sys.stderr, flush=True)
print("LAST bad key old-api-key-1", file=sys.stderr, flush=True)
time.sleep(60)
"#;
fn python() -> String {
std::env::var("JARVIS_TEST_PYTHON").unwrap_or_else(|_| if cfg!(windows) { "python".into() } else { "python3".into() })
}
struct Fake {
root: PathBuf,
out: PathBuf,
}
impl Fake {
fn new(tag: &str) -> Self {
let root = std::env::temp_dir().join(format!("jarvis-fake-worker-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(root.join("voice_worker")).unwrap();
std::fs::write(root.join("voice_worker/__init__.py"), "").unwrap();
std::fs::write(root.join("voice_worker/__main__.py"), FAKE_MAIN).unwrap();
let out = root.join("out");
std::fs::create_dir_all(&out).unwrap();
Self { root, out }
}
fn run(&self, n: usize) -> Value {
serde_json::from_str(&std::fs::read_to_string(self.out.join(format!("run{n}.json"))).unwrap()).unwrap()
}
fn opts(&self, crash_first: bool) -> WorkerProcessOptions {
let mut o = WorkerProcessOptions::new(python(), &self.root, vec!["--melo-python".into(), "/x/python".into()]);
o.env.insert("FAKE_OUT".into(), self.out.display().to_string());
if crash_first {
o.env.insert("FAKE_CRASH_FIRST".into(), "1".into());
}
o.restart_delay = Duration::from_millis(50);
o
}
}
impl Drop for Fake {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
async fn wait_for(rx: &mut tokio::sync::broadcast::Receiver<ProcessEvent>, f: impl Fn(&ProcessEvent) -> bool) -> ProcessEvent {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let e = rx.recv().await.unwrap();
if f(&e) {
return e;
}
}
})
.await
.expect("timed out")
}
#[tokio::test]
async fn runs_the_worker_on_loopback_with_the_token_only_in_the_environment() {
let fake = Fake::new("env");
let p = WorkerProcess::new(fake.opts(false)).unwrap();
let mut rx = p.subscribe();
p.start();
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
let run = fake.run(0);
let port = p.port().to_string();
let argv: Vec<String> = serde_json::from_value(run["argv"].clone()).unwrap();
assert_eq!(argv, ["--host", "127.0.0.1", "--port", &port, "--melo-python", "/x/python"]);
assert_eq!(run["token"], p.token.as_str());
assert!(p.token.len() >= 40);
assert!(!argv.join(" ").contains(&p.token));
assert!(p.url().starts_with("ws://127.0.0.1:"));
p.stop().await;
}
#[tokio::test]
async fn a_dead_worker_restarts_on_the_same_port_with_the_same_token() {
let fake = Fake::new("restart");
let p = WorkerProcess::new(fake.opts(true)).unwrap();
let mut rx = p.subscribe();
p.start();
assert_eq!(wait_for(&mut rx, |e| matches!(e, ProcessEvent::Exited { .. })).await, ProcessEvent::Exited { code: Some(3) });
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
assert_eq!(p.restarts(), 1);
assert_eq!(fake.run(1)["argv"], fake.run(0)["argv"]);
assert_eq!(fake.run(1)["token"], fake.run(0)["token"]);
p.stop().await;
}
#[tokio::test]
async fn stop_ends_the_worker_for_good() {
let fake = Fake::new("stop");
let p = WorkerProcess::new(fake.opts(false)).unwrap();
let mut rx = p.subscribe();
p.start();
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
p.stop().await;
assert!(matches!(rx.recv().await.unwrap(), ProcessEvent::Exited { .. }));
tokio::time::sleep(Duration::from_millis(200)).await;
assert!(rx.try_recv().is_err(), "no restart after stop");
assert_eq!(p.restarts(), 0);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stop_returns_only_after_the_last_output_is_logged() {
// A slow log (each line takes 300 ms) and a worker whose last lines are still unread when it is
// stopped: they must be handled before stop returns, so the next start's secrets (logs.rs,
// set_secrets_in_use) never apply to them.
let fake = Fake::new("drain");
let mut o = fake.opts(false);
o.env.insert("FAKE_LAST_LINES".into(), "1".into());
let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let s2 = seen.clone();
o.line_sink = Some(jarvis_app::worker_process::LineSink(std::sync::Arc::new(move |l: &str| {
std::thread::sleep(Duration::from_millis(300));
s2.lock().unwrap().push(l.to_owned());
})));
let p = WorkerProcess::new(o).unwrap();
let mut rx = p.subscribe();
p.start();
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
tokio::time::timeout(Duration::from_secs(10), async {
while seen.lock().unwrap().is_empty() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
}).await.expect("no output");
p.stop().await;
let lines = seen.lock().unwrap().clone();
assert!(lines.iter().any(|l| l.contains("LAST")), "{lines:?}");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_last_line_logged_after_the_stop_gave_up_is_masked_with_its_runs_secrets() {
// The services started with the old key; the log of the last line is held up past the stop's
// wait, and the next start replaces the values in use before that line is written.
let hub = jarvis_app::logs::LogHub::new(None);
jarvis_app::logs::install(hub.clone());
hub.set_secrets_in_use(vec!["old-api-key-1".into()]);
let fake = Fake::new("late");
let mut o = fake.opts(false);
o.env.insert("FAKE_LAST_LINES".into(), "1".into());
let run = jarvis_app::worker_process::run_sink();
let (started, done) = (std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)));
let (s2, d2) = (started.clone(), done.clone());
o.line_sink = Some(jarvis_app::worker_process::LineSink(std::sync::Arc::new(move |l: &str| {
if l.contains("LAST") {
s2.store(true, std::sync::atomic::Ordering::SeqCst);
std::thread::sleep(Duration::from_secs(7));
(run.0)(l);
d2.store(true, std::sync::atomic::Ordering::SeqCst);
} else {
(run.0)(l);
}
})));
let p = WorkerProcess::new(o).unwrap();
let mut rx = p.subscribe();
p.start();
wait_for(&mut rx, |e| *e == ProcessEvent::Ready).await;
tokio::time::timeout(Duration::from_secs(10), async {
while !started.load(std::sync::atomic::Ordering::SeqCst) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
}).await.expect("no last line");
p.stop().await;
// The next start: only the new key is in use (and stored).
hub.set_secrets(vec!["new-api-key-2".into()]);
hub.set_secrets_in_use(vec!["new-api-key-2".into()]);
tokio::time::timeout(Duration::from_secs(10), async {
while !done.load(std::sync::atomic::Ordering::SeqCst) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
}).await.expect("the last line was never logged");
let lines: Vec<String> = hub.query(None, jarvis_app::logs::Level::Debug, None, None, 100)
.into_iter().map(|l| l.text).collect();
let last = lines.iter().find(|l| l.contains("LAST")).expect("logged");
assert!(last.contains("[비밀값]") && !last.contains("old-api-key-1"), "{lines:?}");
}