fix(app): install cancels whole process trees and silent downloads, and finishes only with the settings written
- A cancelled command ends with every process it started (process group on Unix, taskkill /T on Windows). - Waiting for a response or the next chunk gives way to a cancel, so a silent server cannot hold the install. - Pointing the settings at the runtime is the last install step: a failure shows as failed and can be retried; done means written. - Strata's installer gets its folder as the working folder, not inside the script text. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1
app/Cargo.lock
generated
1
app/Cargo.lock
generated
@@ -1491,6 +1491,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"fs4",
|
||||
"futures-util",
|
||||
"libc",
|
||||
"opus2",
|
||||
"rand 0.9.5",
|
||||
"reqwest",
|
||||
|
||||
@@ -42,3 +42,6 @@ strip = "symbols"
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Security_Cryptography", "Win32_System_SystemInformation"] }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
@@ -140,6 +140,7 @@ on the game PC (`scripts/m0`). The steps, in order:
|
||||
| `stt_model` | Whisper `large-v3-turbo` (faster-whisper) |
|
||||
| `tts_model` | the NLTK data MeloTTS's English g2p reads, then the worker's own `melo_child.py` loads the Korean voice once (downloading it) and must say its warm-up sentence |
|
||||
| `strata` | Strata's own installer (`install.ps1`, `install.sh` elsewhere) with Strata's state, engine and components under `runtime/strata` and no PATH change |
|
||||
| `settings` | `voiceWorker` in the settings pointed at the worker venv's Python with `--melo-python <melo venv> --history memory/history.sqlite --brain` (other settings kept); the install is done only once this is written |
|
||||
|
||||
The LLM itself (Qwen3.6-35B-A3B, about 23 GB) is downloaded by Strata on its first start, which the
|
||||
app runs next (M5, later step).
|
||||
@@ -147,15 +148,15 @@ app runs next (M5, later step).
|
||||
- A finished step leaves `runtime/.done/<id>` holding its version (the pin; for `worker_env` a hash
|
||||
of its requirement files), so later runs skip it and a new pin runs it again. Steps run one at a
|
||||
time; a failure stops the run and the next run starts at that step; cancel stops the running
|
||||
command or download (a download's part file stays, so the next try continues it).
|
||||
command and every process it started (one process group, a job tree on Windows) or the download,
|
||||
also while a server sends nothing (a download's part file stays, so the next try continues it).
|
||||
- Folders go to commands as arguments or as the working folder, never inside a script's text.
|
||||
- Every command runs with the layout's environment: `uv`'s cache and Pythons, `HF_HOME`
|
||||
(`models/hf`), `NLTK_DATA` (`models/nltk`), temp files (`cache/tmp`) and Strata's folders, all
|
||||
inside the data folder. Python comes from `uv` only. Output lines go to the `install` log tab.
|
||||
- Downloads go to `<file>.part`, continue with an HTTP range after a break (a server without ranges
|
||||
starts over), and are checked against the pinned SHA-256; a wrong checksum deletes the part.
|
||||
- When every step is done, `uv`'s download cache is cleaned (the venvs hold copies), and
|
||||
`voiceWorker` in the settings is pointed at the worker venv's Python with
|
||||
`--melo-python <melo venv> --history memory/history.sqlite --brain` (other settings kept).
|
||||
- When every step is done, `uv`'s download cache is cleaned (the venvs hold copies).
|
||||
- With the install complete, the services start the worker with the same environment (models,
|
||||
caches and temp files in the data folder; on Linux also the CUDA wheels' library folders).
|
||||
|
||||
|
||||
@@ -146,13 +146,7 @@ impl AdminApi {
|
||||
}
|
||||
"install_status" => Ok(self.installer()?.status()),
|
||||
"install_start" => {
|
||||
let inst = self.installer()?;
|
||||
let settings = self.dir()?.settings_file();
|
||||
if !inst.start_then(move |layout| {
|
||||
if let Err(e) = crate::install::apply_runtime_settings(&settings, layout) {
|
||||
crate::error_log!("install", "could not point the settings at the runtime: {e}");
|
||||
}
|
||||
}) {
|
||||
if !self.installer()?.start() {
|
||||
return Err("설치가 이미 진행 중입니다".into());
|
||||
}
|
||||
Ok(json!({"started": true}))
|
||||
|
||||
@@ -18,6 +18,22 @@ impl std::fmt::Display for DownloadError {
|
||||
}
|
||||
impl std::error::Error for DownloadError {}
|
||||
|
||||
/// Await `fut`, giving up when `cancel` is set (checked every 200 ms): a server that keeps the
|
||||
/// connection open and sends nothing cannot hold a cancel.
|
||||
async fn or_cancel<T>(fut: impl std::future::Future<Output = T>, cancel: &AtomicBool) -> Result<T, DownloadError> {
|
||||
tokio::pin!(fut);
|
||||
loop {
|
||||
tokio::select! {
|
||||
v = &mut fut => return Ok(v),
|
||||
_ = tokio::time::sleep(std::time::Duration::from_millis(200)) => {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return Err(DownloadError("cancelled".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn e(what: &str) -> impl FnOnce(std::io::Error) -> DownloadError + '_ {
|
||||
move |err| DownloadError(format!("{what}: {err}"))
|
||||
}
|
||||
@@ -49,7 +65,8 @@ pub async fn download(url: &str, dest: &Path, sha256: &str, progress: &(dyn Fn(u
|
||||
if have > 0 {
|
||||
req = req.header(reqwest::header::RANGE, format!("bytes={have}-"));
|
||||
}
|
||||
let resp = req.send().await.map_err(|x| DownloadError(format!("could not download {url}: {x}")))?;
|
||||
let resp = or_cancel(req.send(), cancel).await?
|
||||
.map_err(|x| DownloadError(format!("could not download {url}: {x}")))?;
|
||||
let status = resp.status();
|
||||
if status.as_u16() == 416 && have > 0 {
|
||||
// The part file already holds every byte: check it below.
|
||||
@@ -64,7 +81,15 @@ pub async fn download(url: &str, dest: &Path, sha256: &str, progress: &(dyn Fn(u
|
||||
progress(done, total);
|
||||
if status.as_u16() != 416 {
|
||||
let mut stream = resp.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
loop {
|
||||
let next = match or_cancel(stream.next(), cancel).await {
|
||||
Ok(n) => n,
|
||||
Err(c) => {
|
||||
file.flush().await.ok(); // what arrived stays for the next try
|
||||
return Err(c);
|
||||
}
|
||||
};
|
||||
let Some(chunk) = next else { break };
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
file.flush().await.ok();
|
||||
return Err(DownloadError("cancelled".into()));
|
||||
@@ -168,6 +193,47 @@ mod tests {
|
||||
assert!(!dest2.exists() && !PathBuf::from(format!("{}.part", dest2.display())).exists());
|
||||
}
|
||||
|
||||
/// Accepts, optionally sends headers and the first `bytes`, then holds the connection silently.
|
||||
async fn stall(headers: bool, bytes: usize) -> String {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}/f.bin", l.local_addr().unwrap());
|
||||
tokio::spawn(async move {
|
||||
let mut held = vec![];
|
||||
while let Ok((mut s, _)) = l.accept().await {
|
||||
let mut buf = vec![0u8; 4096];
|
||||
let _ = s.read(&mut buf).await;
|
||||
if headers {
|
||||
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\n\r\n").await;
|
||||
let _ = s.write_all(&vec![7u8; bytes]).await;
|
||||
}
|
||||
held.push(s);
|
||||
}
|
||||
});
|
||||
url
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_silent_server_does_not_hold_a_cancel() {
|
||||
for (headers, bytes) in [(false, 0), (true, 5000)] {
|
||||
let url = stall(headers, bytes).await;
|
||||
let dest = temp_dir("dlstall").join("f.bin");
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let c2 = cancel.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
|
||||
c2.store(true, Ordering::Relaxed);
|
||||
});
|
||||
let t = std::time::Instant::now();
|
||||
let err = download(&url, &dest, &"0".repeat(64), &|_, _| {}, &cancel).await.unwrap_err();
|
||||
assert_eq!(err.0, "cancelled", "headers={headers}");
|
||||
assert!(t.elapsed() < std::time::Duration::from_secs(3), "{:?}", t.elapsed());
|
||||
if headers {
|
||||
let part = std::fs::metadata(format!("{}.part", dest.display())).unwrap().len();
|
||||
assert_eq!(part, bytes as u64, "what arrived stays for the next try");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_keeps_the_part_for_the_next_try() {
|
||||
let (body, sum) = data();
|
||||
|
||||
@@ -164,11 +164,23 @@ impl Ctx {
|
||||
/// Run a program with the layout's environment; each output line becomes the step's note and an
|
||||
/// `install` log line. Fails on a non-zero exit, with the last line.
|
||||
pub async fn run(&self, program: &Path, args: &[&str]) -> Result<(), String> {
|
||||
self.run_in(program, args, None).await
|
||||
}
|
||||
|
||||
/// `run` in folder `cwd` (paths never go into a command string, so no quoting can break).
|
||||
/// The program and everything it starts form one process group (a job on Windows) so a cancel
|
||||
/// ends all of them, not only the first.
|
||||
pub async fn run_in(&self, program: &Path, args: &[&str], cwd: Option<&Path>) -> Result<(), String> {
|
||||
let mut cmd = tokio::process::Command::new(program);
|
||||
cmd.args(args).envs(self.layout.env()).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
if let Some(dir) = cwd {
|
||||
cmd.current_dir(dir);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
|
||||
#[cfg(unix)]
|
||||
cmd.process_group(0);
|
||||
let _ = std::fs::create_dir_all(self.layout.data.join("cache").join("tmp"));
|
||||
let name = program.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
|
||||
crate::info_log!("install", "run {name} {}", args.first().copied().unwrap_or(""));
|
||||
@@ -195,16 +207,18 @@ impl Ctx {
|
||||
let (a, b) = (pump(Box::new(out)), pump(Box::new(err)));
|
||||
let status = loop {
|
||||
tokio::select! {
|
||||
s = child.wait() => break s.map_err(|e| e.to_string())?,
|
||||
s = child.wait() => break Some(s.map_err(|e| e.to_string())?),
|
||||
_ = tokio::time::sleep(std::time::Duration::from_millis(200)) => {
|
||||
if self.cancel.load(Ordering::Relaxed) {
|
||||
let _ = child.kill().await;
|
||||
return Err("cancelled".into());
|
||||
kill_tree(&mut child).await;
|
||||
break None;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let _ = (a.await, b.await);
|
||||
// The pipes close once every process holding them is gone.
|
||||
let _ = tokio::time::timeout(std::time::Duration::from_secs(5), async { let _ = (a.await, b.await); }).await;
|
||||
let Some(status) = status else { return Err("cancelled".into()) };
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -213,6 +227,26 @@ impl Ctx {
|
||||
}
|
||||
}
|
||||
|
||||
/// End `child` and every process it started, and wait for it.
|
||||
async fn kill_tree(child: &mut tokio::process::Child) {
|
||||
if let Some(pid) = child.id() {
|
||||
#[cfg(unix)]
|
||||
// SAFETY: a plain signal to the child's own process group (made with process_group(0)).
|
||||
unsafe {
|
||||
libc::kill(-(pid as i32), libc::SIGKILL);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let mut tk = tokio::process::Command::new("taskkill");
|
||||
tk.args(["/T", "/F", "/PID", &pid.to_string()]).stdin(Stdio::null()).stdout(Stdio::null())
|
||||
.stderr(Stdio::null()).creation_flags(0x0800_0000);
|
||||
let _ = tk.status().await;
|
||||
}
|
||||
}
|
||||
let _ = child.kill().await;
|
||||
let _ = child.wait().await;
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Step: Send + Sync {
|
||||
fn id(&self) -> &'static str;
|
||||
@@ -266,18 +300,15 @@ impl Installer {
|
||||
self.steps.iter().all(|s| self.is_done(s.as_ref()))
|
||||
}
|
||||
|
||||
/// Run the steps not done yet, in order, in the background, then `finished` once all are done;
|
||||
/// false if already running.
|
||||
pub fn start_then(self: &Arc<Self>, finished: impl FnOnce(&Layout) + Send + 'static) -> bool {
|
||||
/// Run the steps not done yet, in order, in the background; false if already running.
|
||||
pub fn start(self: &Arc<Self>) -> bool {
|
||||
if self.running.swap(true, Ordering::SeqCst) {
|
||||
return false;
|
||||
}
|
||||
self.cancel.store(false, Ordering::SeqCst);
|
||||
let me = self.clone();
|
||||
tokio::spawn(async move {
|
||||
if me.run_all().await {
|
||||
finished(&me.layout);
|
||||
}
|
||||
me.run_all().await;
|
||||
me.running.store(false, Ordering::SeqCst);
|
||||
});
|
||||
true
|
||||
@@ -499,12 +530,13 @@ impl Step for StrataStep {
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
let home = ctx.layout.strata_home();
|
||||
std::fs::create_dir_all(&home).map_err(|e| e.to_string())?;
|
||||
// The folder goes in as the working folder, never inside the script text.
|
||||
if cfg!(windows) {
|
||||
let script = format!("Set-Location -LiteralPath '{}'; irm {STRATA_INSTALL}.ps1 | iex", home.display());
|
||||
ctx.run(Path::new("powershell"), &["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &script]).await?;
|
||||
let script = format!("irm {STRATA_INSTALL}.ps1 | iex");
|
||||
ctx.run_in(Path::new("powershell"), &["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &script], Some(&home)).await?;
|
||||
} else {
|
||||
let script = format!("cd '{}' && curl -fsSL {STRATA_INSTALL}.sh -o install.sh && sh install.sh", home.display());
|
||||
ctx.run(Path::new("sh"), &["-c", &script]).await?;
|
||||
let script = format!("curl -fsSL {STRATA_INSTALL}.sh -o install.sh && sh install.sh");
|
||||
ctx.run_in(Path::new("sh"), &["-c", &script], Some(&home)).await?;
|
||||
}
|
||||
if !ctx.layout.strata_exe().exists() {
|
||||
return Err("Strata 설치가 끝났지만 실행 파일이 없습니다".into());
|
||||
@@ -533,10 +565,25 @@ pub fn apply_runtime_settings(settings_file: &Path, layout: &Layout) -> Result<(
|
||||
crate::datadir::write_atomic(settings_file, text.as_bytes()).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// The last step: the settings point at the runtime; a failure shows and can be retried like any step.
|
||||
struct SettingsStep;
|
||||
#[async_trait]
|
||||
impl Step for SettingsStep {
|
||||
fn id(&self) -> &'static str { "settings" }
|
||||
fn label(&self) -> &'static str { "설정 반영" }
|
||||
fn size_hint(&self) -> &'static str { "" }
|
||||
fn version(&self, layout: &Layout) -> String {
|
||||
format!("{}|{}|{}", layout.venv_python("worker").display(), layout.venv_python("melo").display(), layout.src.display())
|
||||
}
|
||||
async fn run(&self, ctx: &Ctx) -> Result<(), String> {
|
||||
apply_runtime_settings(&ctx.layout.data.join("config").join("settings.json"), &ctx.layout)
|
||||
}
|
||||
}
|
||||
|
||||
/// The install steps, in order.
|
||||
pub fn steps() -> Vec<Box<dyn Step>> {
|
||||
vec![Box::new(UvStep), Box::new(WorkerEnvStep), Box::new(MeloEnvStep), Box::new(SttModelStep),
|
||||
Box::new(TtsModelStep), Box::new(StrataStep)]
|
||||
Box::new(TtsModelStep), Box::new(StrataStep), Box::new(SettingsStep)]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -626,6 +673,62 @@ mod tests {
|
||||
assert!(t.elapsed() < std::time::Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_cancel_ends_the_command_and_everything_it_started() {
|
||||
let l = layout("inst6");
|
||||
let status = Arc::new(Mutex::new(vec![StepStatus { id: "x".into(), label: "".into(), size_hint: "".into(),
|
||||
state: StepState::Running, bytes: None, note: None }]));
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let ctx = Ctx { layout: l.clone(), status, index: 0, cancel: cancel.clone() };
|
||||
let pidfile = l.data.join("child.pid");
|
||||
let script = format!("sleep 60 & echo $! > {}; wait", pidfile.display());
|
||||
let c2 = cancel.clone();
|
||||
let pf = pidfile.clone();
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..50 {
|
||||
if pf.exists() { break; }
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
c2.store(true, Ordering::SeqCst);
|
||||
});
|
||||
assert_eq!(ctx.run(Path::new("sh"), &["-c", &script]).await.unwrap_err(), "cancelled");
|
||||
let pid: i32 = std::fs::read_to_string(&pidfile).unwrap().trim().parse().unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
||||
// SAFETY: signal 0 only asks whether the process exists.
|
||||
let alive = unsafe { libc::kill(pid, 0) } == 0;
|
||||
assert!(!alive, "the child started by the command is gone too");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn a_folder_with_a_quote_is_passed_as_the_working_folder() {
|
||||
let l = layout("inst7");
|
||||
let dir = l.data.join("it's here");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let status = Arc::new(Mutex::new(vec![StepStatus { id: "x".into(), label: "".into(), size_hint: "".into(),
|
||||
state: StepState::Running, bytes: None, note: None }]));
|
||||
let ctx = Ctx { layout: l.clone(), status: status.clone(), index: 0, cancel: Arc::new(AtomicBool::new(false)) };
|
||||
ctx.run_in(Path::new("sh"), &["-c", "touch made && pwd"], Some(&dir)).await.unwrap();
|
||||
assert!(dir.join("made").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_settings_step_is_part_of_the_install_and_can_fail_and_be_retried() {
|
||||
let l = layout("inst8");
|
||||
let settings = l.data.join("config").join("settings.json");
|
||||
std::fs::create_dir_all(settings.parent().unwrap()).unwrap();
|
||||
std::fs::write(&settings, "{ broken").unwrap();
|
||||
let i = Installer::new(l.clone(), vec![Box::new(SettingsStep)]);
|
||||
assert!(!i.run_all().await);
|
||||
assert_eq!(i.status()["steps"][0]["state"], "failed");
|
||||
assert!(!i.all_done(), "not done while the settings could not be written");
|
||||
std::fs::write(&settings, "{}").unwrap();
|
||||
assert!(i.run_all().await && i.all_done());
|
||||
assert!(std::fs::read_to_string(&settings).unwrap().contains("--brain"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_settings_point_at_the_installed_runtime_and_keep_the_rest() {
|
||||
let l = layout("inst5");
|
||||
|
||||
Reference in New Issue
Block a user