Files
javis_bot/app/src/download.rs
javis-bot 1a6d6447f6 fix(app): install cancels whole process trees and silent downloads, and finishes only with the settings written
- A cancelled command ends with every process it started (process group on
  Unix, taskkill /T on Windows).
- Waiting for a response or the next chunk gives way to a cancel, so a
  silent server cannot hold the install.
- Pointing the settings at the runtime is the last install step: a failure
  shows as failed and can be retried; done means written.
- Strata's installer gets its folder as the working folder, not inside the
  script text.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 17:17:25 +09:00

254 lines
12 KiB
Rust

//! Downloads for the installer (`app/src/admin.spec.md`, "Install"): resumed where they stopped, and
//! checked against a pinned SHA-256 before they are used.
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use futures_util::StreamExt;
use sha2::{Digest, Sha256};
use tokio::io::AsyncWriteExt;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DownloadError(pub String);
impl std::fmt::Display for DownloadError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for DownloadError {}
/// Await `fut`, giving up when `cancel` is set (checked every 200 ms): a server that keeps the
/// connection open and sends nothing cannot hold a cancel.
async fn or_cancel<T>(fut: impl std::future::Future<Output = T>, cancel: &AtomicBool) -> Result<T, DownloadError> {
tokio::pin!(fut);
loop {
tokio::select! {
v = &mut fut => return Ok(v),
_ = tokio::time::sleep(std::time::Duration::from_millis(200)) => {
if cancel.load(Ordering::Relaxed) {
return Err(DownloadError("cancelled".into()));
}
}
}
}
}
fn e(what: &str) -> impl FnOnce(std::io::Error) -> DownloadError + '_ {
move |err| DownloadError(format!("{what}: {err}"))
}
pub fn sha256_file(path: &Path) -> std::io::Result<String> {
let mut f = std::fs::File::open(path)?;
let mut h = Sha256::new();
std::io::copy(&mut f, &mut h)?;
Ok(format!("{:x}", h.finalize()))
}
/// Download `url` to `dest`, checked against `sha256` (lowercase hex). The bytes go to
/// `dest.part` first; an interrupted download continues from there (HTTP Range), a server that
/// ignores the range starts it over. A wrong checksum deletes the part file, so the next try starts
/// clean. `progress(done, total)` is called as bytes arrive; `cancel` stops it between chunks (the
/// part file stays for the next try). An existing `dest` with the right checksum is kept as it is.
pub async fn download(url: &str, dest: &Path, sha256: &str, progress: &(dyn Fn(u64, Option<u64>) + Sync),
cancel: &AtomicBool) -> Result<(), DownloadError> {
if dest.exists() && sha256_file(dest).map_err(e("could not read the download"))? == sha256 {
return Ok(());
}
if let Some(dir) = dest.parent() {
tokio::fs::create_dir_all(dir).await.map_err(e("could not create the download folder"))?;
}
let part = PathBuf::from(format!("{}.part", dest.display()));
let have = tokio::fs::metadata(&part).await.map(|m| m.len()).unwrap_or(0);
let client = reqwest::Client::builder().build().map_err(|x| DownloadError(x.to_string()))?;
let mut req = client.get(url);
if have > 0 {
req = req.header(reqwest::header::RANGE, format!("bytes={have}-"));
}
let resp = or_cancel(req.send(), cancel).await?
.map_err(|x| DownloadError(format!("could not download {url}: {x}")))?;
let status = resp.status();
if status.as_u16() == 416 && have > 0 {
// The part file already holds every byte: check it below.
} else if !status.is_success() {
return Err(DownloadError(format!("download of {url} failed: HTTP {status}")));
}
let resumed = status.as_u16() == 206;
let total = resp.content_length().map(|n| if resumed { n + have } else { n });
let mut file = tokio::fs::OpenOptions::new().create(true).write(true).append(resumed).truncate(!resumed && status.as_u16() != 416)
.open(&part).await.map_err(e("could not write the download"))?;
let mut done = if resumed || status.as_u16() == 416 { have } else { 0 };
progress(done, total);
if status.as_u16() != 416 {
let mut stream = resp.bytes_stream();
loop {
let next = match or_cancel(stream.next(), cancel).await {
Ok(n) => n,
Err(c) => {
file.flush().await.ok(); // what arrived stays for the next try
return Err(c);
}
};
let Some(chunk) = next else { break };
if cancel.load(Ordering::Relaxed) {
file.flush().await.ok();
return Err(DownloadError("cancelled".into()));
}
let chunk = chunk.map_err(|x| DownloadError(format!("download of {url} broke off: {x}")))?;
file.write_all(&chunk).await.map_err(e("could not write the download"))?;
done += chunk.len() as u64;
progress(done, total);
}
}
file.flush().await.map_err(e("could not write the download"))?;
drop(file);
let got = sha256_file(&part).map_err(e("could not read the download"))?;
if got != sha256 {
let _ = tokio::fs::remove_file(&part).await;
return Err(DownloadError(format!("the download of {url} is damaged (checksum {got}, expected {sha256})")));
}
tokio::fs::rename(&part, dest).await.map_err(e("could not move the download into place"))?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::datadir::temp_dir;
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
/// A file server that honours `Range: bytes=N-` (unless `ranges` is off) and can stop after
/// `cut` bytes of a response, like a broken connection. Records the ranges asked for.
async fn serve(body: Vec<u8>, ranges: bool, cut: Option<usize>) -> (String, Arc<Mutex<Vec<String>>>) {
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}/f.bin", l.local_addr().unwrap());
let asked = Arc::new(Mutex::new(vec![]));
let a2 = asked.clone();
let cut = Arc::new(Mutex::new(cut));
tokio::spawn(async move {
loop {
let Ok((mut s, _)) = l.accept().await else { return };
let mut buf = vec![0u8; 4096];
let n = s.read(&mut buf).await.unwrap_or(0);
let head = String::from_utf8_lossy(&buf[..n]).to_string();
let from = head.lines().find_map(|l| l.strip_prefix("range: bytes=").or(l.strip_prefix("Range: bytes=")))
.and_then(|r| r.trim_end_matches('-').parse::<usize>().ok());
a2.lock().unwrap().push(from.map(|f| f.to_string()).unwrap_or_default());
let (status, slice) = match from {
Some(f) if ranges && f >= body.len() => ("416 Range Not Satisfiable", &body[0..0]),
Some(f) if ranges => ("206 Partial Content", &body[f..]),
_ => ("200 OK", &body[..]),
};
let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", slice.len());
let _ = s.write_all(head.as_bytes()).await;
let send = cut.lock().unwrap().take().map_or(slice.len(), |c| c.min(slice.len()));
let _ = s.write_all(&slice[..send]).await;
}
});
(url, asked)
}
fn data() -> (Vec<u8>, String) {
let b: Vec<u8> = (0..200_000u32).map(|i| (i * 31 % 251) as u8).collect();
let h = format!("{:x}", Sha256::digest(&b));
(b, h)
}
#[tokio::test]
async fn a_broken_download_resumes_where_it_stopped() {
let (body, sum) = data();
let (url, asked) = serve(body.clone(), true, Some(70_000)).await;
let dest = temp_dir("dl1").join("f.bin");
let never = AtomicBool::new(false);
assert!(download(&url, &dest, &sum, &|_, _| {}, &never).await.is_err(), "the first try breaks off");
assert!(!dest.exists());
let seen = Mutex::new(vec![]);
download(&url, &dest, &sum, &|d, t| seen.lock().unwrap().push((d, t)), &never).await.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), body);
assert_eq!(asked.lock().unwrap().as_slice(), ["", "70000"]);
let seen = seen.lock().unwrap();
assert_eq!(seen.first(), Some(&(70_000, Some(200_000))));
assert_eq!(seen.last(), Some(&(200_000, Some(200_000))));
// Present and right: nothing is fetched again.
download(&url, &dest, &sum, &|_, _| {}, &never).await.unwrap();
assert_eq!(asked.lock().unwrap().len(), 2);
}
#[tokio::test]
async fn a_server_without_ranges_starts_over_and_a_bad_checksum_is_thrown_away() {
let (body, sum) = data();
let (url, _) = serve(body.clone(), false, Some(50_000)).await;
let dest = temp_dir("dl2").join("f.bin");
let never = AtomicBool::new(false);
let _ = download(&url, &dest, &sum, &|_, _| {}, &never).await;
download(&url, &dest, &sum, &|_, _| {}, &never).await.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), body);
let wrong = "0".repeat(64);
let dest2 = temp_dir("dl3").join("f.bin");
let err = download(&url, &dest2, &wrong, &|_, _| {}, &never).await.unwrap_err();
assert!(err.0.contains("damaged"));
assert!(!dest2.exists() && !PathBuf::from(format!("{}.part", dest2.display())).exists());
}
/// Accepts, optionally sends headers and the first `bytes`, then holds the connection silently.
async fn stall(headers: bool, bytes: usize) -> String {
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}/f.bin", l.local_addr().unwrap());
tokio::spawn(async move {
let mut held = vec![];
while let Ok((mut s, _)) = l.accept().await {
let mut buf = vec![0u8; 4096];
let _ = s.read(&mut buf).await;
if headers {
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\n\r\n").await;
let _ = s.write_all(&vec![7u8; bytes]).await;
}
held.push(s);
}
});
url
}
#[tokio::test]
async fn a_silent_server_does_not_hold_a_cancel() {
for (headers, bytes) in [(false, 0), (true, 5000)] {
let url = stall(headers, bytes).await;
let dest = temp_dir("dlstall").join("f.bin");
let cancel = Arc::new(AtomicBool::new(false));
let c2 = cancel.clone();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
c2.store(true, Ordering::Relaxed);
});
let t = std::time::Instant::now();
let err = download(&url, &dest, &"0".repeat(64), &|_, _| {}, &cancel).await.unwrap_err();
assert_eq!(err.0, "cancelled", "headers={headers}");
assert!(t.elapsed() < std::time::Duration::from_secs(3), "{:?}", t.elapsed());
if headers {
let part = std::fs::metadata(format!("{}.part", dest.display())).unwrap().len();
assert_eq!(part, bytes as u64, "what arrived stays for the next try");
}
}
}
#[tokio::test]
async fn cancelling_keeps_the_part_for_the_next_try() {
let (body, sum) = data();
let (url, asked) = serve(body.clone(), true, None).await;
let dest = temp_dir("dl4").join("f.bin");
let cancel = AtomicBool::new(false);
let err = download(&url, &dest, &sum, &|d, _| if d > 0 { cancel.store(true, Ordering::Relaxed) }, &cancel)
.await.unwrap_err();
assert_eq!(err.0, "cancelled");
let part = std::fs::metadata(format!("{}.part", dest.display())).unwrap().len();
assert!(part > 0 && part < body.len() as u64);
cancel.store(false, Ordering::Relaxed);
download(&url, &dest, &sum, &|_, _| {}, &cancel).await.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), body);
assert_eq!(asked.lock().unwrap()[1], part.to_string());
}
}