- A cancelled command ends with every process it started (process group on Unix, taskkill /T on Windows). - Waiting for a response or the next chunk gives way to a cancel, so a silent server cannot hold the install. - Pointing the settings at the runtime is the last install step: a failure shows as failed and can be retried; done means written. - Strata's installer gets its folder as the working folder, not inside the script text. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
254 lines
12 KiB
Rust
254 lines
12 KiB
Rust
//! Downloads for the installer (`app/src/admin.spec.md`, "Install"): resumed where they stopped, and
|
|
//! checked against a pinned SHA-256 before they are used.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
use futures_util::StreamExt;
|
|
use sha2::{Digest, Sha256};
|
|
use tokio::io::AsyncWriteExt;
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct DownloadError(pub String);
|
|
|
|
impl std::fmt::Display for DownloadError {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.write_str(&self.0)
|
|
}
|
|
}
|
|
impl std::error::Error for DownloadError {}
|
|
|
|
/// Await `fut`, giving up when `cancel` is set (checked every 200 ms): a server that keeps the
|
|
/// connection open and sends nothing cannot hold a cancel.
|
|
async fn or_cancel<T>(fut: impl std::future::Future<Output = T>, cancel: &AtomicBool) -> Result<T, DownloadError> {
|
|
tokio::pin!(fut);
|
|
loop {
|
|
tokio::select! {
|
|
v = &mut fut => return Ok(v),
|
|
_ = tokio::time::sleep(std::time::Duration::from_millis(200)) => {
|
|
if cancel.load(Ordering::Relaxed) {
|
|
return Err(DownloadError("cancelled".into()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn e(what: &str) -> impl FnOnce(std::io::Error) -> DownloadError + '_ {
|
|
move |err| DownloadError(format!("{what}: {err}"))
|
|
}
|
|
|
|
pub fn sha256_file(path: &Path) -> std::io::Result<String> {
|
|
let mut f = std::fs::File::open(path)?;
|
|
let mut h = Sha256::new();
|
|
std::io::copy(&mut f, &mut h)?;
|
|
Ok(format!("{:x}", h.finalize()))
|
|
}
|
|
|
|
/// Download `url` to `dest`, checked against `sha256` (lowercase hex). The bytes go to
|
|
/// `dest.part` first; an interrupted download continues from there (HTTP Range), a server that
|
|
/// ignores the range starts it over. A wrong checksum deletes the part file, so the next try starts
|
|
/// clean. `progress(done, total)` is called as bytes arrive; `cancel` stops it between chunks (the
|
|
/// part file stays for the next try). An existing `dest` with the right checksum is kept as it is.
|
|
pub async fn download(url: &str, dest: &Path, sha256: &str, progress: &(dyn Fn(u64, Option<u64>) + Sync),
|
|
cancel: &AtomicBool) -> Result<(), DownloadError> {
|
|
if dest.exists() && sha256_file(dest).map_err(e("could not read the download"))? == sha256 {
|
|
return Ok(());
|
|
}
|
|
if let Some(dir) = dest.parent() {
|
|
tokio::fs::create_dir_all(dir).await.map_err(e("could not create the download folder"))?;
|
|
}
|
|
let part = PathBuf::from(format!("{}.part", dest.display()));
|
|
let have = tokio::fs::metadata(&part).await.map(|m| m.len()).unwrap_or(0);
|
|
let client = reqwest::Client::builder().build().map_err(|x| DownloadError(x.to_string()))?;
|
|
let mut req = client.get(url);
|
|
if have > 0 {
|
|
req = req.header(reqwest::header::RANGE, format!("bytes={have}-"));
|
|
}
|
|
let resp = or_cancel(req.send(), cancel).await?
|
|
.map_err(|x| DownloadError(format!("could not download {url}: {x}")))?;
|
|
let status = resp.status();
|
|
if status.as_u16() == 416 && have > 0 {
|
|
// The part file already holds every byte: check it below.
|
|
} else if !status.is_success() {
|
|
return Err(DownloadError(format!("download of {url} failed: HTTP {status}")));
|
|
}
|
|
let resumed = status.as_u16() == 206;
|
|
let total = resp.content_length().map(|n| if resumed { n + have } else { n });
|
|
let mut file = tokio::fs::OpenOptions::new().create(true).write(true).append(resumed).truncate(!resumed && status.as_u16() != 416)
|
|
.open(&part).await.map_err(e("could not write the download"))?;
|
|
let mut done = if resumed || status.as_u16() == 416 { have } else { 0 };
|
|
progress(done, total);
|
|
if status.as_u16() != 416 {
|
|
let mut stream = resp.bytes_stream();
|
|
loop {
|
|
let next = match or_cancel(stream.next(), cancel).await {
|
|
Ok(n) => n,
|
|
Err(c) => {
|
|
file.flush().await.ok(); // what arrived stays for the next try
|
|
return Err(c);
|
|
}
|
|
};
|
|
let Some(chunk) = next else { break };
|
|
if cancel.load(Ordering::Relaxed) {
|
|
file.flush().await.ok();
|
|
return Err(DownloadError("cancelled".into()));
|
|
}
|
|
let chunk = chunk.map_err(|x| DownloadError(format!("download of {url} broke off: {x}")))?;
|
|
file.write_all(&chunk).await.map_err(e("could not write the download"))?;
|
|
done += chunk.len() as u64;
|
|
progress(done, total);
|
|
}
|
|
}
|
|
file.flush().await.map_err(e("could not write the download"))?;
|
|
drop(file);
|
|
let got = sha256_file(&part).map_err(e("could not read the download"))?;
|
|
if got != sha256 {
|
|
let _ = tokio::fs::remove_file(&part).await;
|
|
return Err(DownloadError(format!("the download of {url} is damaged (checksum {got}, expected {sha256})")));
|
|
}
|
|
tokio::fs::rename(&part, dest).await.map_err(e("could not move the download into place"))?;
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::datadir::temp_dir;
|
|
use std::sync::{Arc, Mutex};
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
use tokio::net::TcpListener;
|
|
|
|
/// A file server that honours `Range: bytes=N-` (unless `ranges` is off) and can stop after
|
|
/// `cut` bytes of a response, like a broken connection. Records the ranges asked for.
|
|
async fn serve(body: Vec<u8>, ranges: bool, cut: Option<usize>) -> (String, Arc<Mutex<Vec<String>>>) {
|
|
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let url = format!("http://{}/f.bin", l.local_addr().unwrap());
|
|
let asked = Arc::new(Mutex::new(vec![]));
|
|
let a2 = asked.clone();
|
|
let cut = Arc::new(Mutex::new(cut));
|
|
tokio::spawn(async move {
|
|
loop {
|
|
let Ok((mut s, _)) = l.accept().await else { return };
|
|
let mut buf = vec![0u8; 4096];
|
|
let n = s.read(&mut buf).await.unwrap_or(0);
|
|
let head = String::from_utf8_lossy(&buf[..n]).to_string();
|
|
let from = head.lines().find_map(|l| l.strip_prefix("range: bytes=").or(l.strip_prefix("Range: bytes=")))
|
|
.and_then(|r| r.trim_end_matches('-').parse::<usize>().ok());
|
|
a2.lock().unwrap().push(from.map(|f| f.to_string()).unwrap_or_default());
|
|
let (status, slice) = match from {
|
|
Some(f) if ranges && f >= body.len() => ("416 Range Not Satisfiable", &body[0..0]),
|
|
Some(f) if ranges => ("206 Partial Content", &body[f..]),
|
|
_ => ("200 OK", &body[..]),
|
|
};
|
|
let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", slice.len());
|
|
let _ = s.write_all(head.as_bytes()).await;
|
|
let send = cut.lock().unwrap().take().map_or(slice.len(), |c| c.min(slice.len()));
|
|
let _ = s.write_all(&slice[..send]).await;
|
|
}
|
|
});
|
|
(url, asked)
|
|
}
|
|
|
|
fn data() -> (Vec<u8>, String) {
|
|
let b: Vec<u8> = (0..200_000u32).map(|i| (i * 31 % 251) as u8).collect();
|
|
let h = format!("{:x}", Sha256::digest(&b));
|
|
(b, h)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_broken_download_resumes_where_it_stopped() {
|
|
let (body, sum) = data();
|
|
let (url, asked) = serve(body.clone(), true, Some(70_000)).await;
|
|
let dest = temp_dir("dl1").join("f.bin");
|
|
let never = AtomicBool::new(false);
|
|
assert!(download(&url, &dest, &sum, &|_, _| {}, &never).await.is_err(), "the first try breaks off");
|
|
assert!(!dest.exists());
|
|
let seen = Mutex::new(vec![]);
|
|
download(&url, &dest, &sum, &|d, t| seen.lock().unwrap().push((d, t)), &never).await.unwrap();
|
|
assert_eq!(std::fs::read(&dest).unwrap(), body);
|
|
assert_eq!(asked.lock().unwrap().as_slice(), ["", "70000"]);
|
|
let seen = seen.lock().unwrap();
|
|
assert_eq!(seen.first(), Some(&(70_000, Some(200_000))));
|
|
assert_eq!(seen.last(), Some(&(200_000, Some(200_000))));
|
|
// Present and right: nothing is fetched again.
|
|
download(&url, &dest, &sum, &|_, _| {}, &never).await.unwrap();
|
|
assert_eq!(asked.lock().unwrap().len(), 2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_server_without_ranges_starts_over_and_a_bad_checksum_is_thrown_away() {
|
|
let (body, sum) = data();
|
|
let (url, _) = serve(body.clone(), false, Some(50_000)).await;
|
|
let dest = temp_dir("dl2").join("f.bin");
|
|
let never = AtomicBool::new(false);
|
|
let _ = download(&url, &dest, &sum, &|_, _| {}, &never).await;
|
|
download(&url, &dest, &sum, &|_, _| {}, &never).await.unwrap();
|
|
assert_eq!(std::fs::read(&dest).unwrap(), body);
|
|
|
|
let wrong = "0".repeat(64);
|
|
let dest2 = temp_dir("dl3").join("f.bin");
|
|
let err = download(&url, &dest2, &wrong, &|_, _| {}, &never).await.unwrap_err();
|
|
assert!(err.0.contains("damaged"));
|
|
assert!(!dest2.exists() && !PathBuf::from(format!("{}.part", dest2.display())).exists());
|
|
}
|
|
|
|
/// Accepts, optionally sends headers and the first `bytes`, then holds the connection silently.
|
|
async fn stall(headers: bool, bytes: usize) -> String {
|
|
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let url = format!("http://{}/f.bin", l.local_addr().unwrap());
|
|
tokio::spawn(async move {
|
|
let mut held = vec![];
|
|
while let Ok((mut s, _)) = l.accept().await {
|
|
let mut buf = vec![0u8; 4096];
|
|
let _ = s.read(&mut buf).await;
|
|
if headers {
|
|
let _ = s.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\n\r\n").await;
|
|
let _ = s.write_all(&vec![7u8; bytes]).await;
|
|
}
|
|
held.push(s);
|
|
}
|
|
});
|
|
url
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_silent_server_does_not_hold_a_cancel() {
|
|
for (headers, bytes) in [(false, 0), (true, 5000)] {
|
|
let url = stall(headers, bytes).await;
|
|
let dest = temp_dir("dlstall").join("f.bin");
|
|
let cancel = Arc::new(AtomicBool::new(false));
|
|
let c2 = cancel.clone();
|
|
tokio::spawn(async move {
|
|
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
|
|
c2.store(true, Ordering::Relaxed);
|
|
});
|
|
let t = std::time::Instant::now();
|
|
let err = download(&url, &dest, &"0".repeat(64), &|_, _| {}, &cancel).await.unwrap_err();
|
|
assert_eq!(err.0, "cancelled", "headers={headers}");
|
|
assert!(t.elapsed() < std::time::Duration::from_secs(3), "{:?}", t.elapsed());
|
|
if headers {
|
|
let part = std::fs::metadata(format!("{}.part", dest.display())).unwrap().len();
|
|
assert_eq!(part, bytes as u64, "what arrived stays for the next try");
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn cancelling_keeps_the_part_for_the_next_try() {
|
|
let (body, sum) = data();
|
|
let (url, asked) = serve(body.clone(), true, None).await;
|
|
let dest = temp_dir("dl4").join("f.bin");
|
|
let cancel = AtomicBool::new(false);
|
|
let err = download(&url, &dest, &sum, &|d, _| if d > 0 { cancel.store(true, Ordering::Relaxed) }, &cancel)
|
|
.await.unwrap_err();
|
|
assert_eq!(err.0, "cancelled");
|
|
let part = std::fs::metadata(format!("{}.part", dest.display())).unwrap().len();
|
|
assert!(part > 0 && part < body.len() as u64);
|
|
cancel.store(false, Ordering::Relaxed);
|
|
download(&url, &dest, &sum, &|_, _| {}, &cancel).await.unwrap();
|
|
assert_eq!(std::fs::read(&dest).unwrap(), body);
|
|
assert_eq!(asked.lock().unwrap()[1], part.to_string());
|
|
}
|
|
}
|