238 lines
8.0 KiB
Rust
238 lines
8.0 KiB
Rust
//! Raw archive downloads. Range pages are validated before advancing; no byte caps.
|
|
use crate::{Result, error, http, retention};
|
|
use std::io::{Read, Seek, SeekFrom, Write};
|
|
use wasi::http::types::Method;
|
|
|
|
pub struct Download {
|
|
pub path: String,
|
|
pub bytes: u64,
|
|
}
|
|
|
|
// Only immutable archive GETs are retried, never the command POST. Divide the
|
|
// remaining total budget so a stalled first attempt cannot consume every retry.
|
|
// Failed partial files remain explicitly partial; only verified downloads publish.
|
|
pub fn exec_download(
|
|
h: &str,
|
|
url: &str,
|
|
name: &str,
|
|
extension: &str,
|
|
deadline: u64,
|
|
expected: Option<u64>,
|
|
) -> Result<Download> {
|
|
let mut last = error("exec archive deadline exceeded");
|
|
for remaining in (1..=3).rev() {
|
|
let now = http::now();
|
|
if now >= deadline {
|
|
break;
|
|
}
|
|
let attempt_deadline = now.saturating_add((deadline - now) / remaining);
|
|
match download_verified(h, url, name, extension, attempt_deadline, false, expected) {
|
|
Ok(Some(saved)) => return Ok(saved),
|
|
Ok(None) => return Err(error("missing exec archive")),
|
|
Err(e) => {
|
|
// Validation/local storage errors are not transient network failures.
|
|
let retry = [
|
|
"HTTP failed:",
|
|
"HTTP start:",
|
|
"archive download failed:",
|
|
"deadline exceeded",
|
|
"incomplete archive page",
|
|
"HTTP 500",
|
|
"HTTP 502",
|
|
"HTTP 503",
|
|
"HTTP 504",
|
|
]
|
|
.iter()
|
|
.any(|message| e.message.contains(message));
|
|
if !retry {
|
|
return Err(e);
|
|
}
|
|
eprintln!(
|
|
"box archive GET attempt {} failed: {}",
|
|
4 - remaining,
|
|
e.message
|
|
);
|
|
last = e;
|
|
}
|
|
}
|
|
}
|
|
Err(last)
|
|
}
|
|
|
|
pub fn download(
|
|
h: &str,
|
|
url: &str,
|
|
name: &str,
|
|
extension: &str,
|
|
deadline: u64,
|
|
missing: bool,
|
|
) -> Result<Option<Download>> {
|
|
download_verified(h, url, name, extension, deadline, missing, None)
|
|
}
|
|
|
|
pub fn download_verified(
|
|
h: &str,
|
|
url: &str,
|
|
name: &str,
|
|
extension: &str,
|
|
deadline: u64,
|
|
missing: bool,
|
|
expected: Option<u64>,
|
|
) -> Result<Option<Download>> {
|
|
let mut offset = 0u64;
|
|
let mut total = None;
|
|
let mut output = None;
|
|
let result = (|| {
|
|
loop {
|
|
let range = format!("bytes={offset}-{}", offset.saturating_add(1024 * 1024 - 1));
|
|
let mut response = http::request_options(
|
|
h,
|
|
Method::Get,
|
|
url,
|
|
None,
|
|
30,
|
|
Some(deadline),
|
|
Some(&range),
|
|
missing && offset == 0,
|
|
)?;
|
|
if matches!(response.status, 404 | 405) {
|
|
return Ok(None);
|
|
}
|
|
let page = if response.status == 206 {
|
|
let (start, end, size) =
|
|
parse_range(response.content_range.as_deref().unwrap_or(""))?;
|
|
if start != offset || total.is_some_and(|t| t != size) {
|
|
return Err(error("inconsistent archive range"));
|
|
}
|
|
total = Some(size);
|
|
Some(end - start + 1)
|
|
} else if response.status == 200 && offset == 0 {
|
|
None
|
|
} else {
|
|
return Err(error("unexpected archive response status"));
|
|
};
|
|
if output.is_none() {
|
|
output = Some(retention::create(name, &format!("{extension}.partial"))?);
|
|
}
|
|
let (path, file) = output
|
|
.as_mut()
|
|
.ok_or_else(|| error("archive file unavailable"))?;
|
|
let count = std::io::copy(&mut response, file).map_err(|e| {
|
|
error(format!(
|
|
"archive download failed: {e}; partial file: {path}"
|
|
))
|
|
})?;
|
|
if page.is_some_and(|n| n != count) {
|
|
return Err(error(format!(
|
|
"incomplete archive page; partial file: {path}"
|
|
)));
|
|
}
|
|
offset = offset
|
|
.checked_add(count)
|
|
.ok_or_else(|| error("archive size overflow"))?;
|
|
if page.is_none() || total == Some(offset) {
|
|
if expected.is_some_and(|n| n != offset) {
|
|
return Err(error("archive byte total mismatch"));
|
|
}
|
|
file.flush()?;
|
|
// Reserve a collision-safe final name; replace only our own empty reservation.
|
|
let (complete, reservation) = retention::create(name, extension)?;
|
|
drop(reservation);
|
|
if let Err(e) = std::fs::rename(&*path, &complete) {
|
|
let _ = std::fs::remove_file(&complete);
|
|
return Err(e.into());
|
|
}
|
|
return Ok(Some(Download {
|
|
path: complete,
|
|
bytes: offset,
|
|
}));
|
|
}
|
|
}
|
|
})();
|
|
result.map_err(|mut e: crate::Error| {
|
|
if let Some((path, _)) = output {
|
|
e.message.push_str(&format!("; incomplete archive: {path}"));
|
|
}
|
|
e
|
|
})
|
|
}
|
|
fn parse_range(value: &str) -> Result<(u64, u64, u64)> {
|
|
let parsed = (|| {
|
|
let (span, size) = value.strip_prefix("bytes ")?.split_once('/')?;
|
|
let (start, end) = span.split_once('-')?;
|
|
Some((
|
|
start.parse::<u64>().ok()?,
|
|
end.parse::<u64>().ok()?,
|
|
size.parse::<u64>().ok()?,
|
|
))
|
|
})();
|
|
match parsed {
|
|
Some((s, e, n)) if s <= e && e < n => Ok((s, e, n)),
|
|
_ => Err(error("invalid archive Content-Range")),
|
|
}
|
|
}
|
|
|
|
// Presentation limit only: the complete raw response is always retained on disk.
|
|
pub const BUILD_EXCERPT_BYTES: u64 = 16 * 1024;
|
|
fn build_result(full: Download, service_truncated: bool) -> Result<crate::Log> {
|
|
let mut file = std::fs::File::open(&full.path)?;
|
|
let start = full.bytes.saturating_sub(BUILD_EXCERPT_BYTES);
|
|
file.seek(SeekFrom::Start(start))?;
|
|
let mut bytes = Vec::new();
|
|
file.take(BUILD_EXCERPT_BYTES).read_to_end(&mut bytes)?;
|
|
let mut notices = Vec::new();
|
|
if service_truncated {
|
|
notices.push("Full build-log endpoint unavailable (unsupported service or missing archive); legacy endpoint may return only the last 64 KiB, so earlier bytes may be unavailable.".to_owned());
|
|
}
|
|
if start > 0 {
|
|
notices.push(format!("Inline tail shows raw byte range {start}..{} (end exclusive); full {} bytes retained at {}.", full.bytes, full.bytes, full.path));
|
|
}
|
|
Ok(crate::Log {
|
|
text: String::from_utf8_lossy(&bytes).into_owned(),
|
|
path: full.path,
|
|
bytes: full.bytes,
|
|
service_truncated,
|
|
notice: if notices.is_empty() {
|
|
None
|
|
} else {
|
|
Some(notices.join(" "))
|
|
},
|
|
})
|
|
}
|
|
|
|
pub fn build(h: &str, id: &str, deadline: u64) -> Result<crate::Log> {
|
|
let base = crate::base(id)?;
|
|
let name = format!("{id}-build");
|
|
if let Some(full) = download(h, &format!("{base}/log/full"), &name, "log", deadline, true)? {
|
|
return build_result(full, false);
|
|
}
|
|
let legacy = download(h, &format!("{base}/log"), &name, "log", deadline, false)?
|
|
.ok_or_else(|| error("missing legacy build log"))?;
|
|
build_result(legacy, true)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
#[test]
|
|
fn range_pages_require_valid_totals() {
|
|
assert_eq!(
|
|
parse_range("bytes 0-1048575/1200003").unwrap(),
|
|
(0, 1048575, 1200003)
|
|
);
|
|
assert_eq!(
|
|
parse_range("bytes 1048576-1200002/1200003").unwrap(),
|
|
(1048576, 1200002, 1200003)
|
|
);
|
|
for invalid in [
|
|
"",
|
|
"bytes 1-0/2",
|
|
"bytes 0-2/2",
|
|
"bytes 0-0/0",
|
|
"bytes 0-1/*",
|
|
] {
|
|
assert!(parse_range(invalid).is_err());
|
|
}
|
|
}
|
|
}
|