Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions crates/engine/src/cmd/layer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4226,6 +4226,14 @@ fn add_cog_window(s: &mut Session, p: &Value, url: &str) -> CmdResult {
/// quote the address without its keys.
#[cfg(not(target_arch = "wasm32"))]
pub fn read_cog_window(asked: &WindowAsked) -> Result<CogWindow, String> {
read_cog_window_until(asked, &std::sync::atomic::AtomicBool::new(false))
}

/// [`read_cog_window`], giving up within a moment of `stop` being set (the app's Cancel of the
/// download, or its project closed): in the wait on the header, before each band and in the wait
/// on its tiles. Making the raster and writing its GeoTIFF, once the values are in, don't look.
#[cfg(not(target_arch = "wasm32"))]
pub fn read_cog_window_until(asked: &WindowAsked, stop: &std::sync::atomic::AtomicBool) -> Result<CogWindow, String> {
use crate::render::rustgis_tiles::cog;
use rustgis_data::raster::cog::Level;
let url = asked.url.as_str();
Expand All @@ -4234,7 +4242,7 @@ pub fn read_cog_window(asked: &WindowAsked) -> Result<CogWindow, String> {
let cell = asked.cell;
// The five minutes count from the header's read.
let started = std::time::Instant::now();
let info = cog::info_blocking(url).map_err(named)?;
let info = cog::info_until(url, stop).map_err(named)?;
let layout = &info.layout;
let (pw, ph) = (layout.transform.pixel_width, layout.transform.pixel_height);
if !(pw.is_finite() && ph.is_finite() && pw != 0.0 && ph != 0.0) {
Expand Down Expand Up @@ -4387,7 +4395,8 @@ pub fn read_cog_window(asked: &WindowAsked) -> Result<CogWindow, String> {
let (mut bands, mut rounded, mut overflowed) = (Vec::with_capacity(count), Vec::with_capacity(count), 0u64);
for band in 0..count {
band_in_time(band, count, started.elapsed(), WINDOW_WAIT).map_err(named)?;
let (values, changed, past) = cog::read_values_counted_blocking(&info, url, level, band, (c0, c1), (r0, r1)).map_err(named)?;
let (values, changed, past) =
cog::read_values_counted_blocking(&info, url, level, band, (c0, c1), (r0, r1), stop).map_err(named)?;
bands.push(values);
rounded.push(changed);
overflowed += past;
Expand Down
227 changes: 210 additions & 17 deletions crates/tiles/src/cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,52 @@ fn check_current() -> Result<(), String> {
check_still(&current_still())
}

/// How far a whole download has come, for whoever started it (the app's status bar): the bytes
/// read so far, and the total once the server said it (`Content-Length`; 0 while unknown). A file
/// of several parts (a Shapefile's) adds each part's total as it starts. The bytes are those
/// decoded, so a compressed answer may pass its total. Counted natively only.
#[derive(Debug, Default)]
pub struct Progress {
/// Bytes read.
pub done: AtomicU64,
/// Bytes expected, or 0 while unknown.
pub total: AtomicU64,
}

impl Progress {
/// The bytes read so far.
pub fn received(&self) -> u64 {
self.done.load(Ordering::Relaxed)
}

/// The part read, 0 to 1, once the total is known.
pub fn fraction(&self) -> Option<f32> {
let total = self.total.load(Ordering::Relaxed);
(total > 0).then(|| (self.received() as f64 / total as f64).min(1.0) as f32)
}
}

thread_local! { static PROGRESS: std::cell::RefCell<Option<Arc<Progress>>> = const { std::cell::RefCell::new(None) }; }

/// Runs `f` with downloads it starts counting into `progress` (taken along to their threads, as
/// the request's [`Still`] is).
pub fn with_progress<T>(progress: Option<Arc<Progress>>, f: impl FnOnce() -> T) -> T {
struct Restore(Option<Arc<Progress>>);
impl Drop for Restore {
fn drop(&mut self) {
PROGRESS.with(|p| *p.borrow_mut() = self.0.take());
}
}
let _restore = Restore(PROGRESS.with(|p| p.replace(progress)));
f()
}

/// What downloads started here count into ([`with_progress`]).
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
fn current_progress() -> Option<Arc<Progress>> {
PROGRESS.with(|p| p.borrow().clone())
}

/// Only a live request's failure describes the shared resource. A dropped or retired read
/// must let the next caller retry immediately under its own authority, without classifying cancellation by error text.
pub(crate) fn keep_failure(message: &str, still: &Option<Still>) -> bool {
Expand Down Expand Up @@ -104,6 +150,10 @@ pub const MAX_BYTES: usize = 16 << 20;
/// The error a fetcher gives a request it dropped unfetched (the view moved on): the address is
/// forgotten rather than failed, so it is fetched again when it is next wanted.
pub const DROPPED: &str = "dropped: the view moved on";

/// What a native body's read given up for its request's `Still` says (a download's Cancel): a
/// cancel, not a failure, so it stays out of the network history as [`DROPPED`] does.
pub(crate) const CANCELLED_READ: &str = "cancelled while it was read";
/// The end of a fetcher's error when a range was asked for and the server answered with the
/// whole file: a file read in place by range is downloaded whole instead.
pub const NO_RANGES: &str = "the server doesn't answer byte-range requests";
Expand Down Expand Up @@ -1256,7 +1306,7 @@ impl<'a> RequestDiagnostics<'a> {
pub fn failed(&self, failure: &Failure) {
use crate::failure::Reason;
use rustgis_data::diagnostics::{Network, NetworkKind};
if failure.message == DROPPED {
if failure.message == DROPPED || failure.message.contains(CANCELLED_READ) {
return;
}
// The address as asked for, without the download and range markers this module adds.
Expand Down Expand Up @@ -2546,12 +2596,16 @@ mod native {
pub(super) fn download_within(url: String, timeout: std::time::Duration, omit: &'static [u16], done: DoneDetailed) {
let capture = Capture::current();
let still = super::current_still();
let progress = super::current_progress();
std::thread::spawn(move || {
super::with_optional_still(still, || {
let _scope = capture.enter();
let r =
super::check_current().map_err(Failure::from).and_then(|_| get_detailed_omitting(&download_agent(timeout), &url, omit));
done(r);
super::with_progress(progress, || {
let _scope = capture.enter();
let r = super::check_current()
.map_err(Failure::from)
.and_then(|_| get_detailed_omitting(&download_agent(timeout), &url, omit));
done(r);
})
})
});
}
Expand All @@ -2560,12 +2614,16 @@ mod native {
pub(super) fn download_public_within(url: String, timeout: std::time::Duration, done: DoneDetailed) {
let capture = Capture::current();
let still = super::current_still();
let progress = super::current_progress();
std::thread::spawn(move || {
super::with_optional_still(still, || {
let _scope = capture.enter();
let r =
super::check_current().map_err(Failure::from).and_then(|_| get_detailed_omitting(&public_agent(timeout), &url, &[]));
done(r);
super::with_progress(progress, || {
let _scope = capture.enter();
let r = super::check_current()
.map_err(Failure::from)
.and_then(|_| get_detailed_omitting(&public_agent(timeout), &url, &[]));
done(r);
})
})
});
}
Expand Down Expand Up @@ -2615,6 +2673,47 @@ mod native {
fetch_answer(agent, url, Reach::Local, false, omit).map(|f| f.bytes).map_err(|f| f.message)
}

/// A body's read that gives up, at the next bytes that come, once its request is no longer
/// wanted ([`super::current_still`]: a download's Cancel, or its owner gone), and counts what
/// it reads into the download's [`super::Progress`], if it has one. A server that sends
/// nothing more holds it until the agent's timeout, as before.
struct Wanted<R>(R, Option<std::sync::Arc<super::Progress>>);

impl<R: std::io::Read> Wanted<R> {
/// `body`, a whole answer read for the request on this thread; `total` is what it says it
/// holds. One that doesn't say leaves the download's total unknown (a Shapefile's part
/// sent in chunks after one that said its size), rather than the bar full while it reads.
fn new(body: R, total: Option<u64>) -> Self {
let progress = super::current_progress();
if let Some(p) = &progress {
let known = total.map_or(0, |n| p.received().saturating_add(n));
p.total.store(known, std::sync::atomic::Ordering::Relaxed);
}
Self(body, progress)
}

/// `body`, part of an answer (its last bytes), counted without touching the total.
fn part(body: R) -> Self {
Self(body, super::current_progress())
}
}

impl<R: std::io::Read> std::io::Read for Wanted<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
super::check_current().map_err(|_| std::io::Error::other(super::CANCELLED_READ))?;
Comment thread
elpidioassinaturas marked this conversation as resolved.
Resolved
let n = self.0.read(buf)?;
if let Some(p) = &self.1 {
p.done.fetch_add(n as u64, std::sync::atomic::Ordering::Relaxed);
}
Ok(n)
}
}

/// What an answer says its body holds (`Content-Length`).
fn body_length(resp: &ureq::http::Response<ureq::Body>) -> Option<u64> {
resp.headers().get("content-length")?.to_str().ok()?.trim().parse().ok()
}

/// The User-Agent every request goes with.
const USER_AGENT: &str = concat!("RustGIS/", env!("CARGO_PKG_VERSION"), " (+https://rustgis.app)");

Expand Down Expand Up @@ -2877,10 +2976,7 @@ mod native {
// Read one byte past `n` (no more, and no error at the limit): a longer file is
// reported as a server without ranges, so the file is downloaded whole instead.
let limit = n.min(max as u64).saturating_add(1);
resp.body_mut()
.with_config()
.limit(limit.saturating_add(1))
.reader()
Wanted::part(resp.body_mut().with_config().limit(limit.saturating_add(1)).reader())
.take(limit)
.read_to_end(&mut buf)
.map_err(|e| e.to_string())?;
Expand All @@ -2893,10 +2989,8 @@ mod native {
return Ok(Fetched { bytes: buf, origin });
}
// ureq limits the wire bytes; `take` also caps the decoded bytes when gzip expands them.
resp.body_mut()
.with_config()
.limit(max as u64 + 2)
.reader()
let total = body_length(&resp);
Wanted::new(resp.body_mut().with_config().limit(max as u64 + 2).reader(), total)
.take(max as u64 + 1)
.read_to_end(&mut buf)
.map_err(|e| e.to_string())?;
Expand Down Expand Up @@ -3678,6 +3772,9 @@ mod tests {
let diagnostics = RequestDiagnostics::new(url, "GET");
diagnostics.failed(&Failure::new(url, DROPPED));
assert!(network_records(&history, url).is_empty());
// Nor is a download's Cancel during its read.
diagnostics.failed(&Failure::new(url, CANCELLED_READ));
assert!(network_records(&history, url).is_empty());
diagnostics.failed(&Failure::new(url, "the browser couldn't reach it"));
let records = network_records(&history, url);
assert_eq!(records.len(), 1);
Expand Down Expand Up @@ -4073,6 +4170,102 @@ mod tests {
String::from_utf8_lossy(&request).into_owned()
}

/// A slow download counts its bytes as they come, against the total its answer says, and one
/// no longer wanted (`with_still`: a download's Cancel) stops at the next bytes that come,
/// rather than reading the rest of its body.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn a_slow_download_counts_its_progress_and_stops_once_no_longer_wanted() {
use std::io::Write;
use std::sync::atomic::{AtomicBool, Ordering};
let _turn = crate::cache::turn();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
std::thread::spawn(move || {
let Ok((mut stream, _)) = listener.accept() else { return };
let _ = read_request(&mut stream);
let _ = write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\nConnection: close\r\n\r\n");
// A kilobyte every 20 ms: 20 s for the whole body.
for _ in 0..1000 {
if stream.write_all(&[0u8; 1000]).is_err() {
return;
}
std::thread::sleep(std::time::Duration::from_millis(20));
}
});
let wanted = std::sync::Arc::new(AtomicBool::new(true));
let still_wanted = wanted.clone();
let (tx, rx) = std::sync::mpsc::channel();
let progress = std::sync::Arc::new(super::Progress::default());
super::with_still(std::sync::Arc::new(move || still_wanted.load(Ordering::Relaxed)), || {
super::with_progress(Some(progress.clone()), || {
super::download_limited(&format!("http://{addr}/big.bin"), 2_000_000, Box::new(move |r| tx.send(r).unwrap_or_default()))
})
});
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
while progress.received() < 5_000 {
assert!(std::time::Instant::now() < deadline, "nothing counted: {progress:?}");
std::thread::sleep(std::time::Duration::from_millis(20));
}
assert_eq!(progress.total.load(Ordering::Relaxed), 1_000_000);
let first = progress.received();
std::thread::sleep(std::time::Duration::from_millis(200));
let then = progress.received();
assert!(then > first && then < 1_000_000, "{first} then {then}");
assert!(progress.fraction().is_some_and(|f| f > 0.0 && f < 1.0), "{progress:?}");
wanted.store(false, Ordering::Relaxed);
let r = rx.recv_timeout(std::time::Duration::from_secs(5)).expect("the download kept reading its body");
assert!(r.as_ref().is_err_and(|e| e.contains(super::CANCELLED_READ)), "{r:?}");
}

/// A later part of a download (a Shapefile's) whose answer doesn't say its size leaves the
/// total unknown, so the bar shows megabytes, not 100 % while its bytes still come.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn a_part_of_unknown_size_leaves_the_total_unknown() {
use std::io::Write;
let _turn = crate::cache::turn();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else { return };
let path = read_request(&mut stream).split_whitespace().nth(1).unwrap_or("").to_string();
if path == "/a.shp" {
let _ = write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\n0123456789");
continue;
}
// Chunked, so no length: a 1 kB chunk every 20 ms.
let _ = write!(stream, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n");
for _ in 0..200 {
if write!(stream, "3e8\r\n{}\r\n", "x".repeat(1000)).is_err() {
return;
}
std::thread::sleep(std::time::Duration::from_millis(20));
}
let _ = write!(stream, "0\r\n\r\n");
}
});
let progress = std::sync::Arc::new(super::Progress::default());
let (tx, rx) = std::sync::mpsc::channel();
let first = tx.clone();
super::with_progress(Some(progress.clone()), || {
super::download_limited(&format!("http://{addr}/a.shp"), 1_000_000, Box::new(move |r| first.send(r).unwrap_or_default()))
});
rx.recv_timeout(std::time::Duration::from_secs(10)).unwrap().unwrap();
assert_eq!(progress.fraction(), Some(1.0));
super::with_progress(Some(progress.clone()), || {
super::download_limited(&format!("http://{addr}/a.dbf"), 1_000_000, Box::new(move |r| tx.send(r).unwrap_or_default()))
});
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
while progress.received() < 5_000 {
assert!(std::time::Instant::now() < deadline, "nothing counted: {progress:?}");
std::thread::sleep(std::time::Duration::from_millis(20));
}
assert_eq!(progress.fraction(), None, "{progress:?}");
assert!(rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap().is_ok());
}

#[cfg(not(target_arch = "wasm32"))]
fn serve_recorded(answers: Vec<Vec<u8>>) -> (String, std::sync::mpsc::Receiver<String>) {
use std::io::Write;
Expand Down
Loading
Loading