diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b8269dc359..ce04fe5577 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,6 +20,8 @@ repos: name: fixture projects match the project schema files: ^schema/fixtures/.*\.rgis$ types: [file] + # The upstream JSON/YAML tags do not recognize our .rgis extension. + types_or: [] args: ["--schemafile", "schema/project.schema.json", "--default-filetype", "json"] - repo: local hooks: diff --git a/Cargo.lock b/Cargo.lock index f91d99c7ca..7a7e8123a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4973,6 +4973,7 @@ name = "rustgis-engine" version = "0.6.0" dependencies = [ "base64 0.23.1", + "getrandom 0.4.3", "iana-time-zone", "image", "log", diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 6303a909f6..b00bf78d0c 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -49,7 +49,11 @@ url = { workspace = true } proptest = { workspace = true } # Time slider animations as GIFs (exported by the desktop app only). +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.4", features = ["wasm_js"] } + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +getrandom = "0.4" image = { workspace = true, features = ["gif"] } # The live PostGIS tests make their own tables, and change rows as someone else would. diff --git a/crates/engine/src/cmd/connections.rs b/crates/engine/src/cmd/connections.rs index 944e873768..668b0a593f 100644 --- a/crates/engine/src/cmd/connections.rs +++ b/crates/engine/src/cmd/connections.rs @@ -207,6 +207,7 @@ fn new(s: &mut Session, p: &Value) -> CmdResult { ..Connection::default() }; login_fields(p, &mut c)?; + let key_hint = (c.auth.trim().is_empty() || c.auth.trim().eq_ignore_ascii_case("query")).then(|| c.key_name.trim().to_string()); let explicit = !c.name.trim().is_empty(); let redactions = s.connections.redaction_counts(); let (c, stripped) = s.connections.add(c, explicit)?; @@ -215,19 +216,20 @@ fn new(s: &mut Session, p: &Value) -> CmdResult { } resolve_keys(s, &address_keys(&c.address)); resolve_origins(s, &[&c]); - Ok(shown_stripped(s, &c, stripped, ps(p, "address").unwrap_or(""))) + Ok(shown_stripped(s, &c, stripped, ps(p, "address").unwrap_or(""), key_hint.as_deref())) } /// A connection as `shown` gives it, and what was left out of its address: `stripped`, and the /// names of the key parameters (`strippedParameters`, no values) a query login could send, when /// it isn't one already. What was left out is also said as a notice, so the CLI and Python hear /// it as the app does. -fn shown_stripped(s: &mut Session, c: &Connection, stripped: bool, given: &str) -> Value { +fn shown_stripped(s: &mut Session, c: &Connection, stripped: bool, given: &str, key_hint: Option<&str>) -> Value { use crate::notice_text::NoticeText; let mut out = shown(s, c); if stripped { out["stripped"] = json!(true); - let names = if c.auth == "query" { Vec::new() } else { crate::connections::stripped_parameters(given, &c.address) }; + let names = + if c.auth == "query" { Vec::new() } else { crate::connections::stripped_parameters_with_hint(given, &c.address, key_hint) }; let said = if names.is_empty() { NoticeText::format("A password or token in the address was left out of the saved connection definition.", []) } else { @@ -237,6 +239,22 @@ fn shown_stripped(s: &mut Session, c: &Connection, stripped: bool, given: &str) ) }; s.notify_text(said, false); + let (count, collateral) = crate::connections::collateral_parameters(given, &c.address, key_hint); + if count > 0 { + let args = [ + ("connection", crate::project::short_name(&c.name).into()), + ("count", count.to_string().into()), + ("names", collateral.join(", ").into()), + ]; + let text = if collateral.is_empty() { + "Connection '{connection}': {count} public query component(s) were also left out with credential-bearing pairs; use ampersand separators to keep public options" + } else { + "Connection '{connection}': {count} public query component(s) were also left out with credential-bearing pairs (first names: {names}); use ampersand separators to keep public options" + }; + s.notify_text(NoticeText::format(text, args), false); + out["omittedPublicParameterCount"] = json!(count); + out["omittedPublicParameters"] = json!(collateral); + } if !names.is_empty() { out["strippedParameters"] = json!(names); } @@ -270,6 +288,7 @@ fn edit(s: &mut Session, p: &Value) -> CmdResult { c.options = options(v)?; } login_fields(p, &mut c)?; + let key_hint = (c.auth.trim().is_empty() || c.auth.trim().eq_ignore_ascii_case("query")).then(|| c.key_name.trim().to_string()); // Validate everything, then locate the commit target before touching the store. let (mut prepared, stripped) = s.connections.prepare_update(&old.id, c)?; let database_changed = DATABASE_KINDS.contains(&old.kind.as_str()) && old.address != prepared.address; @@ -325,7 +344,7 @@ fn edit(s: &mut Session, p: &Value) -> CmdResult { s3::disconnect(&c.name); } } - Ok(shown_stripped(s, &c, stripped, ps(p, "address").unwrap_or(""))) + Ok(shown_stripped(s, &c, stripped, ps(p, "address").unwrap_or(""), key_hint.as_deref())) } fn duplicate(s: &mut Session, p: &Value) -> CmdResult { diff --git a/crates/engine/src/cmd/design.rs b/crates/engine/src/cmd/design.rs index 4b102f0fca..423f42a00e 100644 --- a/crates/engine/src/cmd/design.rs +++ b/crates/engine/src/cmd/design.rs @@ -1645,7 +1645,10 @@ fn subtypes(s: &mut Session, p: &Json) -> CmdResult { let base = l.renderer.base_symbol(); // The classes come from every feature, so a definition query doesn't drop any; the // features past them are counted among those it lets through, which are drawn. - let (r, rest) = rustgis_map::Renderer::unique_counted(&l.dataset, &[], &fname, rustgis_map::Ramp::default(), base)?; + let (mut r, rest) = rustgis_map::Renderer::unique_counted(&l.dataset, &[], &fname, rustgis_map::Ramp::default(), base)?; + if l.no_fill == Some(rustgis_map::NoFill::Symbols) { + r.map_symbols(super::layer::show_fill); + } let rows = l.visible_rows(); past = PastClasses { features: drawn_past(&l.dataset, &r, &fname, &rows), ..rest }; l.renderer = r; diff --git a/crates/engine/src/cmd/field.rs b/crates/engine/src/cmd/field.rs index 9f6e721f8a..e66b80d113 100644 --- a/crates/engine/src/cmd/field.rs +++ b/crates/engine/src/cmd/field.rs @@ -49,18 +49,71 @@ pub fn is_field_package(name: &str) -> bool { /// /// It runs where it is called, on the window's thread from Save (`ui.saveItem`), as Package /// Project does: each archive is one read and the zip one write, with no Cancel between them. +#[cfg(test)] pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), String> { + let (name, bytes, notes) = package_output(s, p)?; + for note in notes { + s.notify(note, false); + } + Ok((name, bytes)) +} + +/// Construction notes describe the delivered package, so callers emit them after delivery. +pub(crate) fn package_output(s: &mut Session, p: &Value) -> Result<(String, Vec, Vec), String> { + let mut notes = Vec::new(); + let (name, bytes) = build_package(s, p, &mut notes)?; + Ok((name, bytes, notes)) +} + +fn field_edit(s: &mut Session, label: &'static str, change: impl FnOnce(&mut Session) -> crate::CmdResult) -> crate::CmdResult { + if s.restorable > 0 { change(s) } else { s.guarded(label, true, change) } +} + +fn legacy_user(s: &Session) -> Result<(), String> { + if cfg!(target_arch = "wasm32") || !s.policy_user() { + Err("legacy field recovery requires explicit native User dispatch".into()) + } else { + Ok(()) + } +} + +fn build_package(s: &mut Session, p: &Value, notes: &mut Vec) -> Result<(String, Vec), String> { + macro_rules! note { + ($text:expr, $error:expr $(,)?) => { + if $error { + s.notify($text, true); + } else { + notes.push(($text).to_string()); + } + }; + } + let legacy_return = if p.get("legacyReturn").is_some() { + legacy_user(s)?; + super::b(p, "legacyReturn").ok_or("legacyReturn must be true or false")? + } else { + false + }; + if legacy_return && (s.project.field_origin.is_some() || !s.project.origin_id.is_empty()) { + return Err("legacyReturn cannot override a present field or home identity".into()); + } + + let opened_field = legacy_return + || s.project.field_origin.is_some() + || s.opened_package.as_deref().is_some_and(|p| is_field_package(&p.to_string_lossy())); + if opened_field && s.project.maps.len() != 1 { + return Err("a returning field project must hold only its original field map: remove added maps before sending it back".into()); + } // An offline area's map: named for one of this project's areas, or opened from a field // package (the phone's project holds no areas). A map only named so is packaged as any other. - let opened_field = s.opened_package.as_deref().is_some_and(|p| is_field_package(&p.to_string_lossy())); let returning = s .map() .name .strip_suffix(OFFLINE) .map(str::to_string) - .filter(|area| opened_field || s.project.offline_areas.iter().any(|a| a.name.eq_ignore_ascii_case(area))); - // Only a map coming back from the field (its project opened from a field package) carries its - // photos: going out, an area's map included, the photos named are at home. + .filter(|area| opened_field || s.project.offline_areas.iter().any(|a| a.name.eq_ignore_ascii_case(area))) + .or_else(|| opened_field.then(|| s.map().name.clone())); + // Only a map coming back from the field carries its photos. Its persisted provenance + // survives an ordinary Save As; going out, the photos named are at home. let coming_back = returning.is_some() && opened_field; let (area, mut map) = match returning { Some(area) => { @@ -70,7 +123,7 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S if !rasters.is_empty() { m.release_tiles(&rasters); m.layers.retain(|l| !l.is_raster()); - s.notify(format!("{} raster layer(s) were left out of the field package: it holds feature layers", rasters.len()), false); + note!(format!("{} raster layer(s) were left out of the field package: it holds feature layers", rasters.len()), false); } (area, m) } @@ -79,24 +132,23 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S let features = |m: &rustgis_map::Map| m.layers.iter().filter(|l| !l.is_raster()).map(|l| l.dataset.features.len()).sum::(); let had = features(s.map()); - let made = super::workspace::offline_download(s, p)?; + let made = field_edit(s, "Create field offline area", |s| super::workspace::offline_download(s, p))?; let name = made.get("name").and_then(Value::as_str).ok_or("the offline area wasn't made")?.to_string(); let data = s.project.offline_areas.iter().find(|a| a.name == name).map(|a| a.data.clone()).ok_or("the offline area wasn't made")?; if rasters > 0 { - s.notify( - format!("{rasters} raster layer(s) were left out of the field package: an offline area holds feature layers"), - false, - ); + note!(format!("{rasters} raster layer(s) were left out of the field package: an offline area holds feature layers"), false,); } - s.notify( - format!("Made the offline area '{name}' of this map for the field package: save this project to bring the field's edits back to it later"), + note!( + format!( + "Made the offline area '{name}' of this map for the field package: save this project to bring the field's edits back to it later" + ), false, ); // The area holds the features its extent (the view, unless one is given) touches. let out = had.saturating_sub(features(&data)); if out > 0 { - s.notify( + note!( format!("{out} feature(s) outside the offline area's extent, or without a shape, stayed out of the field package"), false, ); @@ -150,7 +202,7 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S } } if !online.is_empty() { - s.notify( + note!( format!( "{} tile layer(s) in the field package draw from the web and show only online: {}. Add an MBTiles or PMTiles archive for a basemap with no network", online.len(), @@ -160,7 +212,7 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S ); } if !uploaded.is_empty() { - s.notify( + note!( format!( "{} tile archive(s) added to this browser couldn't be put in the field package, as they live only in its memory: {}. Add them to the field from the file", uploaded.len(), @@ -170,18 +222,18 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S ); } if !unread.is_empty() { - s.notify(format!("{} tile archive(s) couldn't be put in the field package: {}", unread.len(), unread.join(", ")), true); + note!(format!("{} tile archive(s) couldn't be put in the field package: {}", unread.len(), unread.join(", ")), true); } // Coming back, the photos taken in the field go with their features. if coming_back { let photos = take_photos(&mut map, &mut budget); if !photos.unread.is_empty() { let unread = &photos.unread; - s.notify(format!("{} photo(s) couldn't be put in the field package: {}", unread.len(), unread.join(", ")), true); + note!(format!("{} photo(s) couldn't be put in the field package: {}", unread.len(), unread.join(", ")), true); } if !photos.not_pictures.is_empty() { let named = &photos.not_pictures; - s.notify( + note!( format!( "{} file(s) a photos field names weren't put in the field package: only pictures go ({}): {}", named.len(), @@ -196,7 +248,48 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S // A project of the one map: what else the project holds (tables, notebooks, models, shared // items and their paths…) stays at home. The custom colour ramps and styles go with it, as its // layers may draw with them. + let origin = if coming_back { + if s.project.field_origin.is_none() { + if !s.project.origin_id.is_empty() { + return Err("the field package has an incomplete origin identity; legacy recovery cannot override it".into()); + } + notes.push("Legacy field return: its origin is unverified; bring it back only with an explicitly verified legacyArea in the home project".into()); + } + s.project.field_origin.clone() + } else { + let ai = s.project.offline_areas.iter().position(|a| a.name.eq_ignore_ascii_case(&area)).ok_or("the offline area was not found")?; + let new_home = s.project.origin_id.is_empty().then(crate::project::field_id).transpose()?; + let new_area = s.project.offline_areas.get(ai).is_some_and(|a| a.id.is_empty()).then(crate::project::field_id).transpose()?; + let assigned_identity = new_home.is_some() || new_area.is_some(); + if assigned_identity { + field_edit(s, "Assign field origin identities", |s| { + if let Some(id) = new_home { + s.project.origin_id = id; + } + if let Some(id) = new_area + && let Some(area) = s.project.offline_areas.get_mut(ai) + { + area.id = id; + } + Ok(Value::Null) + })?; + } + if assigned_identity { + note!("Assigned this legacy offline area an origin identity: save the home project before sending the field package", false); + } + Some(crate::project::FieldOrigin { + project: s.project.origin_id.clone(), + area: s.project.offline_areas.get(ai).ok_or("the offline area was not found")?.id.clone(), + }) + }; + if origin + .as_ref() + .is_some_and(|origin| !crate::project::valid_field_id(&origin.project) || !crate::project::valid_field_id(&origin.area)) + { + return Err("the field origin identity is invalid: make a new field package from the home project".into()); + } let mut project = crate::project::Project::new(area.clone()); + project.field_origin = origin; // Rasters were taken out above, so the map goes as it is. project.maps = vec![map]; let home = &s.project; @@ -209,6 +302,13 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S (home.presentations.len(), "presentation(s)"), (home.workspaces.len(), "imagery workspace(s)"), (home.dl_models.len(), "deep learning model(s)"), + (home.folders.len(), "folder(s)"), + (home.shared.len(), "shared item(s)"), + (home.data_stores.len(), "data store(s)"), + (home.tasks.len(), "task(s)"), + (home.jobs.len(), "job(s)"), + (home.gp_history.len(), "geoprocessing history run(s)"), + (home.offline_areas.iter().filter(|saved| !saved.name.eq_ignore_ascii_case(&area)).count(), "other offline area(s)"), (home.maps.len().saturating_sub(1), "other map(s)"), ] .into_iter() @@ -216,7 +316,7 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S .map(|(n, what)| format!("{n} {what}")) .collect(); if !stayed.is_empty() { - s.notify(format!("The field package holds only the map '{area}': {} stayed in this project", stayed.join(", ")), false); + note!(format!("The field package holds only the map '{area}': {} stayed in this project", stayed.join(", ")), false); } project.ramps = s.project.ramps.clone(); project.styles = s.project.styles.clone(); @@ -232,11 +332,6 @@ pub(crate) fn package(s: &mut Session, p: &Value) -> Result<(String, Vec), S Ok((area, rustgis_data::zip::write(&entries)?)) } -/// Whether `map` is an offline area's map (` (offline)`), packaged to come back. -fn returning_map(map: &rustgis_map::Map) -> bool { - map.name.ends_with(OFFLINE) -} - /// A layer's photos field, by its place: its text field named `photos` or `photo`, whatever the /// case (as the touch interface names it). fn photos_field(layer: &rustgis_map::Layer) -> Option { @@ -443,6 +538,12 @@ fn plain_file(file: &str) -> bool { /// It runs where it is called, on the window's thread, as Sync does: the package is one read, each /// photo one write, and the sync one pass over the area's features, with no Cancel between them. pub(crate) fn bring_back(s: &mut Session, p: &Value) -> Result { + let legacy_area = if p.get("legacyArea").is_some() { + legacy_user(s)?; + Some(super::ps(p, "legacyArea").filter(|name| !name.trim().is_empty()).ok_or("legacyArea must name a verified home offline area")?) + } else { + None + }; let bytes = match super::ps(p, "path") { Some(path) => crate::project::read_file(Path::new(path), MOST_UNPACKED as u64)?, None => { @@ -473,17 +574,46 @@ pub(crate) fn bring_back(s: &mut Session, p: &Value) -> Result { } let text = text.ok_or("the file holds no field package's project")?; let project = Project::from_import_json(&text, None)?; - let mut edited = project - .maps - .into_iter() - .find(returning_map) - .ok_or("the field package holds no offline area's map: package the map in the field (Send Field Package) to bring it back")?; - let area = edited.name.strip_suffix(OFFLINE).unwrap_or_default().to_string(); - let ai = s.project.offline_areas.iter().position(|a| a.name.eq_ignore_ascii_case(&area)).ok_or_else(|| { - format!( - "this project has no offline area '{area}': open the project the field package was made from, as it was saved after packaging" - ) - })?; + let ai = if let Some(area) = legacy_area { + let raw: Value = serde_json::from_str(&text).map_err(|e| e.to_string())?; + // Only entirely absent identities qualify. Null/empty/malformed metadata is not a + // reason to discard a present association; typed parsing already rejects bad IDs. + if raw.get("requires").and_then(Value::as_array).is_some_and(|caps| caps.iter().any(|cap| cap.as_str() == Some("field-origin"))) + || raw.get("field_origin").is_some() + || raw.get("origin_id").is_some() + || project.field_origin.is_some() + || !project.origin_id.is_empty() + { + return Err("legacyArea cannot override a present or incomplete field identity".into()); + } + s.project + .offline_areas + .iter() + .position(|a| a.name.eq_ignore_ascii_case(area)) + .ok_or("legacyArea does not name an offline area in this home project")? + } else { + let origin = project.field_origin.as_ref().ok_or("this legacy field package has no origin identity: explicitly verify its home area and use legacyArea, or make a new field package")?; + if !crate::project::valid_field_id(&origin.project) + || !crate::project::valid_field_id(&origin.area) + || s.project.origin_id.is_empty() + || origin.project != s.project.origin_id + { + return Err( + "the field package belongs to another home project, or has no valid origin identity: open the project it was made from" + .into(), + ); + } + s.project + .offline_areas + .iter() + .position(|a| a.id == origin.area) + .ok_or("this project has no offline area with the field package's identity: open the project as saved after packaging")? + }; + let area = s.project.offline_areas.get(ai).ok_or("no such area")?.name.clone(); + if project.maps.len() != 1 { + return Err("a field package must hold exactly its original field map".into()); + } + let mut edited = project.maps.into_iter().next().ok_or("the field package holds no map")?; // An area of that name in another project isn't this package's: its layers, matched by id, // must be the area's own, of the same kind (a name may have changed in the field), or features // of other layers would be changed. @@ -511,6 +641,13 @@ pub(crate) fn bring_back(s: &mut Session, p: &Value) -> Result { .join("field photos") .join(super::items::safe(&area)), }; + match std::fs::metadata(&dir) { + Ok(metadata) if !metadata.is_dir() => return Err(format!("{}: not a folder; no photo was brought back", dir.display())), + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("{}: {error}; no photo was brought back", dir.display())); + } + _ => {} + } // The path each photo was named by where it was packaged. let before: std::collections::HashMap<&str, &str> = index.lines().filter_map(|l| l.split_once('\t')).collect(); // Where each photo goes, decided before anything is written: the edits are brought back @@ -535,7 +672,8 @@ pub(crate) fn bring_back(s: &mut Session, p: &Value) -> Result { write = false; break; } - _ => path = dir.join(format!("{n}-{name}")), + Err(e) => return Err(format!("{}: {e}; no photo was brought back", path.display())), + Ok(_) => path = dir.join(format!("{n}-{name}")), } } if write && path.file_name().is_some_and(|f| f.to_string_lossy() != name.as_str()) { @@ -598,6 +736,9 @@ pub(crate) fn bring_back(s: &mut Session, p: &Value) -> Result { if strange > 0 { s.notify(format!("{strange} photo(s) in the field package weren't brought back: their names aren't plain file names"), true); } + if legacy_area.is_some() { + s.notify("Legacy field return: applied to the explicitly selected home offline area with an unverified origin; create a fresh field package for further work", false); + } s.notify(summary, false); if let Some(o) = done.as_object_mut() { o.insert("area".into(), json!(area)); @@ -864,7 +1005,10 @@ mod tests { let (_, bytes) = package(&mut s, &json!({"name": "Survey", "extent": [-20.0, -40.0, 60.0, 40.0]})).unwrap(); // What stayed at home is said. let said: Vec = s.notices_since(mark).iter().map(|n| n.text.clone()).collect(); - assert!(said.iter().any(|t| t.contains("stayed in this project") && t.contains("table(s)")), "{said:?}"); + assert!( + said.iter().any(|t| t.contains("stayed in this project") && t.contains("1 table(s)") && t.contains("1 folder(s)")), + "{said:?}" + ); let names: Vec = rustgis_data::zip::read(&bytes).unwrap().into_iter().map(|(n, _)| n).collect(); assert!(names.contains(&"basemaps/site_a_b__.mbtiles".to_string()), "{names:?}"); let into = temp.path().join("unpacked"); @@ -1079,10 +1223,10 @@ mod tests { // one that took a new id was said the first time, and is replaced. let renumbered = said.iter().any(|t| t.contains("took a new id")); assert_eq!(again["added"].as_u64(), Some(u64::from(renumbered)), "{again}"); - // A package for an area this project doesn't have is refused, by name. + // A package for another home is refused before looking up an area by name. let mut other = sample(); let e = other.run("offline.bringBack", &json!({"path": sent.display().to_string()})).unwrap_err(); - assert!(e.contains("no offline area 'Trip'"), "{e}"); + assert!(e.contains("another home project"), "{e}"); } #[test] @@ -1242,6 +1386,39 @@ mod tests { assert!(!temp.path().join("Odd photos").exists()); } + /// An unusable destination is refused before edits land, rather than trying numbered photo + /// names forever. Run it with a completion deadline so a regression fails instead of hanging. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn bringing_back_photos_refuses_a_file_where_their_folder_should_be() { + let temp = Temp::new("photo-folder-is-file"); + let mut home = sample(); + home.path = Some(temp.path().join("home.rgis")); + let (_, bytes) = package(&mut home, &json!({"name": "Site", "extent": [-180.0, -85.0, 180.0, 85.0]})).unwrap(); + let unpacked = temp.path().join("field"); + let text = unpack_into(&bytes, &unpacked).unwrap(); + let mut field = Session::new(project_in(&text, &unpacked).unwrap()); + field.opened_package = Some(temp.path().join("sent.rgisfield")); + let (_, back) = package(&mut field, &Value::Null).unwrap(); + let mut entries = rustgis_data::zip::read(&back).unwrap(); + entries.push(("photos/new.jpg".into(), b"photo".to_vec())); + let path = temp.path().join("back.rgisfield"); + std::fs::write(&path, rustgis_data::zip::write(&entries).unwrap()).unwrap(); + let blocked = temp.path().join("Site photos"); + std::fs::write(&blocked, b"keep this file").unwrap(); + let before = serde_json::to_string(&home.project).unwrap(); + let (send, receive) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let result = home.run("offline.bringBack", &json!({"path": path.display().to_string()})); + let _ = send.send((result, serde_json::to_string(&home.project).unwrap())); + }); + let (result, after) = receive.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); // test-wait: a broken candidate loop must fail instead of hanging the suite + let error = result.unwrap_err(); + assert!(error.contains("no photo was brought back") && error.contains("Site photos"), "{error}"); + assert_eq!(after, before); + assert_eq!(std::fs::read(&blocked).unwrap(), b"keep this file"); + } + #[test] fn a_feature_added_in_the_field_keeps_its_free_id_and_is_found_again() { let temp = Temp::new("free-id"); @@ -1270,6 +1447,117 @@ mod tests { assert_eq!((again["added"].as_u64(), again["deleted"].as_u64()), (Some(0), Some(0)), "{again}"); } + #[test] + fn field_origin_survives_plain_save_and_names_never_choose_the_home_or_area() { + let temp = Temp::new("origin"); + let mut home = sample(); + let params = json!({"name": "Site", "extent": [-180.0, -85.0, 180.0, 85.0]}); + let (_, out) = package(&mut home, ¶ms).unwrap(); + let unpacked = temp.path().join("field"); + let text = unpack_into(&out, &unpacked).unwrap(); + let field_project = project_in(&text, &unpacked).unwrap(); + let origin = field_project.field_origin.clone().unwrap(); + assert_eq!(origin.project, home.project.origin_id); + assert_eq!(origin.area, home.project.offline_areas[0].id); + // Ordinary Save As and reopen have no opened_package hint. The renamed map still + // returns with its original area identity, rather than making a fresh offline area. + let plain = serde_json::to_string(&field_project).unwrap(); + assert!(plain.contains("field-origin")); + let mut field = Session::new(Project::from_json(&plain).unwrap()); + field.map_mut().name = "Renamed on phone".into(); + let (_, back) = package(&mut field, &Value::Null).unwrap(); + let path = temp.path().join("back.rgisfield"); + std::fs::write(&path, &back).unwrap(); + let mut other = sample(); + package(&mut other, ¶ms).unwrap(); + // These projects have identical layer IDs, types and area labels: only their + // persisted origin differs. Refusal occurs before edits or photo installation. + let before = serde_json::to_value(&other.project).unwrap(); + let err = other.run("offline.bringBack", &json!({"path": path})).unwrap_err(); + assert!(err.contains("another home project"), "{err}"); + assert_eq!(serde_json::to_value(&other.project).unwrap(), before); + home.project.offline_areas[0].name = "Renamed at home".into(); + home.run("offline.bringBack", &json!({"path": path})).unwrap(); + // Recreating a same-named area in the correct home gets a fresh ID too. + home.project.offline_areas.clear(); + package(&mut home, ¶ms).unwrap(); + let err = home.run("offline.bringBack", &json!({"path": path})).unwrap_err(); + assert!(err.contains("no offline area with"), "{err}"); + // A legacy file remains readable, but cannot guess a home by name or layer ID. + let mut legacy = field_project; + legacy.field_origin = None; + let bytes = rustgis_data::zip::write(&[("Site.rgis".into(), serde_json::to_vec(&legacy).unwrap())]).unwrap(); + std::fs::write(&path, bytes).unwrap(); + let err = home.run("offline.bringBack", &json!({"path": path})).unwrap_err(); + assert!(err.contains("legacy") && err.contains("make a new field package"), "{err}"); + let mark = home.notice_mark(); + home.run("offline.bringBack", &json!({"path": path, "legacyArea":"Site"})).unwrap(); + assert!(home.notices_since(mark).iter().any(|notice| notice.text.contains("unverified origin"))); + let mut incomplete = serde_json::to_value(&legacy).unwrap(); + incomplete["requires"] = json!(["field-origin"]); + let incomplete = rustgis_data::zip::write(&[("Site.rgis".into(), serde_json::to_vec(&incomplete).unwrap())]).unwrap(); + std::fs::write(&path, incomplete).unwrap(); + let err = home.run("offline.bringBack", &json!({"path":path,"legacyArea":"Site"})).unwrap_err(); + assert!(err.contains("cannot override"), "{err}"); + std::fs::write(&path, &back).unwrap(); + let before = serde_json::to_value(&home.project).unwrap(); + let err = home.run("offline.bringBack", &json!({"path":path,"legacyArea":"Site"})).unwrap_err(); + assert!(err.contains("cannot override"), "{err}"); + assert_eq!(serde_json::to_value(&home.project).unwrap(), before); + // An old field project saved as a plain file can explicitly preserve its edits. + let mut old = Session::new(legacy); + old.map_mut().name = "Renamed old field copy".into(); + let (_, returned) = package(&mut old, &json!({"legacyReturn":true})).unwrap(); + let text = super::super::items::package_text(&returned).unwrap(); + assert!(Project::from_json(&text).unwrap().field_origin.is_none()); + assert!(old.project.offline_areas.is_empty()); + } + + #[test] + fn an_opened_legacy_field_copy_refuses_an_added_map_before_packaging() { + let mut field = Session::new(Project::new("Old field")); + field.opened_package = Some(PathBuf::from("old.rgisfield")); + field.project.maps.push(rustgis_map::Map::new("Added")); + field.project.active_map = 1; + field.clear_history(); + let before = serde_json::to_value(&field.project).unwrap(); + let state = field.state(); + let error = package_output(&mut field, &Value::Null).unwrap_err(); + assert!(error.contains("remove added maps"), "{error}"); + assert_eq!(serde_json::to_value(&field.project).unwrap(), before); + assert_eq!(field.state(), state); + assert_eq!(field.undo_depth(), 0); + assert!(!field.dirty()); + } + + #[test] + fn packaging_records_new_home_and_area_identities_as_an_undoable_change() { + let mut home = sample(); + home.clear_history(); + assert!(!home.dirty()); + package_output(&mut home, &json!({"name":"Recorded","extent":[-180,-85,180,85]})).unwrap(); + assert!(home.dirty()); + assert_eq!(home.undo_depth(), 1); + assert!(!home.project.origin_id.is_empty()); + home.run("edit.undo", &Value::Null).unwrap(); + assert!(home.project.origin_id.is_empty() && home.project.offline_areas.is_empty()); + assert!(!home.dirty()); + } + + #[test] + fn field_package_construction_notes_wait_for_successful_delivery() { + let temp = Temp::new("deferred-notes"); + let mut home = sample(); + let mark = home.notice_mark(); + let (_, _, notes) = super::super::items::item_output(&mut home, "fieldPackage", &json!({"name":"Deferred"})).unwrap(); + assert!(notes.iter().any(|note| note.contains("Made the offline area"))); + assert!(home.notices_since(mark).iter().all(|notice| !notice.text.contains("Made the offline area"))); + let mark = home.notice_mark(); + let error = home.run("offline.package", &json!({"name":"Failed","path":temp.path().join("missing/file.rgisfield")})).unwrap_err(); + assert!(error.contains("file.rgisfield")); + assert!(home.notices_since(mark).iter().all(|notice| !notice.text.contains("Made the offline area 'Failed'"))); + } + #[test] fn a_package_from_another_projects_area_of_the_same_name_is_refused() { let temp = Temp::new("foreign"); @@ -1297,7 +1585,7 @@ mod tests { let path = temp.path().join("back.rgisfield"); std::fs::write(&path, &back).unwrap(); let e = home.run("offline.bringBack", &json!({"path": path.display().to_string()})).unwrap_err(); - assert!(e.contains("layers aren't the area's"), "{e} {ids:?} {theirs:?}"); + assert!(e.contains("another home project"), "{e} {ids:?} {theirs:?}"); } #[test] diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index fe2db33938..5216f7c8a8 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -1676,7 +1676,7 @@ pub fn item_output(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Ve crate::project::ensure_map_imports_complete(map)?; let map = embedded_map(map, &mut MAX_EMBED.clone()); let name = map.name.clone(); - (name, durable_bytes(s, map)?) + (name, durable_bytes_with(s, map, crate::project::map_file_bytes)?) } "layer" => { let id = s.layer_id(p)?; @@ -1686,7 +1686,7 @@ pub fn item_output(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Ve } let layer = embed_layer(l, &mut MAX_EMBED.clone()); let name = layer.name.clone(); - (name, durable_bytes(s, layer)?) + (name, durable_bytes_with(s, layer, crate::project::layer_file_bytes)?) } "styles" => { if s.project.styles.is_empty() { @@ -1723,7 +1723,11 @@ pub fn item_output(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Ve (name, bytes) } "projectPackage" => (s.project.name.clone(), package_project(s)?), - "fieldPackage" => super::field::package(s, p)?, + "fieldPackage" => { + let (name, bytes, notes) = super::field::package_output(s, p)?; + construction_notes.extend(notes); + (name, bytes) + } "mapPackage" | "mobilePackage" => { let tol = f(p, "tolerance").unwrap_or(0.001).clamp(0.0, 1.0); (s.map().name.clone(), package_map(s, kind == "mobilePackage", tol)?) @@ -1794,15 +1798,27 @@ pub fn item_output(s: &mut Session, kind: &str, p: &Value) -> Result<(String, Ve pub(crate) fn save_item(s: &mut Session, kind: &str, p: &Value) -> CmdResult { let written = s.notice_mark(); let path = ps(p, "path").ok_or("path is required")?.to_string(); - let (_, bytes) = item_bytes(s, kind, p)?; + let (_, bytes, notes) = item_output(s, kind, p)?; rustgis_data::io::write_atomic(Path::new(&path), &bytes).map_err(|e| format!("{path}: {e}"))?; + for note in notes { + s.notify(note, false); + } s.retain_written_notices(written); Ok(json!({"path": path, "bytes": bytes.len()})) } /// Consumes the existing item draft, emitting losses after serialization has completed. fn durable_bytes(s: &mut Session, value: T) -> Result, String> { - let (bytes, report) = crate::durable::output_owned(value, to_json)?; + durable_bytes_with(s, value, to_json) +} + +/// Serializes an item through its document writer, after the same durable-field transaction. +fn durable_bytes_with( + s: &mut Session, + value: T, + write: impl FnOnce(&T) -> Result, String>, +) -> Result, String> { + let (bytes, report) = crate::durable::output_owned(value, write)?; crate::durable::notify(s, &report); Ok(bytes) } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index ee3d3f602e..e94c0b4ba7 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -985,7 +985,7 @@ fn class_symbol<'a>(r: &'a mut Renderer, class: &ClassRef) -> Option<&'a mut rus } /// A symbol's fill No Fill made transparent, shown again in its own colour. -fn show_fill(sym: &mut rustgis_map::Symbol) { +pub(super) fn show_fill(sym: &mut rustgis_map::Symbol) { if sym.fill.a() == 0 { sym.fill = sym.fill.with_alpha(shown_fill_alpha()); } @@ -3331,7 +3331,8 @@ enum ConvertedOrigin { #[cfg(not(target_arch = "wasm32"))] pub(crate) fn add_converted(s: &mut Session, p: &Value, label: &str, original: &str, converted: Vec) -> CmdResult { let empty_before = super::map_is_empty(s); - let single = converted.len() == 1; + let parts = converted.len(); + let single = parts == 1; let (mut ids, mut features, mut streamed) = (Vec::new(), 0u64, false); for c in converted { let mut q = json!({"name": c.name}); @@ -3385,6 +3386,15 @@ pub(crate) fn add_converted(s: &mut Session, p: &Value, label: &str, original: & if ids.is_empty() { return Err(format!("{label}: none of its layers could be added")); } + if !single && let Some(name) = ps(p, "name") { + s.notify_text( + crate::notice_text::NoticeText::format( + "Name '{name}' was not used: {parts} converted layer parts keep their generated names", + [("name", crate::project::short_name(name).into()), ("parts", parts.to_string().into())], + ), + false, + ); + } Ok(json!({"layer": ids.first(), "layers": ids, "features": features, "streamed": streamed})) } diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index ba6b0624bc..6b53876b2c 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -57,6 +57,7 @@ pub mod sql; pub mod stac; pub mod stac_area; pub mod table_stats; +pub(crate) mod terrain_blocks; pub mod tile_export; pub mod tiles; pub mod time; diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index 2652ef23de..56b9ca7fa6 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -283,7 +283,7 @@ pub(crate) fn say_conversion_fallbacks(s: &mut Session) { /// Why a block with `extent` can't be drawn on the DEM `dem` (its name, its box and its footprint, /// as `cmd/view.rs` refuses it): no box fits the DEM and the block has none, or its box lies off the /// DEM's footprint. None when it can be drawn. -fn off_dem(extent: Option<[f64; 4]>, dem: Option<&(String, Option<[f64; 4]>, Option<[f64; 4]>)>) -> Option { +pub(super) fn off_dem(extent: Option<[f64; 4]>, dem: Option<&(String, Option<[f64; 4]>, Option<[f64; 4]>)>) -> Option { let (n, bbox, footprint) = dem?; match (*bbox, extent) { (None, None) => Some(format!("no box fits inside '{n}'s extent on the map: give the block an extent [xmin, ymin, xmax, ymax]")), diff --git a/crates/engine/src/cmd/search.rs b/crates/engine/src/cmd/search.rs index 6b58a60a80..d938695b99 100644 --- a/crates/engine/src/cmd/search.rs +++ b/crates/engine/src/cmd/search.rs @@ -724,6 +724,12 @@ pub fn parse_lat_lon(text: &str) -> Option { groups = nums.into_iter().map(|v| (vec![v], None)).collect(); } let [(a, ha), (b, hb)] = <[_; 2]>::try_from(groups).ok()?; + // As in By Coordinates, a hemisphere supplies the sign; a minus sign is ambiguous. + if (ha.is_some() && a.first().is_some_and(|v| v.is_sign_negative())) + || (hb.is_some() && b.first().is_some_and(|v| v.is_sign_negative())) + { + return None; + } let (mut a, mut b) = (axis(&a)?, axis(&b)?); let sign = |v: f64, h: char| if matches!(h, 'S' | 'W') { -v.abs() } else { v }; let (lat, lon) = match (ha, hb) { @@ -1101,9 +1107,23 @@ mod tests { #[test] fn non_coordinates_are_refused() { - for t in - ["Paris", "New York", "95, 200", "40.7", "40.7N 74W 3E", "1 2 3", "40N 50S", "", "12-5", "40° 61' N 74 W", "Seattle 47 -122"] - { + for t in [ + "Paris", + "New York", + "95, 200", + "40.7", + "40.7N 74W 3E", + "1 2 3", + "40N 50S", + "", + "12-5", + "40° 61' N 74 W", + "Seattle 47 -122", + "-40 N, 74 W", + "40 N, -74 E", + "-0 N, 74 W", + "-40 S, 74 W", + ] { assert_eq!(parse_lat_lon(t), None, "{t}"); } } diff --git a/crates/engine/src/cmd/sharing.rs b/crates/engine/src/cmd/sharing.rs index 417bd57a2a..d2b7dfd1f7 100644 --- a/crates/engine/src/cmd/sharing.rs +++ b/crates/engine/src/cmd/sharing.rs @@ -1418,13 +1418,18 @@ fn link_chart(s: &Session, rel: Option<&str>) -> Result<(String, Option) // out; one already in the chart may still be linked. // Up to 200 links drawn from it: destinations the chart has no room for don't count // toward them, so a later one it can draw still is. - let mut a = index.get(&(om, o.id, orow)).copied(); + let origin = (om, o.id, orow); + let mut a = index.get(&origin).copied(); + if a.is_none() && nodes.len() >= MAX_NODES { + continue; + } let mut drawn_here = 0; for &drow in drows { if drawn_here >= MAX_FEATURE_LINKS || links.len() >= MAX_LINKS { break; } - let new = usize::from(a.is_none()) + usize::from(!index.contains_key(&(dm, d.id, drow))); + let destination = (dm, d.id, drow); + let new = usize::from(a.is_none()) + usize::from(destination != origin && !index.contains_key(&destination)); if nodes.len() + new > MAX_NODES { continue; } diff --git a/crates/engine/src/cmd/terrain_blocks.rs b/crates/engine/src/cmd/terrain_blocks.rs new file mode 100644 index 0000000000..84338b7746 --- /dev/null +++ b/crates/engine/src/cmd/terrain_blocks.rs @@ -0,0 +1,339 @@ +//! Validate retained block diagrams when a DEM's deferred pixels become available. Checks are +//! bounded to a fixed grid, once per DEM allocation and block extent, never per raster cell. + +use std::collections::HashSet; +use std::sync::{Arc, Weak}; + +use rustgis_data::raster::Raster; +use rustgis_geom::Coord; +use rustgis_map::view::TerrainSource; + +use crate::Session; +use crate::notice_text::NoticeText; + +const CHECK_CELLS: usize = 64; + +pub(crate) struct Checked { + layer: u64, + pixels: Weak, + extent: Option<[f64; 4]>, +} + +#[derive(Debug)] +pub(crate) struct Coverage { + extent: [f64; 4], + missing: usize, + total: usize, +} + +/// Uses the same admission rule as a loaded DEM, plus a fixed sampling grid that says when +/// heights are missing inside or beyond its footprint. No file or tile request is started. +pub(super) fn check(name: &str, raster: &Raster, extent: Option<[f64; 4]>) -> Result { + let boxes = (name.to_string(), rustgis_render::terrain::raster_box(raster), rustgis_render::terrain::raster_footprint(raster)); + if let Some(reason) = super::project::off_dem(extent, Some(&boxes)) { + return Err(reason); + } + let extent = extent.or(boxes.1).ok_or("no block extent fits the DEM")?; + if raster.width == 0 || raster.height == 0 { + return Err(format!("'{name}' has no height cells for the block")); + } + let mut missing = 0; + for row in 0..=CHECK_CELLS { + for col in 0..=CHECK_CELLS { + let x = extent[0] + (extent[2] - extent[0]) * col as f64 / CHECK_CELLS as f64; + let y = extent[1] + (extent[3] - extent[1]) * row as f64 / CHECK_CELLS as f64; + missing += usize::from(rustgis_render::terrain::dem_height(raster, Coord::new(x, y)).is_none()); + } + } + let total = (CHECK_CELLS + 1) * (CHECK_CELLS + 1); + if missing == total { + // A coarse grid can step over a small valid DEM. Only refuse when a bounded scan + // proves the entire first band has no height; otherwise retain the block and report + // the sampled missing locations. Never scan an unbounded raster on a frame. + let empty = raster.width.checked_mul(raster.height).is_some_and(|cells| cells <= (CHECK_CELLS * CHECK_CELLS) as u32) + && (0..raster.height).all(|row| { + (0..raster.width).all(|col| raster.value(0, i64::from(col), i64::from(row)).is_none_or(|height| !height.is_finite())) + }); + if empty { + return Err(format!("the block has no height from '{name}': its bounded first band has no usable cells")); + } + } + Ok(Coverage { extent, missing, total }) +} + +/// Called after a command commits and when the frame reports newly landed raster pixels. +/// A Weak retains allocation identity without retaining the DEM's pixel buffers. +pub(crate) fn landed(session: &mut Session) { + landed_impl(session, false); +} + +/// Undo/redo restores the recorded view exactly; report unusable restored bounds without +/// immediately deleting them again. Remembering the checked allocation also keeps later frames. +pub(crate) fn after_command(session: &mut Session, id: &str) { + landed_impl(session, matches!(id, "edit.undo" | "edit.redo")); +} + +fn landed_impl(session: &mut Session, restoring: bool) { + // Like in-place overview maintenance, unrelated retained-view edits belong only to the + // native owner/frame. An authorized Undo/Redo only remembers its restored allocation and + // warns; it never drops bounds, so the next native frame cannot undo that explicit Undo. + if !session.policy_ingress_idle() && !restoring { + return; + } + let mut checked = std::mem::take(&mut session.terrain_blocks_checked); + let mut maps = HashSet::new(); + let mut dropped = Vec::new(); + let mut restored = Vec::new(); + let mut partial = Vec::new(); + for map in &session.project.maps { + let map_id = map.instance_id(); + maps.insert(map_id); + let Some(terrain) = &map.view.terrain else { + checked.remove(&map_id); + continue; + }; + let (TerrainSource::Layer(name), Some(block)) = (&terrain.source, &terrain.block) else { + checked.remove(&map_id); + continue; + }; + let Some(layer) = map.layer_by_name(name) else { + checked.remove(&map_id); + continue; + }; + let Some(raster) = layer.raster.as_ref().and_then(|raster| raster.data.as_ref()) else { + checked.remove(&map_id); + continue; + }; + let pixels = Arc::downgrade(raster); + if checked + .get(&map_id) + .is_some_and(|old: &Checked| old.layer == layer.instance_id() && old.pixels.ptr_eq(&pixels) && old.extent == block.extent) + { + continue; + } + checked.insert(map_id, Checked { layer: layer.instance_id(), pixels, extent: block.extent }); + let name = crate::project::short_name(name); + match check(&name, raster, block.extent) { + Err(reason) => { + let detail = format!("{} ({reason})", crate::project::short_name(&map.name)); + if restoring { + restored.push(detail); + } else { + dropped.push((map_id, detail)); + checked.remove(&map_id); + } + } + Ok(coverage) if coverage.missing > 0 => partial.push((crate::project::short_name(&map.name), name, coverage)), + Ok(_) => {} + } + } + checked.retain(|map, _| maps.contains(map)); + session.terrain_blocks_checked = checked; + let details = |first: Vec, total: usize| { + let mut text = first.join("; "); + if total > 3 { + text.push_str(&format!("; and {} more", total - 3)); + } + text + }; + if !dropped.is_empty() { + let ids: HashSet<_> = dropped.iter().map(|(id, _)| *id).collect(); + // Pixel loading is not itself an edit transaction. Record its durable view adjustment + // explicitly, including when this is called from a frame rather than a command. + let changed = session.guarded("Update loaded terrain blocks", true, |session| { + for map in &mut session.project.maps { + if ids.contains(&map.instance_id()) + && let Some(terrain) = &mut map.view.terrain + { + terrain.block = None; + } + } + Ok(serde_json::Value::Null) + }); + if let Err(error) = changed { + session.notify(error, true); + } else { + session.notify_text( + NoticeText::format( + "Terrain: {count} block diagram(s) dropped after their DEM loaded: {details}; the terrain draws the whole map", + [ + ("count", dropped.len().to_string().into()), + ("details", details(dropped.iter().take(3).map(|(_, detail)| detail.clone()).collect(), dropped.len()).into()), + ], + ), + false, + ); + } + } + if !restored.is_empty() { + session.notify_text(NoticeText::format( + "Terrain: {count} restored block diagram(s) kept without usable DEM heights: {details}; change their bounds or remove the blocks to use the DEM", + [("count", restored.len().to_string().into()), ("details", details(restored.iter().take(3).cloned().collect(), restored.len()).into())], + ), false); + } + if !partial.is_empty() { + let missing = partial.iter().map(|(_, _, coverage)| coverage.missing).sum::(); + let total = partial.iter().map(|(_, _, coverage)| coverage.total).sum::(); + let first = partial + .iter() + .take(3) + .map(|(map, name, coverage)| { + let [x0, y0, x1, y1] = coverage.extent; + format!("{map}, DEM '{name}', [{x0:.0}, {y0:.0}, {x1:.0}, {y1:.0}]: {}/{}", coverage.missing, coverage.total) + }) + .collect(); + session.notify_text(NoticeText::format( + "Terrain: {missing} of {total} check samples for {count} block diagram(s) have no DEM height (ground fallback 0 m before water and exaggeration); checked on a {side} by {side} grid in Web Mercator boxes: {details}", + [("missing", missing.to_string().into()), ("total", total.to_string().into()), ("count", partial.len().to_string().into()), + ("side", (CHECK_CELLS + 1).to_string().into()), ("details", details(first, partial.len()).into())], + ), false); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustgis_data::raster::{ColorHint, Crs, GeoTransform}; + use rustgis_map::{ + raster::RasterLayer, + view::{BLOCK_WALLS, Terrain, TerrainBlock}, + }; + + fn pixels(nodata: bool) -> Raster { + Raster::new( + "DEM", + 4, + 4, + vec![vec![if nodata { -9999.0 } else { 12.0 }; 16]], + GeoTransform { origin_x: 1000.0, origin_y: 1400.0, pixel_width: 100.0, pixel_height: -100.0 }, + Crs::WebMercator, + Some(-9999.0), + ColorHint::Gray, + "Float32", + ) + .unwrap() + } + fn pending(extent: [f64; 4]) -> Session { + let mut s = Session::new(crate::Project::default()); + s.map_mut().add_raster("DEM", RasterLayer::unread_in_place("https://invalid.example/dem.tif")); + s.map_mut().view.terrain = Some(Terrain { + source: TerrainSource::Layer("DEM".into()), + exaggeration: 1.0, + block: Some(TerrainBlock { extent: Some(extent), depth: None, walls: BLOCK_WALLS, walls_layer: None }), + water: None, + }); + s.notices.clear(); + s + } + fn load(s: &mut Session, r: Raster) { + s.map_mut().layers.iter_mut().find(|l| l.name == "DEM").unwrap().raster.as_mut().unwrap().data = Some(Arc::new(r)); + } + #[test] + fn a_small_valid_dem_between_grid_samples_keeps_its_block() { + let extent = [900.0, 900.0, 64900.0, 64900.0]; + let raster = pixels(false); + assert!(rustgis_render::terrain::dem_height(&raster, Coord::new(1200.0, 1200.0)).is_some()); + let coverage = check("DEM", &raster, Some(extent)).unwrap(); + assert_eq!(coverage.missing, coverage.total); + let mut session = pending(extent); + load(&mut session, raster); + session.notice_landed_rasters(); + assert!(session.map().view.terrain.as_ref().unwrap().block.is_some()); + assert!(session.notices.iter().any(|notice| notice.text.contains("check samples"))); + } + + #[test] + fn a_named_read_only_command_does_not_drop_unrelated_terrain_blocks() { + use crate::policy::{AgentAccess, Call, Effects, Revocation, Scope}; + struct Permit(bool); + impl Revocation for Permit { + fn allows(&self, effects: Effects) -> bool { + self.0 || effects == Effects::NONE + } + } + let mut session = pending([2000.0, 2000.0, 2200.0, 2200.0]); + load(&mut session, pixels(false)); + session.clear_history(); + let before = serde_json::to_value(&session.project).unwrap(); + let state = session.state(); + let scope = Scope::agent(AgentAccess::Allow, false, "terrain-read", None, Arc::new(Permit(false))); + session.run_scoped(scope, &Call::command("info.diagnostics", serde_json::Value::Null)).unwrap(); + assert_eq!(serde_json::to_value(&session.project).unwrap(), before); + assert_eq!(session.state(), state); + assert_eq!(session.undo_depth(), 0); + assert!(!session.dirty()); + // The actual native frame can perform that maintenance with an undoable edit. + session.notice_landed_rasters(); + assert!(session.map().view.terrain.as_ref().unwrap().block.is_none()); + assert!(session.dirty()); + assert_eq!(session.undo_depth(), 1); + // Full access may explicitly Undo the native edit. Its cache bookkeeping is harmless + // and necessary: the following frame must keep the recorded bounds restored by Undo. + let owner = Scope::agent(AgentAccess::Full, true, "terrain-undo", None, Arc::new(Permit(true))); + session.run_scoped(owner, &Call::command("edit.undo", serde_json::Value::Null)).unwrap(); + assert!(session.map().view.terrain.as_ref().unwrap().block.is_some()); + session.notice_landed_rasters(); + assert!(session.map().view.terrain.as_ref().unwrap().block.is_some()); + assert!(!session.dirty()); + assert_eq!(session.undo_depth(), 0); + } + + #[test] + fn deferred_dem_revalidates_disjoint_block_once_without_removing_terrain() { + let mut s = pending([2000.0, 2000.0, 2200.0, 2200.0]); + s.notice_landed_rasters(); + assert!(s.map().view.terrain.as_ref().unwrap().block.is_some()); + assert!(s.notices.is_empty()); + load(&mut s, pixels(false)); + s.notice_landed_rasters(); + assert!(s.map().view.terrain.as_ref().unwrap().block.is_none()); + let count = s.notices.iter().filter(|n| n.text.contains("1 block diagram(s) dropped")).count(); + assert_eq!(count, 1); + s.notice_landed_rasters(); + assert_eq!(s.notices.iter().filter(|n| n.text.contains("block diagram(s) dropped")).count(), 1); + assert!(s.dirty()); + assert_eq!(s.undo_depth(), 1); + s.run("edit.undo", &serde_json::Value::Null).unwrap(); + assert!(s.map().view.terrain.as_ref().unwrap().block.is_some()); + assert!(!s.dirty()); + s.notice_landed_rasters(); + assert!(s.map().view.terrain.as_ref().unwrap().block.is_some(), "a frame must not undo the user's Undo"); + assert_eq!(s.undo_depth(), 0); + s.run("edit.redo", &serde_json::Value::Null).unwrap(); + assert!(s.map().view.terrain.as_ref().unwrap().block.is_none()); + assert!(s.dirty()); + } + #[test] + fn loaded_dem_without_heights_is_refused_before_terrain_changes() { + let mut s = pending([1100.0, 1100.0, 1300.0, 1300.0]); + load(&mut s, pixels(true)); + let before = s.map().view.terrain.clone(); + let error = s + .run("map.terrain", &serde_json::json!({"layer": "DEM", "exaggeration": 2, "block": {"extent": [1100, 1100, 1300, 1300]}})) + .unwrap_err(); + assert!(error.contains("no height"), "{error}"); + assert_eq!(s.map().view.terrain, before); + assert!(!s.notices.iter().any(|notice| notice.text.contains("block diagram(s) dropped"))); + } + + #[test] + fn partial_dem_says_sample_count_and_box_once_per_allocation_clean_dem_says_none() { + let mut s = pending([900.0, 1000.0, 1300.0, 1400.0]); + load(&mut s, pixels(false)); + s.notice_landed_rasters(); + let notices: Vec<_> = s.notices.iter().filter(|n| n.text.contains("check samples")).collect(); + assert_eq!(notices.len(), 1); + assert!(notices[0].text.contains("4225") && notices[0].text.contains("[900, 1000, 1300, 1400]")); + assert!(s.map().view.terrain.as_ref().unwrap().block.is_some()); + s.notice_landed_rasters(); + assert_eq!(s.notices.iter().filter(|n| n.text.contains("check samples")).count(), 1); + load(&mut s, pixels(false)); + s.notice_landed_rasters(); + assert_eq!(s.notices.iter().filter(|n| n.text.contains("check samples")).count(), 2); + let mut clean = pending([1100.0, 1100.0, 1300.0, 1300.0]); + load(&mut clean, pixels(false)); + clean.notice_landed_rasters(); + assert!(clean.notices.is_empty()); + assert!(check("DEM", &pixels(true), Some([1100.0, 1100.0, 1300.0, 1300.0])).unwrap_err().contains("no height")); + } +} diff --git a/crates/engine/src/cmd/view.rs b/crates/engine/src/cmd/view.rs index c5bd706ab1..46206c7c2b 100644 --- a/crates/engine/src/cmd/view.rs +++ b/crates/engine/src/cmd/view.rs @@ -547,6 +547,17 @@ fn terrain(s: &mut Session, p: &Value) -> CmdResult { } return Err(e); } + // Data may arrive after this command. Loaded DEMs can be checked now; deferred ones are + // checked by notice_landed_rasters before their next frame, or after a successful command. + if let (TerrainSource::Layer(name), Some(block)) = (&t.source, &t.block) + && let Some(raster) = m.layer_by_name(name).and_then(|layer| layer.raster.as_ref()).and_then(|raster| raster.data.as_ref()) + && let Err(reason) = super::terrain_blocks::check(&crate::project::short_name(name), raster, block.extent) + { + if let Some(old) = old_terrain { + m.view.terrain = Some(old); + } + return Err(format!("{reason}{kept} (Terrain stays as it was)")); + } // The box the block is cut to, as numbers: the DEM's (an approximation for one turned by its // projection) when none was given. out["block"] = t.block.as_ref().map_or(Value::Null, |b| { @@ -561,6 +572,9 @@ fn terrain(s: &mut Session, p: &Value) -> CmdResult { }; json!({"extent": extent, "extent_from": from, "depth": b.depth, "walls": walls}) }); + if matches!(t.source, TerrainSource::Layer(_)) && t.block.is_some() && dem_box.is_none() { + out["block"]["pendingDem"] = json!(true); + } // The terrain keeps its DEM by name: with two layers of that name (a rename can make them), // it would draw the first one's heights. Checked for a terrain kept on too. if let TerrainSource::Layer(n) = &t.source diff --git a/crates/engine/src/cmd/workspace.rs b/crates/engine/src/cmd/workspace.rs index 5f6e28affd..486cd06cda 100644 --- a/crates/engine/src/cmd/workspace.rs +++ b/crates/engine/src/cmd/workspace.rs @@ -249,21 +249,22 @@ pub fn commands() -> Vec { let path = super::ps(p, "path").ok_or("path is required")?.to_string(); super::field::not_the_project(s, std::path::Path::new(&path))?; let had = s.project.offline_areas.len(); - let (area, bytes) = super::field::package(s, p)?; + let (area, bytes, notes) = super::field::package_output(s, p)?; if let Err(e) = rustgis_data::io::write_atomic(std::path::Path::new(&path), &bytes) { super::field::forget_new_areas(s, had); return Err(format!("{path}: {e}")); } + for note in notes { s.notify(note, false); } s.retain_written_notices(written); Ok(super::field::packaged(&path, &area, bytes.len())) }) .roles(&[]) .effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)) - .params(r#"{"path": "*.rgisfield", "name"?: offline area name, "extent"?: [minLon, minLat, maxLon, maxLat]}"#), + .params(r#"{"path": "*.rgisfield", "name"?: offline area name, "extent"?: [minLon, minLat, maxLon, maxLat], "legacyReturn"?: "native User only: send an old ID-less field project saved as a plain project; its return requires explicit legacyArea confirmation at home"}"#), C::new("offline.bringBack", "Bring Back Edits", "Data › Offline", super::field::bring_back) .roles(&[]) .effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Files)) - .params(r#"{"path"?: "*.rgisfield", "data"?: base64 of one}"#), + .params(r#"{"path"?: "*.rgisfield", "data"?: base64 of one, "legacyArea"?: "native User only: explicitly verified home offline area for an entirely ID-less legacy return; cannot override any present or incomplete identity"}"#), C::new("offline.sync", "Sync", "Data › Offline", offline_sync).roles(&[]).effects(crate::policy::Effects::NONE).params(r#"{"name": offline area}"#), C::new("offline.remove", "Remove", "Data › Offline", |s, p| { let n = ps(p, "name").ok_or("name is required")?; @@ -678,10 +679,13 @@ pub(crate) fn offline_download(s: &mut Session, p: &Value) -> CmdResult { if s.project.offline_areas.len() >= MAX_AREAS { return Err(format!("a project keeps at most {MAX_AREAS} offline areas; remove one first")); } + let id = crate::project::field_id()?; + let origin_id = if s.project.origin_id.is_empty() { crate::project::field_id()? } else { s.project.origin_id.clone() }; let data = snapshot(s.map_mut(), &e); let features: usize = data.layers.iter().map(|l| l.dataset.features.len()).sum(); let t = now(); - s.project.offline_areas.push(OfflineArea { name: name.clone(), map: s.map().name.clone(), extent: e, created: t, synced: t, data }); + s.project.origin_id = origin_id; + s.project.offline_areas.push(OfflineArea { id, name: name.clone(), map: s.map().name.clone(), extent: e, created: t, synced: t, data }); Ok(json!({"name": name, "features": features})) } diff --git a/crates/engine/src/connections.rs b/crates/engine/src/connections.rs index fa089f9c80..df94623501 100644 --- a/crates/engine/src/connections.rs +++ b/crates/engine/src/connections.rs @@ -159,16 +159,17 @@ impl Kept { /// keeps apart from them): a bare `key` is a secret beside a web address, and an address /// holding a secret doesn't stop being one. Only then are the classified entries removed. fn without_secrets(&mut self, mut o: Map, address: &str) -> Map { - let secret: Vec = o + // Count address-bearing siblings once; unknown preference fields are not capped. + let addresses = + o.iter().filter(|(k, v)| !k.eq_ignore_ascii_case("data") && !k.eq_ignore_ascii_case("text") && web_address(v, 0)).count(); + let beside = web_address(&Value::String(address.into()), 0); + let secret: BTreeSet = o .iter() .filter(|(k, v)| { let mut probe = Map::new(); probe.insert((*k).clone(), (*v).clone()); - if web_address(&Value::String(address.into()), 0) - || o.iter().any(|(other, v)| { - other != *k && !other.eq_ignore_ascii_case("data") && !other.eq_ignore_ascii_case("text") && web_address(v, 0) - }) - { + let own_address = !k.eq_ignore_ascii_case("data") && !k.eq_ignore_ascii_case("text") && web_address(v, 0); + if beside || addresses > usize::from(own_address) { probe.insert("\u{1}beside".into(), "https://a".into()); } sources::holds_secret(&Value::Object(probe)) @@ -346,21 +347,63 @@ pub(crate) fn reserve_query_name(c: &Connection) -> Result &str { + let rest = match address.split_once("://") { + Some((_, rest)) => rustgis_data::source::pgconn::userinfo_end(rest).and_then(|i| rest.get(i + 1..)).unwrap_or(rest), + None => address, + }; + rest.split('#').next().and_then(|rest| rest.split_once('?')).map_or("", |(_, query)| query) +} + +/// Legacy semicolon pairs are ambiguous after a credential-bearing pair: their text may be +/// part of its secret value. Retain public prefixes and the first key, withholding its suffix. +fn public_query_pairs<'a>(address: &'a str, hint: Option<&str>) -> Vec<&'a str> { + let query = public_query(address); + let context = rustgis_data::address::QueryNames::of(query); + let mut pairs = Vec::new(); + for segment in query.split('&') { + for pair in segment.split(';').filter(|pair| !pair.is_empty()) { + pairs.push(pair); + let name = pair.split_once('=').map_or(pair, |(name, _)| name); + if rustgis_data::address::name_class(name, &context) != rustgis_data::address::NameClass::Plain + || hint.is_some_and(|hint| name.eq_ignore_ascii_case(hint)) + { + break; + } + } + } + pairs +} + /// The names (never the values) of the query parameters `given` had that `saved` lacks, which a /// query login can carry: the keys an address held and saving left out, to offer as the login's /// parameter. At most four, each once. Only `name=value` pairs count: a bare segment can't be a /// login's parameter, and when it is a key written as a name, its text is the secret. pub fn stripped_parameters(given: &str, saved: &str) -> Vec { + stripped_parameters_with_hint(given, saved, None) +} + +/// Also offers an unregistered parameter the caller explicitly declared as a query-key hint. +/// The hint is per call: merely previewing a definition must not register a login name. +pub(crate) fn stripped_parameters_with_hint(given: &str, saved: &str, hint: Option<&str>) -> Vec { use crate::render::rustgis_tiles::logins; - fn names(address: &str) -> Vec<&str> { - let query = address.split('#').next().and_then(|a| a.split_once('?')).map_or("", |(_, query)| query); - query.split('&').filter_map(|pair| pair.split_once('=').map(|(name, _)| name)).filter(|name| !name.is_empty()).collect() + fn names<'a>(address: &'a str, hint: Option<&str>) -> Vec<&'a str> { + public_query_pairs(address, hint) + .into_iter() + .filter_map(|pair| pair.split_once('=').map(|(name, _)| name)) + .filter(|name| !name.is_empty()) + .collect() } - let kept = names(saved); + let context = rustgis_data::address::QueryNames::of(public_query(given)); + let kept = names(saved, None); let mut left_out: Vec = Vec::new(); - for name in names(given) { + for name in names(given, hint) { if left_out.len() < 4 && logins::valid_query_name(name) + && (hint.is_some_and(|hint| name.eq_ignore_ascii_case(hint)) + || rustgis_data::address::name_class(name, &context) == rustgis_data::address::NameClass::Credential) && !kept.contains(&name) && !left_out.iter().any(|known| known.eq_ignore_ascii_case(name)) { @@ -370,6 +413,41 @@ pub fn stripped_parameters(given: &str, saved: &str) -> Vec { left_out } +/// Public query components lost with a legacy credential-bearing ampersand pair. Count every +/// lost component, including repeated names; report at most four safe names, never bare text or +/// values. These are diagnostics, not query-login suggestions. +pub(crate) fn collateral_parameters(given: &str, saved: &str, hint: Option<&str>) -> (usize, Vec) { + let context = rustgis_data::address::QueryNames::of(public_query(given)); + let mut kept = std::collections::HashMap::<&str, usize>::new(); + for pair in public_query_pairs(saved, None) { + *kept.entry(pair).or_default() += 1; + } + let (mut count, mut names) = (0usize, Vec::::new()); + for pair in public_query_pairs(given, hint) { + let name = pair.split_once('=').map_or(pair, |(name, _)| name); + if rustgis_data::address::name_class(name, &context) != rustgis_data::address::NameClass::Plain + || hint.is_some_and(|hint| name.eq_ignore_ascii_case(hint)) + { + continue; + } + if let Some(remaining) = kept.get_mut(pair) + && *remaining > 0 + { + *remaining -= 1; + continue; + } + count += 1; + if names.len() < 4 + && pair.contains('=') + && crate::render::rustgis_tiles::logins::valid_query_name(name) + && !names.iter().any(|known| known.eq_ignore_ascii_case(name)) + { + names.push(name.to_string()); + } + } + (count, names) +} + /// The origin (`scheme://host[:port]`) a web-service connection's login goes to: the only one. pub fn login_origin(c: &Connection) -> Option { crate::render::rustgis_tiles::logins::origin(&c.address) @@ -395,8 +473,11 @@ pub fn clean(mut c: Connection) -> Result<(Connection, bool), String> { return Err("a connection address must be at most 2048 bytes and contain no control characters".into()); } // A query login's own parameter goes before anything quotes the address (an error included): - // its name may not be registered yet, so the generic redaction can't see it. - let query_name = Some(c.key_name.trim()).filter(|name| c.auth.trim().eq_ignore_ascii_case("query") && !name.is_empty()); + // its name may not be registered yet, so the generic redaction can't see it. A name given + // without a method still declares a possible key, as it does in the replay log. An explicit + // other method (`none` included) does not select a query parameter. + let query_name = Some(c.key_name.trim()) + .filter(|name| (c.auth.trim().is_empty() || c.auth.trim().eq_ignore_ascii_case("query")) && !name.is_empty()); let given = query_name.map_or_else(|| t.to_string(), |name| crate::render::rustgis_tiles::logins::without_query_parameter(t, name)); let stored = sources::redact(&given); let stripped = stored != t; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9e60d4d2e5..4ee739578d 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -801,6 +801,8 @@ pub struct Session { /// The count of cloud-optimised GeoTIFF headers landed when [`Session::notice_landed_rasters`] /// last looked back at the dates a dated series has shown (`cog::landed_count`). cog_headers_seen: u64, + /// Last loaded DEM and block extent validated for each live map; no pixel buffers retained. + terrain_blocks_checked: std::collections::HashMap, /// Layer whose attribute table is open. pub table_layer: Option, /// The standalone table shown in the attribute table instead of a layer, by its id. @@ -1247,6 +1249,7 @@ impl Session { crs_used_elsewhere: Vec::new(), grid_notes_checked: None, cog_headers_seen: 0, + terrain_blocks_checked: Default::default(), table_layer: None, table_standalone: None, clipboard: None, @@ -1635,6 +1638,9 @@ impl Session { render::rustgis_tiles::mbtiles::close_unused(files.into_iter()); } // Pixels the command read in place (a project, an import) say what they lost. + if res.is_ok() { + cmd::terrain_blocks::after_command(self, id); + } self.notice_landed_pixels(); // And the coordinate systems that needed a missing datum grid, after every command. self.say_grid_notes(); @@ -2185,6 +2191,7 @@ impl Session { /// Once for each layer's pixels, failure or file, one notice of each kind; the app calls it /// each frame. pub fn notice_landed_rasters(&mut self) { + cmd::terrain_blocks::landed(self); self.notice_landed_pixels(); // From frames too: a system a frame first drew needs no command after it to be said. self.say_grid_notes_now_and_then(); diff --git a/crates/engine/src/model.rs b/crates/engine/src/model.rs index 53b0c7ac3f..65506415da 100644 --- a/crates/engine/src/model.rs +++ b/crates/engine/src/model.rs @@ -520,6 +520,9 @@ pub fn graph_from_json_counted(v: &Value) -> Result<(ModelGraph, GraphDropped), for n in nodes { let Ok(mut node) = serde_json::from_value::(n.clone()) else { dropped.nodes += 1; + if let Some(id) = n.get("id").and_then(Value::as_str) { + given_dropped.push(id.to_string()); + } continue; }; let id: String = node.id.trim().chars().take(MAX_NODE_ID).collect(); diff --git a/crates/engine/src/policy/dispatch.rs b/crates/engine/src/policy/dispatch.rs index f69e5742d7..182ef9c6cb 100644 --- a/crates/engine/src/policy/dispatch.rs +++ b/crates/engine/src/policy/dispatch.rs @@ -717,8 +717,12 @@ impl Session { // command of an extension that is off, as `Session::run` does: the one asked for, and // the one a completion finishes. if let Some(why) = self.extensions().blocks(id) { - // A completion's raster is let go, as `finish_raster` does. - if let Some(run) = params.get("ran").and_then(Value::as_u64) { + // Only its own completion may release the raster, as `finish_raster` does. + if let Some(run) = params.get("ran").and_then(Value::as_u64) + && let Some(c) = self.raster_results.iter().find(|(n, _)| *n == run).and_then(|(_, m)| m.continuation.as_ref()) + && c.node.id() == id + && self.check_delivery_owner(c).is_ok() + { self.raster_results.retain(|(n, _)| *n != run); } return Err(why); diff --git a/crates/engine/src/policy/provenance.rs b/crates/engine/src/policy/provenance.rs index 5f87c97152..4858c7495b 100644 --- a/crates/engine/src/policy/provenance.rs +++ b/crates/engine/src/policy/provenance.rs @@ -89,7 +89,10 @@ impl Session { "model.new" => "model", _ => return, }; - if p.get("fromHistory").and_then(Value::as_bool) == Some(true) && p.get("steps").is_none() && self.history_from_file > 0 { + if p.get("fromHistory").and_then(Value::as_bool) == Some(true) + && p.get("steps").is_none_or(Value::is_null) + && self.history_from_file > 0 + { return; } let name = if kind == "notebook" { diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index db4620d0e8..e9dca33143 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -8,8 +8,11 @@ use serde::{Deserialize, Serialize}; use crate::GpRecord; +#[path = "project_capabilities.rs"] +mod capabilities; #[path = "project_label_losses.rs"] pub(crate) mod label_losses; +pub(crate) use capabilities::{layer_file_bytes, map_file_bytes}; /// A map export cannot keep runtime-only download reservations, even in a larger project. pub(crate) fn ensure_map_imports_complete(map: &Map) -> Result<(), String> { @@ -59,15 +62,15 @@ pub fn open_command(path: &str) -> &'static str { /// RustGIS release version. Advance it when an older reader would lose meaningful saved /// information on resaving; within format 2, such an addition is named in `requires` instead /// (see [`READ_FORMAT`] and [`CAPABILITIES`]). -pub const FORMAT: u32 = 1; -/// The newest format this RustGIS reads. Format 2 is read a release before anything writes it +pub const FORMAT: u32 = 2; +/// The newest format this RustGIS reads. Format 2 was made readable before writing began /// (#1853): a document of format 2 opens when this RustGIS knows every capability it `requires`, /// so what a later RustGIS adds to format 2, and an older one would lose, is listed there and /// refused by name instead. pub const READ_FORMAT: u32 = 2; -/// The capabilities a saved document may require that this RustGIS reads. None yet: format 2 is -/// format 1's content, and each later addition an older reader would lose comes with its name. -pub const CAPABILITIES: &[&str] = &[]; +/// The saved additions this RustGIS understands. Writers require only capabilities used by +/// their payload, so a format-2 reader without them refuses instead of discarding their fields. +pub const CAPABILITIES: &[&str] = &["terrain-block", "terrain-water", "shaded-relief-effects", "normalized-legend-urls", "field-origin"]; /// The most entries a document's `requires` may list: past it the document is refused without /// the rest being read. pub(crate) const MAX_REQUIRES: usize = 256; @@ -338,6 +341,7 @@ pub(crate) fn public_login_params(value: &serde_json::Value) -> Option, #[cfg_attr(feature = "schema", schemars(with = "Vec"))] pub maps: Vec, pub active_map: usize, @@ -560,10 +570,31 @@ pub struct Job { pub updated: f64, } +/// A field copy's exact home and offline-area identities (128 random bits each). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct FieldOrigin { + pub project: String, + pub area: String, +} + +pub(crate) fn field_id() -> Result { + let mut bits = [0u8; 16]; + getrandom::fill(&mut bits).map_err(|e| format!("cannot make a field-package identity: {e}"))?; + Ok(bits.iter().map(|byte| format!("{byte:02x}")).collect()) +} + +pub(crate) fn valid_field_id(id: &str) -> bool { + id.len() == 32 && id.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + /// A map's data clipped to an extent, to take into the field. #[derive(Clone, Debug, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] pub struct OfflineArea { + /// Persistent identity, independent of the editable display name. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub id: String, pub name: String, /// The map it came from (by name). pub map: String, @@ -1039,6 +1070,8 @@ impl Project { read_notices: Vec::new(), format: FORMAT, name: name.into(), + origin_id: String::new(), + field_origin: None, maps: vec![Map::new("Map")], active_map: 0, tables: Vec::new(), @@ -1619,6 +1652,13 @@ impl Project { format!("not a RustGIS project: {msg}") } })?; + let mut area_ids = std::collections::HashSet::new(); + if (!p.origin_id.is_empty() && !valid_field_id(&p.origin_id)) + || p.field_origin.as_ref().is_some_and(|origin| !valid_field_id(&origin.project) || !valid_field_id(&origin.area)) + || p.offline_areas.iter().any(|area| !area.id.is_empty() && (!valid_field_id(&area.id) || !area_ids.insert(area.id.as_str()))) + { + return Err("not a RustGIS project: an offline field identity is invalid or duplicated".into()); + } p.ensure_imports_complete().map_err(|_| "a project file cannot contain runtime GeoLibre download reservations".to_string())?; // (A backstop: `check_format` has refused it already.) if p.format > READ_FORMAT { @@ -2205,6 +2245,8 @@ impl Durable for Project { read_notices: _, format: _, name: _, + origin_id: _, + field_origin: _, maps, active_map: _, tables, @@ -2233,7 +2275,7 @@ impl Durable for Project { p.child(Owner::Map(i), v); } for (i, v) in offline_areas.iter().enumerate() { - let OfflineArea { name: _, map: _, extent: _, created: _, synced: _, data } = v; + let OfflineArea { id: _, name: _, map: _, extent: _, created: _, synced: _, data } = v; p.child(Owner::Area(i), data); } for (i, v) in tables.iter().enumerate() { diff --git a/crates/engine/src/project_capabilities.rs b/crates/engine/src/project_capabilities.rs new file mode 100644 index 0000000000..d3cc617124 --- /dev/null +++ b/crates/engine/src/project_capabilities.rs @@ -0,0 +1,190 @@ +//! Required capabilities at saved-document boundaries. The derived Project fields stay the +//! serialization source of truth; the header is computed from the payload, never cached. + +use rustgis_map::{Layer, Map, raster::RasterRenderer}; +use serde::{Deserialize, Serialize}; + +use super::{FORMAT, Project}; + +impl Serialize for Project { + fn serialize(&self, serializer: S) -> Result { + #[derive(Serialize)] + struct Document<'a> { + #[serde(skip_serializing_if = "Vec::is_empty")] + requires: Vec<&'static str>, + #[serde(flatten, with = "Project")] + project: &'a Project, + } + Document { requires: self.required_capabilities(), project: self }.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Project { + fn deserialize>(deserializer: D) -> Result { + Project::deserialize(deserializer) + } +} + +impl Project { + /// Required features of all maps, including snapshots that can later be restored. + pub(crate) fn required_capabilities(&self) -> Vec<&'static str> { + let mut required = map_capabilities(self.maps.iter().chain(self.offline_areas.iter().map(|a| &a.data))); + if !self.origin_id.is_empty() || self.field_origin.is_some() || self.offline_areas.iter().any(|area| !area.id.is_empty()) { + required.push("field-origin"); + } + required + } +} + +fn relief_effects(layer: &Layer) -> bool { + layer.raster.as_ref().is_some_and(|r| { + matches!(&r.style.renderer, RasterRenderer::ShadedRelief { shadow, sky, fill, rock, .. } + if *shadow != 0.0 || *sky != 0.0 || *fill != 0.0 || rock.is_some()) + }) +} + +fn map_capabilities<'a>(maps: impl IntoIterator) -> Vec<&'static str> { + let (mut block, mut water, mut relief, mut legends) = (false, false, false, false); + for map in maps { + if let Some(t) = &map.view.terrain { + block |= t.block.is_some(); + water |= t.water.is_some(); + } + relief |= map.layers.iter().any(relief_effects); + legends |= map.tiles.iter().any(|tile| { + tile.legends.iter().any(|legend| rustgis_data::login_address::inner(&legend.url).len() > rustgis_map::tiles::MAX_LEGEND_URL) + }); + } + [(block, "terrain-block"), (water, "terrain-water"), (relief, "shaded-relief-effects"), (legends, "normalized-legend-urls")] + .into_iter() + .filter_map(|(used, name)| used.then_some(name)) + .collect() +} + +#[derive(Serialize)] +struct Item<'a, T> { + format: u32, + #[serde(skip_serializing_if = "Vec::is_empty")] + requires: Vec<&'static str>, + #[serde(flatten)] + value: &'a T, +} + +/// A map file has its own header, independent of the project it came from. +pub(crate) fn map_file_bytes(map: &Map) -> Result, String> { + serde_json::to_vec(&Item { format: FORMAT, requires: map_capabilities([map]), value: map }).map_err(|e| e.to_string()) +} + +/// A layer file keeps the same renderer capabilities as the layer saved inside a project. +pub(crate) fn layer_file_bytes(layer: &Layer) -> Result, String> { + let requires = if relief_effects(layer) { vec!["shaded-relief-effects"] } else { Vec::new() }; + serde_json::to_vec(&Item { format: FORMAT, requires, value: layer }).map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{Value, json}; + + fn json_bytes(bytes: &[u8]) -> Value { + serde_json::from_slice(bytes).unwrap() + } + + #[test] + fn field_identity_is_preserved_and_required_in_a_plain_saved_project() { + let project = Project::from_json(include_str!("../../../schema/fixtures/field_origin.rgis")).unwrap(); + let saved = serde_json::to_value(&project).unwrap(); + assert_eq!(saved["requires"], json!(["field-origin"])); + let reopened = Project::from_json(&saved.to_string()).unwrap(); + assert_eq!(reopened.origin_id, project.origin_id); + assert_eq!(reopened.field_origin, project.field_origin); + let mut invalid = saved.clone(); + invalid["field_origin"]["area"] = json!("not-an-id"); + assert!(Project::from_json(&invalid.to_string()).unwrap_err().contains("identity")); + let mut clean = Project::new("Legacy"); + assert!(serde_json::to_value(&clean).unwrap().get("requires").is_none()); + clean.origin_id = "0123456789abcdef0123456789abcdef".into(); + assert_eq!(serde_json::to_value(&clean).unwrap()["requires"], json!(["field-origin"])); + } + + #[test] + fn normalized_legend_capability_follows_maps_and_offline_snapshots() { + let mut p = Project::from_json(include_str!("../../../schema/fixtures/normalized_legend.rgis")).unwrap(); + assert_eq!(p.map().tiles[0].legends.len(), 1); + let url = p.map().tiles[0].legends[0].url.clone(); + assert!(url.len() > rustgis_map::tiles::MAX_LEGEND_URL); + let saved = serde_json::to_value(&p).unwrap(); + assert_eq!(saved["requires"], json!(["normalized-legend-urls"])); + assert_eq!(json_bytes(&map_file_bytes(p.map()).unwrap())["requires"], saved["requires"]); + p.offline_areas.push(super::super::OfflineArea { + id: String::new(), + name: "Field".into(), + map: "Map".into(), + extent: rustgis_geom::Extent::new(0.0, 0.0, 1.0, 1.0), + created: 0.0, + synced: 0.0, + data: p.map().clone(), + }); + p.maps[0].tiles.clear(); + assert_eq!(serde_json::to_value(&p).unwrap()["requires"], json!(["normalized-legend-urls"])); + let reopened = Project::from_json(&saved.to_string()).unwrap(); + assert_eq!(reopened.map().tiles[0].legends[0].url, url); + } + + /// Legacy files still open; a save names terrain features an older format-2 reader loses. + #[test] + fn terrain_headers_follow_the_saved_payload_including_offline_snapshots() { + let mut p = Project::from_json(include_str!("../../../schema/fixtures/terrain_water.rgis")).unwrap(); + let saved = serde_json::to_value(&p).unwrap(); + assert_eq!(saved["format"], 2); + assert_eq!(saved["requires"], json!(["terrain-water"])); + assert_eq!(json_bytes(&map_file_bytes(p.map()).unwrap())["requires"], saved["requires"]); + let loaded = Project::from_json(&saved.to_string()).unwrap(); + assert_eq!(loaded.map().view.terrain, p.map().view.terrain); + p.offline_areas.push(super::super::OfflineArea { + id: String::new(), + name: "Field".into(), + map: "Map".into(), + extent: rustgis_geom::Extent::new(0.0, 0.0, 1.0, 1.0), + created: 0.0, + synced: 0.0, + data: p.map().clone(), + }); + p.maps[0].view.terrain = None; + assert_eq!(serde_json::to_value(&p).unwrap()["requires"], json!(["terrain-water"])); + p.offline_areas.clear(); + let clean = serde_json::to_value(&p).unwrap(); + assert!(clean.get("requires").is_none()); + assert!(json_bytes(&map_file_bytes(p.map()).unwrap()).get("requires").is_none()); + assert!(Project::from_json(include_str!("../../../schema/fixtures/terrain_water_v2.rgis")).is_ok()); + let mut unknown = saved; + unknown["requires"] = json!(["future-terrain"]); + assert!(Project::from_json(&unknown.to_string()).unwrap_err().contains("future-terrain")); + } + + /// Nondefault relief effects survive project, map and layer files; ordinary relief needs none. + #[test] + fn relief_headers_cover_each_effect_and_clean_controls() { + let raster: rustgis_map::raster::RasterLayer = serde_json::from_value(json!({ + "style": {"renderer": {"ShadedRelief": {"band": 0, "ramp": null, "azimuth": 315.0, "altitude": 45.0, "z_factor": 1.0}}} + })) + .unwrap(); + let clean = Layer::new_raster(1, "DEM", raster); + for field in ["shadow", "sky", "fill", "rock"] { + let mut value = serde_json::to_value(&clean).unwrap(); + value["raster"]["style"]["renderer"]["ShadedRelief"][field] = + if field == "rock" { serde_json::to_value(rustgis_map::raster::Rock::DEFAULT).unwrap() } else { json!(0.4) }; + let layer: Layer = serde_json::from_value(value).unwrap(); + let mut p = Project::new("Relief"); + p.maps[0].layers.push(layer.clone()); + for document in [ + serde_json::to_value(&p).unwrap(), + json_bytes(&map_file_bytes(p.map()).unwrap()), + json_bytes(&layer_file_bytes(&layer).unwrap()), + ] { + assert_eq!(document["requires"], json!(["shaded-relief-effects"]), "{field}: {document}"); + } + } + assert!(json_bytes(&layer_file_bytes(&clean).unwrap()).get("requires").is_none()); + } +} diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 339d479372..ab8688e623 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -4,7 +4,7 @@ //! without their passwords, tokens or signatures. use rustgis_data::source::credentials::{redact_query, secret_key}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value, json}; @@ -89,17 +89,17 @@ impl Kept { /// holding a secret doesn't stop being one; `known` is the row's known details (its `extra`), a /// sibling too. Only then are the classified entries removed. fn without_secrets(&mut self, mut o: Map, address: &str, known: &Value) -> Map { - let secret: Vec = o + // Count address-bearing siblings once; unknown preference fields are not capped. + let addresses = + o.iter().filter(|(k, v)| !k.eq_ignore_ascii_case("data") && !k.eq_ignore_ascii_case("text") && web_address(v, 0)).count(); + let beside = web_address(&Value::String(address.into()), 0) || web_address(known, 0); + let secret: BTreeSet = o .iter() .filter(|(k, v)| { let mut probe = Map::new(); probe.insert((*k).clone(), (*v).clone()); - if web_address(&Value::String(address.into()), 0) - || web_address(known, 0) - || o.iter().any(|(other, v)| { - other != *k && !other.eq_ignore_ascii_case("data") && !other.eq_ignore_ascii_case("text") && web_address(v, 0) - }) - { + let own_address = !k.eq_ignore_ascii_case("data") && !k.eq_ignore_ascii_case("text") && web_address(v, 0); + if beside || addresses > usize::from(own_address) { probe.insert("\u{1}beside".into(), "https://a".into()); } holds_secret(&Value::Object(probe)) @@ -797,7 +797,7 @@ pub(crate) fn strip_capabilities(address: &str) -> String { fn strip_capabilities_at(address: &str, depth: usize) -> String { if depth >= 8 { - return String::new(); + return "[address omitted]".into(); } let text = address.trim_start(); if let Some((head, archive)) = template_archive(text) { @@ -866,6 +866,10 @@ pub fn scrub_s3_wrappers(text: &str) -> String { } /// Recursively redacts known URL fields in a JSON value for non-User callers. +/// `tilesets`, `describe` and `capabilities` name protocol documents' addresses, not their +/// contents: respectively `ogc::tileset`, `describe_coverage_url` and Identify's `next_fetch`. +/// Supplied protocol bodies are parsed separately; Identify's input `capabilities` parameter +/// contains XML, while its result field of the same name is a GetCapabilities address. pub(crate) fn redact_url_fields(value: &mut Value) { match value { Value::Object(map) => { @@ -1035,6 +1039,19 @@ fn build_library() -> Vec<(&'static str, Source)> { mod tests { use super::*; + #[test] + fn deeply_nested_capability_addresses_say_when_the_inner_address_is_omitted() { + let public = "https://example.invalid/data.tif"; + assert_eq!(strip_capabilities(public), public); + let prefix = crate::render::rustgis_tiles::s3::ANONYMOUS; + for depth in [8, 32] { + let nested = format!("{}{public}", prefix.repeat(depth)); + let shown = strip_capabilities(&nested); + assert!(shown.ends_with("[address omitted]"), "{shown}"); + assert!(!shown.contains(public), "the unvisited inner address is not returned"); + } + } + #[test] fn restored_sources_never_reuse_a_previous_process_login_binding() { let address = rustgis_data::s3_address::bind("0123456789abcdef", "https://s3.example.org", "s3://data/a.fgb").unwrap(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 9394fd1d99..7ff5306785 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -6472,6 +6472,7 @@ fn opening_projects_reports_moved_label_and_callout_adjustments() { transparency: 0.0, }); project.offline_areas.push(crate::project::OfflineArea { + id: String::new(), name: "Field copy".into(), map: "Map".into(), extent: rustgis_geom::Extent::new(-1.0, -1.0, 1.0, 1.0), @@ -12849,6 +12850,17 @@ fn a_link_chart_says_what_its_limits_leave_out() { let shows = "300 of the 301 related features and 20000 of the 20200 links"; assert_eq!(said(&s, mark, "Mesh"), [format!("Mesh: the link chart shows {shows}; {limits}")]); assert!(html.contains(&format!("Shows {shows}."))); + // A self-link needs one node, including the last available slot. Counting its endpoints + // twice would stop at 2,999 even though all 3,000 features and links fit. + let keys: Vec = (0..3000).map(|i| format!("self{i}")).collect(); + add(&mut s, "self.csv", &keys); + s.run("data.relate", &json!({"name": "Self", "origin": "self", "originKey": "key", "destination": "self", "destinationKey": "key"})) + .unwrap(); + let mark = s.notice_mark(); + s.run("share.linkChart", &json!({"relationship": "Self"})).unwrap(); + let (links, nodes, html) = drawn("Self"); + assert_eq!((links, nodes), (3000, 3000)); + assert!(said(&s, mark, "Self").is_empty() && !html.contains("Shows ")); let _ = std::fs::remove_dir_all(dir); } @@ -17551,7 +17563,12 @@ fn split_sources_keep_their_points_on_top() { let origin = |family| rustgis_map::source::DuckdbOrigin { sql: Some("SELECT 1".into()), database: None, table: None, crs: None, family }; let parts = duckdb_parts(&mut s, "Trip", duckdb::Output { parts, skipped: 0, other_family: 0 }, origin); - add_converted(&mut s, &json!({}), "the query", "DuckDB query", parts).unwrap(); + let requested = "Requested".repeat(500); + add_converted(&mut s, &json!({"name": requested}), "the query", "DuckDB query", parts).unwrap(); + let warning: Vec<_> = s.notices.iter().filter(|n| n.text.contains("converted layer parts keep their generated names")).collect(); + assert_eq!(warning.len(), 1); + assert!(warning[0].text.contains("3 converted layer parts") && warning[0].text.len() < 300); + assert!(!warning[0].text.contains(&requested)); assert_eq!(names(&s), ["Trip (points)", "Trip (lines)", "Trip (polygons)"]); // A GDAL file with two mixed layers: each keeps its points on top, and the file's second layer @@ -17572,6 +17589,18 @@ fn split_sources_keep_their_points_on_top() { add_converted(&mut s, &json!({}), &original, &original, gdal_parts(converted, &original, None)).unwrap(); assert_eq!(names(&s), ["b (points)", "b (lines)", "b (polygons)", "a (points)", "a (lines)", "a (polygons)"]); + assert!(!s.notices.iter().any(|n| n.text.contains("converted layer parts keep their generated names")), "no ignored name was supplied"); + let mut single = Session::new(Project::default()); + let output = duckdb::Output { + parts: vec![duckdb::Part { path: write("single", Family::Points), family: Family::Points, count: 1 }], + skipped: 0, + other_family: 0, + }; + let one = duckdb_parts(&mut single, "Generated", output, origin); + add_converted(&mut single, &json!({"name": "Requested"}), "the query", "DuckDB query", one).unwrap(); + assert_eq!(names(&single), ["Requested"]); + assert!(!single.notices.iter().any(|n| n.text.contains("converted layer parts keep their generated names"))); + // A file the built-in readers split by type (a KML here, as GPX, OSM or DXF): the same order, // and the first layer added (the polygons, away from the rest) still frames an empty map on // its own extent, not on all the data's. @@ -21960,7 +21989,8 @@ fn deleting_a_field_clears_the_label_expressions_reading_it() { ["Countries: its label expressions read the deleted field tag, so they are removed: Default's size; Big's text, size and angle"] ); let saved = serde_json::to_string(&got).unwrap(); - assert!(!saved.contains("tag"), "{saved}"); + let reopened: rustgis_map::Labeling = serde_json::from_str(&saved).unwrap(); + assert_eq!(reopened, got, "the saved label expressions stay cleared"); // Undo brings the field and the expressions back. s.run("edit.undo", &Value::Null).unwrap(); let got = labels(&s); @@ -22018,6 +22048,16 @@ fn deleting_a_field_is_refused_while_the_time_slider_reads_it() { assert_eq!(s.undo_depth(), depth); assert!(s.map().layer_by_name("Events").and_then(|l| l.dataset.field_index("when")).is_some()); assert_eq!(shown(&s), ["e0"]); + // Turning off the map slider retains its layer binding; deletion still refuses without + // an undo step, even though every row is now shown. + s.run("time.set", &json!({"off": true})).unwrap(); + let off_depth = s.undo_depth(); + assert!(s.map().time.is_none()); + assert!(s.map().layer_by_name("Events").and_then(|layer| layer.time.as_ref()).is_some()); + let error = s.run("edit.deleteField", &json!({"layer": "Events", "field": "when"})).unwrap_err(); + assert!(error.contains("unbind it first"), "{error}"); + assert_eq!(s.undo_depth(), off_depth); + assert!(s.map().layer_by_name("Events").and_then(|layer| layer.dataset.field_index("when")).is_some()); // Another field goes; once unbound, the slider's field goes too. s.run("edit.deleteField", &json!({"layer": "Events", "field": "ms"})).unwrap(); s.run("time.unbind", &Value::Null).unwrap(); @@ -30683,7 +30723,7 @@ fn keys_read_by_commands() -> Vec<(String, std::collections::BTreeSet)> let call = if *m == module { name.as_str() } else { qualified.as_str() }; let key = format!("{m}::{name}"); // A dispatcher by kind: its arms read keys for other commands. - if key == "items::item_bytes" { + if matches!(key.as_str(), "items::item_bytes" | "items::item_output") { continue; } if !seen.contains(&key) && args.iter().any(|a| calls(&body, call, a)) { @@ -34265,6 +34305,9 @@ fn a_new_renderer_from_a_style_or_subtypes_ends_no_fill() { assert!(said.iter().any(|t| t.contains("turning No Fill off drops")), "{said:?}"); let l = s.map().layer(id).unwrap(); assert!(l.no_fill.is_none() && l.hidden_fills.is_none()); + let rustgis_map::Renderer::Unique { classes, other, .. } = &l.renderer else { panic!("subtype classes") }; + assert!(classes.iter().all(|(_, symbol)| symbol.fill.a() > 0)); + assert!(other.fill.a() > 0, "the new renderer's fills are drawn after No Fill ends"); } /// A class's fill colour chosen while No Fill is on is recorded for Restore Fill only while the @@ -35845,6 +35888,59 @@ fn model_graph_drops_are_noticed() { assert_eq!(saved, Some(2), "the connection naming \" i2 \" follows its trimmed id"); } +#[test] +fn imported_history_with_null_steps_does_not_make_a_trusted_model() { + let mut s = world(); + s.run("gp.run", &json!({"tool": "buffer", "params": {"input": "Cities", "distance": 100}})).unwrap(); + s.history_from_file = 1; + for (name, null_steps) in [("Omitted steps", false), ("Null steps", true)] { + let mut params = json!({"name": name, "fromHistory": true}); + if null_steps { + params["steps"] = Value::Null; + } + s.run("model.new", ¶ms).unwrap(); + assert!(!s.item_trusted("model", name), "a history model remains imported with null_steps={null_steps}"); + } + s.history_from_file = 0; + s.run("model.new", &json!({"name": "Local history", "fromHistory": true, "steps": null})).unwrap(); + assert!(s.item_trusted("model", "Local history")); +} + +#[test] +fn unreadable_model_nodes_do_not_redirect_their_connections() { + for unreadable_first in [false, true] { + let kept = json!({"id": " A ", "kind": "input", "layer": "Cities"}); + let unreadable = json!({"id": "A", "kind": "unknown"}); + let mut nodes = if unreadable_first { vec![unreadable, kept] } else { vec![kept, unreadable] }; + nodes.push(json!({"id": "out", "kind": "output", "name": "Result"})); + let (graph, dropped) = crate::model::graph_from_json_counted(&json!({ + "nodes": nodes, + "edges": [ + {"from": "A", "fromPort": "out", "to": "out", "toPort": "in"}, + {"from": " A ", "fromPort": "out", "to": "out", "toPort": "in"} + ] + })) + .unwrap(); + assert_eq!(dropped.nodes, 1); + assert_eq!(dropped.edges, 1, "the unreadable node's connection must be omitted"); + assert_eq!(graph.edges.len(), 1, "the valid node's original ID still resolves"); + assert_eq!(graph.edges[0].from, "A"); + } + // A readable node with the exact same original ID still owns its connections. + let (graph, dropped) = crate::model::graph_from_json_counted(&json!({ + "nodes": [ + {"id": "A", "kind": "unknown"}, + {"id": "A", "kind": "input", "layer": "Cities"}, + {"id": "out", "kind": "output", "name": "Result"} + ], + "edges": [{"from": "A", "fromPort": "out", "to": "out", "toPort": "in"}] + })) + .unwrap(); + assert_eq!(dropped.nodes, 1); + assert_eq!(dropped.edges, 0); + assert_eq!(graph.edges.len(), 1); +} + /// ModelBuilder graphs: validated, run in flow order, intermediate results dropped, outputs named. fn model_graphs(s: &mut Session) { let graph = json!({ @@ -43551,6 +43647,37 @@ fn plugin_rasters_with_a_null_path_need_no_files() { rustgis_plugins::unload("eng-null-path"); } +/// A disabled command rejects foreign completions without consuming the rightful owner's raster. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn disabled_extensions_do_not_release_another_callers_raster() { + use crate::cmd::workflows::{raster_land, raster_start}; + use crate::policy::{AgentAccess, Effects, Revocation, Scope}; + use rustgis_analysis::control::Control; + struct Live; + impl Revocation for Live { + fn allows(&self, _: Effects) -> bool { + true + } + } + let mut s = world(); + s.run("layer.addDataBytes", &json!({"fileName": "dem.tif", "data": dem_b64()})).unwrap(); + let p = json!({"layer": "dem", "function": "hillshade"}); + let run = raster_start(&mut s, "raster.function", &p).unwrap(); + let id = run.id; + let made = run.run(&Control::new()).unwrap(); + assert_eq!(raster_land(&mut s, id, made), Ok(true)); + s.run("extension.disable", &json!({"id": "imagery"})).unwrap(); + let foreign = Scope::agent(AgentAccess::Refuse, true, "foreign-raster", None, std::sync::Arc::new(Live)); + assert!(s.with_scope(foreign, |s| s.run("raster.function", &json!({"ran": id}))).unwrap().is_err()); + assert!(s.raster_results.iter().any(|(n, _)| *n == id), "foreign rejection preserves the result"); + assert!(s.run("dl.remove", &json!({"ran": id, "name": "unrelated"})).is_err()); + assert!(s.raster_results.iter().any(|(n, _)| *n == id), "another command cannot consume it"); + s.run("extension.enable", &json!({"id": "imagery"})).unwrap(); + assert!(s.finish_raster("raster.function", id, &p).is_ok()); + assert!(!s.raster_results.iter().any(|(n, _)| *n == id)); +} + /// The Analysis pane's single-pass raster tools made on another thread (#1291 step 3): each raster /// handed over (`ran`) is the one its command makes here, bit for bit and with what the command /// says of it besides, and a cancelled run is an error saying so. @@ -45650,9 +45777,8 @@ fn importing_a_file_that_is_neither_map_nor_project_says_so() { assert!(e.contains("Update RustGIS"), "{e}"); } -/// Format 2 is read a release before anything writes it (#1853 piece 1): a project of format 2 -/// opens, and is saved again in the format this RustGIS writes; one needing a capability this -/// RustGIS doesn't know is refused naming it; a newer format is refused as before. +/// A project of format 2 opens and is saved again in the format this RustGIS writes; one needing +/// an unknown capability is refused naming it, and a newer format is refused as before. #[test] fn format_2_is_read_unless_it_needs_what_this_rustgis_cannot_read() { let s = world(); @@ -45664,8 +45790,8 @@ fn format_2_is_read_unless_it_needs_what_this_rustgis_cannot_read() { } v.to_string() }; - // Format 2, with no capabilities, an empty list or none: read whole, and written back as - // format 1, the format this RustGIS writes. + // Format 2, with no capabilities, an empty list or none: read whole, and written back in + // the current format without inventing required capabilities. for requires in [None, Some(json!([])), Some(Value::Null)] { let p = Project::from_json(&as_format(2, requires.clone())).unwrap(); assert_eq!(p.format, 2); @@ -46586,6 +46712,7 @@ fn opening_a_diagram_that_wont_show_says_why() { let mut offline = Session::new(Project::from_json(&clean).unwrap()); let snapshot = s.map().clone(); offline.project.offline_areas.push(crate::project::OfflineArea { + id: String::new(), name: "Trip".into(), map: snapshot.name.clone(), extent: rustgis_geom::Extent::new(-10.0, -10.0, 10.0, 10.0), diff --git a/crates/engine/src/tests/credential_output.rs b/crates/engine/src/tests/credential_output.rs index 7f879c8230..567e5b06c5 100644 --- a/crates/engine/src/tests/credential_output.rs +++ b/crates/engine/src/tests/credential_output.rs @@ -520,10 +520,10 @@ fn durable_future_documents_keep_their_exact_text_unchecked() { assert_eq!(kept.contains(SECRET), unchecked == 1, "{kept}"); } // A task document's reader doesn't look at `requires`, so listing one doesn't keep it unchecked; - // of format 2, which its typed copy would lose, it stays as written, as before. + // a future format stays as written because its typed copy could lose new fields. for (task, unchecked) in [ (json!({"name": "t", "requires": ["z-values"], "steps": []}), 0), - (json!({"format": crate::project::READ_FORMAT, "name": "t", "steps": []}), 1), + (json!({"format": crate::project::READ_FORMAT + 1, "name": "t", "steps": []}), 1), ] { let (_, report) = crate::roles::command("task.import", &json!({"text": task.to_string()}), 0); assert_eq!(report.unchecked, unchecked, "{task}"); @@ -844,7 +844,9 @@ fn durable_database_origins_clean_each_owned_field_without_opening_a_database() let mark = s.notice_mark(); let (_, bytes) = crate::cmd::items::item_bytes(&mut s, "layer", &json!({"layer":1})).unwrap(); let written: Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(written, expected["maps"][0]["layers"][0]); + let mut expected_layer = expected["maps"][0]["layers"][0].clone(); + expected_layer["format"] = json!(crate::project::FORMAT); + assert_eq!(written, expected_layer); assert_eq!(s.notices_since(mark).len(), usize::from(private), "layer export has one aggregate notice or none"); } assert_eq!(serde_json::to_value(&s.project).unwrap(), before, "{role}: all output rolls back live fields"); diff --git a/crates/engine/src/tests/scopes_and_redaction.rs b/crates/engine/src/tests/scopes_and_redaction.rs index 8288524aa9..ce9d9ea921 100644 --- a/crates/engine/src/tests/scopes_and_redaction.rs +++ b/crates/engine/src/tests/scopes_and_redaction.rs @@ -736,6 +736,65 @@ fn outputs_redact_urls_capabilities_and_local_paths_across_scopes() { rustgis_plugins::unload("plug"); } +/// Legacy field recovery is reserved to direct native User dispatch. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn legacy_field_recovery_cannot_be_requested_by_restricted_ingress() { + use base64::Engine; + let mut s = world(); + let (_, bytes) = crate::cmd::field::package(&mut s, &json!({"name":"Site","extent":[-180,-85,180,85]})).unwrap(); + let entries = rustgis_data::zip::read(&bytes).unwrap(); + let (_, project) = entries.iter().find(|(name, _)| name.ends_with(".rgis")).unwrap(); + let mut project: Value = serde_json::from_slice(project).unwrap(); + project["format"] = json!(1); + project.as_object_mut().unwrap().remove("field_origin"); + project.as_object_mut().unwrap().remove("requires"); + let bytes = rustgis_data::zip::write(&[("Site.rgis".into(), serde_json::to_vec(&project).unwrap())]).unwrap(); + let params = json!({"data":base64::engine::general_purpose::STANDARD.encode(bytes),"legacyArea":"Site"}); + let (before, state, depth) = (serde_json::to_value(&s.project).unwrap(), s.state(), s.undo_depth()); + let error = + s.run_scoped(agent_scope(AgentAccess::Allow, "legacy-recovery"), &Call::command("offline.bringBack", params.clone())).unwrap_err(); + assert!(error.contains("native User"), "{error}"); + let (scope, _) = assistant_scope(7117); + assert!(s.run_scoped(scope, &Call::command("offline.bringBack", params.clone())).is_err()); + isolate_plugins(); + plugin("legacy-recovery-files", &["edit", "files"]); + let error = forward(&mut s, "legacy-recovery-files", "offline.bringBack", params.clone()).unwrap_err(); + assert!(error.contains("native User"), "{error}"); + rustgis_plugins::unload("legacy-recovery-files"); + let result = + s.narrowed(Narrow::Item { kind: "task", name: "legacy recovery".into() }, |s| s.run("offline.bringBack", ¶ms)).unwrap(); + assert!(result.is_err()); + assert_eq!(serde_json::to_value(&s.project).unwrap(), before); + assert_eq!((s.state(), s.undo_depth()), (state, depth)); +} + +/// Protocol document selectors are URLs in actual command results, including restricted ones. +#[test] +fn service_result_document_fields_are_addresses_and_keep_public_query_values() { + let params = json!({"url":"https://protocol-fields.example/catalog", "type":"ogcapi", + "text":json!({"collections":[{"id":"roads","title":"Roads","links":[ + {"rel":"tilesets-map","type":"application/json","href":"https://protocol-fields.example/tiles?f=json&style=light"} + ]}]}).to_string()}); + let mut s = world(); + let native = s.run("service.layers", ¶ms).unwrap(); + let restricted = s.run_scoped(agent_scope(AgentAccess::Allow, "protocol-fields"), &Call::command("service.layers", params)).unwrap(); + for result in [&native, &restricted] { + let address = result["layers"].as_array().unwrap().iter().find_map(|layer| layer["tilesets"].as_str()).unwrap(); + assert_eq!(address, "https://protocol-fields.example/tiles?f=json&style=light"); + } + let coverage = r#"soilSoil"#; + let result = s + .run_scoped( + agent_scope(AgentAccess::Allow, "protocol-fields-wcs"), + &Call::command("service.layers", json!({"url":"https://protocol-fields.example/wcs", "text":coverage})), + ) + .unwrap(); + let describe = result["layers"][0]["offer"]["describe"].as_str().unwrap(); + assert!(describe.starts_with("https://protocol-fields.example/wcs?"), "{describe}"); + assert!(describe.to_ascii_lowercase().contains("request=describecoverage"), "{describe}"); +} + #[test] fn an_agent_url_request_cannot_clear_a_document_restriction() { use crate::render::rustgis_tiles::{address::Reach, cache}; diff --git a/crates/engine/src/tests/service_logins.rs b/crates/engine/src/tests/service_logins.rs index a81ec8113e..2fc18ad1d6 100644 --- a/crates/engine/src/tests/service_logins.rs +++ b/crates/engine/src/tests/service_logins.rs @@ -2714,6 +2714,18 @@ fn a_key_in_the_address_is_left_out_and_its_parameter_offered_as_the_login() { ) .unwrap(); assert!(plain.get("stripped").is_none(), "{plain}"); + // A declared query key is a credential even when its name otherwise looks public. + let mark = s.notice_mark(); + let custom = s.run("connection.new", &json!({"kind":"wmts","name":"Declared query key","address":format!("{origin}/declared?zqdeclared757=RGSYNTHdeclared757&layer=a"),"auth":"query","keyName":"zqdeclared757"})).unwrap(); + assert_eq!(custom["address"], format!("{origin}/declared?layer=a")); + assert!(custom.get("omittedPublicParameterCount").is_none(), "{custom}"); + assert!(custom.get("omittedPublicParameters").is_none(), "{custom}"); + let edited_custom = s + .run("connection.edit", &json!({"id":custom["id"],"address":format!("{origin}/declared?zqdeclared757=RGSYNTHedited757&layer=a")})) + .unwrap(); + assert!(edited_custom.get("omittedPublicParameterCount").is_none(), "{edited_custom}"); + assert!(edited_custom.get("omittedPublicParameters").is_none(), "{edited_custom}"); + assert!(s.notices_since(mark).iter().all(|n| !n.text.contains("public query component"))); // An edit that moves a key into the address is the same. let edited = s.run("connection.edit", &json!({"id":clean["id"],"address":format!("{origin}/clean?token=edited-757")})).unwrap(); assert_eq!(edited["stripped"], true); @@ -2727,6 +2739,73 @@ fn a_key_in_the_address_is_left_out_and_its_parameter_offered_as_the_login() { assert!(bare.get("strippedParameters").is_none(), "{bare}"); let said = format!("{bare}{:?}", s.notices_since(mark)); assert!(!said.contains("RGSYNTHbare757"), "{said}"); + // Legacy credential-bearing ampersand pairs are dropped conservatively. Public options + // lost alongside a key are counted separately, never offered as credential names. + let legacy_mark = s.notice_mark(); + let legacy = s + .run( + "connection.new", + &json!({"kind":"wmts","name":"Legacy separators","address":format!("{origin}/legacy?x=1;token=RGSYNTHlegacy757")}), + ) + .unwrap(); + assert_eq!(legacy["stripped"], true); + assert_eq!(legacy["strippedParameters"], json!(["token"])); + assert_eq!(legacy["address"], format!("{origin}/legacy")); + assert_eq!(legacy["omittedPublicParameterCount"], 1); + assert_eq!(legacy["omittedPublicParameters"], json!(["x"])); + let collateral: Vec<_> = s.notices_since(legacy_mark).iter().filter(|n| n.text.contains("public query component")).collect(); + assert_eq!(collateral.len(), 1); + assert!(collateral[0].text.contains("Legacy separators") && collateral[0].text.contains("first names: x")); + assert!(!collateral[0].text.contains("RGSYNTHlegacy757")); + let repeated = s.run("connection.new", &json!({"kind":"wmts","name":"Repeated public option","address":format!("{origin}/legacy?x=1;token=RGSYNTHlegacyRepeated&x=1&style=light")})).unwrap(); + assert_eq!(repeated["address"], format!("{origin}/legacy?x=1&style=light")); + assert_eq!(repeated["omittedPublicParameterCount"], 1); + assert_eq!(repeated["omittedPublicParameters"], json!(["x"])); + + // Semicolon suffixes after a key may be its value, never public names or login hints. + for (i, suffix) in ["RGSYNTHopaque757=x", "tokenRGSYNTHopaque757=x"].iter().enumerate() { + let mark = s.notice_mark(); + let result = s + .run( + "connection.new", + &json!({"kind":"wmts","name":format!("Private suffix {i}"),"address":format!("{origin}/private?token=p;{suffix}")}), + ) + .unwrap(); + let shown = format!("{result}{:?}", s.notices_since(mark)); + assert!(!shown.contains("RGSYNTHopaque757"), "{shown}"); + assert!(result.get("omittedPublicParameters").is_none(), "{result}"); + assert_eq!(result["strippedParameters"], json!(["token"])); + } + let mark = s.notice_mark(); + let userinfo = s + .run( + "connection.new", + &json!({"kind":"wmts","name":"Private authority","address":"https://u:p?RGSYNTHopaque757=x@host.example/wmts"}), + ) + .unwrap(); + let shown = format!("{userinfo}{:?}", s.notices_since(mark)); + assert!(!shown.contains("RGSYNTHopaque757"), "{shown}"); + assert!(userinfo.get("omittedPublicParameters").is_none() && userinfo.get("strippedParameters").is_none(), "{userinfo}"); + + // A custom key name is a redaction hint even before a method is chosen; the saved definition + // must be as safe as its replay log. An explicit no-login method leaves ordinary data alone. + let no_method = s + .run("connection.new", &json!({"kind":"wmts","name":"Custom without method","address":format!("{origin}/custom?zqwithoutmethod757=RGSYNTHnomethod757&layer=a"),"keyName":"zqwithoutmethod757"})) + .unwrap(); + assert_eq!(no_method["stripped"], true); + assert_eq!(no_method["strippedParameters"], json!(["zqwithoutmethod757"])); + assert_eq!(no_method["address"], format!("{origin}/custom?layer=a")); + assert!(no_method.get("omittedPublicParameterCount").is_none()); + assert!(!s.connections.to_json().contains("RGSYNTHnomethod757")); + let public_hint = s.run("connection.new", &json!({"kind":"wmts","name":"Public name used as key hint","address":format!("{origin}/custom?layer=public-code&style=light"),"keyName":"layer"})).unwrap(); + assert_eq!(public_hint["address"], format!("{origin}/custom?style=light")); + assert_eq!(public_hint["strippedParameters"], json!(["layer"])); + assert!(public_hint.get("omittedPublicParameterCount").is_none()); + let explicit_none = s + .run("connection.new", &json!({"kind":"wmts","name":"Explicit no login","address":format!("{origin}/none?zqordinary757=public-code"),"keyName":"zqordinary757","auth":"none"})) + .unwrap(); + assert!(explicit_none.get("stripped").is_none()); + assert_eq!(explicit_none["address"], format!("{origin}/none?zqordinary757=public-code")); } /// A key in a query login is sent on the wire to the origin (and only as the parameter the diff --git a/crates/engine/tests/project_save.rs b/crates/engine/tests/project_save.rs index 88e8f3545a..f49445fd6c 100644 --- a/crates/engine/tests/project_save.rs +++ b/crates/engine/tests/project_save.rs @@ -170,7 +170,7 @@ fn file_compatibility_rejects_future_formats_before_importing() -> Result<(), St Ok(()) } -/// Existing writers keep bare payloads, while readers accept legacy and explicitly supported metadata. +/// Map/layer writers declare compatibility; readers also accept their legacy bare payloads. #[test] fn file_compatibility_keeps_current_and_legacy_companion_files() -> Result<(), String> { use rustgis_engine::{data::Dataset, data::GeometryType, map::Layer}; @@ -182,7 +182,11 @@ fn file_compatibility_keeps_current_and_legacy_companion_files() -> Result<(), S { let (_, bytes) = rustgis_engine::cmd::items::item_bytes(&mut original, kind, ¶ms)?; let value: Value = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; - assert!(value.get("format").is_none(), "current writer unchanged: {kind}"); + if kind == "styles" { + assert!(value.get("format").is_none(), "style arrays retain their legacy shape"); + } else { + assert_eq!(value["format"], rustgis_engine::project::FORMAT, "{kind}"); + } for declared in [false, true] { if declared && kind == "styles" { continue; // Style writers still emit the existing array, with no wrapper. @@ -190,6 +194,9 @@ fn file_compatibility_keeps_current_and_legacy_companion_files() -> Result<(), S let mut payload = value.clone(); if declared { payload["format"] = json!(rustgis_engine::project::FORMAT); + } else if let Some(object) = payload.as_object_mut() { + object.remove("format"); + object.remove("requires"); } let mut loaded = Session::new(Project::new("Loaded")); loaded.run(command, &json!({"text": payload.to_string()}))?; diff --git a/crates/render/src/terrain.rs b/crates/render/src/terrain.rs index c7d404a90c..976cd44a3b 100644 --- a/crates/render/src/terrain.rs +++ b/crates/render/src/terrain.rs @@ -449,9 +449,17 @@ fn raster_corners(r: &Raster) -> Option<([f64; 4], [f64; 4])> { Some((xs, ys)) } -/// A DEM's height at a Web Mercator point, read bilinearly from its first band; nodata and -/// points off it give `None`. +/// A DEM's first-band height at a Web Mercator point, before exaggeration or water. Uses +/// the same bounded bilinear sampling and full-turn wrapping as [`Ground`]; nodata, empty +/// grids and points off the DEM give `None`. +pub fn dem_height(raster: &Raster, at: Coord) -> Option { + raster_height(raster, at, crate::raster::full_turn_width(raster)) +} + fn raster_height(r: &Raster, m: Coord, wrap_width: Option) -> Option { + if r.width == 0 || r.height == 0 { + return None; + } let n = r.transformer().from_merc(m)?; let (px, py) = r.transform.to_pixel(n); let px = r.wrap_col(px); @@ -584,6 +592,29 @@ mod tests { Ground { heights: Heights::Raster { raster, wrap_width }, exaggeration: 1.0, body: 1.0, water: None } } + #[test] + fn dem_sampler_refuses_empty_grid_dimensions() { + let mut raster = Raster::new( + "DEM", + 2, + 1, + vec![vec![10.0, 20.0]], + GeoTransform { origin_x: 0.0, origin_y: 100.0, pixel_width: 100.0, pixel_height: -100.0 }, + Crs::WebMercator, + None, + ColorHint::Gray, + "Float32", + ) + .unwrap(); + let at = Coord::new(0.0, 50.0); + assert!(dem_height(&raster, at).is_some()); + raster.width = 0; + assert_eq!(dem_height(&raster, at), None); + raster.width = 2; + raster.height = 0; + assert_eq!(dem_height(&raster, Coord::new(50.0, 100.0)), None); + } + #[test] fn a_blocks_base_goes_below_the_nodata_drawn_at_zero() { // #1914: ground 5000 m high beside nodata (drawn at 0): the base is 1000 m below 0, not diff --git a/deny.toml b/deny.toml index a351c0ded6..78e882a36d 100644 --- a/deny.toml +++ b/deny.toml @@ -36,6 +36,5 @@ ignore = true unknown-registry = "deny" unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] -# terrio-crs until its next release on crates.io (Cargo.toml). -allow-git = ["https://github.com/opengeos/terrio"] +allow-git = [] required-git-spec = "rev" diff --git a/docs/maintaining.md b/docs/maintaining.md index 4385534450..b79b37352c 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -95,6 +95,14 @@ Each release adds a small project and preferences file written by that release a `released_projects_and_preferences_still_open`. Record a skip in the fixtures' README when a release has no such file format. Current-writer examples do not replace these historical files. +New project saves and map/layer exports write format 2. Their `requires` header names only +features present in the saved payload: `terrain-block`, `terrain-water`, +`shaded-relief-effects` (cast shadows, sky visibility, fill light or rock colour) and +`normalized-legend-urls` for normalized legend addresses. Project +requirements include offline map snapshots. Earlier format-2 readers refuse unknown required +capabilities instead of opening and later saving away those settings; legacy format-1 files +still open. + ## New dependencies A proposal for a dependency answers: diff --git a/docs/user-guide/field-work.md b/docs/user-guide/field-work.md index 3383e9b8ce..977d2edb53 100644 --- a/docs/user-guide/field-work.md +++ b/docs/user-guide/field-work.md @@ -159,6 +159,48 @@ Give one package to one phone at a time, and make a new package after each Bring phone's edits to the same package, brought back after the first's, would undo the first's where they differ. A package for an area the project doesn't have is refused. +### Origin identities and older field copies + +Download Map gives each new offline area and its home project stable identities. New field +packages preserve those identities, including after Save As and reopening. Bring Back checks +both before applying edits or writing photos. Packaging an older area can also add identities +to its home project: save that project so the return can be matched to it. Undo can remove the +newly created area and identities too. + +Saving a project with these identities declares the `field-origin` capability, even when the +offline area has never been sent as a field package. Earlier RustGIS releases that do not +support this capability refuse the saved project rather than discard its return association. + +A field copy made before these identities were introduced still opens, and Send Field Package +preserves its edits and photos as an unverified legacy return. Ordinary Bring Back refuses to +guess its home from names. After verifying which original home offline area it belongs to, a +user can explicitly recover it from the native command line: + +```sh +rustgis-cli run --project home.rgis \ + --cmd 'offline.bringBack={"path":"legacy-return.rgisfield","legacyArea":"Site"}' \ + --save recovered-home.rgis +``` + +`legacyArea` accepts entirely ID-less returns and direct User dispatch. Assistant, MCP, plugin +and imported-item calls cannot select this recovery path, and it cannot override present, +malformed or incomplete identities. The result reports the unverified origin; normal conflict, +photo and Undo handling still apply. Make a fresh field package for further edits and save its +home project. + +If the older field copy was saved as a plain `.rgis` project, send its existing edits and photos +first with the explicit legacy-return option: + +```sh +rustgis-cli run --project edited-field.rgis \ + --cmd 'offline.package={"path":"legacy-return.rgisfield","legacyReturn":true}' +``` + +Use the project and photo files on the device where those edits were saved, or copy its +referenced photos along with it. This preserves the old copy's data without inventing an origin +identity. A newly identified package whose home cannot be found requires the home project saved +after packaging; legacy recovery cannot change that association. + ## What differs on a phone - **Plugins** can't be installed on an iPhone or iPad: App Store rules don't allow an app to diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 9764ce6131..b47f7ec13f 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2387,6 +2387,7 @@ dependencies = [ "crs-definitions", "csv", "encoding_rs", + "flate2", "flatgeobuf", "geo", "geozero", @@ -3059,8 +3060,9 @@ dependencies = [ [[package]] name = "terrio-crs" -version = "0.2.3" -source = "git+https://github.com/opengeos/terrio?rev=02068f6fef3c4326199e878687712c41e64552f1#02068f6fef3c4326199e878687712c41e64552f1" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75b204c693afa267f00b04e9bf7a4158a6132cd0d0213bfe0e1317aa59d6b33" dependencies = [ "geographiclib-rs", "libc", diff --git a/schema/fixtures/field_origin.rgis b/schema/fixtures/field_origin.rgis new file mode 100644 index 0000000000..49c42098ae --- /dev/null +++ b/schema/fixtures/field_origin.rgis @@ -0,0 +1,98 @@ +{ + "format": 2, + "name": "Field identity", + "maps": [ + { + "name": "Map", + "layers": [], + "basemap": "Paper", + "camera": { + "center": { + "x": 0.0, + "y": 2000000.0 + }, + "res": 30000.0, + "bearing": 0.0, + "pitch": 0.0 + }, + "bookmarks": [], + "graticule": true, + "projection": "WebMercator", + "popup_mode": "Topmost", + "label_display": { + "paused": false, + "locked": false, + "show_unplaced": false + }, + "editing": { + "relationships": [], + "snapping": { + "enabled": false, + "tolerance_px": 10.0, + "vertex": true, + "edge": true, + "end": true + }, + "topology": [], + "settings": { + "enabled": true, + "show_vertices": true, + "elevation": "None", + "ground_to_grid": null + } + }, + "swipe": null, + "view": { + "effects": { + "stars": true, + "sun_time": null, + "weather": "None", + "vignette": false, + "edges": false, + "tilt_shift": false, + "grain": false, + "brightness": 0.0, + "contrast": 0.0, + "saturation": 0.0, + "sepia": 0.0 + }, + "wireframe": false, + "clip_distance": null + }, + "animation": [], + "next_layer_id": 1 + } + ], + "active_map": 0, + "tables": [], + "next_table_id": 1, + "gp_history": [], + "layouts": [], + "folders": [], + "styles": [], + "ramps": [], + "tasks": [], + "gp_env": { + "extent": null, + "cell_size": null + }, + "models": [], + "workspaces": [], + "link_views": false, + "jobs": [], + "offline_areas": [], + "reports": [], + "presentations": [], + "notebooks": [], + "dl_models": [], + "shared": [], + "data_stores": [], + "requires": [ + "field-origin" + ], + "origin_id": "0123456789abcdef0123456789abcdef", + "field_origin": { + "project": "fedcba9876543210fedcba9876543210", + "area": "00112233445566778899aabbccddeeff" + } +} diff --git a/schema/fixtures/normalized_legend.rgis b/schema/fixtures/normalized_legend.rgis new file mode 100644 index 0000000000..1277a17a5c --- /dev/null +++ b/schema/fixtures/normalized_legend.rgis @@ -0,0 +1,105 @@ +{ + "format": 2, + "name": "Encoded legend", + "maps": [ + { + "name": "Map", + "layers": [], + "basemap": "Paper", + "camera": { + "center": { + "x": 0.0, + "y": 2000000.0 + }, + "res": 30000.0, + "bearing": 0.0, + "pitch": 0.0 + }, + "bookmarks": [], + "graticule": true, + "projection": "WebMercator", + "popup_mode": "Topmost", + "label_display": { + "paused": false, + "locked": false, + "show_unplaced": false + }, + "editing": { + "relationships": [], + "snapping": { + "enabled": false, + "tolerance_px": 10.0, + "vertex": true, + "edge": true, + "end": true + }, + "topology": [], + "settings": { + "enabled": true, + "show_vertices": true, + "elevation": "None", + "ground_to_grid": null + } + }, + "swipe": null, + "view": { + "effects": { + "stars": true, + "sun_time": null, + "weather": "None", + "vignette": false, + "edges": false, + "tilt_shift": false, + "grain": false, + "brightness": 0.0, + "contrast": 0.0, + "saturation": 0.0, + "sepia": 0.0 + }, + "wireframe": false, + "clip_distance": null + }, + "animation": [], + "next_layer_id": 1, + "tiles": [ + { + "name": "Legend", + "kind": "raster", + "url": "https://tiles.example/{z}/{x}/{y}.png", + "legends": [ + { + "url": "https://legend.example/image?label=%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20" + } + ] + } + ] + } + ], + "active_map": 0, + "tables": [], + "next_table_id": 1, + "gp_history": [], + "layouts": [], + "folders": [], + "styles": [], + "ramps": [], + "tasks": [], + "gp_env": { + "extent": null, + "cell_size": null + }, + "models": [], + "workspaces": [], + "link_views": false, + "jobs": [], + "offline_areas": [], + "reports": [], + "presentations": [], + "notebooks": [], + "dl_models": [], + "shared": [], + "data_stores": [], + "requires": [ + "normalized-legend-urls" + ] +} diff --git a/schema/fixtures/terrain_water_v2.rgis b/schema/fixtures/terrain_water_v2.rgis new file mode 100644 index 0000000000..d01971880f --- /dev/null +++ b/schema/fixtures/terrain_water_v2.rgis @@ -0,0 +1 @@ +{"format":2,"name":"Terrain water","maps":[{"name":"Map","layers":[],"basemap":"Paper","camera":{"center":{"x":0.0,"y":2000000.0},"res":30000.0,"bearing":0.0,"pitch":0.0},"bookmarks":[],"graticule":true,"projection":"WebMercator","popup_mode":"Topmost","label_display":{"paused":false,"locked":false,"show_unplaced":false},"editing":{"relationships":[],"snapping":{"enabled":false,"tolerance_px":10.0,"vertex":true,"edge":true,"end":true},"topology":[],"settings":{"enabled":true,"show_vertices":true,"elevation":"None","ground_to_grid":null}},"swipe":null,"view":{"effects":{"stars":true,"sun_time":null,"weather":"None","vignette":false,"edges":false,"tilt_shift":false,"grain":false,"brightness":0.0,"contrast":0.0,"saturation":0.0,"sepia":0.0},"wireframe":false,"clip_distance":null,"terrain":{"source":{"Layer":"dem"},"exaggeration":2.0,"water":{"layer":"lagos","level":"nivel"}}},"animation":[],"next_layer_id":1}],"active_map":0,"tables":[],"next_table_id":1,"gp_history":[],"layouts":[],"folders":[],"styles":[],"ramps":[],"tasks":[],"gp_env":{"extent":null,"cell_size":null},"models":[],"workspaces":[],"link_views":false,"jobs":[],"offline_areas":[],"reports":[],"presentations":[],"notebooks":[],"dl_models":[],"shared":[],"data_stores":[],"requires":["terrain-water"]} diff --git a/schema/project.schema.json b/schema/project.schema.json index ec06d355ec..d3fb967e05 100644 --- a/schema/project.schema.json +++ b/schema/project.schema.json @@ -162,6 +162,22 @@ ], "type": "object" }, + "FieldOrigin": { + "description": "A field copy's exact home and offline-area identities (128 random bits each).", + "properties": { + "area": { + "type": "string" + }, + "project": { + "type": "string" + } + }, + "required": [ + "project", + "area" + ], + "type": "object" + }, "FieldType": { "oneOf": [ { @@ -602,6 +618,10 @@ "extent": { "$ref": "#/$defs/Extent" }, + "id": { + "description": "Persistent identity, independent of the editable display name.", + "type": "string" + }, "map": { "description": "The map it came from (by name).", "type": "string" @@ -875,7 +895,7 @@ }, "$id": "https://raw.githubusercontent.com/opengeos/RustGIS/main/schema/project.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "A RustGIS project file, as RustGIS reads it. Generated from the Rust types by `cargo xtask schema`; don't edit it by hand. It writes format 1 and reads formats up to 2.", + "description": "A RustGIS project file, as RustGIS reads it. Generated from the Rust types by `cargo xtask schema`; don't edit it by hand. It writes format 2 and reads formats up to 2.", "properties": { "active_map": { "format": "uint", @@ -898,6 +918,17 @@ }, "type": "array" }, + "field_origin": { + "anyOf": [ + { + "$ref": "#/$defs/FieldOrigin" + }, + { + "type": "null" + } + ], + "description": "The home and area this field copy belongs to, retained through ordinary saves." + }, "folders": { "default": [], "description": "Folder connections shown in the Browser.", @@ -907,7 +938,7 @@ "type": "array" }, "format": { - "default": 1, + "default": 2, "description": "The format read from disk; serialization always declares the current writer's format.", "format": "uint32", "maximum": 2, @@ -990,6 +1021,10 @@ }, "type": "array" }, + "origin_id": { + "description": "Persistent home-project identity for offline returns; legacy projects acquire it when packaged.", + "type": "string" + }, "presentations": { "default": [], "description": "Presentations: pages of maps, text, images and videos.", @@ -1017,7 +1052,13 @@ "requires": { "description": "The capabilities this file needs past its format, each one this RustGIS knows.", "items": { - "enum": [], + "enum": [ + "terrain-block", + "terrain-water", + "shaded-relief-effects", + "normalized-legend-urls", + "field-origin" + ], "type": "string" }, "maxItems": 256, @@ -1080,7 +1121,13 @@ ], "title": "RustGIS project (.rgis)", "type": "object", - "x-rustgis-capabilities": [], + "x-rustgis-capabilities": [ + "terrain-block", + "terrain-water", + "shaded-relief-effects", + "normalized-legend-urls", + "field-origin" + ], "x-rustgis-reads-format": 2, - "x-rustgis-writes-format": 1 + "x-rustgis-writes-format": 2 }