Repository navigation
feat(annotation): add image-aware pane and original-pixel drawing #2290
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
e0a61c2
4f88e4c
80b2d27
7433506
4245edd
aa5512b
8e38c3d
c9f0889
a0d6b23
7ba682b
d2ebb4a
8d155ca
62c02a1
0eb681e
8712d19
58b4279
84500c8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| //! Runtime identity of the local GeoTIFF used to decode a displayed raster. | ||
| //! Metadata comparison, not a content hash: rewrites preserving every compared field are not detected. | ||
| //! Windows compares length, modification and creation times; Unix also compares device, inode and ctime. | ||
| use std::{ | ||
| fs::Metadata, | ||
| path::{Path, PathBuf}, | ||
| }; | ||
|
|
||
| #[derive(Clone, Debug, PartialEq, Eq)] | ||
| pub(super) struct FileRevision { | ||
| path: PathBuf, | ||
| len: u64, | ||
| modified: std::time::SystemTime, | ||
| #[cfg(unix)] | ||
| identity: (u64, u64, i64, i64), | ||
| #[cfg(windows)] | ||
| created: u64, | ||
| } | ||
|
|
||
| impl FileRevision { | ||
| pub(super) fn capture(path: &Path, metadata: &Metadata) -> std::io::Result<Self> { | ||
| #[cfg(unix)] | ||
| use std::os::unix::fs::MetadataExt; | ||
| #[cfg(windows)] | ||
| use std::os::windows::fs::MetadataExt; | ||
| Ok(Self { | ||
| path: std::path::absolute(path)?, | ||
| len: metadata.len(), | ||
| modified: metadata.modified()?, | ||
| #[cfg(unix)] | ||
| identity: (metadata.dev(), metadata.ino(), metadata.ctime(), metadata.ctime_nsec()), | ||
| #[cfg(windows)] | ||
| created: metadata.creation_time(), | ||
| }) | ||
| } | ||
|
|
||
| fn check_metadata(&self, path: &Path, metadata: &Metadata) -> std::result::Result<(), String> { | ||
| let current = Self::capture(path, metadata).map_err(|e| format!("Could not verify the current image file metadata: {e}"))?; | ||
| if *self == current { | ||
| Ok(()) | ||
| } else { | ||
| Err("The image file changed since its pixels were loaded. Use Reload Data, then retry the operation.".into()) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl super::Raster { | ||
| fn original_revision(&self) -> std::result::Result<&FileRevision, String> { | ||
| match &self.source_file { | ||
| Some(Ok(original)) => Ok(original), | ||
| Some(Err(error)) => Err(format!( | ||
| "The original image file revision could not be captured ({error}). Check filesystem metadata support/access, or export the image as a GeoTIFF to another location and add it." | ||
| )), | ||
| None => Err("The displayed image has no verified original file revision. Use Reload Data, then retry the operation.".into()), | ||
| } | ||
| } | ||
|
|
||
| /// Checks fresh metadata at the live path on the reader's worker, never per frame. | ||
| pub fn check_original_file(&self, path: &Path) -> std::result::Result<(), String> { | ||
| let original = self.original_revision()?; | ||
| let metadata = std::fs::metadata(path).map_err(|e| format!("Could not read the current image file metadata: {e}"))?; | ||
| original.check_metadata(path, &metadata) | ||
| } | ||
|
|
||
| /// Checks fresh metadata from this open handle against the original source at `path`. | ||
| /// This does not verify that the live path still names the handle; check_original_file does. | ||
| pub fn check_original_handle(&self, path: &Path, file: &std::fs::File) -> std::result::Result<(), String> { | ||
| let original = self.original_revision()?; | ||
| let metadata = file.metadata().map_err(|e| format!("Could not read the open image file metadata: {e}"))?; | ||
| original.check_metadata(path, &metadata) | ||
| } | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| #[test] | ||
| fn annotation_grid_revision_detects_replacement_with_the_same_length_and_modification_time() { | ||
| let folder = tempfile::tempdir().unwrap(); | ||
| let path = folder.path().join("image.tif"); | ||
| let bytes = super::super::geotiff::write(&super::super::tests::small(), 0).unwrap(); | ||
| std::fs::write(&path, &bytes).unwrap(); | ||
| let raster = super::super::geotiff::read_file(&path, "image").unwrap(); | ||
| let original = std::fs::File::open(&path).unwrap(); | ||
| let metadata = original.metadata().unwrap(); | ||
| raster.check_original_file(&path).unwrap(); | ||
| raster.check_original_handle(&path, &original).unwrap(); | ||
| let replacement = folder.path().join("replacement.tif"); | ||
| std::fs::write(&replacement, &bytes).unwrap(); | ||
| let file = std::fs::File::options().write(true).open(&replacement).unwrap(); | ||
| file.set_times(std::fs::FileTimes::new().set_modified(metadata.modified().unwrap())).unwrap(); | ||
| drop(file); | ||
| std::fs::remove_file(&path).unwrap(); | ||
| std::fs::rename(&replacement, &path).unwrap(); | ||
| let current = std::fs::metadata(&path).unwrap(); | ||
| assert_eq!(metadata.len(), current.len()); | ||
| assert_eq!(metadata.modified().unwrap(), current.modified().unwrap()); | ||
| // Retaining old metadata cannot make the live-path API accept the replacement. | ||
| assert!(raster.check_original_file(&path).unwrap_err().contains("Reload Data")); | ||
| let replacement_handle = std::fs::File::open(&path).unwrap(); | ||
| assert!(raster.check_original_handle(&path, &replacement_handle).is_err()); | ||
| } | ||
|
|
||
| #[test] | ||
| fn annotation_grid_handle_validation_rejects_a_different_file() { | ||
| let folder = tempfile::tempdir().unwrap(); | ||
| let path = folder.path().join("image.tif"); | ||
| let other = folder.path().join("other.tif"); | ||
| let bytes = super::super::geotiff::write(&super::super::tests::small(), 0).unwrap(); | ||
| std::fs::write(&path, &bytes).unwrap(); | ||
| let mut other_bytes = bytes.clone(); | ||
| other_bytes.push(0); // Distinct metadata even on filesystems with coarse timestamps. | ||
| std::fs::write(&other, &other_bytes).unwrap(); | ||
| let raster = super::super::geotiff::read_file(&path, "image").unwrap(); | ||
| assert!(raster.check_original_handle(&path, &std::fs::File::open(&other).unwrap()).is_err()); | ||
| raster.check_original_file(&path).unwrap(); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1328,9 +1328,52 @@ pub fn read_file(path: &std::path::Path, name: &str) -> Result<Raster> { | |
| /// [`read_file`], with the no-data value a sidecar gives (see [`read_limited_with_nodata`]). | ||
| #[cfg(not(target_arch = "wasm32"))] | ||
| pub(super) fn read_file_with_nodata(path: &std::path::Path, name: &str, sidecar_nodata: Option<f64>) -> Result<Raster> { | ||
| let path = path.to_path_buf(); | ||
| let open = move || -> std::io::Result<Reader<'static>> { reader(std::io::BufReader::new(crate::provenance::open(&path)?)) }; | ||
| read_limited_with_nodata(open().map_err(err)?, name, max_samples(), Some(&open), sidecar_nodata) | ||
| let file = crate::provenance::open(path).map_err(err)?; | ||
| let revision = file.metadata().and_then(|metadata| super::file_revision::FileRevision::capture(path, &metadata)); | ||
| read_file_with_revision(path, name, sidecar_nodata, file, revision) | ||
| } | ||
|
|
||
| #[cfg(not(target_arch = "wasm32"))] | ||
| fn load_revision_error(reason: impl std::fmt::Display) -> String { | ||
| format!( | ||
| "{reason}. Wait until the file is stable, or copy it to a stable local location. Retry Add Data; for an existing layer, retry Reload Data." | ||
| ) | ||
| } | ||
|
|
||
| #[cfg(not(target_arch = "wasm32"))] | ||
| fn read_file_with_revision( | ||
| path: &std::path::Path, | ||
| name: &str, | ||
| sidecar_nodata: Option<f64>, | ||
| file: std::fs::File, | ||
| revision: std::io::Result<super::file_revision::FileRevision>, | ||
| ) -> Result<Raster> { | ||
| let revision = revision.map_err(|e| e.to_string()); | ||
| let open = || -> std::io::Result<Reader<'static>> { | ||
| let file = crate::provenance::open(path)?; | ||
| if let Ok(original) = &revision { | ||
| let current = file | ||
| .metadata() | ||
| .and_then(|metadata| super::file_revision::FileRevision::capture(path, &metadata)) | ||
| .map_err(|e| std::io::Error::other(load_revision_error(format!("Could not verify the image file while reading: {e}"))))?; | ||
| if *original != current { | ||
| return Err(std::io::Error::other(load_revision_error("The image file changed while its pixels were read"))); | ||
| } | ||
| } | ||
| reader(std::io::BufReader::new(file)) | ||
| }; | ||
| let mut raster = | ||
| read_limited_with_nodata(reader(std::io::BufReader::new(file)).map_err(err)?, name, max_samples(), Some(&open), sidecar_nodata)?; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Check the file revision before returning a decode error. If the source is truncated after the decoder reads its header, a strip read can fail before the final revision check. Add Data and Reload Data then report the decode error without the documented instructions to wait for a stable file and retry. Keep the decode result until after revision validation. If cancellation caused the failure, preserve it. If the revision still matches, return the original decode error. If the revision changed or can no longer be read, return the recovery message.
|
||
| if let Ok(original) = &revision { | ||
| let current = std::fs::metadata(path) | ||
| .and_then(|metadata| super::file_revision::FileRevision::capture(path, &metadata)) | ||
| .map_err(|e| err(load_revision_error(format!("Could not verify the image file after reading: {e}"))))?; | ||
| if *original != current { | ||
| return Err(err(load_revision_error("The image file changed while its pixels were read"))); | ||
| } | ||
| } | ||
| raster.source_file = Some(revision); | ||
| Ok(raster) | ||
| } | ||
|
|
||
| /// [`read`], holding at most `max` samples (the tests' small limit exercises the overviews), with | ||
|
|
@@ -2029,6 +2072,77 @@ mod tests { | |
| use super::*; | ||
| use crate::raster::tests::small; | ||
|
|
||
| #[cfg(not(target_arch = "wasm32"))] | ||
| #[test] | ||
| fn annotation_grid_geotiff_open_failure_retains_format_context() { | ||
| let dir = tempfile::tempdir().unwrap(); | ||
| let error = read_file(&dir.path().join("missing.tif"), "missing").unwrap_err().to_string(); | ||
| assert!(error.contains("GeoTIFF:"), "{error}"); | ||
| } | ||
|
|
||
| #[cfg(not(target_arch = "wasm32"))] | ||
| #[test] | ||
| fn annotation_grid_load_change_and_missing_path_refuse_with_add_data_recovery() { | ||
| use std::io::Write; | ||
| for removed in [false, true] { | ||
| let dir = tempfile::tempdir().unwrap(); | ||
| let path = dir.path().join("image.tif"); | ||
| let pixels = small(); | ||
| let bytes = write(&pixels, 0).unwrap(); | ||
| std::fs::write(&path, &bytes).unwrap(); | ||
| let file = std::fs::File::open(&path).unwrap(); | ||
| let revision = super::super::file_revision::FileRevision::capture(&path, &file.metadata().unwrap()).unwrap(); | ||
| // Inject a change between the initial source snapshot and the final load check, | ||
| // without timers or filesystem races. The already-open TIFF remains decodable. | ||
| if removed { | ||
| std::fs::remove_file(&path).unwrap(); | ||
| } else { | ||
| std::fs::OpenOptions::new().append(true).open(&path).unwrap().write_all(&[0]).unwrap(); | ||
| } | ||
| let error = read_file_with_revision(&path, "image", None, file, Ok(revision)).unwrap_err().to_string(); | ||
| assert!(error.contains("GeoTIFF:") && error.contains("Retry Add Data"), "{error}"); | ||
| assert!(error.contains("stable local location") && error.contains("for an existing layer, retry Reload Data"), "{error}"); | ||
| assert!(!error.contains("Use Reload Data, then retry the operation"), "{error}"); | ||
| assert!(error.contains(if removed { "Could not verify" } else { "file changed" }), "{error}"); | ||
| // Once stable, an ordinary Add Data read succeeds and has a verified grid revision. | ||
| std::fs::write(&path, &bytes).unwrap(); | ||
| let loaded = read_file(&path, "image").unwrap(); | ||
| assert_eq!(loaded.bands[0].data, pixels.bands[0].data); | ||
| loaded.check_original_file(&path).unwrap(); | ||
| } | ||
| } | ||
|
|
||
| #[cfg(not(target_arch = "wasm32"))] | ||
| #[test] | ||
| fn annotation_grid_capture_failure_preserves_pixels_and_reports_the_original_cause() { | ||
| let dir = tempfile::tempdir().unwrap(); | ||
| let path = dir.path().join("image.tif"); | ||
| let pixels = small(); | ||
| std::fs::write(&path, write(&pixels, 0).unwrap()).unwrap(); | ||
| let file = std::fs::File::open(&path).unwrap(); | ||
| let raster = read_file_with_revision( | ||
| &path, | ||
| "image", | ||
| None, | ||
| file, | ||
| Err(std::io::Error::new(std::io::ErrorKind::Unsupported, "modification timestamps unavailable")), | ||
| ) | ||
| .unwrap(); | ||
| assert_eq!((raster.width, raster.height), (pixels.width, pixels.height)); | ||
| assert_eq!(raster.bands[0].data, pixels.bands[0].data); | ||
| let error = raster.check_original_file(&path).unwrap_err(); | ||
| assert!(error.contains("modification timestamps unavailable"), "{error}"); | ||
| assert!(error.contains("metadata support/access") && error.contains("another location"), "{error}"); | ||
| assert!(!error.contains("Reload Data"), "{error}"); | ||
| assert!( | ||
| raster | ||
| .clone() | ||
| .check_original_handle(&path, &std::fs::File::open(&path).unwrap()) | ||
| .unwrap_err() | ||
| .contains("modification timestamps unavailable") | ||
| ); | ||
| } | ||
|
|
||
| #[cfg(not(target_arch = "wasm32"))] | ||
| #[test] | ||
| fn chunked_reads_count_progress_and_stop_before_or_during_serial_and_parallel_work() { | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The revision compares a path resolved against the working directory (low confidence, about 25%).
capturestoresstd::path::absolute(path)?inpath, and that field takes part in the derivedPartialEq. The live-path check (check_original_file) rebuilds the revision with the cwd at check time. If the process working directory changes between load and grid read (for example a Pythonos.chdir()with a relative layer path), the equality fails and the user sees "The image file changed since its pixels were loaded" even though the file is untouched. The only recovery offered is Reload Data.The stored path doesn't need to be part of the comparison: the callers already pass the path explicitly. Dropping the
pathfield (and theabsolutecall, which also adds a failure source tocapture) would make the check depend only on length, mtime and the platform identity, which is what the module doc describes.