From 3b06ebb758593f64ff309d1b1bea8dbe808d29ad Mon Sep 17 00:00:00 2001 From: Adam Harris Date: Tue, 27 Oct 2026 23:41:56 -0600 Subject: [PATCH] feat(manifest): a manifest can say its files are held elsewhere held: {by, at} names the machine that keeps a manifest's files, and where to ask it for one. Where the covered directory is present nothing changes. Where it is not, the rows are held elsewhere rather than missing: check reports nothing, manifest says who holds them, --verify verifies nothing and exits non-zero, and --update and --add refuse rather than empty the record from a directory this device does not have. Without held, an absent root is a broken link as before. Workspace::set_manifest_held and manifest --held-by/--held-at/--not-held set it, rewriting the manifest and re-pinning the node in one change set. ManifestStatus gains held and elsewhere. For a photo archive on a NAS whose workspace travels to devices that will never carry 1.66 TB. Also puts manifests.md section 3's closing paragraphs back above 3.1, where they belong. --- docs/manifests.md | 54 ++++++++++++---- prov-cli/src/check.rs | 41 ++++++++++++ prov-cli/src/cli.rs | 11 ++++ prov-cli/src/main.rs | 12 +++- prov-graph/src/manifest.rs | 121 ++++++++++++++++++++++++++++++++++ prov/src/lib.rs | 1 + prov/src/manifest.rs | 129 ++++++++++++++++++++++++++++++++++++- prov/src/mutate/gather.rs | 1 + prov/src/validate.rs | 13 ++-- 9 files changed, 362 insertions(+), 21 deletions(-) diff --git a/docs/manifests.md b/docs/manifests.md index 40ee388..034a06f 100644 --- a/docs/manifests.md +++ b/docs/manifests.md @@ -79,6 +79,7 @@ spec §5's MUST, which a list of rows prov re-lays-out plainly falls under. | `files` | yes | The rows. Absent or null means an empty directory. | | `verbatim` | no | `true` claims every file under `root`, documents included (§3.1). Absent means `false`; any value but a boolean is refused. | | `exclude` | no | Patterns for files under `root` the manifest does not claim (§3.2). A list of strings; a pattern outside the syntax is refused. | +| `held` | no | Who keeps the files when not every device has them (§3.5): `by`, a name, required; `at`, an address, optional. Anything else is refused. | Each row: @@ -123,6 +124,19 @@ directory then rewrites one line instead of ten thousand. - A nested manifest's directory belongs to its own node, so two manifests can never claim one file. +Completeness is what makes the interesting question answerable. A covered file is +not a document, so §8's orphan walk cannot see it, and it is not a link, so the +census cannot either. Without a claim over the whole directory, a photograph +could vanish and no prov surface would ever say so. + +**Hashes are optional, per row.** A manifest with no hashes is an *inventory* — +what is supposed to be here. One with hashes is that plus a fixity baseline. +Hashing reads every file, at mint and at every refresh, which is a real cost over +an archive and a reasonable one to decline for a directory of scans while still +wanting checksums everywhere else (`attach --manifest --no-hash`). A refresh +**preserves the mode**: an inventory that quietly began recording digests would +be claiming a guarantee nobody asked for. + ### 3.1 Verbatim: claiming the documents too Some directories are one artifact, not an archive: a built web app, whose @@ -214,18 +228,35 @@ the same either way, and a backend that cannot read in parts reads the file whole, as before. This holds for every checksum prov records over a payload: manifest rows, attachment sidecars, inline rows, and `check`. -Completeness is what makes the interesting question answerable. A covered file is -not a document, so §8's orphan walk cannot see it, and it is not a link, so the -census cannot either. Without a claim over the whole directory, a photograph -could vanish and no prov surface would ever say so. +### 3.5 Held: files kept on another machine -**Hashes are optional, per row.** A manifest with no hashes is an *inventory* — -what is supposed to be here. One with hashes is that plus a fixity baseline. -Hashing reads every file, at mint and at every refresh, which is a real cost over -an archive and a reasonable one to decline for a directory of scans while still -wanting checksums everywhere else (`attach --manifest --no-hash`). A refresh -**preserves the mode**: an inventory that quietly began recording digests would -be claiming a guarantee nobody asked for. +An archive too large for every device stays on the machine that keeps it, +while its workspace — the description and the manifest — travels. On a +device without the files, every row would read as missing and the root as a +broken link. `held` says that is expected: + +```yaml +root: photo/ +held: + by: Harris NAS # the keeper, named for a person + at: https://nas.example:8765/ # where to ask it for a file; optional +``` + +- **Where `root` is a directory, nothing changes.** That is the keeper, or a + full copy, and the manifest is checked, refreshed and verified exactly as + an unheld one. +- **Where it is not, the rows are held elsewhere.** `check` says nothing + about them; `manifest` reports *held by Harris NAS; not on this device*; + `--verify` verifies nothing and says so, exiting non-zero; and `--update` + and `--add` refuse, because rebuilding the rows from a directory that is + not there would empty the record. +- **Without `held`, an absent root is a broken link, as ever.** A manifest + must say its files are elsewhere before their absence stops counting. + +Neither `by` nor `at` is a secret: who may ask the keeper is the keeper's to +decide. `manifest TARGET --held-by NAME [--held-at ADDRESS]` sets it, and +`--not-held` clears it; either rewrites the manifest and re-pins the node. +Set it where the files are, before the manifest travels. ## 4. The chain @@ -383,6 +414,7 @@ Each verb is one change set: moved, written and removed together. | `manifest TARGET --update` | rebuild the rows from the directory as it is now, keeping each row's fields, re-stamping the node | | `manifest TARGET --add FILE...` | append rows for the named files only, checksumming just those; every other row stays, its file present or not (§5) | | `manifest TARGET --verify` | re-read every listed file and compare its checksum | +| `manifest TARGET --held-by NAME [--held-at ADDRESS]` | say who keeps the files, so a device without them reads them as held elsewhere (§3.5); `--not-held` clears it | | `gather ATTACHMENT... --into DIR [--title T] [--discard FIELD] [--dry-run]` | attachments of one index into one manifest over `DIR` (§7) | | `scatter TARGET [--into INDEX] [--discard FIELD] [--dry-run]` | a manifest back into one attachment per file (§7) | diff --git a/prov-cli/src/check.rs b/prov-cli/src/check.rs index c70a52e..1a8219d 100644 --- a/prov-cli/src/check.rs +++ b/prov-cli/src/check.rs @@ -359,11 +359,30 @@ pub(crate) fn cmd_manifest( update: bool, verify: bool, add: &[PathBuf], + held: Option>, ) -> CmdResult { let mut session = Session::open()?; let target_rel = ws_rel(&session.ctx, target)?; let node = resolve_manifest_node(&session.ws, &target_rel)?; + if let Some(held) = held { + let said = match &held { + Some(held) => format!("held by {}", held.by), + None => "not held".to_string(), + }; + let wrote = block_on(session.ws.set_manifest_held(&node, held))?; + session.commit()?; + let status = block_on(session.ws.manifest_status(&node))? + .ok_or_else(|| format!("{} declares no manifest", node.display()))?; + if wrote { + eprintln!("{}: now {said}", status.manifest.display()); + } else { + eprintln!("{}: already {said}", status.manifest.display()); + } + println!("{}", status.manifest.display()); + return Ok(ExitCode::SUCCESS); + } + if !add.is_empty() { let paths = add .iter() @@ -401,6 +420,16 @@ pub(crate) fn cmd_manifest( } if verify { + let status = block_on(session.ws.manifest_status(&node))? + .ok_or_else(|| format!("{} declares no manifest", node.display()))?; + if let (true, Some(held)) = (status.elsewhere, &status.held) { + eprintln!( + "{}: nothing verified — its files are held by {} and are not on this device", + status.manifest.display(), + held.by + ); + return Ok(ExitCode::FAILURE); + } let findings = block_on(session.ws.verify_manifest(&node))?; for finding in &findings { println!("{finding}"); @@ -427,6 +456,18 @@ pub(crate) fn cmd_manifest( ", no checksums (an inventory)" } ); + if let Some(held) = &status.held { + let at = held + .at + .as_deref() + .map(|at| format!(", at {at}")) + .unwrap_or_default(); + if status.elsewhere { + eprintln!(" held by {}{at}; not on this device", held.by); + } else { + eprintln!(" held by {}{at}", held.by); + } + } for path in &status.missing { eprintln!(" missing: {}", path.display()); } diff --git a/prov-cli/src/cli.rs b/prov-cli/src/cli.rs index ffeea6b..4f60025 100644 --- a/prov-cli/src/cli.rs +++ b/prov-cli/src/cli.rs @@ -648,6 +648,17 @@ pub(crate) enum Command { /// a document, or one already listed. #[arg(long, value_name = "FILE", num_args = 1..)] add: Vec, + /// Say that NAME keeps the files, so a device that carries the + /// manifest without them reads them as held elsewhere rather than + /// missing. Set it where the files are, before the manifest travels. + #[arg(long, value_name = "NAME", conflicts_with_all = ["update", "verify", "add", "not_held"])] + held_by: Option, + /// Where a device asks the keeper for a file. Not a secret. + #[arg(long, value_name = "ADDRESS", requires = "held_by")] + held_at: Option, + /// Say that every device carrying the manifest has its files again. + #[arg(long, conflicts_with_all = ["update", "verify", "add"])] + not_held: bool, }, /// Gather attachments into one manifest: their files move into a new /// directory, their sidecars go, and one node takes the first one's place diff --git a/prov-cli/src/main.rs b/prov-cli/src/main.rs index c54d51a..516ff97 100644 --- a/prov-cli/src/main.rs +++ b/prov-cli/src/main.rs @@ -170,7 +170,17 @@ fn main() -> ExitCode { update, verify, add, - } => check::cmd_manifest(&target, update, verify, &add), + held_by, + held_at, + not_held, + } => { + let held = match held_by { + Some(by) => Some(Some(prov::Held { by, at: held_at })), + None if not_held => Some(None), + None => None, + }; + check::cmd_manifest(&target, update, verify, &add, held) + } Command::Gather { cards, into, diff --git a/prov-graph/src/manifest.rs b/prov-graph/src/manifest.rs index f387305..b7186d9 100644 --- a/prov-graph/src/manifest.rs +++ b/prov-graph/src/manifest.rs @@ -61,6 +61,13 @@ //! drift. An excluded directory is not walked at all. See //! [`Manifest::excludes`] for the syntax. //! +//! And a manifest may say its files are kept somewhere else: `held:` names the +//! machine that holds them (`by`) and, optionally, where to ask it for one +//! (`at`). On a device without the covered directory, a held manifest's rows +//! are elsewhere rather than missing — a photo archive on a NAS, described in a +//! library every device carries, whose files only the NAS has. Where the +//! directory is present nothing changes. See [`Held`]. +//! //! This module is the model and its serialization, which is all the read core //! needs; the verbs that build, refresh and verify one are `prov`'s. @@ -85,6 +92,9 @@ pub const VERBATIM_KEY: &str = "verbatim"; /// not claim — see [`Manifest::excludes`]. pub const EXCLUDE_KEY: &str = "exclude"; +/// The manifest key saying who keeps the covered files — see [`Held`]. +pub const HELD_KEY: &str = "held"; + /// The manifest key holding the rows. pub const FILES_KEY: &str = "files"; @@ -113,6 +123,67 @@ pub struct ManifestEntry { pub fields: Mapping, } +/// Who keeps a manifest's files, when they are not on every device that +/// carries the manifest ([`HELD_KEY`]). +/// +/// Written as `held: {by: Harris NAS, at: https://nas.example/}`. Neither is a +/// secret: who may ask is the keeper's to decide, never settled by knowing the +/// address. +/// +/// It changes what an absent root means and nothing else. Without it, a +/// covered directory that is not there is a broken link and every row is +/// missing; with it, the rows are held elsewhere, `check` says nothing about +/// them, and the verbs that would rebuild the rows from the directory refuse +/// rather than empty the manifest. Where the directory is present — on the +/// keeper itself — the manifest is checked exactly as an unheld one. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Held { + /// The keeper, named for a person: `Harris NAS`. + pub by: String, + /// Where a device asks the keeper for a file, if it can be asked at all. + pub at: Option, +} + +impl Held { + fn from_value(value: &Value) -> Result { + let refuse = || { + Error::Structure(format!( + "manifest `{HELD_KEY}` must be `{{by: , at:
}}`" + )) + }; + let map = value.as_mapping().ok_or_else(refuse)?; + let mut by = None; + let mut at = None; + for (key, value) in map { + match key.as_str() { + "by" => by = Some(value.as_str().ok_or_else(refuse)?), + "at" => match value { + Value::Null => {} + value => at = Some(value.as_str().ok_or_else(refuse)?.to_string()), + }, + _ => return Err(refuse()), + } + } + let by = by + .map(str::trim) + .filter(|by| !by.is_empty()) + .ok_or_else(refuse)?; + Ok(Held { + by: by.to_string(), + at, + }) + } + + fn to_value(&self) -> Value { + let mut map = Mapping::new(); + map.insert("by".into(), Value::String(self.by.clone())); + if let Some(at) = &self.at { + map.insert("at".into(), Value::String(at.clone())); + } + Value::Mapping(map) + } +} + /// A parsed manifest document: the directory it claims, and the files it says /// are in it. #[derive(Debug, Clone, PartialEq, Default)] @@ -129,6 +200,8 @@ pub struct Manifest { /// Patterns for files under `root` that the manifest does not claim /// ([`EXCLUDE_KEY`]), as written. See [`excludes`](Self::excludes). pub exclude: Vec, + /// Who keeps the files, when not every device has them ([`HELD_KEY`]). + pub held: Option, } impl Manifest { @@ -177,6 +250,10 @@ impl Manifest { patterns } }; + let held = match meta.get(HELD_KEY) { + None | Some(Value::Null) => None, + Some(value) => Some(Held::from_value(value)?), + }; let rows = match meta.get(FILES_KEY) { // A manifest over an empty directory has no rows at all, which is a // legitimate state (`files:` written as null, or absent). @@ -218,6 +295,7 @@ impl Manifest { files, verbatim, exclude, + held, }) } @@ -234,6 +312,9 @@ impl Manifest { if self.verbatim { map.insert(VERBATIM_KEY.into(), Value::Bool(true)); } + if let Some(held) = &self.held { + map.insert(HELD_KEY.into(), held.to_value()); + } if !self.exclude.is_empty() { map.insert( EXCLUDE_KEY.into(), @@ -699,6 +780,46 @@ mod tests { assert_eq!(parse(&text(&excluding)).unwrap(), excluding); } + #[test] + fn held_round_trips_and_is_left_out_when_absent() { + let text = |m: &Manifest| { + crate::meta::serialize_mapping( + &m.to_mapping("Photo — manifest"), + crate::MetaFormat::Yaml, + ) + .unwrap() + }; + let plain = parse("root: photo/\n").unwrap(); + assert_eq!(plain.held, None); + assert!(!text(&plain).contains("held"), "{}", text(&plain)); + let held = + parse("root: photo/\nheld:\n by: Harris NAS\n at: https://nas.example/\n").unwrap(); + assert_eq!( + held.held, + Some(Held { + by: "Harris NAS".into(), + at: Some("https://nas.example/".into()), + }) + ); + assert_eq!(parse(&text(&held)).unwrap(), held); + let nameless = parse("root: photo/\nheld: {by: Harris NAS}\n").unwrap(); + assert_eq!(nameless.held.unwrap().at, None); + } + + #[test] + fn a_held_it_cannot_read_is_refused() { + for bad in [ + "held: Harris NAS\n", + "held: {at: https://nas.example/}\n", + "held: {by: ' '}\n", + "held: {by: 3}\n", + "held: {by: NAS, where: attic}\n", + ] { + let text = format!("root: photo/\n{bad}"); + assert!(parse(&text).is_err(), "{bad}"); + } + } + #[test] fn an_exclude_it_cannot_read_is_refused() { for bad in [ diff --git a/prov/src/lib.rs b/prov/src/lib.rs index de10e40..d6f1430 100644 --- a/prov/src/lib.rs +++ b/prov/src/lib.rs @@ -307,6 +307,7 @@ pub use mutate::{ PlannedIndex, RegroupOptions, Reordered, Reparented, ScatterPlan, }; pub use prov_exports::ExportSpec; +pub use prov_graph::manifest::Held; pub use prov_views::ViewSpec; pub use provenance::{Actor, Confirmation, Confirmations, Generated, Tier}; pub use remedy::{Fix, Remedy, RemedyKind, Warrant}; diff --git a/prov/src/manifest.rs b/prov/src/manifest.rs index 8b9ea04..d9a27e7 100644 --- a/prov/src/manifest.rs +++ b/prov/src/manifest.rs @@ -127,6 +127,12 @@ pub struct ManifestStatus { pub missing: Vec, /// Opaque files under `root` that no row claims, relative to `root`. pub extra: Vec, + /// Who keeps the files, when the manifest says ([`Held`](prov_graph::manifest::Held)). + pub held: Option, + /// Whether the files are held elsewhere and not here: the manifest is + /// held and its directory is absent. Then nothing is missing or extra — + /// there was nothing here to compare. + pub elsewhere: bool, } impl ManifestStatus { @@ -230,6 +236,7 @@ impl Workspace { files, verbatim: claim.verbatim, exclude: claim.exclude.clone(), + held: claim.held.clone(), }; manifest.sort(); Ok(manifest) @@ -289,8 +296,13 @@ impl Workspace { return Ok(None); }; let root = manifest.checked_root(&manifest_doc)?; - let on_disk = self.graph().scan_manifest(&manifest_doc, &manifest).await?; - let (missing, extra) = prov_graph::manifest::diff(&manifest.files, &on_disk); + let elsewhere = manifest.held.is_some() && !self.is_dir_here(&root).await; + let (missing, extra) = if elsewhere { + (Vec::new(), Vec::new()) + } else { + let on_disk = self.graph().scan_manifest(&manifest_doc, &manifest).await?; + prov_graph::manifest::diff(&manifest.files, &on_disk) + }; Ok(Some(ManifestStatus { node: node.to_path_buf(), manifest: manifest_doc, @@ -299,9 +311,39 @@ impl Workspace { listed: manifest.files.len(), missing, extra, + held: manifest.held, + elsewhere, })) } + /// Whether `dir` is a directory on this device — following a link, as the + /// scan does. + pub(crate) async fn is_dir_here(&self, dir: &Path) -> bool { + self.graph() + .stat(dir) + .await + .map(|m| m.is_dir()) + .unwrap_or(false) + } + + /// Refuse to rebuild or extend a held manifest's rows from a directory + /// that is not here: the scan would find nothing, and an update would empty + /// the record of an archive this device simply does not carry. + async fn refuse_held_elsewhere(&self, manifest_doc: &Path, manifest: &Manifest) -> Result<()> { + let Some(held) = &manifest.held else { + return Ok(()); + }; + let root = manifest.checked_root(manifest_doc)?; + if self.is_dir_here(&root).await { + return Ok(()); + } + Err(Error::Structure(format!( + "{}: its files are held by {} and are not on this device; refresh it where they are", + manifest_doc.display(), + held.by + ))) + } + /// **Deep** verification of the manifest `node` declares: read every listed /// file and compare its bytes against the recorded digest, one /// [`Finding::ManifestMismatch`] per file that disagrees. @@ -370,6 +412,7 @@ impl Workspace { node.display() ))); }; + self.refuse_held_elsewhere(&manifest_doc, ¤t).await?; // An empty manifest has no mode to preserve, so it takes the // workspace's: a directory that was empty when it was covered and has // files in it now should record them the way this workspace records @@ -512,6 +555,7 @@ impl Workspace { if paths.is_empty() { return Ok((update, Vec::new())); } + self.refuse_held_elsewhere(&manifest_doc, ¤t).await?; let root = current.checked_root(&manifest_doc)?; let mut listed: std::collections::BTreeSet = current.files.iter().map(|e| e.path.clone()).collect(); @@ -844,6 +888,41 @@ impl Workspace { Ok(update) } + /// Say who keeps `node`'s files — or, with `None`, that every device + /// carrying the manifest has them — and re-pin the node over the result, + /// in one change set. Answers whether anything was written: setting what + /// is already set writes nothing. + /// + /// The rows are kept as they are. Marking a manifest held is what makes it + /// safe to carry to a device without its directory, so it is done where + /// the files are, before the manifest travels. + pub async fn set_manifest_held( + &mut self, + node: &Path, + held: Option, + ) -> Result { + let node = link::normalize(node); + let Some((manifest_doc, mut manifest)) = self.manifest_of(&node).await? else { + return Err(Error::Structure(format!( + "{} declares no manifest", + node.display() + ))); + }; + if manifest.held == held { + return Ok(false); + } + manifest.held = held; + let writes = self + .manifest_rewrite(&node, &manifest_doc, &manifest) + .await?; + let mut cs = self.change(); + for (path, text) in writes { + cs.write(&path, text); + } + self.commit(cs).await?; + Ok(true) + } + /// Append rows for `paths` — workspace-relative files under `node`'s /// covered root — to its manifest, and re-stamp the node's `content_hash` /// over the result in the same change set. Returns the rows added, in @@ -1700,6 +1779,52 @@ mod tests { dir } + #[test] + fn a_held_manifest_is_elsewhere_where_its_directory_is_not() { + use prov_graph::manifest::Held; + let dir = photos("held"); + let mut w = ws(&dir); + let node = block_on(w.attach_manifest(Path::new("photos"), Path::new("index.md"))).unwrap(); + let nas = Held { + by: "Harris NAS".into(), + at: Some("https://nas.example/".into()), + }; + assert!(block_on(w.set_manifest_held(&node, Some(nas.clone()))).unwrap()); + assert!(!block_on(w.set_manifest_held(&node, Some(nas.clone()))).unwrap()); + let manifest = read(&dir, "photos.manifest.yaml"); + assert!(manifest.contains("by: Harris NAS"), "{manifest}"); + assert_eq!(block_on(w.check("index.md")).unwrap(), vec![]); + + // On the keeper, a held manifest is checked as any other. + write(&dir, "photos/stray.jpg", [0xff, 0xd8, 0x09]); + assert!( + !block_on(w.manifest_status(&node)) + .unwrap() + .unwrap() + .agrees() + ); + std::fs::remove_file(dir.join("photos/stray.jpg")).unwrap(); + + // Carried to a device without the photographs: elsewhere, not missing. + std::fs::rename(dir.join("photos"), dir.join("away")).unwrap(); + let status = block_on(w.manifest_status(&node)).unwrap().unwrap(); + assert!(status.elsewhere && status.agrees(), "{status:?}"); + assert_eq!(status.held, Some(nas)); + assert_eq!(status.listed, 2); + assert_eq!(block_on(w.check("index.md")).unwrap(), vec![]); + + // And nothing here may rebuild the rows from a directory it lacks. + let before = read(&dir, "photos.manifest.yaml"); + let refused = block_on(w.update_manifest(&node)).unwrap_err().to_string(); + assert!(refused.contains("held by Harris NAS"), "{refused}"); + assert!(block_on(w.extend_manifest(&node, &[PathBuf::from("photos/c.jpg")])).is_err()); + assert_eq!(read(&dir, "photos.manifest.yaml"), before); + + // Unheld, the same absence is a broken root again. + assert!(block_on(w.set_manifest_held(&node, None)).unwrap()); + assert!(!block_on(w.check("index.md")).unwrap().is_empty()); + } + #[test] fn excluded_files_are_neither_listed_nor_drift() { let dir = nas_share("exclude"); diff --git a/prov/src/mutate/gather.rs b/prov/src/mutate/gather.rs index 1d54303..5ea62be 100644 --- a/prov/src/mutate/gather.rs +++ b/prov/src/mutate/gather.rs @@ -520,6 +520,7 @@ impl Workspace { files: entries, verbatim: false, exclude: Vec::new(), + held: None, }; manifest.sort(); let manifest_text = prov_graph::meta::serialize_mapping( diff --git a/prov/src/validate.rs b/prov/src/validate.rs index ed0501c..06eb978 100644 --- a/prov/src/validate.rs +++ b/prov/src/validate.rs @@ -2522,13 +2522,12 @@ impl Workspace { } }; let root = manifest.covered_root(&manifest_doc); - if !self - .graph() - .stat(&root) - .await - .map(|m| m.is_dir()) - .unwrap_or(false) - { + if !self.is_dir_here(&root).await { + // Its files are kept somewhere else, and this device was told + // so: nothing here to compare, and nothing broken. + if manifest.held.is_some() { + continue; + } findings.push(Finding::BrokenLink { doc: manifest_doc, site: LinkSite::field(prov_graph::manifest::ROOT_KEY),