diff --git a/sdks/node/lib/native-contracts.ts b/sdks/node/lib/native-contracts.ts index 060c7ea4a..7df377d52 100644 --- a/sdks/node/lib/native-contracts.ts +++ b/sdks/node/lib/native-contracts.ts @@ -360,7 +360,19 @@ export interface NativeBoxConnection { export type JsCloneOptions = Record; -export type JsExportOptions = Record; +export interface JsExportOptions { + /** + * Write a directory of content-addressed objects instead of one `.boxlite` + * file, so mirroring it to object storage transfers only the objects the + * destination lacks. + */ + asDirectory?: boolean; + /** + * Publish into a shared layer store under this archive name (requires + * `asDirectory`; the destination is then the store root). + */ + archiveName?: string; +} export interface JsBox { readonly id: string; diff --git a/sdks/node/src/snapshot_options.rs b/sdks/node/src/snapshot_options.rs index 8387cd912..763dbb687 100644 --- a/sdks/node/src/snapshot_options.rs +++ b/sdks/node/src/snapshot_options.rs @@ -14,14 +14,25 @@ impl From for SnapshotOptions { } } -/// Options for exporting a box (forward-compatible placeholder). +/// Options for exporting a box. #[napi(object)] #[derive(Clone, Debug)] -pub struct JsExportOptions {} +pub struct JsExportOptions { + /// Write a directory of content-addressed objects instead of one + /// `.boxlite` file, so mirroring it to object storage transfers only the + /// objects the destination lacks. + pub as_directory: Option, + /// Publish into a shared layer store under this archive name (requires + /// `asDirectory`; the destination is then the store root). + pub archive_name: Option, +} impl From for ExportOptions { - fn from(_js: JsExportOptions) -> Self { - ExportOptions {} + fn from(js: JsExportOptions) -> Self { + ExportOptions { + as_directory: js.as_directory.unwrap_or(false), + archive_name: js.archive_name, + } } } @@ -48,7 +59,10 @@ mod tests { #[test] fn export_options_from_js() { - let js = JsExportOptions {}; + let js = JsExportOptions { + as_directory: None, + archive_name: None, + }; let _opts: ExportOptions = js.into(); } diff --git a/sdks/python/src/snapshot_options.rs b/sdks/python/src/snapshot_options.rs index 65530f1e7..32d0ce793 100644 --- a/sdks/python/src/snapshot_options.rs +++ b/sdks/python/src/snapshot_options.rs @@ -22,22 +22,39 @@ impl From for SnapshotOptions { } } -/// Options for exporting a box (forward-compatible placeholder). +/// Options for exporting a box. #[pyclass(name = "ExportOptions")] #[derive(Clone)] -pub(crate) struct PyExportOptions {} +pub(crate) struct PyExportOptions { + /// Write a directory of content-addressed objects instead of one + /// `.boxlite` file, so mirroring it to object storage transfers only the + /// objects the destination lacks. + #[pyo3(get, set)] + pub(crate) as_directory: bool, + /// Publish into a shared layer store under this archive name (requires + /// `as_directory`; the destination is then the store root). + #[pyo3(get, set)] + pub(crate) archive_name: Option, +} #[pymethods] impl PyExportOptions { #[new] - fn new() -> Self { - Self {} + #[pyo3(signature = (as_directory = false, archive_name = None))] + fn new(as_directory: bool, archive_name: Option) -> Self { + Self { + as_directory, + archive_name, + } } } impl From for ExportOptions { - fn from(_py: PyExportOptions) -> Self { - ExportOptions {} + fn from(py: PyExportOptions) -> Self { + ExportOptions { + as_directory: py.as_directory, + archive_name: py.archive_name, + } } } diff --git a/src/boxlite/src/db/base_disk.rs b/src/boxlite/src/db/base_disk.rs index fd1c65b1f..b245389ca 100644 --- a/src/boxlite/src/db/base_disk.rs +++ b/src/boxlite/src/db/base_disk.rs @@ -96,8 +96,8 @@ impl BaseDiskStore { let conn = self.db.conn(); db_err!(conn.execute( "INSERT INTO base_disk \ - (id, source_box_id, name, kind, base_path, created_at, json) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + (id, source_box_id, name, kind, base_path, created_at, json, digest) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", rusqlite::params![ &disk.id, &disk.source_box_id, @@ -106,11 +106,43 @@ impl BaseDiskStore { &disk.disk_info.base_path, disk.created_at, json, + &disk.digest, ], ))?; Ok(()) } + /// Find a base disk by its content digest. + /// + /// Only layers whose digest has already been computed are visible here; + /// see [`BaseDisk::digest`] for why it is filled in lazily. + pub(crate) fn find_by_digest(&self, digest: &str) -> BoxliteResult> { + let conn = self.db.conn(); + let result = db_err!( + conn.query_row( + "SELECT id, source_box_id, name, kind, base_path, \ + created_at, json FROM base_disk WHERE digest = ?1", + rusqlite::params![digest], + row_to_record, + ) + .optional() + )?; + Ok(result) + } + + /// Record a layer's content digest, in both the indexed column and the + /// JSON blob so the two cannot drift. + pub(crate) fn set_digest(&self, id: &BaseDiskID, digest: &str) -> BoxliteResult<()> { + let conn = self.db.conn(); + db_err!(conn.execute( + "UPDATE base_disk \ + SET digest = ?2, json = json_set(json, '$.digest', ?2) \ + WHERE id = ?1", + rusqlite::params![id, digest], + ))?; + Ok(()) + } + /// Find a base disk by its ID. #[allow(dead_code)] // used in lineage.rs tests pub(crate) fn find_by_id(&self, id: &BaseDiskID) -> BoxliteResult> { @@ -330,6 +362,7 @@ mod tests { size_bytes: 512, }, created_at: chrono::Utc::now().timestamp(), + digest: None, } } @@ -686,6 +719,7 @@ mod tests { size_bytes: 1024, }, created_at: 1700000000, + digest: None, }; store.insert(&disk).unwrap(); diff --git a/src/boxlite/src/db/migration/mod.rs b/src/boxlite/src/db/migration/mod.rs index 98dcd268e..26aededbf 100644 --- a/src/boxlite/src/db/migration/mod.rs +++ b/src/boxlite/src/db/migration/mod.rs @@ -11,6 +11,7 @@ mod v5_to_v6; mod v6_to_v7; mod v7_to_v8; mod v8_to_v9; +mod v9_to_v10; use std::path::Path; @@ -81,5 +82,6 @@ fn all_migrations() -> Vec> { Box::new(v6_to_v7::MoveDisksAndAddBaseDisk), Box::new(v7_to_v8::RenameNetworkSpec), Box::new(v8_to_v9::PreservePublishedPorts), + Box::new(v9_to_v10::AddBaseDiskDigest), ] } diff --git a/src/boxlite/src/db/migration/v6_to_v7.rs b/src/boxlite/src/db/migration/v6_to_v7.rs index 24e73694d..28bb50eef 100644 --- a/src/boxlite/src/db/migration/v6_to_v7.rs +++ b/src/boxlite/src/db/migration/v6_to_v7.rs @@ -18,6 +18,29 @@ use crate::db::schema; use crate::runtime::id::BaseDiskID; use crate::runtime::id::BaseDiskIDMint; +/// The `base_disk` table exactly as v7 created it. +/// +/// Frozen on purpose: a migration must reproduce the schema of its own era, so +/// it cannot read `schema::BASE_DISK_TABLE`. That constant tracks the current +/// schema, and every column later added to it would otherwise appear here too +/// — making the `ALTER TABLE` in the migration that introduces the column fail +/// with "duplicate column name" for anyone upgrading from v6 or earlier. +const V7_BASE_DISK_TABLE: &str = r#" +CREATE TABLE IF NOT EXISTS base_disk ( + id TEXT PRIMARY KEY NOT NULL, + source_box_id TEXT NOT NULL, + name TEXT, + kind TEXT NOT NULL CHECK(kind IN ('snapshot', 'clone_base', 'rootfs')), + base_path TEXT NOT NULL, + created_at INTEGER NOT NULL, + json TEXT NOT NULL, + UNIQUE(source_box_id, name) +); +CREATE INDEX IF NOT EXISTS idx_base_disk_source ON base_disk(source_box_id); +CREATE INDEX IF NOT EXISTS idx_base_disk_kind ON base_disk(kind); +CREATE INDEX IF NOT EXISTS idx_base_disk_path ON base_disk(base_path); +"#; + pub(crate) struct MoveDisksAndAddBaseDisk; impl Migration for MoveDisksAndAddBaseDisk { @@ -33,7 +56,7 @@ impl Migration for MoveDisksAndAddBaseDisk { fn run(&self, conn: &Connection, home_dir: Option<&Path>) -> BoxliteResult<()> { // 1. Create base_disk table (for clone bases and rootfs cache). - db_err!(conn.execute_batch(schema::BASE_DISK_TABLE))?; + db_err!(conn.execute_batch(V7_BASE_DISK_TABLE))?; // 2. Create snapshot table (for per-box snapshots). db_err!(conn.execute_batch(schema::SNAPSHOT_TABLE))?; @@ -307,7 +330,7 @@ mod tests { /// Create an in-memory DB with the base_disk table for migration tests. fn test_db() -> Connection { let conn = Connection::open_in_memory().unwrap(); - conn.execute_batch(schema::BASE_DISK_TABLE).unwrap(); + conn.execute_batch(V7_BASE_DISK_TABLE).unwrap(); conn.execute_batch(schema::SNAPSHOT_TABLE).unwrap(); conn.execute_batch(schema::BASE_DISK_REF_TABLE).unwrap(); conn diff --git a/src/boxlite/src/db/migration/v9_to_v10.rs b/src/boxlite/src/db/migration/v9_to_v10.rs new file mode 100644 index 000000000..35a5dfe52 --- /dev/null +++ b/src/boxlite/src/db/migration/v9_to_v10.rs @@ -0,0 +1,105 @@ +//! Migration v9 → v10: Add a content digest column to `base_disk`. +//! +//! Layers are addressed by content when they travel between hosts, so a base +//! needs a digest that can be looked up without scanning every JSON blob. The +//! column is nullable and left empty here: a base is immutable, so its digest +//! is computed once, on first use, rather than by hashing every existing layer +//! during startup. + +use std::path::Path; + +use rusqlite::Connection; + +use boxlite_shared::errors::{BoxliteError, BoxliteResult}; + +use super::{Migration, db_err}; + +pub(crate) struct AddBaseDiskDigest; + +impl Migration for AddBaseDiskDigest { + fn source_version(&self) -> i32 { + 9 + } + fn target_version(&self) -> i32 { + 10 + } + fn description(&self) -> &str { + "Add base_disk.digest column and index" + } + + fn run(&self, conn: &Connection, _home_dir: Option<&Path>) -> BoxliteResult<()> { + db_err!(conn.execute("ALTER TABLE base_disk ADD COLUMN digest TEXT", []))?; + db_err!(conn.execute( + "CREATE INDEX IF NOT EXISTS idx_base_disk_digest ON base_disk(digest)", + [], + ))?; + + tracing::info!("Added base_disk.digest column (populated lazily on first use)"); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v9_base_disk_table(conn: &Connection) { + conn.execute_batch( + r#"CREATE TABLE base_disk ( + id TEXT PRIMARY KEY NOT NULL, + source_box_id TEXT NOT NULL, + name TEXT, + kind TEXT NOT NULL, + base_path TEXT NOT NULL, + created_at INTEGER NOT NULL, + json TEXT NOT NULL + );"#, + ) + .unwrap(); + conn.execute( + "INSERT INTO base_disk (id, source_box_id, name, kind, base_path, created_at, json) \ + VALUES ('abc', 'box1', NULL, 'clone_base', '/bases/abc.qcow2', 1, '{}')", + [], + ) + .unwrap(); + } + + #[test] + fn existing_rows_survive_with_a_null_digest() { + let conn = Connection::open_in_memory().unwrap(); + v9_base_disk_table(&conn); + + AddBaseDiskDigest.run(&conn, None).unwrap(); + + // A pre-existing layer keeps a NULL digest, so the lazy computation + // path — not the migration — is what fills it in. Hashing every base + // here would read every cached layer on the first startup after an + // upgrade. + let digest: Option = conn + .query_row("SELECT digest FROM base_disk WHERE id = 'abc'", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(digest, None); + } + + #[test] + fn digest_column_is_writable_after_migration() { + let conn = Connection::open_in_memory().unwrap(); + v9_base_disk_table(&conn); + + AddBaseDiskDigest.run(&conn, None).unwrap(); + conn.execute( + "UPDATE base_disk SET digest = 'sha256:dead' WHERE id = 'abc'", + [], + ) + .unwrap(); + + let digest: Option = conn + .query_row("SELECT digest FROM base_disk WHERE id = 'abc'", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(digest.as_deref(), Some("sha256:dead")); + } +} diff --git a/src/boxlite/src/db/schema.rs b/src/boxlite/src/db/schema.rs index c419be10f..03abc75c8 100644 --- a/src/boxlite/src/db/schema.rs +++ b/src/boxlite/src/db/schema.rs @@ -7,7 +7,7 @@ //! Each table has queryable columns for efficient filtering + JSON blob for full data. /// Current schema version. -pub const SCHEMA_VERSION: i32 = 9; +pub const SCHEMA_VERSION: i32 = 10; /// Schema version tracking table. pub const SCHEMA_VERSION_TABLE: &str = r#" @@ -115,11 +115,13 @@ CREATE TABLE IF NOT EXISTS base_disk ( base_path TEXT NOT NULL, created_at INTEGER NOT NULL, json TEXT NOT NULL, + digest TEXT, UNIQUE(source_box_id, name) ); CREATE INDEX IF NOT EXISTS idx_base_disk_source ON base_disk(source_box_id); CREATE INDEX IF NOT EXISTS idx_base_disk_kind ON base_disk(kind); CREATE INDEX IF NOT EXISTS idx_base_disk_path ON base_disk(base_path); +CREATE INDEX IF NOT EXISTS idx_base_disk_digest ON base_disk(digest); "#; /// Base disk reference table (added in v7). diff --git a/src/boxlite/src/disk/base_disk.rs b/src/boxlite/src/disk/base_disk.rs index c2692d2b5..89011fef8 100644 --- a/src/boxlite/src/disk/base_disk.rs +++ b/src/boxlite/src/disk/base_disk.rs @@ -19,9 +19,11 @@ use std::collections::HashSet; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::{Duration, SystemTime}; use boxlite_shared::errors::BoxliteResult; +use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use crate::db::base_disk::BaseDiskStore; @@ -64,9 +66,69 @@ pub struct BaseDisk { #[serde(flatten)] pub disk_info: super::DiskInfo, pub created_at: i64, + /// Content digest (`sha256:`) of the layer file, or `None` until one + /// is needed. + /// + /// Computed lazily rather than at creation: `create_base_disk` forks a + /// layer with a `rename(2)`, and hashing there would turn an O(1) + /// operation into a full read of the disk on every clone. A base is + /// immutable once created, so the digest is stable and only has to be + /// computed once — see [`BaseDiskManager::digest_of`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub digest: Option, } use crate::disk::constants::filenames as disk_filenames; +/// Sentinel `source_box_id` for layers that arrived in an archive rather than +/// being forked from a box on this host. +const IMPORTED_SOURCE: &str = "__imported__"; + +/// Canonical digest of an immutable layer that has no store record, cached in a +/// file beside it. +/// +/// The image disk is the case this exists for: it lives in the image cache +/// under a path derived from its image digest, nothing rewrites it, and it is +/// typically the largest layer in a chain. Hashing it on every export is the +/// single most expensive thing export does. +/// +/// A stale sidecar is not a risk here — the file it names is addressed by +/// content and installed atomically, so a given path always holds the same +/// bytes. The write is best-effort: losing it only costs a rehash. +fn sidecar_digest(path: &Path) -> BoxliteResult { + let sidecar = path.with_extension(format!( + "{}.digest", + path.extension().unwrap_or_default().to_string_lossy() + )); + + if let Ok(cached) = std::fs::read_to_string(&sidecar) { + let cached = cached.trim(); + if cached + .strip_prefix("sha256:") + .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())) + { + return Ok(cached.to_string()); + } + } + + let digest = crate::litebox::archive::CanonicalLayer::open(path)?.digest()?; + let staging = sidecar.with_extension(format!( + "{}.{}.partial", + sidecar.extension().unwrap_or_default().to_string_lossy(), + uuid::Uuid::new_v4() + )); + if let Err(e) = + std::fs::write(&staging, &digest).and_then(|()| std::fs::rename(&staging, &sidecar)) + { + let _ = std::fs::remove_file(&staging); + tracing::debug!( + path = %sidecar.display(), + error = %e, + "Could not cache layer digest; it will be recomputed next export" + ); + } + Ok(digest) +} + /// Manages the lifecycle of clone base disks. /// /// All base disks are flat files under `bases_dir/` named by `BaseDiskID`. @@ -78,6 +140,7 @@ use crate::disk::constants::filenames as disk_filenames; pub(crate) struct BaseDiskManager { bases_dir: PathBuf, store: BaseDiskStore, + lifecycle_lock: Arc>, } impl BaseDiskManager { @@ -91,7 +154,11 @@ impl BaseDiskManager { let bases_dir = bases_dir .canonicalize() .unwrap_or_else(|_| bases_dir.clone()); - Self { bases_dir, store } + Self { + bases_dir, + store, + lifecycle_lock: Arc::new(Mutex::new(())), + } } /// Expose the underlying store for direct queries (list, find, etc.). @@ -99,6 +166,10 @@ impl BaseDiskManager { &self.store } + pub(crate) fn lock_lifecycle(&self) -> parking_lot::MutexGuard<'_, ()> { + self.lifecycle_lock.lock() + } + /// The bases root directory. #[allow(dead_code)] // used in tests pub(crate) fn bases_dir(&self) -> &Path { @@ -321,6 +392,7 @@ impl BaseDiskManager { kind, disk_info, created_at: now, + digest: None, }; self.store.insert(&disk)?; @@ -330,11 +402,96 @@ impl BaseDiskManager { Ok(disk) } + /// Install an already-verified layer blob as a base disk with a known digest. + /// + /// Used by import: the caller has checked the blob hashes to `digest`, so + /// the digest is recorded up front rather than lazily. The blob is moved, + /// not copied — it lives in the import's temp directory and is about to be + /// discarded. + /// + /// The digest stays valid after the caller relinks the installed file, + /// because it names the layer's canonical (backing-cleared) form rather + /// than the bytes currently on disk — see + /// [`crate::litebox::archive::CanonicalLayer`]. + pub(crate) fn install_layer(&self, blob: &Path, digest: &str) -> BoxliteResult { + let base_disk_id = BaseDiskIDMint::mint(); + let base_file = self.bases_dir.join(format!("{}.qcow2", base_disk_id)); + + crate::litebox::archive::move_file(blob, &base_file)?; + + let size_bytes = std::fs::metadata(&base_file).map(|m| m.len()).unwrap_or(0); + let disk = BaseDisk { + id: base_disk_id, + // Not forked from any box on this host — it arrived in an archive. + source_box_id: IMPORTED_SOURCE.to_string(), + name: None, + kind: BaseDiskKind::CloneBase, + disk_info: super::DiskInfo { + base_path: base_file + .canonicalize() + .unwrap_or(base_file.clone()) + .to_string_lossy() + .to_string(), + container_disk_bytes: size_bytes, + size_bytes, + }, + created_at: chrono::Utc::now().timestamp(), + digest: Some(digest.to_string()), + }; + self.store.insert(&disk)?; + Ok(disk) + } + + /// The content digest of a layer already registered in the store, + /// computing and caching it on first call. + /// + /// A base is immutable, so the digest is stable and the hash is paid once + /// per layer for the lifetime of the store — which is what keeps repeat + /// exports of boxes sharing a base cheap. + /// + /// Returns `None` for a path that is not a registered base (an image + /// backing file, a raw rootfs), whose digest the caller must compute + /// itself; nothing durable exists to cache it against. + pub(crate) fn digest_of(&self, layer_path: &Path) -> BoxliteResult> { + let canonical = layer_path + .canonicalize() + .unwrap_or_else(|_| layer_path.to_path_buf()); + let Some(record) = self.store.find_by_base_path(&canonical.to_string_lossy())? else { + // Not a registered base — the image disk is the one that matters + // here, and it is the largest layer in a chain. Its own cache is + // keyed by image digest and its contents never change, so a sidecar + // is enough to keep export from re-reading hundreds of megabytes + // every time. + return sidecar_digest(&canonical).map(Some); + }; + + if let Some(digest) = record.disk.digest { + return Ok(Some(digest)); + } + + let digest = crate::litebox::archive::CanonicalLayer::open(&canonical)?.digest()?; + // A cache write that loses a race is harmless: the digest is a pure + // function of immutable content, so both writers store the same value. + if let Err(e) = self.store.set_digest(&record.disk.id, &digest) { + tracing::warn!( + base_disk_id = %record.disk.id, + error = %e, + "Failed to cache base disk digest; it will be recomputed next time" + ); + } + Ok(Some(digest)) + } + /// Attempt to garbage-collect a clone base by ID and cascade to parent. /// /// Queries the `base_disk_ref` table for dependents. If none exist, /// deletes the base (DB record + file) and cascades to the parent base. pub(crate) fn try_gc_base(&self, base_disk_id: &BaseDiskID) { + let _lifecycle = self.lifecycle_lock.lock(); + self.try_gc_base_locked(base_disk_id); + } + + fn try_gc_base_locked(&self, base_disk_id: &BaseDiskID) { let record = match self.store.find_by_id(base_disk_id) { Ok(Some(r)) => r, _ => return, @@ -368,7 +525,7 @@ impl BaseDiskManager { && let Ok(Some(parent_record)) = self.store.find_by_base_path(&parent_path.to_string_lossy()) { - self.try_gc_base(parent_record.id()); + self.try_gc_base_locked(parent_record.id()); } } @@ -435,6 +592,47 @@ mod tests { (dir, mgr) } + /// A layer outside the base store — the image disk — must not be re-read on + /// every export; it is the largest layer in a typical chain. + #[test] + fn digest_of_an_unregistered_layer_is_cached_beside_it() { + let (dir, mgr) = setup(); + let image_disk = dir.path().join("sha256-abc.ext4"); + std::fs::write(&image_disk, b"raw ext4 bytes").unwrap(); + + let first = mgr.digest_of(&image_disk).unwrap().expect("a digest"); + + // Prove the second call answers from the sidecar rather than the file: + // replace the file's contents and require the answer not to change. + let sidecar = dir.path().join("sha256-abc.ext4.digest"); + assert!( + sidecar.exists(), + "expected a sidecar at {}", + sidecar.display() + ); + std::fs::write(&image_disk, b"different bytes entirely").unwrap(); + + let second = mgr.digest_of(&image_disk).unwrap().expect("a digest"); + assert_eq!( + first, second, + "the cached digest must be returned without re-reading the layer" + ); + } + + #[test] + fn digest_of_ignores_a_truncated_sidecar() { + let (dir, mgr) = setup(); + let image_disk = dir.path().join("sha256-def.ext4"); + let sidecar = dir.path().join("sha256-def.ext4.digest"); + std::fs::write(&image_disk, b"raw ext4 bytes").unwrap(); + std::fs::write(&sidecar, "sha256:1234").unwrap(); + + let digest = mgr.digest_of(&image_disk).unwrap().expect("a digest"); + + assert_eq!(digest.len(), "sha256:".len() + 64); + assert_eq!(std::fs::read_to_string(sidecar).unwrap(), digest); + } + /// Helper: create a minimal qcow2 file with an optional backing file path. fn write_qcow2_with_backing(path: &Path, backing: Option<&str>) { use std::io::Write; @@ -627,6 +825,7 @@ mod tests { size_bytes: 0, }, created_at: 0, + digest: None, }; mgr.store().insert(&disk).unwrap(); @@ -666,6 +865,7 @@ mod tests { size_bytes: 0, }, created_at: 0, + digest: None, }; mgr.store().insert(&bd1).unwrap(); @@ -683,6 +883,7 @@ mod tests { size_bytes: 0, }, created_at: 0, + digest: None, }; mgr.store().insert(&bd2).unwrap(); @@ -726,6 +927,7 @@ mod tests { size_bytes: 0, }, created_at: 0, + digest: None, }; mgr.store().insert(&disk).unwrap(); @@ -760,6 +962,7 @@ mod tests { size_bytes: 0, }, created_at: 0, + digest: None, }; mgr.store().insert(&disk).unwrap(); @@ -812,6 +1015,7 @@ mod tests { id: base_id(id), source_box_id: "__global__".to_string(), name: Some(id.to_string()), + digest: None, kind, disk_info: DiskInfo { base_path: path.to_string_lossy().to_string(), diff --git a/src/boxlite/src/disk/mod.rs b/src/boxlite/src/disk/mod.rs index 1f82d791b..8919cad70 100644 --- a/src/boxlite/src/disk/mod.rs +++ b/src/boxlite/src/disk/mod.rs @@ -136,6 +136,7 @@ pub(crate) use base_disk::{BaseDisk, BaseDiskKind, BaseDiskManager}; pub use ext4::{create_ext4_from_dir, inject_file_into_ext4}; pub use qcow2::{ BackingFormat, Qcow2Helper, is_backing_dependency, read_backing_chain, read_backing_file_path, + set_backing_file_path, }; // ============================================================================ diff --git a/src/boxlite/src/disk/qcow2.rs b/src/boxlite/src/disk/qcow2.rs index 74ccdc6dd..1b4bfd0bf 100644 --- a/src/boxlite/src/disk/qcow2.rs +++ b/src/boxlite/src/disk/qcow2.rs @@ -250,6 +250,13 @@ impl Qcow2Helper { /// Equivalent to: `qemu-img convert -O qcow2 ` /// /// Errors on compressed clusters (bit 62 in L2 entries). + /// + /// Retained with no caller since export switched to shipping layers: + /// `MAX_BACKING_CHAIN_DEPTH` caps a chain at 8, so collapsing a chain back + /// into one image is the compaction step that keeps clone-heavy lineages + /// under the cap. Deleting it would only mean rewriting it — see + /// `docs/investigations/incremental-export-import.md`. + #[allow(dead_code)] pub fn flatten(src: &Path, dst: &Path) -> BoxliteResult<()> { use std::io::{Seek, SeekFrom, Write}; @@ -478,6 +485,7 @@ impl Qcow2Helper { /// Open the full backing chain starting from `path`. /// /// Returns layers from top (index 0) to base (last index). + #[allow(dead_code)] fn open_flatten_chain(path: &Path) -> BoxliteResult> { let mut chain = Vec::new(); let mut current_path = path.to_path_buf(); @@ -987,8 +995,8 @@ pub fn set_backing_file_path(qcow2_path: &Path, new_backing: &Path) -> BoxliteRe )) })?; - // Read header: magic (4) + version (4) + backing_file_offset (8) + backing_file_size (4) - let mut header = [0u8; 20]; + // Read through the L1 table offset so the backing path cannot overwrite metadata. + let mut header = [0u8; 48]; file.read_exact(&mut header).map_err(|e| { BoxliteError::Storage(format!( "Failed to read qcow2 header from {}: {}", @@ -1009,10 +1017,28 @@ pub fn set_backing_file_path(qcow2_path: &Path, new_backing: &Path) -> BoxliteRe let backing_offset = u64::from_be_bytes(header[8..16].try_into().unwrap()); let old_backing_size = u32::from_be_bytes(header[16..20].try_into().unwrap()); + let l1_table_offset = u64::from_be_bytes(header[40..48].try_into().unwrap()); + // A zero size with a valid offset is the canonical form an archive ships: + // the path was blanked so the layer hashes the same on every host, but the + // region it lived in is still reserved, so a new path can be written there. if backing_offset == 0 { return Err(BoxliteError::Storage(format!( - "Cannot rebase {}: no existing backing file reference", + "Cannot rebase {}: no reserved backing file region", + qcow2_path.display() + ))); + } + let backing_end = backing_offset + .checked_add(new_backing_bytes.len() as u64) + .ok_or_else(|| { + BoxliteError::Storage(format!( + "Cannot rebase {}: backing path length overflows", + qcow2_path.display() + )) + })?; + if l1_table_offset != 0 && backing_end > l1_table_offset { + return Err(BoxliteError::Storage(format!( + "Cannot rebase {}: backing path reaches qcow2 metadata", qcow2_path.display() ))); } @@ -1167,6 +1193,7 @@ pub fn is_backing_dependency(target: &Path, chain_root: &Path) -> bool { const QCOW2_MAGIC: u32 = 0x514649fb; /// A layer in a QCOW2 backing chain, used during flatten. +#[allow(dead_code)] enum FlattenLayer { /// A QCOW2 layer with L1/L2 indirection. Qcow2 { @@ -1622,7 +1649,48 @@ mod tests { let result = set_backing_file_path(&qcow2_path, &new_backing); assert!(result.is_err()); let err = result.unwrap_err().to_string(); - assert!(err.contains("no existing backing file reference")); + assert!( + err.contains("no reserved backing file region"), + "got: {err}" + ); + } + + /// A layer shipped in an archive has its backing path blanked but the + /// region it occupied still reserved, so an importer can write the parent + /// it chose. Rebasing must accept that, or every layered import fails. + #[test] + fn test_set_backing_file_path_accepts_a_blanked_reservation() { + let dir = TempDir::new().unwrap(); + let qcow2_path = dir.path().join("canonical.qcow2"); + + // A real child, then blanked the way CanonicalLayer presents it: + // offset preserved, size zeroed, path bytes zeroed. + write_qcow2_with_backing(&qcow2_path, Some("/exporter/bases/parent.qcow2")); + let mut bytes = std::fs::read(&qcow2_path).unwrap(); + let offset = u64::from_be_bytes(bytes[8..16].try_into().unwrap()) as usize; + let size = u32::from_be_bytes(bytes[16..20].try_into().unwrap()) as usize; + bytes[16..20].fill(0); + bytes[offset..offset + size].fill(0); + std::fs::write(&qcow2_path, &bytes).unwrap(); + + assert_eq!( + read_backing_file_path(&qcow2_path).unwrap(), + None, + "a blanked reservation must read as having no backing file" + ); + + let new_backing = dir.path().join("local-parent.qcow2"); + std::fs::write(&new_backing, vec![0u8; 512]).unwrap(); + set_backing_file_path(&qcow2_path, &new_backing) + .expect("rebase onto a blanked reservation"); + + let expected = new_backing.canonicalize().unwrap(); + assert_eq!( + read_backing_file_path(&qcow2_path) + .unwrap() + .map(std::path::PathBuf::from), + Some(expected) + ); } #[test] @@ -1639,6 +1707,25 @@ mod tests { assert!(err.contains("canonicalize")); } + #[test] + fn test_set_backing_file_path_rejects_metadata_overlap() { + let dir = TempDir::new().unwrap(); + let qcow2_path = dir.path().join("test.qcow2"); + let old_backing = dir.path().join("old.raw"); + std::fs::write(&old_backing, vec![0u8; 512]).unwrap(); + write_qcow2_with_backing(&qcow2_path, Some(&old_backing.to_string_lossy())); + + let mut bytes = std::fs::read(&qcow2_path).unwrap(); + bytes[40..48].copy_from_slice(&520u64.to_be_bytes()); + std::fs::write(&qcow2_path, bytes).unwrap(); + + let new_backing = dir.path().join("new.raw"); + std::fs::write(&new_backing, vec![0u8; 512]).unwrap(); + let error = set_backing_file_path(&qcow2_path, &new_backing) + .expect_err("backing path must not overwrite the L1 table"); + assert!(error.to_string().contains("reaches qcow2 metadata")); + } + #[test] fn test_set_backing_file_path_invalid_qcow2() { let dir = TempDir::new().unwrap(); diff --git a/src/boxlite/src/lib.rs b/src/boxlite/src/lib.rs index a89f2123d..4440df5ee 100644 --- a/src/boxlite/src/lib.rs +++ b/src/boxlite/src/lib.rs @@ -42,6 +42,7 @@ pub use disk::DiskInfo; pub use event_listener::{AuditEvent, AuditEventKind, AuditEventListener, EventListener}; pub use litebox::SnapshotHandle; pub use litebox::archive::ArchiveManifest; +pub use litebox::archive_store::{ArchiveStore, GcReport}; pub use litebox::snapshot_mgr::SnapshotInfo; pub use litebox::{ BoxCommand, CopyOptions, ExecResult, ExecStderr, ExecStdin, ExecStdout, Execution, ExecutionId, diff --git a/src/boxlite/src/litebox/archive.rs b/src/boxlite/src/litebox/archive.rs index 213379bb0..ab5e8b8a9 100644 --- a/src/boxlite/src/litebox/archive.rs +++ b/src/boxlite/src/litebox/archive.rs @@ -3,15 +3,12 @@ //! Handles `.boxlite` archive files: zstd-compressed tarballs containing //! disk images and a JSON manifest. -use std::io::Write; use std::path::Path; use boxlite_shared::errors::{BoxliteError, BoxliteResult}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::disk::constants::filenames as disk_filenames; - /// Manifest filename inside the archive. pub(crate) const MANIFEST_FILENAME: &str = "manifest.json"; @@ -34,8 +31,27 @@ pub(crate) const CAPABILITY_POLICY_ARCHIVE_VERSION: u32 = 4; /// v5, and the importer canonicalizes anything below it. pub(crate) const PUBLISHED_PORTS_ARCHIVE_VERSION: u32 = 5; +/// First archive version that carries the box's disk as a layer chain. +/// +/// Up to v5 an archive held one flattened `disk.qcow2`. A v6 archive holds +/// `layers/` blobs plus the order to relink them in, which an older importer +/// cannot reassemble — it would find no container disk at all — so stamping v6 +/// makes it refuse the archive rather than fail obscurely. +pub(crate) const LAYERED_ARCHIVE_VERSION: u32 = 6; + /// Maximum archive version this build can import. -pub(crate) const MAX_SUPPORTED_VERSION: u32 = PUBLISHED_PORTS_ARCHIVE_VERSION; +pub(crate) const MAX_SUPPORTED_VERSION: u32 = LAYERED_ARCHIVE_VERSION; + +/// Directory holding layer blobs inside a layered archive. +pub(crate) const LAYERS_DIR: &str = "layers"; + +/// Tar entry name for a layer blob, derived from its digest. +/// +/// The `sha256:` prefix is dropped so the name stays a plain path component. +pub(crate) fn layer_entry_name(digest: &str) -> String { + let hex = digest.strip_prefix("sha256:").unwrap_or(digest); + format!("{LAYERS_DIR}/{hex}") +} /// Pick the archive format an exported box needs. pub(crate) fn archive_version_for_options(options: &crate::runtime::options::BoxOptions) -> u32 { @@ -48,6 +64,42 @@ pub(crate) fn archive_version_for_options(options: &crate::runtime::options::Box } } +/// Format of a layer blob, which decides how its child references it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LayerFormat { + Qcow2, + /// A raw image, only ever the bottom of a chain (the image disk). + Raw, +} + +/// One layer of a box's disk chain, addressed by content. +/// +/// Carries no path: an importer resolves a layer against its own store and +/// picks where it lands, so nothing an archive says can point a backing file +/// at a host path of the archive's choosing. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ArchiveLayer { + /// `sha256:` of the layer blob. + pub digest: String, + /// Format of this layer's blob. + pub format: LayerFormat, + /// Virtual size in bytes (qcow2 layers only; 0 for raw). + #[serde(default)] + pub virtual_size: u64, + /// The OCI image this layer is the disk for, when it is one. + /// + /// The bottom of every chain is the image's ext4, and `mke2fs` writes a + /// random filesystem UUID and timestamps into it — so two hosts building + /// the same image produce different bytes and `digest` can never match + /// across them. The image digest can: it is a hash of the OCI layer + /// digests (images/object.rs), identical everywhere. An importer that + /// already holds this image's disk uses its own copy and never writes the + /// blob, which is the only form of cross-host reuse this layer can have. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_digest: Option, +} + /// Archive manifest stored as `manifest.json` inside exported archives. /// /// v1: plain tar, no checksums @@ -55,9 +107,10 @@ pub(crate) fn archive_version_for_options(options: &crate::runtime::options::Box /// v3: adds `box_options` for full configuration preservation /// v4: `box_options.advanced` carries a custom capability policy /// v5: `ports` carry publication semantics (automatic host port, bind IP) +/// v6: the container disk travels as a chain of content-addressed layers #[derive(Debug, Serialize, Deserialize)] pub struct ArchiveManifest { - /// Archive format version (1 through 5). + /// Archive format version (1 through 6). pub version: u32, /// Original box name (optional, may be renamed on import). pub box_name: Option, @@ -70,18 +123,301 @@ pub struct ArchiveManifest { pub guest_disk_checksum: String, /// SHA-256 checksum of the container disk. pub container_disk_checksum: String, + /// The container disk's layer chain, ordered base first, top last (v6+). + /// + /// Empty for v1–v5, whose container disk is a single flattened image. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub layers: Vec, /// Timestamp when the archive was created. pub exported_at: String, } // ── Build ─────────────────────────────────────────────────────────────── -/// Build a zstd-compressed tar archive. -pub(crate) fn build_zstd_tar_archive( +/// A layer's bytes with its backing-file pointer zeroed. +/// +/// A qcow2's digest covers its header, and the header holds the *absolute +/// local path* of its parent. Hashing a layer as it sits on disk would +/// therefore mix in where that host happens to keep the parent, so the same +/// logical layer would hash differently on every machine and content +/// addressing could never match anything across hosts. It would also go stale +/// the moment an importer relinks the file, and leak the exporting host's +/// directory layout into the archive. +/// +/// The canonical form is what travels and what gets hashed: identical to the +/// file except `backing_file_offset`, `backing_file_size`, and the path string +/// they point at read as zeroes. Length is unchanged, so this streams — no +/// temporary copy of a multi-hundred-megabyte layer. +pub(crate) struct CanonicalLayer { + file: std::fs::File, + len: u64, + pos: u64, + /// Byte ranges to serve as zeroes, in ascending order. + holes: Vec<(u64, u64)>, +} + +impl CanonicalLayer { + /// Header bytes covering `backing_file_size` only. + /// + /// `backing_file_offset` is deliberately preserved: it is a location + /// *within* the file, identical on every host for a layer boxlite wrote, + /// and an importer needs it to know where to put the parent path it picks. + /// The size is zeroed because it would otherwise leak — and make the digest + /// depend on — how long the exporting host's path happened to be. + const BACKING_SIZE_FIELD: (u64, u64) = (16, 20); + + pub(crate) fn open(path: &Path) -> BoxliteResult { + use std::io::Read; + + let mut file = std::fs::File::open(path).map_err(|e| { + BoxliteError::Storage(format!("Failed to open layer {}: {}", path.display(), e)) + })?; + let len = file + .metadata() + .map_err(|e| { + BoxliteError::Storage(format!("Failed to stat layer {}: {}", path.display(), e)) + })? + .len(); + + let mut head = [0u8; 20]; + let holes = match file.read_exact(&mut head) { + Ok(()) if u32::from_be_bytes(head[0..4].try_into().unwrap()) == 0x5146_49fb => { + let backing_offset = u64::from_be_bytes(head[8..16].try_into().unwrap()); + let backing_size = u32::from_be_bytes(head[16..20].try_into().unwrap()) as u64; + let mut holes = vec![Self::BACKING_SIZE_FIELD]; + if backing_offset != 0 && backing_size != 0 { + holes.push((backing_offset, backing_offset + backing_size)); + } + holes.sort_unstable(); + holes + } + // A raw layer (the image disk) has no header to normalize. + _ => Vec::new(), + }; + + use std::io::Seek; + file.rewind().map_err(|e| { + BoxliteError::Storage(format!("Failed to rewind {}: {}", path.display(), e)) + })?; + + Ok(Self { + file, + len, + pos: 0, + holes, + }) + } + + pub(crate) fn len(&self) -> u64 { + self.len + } + + /// The layer's canonical digest, consuming the reader. + pub(crate) fn digest(mut self) -> BoxliteResult { + use std::io::Read; + + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; 64 * 1024]; + loop { + let n = self + .read(&mut buf) + .map_err(|e| BoxliteError::Storage(format!("Failed to read layer: {}", e)))?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(format!("sha256:{}", hex::encode(hasher.finalize()))) + } +} + +impl std::io::Read for CanonicalLayer { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.file.read(buf)?; + let start = self.pos; + let end = start + n as u64; + + for &(hole_start, hole_end) in &self.holes { + let from = hole_start.max(start); + let to = hole_end.min(end); + if from < to { + let lo = (from - start) as usize; + let hi = (to - start) as usize; + buf[lo..hi].fill(0); + } + } + + self.pos = end; + Ok(n) + } +} + +/// Write the archive as a directory of separately addressed objects. +/// +/// Produces `manifest.json` and `layers/{hex}.zst`, one object per layer, each +/// holding that layer's [`CanonicalLayer`] bytes compressed on its own. The +/// point is that every object is immutable and named by its content, so +/// mirroring the directory to object storage uploads only what is missing — +/// the sync tool's existence check is the whole negotiation. A layer that two +/// exports share is written to the same name and transferred once. +/// +/// The manifest is written last. A reader that finds it can rely on every +/// layer it names already being present, which is what makes an interrupted +/// mirror safe to retry rather than a half-published archive. +pub(crate) fn build_layered_directory( + output_dir: &Path, + manifest_json: &str, + layers: &[(String, std::path::PathBuf)], + compression_level: i32, +) -> BoxliteResult<()> { + write_layer_objects(output_dir, layers, compression_level)?; + + let manifest_path = output_dir.join(MANIFEST_FILENAME); + std::fs::write(&manifest_path, manifest_json).map_err(|e| { + BoxliteError::Storage(format!( + "Failed to write {}: {}", + manifest_path.display(), + e + )) + })?; + + Ok(()) +} + +/// Write each layer as a content-named `.zst` object under `root/layers/`. +/// +/// An object that already exists is left alone — it was put there by an +/// earlier export sharing that layer, which is what makes repeat exports into +/// the same place incremental. New objects are written under a temporary name +/// and renamed, so a reader never sees a half-written object under a name +/// that promises specific content. +fn write_layer_objects( + root: &Path, + layers: &[(String, std::path::PathBuf)], + compression_level: i32, +) -> BoxliteResult<()> { + let layers_dir = root.join(LAYERS_DIR); + std::fs::create_dir_all(&layers_dir).map_err(|e| { + BoxliteError::Storage(format!( + "Failed to create layer directory {}: {}", + layers_dir.display(), + e + )) + })?; + + for (digest, path) in layers { + let object = root.join(format!("{}.zst", layer_entry_name(digest))); + if object.exists() { + tracing::debug!(digest = %digest, "Layer object already written, leaving it"); + continue; + } + + let staging = object.with_extension("zst.partial"); + let mut layer = CanonicalLayer::open(path)?; + let file = std::fs::File::create(&staging).map_err(|e| { + BoxliteError::Storage(format!("Failed to create {}: {}", staging.display(), e)) + })?; + let mut encoder = zstd::Encoder::new(file, compression_level) + .map_err(|e| BoxliteError::Storage(format!("Failed to create zstd encoder: {}", e)))?; + std::io::copy(&mut layer, &mut encoder).map_err(|e| { + BoxliteError::Storage(format!("Failed to write layer {}: {}", digest, e)) + })?; + encoder.finish().map_err(|e| { + BoxliteError::Storage(format!("Failed to finish layer {}: {}", digest, e)) + })?; + move_file(&staging, &object)?; + } + + Ok(()) +} + +/// Directory holding one manifest per archive inside a layer store. +pub(crate) const STORE_ARCHIVES_DIR: &str = "archives"; + +/// Publish an archive into a shared layer store. +/// +/// A store is `archives/.json` manifests over one `layers/` pool, so N +/// archives share every layer they have in common; the manifests are what +/// holds a layer in the pool, and [`crate::ArchiveStore::gc`] sweeps whatever +/// no manifest names. The manifest is written last and lands by rename: a +/// reader that finds it can rely on every layer it names having been written +/// before it. +/// +/// Publishing races the sweeper. A concurrent gc can read the pool between +/// this export's object writes and its manifest landing, see objects no +/// manifest references yet, and — if they are older than its grace period — +/// delete them. The manifest is therefore re-checked after it lands and any +/// swept object is rewritten; the grace period is what makes that window +/// finite, and the repair is what closes it. +pub(crate) fn build_store_archive( + store_root: &Path, + name: &str, + manifest_json: &str, + layers: &[(String, std::path::PathBuf)], + compression_level: i32, +) -> BoxliteResult { + validate_store_archive_name(name)?; + + write_layer_objects(store_root, layers, compression_level)?; + + let archives_dir = store_root.join(STORE_ARCHIVES_DIR); + std::fs::create_dir_all(&archives_dir).map_err(|e| { + BoxliteError::Storage(format!( + "Failed to create {}: {}", + archives_dir.display(), + e + )) + })?; + let manifest_path = archives_dir.join(format!("{name}.json")); + let staging = archives_dir.join(format!(".{name}.json.partial")); + std::fs::write(&staging, manifest_json).map_err(|e| { + BoxliteError::Storage(format!("Failed to write {}: {}", staging.display(), e)) + })?; + move_file(&staging, &manifest_path)?; + + // Repair anything a concurrent sweep took between our object writes and + // the manifest landing. From here on the manifest holds the references, + // so a rewritten object stays. + for (digest, path) in layers { + let object = store_root.join(format!("{}.zst", layer_entry_name(digest))); + if !object.exists() { + tracing::warn!(digest = %digest, "Layer object swept mid-publish; rewriting"); + write_layer_objects( + store_root, + std::slice::from_ref(&(digest.clone(), path.clone())), + compression_level, + )?; + } + } + + Ok(manifest_path) +} + +/// Refuse an archive name that could escape `archives/` or collide with the +/// staging convention. +pub(crate) fn validate_store_archive_name(name: &str) -> BoxliteResult<()> { + let bad = name.is_empty() + || name.starts_with('.') + || name + .chars() + .any(|c| c == '/' || c == '\\' || c == ':' || c.is_control()); + if bad { + return Err(BoxliteError::InvalidArgument(format!( + "invalid archive name {name:?}: must be non-empty, not start with '.', and contain no path separators" + ))); + } + Ok(()) +} + +/// Build a zstd-compressed tar archive holding a manifest and layer blobs. +/// +/// `layers` pairs each layer's digest with the file to read it from, in the +/// same order as the manifest's layer list. Each layer travels in its +/// [`CanonicalLayer`] form. +pub(crate) fn build_layered_archive( output_path: &Path, manifest_path: &Path, - container_disk: &Path, - guest_disk: Option<&Path>, + layers: &[(String, std::path::PathBuf)], compression_level: i32, ) -> BoxliteResult<()> { let file = std::fs::File::create(output_path).map_err(|e| { @@ -94,44 +430,32 @@ pub(crate) fn build_zstd_tar_archive( let encoder = zstd::Encoder::new(file, compression_level) .map_err(|e| BoxliteError::Storage(format!("Failed to create zstd encoder: {}", e)))?; - let mut builder = tar::Builder::new(encoder); - append_archive_files(&mut builder, manifest_path, container_disk, guest_disk)?; - let encoder = builder - .into_inner() - .map_err(|e| BoxliteError::Storage(format!("Failed to finalize tar: {}", e)))?; - encoder - .finish() - .map_err(|e| BoxliteError::Storage(format!("Failed to finish zstd compression: {}", e)))?; - - Ok(()) -} - -fn append_archive_files( - builder: &mut tar::Builder, - manifest_path: &Path, - container_disk: &Path, - guest_disk: Option<&Path>, -) -> BoxliteResult<()> { builder .append_path_with_name(manifest_path, MANIFEST_FILENAME) .map_err(|e| BoxliteError::Storage(format!("Failed to add manifest to archive: {}", e)))?; - builder - .append_path_with_name(container_disk, disk_filenames::CONTAINER_DISK) - .map_err(|e| { - BoxliteError::Storage(format!("Failed to add container disk to archive: {}", e)) - })?; - - if let Some(guest) = guest_disk { + for (digest, path) in layers { + let layer = CanonicalLayer::open(path)?; + let mut header = tar::Header::new_gnu(); + header.set_size(layer.len()); + header.set_mode(0o600); + header.set_cksum(); builder - .append_path_with_name(guest, disk_filenames::GUEST_ROOTFS_DISK) + .append_data(&mut header, layer_entry_name(digest), layer) .map_err(|e| { - BoxliteError::Storage(format!("Failed to add guest rootfs disk to archive: {}", e)) + BoxliteError::Storage(format!("Failed to add layer {} to archive: {}", digest, e)) })?; } + let encoder = builder + .into_inner() + .map_err(|e| BoxliteError::Storage(format!("Failed to finalize tar: {}", e)))?; + encoder + .finish() + .map_err(|e| BoxliteError::Storage(format!("Failed to finish zstd compression: {}", e)))?; + Ok(()) } @@ -139,6 +463,7 @@ fn append_archive_files( /// Zstd magic bytes: `0x28B52FFD` (little-endian in file). const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD]; +pub(crate) const MAX_ARCHIVE_OUTPUT: u64 = 128 * 1024 * 1024 * 1024; /// Extract an archive, detecting format via magic bytes (zstd or plain tar). pub(crate) fn extract_archive(archive_path: &Path, dest_dir: &Path) -> BoxliteResult<()> { @@ -179,9 +504,11 @@ pub(crate) fn extract_archive(archive_path: &Path, dest_dir: &Path) -> BoxliteRe } fn extract_zstd_tar(file: std::fs::File, dest_dir: &Path) -> BoxliteResult<()> { + use std::io::Read; + let decoder = zstd::Decoder::new(file) .map_err(|e| BoxliteError::Storage(format!("Failed to create zstd decoder: {}", e)))?; - let mut archive = tar::Archive::new(decoder); + let mut archive = tar::Archive::new(decoder.take(MAX_ARCHIVE_OUTPUT.saturating_add(1))); archive .unpack(dest_dir) .map_err(|e| BoxliteError::Storage(format!("Failed to extract zstd tar: {}", e)))?; @@ -263,6 +590,59 @@ pub(crate) fn sha256_file(path: &Path) -> BoxliteResult { Ok(format!("sha256:{}", hex::encode(hasher.finalize()))) } +/// Decompress one layer object from a mirrored archive directory. +/// +/// Only called for a layer the importer has decided it actually needs, which +/// is the point of the directory form: an object the host already holds is +/// never read, let alone decompressed. +pub(crate) fn extract_layer_object( + archive_dir: &Path, + digest: &str, + dest: &Path, + virtual_size: u64, + remaining_output: u64, +) -> BoxliteResult { + use std::io::Read; + + let object = archive_dir.join(format!("{}.zst", layer_entry_name(digest))); + let file = std::fs::File::open(&object).map_err(|e| { + BoxliteError::Storage(format!( + "Archive directory is missing layer {}: {}", + object.display(), + e + )) + })?; + let layer_limit = if virtual_size > 0 { + virtual_size + } else { + MAX_ARCHIVE_OUTPUT + }; + let max_output = layer_limit.min(remaining_output); + let mut decoder = zstd::Decoder::new(file) + .map_err(|e| BoxliteError::Storage(format!("Failed to read layer {}: {}", digest, e)))?; + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent).map_err(|e| { + BoxliteError::Storage(format!("Failed to create {}: {}", parent.display(), e)) + })?; + } + let mut out = std::fs::File::create(dest).map_err(|e| { + BoxliteError::Storage(format!("Failed to create {}: {}", dest.display(), e)) + })?; + let written = std::io::copy( + &mut decoder.by_ref().take(max_output.saturating_add(1)), + &mut out, + ) + .map_err(|e| BoxliteError::Storage(format!("Failed to unpack layer {}: {}", digest, e)))?; + if written > max_output { + drop(out); + let _ = std::fs::remove_file(dest); + return Err(BoxliteError::Storage(format!( + "Layer {digest} exceeds its decompression limit" + ))); + } + Ok(written) +} + #[cfg(test)] mod tests { use super::*; @@ -411,6 +791,23 @@ mod tests { assert_eq!(content, "hello from plain tar"); } + #[test] + fn layer_object_decompression_is_bounded() { + let dir = tempdir().unwrap(); + let archive_dir = dir.path().join("archive"); + std::fs::create_dir_all(archive_dir.join(LAYERS_DIR)).unwrap(); + let digest = format!("sha256:{}", "a".repeat(64)); + let object = archive_dir.join(format!("{}.zst", layer_entry_name(&digest))); + std::fs::write(&object, zstd::encode_all(&b"too large"[..], 3).unwrap()).unwrap(); + let dest = dir.path().join("layer"); + + let error = extract_layer_object(&archive_dir, &digest, &dest, 1, MAX_ARCHIVE_OUTPUT) + .expect_err("decompression must stop at the declared virtual size"); + + assert!(error.to_string().contains("decompression limit")); + assert!(!dest.exists(), "a rejected partial layer must be removed"); + } + #[test] fn test_move_file_same_filesystem() { let dir = tempdir().unwrap(); @@ -452,18 +849,33 @@ mod tests { let extract_dir = dir.path().join("extracted"); std::fs::create_dir_all(&extract_dir).unwrap(); - // Create test files let manifest_path = dir.path().join(MANIFEST_FILENAME); - let container_path = dir.path().join("container.qcow2"); - std::fs::write(&manifest_path, r#"{"version":2}"#).unwrap(); - std::fs::write(&container_path, "fake-container-disk").unwrap(); - - build_zstd_tar_archive(&archive_path, &manifest_path, &container_path, None, 3).unwrap(); + let base = dir.path().join("base.bin"); + let top = dir.path().join("top.bin"); + std::fs::write(&manifest_path, r#"{"version":6}"#).unwrap(); + std::fs::write(&base, "fake-base-layer").unwrap(); + std::fs::write(&top, "fake-top-layer").unwrap(); + + let layers = vec![ + ("sha256:aaa".to_string(), base), + ("sha256:bbb".to_string(), top), + ]; + build_layered_archive(&archive_path, &manifest_path, &layers, 3).unwrap(); extract_archive(&archive_path, &extract_dir).unwrap(); assert_eq!( std::fs::read_to_string(extract_dir.join(MANIFEST_FILENAME)).unwrap(), - r#"{"version":2}"# + r#"{"version":6}"# + ); + // Each layer lands under the name its digest implies, which is how the + // importer finds a blob it only knows by content. + assert_eq!( + std::fs::read_to_string(extract_dir.join(layer_entry_name("sha256:aaa"))).unwrap(), + "fake-base-layer" + ); + assert_eq!( + std::fs::read_to_string(extract_dir.join(layer_entry_name("sha256:bbb"))).unwrap(), + "fake-top-layer" ); } } diff --git a/src/boxlite/src/litebox/archive_store.rs b/src/boxlite/src/litebox/archive_store.rs new file mode 100644 index 000000000..541085869 --- /dev/null +++ b/src/boxlite/src/litebox/archive_store.rs @@ -0,0 +1,351 @@ +//! A shared layer store: many archives over one content-addressed pool. +//! +//! The layout is the one an incremental mirror wants: +//! +//! ```text +//! /archives/.json one manifest per archive — the references +//! /layers/.zst shared pool, named by content +//! ``` +//! +//! Archives of different boxes share every layer they have in common, so the +//! pool grows by what is new, not by what is exported. What holds a layer in +//! the pool is the manifests that name it; [`ArchiveStore::gc`] sweeps the +//! rest. Deleting an archive is deleting its manifest — the bytes it pinned +//! stay until a sweep finds them unreferenced. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use boxlite_shared::errors::{BoxliteError, BoxliteResult}; + +use super::archive::{ + ArchiveManifest, LAYERS_DIR, MANIFEST_FILENAME, STORE_ARCHIVES_DIR, layer_entry_name, + validate_store_archive_name, +}; + +/// What a sweep did, and what it left alone. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct GcReport { + /// Unreferenced objects removed. + pub swept: usize, + /// Bytes those objects held. + pub bytes_freed: u64, + /// Unreferenced objects left in place because they are younger than the + /// grace period — possibly a publish in flight whose manifest has not + /// landed yet. + pub kept_in_grace: usize, +} + +/// A handle on a shared layer store. +/// +/// Purely a view over a directory: opening one creates nothing and holds no +/// lock. Publishing into a store is an export with +/// `ExportOptions { as_directory: true, archive_name: Some(..) }` whose +/// destination is the store root. +pub struct ArchiveStore { + root: PathBuf, +} + +impl ArchiveStore { + /// Open a store rooted at `root`. + /// + /// Refuses a single-archive directory (a root `manifest.json`): its + /// layers all belong to that one archive, and sweeping it with an empty + /// `archives/` beside it would destroy them. + pub fn open(root: impl Into) -> BoxliteResult { + let root = root.into(); + if root.join(MANIFEST_FILENAME).exists() { + return Err(BoxliteError::InvalidArgument(format!( + "{} is a single-archive directory, not a store: it has a root {}", + root.display(), + MANIFEST_FILENAME + ))); + } + Ok(Self { root }) + } + + /// Names of the archives published here, in no particular order. + pub fn archives(&self) -> BoxliteResult> { + let dir = self.root.join(STORE_ARCHIVES_DIR); + if !dir.is_dir() { + return Ok(Vec::new()); + } + let mut names = Vec::new(); + for entry in read_dir(&dir)? { + let path = entry?.path(); + if path.extension().is_some_and(|e| e == "json") + && let Some(stem) = path.file_stem().and_then(|s| s.to_str()) + && !stem.starts_with('.') + { + names.push(stem.to_string()); + } + } + Ok(names) + } + + /// Path of a published archive's manifest — the value to hand to import. + pub fn archive_path(&self, name: &str) -> BoxliteResult { + validate_store_archive_name(name)?; + Ok(self + .root + .join(STORE_ARCHIVES_DIR) + .join(format!("{name}.json"))) + } + + /// Drop an archive's manifest, releasing its hold on the pool. + /// + /// Returns whether it existed. The layers it pinned stay on disk until + /// [`gc`](Self::gc) finds them unreferenced — removal is cheap and safe, + /// reclamation is the sweep's job. + pub fn remove(&self, name: &str) -> BoxliteResult { + let path = self.archive_path(name)?; + match std::fs::remove_file(&path) { + Ok(()) => Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(BoxliteError::Storage(format!( + "Failed to remove {}: {}", + path.display(), + e + ))), + } + } + + /// Sweep pool objects no manifest references. + /// + /// Fails closed: a manifest that cannot be parsed aborts the sweep, since + /// its references are unknown and anything deleted might be them. Objects + /// younger than `grace` are kept even when unreferenced — a publish writes + /// its objects before its manifest, so a sweep that runs inside that + /// window sees layers nothing names yet. The grace period must exceed the + /// longest publish that can run concurrently; the publisher additionally + /// re-checks its objects after the manifest lands and rewrites anything a + /// sweep took. + /// + /// Only content-named `.zst` objects and stale `.partial` staging files + /// are candidates; anything else in the pool directory is left untouched. + pub fn gc(&self, grace: Duration) -> BoxliteResult { + let archives_dir = self.root.join(STORE_ARCHIVES_DIR); + if !archives_dir.is_dir() { + return Err(BoxliteError::InvalidArgument(format!( + "{} is not an archive store: it has no {}/ directory", + self.root.display(), + STORE_ARCHIVES_DIR + ))); + } + + let mut referenced: HashSet = HashSet::new(); + for entry in read_dir(&archives_dir)? { + let path = entry?.path(); + if !path.extension().is_some_and(|e| e == "json") { + continue; + } + let text = std::fs::read_to_string(&path).map_err(|e| { + BoxliteError::Storage(format!("Failed to read {}: {}", path.display(), e)) + })?; + let manifest: ArchiveManifest = serde_json::from_str(&text).map_err(|e| { + BoxliteError::Storage(format!( + "Refusing to sweep: {} is not a readable manifest ({}); its references are unknown", + path.display(), + e + )) + })?; + for layer in &manifest.layers { + referenced.insert( + self.root + .join(format!("{}.zst", layer_entry_name(&layer.digest))), + ); + } + } + + let layers_dir = self.root.join(LAYERS_DIR); + let mut report = GcReport::default(); + if !layers_dir.is_dir() { + return Ok(report); + } + let now = std::time::SystemTime::now(); + for entry in read_dir(&layers_dir)? { + let path = entry?.path(); + let name = match path.file_name().and_then(|n| n.to_str()) { + Some(n) => n, + None => continue, + }; + let sweepable = is_object_name(name) || name.ends_with(".partial"); + if !sweepable || referenced.contains(&path) { + continue; + } + let meta = match std::fs::metadata(&path) { + Ok(m) => m, + // Raced with another sweeper or a publisher's rename. + Err(_) => continue, + }; + let age = meta + .modified() + .ok() + .and_then(|m| now.duration_since(m).ok()) + .unwrap_or(Duration::ZERO); + if age < grace { + report.kept_in_grace += 1; + continue; + } + match std::fs::remove_file(&path) { + Ok(()) => { + report.swept += 1; + report.bytes_freed += meta.len(); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(BoxliteError::Storage(format!( + "Failed to sweep {}: {}", + path.display(), + e + ))); + } + } + } + Ok(report) + } +} + +/// Whether a pool filename is a content-named object (`.zst`). +fn is_object_name(name: &str) -> bool { + name.strip_suffix(".zst") + .is_some_and(|hex| !hex.is_empty() && hex.chars().all(|c| c.is_ascii_hexdigit())) +} + +fn read_dir(dir: &Path) -> BoxliteResult { + std::fs::read_dir(dir) + .map_err(|e| BoxliteError::Storage(format!("Failed to read {}: {}", dir.display(), e))) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A store with hand-written manifests and objects, no boxes involved. + fn store(root: &Path) -> ArchiveStore { + std::fs::create_dir_all(root.join(STORE_ARCHIVES_DIR)).unwrap(); + std::fs::create_dir_all(root.join(LAYERS_DIR)).unwrap(); + ArchiveStore::open(root).unwrap() + } + + fn publish(root: &Path, name: &str, digests: &[&str]) { + for d in digests { + std::fs::write(root.join(LAYERS_DIR).join(format!("{d}.zst")), d).unwrap(); + } + let layers: Vec = digests + .iter() + .map(|d| format!(r#"{{"digest":"sha256:{d}","format":"qcow2"}}"#)) + .collect(); + let manifest = format!( + r#"{{"version":6,"box_name":null,"image":"t","guest_disk_checksum":"","container_disk_checksum":"","layers":[{}],"exported_at":"2026-07-31T00:00:00Z"}}"#, + layers.join(",") + ); + std::fs::write( + root.join(STORE_ARCHIVES_DIR).join(format!("{name}.json")), + manifest, + ) + .unwrap(); + } + + fn pool(root: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(root.join(LAYERS_DIR)) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + v.sort(); + v + } + + /// The pool holds the union; a layer stays while any manifest names it. + #[test] + fn a_layer_stays_while_any_archive_references_it() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let root = temp.path(); + let s = store(root); + publish(root, "one", &["aa", "bb"]); + publish(root, "two", &["aa", "cc"]); + + assert!(s.remove("two").unwrap()); + let report = s.gc(Duration::ZERO).unwrap(); + assert_eq!(report.swept, 1, "only the layer unique to `two` goes"); + assert_eq!(pool(root), vec!["aa.zst", "bb.zst"]); + + assert!(s.remove("one").unwrap()); + let report = s.gc(Duration::ZERO).unwrap(); + assert_eq!(report.swept, 2); + assert!(pool(root).is_empty()); + assert!(!s.remove("one").unwrap(), "second removal reports absence"); + } + + /// Unreferenced but young objects survive: a publish in flight writes its + /// objects before its manifest, and the sweeper must not eat them. + #[test] + fn an_unreferenced_object_inside_the_grace_period_is_kept() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let root = temp.path(); + let s = store(root); + std::fs::write(root.join(LAYERS_DIR).join("dd.zst"), "orphan").unwrap(); + + let report = s.gc(Duration::from_secs(3600)).unwrap(); + assert_eq!((report.swept, report.kept_in_grace), (0, 1)); + assert_eq!(pool(root), vec!["dd.zst"]); + + let report = s.gc(Duration::ZERO).unwrap(); + assert_eq!(report.swept, 1); + } + + /// A manifest that cannot be parsed aborts the sweep entirely: its + /// references are unknown, so nothing is safe to delete. + #[test] + fn a_corrupt_manifest_aborts_the_sweep() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let root = temp.path(); + let s = store(root); + publish(root, "good", &["aa"]); + std::fs::write(root.join(LAYERS_DIR).join("ee.zst"), "orphan").unwrap(); + std::fs::write( + root.join(STORE_ARCHIVES_DIR).join("bad.json"), + "not a manifest", + ) + .unwrap(); + + let err = s.gc(Duration::ZERO).unwrap_err(); + assert!(err.to_string().contains("Refusing to sweep"), "{err}"); + assert_eq!(pool(root), vec!["aa.zst", "ee.zst"], "nothing was deleted"); + } + + /// Files that are not content-named objects are never sweep candidates. + #[test] + fn foreign_files_in_the_pool_are_left_alone() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let root = temp.path(); + let s = store(root); + std::fs::write(root.join(LAYERS_DIR).join("README.txt"), "notes").unwrap(); + std::fs::write(root.join(LAYERS_DIR).join("zz.zst.partial"), "stale").unwrap(); + + let report = s.gc(Duration::ZERO).unwrap(); + assert_eq!(report.swept, 1, "only the stale staging file goes"); + assert_eq!(pool(root), vec!["README.txt"]); + } + + /// A single-archive directory must not be opened as a store: an empty + /// archives/ beside a root manifest would make every layer look orphaned. + #[test] + fn a_single_archive_directory_is_refused() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let root = temp.path(); + std::fs::write(root.join(MANIFEST_FILENAME), "{}").unwrap(); + assert!(ArchiveStore::open(root).is_err()); + } + + /// Archive names are path components, nothing more. + #[test] + fn a_path_escaping_archive_name_is_refused() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let s = store(temp.path()); + for bad in ["../evil", "a/b", "", ".hidden", "a\\b"] { + assert!(s.archive_path(bad).is_err(), "{bad:?} must be refused"); + } + } +} diff --git a/src/boxlite/src/litebox/box_impl.rs b/src/boxlite/src/litebox/box_impl.rs index 18a375424..91090818b 100644 --- a/src/boxlite/src/litebox/box_impl.rs +++ b/src/boxlite/src/litebox/box_impl.rs @@ -106,6 +106,50 @@ impl LiveState { } } +/// How long to wait for the guest to freeze its filesystems. +/// +/// `FIFREEZE` does not fail under write load — it blocks until the filesystem +/// has flushed, so a busy guest simply takes longer. The old 5s was short +/// enough that a moderately busy box would time out routinely, which under +/// [`QuiescePolicy::RequireFrozen`] would turn into a refused export; 30s +/// still produced spurious refusals when the host itself was saturated +/// (observed with four VMs booting in parallel: the running-box export flaked +/// while a lone run passed). An export is not latency-sensitive, so the +/// ceiling is generous — it exists only to bound a guest that is wedged or +/// has no agent, not to keep a busy one on schedule. +const GUEST_QUIESCE_TIMEOUT: Duration = Duration::from_secs(60); + +/// Decide whether an operation may proceed given how the freeze went. +/// +/// Split out from the quiesce bracket so the refusal contract — which error +/// class, and whether the message tells the caller what to do about it — is +/// testable without a running VM. +fn ensure_frozen_enough(box_id: &BoxID, frozen: bool, policy: QuiescePolicy) -> BoxliteResult<()> { + if frozen || policy == QuiescePolicy::BestEffort { + return Ok(()); + } + Err(BoxliteError::InvalidState(format!( + "Cannot export box {}: the guest did not freeze its filesystems within {}s, so the \ + archive would only be crash-consistent — the disk equivalent of pulling the power cord, \ + with the guest's unwritten page cache lost. Stop the box and export it again for a \ + consistent archive.", + box_id, + GUEST_QUIESCE_TIMEOUT.as_secs() + ))) +} + +/// What a failed guest freeze means for the operation being wrapped. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum QuiescePolicy { + /// Carry on without a freeze. The disk view is crash-consistent — as if the + /// machine lost power — which is acceptable when the result is a fresh COW + /// fork the caller is about to boot anyway. + BestEffort, + /// Abandon the operation if the guest will not freeze, rather than hand back + /// a crash-consistent result that looks indistinguishable from a good one. + RequireFrozen, +} + // ============================================================================ // BOX IMPL // ============================================================================ @@ -1175,6 +1219,23 @@ impl BoxImpl { /// Guest RPCs are best-effort with timeout — failure degrades to /// crash-consistent (SIGSTOP-only), not operation failure. pub(crate) async fn with_quiesce_async(&self, fut: Fut) -> BoxliteResult + where + Fut: Future>, + { + self.with_quiesce_policy(QuiescePolicy::BestEffort, fut) + .await + } + + /// `with_quiesce_async`, but the caller chooses what a failed freeze means. + /// + /// With [`QuiescePolicy::RequireFrozen`] the operation is abandoned before + /// the VM is even stopped, so a caller that needs a filesystem-consistent + /// view never silently receives a crash-consistent one. + pub(crate) async fn with_quiesce_policy( + &self, + policy: QuiescePolicy, + fut: Fut, + ) -> BoxliteResult where Fut: Future>, { @@ -1201,11 +1262,18 @@ impl BoxImpl { let t0 = Instant::now(); - // Phase 1: Freeze guest I/O (best-effort, 5s timeout) + // Phase 1: Freeze guest I/O let t_quiesce = Instant::now(); let frozen = self.guest_quiesce().await; let quiesce_ms = t_quiesce.elapsed().as_millis() as u64; + // A timed-out quiesce may have frozen the guest before its reply was + // dropped. Thaw before a strict policy refuses the operation. + if !frozen && policy == QuiescePolicy::RequireFrozen { + self.guest_thaw().await; + } + ensure_frozen_enough(&self.config.id, frozen, policy)?; + // Phase 2: SIGSTOP — pause vCPUs // SAFETY: sending SIGSTOP to a known valid PID that we own (shim process). let ret = unsafe { libc::kill(pid, libc::SIGSTOP) }; @@ -1271,7 +1339,7 @@ impl BoxImpl { return false; }; - let result = tokio::time::timeout(Duration::from_secs(5), async { + let result = tokio::time::timeout(GUEST_QUIESCE_TIMEOUT, async { let mut guest = live.guest_session.guest().await?; guest.quiesce().await }) @@ -1442,6 +1510,50 @@ mod tests { use chrono::Utc; use tempfile::TempDir; + /// An export whose freeze failed must be refused, not silently downgraded: + /// a crash-consistent archive is indistinguishable from a good one. + #[test] + fn a_failed_freeze_refuses_the_export() { + let id = BoxIDMint::mint(); + let err = ensure_frozen_enough(&id, false, QuiescePolicy::RequireFrozen) + .expect_err("an unfrozen guest must not yield an archive"); + + assert!( + matches!(err, BoxliteError::InvalidState(_)), + "expected InvalidState, got {err:?}" + ); + let msg = err.to_string(); + // The caller can only act on this if the message says what to do. + assert!( + msg.contains("crash-consistent"), + "message must name the hazard: {msg}" + ); + assert!( + msg.contains("Stop the box"), + "message must state the remedy: {msg}" + ); + assert!( + msg.contains(&GUEST_QUIESCE_TIMEOUT.as_secs().to_string()), + "message must state how long it waited: {msg}" + ); + } + + /// Clone and snapshot fork a disk the caller boots straight away, so they + /// keep the old behaviour rather than failing under write load. + #[test] + fn best_effort_still_proceeds_without_a_freeze() { + let id = BoxIDMint::mint(); + ensure_frozen_enough(&id, false, QuiescePolicy::BestEffort) + .expect("best-effort must tolerate an unfrozen guest"); + } + + #[test] + fn a_successful_freeze_proceeds_under_either_policy() { + let id = BoxIDMint::mint(); + ensure_frozen_enough(&id, true, QuiescePolicy::RequireFrozen).expect("frozen is enough"); + ensure_frozen_enough(&id, true, QuiescePolicy::BestEffort).expect("frozen is enough"); + } + fn published_ports(info: &BoxInfo) -> Option<&[PublishedPort]> { info.network .as_ref() diff --git a/src/boxlite/src/litebox/clone_export.rs b/src/boxlite/src/litebox/clone_export.rs index 15fe3f8d9..4b88781f2 100644 --- a/src/boxlite/src/litebox/clone_export.rs +++ b/src/boxlite/src/litebox/clone_export.rs @@ -5,7 +5,7 @@ use std::time::Instant; use boxlite_shared::errors::{BoxliteError, BoxliteResult}; -use super::box_impl::BoxImpl; +use super::box_impl::{BoxImpl, QuiescePolicy}; use crate::disk::BaseDiskKind; use crate::disk::constants::filenames as disk_filenames; use crate::disk::{BackingFormat, Qcow2Helper}; @@ -172,7 +172,7 @@ impl BoxImpl { pub(crate) async fn export_box( &self, - _options: crate::runtime::options::ExportOptions, + options: crate::runtime::options::ExportOptions, dest: &std::path::Path, ) -> BoxliteResult { let t0 = Instant::now(); @@ -181,34 +181,62 @@ impl BoxImpl { let box_home = self.config.box_home.clone(); let runtime_layout = self.runtime.layout.clone(); - // Phase 1: Flatten disks inside quiesce bracket (VM paused only for this). - // Flatten reads live qcow2 chains and must see consistent disk state. - let flatten_result = self - .with_quiesce_async(async { + // Phase 1: Capture the chain inside the quiesce bracket (VM paused). + // Only the top overlay is live, so only it has to be copied; the bases + // below it are immutable and are read in place at archive time. + // + // An archive is expected to restore into a working box, so a failed + // freeze is refused rather than silently downgraded: SIGSTOP alone + // pauses the vCPUs but leaves the guest's dirty page cache unwritten, + // producing the disk equivalent of pulling the power cord. That archive + // is indistinguishable from a good one, which makes it worse than no + // archive at all. Clone and snapshot keep the best-effort policy — + // their output is a COW fork the caller boots immediately, not an + // artifact someone will restore from months later. + let capture = self + .with_quiesce_policy(QuiescePolicy::RequireFrozen, async { let bh = box_home.clone(); let rl = runtime_layout.clone(); - tokio::task::spawn_blocking(move || do_export_flatten(&bh, &rl)) + tokio::task::spawn_blocking(move || do_export_capture(&bh, &rl)) .await .map_err(|e| { - BoxliteError::Internal(format!("Export flatten task panicked: {}", e)) + BoxliteError::Internal(format!("Export capture task panicked: {}", e)) })? }) .await?; - // Phase 2: Checksum + manifest + archive run with VM resumed. - // These only read static temp files, no disk consistency needed. + // Phase 2: Digest + manifest + archive run with the VM resumed. Every + // input is now either a temp copy or an immutable base. let config_name = self.config.name.clone(); let config_options = self.config.options.clone(); let box_id_str = self.id().to_string(); let dest = dest.to_path_buf(); + let as_directory = options.as_directory; + let archive_name = options.archive_name.clone(); + // A store publish is a directory-form export with a different landing + // spot for the manifest; without as_directory the name has no meaning. + if archive_name.is_some() && !as_directory { + return Err(BoxliteError::InvalidArgument( + "archive_name requires as_directory: a store is a directory of layer objects" + .into(), + )); + } + let base_disk_mgr = self.runtime.base_disk_mgr.clone(); + let image_disks_dir = self.runtime.layout.image_layout().disk_images_dir(); let result = tokio::task::spawn_blocking(move || { do_export_finalize( - flatten_result, + capture, + &base_disk_mgr, + &image_disks_dir, config_name.as_deref(), &config_options, &box_id_str, - &dest, + match (archive_name.as_deref(), as_directory) { + (Some(name), _) => ExportDest::Store { root: &dest, name }, + (None, true) => ExportDest::Directory(&dest), + (None, false) => ExportDest::File(&dest), + }, ) }) .await @@ -225,26 +253,39 @@ impl BoxImpl { } } -/// Intermediate result from flatten phase, passed to finalize phase. -struct FlattenResult { +/// The box's disk chain as captured under quiesce, base first and top last. +struct ChainCapture { temp_dir: tempfile::TempDir, - flat_container: std::path::PathBuf, - flat_guest: Option, - flatten_ms: u64, + /// Files to read each layer from. The last entry is a temp copy of the + /// live top overlay; the rest are immutable bases read in place. + layer_paths: Vec, + capture_ms: u64, } -/// Phase 1: Flatten qcow2 disk chains into standalone images. +/// Phase 1: Capture the container disk's layer chain. /// Runs inside the quiesce bracket — this is the only part that needs disk consistency. -fn do_export_flatten( +/// +/// The chain is exported as layers rather than flattened into one image. The +/// layers below the top are immutable and shared: every box's container disk is +/// a COW child of the image disk, so the image layer is identical across every +/// box built from it and an importer that already holds it skips the transfer +/// entirely. Flattening would erase exactly that structure, and measured on a +/// real box it does not even buy a smaller archive. +/// +/// The guest rootfs disk is deliberately not exported. It is a thin COW overlay +/// over the host-global guest rootfs cache (`bases/{id}.ext4`, keyed by the +/// bootstrap image + guest binary version), holds no user state, and is +/// recreated from the importing host's own cache on first start — the same way +/// clone and snapshot-restore already treat it. +fn do_export_capture( box_home: &std::path::Path, runtime_layout: &crate::runtime::layout::FilesystemLayout, -) -> BoxliteResult { - use crate::disk::Qcow2Helper; +) -> BoxliteResult { use crate::disk::constants::filenames as disk_filenames; + use crate::disk::{read_backing_chain, read_backing_file_path}; let disks_dir = box_home.join("disks"); let container_disk = disks_dir.join(disk_filenames::CONTAINER_DISK); - let guest_disk = disks_dir.join(disk_filenames::GUEST_ROOTFS_DISK); if !container_disk.exists() { return Err(BoxliteError::Storage(format!( @@ -256,55 +297,155 @@ fn do_export_flatten( let temp_dir = tempfile::tempdir_in(runtime_layout.temp_dir()) .map_err(|e| BoxliteError::Storage(format!("Failed to create temp directory: {}", e)))?; - let t_flatten = Instant::now(); - let flat_container = temp_dir.path().join(disk_filenames::CONTAINER_DISK); - Qcow2Helper::flatten(&container_disk, &flat_container)?; + let t_capture = Instant::now(); + + // Only the top overlay can still be written to, so it is the only layer + // that has to be copied while the VM is paused. + let top_copy = temp_dir.path().join(disk_filenames::CONTAINER_DISK); + std::fs::copy(&container_disk, &top_copy).map_err(|e| { + BoxliteError::Storage(format!( + "Failed to copy container disk {}: {}", + container_disk.display(), + e + )) + })?; + + // read_backing_chain yields the backing files below `container_disk`, + // nearest first, so reversing puts the deepest base at index 0. + let chain = read_backing_chain(&container_disk); + let deepest = chain.last().unwrap_or(&container_disk); + if is_qcow2(deepest) && read_backing_file_path(deepest)?.is_some() { + return Err(BoxliteError::Storage(format!( + "Cannot export {}: backing chain is incomplete", + container_disk.display() + ))); + } + let mut layer_paths: Vec = chain.into_iter().rev().collect(); + layer_paths.push(top_copy); - let flat_guest = if guest_disk.exists() { - let flat = temp_dir.path().join(disk_filenames::GUEST_ROOTFS_DISK); - Qcow2Helper::flatten(&guest_disk, &flat)?; - Some(flat) - } else { - None - }; - let flatten_ms = t_flatten.elapsed().as_millis() as u64; + let capture_ms = t_capture.elapsed().as_millis() as u64; - Ok(FlattenResult { + Ok(ChainCapture { temp_dir, - flat_container, - flat_guest, - flatten_ms, + layer_paths, + capture_ms, }) } +/// The OCI image digest a layer is the disk for, if it is one. +/// +/// Image disks live in the image cache under a filename derived from the image +/// digest (`images/image_disk.rs`), so the digest is read back from the path +/// rather than by resolving the image again — export must not depend on the +/// registry being reachable. +fn image_digest_of(path: &std::path::Path, image_disks_dir: &std::path::Path) -> Option { + let path = path.canonicalize().ok()?; + let image_disks_dir = image_disks_dir.canonicalize().ok()?; + if path.parent() != Some(image_disks_dir.as_path()) { + return None; + } + let stem = path.file_stem()?.to_str()?; + // `sha256:` is stored as `sha256-.ext4`. + let (algo, hex) = stem.split_once('-')?; + if algo != "sha256" || hex.is_empty() { + return None; + } + Some(format!("{algo}:{hex}")) +} + +/// Whether a file starts with the qcow2 magic, deciding how a child references it. +fn is_qcow2(path: &std::path::Path) -> bool { + use std::io::Read; + let Ok(mut f) = std::fs::File::open(path) else { + return false; + }; + let mut magic = [0u8; 4]; + f.read_exact(&mut magic).is_ok() && u32::from_be_bytes(magic) == 0x5146_49fb +} + +/// Where an export lands, and in which form. +enum ExportDest<'a> { + /// One `.boxlite` file; a directory here means "name the file inside it". + File(&'a std::path::Path), + /// A mirrorable directory of layer objects, used exactly as given. + Directory(&'a std::path::Path), + /// A shared layer store: the manifest lands at `archives/.json` + /// and the layers join the store's pool. + Store { + root: &'a std::path::Path, + name: &'a str, + }, +} + /// Phase 2: Checksum, manifest, and archive. /// Runs after the VM resumes — only reads static temp files. fn do_export_finalize( - flatten: FlattenResult, + capture: ChainCapture, + base_disk_mgr: &crate::disk::BaseDiskManager, + image_disks_dir: &std::path::Path, config_name: Option<&str>, config_options: &crate::runtime::options::BoxOptions, box_id_str: &str, - dest: &std::path::Path, + dest: ExportDest<'_>, ) -> BoxliteResult { use super::archive::{ - ArchiveManifest, MANIFEST_FILENAME, archive_version_for_options, build_zstd_tar_archive, - sha256_file, + ArchiveLayer, ArchiveManifest, CanonicalLayer, LAYERED_ARCHIVE_VERSION, LayerFormat, + MANIFEST_FILENAME, archive_version_for_options, build_layered_archive, + build_layered_directory, build_store_archive, }; + use crate::disk::Qcow2Helper; - let output_path = if dest.is_dir() { - let name = config_name.unwrap_or("box"); - dest.join(format!("{}.boxlite", name)) - } else { - dest.to_path_buf() + // In directory mode `dest` *is* the directory to mirror, so it is used as + // given — appending a name would bury the layout a level down and break + // repeat exports into the same place, which is what makes the transfer + // incremental. + let output_path = match dest { + ExportDest::Directory(dir) | ExportDest::Store { root: dir, .. } => dir.to_path_buf(), + ExportDest::File(path) if path.is_dir() => { + let name = config_name.unwrap_or("box"); + path.join(format!("{}.boxlite", name)) + } + ExportDest::File(path) => path.to_path_buf(), }; - let t_checksum = Instant::now(); - let container_disk_checksum = sha256_file(&flatten.flat_container)?; - let guest_disk_checksum = match flatten.flat_guest { - Some(ref fg) => sha256_file(fg)?, - None => String::new(), - }; - let checksum_ms = t_checksum.elapsed().as_millis() as u64; + let t_digest = Instant::now(); + let last = capture.layer_paths.len().saturating_sub(1); + let mut layers = Vec::with_capacity(capture.layer_paths.len()); + let mut blobs = Vec::with_capacity(capture.layer_paths.len()); + + for (i, path) in capture.layer_paths.iter().enumerate() { + // Every layer below the top is immutable, so its digest is cached and a + // repeat export never re-reads it — which matters most for the image + // disk, usually the largest layer in the chain. The top layer is a + // fresh temp copy that will be gone in a moment, so there is nothing to + // cache it against and no point trying. + let digest = if i == last { + CanonicalLayer::open(path)?.digest()? + } else { + match base_disk_mgr.digest_of(path)? { + Some(cached) => cached, + None => CanonicalLayer::open(path)?.digest()?, + } + }; + + let qcow2 = is_qcow2(path); + layers.push(ArchiveLayer { + image_digest: image_digest_of(path, image_disks_dir), + digest: digest.clone(), + format: if qcow2 { + LayerFormat::Qcow2 + } else { + LayerFormat::Raw + }, + virtual_size: if qcow2 { + Qcow2Helper::qcow2_virtual_size(path)? + } else { + 0 + }, + }); + blobs.push((digest, path.clone())); + } + let digest_ms = t_digest.elapsed().as_millis() as u64; let image = match &config_options.rootfs { crate::runtime::options::RootfsSpec::Image(img) => img.clone(), @@ -312,38 +453,325 @@ fn do_export_finalize( }; let manifest = ArchiveManifest { - version: archive_version_for_options(config_options), + // A layered archive is unreadable to a pre-v6 importer, so it is + // stamped v6 regardless of what the options alone would need. + version: archive_version_for_options(config_options).max(LAYERED_ARCHIVE_VERSION), box_name: config_name.map(|s| s.to_string()), image, box_options: Some(config_options.clone()), - guest_disk_checksum, - container_disk_checksum, + // Kept for wire compatibility with importers that still expect the + // fields; v6 carries per-layer digests instead. + guest_disk_checksum: String::new(), + container_disk_checksum: String::new(), + layers, exported_at: chrono::Utc::now().to_rfc3339(), }; let manifest_json = serde_json::to_string_pretty(&manifest) .map_err(|e| BoxliteError::Internal(format!("Failed to serialize manifest: {}", e)))?; - let manifest_path = flatten.temp_dir.path().join(MANIFEST_FILENAME); - std::fs::write(&manifest_path, manifest_json)?; let t_archive = Instant::now(); - build_zstd_tar_archive( - &output_path, - &manifest_path, - &flatten.flat_container, - flatten.flat_guest.as_deref(), - 3, - )?; + let output_path = match dest { + ExportDest::Store { name, .. } => { + build_store_archive(&output_path, name, &manifest_json, &blobs, 3)? + } + ExportDest::Directory(_) => { + build_layered_directory(&output_path, &manifest_json, &blobs, 3)?; + output_path + } + ExportDest::File(_) => { + let manifest_path = capture.temp_dir.path().join(MANIFEST_FILENAME); + std::fs::write(&manifest_path, &manifest_json)?; + build_layered_archive(&output_path, &manifest_path, &blobs, 3)?; + output_path + } + }; let archive_ms = t_archive.elapsed().as_millis() as u64; tracing::info!( box_id = %box_id_str, output = %output_path.display(), - flatten_ms = flatten.flatten_ms, - checksum_ms, + layers = blobs.len(), + capture_ms = capture.capture_ms, + digest_ms, archive_ms, - "Exported box to archive" + "Exported box to layered archive" ); Ok(crate::runtime::options::BoxArchive::new(output_path)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::runtime::layout::{FilesystemLayout, FsLayoutConfig}; + + /// Entry paths inside a built `.boxlite` archive. + fn archive_entry_names(archive_path: &std::path::Path) -> Vec { + let file = std::fs::File::open(archive_path).expect("open archive"); + let decoder = zstd::Decoder::new(file).expect("zstd decoder"); + let mut archive = tar::Archive::new(decoder); + archive + .entries() + .expect("read entries") + .map(|e| { + e.expect("entry") + .path() + .expect("entry path") + .to_string_lossy() + .into_owned() + }) + .collect() + } + + /// Build a manager over a real store in `home`. + fn test_base_disk_mgr(home: &std::path::Path) -> crate::disk::BaseDiskManager { + let bases_dir = home.join("bases"); + std::fs::create_dir_all(&bases_dir).unwrap(); + let db = crate::db::Database::open(&home.join("boxlite.db")).unwrap(); + crate::disk::BaseDiskManager::new(bases_dir, crate::db::base_disk::BaseDiskStore::new(db)) + } + + /// A box home holding a two-layer chain: `disk.qcow2` over a base. + fn chained_box_home(home: &std::path::Path) -> std::path::PathBuf { + let box_home = home.join("box"); + let disks = box_home.join("disks"); + std::fs::create_dir_all(&disks).unwrap(); + + let base = home.join("base.qcow2"); + let vsize = Qcow2Helper::create_disk(&base, true).unwrap().leak(); + let vsize = Qcow2Helper::qcow2_virtual_size(&vsize).unwrap(); + Qcow2Helper::create_cow_child_disk( + &base, + crate::disk::BackingFormat::Qcow2, + &disks.join(disk_filenames::CONTAINER_DISK), + vsize, + ) + .unwrap() + .leak(); + + // Present, as on any started box — and never exported. + Qcow2Helper::create_disk(&disks.join(disk_filenames::GUEST_ROOTFS_DISK), true) + .unwrap() + .leak(); + box_home + } + + /// Re-exporting the same box leaves existing layer objects untouched. + /// + /// The shared base keeps its mtime — the object was not rewritten — which + /// is the property a sync tool needs for "mirror this directory" to + /// transfer only missing objects. + #[test] + fn a_reexport_into_the_same_directory_skips_existing_objects() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let home = temp.path(); + let out = home.join("mirror"); + + export_with(home, &out, true); + let layers: Vec<_> = std::fs::read_dir(out.join("layers")) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + assert!(!layers.is_empty(), "directory export must produce objects"); + let stamps: Vec<_> = layers + .iter() + .map(|p| std::fs::metadata(p).unwrap().modified().unwrap()) + .collect(); + + // Re-export the same box into the same mirror. + export_with(home, &out, true); + + for (path, before) in layers.iter().zip(&stamps) { + assert_eq!( + &std::fs::metadata(path).unwrap().modified().unwrap(), + before, + "{} was rewritten on re-export", + path.display() + ); + } + assert!(out.join("manifest.json").exists()); + } + + /// Publishing the same chain under two names shares every object; the + /// pool empties only when the last manifest referencing it is gone. + #[test] + fn a_store_frees_a_layer_only_with_its_last_reference() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let home = temp.path(); + let store_root = home.join("store"); + + let first = export_named(home, &store_root, true, Some("monday")); + assert!(first.path().ends_with("archives/monday.json")); + let objects: Vec<_> = std::fs::read_dir(store_root.join("layers")) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + assert!(!objects.is_empty()); + let stamps: Vec<_> = objects + .iter() + .map(|p| std::fs::metadata(p).unwrap().modified().unwrap()) + .collect(); + + export_named(home, &store_root, true, Some("tuesday")); + for (path, before) in objects.iter().zip(&stamps) { + assert_eq!( + &std::fs::metadata(path).unwrap().modified().unwrap(), + before, + "{} was rewritten by the second publish", + path.display() + ); + } + + let store = crate::ArchiveStore::open(&store_root).unwrap(); + let mut names = store.archives().unwrap(); + names.sort(); + assert_eq!(names, ["monday", "tuesday"]); + + // Identical chains: dropping one name must free nothing. + assert!(store.remove("monday").unwrap()); + let report = store.gc(std::time::Duration::ZERO).unwrap(); + assert_eq!(report.swept, 0, "tuesday still references every layer"); + + assert!(store.remove("tuesday").unwrap()); + let report = store.gc(std::time::Duration::ZERO).unwrap(); + assert_eq!(report.swept, objects.len()); + assert_eq!( + std::fs::read_dir(store_root.join("layers")) + .unwrap() + .count(), + 0 + ); + } + + fn export_to_archive(home: &std::path::Path) -> crate::runtime::options::BoxArchive { + export_with(home, &home.join("out.boxlite"), false) + } + + fn export_with( + home: &std::path::Path, + dest: &std::path::Path, + as_directory: bool, + ) -> crate::runtime::options::BoxArchive { + export_named(home, dest, as_directory, None) + } + + fn export_named( + home: &std::path::Path, + dest: &std::path::Path, + as_directory: bool, + archive_name: Option<&str>, + ) -> crate::runtime::options::BoxArchive { + let layout = FilesystemLayout::new(home.to_path_buf(), FsLayoutConfig::default()); + std::fs::create_dir_all(layout.temp_dir()).unwrap(); + let box_home = chained_box_home(home); + let capture = do_export_capture(&box_home, &layout).expect("capture"); + do_export_finalize( + capture, + &test_base_disk_mgr(home), + &home.join("images").join("disk-images"), + Some("some-box"), + &crate::runtime::options::BoxOptions::default(), + "box-id", + match (archive_name, as_directory) { + (Some(name), _) => ExportDest::Store { root: dest, name }, + (None, true) => ExportDest::Directory(dest), + (None, false) => ExportDest::File(dest), + }, + ) + .expect("finalize") + } + + /// Export ships the disk chain as layers instead of flattening it, so an + /// importer that already holds a layer can skip transferring it. + #[test] + fn export_emits_one_blob_per_chain_layer() { + let home = tempfile::tempdir_in("/tmp").expect("home dir"); + let archive = export_to_archive(home.path()); + + let entries = archive_entry_names(archive.path()); + let blobs: Vec<_> = entries + .iter() + .filter(|e| e.starts_with("layers/")) + .collect(); + assert_eq!( + blobs.len(), + 2, + "expected one blob per chain layer (base + overlay), got {entries:?}" + ); + assert!( + !entries.iter().any(|e| e == disk_filenames::CONTAINER_DISK), + "a layered archive carries no flattened disk, got {entries:?}" + ); + } + + /// The guest rootfs disk is host-global state the importing host rebuilds + /// from its own version-keyed cache, so it must never travel in an archive. + #[test] + fn export_omits_the_guest_rootfs_disk() { + let home = tempfile::tempdir_in("/tmp").expect("home dir"); + let archive = export_to_archive(home.path()); + + let entries = archive_entry_names(archive.path()); + assert!( + !entries + .iter() + .any(|e| e.ends_with(disk_filenames::GUEST_ROOTFS_DISK)), + "archive must not carry the guest rootfs disk, got {entries:?}" + ); + } + + #[test] + fn export_refuses_an_incomplete_backing_chain() { + let temp = tempfile::TempDir::new_in("/tmp").unwrap(); + let home = temp.path(); + let layout = FilesystemLayout::new(home.to_path_buf(), FsLayoutConfig::default()); + std::fs::create_dir_all(layout.temp_dir()).unwrap(); + let box_home = chained_box_home(home); + let container = box_home.join("disks").join(disk_filenames::CONTAINER_DISK); + let nearest = crate::disk::read_backing_file_path(&container) + .unwrap() + .map(std::path::PathBuf::from) + .expect("container backing"); + std::fs::remove_file(nearest).unwrap(); + + let error = match do_export_capture(&box_home, &layout) { + Ok(_) => panic!("an incomplete chain must not produce an archive"), + Err(error) => error, + }; + + assert!(error.to_string().contains("backing chain is incomplete")); + } + + /// Layers are ordered base first, so an importer can materialize each + /// layer's parent before relinking it. + #[test] + fn manifest_orders_layers_base_first() { + let home = tempfile::tempdir_in("/tmp").expect("home dir"); + let archive = export_to_archive(home.path()); + + let file = std::fs::File::open(archive.path()).unwrap(); + let mut tar = tar::Archive::new(zstd::Decoder::new(file).unwrap()); + let mut manifest_json = String::new(); + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + if entry.path().unwrap().to_string_lossy() == super::super::archive::MANIFEST_FILENAME { + use std::io::Read; + entry.read_to_string(&mut manifest_json).unwrap(); + } + } + let manifest: super::super::archive::ArchiveManifest = + serde_json::from_str(&manifest_json).unwrap(); + + assert_eq!(manifest.layers.len(), 2, "{:?}", manifest.layers); + // The base has no backing file of its own; the overlay sits on top. + assert_eq!( + manifest.version, + super::super::archive::LAYERED_ARCHIVE_VERSION + ); + assert!( + manifest.layers[0].digest != manifest.layers[1].digest, + "layers must be distinct blobs" + ); + } +} diff --git a/src/boxlite/src/litebox/mod.rs b/src/boxlite/src/litebox/mod.rs index 742b04440..8aa58d921 100644 --- a/src/boxlite/src/litebox/mod.rs +++ b/src/boxlite/src/litebox/mod.rs @@ -3,6 +3,7 @@ //! Provides lazy initialization and execution capabilities for isolated boxes. pub(crate) mod archive; +pub mod archive_store; pub(crate) mod box_impl; mod clone_export; pub(crate) mod config; diff --git a/src/boxlite/src/rest/litebox.rs b/src/boxlite/src/rest/litebox.rs index 041a30cc3..b744face4 100644 --- a/src/boxlite/src/rest/litebox.rs +++ b/src/boxlite/src/rest/litebox.rs @@ -410,6 +410,15 @@ impl BoxBackend for RestBox { ) -> BoxliteResult { self.client.require_export_enabled().await?; + // The wire format is one HTTP body; a directory of objects has no + // representation there. Refusing is better than silently handing back + // a single file the caller intends to mirror somewhere. + if options.as_directory || options.archive_name.is_some() { + return Err(BoxliteError::Unsupported( + "directory-form export is not available over REST; export locally and mirror the directory".into(), + )); + } + let box_id = self.box_id_str(); let path = format!("/boxes/{}/export", box_id); let req = ExportBoxRequest::from_options(&options); diff --git a/src/boxlite/src/rootfs/guest.rs b/src/boxlite/src/rootfs/guest.rs index 6277d6197..b5cc1954c 100644 --- a/src/boxlite/src/rootfs/guest.rs +++ b/src/boxlite/src/rootfs/guest.rs @@ -472,6 +472,7 @@ impl GuestRootfsManager { size_bytes, }, created_at: chrono::Utc::now().timestamp(), + digest: None, }; if let Err(e) = self.base_disk_mgr.store().insert(&disk) { @@ -654,6 +655,7 @@ mod tests { size_bytes: 100, }, created_at: chrono::Utc::now().timestamp(), + digest: None, }) .unwrap(); } diff --git a/src/boxlite/src/runtime/import.rs b/src/boxlite/src/runtime/import.rs index 76f427429..0a37029b5 100644 --- a/src/boxlite/src/runtime/import.rs +++ b/src/boxlite/src/runtime/import.rs @@ -1,6 +1,7 @@ //! Box import from `.boxlite` archives. -use std::path::Path; +use std::cell::Cell; +use std::path::{Path, PathBuf}; use std::sync::Arc; use boxlite_shared::errors::{BoxliteError, BoxliteResult}; @@ -8,10 +9,12 @@ use boxlite_shared::errors::{BoxliteError, BoxliteResult}; use crate::disk::constants::filenames as disk_filenames; use crate::litebox::LiteBox; use crate::litebox::archive::{ - ArchiveManifest, MANIFEST_FILENAME, MAX_SUPPORTED_VERSION, PUBLISHED_PORTS_ARCHIVE_VERSION, - extract_archive, move_file, sha256_file, + ArchiveLayer, ArchiveManifest, CanonicalLayer, LayerFormat, MANIFEST_FILENAME, + MAX_SUPPORTED_VERSION, PUBLISHED_PORTS_ARCHIVE_VERSION, extract_archive, layer_entry_name, + move_file, sha256_file, }; use crate::runtime::advanced_options::SecurityOptions; +use crate::runtime::id::BaseDiskID; use crate::runtime::options::{ ArchiveImportPolicy, BoxArchive, BoxOptions, RootfsSpec, normalize_legacy_ports, }; @@ -52,13 +55,77 @@ pub(crate) async fn import_box( let staging_dir = temp_dir.path().join("staging"); let temp_path = temp_dir.path().to_path_buf(); let staging_clone = staging_dir.clone(); - tokio::task::spawn_blocking(move || install_disks(&temp_path, &staging_clone)) - .await - .map_err(|e| BoxliteError::Internal(format!("Import install task panicked: {}", e)))??; + let layers = manifest.layers.clone(); + let base_disk_mgr = runtime.base_disk_mgr.clone(); + let image_disks_dir = runtime.layout.image_layout().disk_images_dir(); + // Layers are pinned to this token the moment each one lands, and the token + // is only released once the box owns them. Without it a layer sits + // unreferenced between installation and provisioning, where a concurrent + // `box rm` would GC it out from under this import — and anything installed + // before a mid-way failure would leak, since nothing else ever collects a + // base with no dependents. + let token = format!("__importing__{}", uuid::Uuid::new_v4()); + let token_for_task = token.clone(); + // The directory form keeps its objects where they are; the scratch dir is + // only where the ones actually wanted get unpacked. + let blobs = if let Some(store_root) = store_manifest(archive.path()) { + LayerBlobs::directory(store_root, temp_path.clone()) + } else if archive.path().is_dir() { + LayerBlobs::directory(archive.path().to_path_buf(), temp_path.clone()) + } else { + LayerBlobs::Extracted(temp_path.clone()) + }; + let install_task = tokio::task::spawn_blocking(move || { + if layers.is_empty() { + install_disks(&temp_path, &staging_clone).map(|()| Vec::new()) + } else { + install_layers( + &layers, + &blobs, + &staging_clone, + &base_disk_mgr, + &token_for_task, + &image_disks_dir, + ) + } + }); + let install = match install_task.await { + Ok(install) => install, + Err(e) => { + release_import_token(&runtime.base_disk_mgr, &token); + return Err(BoxliteError::Internal(format!( + "Import install task panicked: {e}" + ))); + } + }; + + let installed = match install { + Ok(installed) => installed, + Err(e) => { + release_import_token(&runtime.base_disk_mgr, &token); + return Err(e); + } + }; - let litebox = runtime + let litebox = match runtime .provision_box(staging_dir, name, options, BoxStatus::Stopped) - .await?; + .await + { + Ok(litebox) => litebox, + Err(e) => { + release_import_token(&runtime.base_disk_mgr, &token); + return Err(e); + } + }; + + // Hand ownership to the box before dropping the token, so the layers are + // never momentarily unreferenced. + handoff_import_refs( + &runtime.base_disk_mgr, + &installed, + &token, + litebox.id().as_ref(), + ); tracing::info!( box_id = %litebox.id(), @@ -69,6 +136,46 @@ pub(crate) async fn import_box( Ok(litebox) } +/// Drop an import's provisional refs and collect anything they were the last +/// reference to. +/// +/// After a successful import the box holds its own refs, so this only releases +/// the token. After a failure it is what stops half-installed layers from +/// accumulating in `bases/` forever. +fn handoff_import_refs( + base_disk_mgr: &crate::disk::BaseDiskManager, + installed: &[BaseDiskID], + token: &str, + box_id: &str, +) { + for base_id in installed { + if let Err(e) = base_disk_mgr.store().add_ref(base_id, box_id) { + tracing::warn!( + box_id, + base_disk_id = %base_id, + error = %e, + "Failed to record base disk ref; retaining import token refs" + ); + return; + } + } + release_import_token(base_disk_mgr, token); +} + +fn release_import_token(base_disk_mgr: &crate::disk::BaseDiskManager, token: &str) { + let store = base_disk_mgr.store(); + let released = match store.remove_all_refs_for_box(token) { + Ok(ids) => ids, + Err(e) => { + tracing::warn!(error = %e, "Failed to release import token refs"); + return; + } + }; + for id in released { + base_disk_mgr.try_gc_base(&id); + } +} + /// Read the persisted configuration, falling back to the v1/v2 image field. /// /// An archive is untrusted input, so its options are validated here rather @@ -135,9 +242,20 @@ fn extract_and_validate( let temp_dir = tempfile::tempdir_in(layout.temp_dir()) .map_err(|e| BoxliteError::Storage(format!("Failed to create temp directory: {}", e)))?; - extract_archive(archive_path, temp_dir.path())?; - - let manifest_path = temp_dir.path().join(MANIFEST_FILENAME); + // A mirrored archive directory is already in the layout an extraction + // would produce, except its layers are still compressed and are unpacked + // one at a time, only if wanted. Copying it here first would throw that + // away, so only the single-file form is extracted. A store archive is the + // same thing again, one level up: the path is the manifest itself, at + // `/archives/.json` over the store's shared pool. + let manifest_path = if let Some(_root) = store_manifest(archive_path) { + archive_path.to_path_buf() + } else if archive_path.is_dir() { + archive_path.join(MANIFEST_FILENAME) + } else { + extract_archive(archive_path, temp_dir.path())?; + temp_dir.path().join(MANIFEST_FILENAME) + }; if !manifest_path.exists() { return Err(BoxliteError::Storage( "Invalid archive: manifest.json not found".to_string(), @@ -155,6 +273,18 @@ fn extract_and_validate( ))); } + // A layered archive carries `layers/` blobs instead of a flattened disk; + // each is checked against its own digest as it is installed. + if !manifest.layers.is_empty() { + for layer in &manifest.layers { + validate_sha256_digest(&layer.digest, "layer")?; + if let Some(image_digest) = &layer.image_digest { + validate_sha256_digest(image_digest, "image")?; + } + } + return Ok((manifest, temp_dir)); + } + let extracted_container = temp_dir.path().join(disk_filenames::CONTAINER_DISK); if !extracted_container.exists() { return Err(BoxliteError::Storage(format!( @@ -174,21 +304,342 @@ fn extract_and_validate( } } - let extracted_guest = temp_dir.path().join(disk_filenames::GUEST_ROOTFS_DISK); - if extracted_guest.exists() && !manifest.guest_disk_checksum.is_empty() { - let actual = sha256_file(&extracted_guest)?; - if actual != manifest.guest_disk_checksum { - return Err(BoxliteError::Storage(format!( - "Guest disk checksum mismatch: expected {}, got {}", - manifest.guest_disk_checksum, actual - ))); + // A guest rootfs disk carried by an older archive is ignored, so it is + // neither checksummed nor installed — see `install_disks`. + + Ok((manifest, temp_dir)) +} + +/// Materialize a layered archive's chain and relink it, returning the ids of +/// the base disks the imported box now depends on. +/// +/// A layer already present locally — same content digest — is reused as-is and +/// its blob is never written, which is where cross-box dedup comes from: every +/// box built from an image shares that image's layer. +/// +/// Security: the manifest carries digests, never paths. Each child is relinked +/// to a path *this* function chose and canonicalized locally, and the resulting +/// header is read back and checked, so a crafted archive cannot aim a backing +/// file at a host path of its choosing. Every blob is verified against its +/// declared digest before anything points at it. +fn install_layers( + layers: &[ArchiveLayer], + blobs: &LayerBlobs, + box_home: &Path, + base_disk_mgr: &crate::disk::BaseDiskManager, + token: &str, + image_disks_dir: &Path, +) -> BoxliteResult> { + let Some((top, bases)) = layers.split_last() else { + return Err(BoxliteError::Storage( + "Invalid archive: layered manifest has no layers".to_string(), + )); + }; + + let disks_dir = box_home.join("disks"); + std::fs::create_dir_all(&disks_dir).map_err(|e| { + BoxliteError::Storage(format!( + "Failed to create disks directory {}: {}", + disks_dir.display(), + e + )) + })?; + + // Materialize the bases bottom-up, so each layer's parent already exists + // by the time it is relinked. + let mut base_ids = Vec::new(); + let mut parent: Option = None; + for layer in bases { + let (path, id, freshly_installed) = resolve_layer( + layer, + blobs, + base_disk_mgr, + token, + parent.as_deref(), + image_disks_dir, + )?; + if let Some(id) = id { + base_ids.push(id); + } + if freshly_installed { + match &parent { + Some(parent_path) => relink(&path, parent_path)?, + // The deepest layer stands alone. Without this an archive + // could ship a base whose header already points at any host + // path — the chain is granted to the sandbox at start, so that + // would hand the guest an arbitrary file. + None => validate_no_backing_references(&path)?, + } } + parent = Some(path); } - Ok((manifest, temp_dir)) + // The top layer is the box's own container disk. + let container = disks_dir.join(disk_filenames::CONTAINER_DISK); + let blob = blobs.materialize(top)?; + verify_layer_digest(&blob, &top.digest)?; + verify_layer_format(&blob, top)?; + move_file(&blob, &container)?; + + match &parent { + Some(parent_path) => relink(&container, parent_path)?, + // A single-layer chain stands alone, so it must not reference anything. + None => validate_no_backing_references(&container)?, + } + + Ok(base_ids) +} + +/// Where a layer's bytes come from while an archive is being installed. +/// +/// The two forms differ in *when* a blob costs anything. A `.boxlite` file is +/// one stream, so every layer is already unpacked by the time anything is +/// decided. A mirrored directory holds each layer as its own object, so a +/// layer the host already has is never opened — which is the only reason the +/// directory form saves work rather than just rearranging it. +enum LayerBlobs { + Extracted(PathBuf), + Directory { + archive_dir: PathBuf, + scratch: PathBuf, + remaining_output: Cell, + }, +} + +/// The store root, if this archive path is a store manifest. +/// +/// A store archive is addressed by its manifest file, +/// `/archives/.json`; its layers live in the store's shared pool +/// at `/layers/`. Anything else — a `.boxlite` file, a mirrored +/// archive directory — is not a store manifest. +fn store_manifest(archive_path: &Path) -> Option { + if !archive_path.is_file() || archive_path.extension()? != "json" { + return None; + } + let archives_dir = archive_path.parent()?; + if archives_dir.file_name()? != crate::litebox::archive::STORE_ARCHIVES_DIR { + return None; + } + archives_dir.parent().map(Path::to_path_buf) +} + +impl LayerBlobs { + fn directory(archive_dir: PathBuf, scratch: PathBuf) -> Self { + Self::directory_with_limit( + archive_dir, + scratch, + crate::litebox::archive::MAX_ARCHIVE_OUTPUT, + ) + } + + fn directory_with_limit(archive_dir: PathBuf, scratch: PathBuf, limit: u64) -> Self { + Self::Directory { + archive_dir, + scratch, + remaining_output: Cell::new(limit), + } + } + + /// Produce a path to this layer's bytes, unpacking it only if needed. + fn materialize(&self, layer: &ArchiveLayer) -> BoxliteResult { + validate_sha256_digest(&layer.digest, "layer")?; + match self { + Self::Extracted(dir) => Ok(dir.join(layer_entry_name(&layer.digest))), + Self::Directory { + archive_dir, + scratch, + remaining_output, + } => { + let dest = scratch.join(layer_entry_name(&layer.digest)); + if !dest.exists() { + let written = crate::litebox::archive::extract_layer_object( + archive_dir, + &layer.digest, + &dest, + layer.virtual_size, + remaining_output.get(), + )?; + remaining_output.set(remaining_output.get().saturating_sub(written)); + } + Ok(dest) + } + } + } +} + +fn validate_sha256_digest<'a>(digest: &'a str, kind: &str) -> BoxliteResult<&'a str> { + digest + .strip_prefix("sha256:") + .filter(|hex| hex.len() == 64 && hex.bytes().all(|byte| byte.is_ascii_hexdigit())) + .ok_or_else(|| { + BoxliteError::Storage(format!("Invalid archive: malformed {kind} digest {digest}")) + }) +} + +/// Fail unless a blob hashes to the digest the manifest declared for it. +fn verify_layer_digest(path: &Path, digest: &str) -> BoxliteResult<()> { + if !path.exists() { + return Err(BoxliteError::Storage(format!( + "Invalid archive: layer {digest} is missing from the archive" + ))); + } + // Compare canonical forms: a shipped blob has its backing pointer zeroed, + // and so does the digest that names it. + let actual = CanonicalLayer::open(path)?.digest()?; + if actual != digest { + return Err(BoxliteError::Storage(format!( + "Layer digest mismatch: expected {digest}, got {actual}" + ))); + } + Ok(()) +} + +/// Fail unless a blob's on-disk format is the one the manifest declared. +/// +/// Only the deepest layer may be raw; anything above it must be qcow2 to carry +/// a backing pointer at all. Checking here keeps a mislabelled layer from +/// reaching `relink`, whose failure would be reported as a rebase error rather +/// than as the malformed archive it is. +fn verify_layer_format(path: &Path, layer: &ArchiveLayer) -> BoxliteResult<()> { + let is_qcow2 = qcow2_magic(path); + let declared_qcow2 = layer.format == LayerFormat::Qcow2; + if is_qcow2 != declared_qcow2 { + return Err(BoxliteError::Storage(format!( + "Layer {} declares format {:?} but its blob is {}", + layer.digest, + layer.format, + if is_qcow2 { "qcow2" } else { "raw" } + ))); + } + Ok(()) +} + +fn qcow2_magic(path: &Path) -> bool { + use std::io::Read; + let Ok(mut f) = std::fs::File::open(path) else { + return false; + }; + let mut magic = [0u8; 4]; + f.read_exact(&mut magic).is_ok() && u32::from_be_bytes(magic) == 0x5146_49fb } -/// Validate disk security and move disks into box_home/disks/. +/// Return where a layer lives locally, installing it if this host lacks it. +/// +/// A local layer is reused only when its chain already matches the one the +/// archive describes — that is, its backing file is exactly `parent`. A layer's +/// digest names its canonical form, which says nothing about which parent it +/// sits on, so the same layer can legitimately exist over different parents. +/// Relinking a reused base to satisfy this archive would rewrite a file other +/// boxes and snapshots are actively backed by, silently re-pointing them at +/// content this archive supplied; installing a private copy instead costs +/// space but cannot corrupt anything. +/// +/// The returned id is `Some` only when a base disk record exists to reference, +/// which is what keeps a newly installed layer from being garbage-collected. +/// The bool reports whether the file was freshly installed, and so is safe for +/// the caller to relink. +fn resolve_layer( + layer: &ArchiveLayer, + blobs: &LayerBlobs, + base_disk_mgr: &crate::disk::BaseDiskManager, + token: &str, + parent: Option<&Path>, + image_disks_dir: &Path, +) -> BoxliteResult<(PathBuf, Option, bool)> { + validate_sha256_digest(&layer.digest, "layer")?; + + // An image disk this host already built is preferred over the archived + // copy, and is the only cross-host reuse available for that layer: its + // bytes differ on every host (mke2fs writes a random UUID), so `digest` + // cannot match, but the image digest can. Reusing it also keeps the box on + // the host's own correctly-built disk rather than a foreign one. + // + // No base disk id is returned because the image cache owns this file and + // manages its own lifecycle — it must not be pulled into base-disk GC. + if let Some(image_digest) = &layer.image_digest { + let hex = validate_sha256_digest(image_digest, "image")?; + let local = image_disks_dir.join(format!("sha256-{hex}.ext4")); + if local.is_file() { + tracing::debug!( + image_digest = %image_digest, + "Image disk already built locally, skipping the archived copy" + ); + return Ok((local, None, false)); + } + } + + // Keep lookup/install and the provisional ref pin indivisible with GC. + // Otherwise GC can observe a zero-ref record between the lookup and pin, + // then delete the file while this import is adopting it. + let lifecycle = base_disk_mgr.lock_lifecycle(); + if let Some(existing) = base_disk_mgr.store().find_by_digest(&layer.digest)? { + let path = PathBuf::from(&existing.disk.disk_info.base_path); + if path.exists() && backing_matches(&path, parent) { + base_disk_mgr.store().add_ref(&existing.disk.id, token)?; + tracing::debug!(digest = %layer.digest, "Layer already present, skipping transfer"); + return Ok((path, Some(existing.disk.id), false)); + } + // Either the record outlived its file, or the local copy sits on a + // different parent; install a private copy below. + } + + let blob = blobs.materialize(layer)?; + verify_layer_digest(&blob, &layer.digest)?; + verify_layer_format(&blob, layer)?; + let installed = base_disk_mgr.install_layer(&blob, &layer.digest)?; + if let Err(error) = base_disk_mgr.store().add_ref(&installed.id, token) { + drop(lifecycle); + base_disk_mgr.try_gc_base(&installed.id); + return Err(error); + } + Ok((installed.disk_info.to_path_buf(), Some(installed.id), true)) +} + +/// Whether `path`'s backing file is already exactly `parent`. +fn backing_matches(path: &Path, parent: Option<&Path>) -> bool { + let actual = crate::disk::read_backing_file_path(path) + .ok() + .flatten() + .map(PathBuf::from); + match (actual, parent) { + (None, None) => true, + (Some(actual), Some(parent)) => { + let expected = parent + .canonicalize() + .unwrap_or_else(|_| parent.to_path_buf()); + actual == expected + } + _ => false, + } +} + +/// Point a child qcow2 at a parent path chosen by this host, then prove it took. +fn relink(child: &Path, parent: &Path) -> BoxliteResult<()> { + crate::disk::set_backing_file_path(child, parent)?; + + let expected = parent + .canonicalize() + .unwrap_or_else(|_| parent.to_path_buf()); + match crate::disk::read_backing_file_path(child)? { + Some(actual) if Path::new(&actual) == expected => Ok(()), + other => Err(BoxliteError::InvalidState(format!( + "Refusing imported disk '{}': backing file is {:?} after relink, expected {}", + child.display(), + other, + expected.display() + ))), + } +} + +/// Validate disk security and move the container disk into box_home/disks/. +/// +/// The guest rootfs disk is never installed, even when an older archive carries +/// one. It holds no user state, and letting an archived copy win would bypass +/// the importing host's own version-keyed guest rootfs cache: export flattens +/// the overlay, so the archived disk has no backing reference and +/// `validate_reusable_guest_rootfs_disk` would accept it verbatim. Leaving it +/// absent makes the next start rebuild the overlay from the local cache, which +/// is what clone and snapshot-restore already do. fn install_disks(temp_dir: &Path, box_home: &Path) -> BoxliteResult<()> { // Security: Reject imported disks that reference backing files. // A crafted archive could include a qcow2 with a backing reference to @@ -196,11 +647,6 @@ fn install_disks(temp_dir: &Path, box_home: &Path) -> BoxliteResult<()> { let extracted_container = temp_dir.join(disk_filenames::CONTAINER_DISK); validate_no_backing_references(&extracted_container)?; - let extracted_guest = temp_dir.join(disk_filenames::GUEST_ROOTFS_DISK); - if extracted_guest.exists() { - validate_no_backing_references(&extracted_guest)?; - } - let disks_dir = box_home.join("disks"); std::fs::create_dir_all(&disks_dir).map_err(|e| { BoxliteError::Storage(format!( @@ -215,13 +661,6 @@ fn install_disks(temp_dir: &Path, box_home: &Path) -> BoxliteResult<()> { &disks_dir.join(disk_filenames::CONTAINER_DISK), )?; - if extracted_guest.exists() { - move_file( - &extracted_guest, - &disks_dir.join(disk_filenames::GUEST_ROOTFS_DISK), - )?; - } - Ok(()) } @@ -238,6 +677,435 @@ pub(crate) fn validate_no_backing_references(disk_path: &Path) -> BoxliteResult< Ok(()) } +#[cfg(test)] +#[cfg(test)] +mod layered_install_tests { + use super::*; + use crate::litebox::archive::CanonicalLayer; + + fn mgr(home: &Path) -> crate::disk::BaseDiskManager { + let bases = home.join("bases"); + std::fs::create_dir_all(&bases).unwrap(); + let db = crate::db::Database::open(&home.join("boxlite.db")).unwrap(); + crate::disk::BaseDiskManager::new(bases, crate::db::base_disk::BaseDiskStore::new(db)) + } + + /// Write a blob into the extracted-archive layout and describe it. + /// + /// `tag` makes each layer's content unique, so distinct layers get distinct + /// digests. A layer that will be relinked must be staged with some backing + /// path — `set_backing_file_path` can only rewrite a pointer that exists, + /// which is also true of the real layers this stands in for: every layer + /// above the image disk is a COW child. + fn stage(temp: &Path, tag: u8, backing: Option<&str>) -> ArchiveLayer { + let scratch = temp.join("scratch.qcow2"); + crate::disk::qcow2::write_test_qcow2(&scratch, backing); + // Perturb a byte outside the header and the backing-path region so the + // canonical digests differ per layer. + let mut bytes = std::fs::read(&scratch).unwrap(); + bytes[900] = tag; + std::fs::write(&scratch, &bytes).unwrap(); + + let digest = CanonicalLayer::open(&scratch).unwrap().digest().unwrap(); + let layer = ArchiveLayer { + image_digest: None, + digest: digest.clone(), + format: LayerFormat::Qcow2, + virtual_size: 0, + }; + let dest = temp.join(layer_entry_name(&digest)); + std::fs::create_dir_all(dest.parent().unwrap()).unwrap(); + std::fs::rename(&scratch, &dest).unwrap(); + layer + } + + #[test] + fn a_failed_box_ref_keeps_the_import_token_refs() { + let home = tempfile::TempDir::new_in("/tmp").unwrap(); + let temp = home.path().join("extract"); + std::fs::create_dir_all(&temp).unwrap(); + let mgr = mgr(home.path()); + let bottom = stage(&temp, 1, None); + let top = stage(&temp, 2, Some(FOREIGN_PARENT)); + let installed = install_layers( + &[bottom, top], + &LayerBlobs::Extracted(temp), + &home.path().join("box"), + &mgr, + "import-token", + &home.path().join("images"), + ) + .expect("install"); + assert!(!installed.is_empty(), "the bottom layer must be installed"); + + let conn = rusqlite::Connection::open(home.path().join("boxlite.db")).unwrap(); + conn.execute_batch( + "CREATE TRIGGER reject_box_ref + BEFORE INSERT ON base_disk_ref + WHEN NEW.box_id = 'box-id' + BEGIN + SELECT RAISE(FAIL, 'forced add_ref failure'); + END;", + ) + .unwrap(); + drop(conn); + + handoff_import_refs(&mgr, &installed, "import-token", "box-id"); + + for id in installed { + let dependents = mgr.store().dependent_boxes(&id).unwrap(); + assert!(dependents.contains(&"import-token".to_string())); + assert!(!dependents.contains(&"box-id".to_string())); + } + } + + #[test] + fn a_failed_import_token_ref_collects_the_installed_layer() { + let home = tempfile::TempDir::new_in("/tmp").unwrap(); + let temp = home.path().join("extract"); + std::fs::create_dir_all(&temp).unwrap(); + let mgr = mgr(home.path()); + let bottom = stage(&temp, 1, None); + let digest = bottom.digest.clone(); + + let conn = rusqlite::Connection::open(home.path().join("boxlite.db")).unwrap(); + conn.execute_batch( + "CREATE TRIGGER reject_import_ref + BEFORE INSERT ON base_disk_ref + WHEN NEW.box_id = 'import-token' + BEGIN + SELECT RAISE(FAIL, 'forced add_ref failure'); + END;", + ) + .unwrap(); + drop(conn); + + install_layers( + &[bottom, stage(&temp, 2, Some(FOREIGN_PARENT))], + &LayerBlobs::Extracted(temp), + &home.path().join("box"), + &mgr, + "import-token", + &home.path().join("images"), + ) + .expect_err("the forced token ref failure must abort the import"); + + assert!( + mgr.store().find_by_digest(&digest).unwrap().is_none(), + "the unreferenced layer record must be collected" + ); + } + + /// A directory archive's objects are opened only when actually wanted. + /// + /// The host already holds the bottom layer, so its object in the mirror is + /// replaced with garbage that would fail digest verification the moment + /// anything read it. The import must succeed anyway — proof the object was + /// never opened, which is what makes the directory form cheaper than the + /// single file rather than just differently shaped. + #[test] + fn a_layer_the_host_already_holds_is_never_read_from_the_directory() { + let home = tempfile::TempDir::new_in("/tmp").unwrap(); + let temp = home.path().join("extract"); + std::fs::create_dir_all(&temp).unwrap(); + let mgr = mgr(home.path()); + + // First import, single-file form: installs both layers locally. + let bottom = stage(&temp, 1, None); + let top = stage(&temp, 2, Some(FOREIGN_PARENT)); + // Read before the first install consumes the blob by moving it. + let top_blob = std::fs::read(temp.join(layer_entry_name(&top.digest))).unwrap(); + install_layers( + &[bottom.clone(), top.clone()], + &LayerBlobs::Extracted(temp.clone()), + &home.path().join("box1"), + &mgr, + "tok1", + &home.path().join("images"), + ) + .expect("first import"); + + // Second import of the same box, directory form. The bottom's object + // is garbage; the top's object is real (a top layer is always fresh). + let mirror = home.path().join("mirror"); + let layers_dir = mirror.join("layers"); + std::fs::create_dir_all(&layers_dir).unwrap(); + let hex = |d: &str| d.strip_prefix("sha256:").unwrap().to_string(); + std::fs::write( + layers_dir.join(format!("{}.zst", hex(&bottom.digest))), + b"not zstd, not the layer, not anything", + ) + .unwrap(); + std::fs::write( + layers_dir.join(format!("{}.zst", hex(&top.digest))), + zstd::encode_all(&top_blob[..], 3).unwrap(), + ) + .unwrap(); + + let scratch = home.path().join("scratch"); + std::fs::create_dir_all(&scratch).unwrap(); + install_layers( + &[bottom, top], + &LayerBlobs::directory(mirror, scratch), + &home.path().join("box2"), + &mgr, + "tok2", + &home.path().join("images"), + ) + .expect("a held layer's garbage object must never be read"); + } + + /// The image layer's bytes differ on every host, so content addressing can + /// never reuse it. Its image digest can — and the host's own build is the + /// one the box should sit on. + #[test] + fn a_locally_built_image_disk_is_used_instead_of_the_archived_one() { + let home = tempfile::TempDir::new_in("/tmp").unwrap(); + let temp = home.path().join("extract"); + std::fs::create_dir_all(&temp).unwrap(); + let images = home.path().join("images"); + std::fs::create_dir_all(&images).unwrap(); + + // This host already built the image disk. + let image_digest = format!("sha256:{}", "f".repeat(64)); + let local = images.join(format!("sha256-{}.ext4", "f".repeat(64))); + std::fs::write(&local, b"the host's own build").unwrap(); + + // The archive carries its own, byte-different copy of that layer. + let mut bottom = stage(&temp, 1, None); + bottom.image_digest = Some(image_digest); + let archived_blob = temp.join(layer_entry_name(&bottom.digest)); + let top = stage(&temp, 2, Some(FOREIGN_PARENT)); + + install_layers( + &[bottom, top], + &LayerBlobs::Extracted(temp.clone()), + &home.path().join("box"), + &mgr(home.path()), + "tok", + &images, + ) + .expect("import"); + + // The archived blob is still sitting in the extract dir: nothing + // consumed it, because the local image disk won. + assert!( + archived_blob.exists(), + "the archived image layer must be left untouched" + ); + assert_eq!( + std::fs::read(&local).unwrap(), + b"the host's own build", + "the local image disk must not be overwritten" + ); + // And the box's disk is chained onto that local copy. + let container = home.path().join("box").join("disks").join("disk.qcow2"); + assert_eq!( + crate::disk::read_backing_file_path(&container) + .unwrap() + .map(PathBuf::from), + Some(local.canonicalize().unwrap()), + "the imported box must read through the host's own image disk" + ); + } + + #[test] + fn a_malformed_image_digest_cannot_escape_the_image_cache() { + let home = tempfile::TempDir::new_in("/tmp").unwrap(); + let temp = home.path().join("extract"); + std::fs::create_dir_all(&temp).unwrap(); + let images = home.path().join("images"); + std::fs::create_dir_all(&images).unwrap(); + + let mut bottom = stage(&temp, 1, None); + bottom.image_digest = Some("sha256:../../bases/victim".to_string()); + let top = stage(&temp, 2, Some(FOREIGN_PARENT)); + + let error = install_layers( + &[bottom, top], + &LayerBlobs::Extracted(temp), + &home.path().join("box"), + &mgr(home.path()), + "tok", + &images, + ) + .expect_err("manifest paths must not escape the image cache"); + + assert!( + error.to_string().contains("malformed image digest"), + "got: {error}" + ); + } + + #[test] + fn a_malformed_layer_digest_cannot_escape_the_scratch_directory() { + let home = tempfile::TempDir::new_in("/tmp").unwrap(); + let archive_dir = home.path().join("archive"); + let scratch = home.path().join("scratch"); + std::fs::create_dir_all(&archive_dir).unwrap(); + std::fs::create_dir_all(&scratch).unwrap(); + let escaped = home.path().join("escaped"); + let layer = ArchiveLayer { + digest: "sha256:../../escaped".to_string(), + format: LayerFormat::Raw, + virtual_size: 1, + image_digest: None, + }; + + let error = LayerBlobs::directory(archive_dir, scratch) + .materialize(&layer) + .expect_err("a malformed digest must be rejected before filesystem access"); + + assert!(error.to_string().contains("malformed layer digest")); + assert!(!escaped.exists(), "validation must happen before any write"); + } + + #[test] + fn directory_layers_share_one_decompression_budget() { + let home = tempfile::TempDir::new_in("/tmp").unwrap(); + let archive_dir = home.path().join("archive"); + let scratch = home.path().join("scratch"); + std::fs::create_dir_all(archive_dir.join("layers")).unwrap(); + std::fs::create_dir_all(&scratch).unwrap(); + let make_layer = |byte: u8| { + let digest = format!("sha256:{}", format!("{byte:02x}").repeat(32)); + let object = archive_dir.join(format!("{}.zst", layer_entry_name(&digest))); + std::fs::write(object, zstd::encode_all(&[byte; 4][..], 3).unwrap()).unwrap(); + ArchiveLayer { + digest, + format: LayerFormat::Raw, + virtual_size: 0, + image_digest: None, + } + }; + let first = make_layer(1); + let second = make_layer(2); + let blobs = LayerBlobs::directory_with_limit(archive_dir, scratch.clone(), 6); + + blobs.materialize(&first).expect("first layer fits"); + let error = blobs + .materialize(&second) + .expect_err("all directory layers must share the aggregate limit"); + + assert!(error.to_string().contains("decompression limit")); + assert!( + !scratch.join(layer_entry_name(&second.digest)).exists(), + "a layer rejected by the aggregate limit must not remain" + ); + } + + /// A stand-in for the exporter's local backing path, which import must + /// replace with one of its own choosing. + const FOREIGN_PARENT: &str = "/exporter/bases/whatever.qcow2"; + + /// The deepest layer's backing pointer is attacker-controlled data. The + /// chain is granted to the sandbox at start, so honouring it would hand the + /// guest an arbitrary host file. + #[test] + fn a_bottom_layer_pointing_at_a_host_path_is_refused() { + let home = tempfile::tempdir_in("/tmp").unwrap(); + let temp = home.path().join("extracted"); + std::fs::create_dir_all(&temp).unwrap(); + + let evil = stage(&temp, 1, Some("/etc/shadow")); + let top = stage(&temp, 2, Some(FOREIGN_PARENT)); + + let err = install_layers( + &[evil, top], + &LayerBlobs::Extracted(temp.clone()), + &home.path().join("box"), + &mgr(home.path()), + "tok", + &temp.join("images"), + ) + .expect_err("a bottom layer with a backing reference must be refused"); + let msg = err.to_string(); + assert!(msg.contains("backing file reference"), "got: {msg}"); + assert!(msg.contains("/etc/shadow"), "got: {msg}"); + } + + /// A layer already held locally may sit on a different parent than this + /// archive describes. Relinking it would rewrite a file other boxes are + /// backed by, re-pointing them at content this archive supplied. + #[test] + fn a_reused_layer_on_a_different_parent_is_copied_not_rewritten() { + let home = tempfile::tempdir_in("/tmp").unwrap(); + let temp = home.path().join("extracted"); + std::fs::create_dir_all(&temp).unwrap(); + let mgr = mgr(home.path()); + + // A shared base already installed locally, backed by nothing. + let shared = stage(&temp, 1, Some(FOREIGN_PARENT)); + let shared_blob = temp.join(layer_entry_name(&shared.digest)); + let victim_copy = temp.join("victim-source.qcow2"); + std::fs::copy(&shared_blob, &victim_copy).unwrap(); + let installed = mgr.install_layer(&victim_copy, &shared.digest).unwrap(); + let victim_path = installed.disk_info.to_path_buf(); + let victim_before = std::fs::read(&victim_path).unwrap(); + + // An archive that puts that same layer on top of a new parent. + let new_parent = stage(&temp, 2, None); + let top = stage(&temp, 3, Some(FOREIGN_PARENT)); + install_layers( + &[new_parent, shared, top], + &LayerBlobs::Extracted(temp.clone()), + &home.path().join("box"), + &mgr, + "tok", + &temp.join("images"), + ) + .expect("import should succeed by copying, not by rewriting"); + + assert_eq!( + std::fs::read(&victim_path).unwrap(), + victim_before, + "the pre-existing shared base must not be modified" + ); + } + + /// A digest names the canonical form, so relinking an installed layer must + /// not invalidate it — otherwise re-exporting an imported box yields an + /// archive no other host can read. + #[test] + fn a_relinked_layer_still_matches_its_recorded_digest() { + let home = tempfile::tempdir_in("/tmp").unwrap(); + let temp = home.path().join("extracted"); + std::fs::create_dir_all(&temp).unwrap(); + let mgr = mgr(home.path()); + + let bottom = stage(&temp, 1, None); + let middle = stage(&temp, 2, Some(FOREIGN_PARENT)); + let middle_digest = middle.digest.clone(); + let top = stage(&temp, 3, Some(FOREIGN_PARENT)); + + install_layers( + &[bottom, middle, top], + &LayerBlobs::Extracted(temp.clone()), + &home.path().join("box"), + &mgr, + "tok", + &temp.join("images"), + ) + .expect("install"); + + // The middle layer was relinked onto the bottom one; its canonical + // digest must be unchanged. + let record = mgr + .store() + .find_by_digest(&middle_digest) + .unwrap() + .expect("middle layer recorded under its digest"); + let on_disk = CanonicalLayer::open(&record.disk.disk_info.to_path_buf()) + .unwrap() + .digest() + .unwrap(); + assert_eq!( + on_disk, middle_digest, + "canonical digest must survive relinking" + ); + } +} + #[cfg(test)] mod tests { use super::*; @@ -251,6 +1119,7 @@ mod tests { box_options: Some(options), guest_disk_checksum: String::new(), container_disk_checksum: String::new(), + layers: Vec::new(), exported_at: "2026-07-26T00:00:00Z".to_string(), } } @@ -409,6 +1278,7 @@ mod tests { }), guest_disk_checksum: String::new(), container_disk_checksum: String::new(), + layers: Vec::new(), exported_at: "2026-01-01T00:00:00Z".into(), }; diff --git a/src/boxlite/src/runtime/options.rs b/src/boxlite/src/runtime/options.rs index a84b0082c..ec91f2d88 100644 --- a/src/boxlite/src/runtime/options.rs +++ b/src/boxlite/src/runtime/options.rs @@ -898,7 +898,27 @@ pub struct SnapshotOptions {} /// Forward-compatible options for exporting a box archive. #[derive(Debug, Clone, Default)] -pub struct ExportOptions {} +pub struct ExportOptions { + /// Write the archive as a directory of individually addressed objects + /// rather than a single `.boxlite` file. + /// + /// The layout is a `manifest.json` beside a `layers/` directory holding one + /// compressed object per layer, named by the layer's digest. Because a + /// layer is immutable and named by its content, syncing that directory to + /// object storage transfers only the objects the destination lacks — an + /// `aws s3 sync` or `mc mirror` already skips the rest, with no protocol + /// between the two ends. The single-file form cannot do that: it is one + /// opaque blob that changes completely between exports. + pub as_directory: bool, + /// Publish into a shared layer store under this archive name. + /// + /// Requires `as_directory`. `dest` is then the store root: the manifest + /// lands at `archives/.json` and the layers join the store's shared + /// `layers/` pool, so archives of different boxes share every layer they + /// have in common. What holds a layer in the pool is the manifests that + /// name it; `ArchiveStore::gc` sweeps the rest. + pub archive_name: Option, +} /// Forward-compatible options for cloning a box. #[derive(Debug, Clone, Default)] diff --git a/src/boxlite/src/runtime/rt_impl.rs b/src/boxlite/src/runtime/rt_impl.rs index a6c5c6464..b59de2c54 100644 --- a/src/boxlite/src/runtime/rt_impl.rs +++ b/src/boxlite/src/runtime/rt_impl.rs @@ -3174,6 +3174,7 @@ mod tests { size_bytes: 0, }, created_at: 0, + digest: None, }; runtime.base_disk_mgr.store().insert(&base_disk).unwrap(); runtime @@ -3258,6 +3259,7 @@ mod tests { size_bytes: 0, }, created_at: 0, + digest: None, }; runtime.base_disk_mgr.store().insert(&base_disk).unwrap(); runtime diff --git a/src/boxlite/tests/clone_export_import.rs b/src/boxlite/tests/clone_export_import.rs index 962291c7a..532075728 100644 --- a/src/boxlite/tests/clone_export_import.rs +++ b/src/boxlite/tests/clone_export_import.rs @@ -121,6 +121,110 @@ async fn test_export_import_roundtrip() { let _ = runtime.shutdown(Some(common::TEST_SHUTDOWN_TIMEOUT)).await; } +#[tokio::test] +async fn test_directory_export_import_roundtrip() { + let home = boxlite_test_utils::home::PerTestBoxHome::new(); + let runtime = BoxliteRuntime::new(BoxliteOptions { + home_dir: home.path.clone(), + image_registries: common::test_registries(), + }) + .expect("create runtime"); + let source = create_stopped_box(&runtime).await; + + let export_dir = TempDir::new_in("/tmp").unwrap(); + let mirror = export_dir.path().join("mirror"); + + let archive = source + .export( + ExportOptions { + as_directory: true, + ..Default::default() + }, + &mirror, + ) + .await + .expect("Failed to export box as directory"); + + // The archive is the directory itself: a manifest beside layer objects. + assert!(archive.path().is_dir()); + assert!(archive.path().join("manifest.json").exists()); + let objects = std::fs::read_dir(archive.path().join("layers")) + .expect("layers dir") + .count(); + assert!(objects >= 1, "expected at least one layer object"); + + let imported = runtime + .import_box(archive, Some("imported-from-dir".to_string())) + .await + .expect("Failed to import box from directory"); + + let info = imported.info().await.expect("get imported box info"); + assert_eq!(info.name.as_deref(), Some("imported-from-dir")); + assert_eq!(info.status, BoxStatus::Stopped); + + imported + .start() + .await + .expect("Failed to start imported box"); + imported.stop().await.expect("Failed to stop imported box"); + + let _ = runtime.shutdown(Some(common::TEST_SHUTDOWN_TIMEOUT)).await; +} + +#[tokio::test] +async fn test_store_export_import_roundtrip() { + let home = boxlite_test_utils::home::PerTestBoxHome::new(); + let runtime = BoxliteRuntime::new(BoxliteOptions { + home_dir: home.path.clone(), + image_registries: common::test_registries(), + }) + .expect("create runtime"); + let source = create_stopped_box(&runtime).await; + + let export_dir = TempDir::new_in("/tmp").unwrap(); + let store_root = export_dir.path().join("store"); + + let archive = source + .export( + ExportOptions { + as_directory: true, + archive_name: Some("backup".to_string()), + }, + &store_root, + ) + .await + .expect("Failed to publish box into store"); + + // The archive is addressed by its manifest inside the store. + assert!(archive.path().ends_with("archives/backup.json")); + assert!(store_root.join("layers").is_dir()); + let store = boxlite::ArchiveStore::open(&store_root).expect("open store"); + assert_eq!(store.archives().expect("list"), vec!["backup".to_string()]); + + let imported = runtime + .import_box(archive, Some("imported-from-store".to_string())) + .await + .expect("Failed to import box from store"); + + let info = imported.info().await.expect("get imported box info"); + assert_eq!(info.name.as_deref(), Some("imported-from-store")); + assert_eq!(info.status, BoxStatus::Stopped); + + imported + .start() + .await + .expect("Failed to start imported box"); + imported.stop().await.expect("Failed to stop imported box"); + + // Dropping the only manifest lets a sweep empty the pool. + assert!(store.remove("backup").expect("remove")); + let report = store.gc(std::time::Duration::ZERO).expect("gc"); + assert!(report.swept >= 1); + assert_eq!(store_root.join("layers").read_dir().unwrap().count(), 0); + + let _ = runtime.shutdown(Some(common::TEST_SHUTDOWN_TIMEOUT)).await; +} + #[tokio::test] async fn test_export_import_preserves_box_options() { let home = boxlite_test_utils::home::PerTestBoxHome::new(); @@ -255,6 +359,34 @@ async fn test_export_running_box() { imported.start().await.expect("Start imported box"); imported.stop().await.expect("Stop imported box"); + let mirror = export_dir.path().join("running-mirror"); + let directory_archive = source + .export( + ExportOptions { + as_directory: true, + ..Default::default() + }, + &mirror, + ) + .await + .expect("Directory export on running box should succeed"); + assert!(directory_archive.path().join("manifest.json").exists()); + let imported_directory = runtime + .import_box( + directory_archive, + Some("imported-running-directory".to_string()), + ) + .await + .expect("Directory archive from running box should import"); + imported_directory + .start() + .await + .expect("Start directory-imported box"); + imported_directory + .stop() + .await + .expect("Stop directory-imported box"); + source.stop().await.expect("Stop source box"); let _ = runtime.shutdown(Some(common::TEST_SHUTDOWN_TIMEOUT)).await; diff --git a/src/boxlite/tests/minio_backup_roundtrip.rs b/src/boxlite/tests/minio_backup_roundtrip.rs new file mode 100644 index 000000000..ab6ec3154 --- /dev/null +++ b/src/boxlite/tests/minio_backup_roundtrip.rs @@ -0,0 +1,159 @@ +//! Real backup round-trip through MinIO. +//! +//! Proves the claim that backing a box up to S3-compatible object storage +//! needs nothing inside boxlite: export produces a file, any S3 client moves +//! it, and import reads it back. The archive crosses a real MinIO server — +//! uploaded, deleted locally, re-downloaded — before being imported and run. +//! +//! Requires a MinIO reachable at `BOXLITE_TEST_S3_ENDPOINT` (default +//! http://127.0.0.1:29000) with the bucket `BOXLITE_TEST_S3_BUCKET` +//! (default boxlite-backup). Skips itself when that is absent, so it never +//! fails a normal test run. + +mod common; + +use boxlite::runtime::options::{BoxliteOptions, ExportOptions}; +use boxlite::{BoxCommand, BoxliteRuntime}; +use std::path::Path; +use std::process::Command; +use tempfile::TempDir; + +fn endpoint() -> String { + std::env::var("BOXLITE_TEST_S3_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:29000".to_string()) +} + +fn bucket() -> String { + std::env::var("BOXLITE_TEST_S3_BUCKET").unwrap_or_else(|_| "boxlite-backup".to_string()) +} + +/// Run the aws CLI against the MinIO endpoint. +fn aws(args: &[&str]) -> std::process::Output { + Command::new("aws") + .env("AWS_ACCESS_KEY_ID", "minioadmin") + .env("AWS_SECRET_ACCESS_KEY", "minioadmin") + .env("AWS_DEFAULT_REGION", "us-east-1") + .arg("--endpoint-url") + .arg(endpoint()) + .args(args) + .output() + .expect("run aws cli") +} + +fn minio_available() -> bool { + let out = aws(&["s3", "ls", &format!("s3://{}", bucket())]); + out.status.success() +} + +fn sha256(path: &Path) -> String { + let out = Command::new("shasum") + .args(["-a", "256"]) + .arg(path) + .output() + .expect("shasum"); + String::from_utf8_lossy(&out.stdout) + .split_whitespace() + .next() + .unwrap_or_default() + .to_string() +} + +#[tokio::test] +async fn a_box_survives_a_round_trip_through_minio() { + if !minio_available() { + eprintln!("skipping: no MinIO bucket {} at {}", bucket(), endpoint()); + return; + } + + let home = boxlite_test_utils::home::PerTestBoxHome::new(); + let runtime = BoxliteRuntime::new(BoxliteOptions { + home_dir: home.path.clone(), + image_registries: common::test_registries(), + }) + .expect("create runtime"); + + // A box carrying a marker only a genuine restore can reproduce. + let source = runtime + .create(common::alpine_opts(), Some("minio-src".to_string())) + .await + .expect("create box"); + source.start().await.expect("start"); + let marker = "backed-up-through-minio"; + let cmd = BoxCommand::new("sh").args(["-c", &format!("echo {marker} > /root/marker")]); + source.exec(cmd).await.expect("exec").wait().await.ok(); + source.stop().await.expect("stop"); + + // Export, then hand the file to object storage and forget it locally. + let export_dir = TempDir::new_in("/tmp").unwrap(); + let archive = source + .export(ExportOptions::default(), export_dir.path()) + .await + .expect("export"); + let local_digest = sha256(archive.path()); + let size = std::fs::metadata(archive.path()).unwrap().len(); + let key = format!("s3://{}/minio-roundtrip.boxlite", bucket()); + + let up = aws(&["s3", "cp", &archive.path().to_string_lossy(), &key]); + assert!( + up.status.success(), + "upload failed: {}", + String::from_utf8_lossy(&up.stderr) + ); + std::fs::remove_file(archive.path()).expect("drop the local archive"); + assert!(!archive.path().exists(), "the archive must be gone locally"); + + // Pull it back from MinIO and require the bytes to be identical. + let restore_dir = TempDir::new_in("/tmp").unwrap(); + let restored = restore_dir.path().join("restored.boxlite"); + let down = aws(&["s3", "cp", &key, &restored.to_string_lossy()]); + assert!( + down.status.success(), + "download failed: {}", + String::from_utf8_lossy(&down.stderr) + ); + assert_eq!( + sha256(&restored), + local_digest, + "MinIO must return the archive byte-for-byte" + ); + println!("\n=== archive crossed MinIO: {size} bytes, sha256 {local_digest} ==="); + + // Import the downloaded archive and prove the box actually works. + let imported = runtime + .import_box( + boxlite::runtime::options::BoxArchive::new(restored), + Some("minio-restored".to_string()), + ) + .await + .expect("import the archive fetched from MinIO"); + + imported.start().await.expect("start the restored box"); + let read_back = BoxCommand::new("cat").args(["/root/marker"]); + let mut exec = imported + .exec(read_back) + .await + .expect("exec on restored box"); + + let mut stdout = String::new(); + if let Some(mut stream) = exec.stdout() { + use futures::StreamExt; + while let Some(chunk) = stream.next().await { + stdout.push_str(&chunk); + } + } + let result = exec.wait().await.expect("wait"); + assert_eq!(result.exit_code, 0, "reading the marker must succeed"); + assert_eq!( + stdout.trim(), + marker, + "restored box must carry the marker written before backup" + ); + println!( + "=== restored box returned the marker: {:?} ===\n", + stdout.trim() + ); + + imported.stop().await.expect("stop"); + let _ = aws(&["s3", "rm", &key]); + let _ = runtime.shutdown(Some(common::TEST_SHUTDOWN_TIMEOUT)).await; +}