feat(archive): shared layer store with reference-counted sweep

sdks/node/lib/native-contracts.ts

@@ -360,7 +360,19 @@ export interface NativeBoxConnection {
 
 export type JsCloneOptions = Record<string, never>;
 
-export type JsExportOptions = Record<string, never>;
+export interface JsExportOptions {
+  /**
+   * Write a directory of content-addressed objects instead of one `.boxlite`
+   * file, so mirroring it to object storage transfers only the objects the
+   * destination lacks.
+   */
+  asDirectory?: boolean;
+  /**
+   * Publish into a shared layer store under this archive name (requires
+   * `asDirectory`; the destination is then the store root).
+   */
+  archiveName?: string;
+}
 
 export interface JsBox {
   readonly id: string;

sdks/node/src/snapshot_options.rs

@@ -14,14 +14,25 @@ impl From<JsSnapshotOptions> for SnapshotOptions {
     }
 }
 
-/// Options for exporting a box (forward-compatible placeholder).
+/// Options for exporting a box.
 #[napi(object)]
 #[derive(Clone, Debug)]
-pub struct JsExportOptions {}
+pub struct JsExportOptions {
+    /// Write a directory of content-addressed objects instead of one
+    /// `.boxlite` file, so mirroring it to object storage transfers only the
+    /// objects the destination lacks.
+    pub as_directory: Option<bool>,
+    /// Publish into a shared layer store under this archive name (requires
+    /// `asDirectory`; the destination is then the store root).
+    pub archive_name: Option<String>,
+}
 
 impl From<JsExportOptions> for ExportOptions {
-    fn from(_js: JsExportOptions) -> Self {
-        ExportOptions {}
+    fn from(js: JsExportOptions) -> Self {
+        ExportOptions {
+            as_directory: js.as_directory.unwrap_or(false),
+            archive_name: js.archive_name,
+        }
     }
 }
 
@@ -48,7 +59,10 @@ mod tests {
 
     #[test]
     fn export_options_from_js() {
-        let js = JsExportOptions {};
+        let js = JsExportOptions {
+            as_directory: None,
+            archive_name: None,
+        };
         let _opts: ExportOptions = js.into();
     }
 

sdks/python/src/snapshot_options.rs

@@ -22,22 +22,39 @@ impl From<PySnapshotOptions> for SnapshotOptions {
     }
 }
 
-/// Options for exporting a box (forward-compatible placeholder).
+/// Options for exporting a box.
 #[pyclass(name = "ExportOptions")]
 #[derive(Clone)]
-pub(crate) struct PyExportOptions {}
+pub(crate) struct PyExportOptions {
+    /// Write a directory of content-addressed objects instead of one
+    /// `.boxlite` file, so mirroring it to object storage transfers only the
+    /// objects the destination lacks.
+    #[pyo3(get, set)]
+    pub(crate) as_directory: bool,
+    /// Publish into a shared layer store under this archive name (requires
+    /// `as_directory`; the destination is then the store root).
+    #[pyo3(get, set)]
+    pub(crate) archive_name: Option<String>,
+}
 
 #[pymethods]
 impl PyExportOptions {
     #[new]
-    fn new() -> Self {
-        Self {}
+    #[pyo3(signature = (as_directory = false, archive_name = None))]
+    fn new(as_directory: bool, archive_name: Option<String>) -> Self {
+        Self {
+            as_directory,
+            archive_name,
+        }
     }
 }
 
 impl From<PyExportOptions> for ExportOptions {
-    fn from(_py: PyExportOptions) -> Self {
-        ExportOptions {}
+    fn from(py: PyExportOptions) -> Self {
+        ExportOptions {
+            as_directory: py.as_directory,
+            archive_name: py.archive_name,
+        }
     }
 }
 

src/boxlite/src/db/base_disk.rs

@@ -96,8 +96,8 @@ impl BaseDiskStore {
         let conn = self.db.conn();
         db_err!(conn.execute(
             "INSERT INTO base_disk \
-             (id, source_box_id, name, kind, base_path, created_at, json) \
-             VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
+             (id, source_box_id, name, kind, base_path, created_at, json, digest) \
+             VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
             rusqlite::params![
                 &disk.id,
                 &disk.source_box_id,
@@ -106,11 +106,43 @@ impl BaseDiskStore {
                 &disk.disk_info.base_path,
                 disk.created_at,
                 json,
+                &disk.digest,
             ],
         ))?;
         Ok(())
     }
 
+    /// Find a base disk by its content digest.
+    ///
+    /// Only layers whose digest has already been computed are visible here;
+    /// see [`BaseDisk::digest`] for why it is filled in lazily.
+    pub(crate) fn find_by_digest(&self, digest: &str) -> BoxliteResult<Option<BaseDiskInfo>> {
+        let conn = self.db.conn();
+        let result = db_err!(
+            conn.query_row(
+                "SELECT id, source_box_id, name, kind, base_path, \
+                 created_at, json FROM base_disk WHERE digest = ?1",
+                rusqlite::params![digest],
+                row_to_record,
+            )
+            .optional()
+        )?;
+        Ok(result)
+    }
+
+    /// Record a layer's content digest, in both the indexed column and the
+    /// JSON blob so the two cannot drift.
+    pub(crate) fn set_digest(&self, id: &BaseDiskID, digest: &str) -> BoxliteResult<()> {
+        let conn = self.db.conn();
+        db_err!(conn.execute(
+            "UPDATE base_disk \
+             SET digest = ?2, json = json_set(json, '$.digest', ?2) \
+             WHERE id = ?1",
+            rusqlite::params![id, digest],
+        ))?;
+        Ok(())
+    }
+
     /// Find a base disk by its ID.
     #[allow(dead_code)] // used in lineage.rs tests
     pub(crate) fn find_by_id(&self, id: &BaseDiskID) -> BoxliteResult<Option<BaseDiskInfo>> {
@@ -330,6 +362,7 @@ mod tests {
                 size_bytes: 512,
             },
             created_at: chrono::Utc::now().timestamp(),
+            digest: None,
         }
     }
 
@@ -686,6 +719,7 @@ mod tests {
                 size_bytes: 1024,
             },
             created_at: 1700000000,
+            digest: None,
         };
         store.insert(&disk).unwrap();
 

src/boxlite/src/db/migration/mod.rs

@@ -11,6 +11,7 @@ mod v5_to_v6;
 mod v6_to_v7;
 mod v7_to_v8;
 mod v8_to_v9;
+mod v9_to_v10;
 
 use std::path::Path;
 
@@ -81,5 +82,6 @@ fn all_migrations() -> Vec<Box<dyn Migration>> {
         Box::new(v6_to_v7::MoveDisksAndAddBaseDisk),
         Box::new(v7_to_v8::RenameNetworkSpec),
         Box::new(v8_to_v9::PreservePublishedPorts),
+        Box::new(v9_to_v10::AddBaseDiskDigest),
     ]
 }

src/boxlite/src/db/migration/v6_to_v7.rs

@@ -18,6 +18,29 @@ use crate::db::schema;
 use crate::runtime::id::BaseDiskID;
 use crate::runtime::id::BaseDiskIDMint;
 
+/// The `base_disk` table exactly as v7 created it.
+///
+/// Frozen on purpose: a migration must reproduce the schema of its own era, so
+/// it cannot read `schema::BASE_DISK_TABLE`. That constant tracks the current
+/// schema, and every column later added to it would otherwise appear here too
+/// — making the `ALTER TABLE` in the migration that introduces the column fail
+/// with "duplicate column name" for anyone upgrading from v6 or earlier.
+const V7_BASE_DISK_TABLE: &str = r#"
+CREATE TABLE IF NOT EXISTS base_disk (
+    id TEXT PRIMARY KEY NOT NULL,
+    source_box_id TEXT NOT NULL,
+    name TEXT,
+    kind TEXT NOT NULL CHECK(kind IN ('snapshot', 'clone_base', 'rootfs')),
+    base_path TEXT NOT NULL,
+    created_at INTEGER NOT NULL,
+    json TEXT NOT NULL,
+    UNIQUE(source_box_id, name)
+);
+CREATE INDEX IF NOT EXISTS idx_base_disk_source ON base_disk(source_box_id);
+CREATE INDEX IF NOT EXISTS idx_base_disk_kind ON base_disk(kind);
+CREATE INDEX IF NOT EXISTS idx_base_disk_path ON base_disk(base_path);
+"#;
+
 pub(crate) struct MoveDisksAndAddBaseDisk;
 
 impl Migration for MoveDisksAndAddBaseDisk {
@@ -33,7 +56,7 @@ impl Migration for MoveDisksAndAddBaseDisk {
 
     fn run(&self, conn: &Connection, home_dir: Option<&Path>) -> BoxliteResult<()> {
         // 1. Create base_disk table (for clone bases and rootfs cache).
-        db_err!(conn.execute_batch(schema::BASE_DISK_TABLE))?;
+        db_err!(conn.execute_batch(V7_BASE_DISK_TABLE))?;
 
         // 2. Create snapshot table (for per-box snapshots).
         db_err!(conn.execute_batch(schema::SNAPSHOT_TABLE))?;
@@ -307,7 +330,7 @@ mod tests {
     /// Create an in-memory DB with the base_disk table for migration tests.
     fn test_db() -> Connection {
         let conn = Connection::open_in_memory().unwrap();
-        conn.execute_batch(schema::BASE_DISK_TABLE).unwrap();
+        conn.execute_batch(V7_BASE_DISK_TABLE).unwrap();
         conn.execute_batch(schema::SNAPSHOT_TABLE).unwrap();
         conn.execute_batch(schema::BASE_DISK_REF_TABLE).unwrap();
         conn

src/boxlite/src/db/migration/v9_to_v10.rs

@@ -0,0 +1,105 @@
+//! Migration v9 → v10: Add a content digest column to `base_disk`.
+//!
+//! Layers are addressed by content when they travel between hosts, so a base
+//! needs a digest that can be looked up without scanning every JSON blob. The
+//! column is nullable and left empty here: a base is immutable, so its digest
+//! is computed once, on first use, rather than by hashing every existing layer
+//! during startup.
+
+use std::path::Path;
+
+use rusqlite::Connection;
+
+use boxlite_shared::errors::{BoxliteError, BoxliteResult};
+
+use super::{Migration, db_err};
+
+pub(crate) struct AddBaseDiskDigest;
+
+impl Migration for AddBaseDiskDigest {
+    fn source_version(&self) -> i32 {
+        9
+    }
+    fn target_version(&self) -> i32 {
+        10
+    }
+    fn description(&self) -> &str {
+        "Add base_disk.digest column and index"
+    }
+
+    fn run(&self, conn: &Connection, _home_dir: Option<&Path>) -> BoxliteResult<()> {
+        db_err!(conn.execute("ALTER TABLE base_disk ADD COLUMN digest TEXT", []))?;
+        db_err!(conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_base_disk_digest ON base_disk(digest)",
+            [],
+        ))?;
+
+        tracing::info!("Added base_disk.digest column (populated lazily on first use)");
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn v9_base_disk_table(conn: &Connection) {
+        conn.execute_batch(
+            r#"CREATE TABLE base_disk (
+                id TEXT PRIMARY KEY NOT NULL,
+                source_box_id TEXT NOT NULL,
+                name TEXT,
+                kind TEXT NOT NULL,
+                base_path TEXT NOT NULL,
+                created_at INTEGER NOT NULL,
+                json TEXT NOT NULL
+            );"#,
+        )
+        .unwrap();
+        conn.execute(
+            "INSERT INTO base_disk (id, source_box_id, name, kind, base_path, created_at, json) \
+             VALUES ('abc', 'box1', NULL, 'clone_base', '/bases/abc.qcow2', 1, '{}')",
+            [],
+        )
+        .unwrap();
+    }
+
+    #[test]
+    fn existing_rows_survive_with_a_null_digest() {
+        let conn = Connection::open_in_memory().unwrap();
+        v9_base_disk_table(&conn);
+
+        AddBaseDiskDigest.run(&conn, None).unwrap();
+
+        // A pre-existing layer keeps a NULL digest, so the lazy computation
+        // path — not the migration — is what fills it in. Hashing every base
+        // here would read every cached layer on the first startup after an
+        // upgrade.
+        let digest: Option<String> = conn
+            .query_row("SELECT digest FROM base_disk WHERE id = 'abc'", [], |r| {
+                r.get(0)
+            })
+            .unwrap();
+        assert_eq!(digest, None);
+    }
+
+    #[test]
+    fn digest_column_is_writable_after_migration() {
+        let conn = Connection::open_in_memory().unwrap();
+        v9_base_disk_table(&conn);
+
+        AddBaseDiskDigest.run(&conn, None).unwrap();
+        conn.execute(
+            "UPDATE base_disk SET digest = 'sha256:dead' WHERE id = 'abc'",
+            [],
+        )
+        .unwrap();
+
+        let digest: Option<String> = conn
+            .query_row("SELECT digest FROM base_disk WHERE id = 'abc'", [], |r| {
+                r.get(0)
+            })
+            .unwrap();
+        assert_eq!(digest.as_deref(), Some("sha256:dead"));
+    }
+}

src/boxlite/src/db/schema.rs

@@ -7,7 +7,7 @@
 //! Each table has queryable columns for efficient filtering + JSON blob for full data.
 
 /// Current schema version.
-pub const SCHEMA_VERSION: i32 = 9;
+pub const SCHEMA_VERSION: i32 = 10;
 
 /// Schema version tracking table.
 pub const SCHEMA_VERSION_TABLE: &str = r#"
@@ -115,11 +115,13 @@ CREATE TABLE IF NOT EXISTS base_disk (
     base_path TEXT NOT NULL,
     created_at INTEGER NOT NULL,
     json TEXT NOT NULL,
+    digest TEXT,
     UNIQUE(source_box_id, name)
 );
 CREATE INDEX IF NOT EXISTS idx_base_disk_source ON base_disk(source_box_id);
 CREATE INDEX IF NOT EXISTS idx_base_disk_kind ON base_disk(kind);
 CREATE INDEX IF NOT EXISTS idx_base_disk_path ON base_disk(base_path);
+CREATE INDEX IF NOT EXISTS idx_base_disk_digest ON base_disk(digest);
 "#;
 
 /// Base disk reference table (added in v7).