Enhance blob metadata validation and simplify API structure

Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["nydus", "nydus-backend", "nydus-config", "nydus-core", "nydus-error", "nydus-format", "nydus-storage", "nydus-telemetry"]
 
 [workspace.dependencies]
-clap = { version = "4", features = ["derive", "env"] }
+clap = { version = "4", features = ["derive", "env", "string"] }
 libc = "0.2"
 memmap2 = "0.9"
 serde = { version = "1", features = ["derive"] }

docs/nydus.md

@@ -1062,7 +1062,7 @@ first logical external data block starts at offset 0
 
 blob_meta then maps that logical byte offset to a compressed range in the full
 blob's data region. The block is mapped to its block_group by
-`block_group_index = blkaddr >> block_group_block_bits`, and the block_group entry gives the
+`block_group_index = blkaddr >> block_group_block_count_bits`, and the block_group entry gives the
 encoded `compressed_offset` (for example 0 for the first encoded block_group).
 ```
 
@@ -1113,8 +1113,8 @@ embedded blob meta region
 | block_groups_offset                 |
 | chunk_count                   |
 | block_group_count                   |
-| chunk_block_bits (u8)         |
-| block_group_block_bits (u8 + pad)   |
+| chunk_block_count_bits (u8)         |
+| block_group_block_count_bits (u8 + pad)   |
 | reserved tail (compat area)   |
 +-------------------------------+
 | chunk entries                 |
@@ -1167,16 +1167,16 @@ Header details:
 	chunk table.
 - `chunk_count` is the number of chunk entries.
 - `block_group_count` is the number of compressed block group entries.
-- `chunk_block_bits` is log2 of the EROFS chunk size in 4 KiB blocks:
-	`chunk_size = 4096 << chunk_block_bits`, so the default 1 MiB chunk stores
+- `chunk_block_count_bits` is log2 of the EROFS chunk size in 4 KiB blocks:
+	`chunk_size = 4096 << chunk_block_count_bits`, so the default 1 MiB chunk stores
 	8. Storing the exponent EROFS-style (the same quantity as `chunk_format &
 	EROFS_CHUNK_FORMAT_BLKBITS_MASK`) makes non-power-of-two chunk sizes
 	unrepresentable and feeds the shift-based offset math directly.
-- `block_group_block_bits` is log2 of the per-block group block count, same
-	representation as `chunk_block_bits` (the default 4 MiB block group stores 10).
-	Every block group except the last is exactly `1 << block_group_block_bits` blocks, so
+- `block_group_block_count_bits` is log2 of the per-block group block count, same
+	representation as `chunk_block_count_bits` (the default 4 MiB block group stores 10).
+	Every block group except the last is exactly `1 << block_group_block_count_bits` blocks, so
 	the read path maps a block to its block group with
-	`block_group_index = block_id >> block_group_block_bits` in O(1). The two exponents
+	`block_group_index = block_id >> block_group_block_count_bits` in O(1). The two exponents
 	are adjacent `u8`s at offset 48; the six bytes after them are reserved.
 - The header is one EROFS block (4096 bytes): the chunk table starts block
 	aligned by construction, and everything between the last field and the end
@@ -1206,7 +1206,7 @@ Block group details:
 
 - Block groups are formed by packing whole decoded blocks up to `--block-group-size`
 	regardless of chunk boundaries, then compressing the batch as one unit. So
-	every block group but the last is exactly `1 << block_group_block_bits` blocks.
+	every block group but the last is exactly `1 << block_group_block_count_bits` blocks.
 - `uncompressed_block_offset` is the decoded cache 4 KiB block offset for the
 	block group. Block groups are dense and contiguous in the decoded address space.
 - `compressed_offset` is the encoded payload's byte offset within the data
@@ -1355,7 +1355,7 @@ ondemand blob — named by SHA256(full blob), one new nydus layer
 Every block group entry in the ondemand blob is a **redirect**: instead of
 describing this blob's own decoded address space, it names the source block group
 it is a copy of. Block group sizes follow the source block groups, so the uniform-size
-invariant is relaxed and the O(1) `block >> block_group_block_bits` lookup is never
+invariant is relaxed and the O(1) `block >> block_group_block_count_bits` lookup is never
 used on an ondemand blob:
 
 ```text
@@ -1471,7 +1471,7 @@ When mounting with `--bootstrap + --blob-dir`:
 	the cache directory. The cache verifies the blob meta header crc32c before
 	mmaping the cached file and using its chunk entries.
 7. Reads use logical uncompressed offsets from inode chunk indexes. The cache
-	layer maps an offset to its block group in O(1) with `block >> block_group_block_bits`,
+	layer maps an offset to its block group in O(1) with `block >> block_group_block_count_bits`,
 	ensures every block group covering the requested range is fetched and decoded from
 	the data region (validating block group CRC32C), and then reads the bytes straight
 	out of the cache file. The cache file mirrors the dense decoded address space,
@@ -1548,7 +1548,7 @@ Per-blob prefetch streams block groups into the cache:
 
 - The blob meta block groups are the compression/cache unit. Prefetch reads the data
 	region in windows that accumulate consecutive block groups up to the default block group
-	uncompressed size (1 MiB), so each window decode covers one or more block groups.
+	uncompressed size (4 MiB), so each window decode covers one or more block groups.
 - For each window it issues a single contiguous backend range read, then decodes
 	each contained block group (plain copy or zstd), validates length and CRC32C, writes
 	the decoded bytes to the cache file at the block group's uncompressed offset, and

nydus-backend/src/local.rs

@@ -198,10 +198,7 @@ impl BlobBackend for Local {
     fn blob_metadata(&self, blob_id: &[u8; SHA256_DIGEST_SIZE]) -> io::Result<BlobMetadata> {
         let source = self.resolved_source(blob_id)?;
         let data = self.read_blob_metadata_bytes(&source)?;
-        BlobMetadata::loader()
-            .blob_id(*blob_id)
-            .from_bytes(&data)
-            .map_err(io::Error::other)
+        BlobMetadata::from_bytes(&data, false).map_err(io::Error::other)
     }
 
     fn save_blob_metadata(&self, blob_id: &[u8; SHA256_DIGEST_SIZE], dst: &Path) -> io::Result<()> {
@@ -260,16 +257,16 @@ fn probe_full_blob_source(
 mod tests {
     use super::*;
     use crate::ReadKind;
-    use nydus_format::blob::{BlobMetadataBlockGroup, BlobMetadataChunk};
+    use nydus_format::blob::{BlobMetadataBlockGroup, BlobMetadataChunk, BlobMetadataCompressor};
     use nydus_format::utils::sha256_bytes;
     use tempfile::tempdir;
 
-    fn blob_metadata(blob_id: [u8; SHA256_DIGEST_SIZE], payload: &[u8]) -> BlobMetadata {
-        BlobMetadata::from_parts(
-            blob_id,
+    fn blob_metadata(payload: &[u8]) -> BlobMetadata {
+        BlobMetadata::new(
+            BlobMetadataCompressor::None,
             1,
-            vec![BlobMetadataBlockGroup::new(0, 1, 0, 4096, crc32c::crc32c(payload)).unwrap()],
             vec![BlobMetadataChunk::new(*blake3::hash(payload).as_bytes(), 0, 1).unwrap()],
+            vec![BlobMetadataBlockGroup::new(0, 1, 0, 4096, crc32c::crc32c(payload)).unwrap()],
         )
         .unwrap()
     }
@@ -280,13 +277,8 @@ mod tests {
     fn local_backend_reads_full_blob_file_and_sidecar_meta() {
         let dir = tempdir().unwrap();
         let payload = vec![0xabu8; 4096];
-        let data_blob_id = sha256_bytes(&payload);
-        let full_blob_id = write_minimal_full_blob(
-            dir.path(),
-            &payload,
-            &blob_metadata(data_blob_id, &payload),
-            true,
-        );
+        let full_blob_id =
+            write_minimal_full_blob(dir.path(), &payload, &blob_metadata(&payload), true);
 
         let backend = Local::new(dir.path().to_path_buf());
         let blob_metadata = backend.blob_metadata(&full_blob_id).unwrap();
@@ -309,12 +301,8 @@ mod tests {
         let dir = tempdir().unwrap();
         let payload = vec![0xcdu8; 4096];
         let data_blob_id = sha256_bytes(&payload);
-        let full_blob_id = write_minimal_full_blob(
-            dir.path(),
-            &payload,
-            &blob_metadata(data_blob_id, &payload),
-            false,
-        );
+        let full_blob_id =
+            write_minimal_full_blob(dir.path(), &payload, &blob_metadata(&payload), false);
         let backend = Local::new(dir.path().to_path_buf());
 
         let blob_metadata = backend.blob_metadata(&full_blob_id).unwrap();

nydus-backend/src/registry/mod.rs

@@ -649,9 +649,7 @@ impl Registry {
             ReadContext::raw(ReadKind::OnDemand),
         )?;
 
-        BlobMetadata::loader()
-            .blob_id(*blob_id)
-            .from_bytes(&blob_metadata_bytes)
+        BlobMetadata::from_bytes(&blob_metadata_bytes, false)
             .map_err(|err| RegistryError::Io(io::Error::other(err)))
     }
 

nydus-core/src/blob.rs

@@ -130,7 +130,7 @@ impl Blobs {
                     blocks: info.blocks,
                     cache_size,
                     cache_path,
-                    is_redirect: cache.is_redirect_blob(),
+                    is_redirect: cache.is_redirect(),
                 })
             })
             .collect()

nydus-core/src/reader/data.rs

@@ -6,7 +6,7 @@ use nydus_format::erofs::{
     EROFS_CHUNK_INDEX_SIZE, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE,
     EROFS_INODE_FLAT_PLAIN, EROFS_NULL_ADDR,
 };
-use nydus_format::utils::round_up;
+use nydus_format::utils::align_up_usize;
 
 use super::{ErofsReader, RawBlobInfo};
 
@@ -135,7 +135,8 @@ impl ErofsReader {
         let nchunks = inode.size().div_ceil(chunk_size) as usize;
         let inode_offset = self.nid_to_offset(nid);
         let header_size = inode.header_size() + inode.xattr_size();
-        let index_offset = inode_offset + round_up(header_size, EROFS_CHUNK_INDEX_SIZE);
+        let index_offset = inode_offset
+            + align_up_usize(header_size, EROFS_CHUNK_INDEX_SIZE).expect("alignment overflowed");
         let index_total = nchunks * EROFS_CHUNK_INDEX_SIZE;
         self.mmap_slice(index_offset, index_total)
     }

nydus-core/src/reader/metadata.rs

@@ -5,7 +5,7 @@ use nydus_format::erofs::{
     EROFS_INODE_EXTENDED_SIZE, EROFS_INODE_FLAT_INLINE, EROFS_INODE_FLAT_PLAIN,
     EROFS_XATTR_ENTRY_HEADER_SIZE, EROFS_XATTR_IBODY_HEADER_SIZE,
 };
-use nydus_format::utils::round_up;
+use nydus_format::utils::align_up_usize;
 
 use super::{ErofsReader, RawDirEntry};
 
@@ -288,7 +288,7 @@ impl ErofsReader {
             result.push((full_name, value));
 
             // Advance to next entry (4-byte aligned)
-            pos = round_up(value_end, XATTR_ENTRY_ALIGN);
+            pos = align_up_usize(value_end, XATTR_ENTRY_ALIGN).expect("alignment overflowed");
         }
 
         Ok(result)

nydus-core/src/reader/mod.rs

@@ -276,8 +276,8 @@ impl ErofsReader {
     /// Return whether the blob identified by `blob_index` is an "ondemand"
     /// redirect blob (produced by `nydus optimize`). Opens the blob cache,
     /// which reads the local blob meta but performs no data prefetch.
-    pub fn is_redirect_blob(&self, blob_index: u16) -> io::Result<bool> {
-        self.blobs.is_redirect_blob(blob_index)
+    pub fn is_redirect(&self, blob_index: u16) -> io::Result<bool> {
+        self.blobs.is_redirect(blob_index)
     }
 
     /// Prefetch every block group of the blob identified by `blob_index`. An

nydus-format/src/blob/algorithm.rs

@@ -6,13 +6,16 @@ use crate::blob::metadata::BlobMetadataFlags;
 use crate::error::{Error, Result};
 use std::fmt;
 
+/// The block group payload compressor a blob meta declares. `None` is the
+/// absent-flag state: payloads are stored raw.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum BlobMetadataCompressor {
     None,
     Zstd,
 }
 
 impl BlobMetadataCompressor {
+    /// The flag bit encoding this compressor, empty for `None`.
     pub fn flag(self) -> BlobMetadataFlags {
         match self {
             Self::None => BlobMetadataFlags::empty(),
@@ -21,6 +24,8 @@ impl BlobMetadataCompressor {
     }
 }
 
+/// The lowercase algorithm name, as surfaced in the `build` and `check`
+/// summaries.
 impl fmt::Display for BlobMetadataCompressor {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(match self {
@@ -42,19 +47,24 @@ impl From<BlobMetadataFlags> for BlobMetadataCompressor {
     }
 }
 
+/// The chunk digest algorithm a blob meta declares, always explicit (see
+/// the `TryFrom` below).
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum BlobMetadataDigester {
     Blake3,
 }
 
 impl BlobMetadataDigester {
+    /// The flag bit encoding this digester.
     pub fn flag(self) -> BlobMetadataFlags {
         match self {
             Self::Blake3 => BlobMetadataFlags::DIGESTER_BLAKE3,
         }
     }
 }
 
+/// The lowercase algorithm name, as surfaced in the `build` and `check`
+/// summaries.
 impl fmt::Display for BlobMetadataDigester {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(match self {