diff --git a/Cargo.lock b/Cargo.lock index ec249db0d6e..7ab71f02776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,17 +126,11 @@ dependencies = [ "rustversion", ] -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "asn1-rs" @@ -463,16 +457,15 @@ checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6" [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" dependencies = [ - "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.2.17", + "cpufeatures 0.3.0", ] [[package]] @@ -2579,6 +2572,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2889,6 +2891,7 @@ dependencies = [ "hyper-util", "libc", "libublk", + "lz4_flex", "memmap2", "nydus-backend", "nydus-config", @@ -2966,6 +2969,7 @@ dependencies = [ "nydus-storage", "tempfile", "tracing", + "zstd", ] [[package]] @@ -2991,6 +2995,7 @@ dependencies = [ "sha2 0.10.9", "tempfile", "thiserror 2.0.18", + "zstd", ] [[package]] @@ -3000,6 +3005,7 @@ dependencies = [ "blake3", "crc32c", "libc", + "lz4_flex", "memmap2", "nydus-backend", "nydus-config", @@ -5384,6 +5390,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + [[package]] name = "typeid" version = "1.0.3" diff --git a/Makefile b/Makefile index 7cfab260c9a..4d2b57494c0 100644 --- a/Makefile +++ b/Makefile @@ -211,7 +211,7 @@ test-nbd: release # is missing (nbd/ublk modules, Linux < 6.15 for fanotify) are skipped # individually. Requires root and fio; fanotify needs an ext4 cache dir — # TMPDIR is set to the repo's .test-tmp/ on the working tree filesystem. -test-bench: FEATURES=cli,fuse,nbd,ublk,fanotify +test-bench: FEATURES=cli,fuse,nbd,ublk,fanotify,fileio test-bench: release @test -n "$(GO_BIN)" || { echo "go not found; set GO=/abs/path/to/go or GO_BIN=/abs/path/to/go"; exit 1; } mkdir -p $(CURDIR)/.test-tmp diff --git a/README.md b/README.md index 1d9451ddd55..1ab807d9dc0 100644 --- a/README.md +++ b/README.md @@ -19,16 +19,17 @@ redesign in Rust. Compared with Nydus v2 (RAFS), v3 brings: - **CLI-friendly** — non-core capabilities are removed and binary components reduced: one `nydus` binary (`build` / `merge` / `check` / `optimize` / `fuse` / `ublk` / `uffd` / `fanotify`) plus the `nydusify` image orchestrator. Each layer is one - self-contained blob artifact (`data + bootstrap + blob meta + footer`) - named by its SHA256, with an optional standalone metadata-only bootstrap. + self-contained blob artifact (`data + zstd-compressed bootstrap + blob + meta + footer`) named by its SHA256, with an optional standalone + metadata-only bootstrap. - **Native EROFS format** — a fully standard EROFS layout compatible with erofs-utils and kernel mounting; filesystem and chunk metadata are fetched in bulk up front via the compact bootstrap, then file data loads on demand. -- **Decoupled dedup and compression units** — `--chunk-size` sets the - deduplication granularity (BLAKE3) while `--compress-size` sets the - compression and read unit (zstd, default 4 MiB), no longer tied to file - chunks: better compression efficiency and less read amplification, with - CRC32C validation enforced on every read path. +- **Decoupled chunking and compression units** — `--chunk-size` sets the + file chunk granularity while `--compress-size` sets the compression and + read unit (zstd, default 4 MiB), no longer tied to file chunks: better + compression efficiency and less read amplification, with CRC32C + validation enforced on every read path. - **On-demand loading** — file reads map to compressed groups through an O(1) logical-address lookup; only the touched groups are fetched, validated, decoded, and cached. @@ -87,63 +88,161 @@ is loaded on demand at runtime (that cost shows up inside Ready). cuts cold-start E2E from 22.62s to 8.94s (~2.5×), and against Nydus v2 (row 4 vs 6) from 17.75s to 8.94s (~2×). -### Read-path transport comparison (FUSE / NBD / ublk / fanotify) +### Read-path transport comparison (FUSE / NBD / ublk / fileio / fanotify) -`make test-bench` runs a unified cold-start benchmark where every serving -mode mounts the SAME locally built image (local backend, prefetch disabled, -separate caches), so the comparison isolates the read transport: +The table below compares every serving mode on a REAL application cold +start: all modes mount the SAME locally built image (separate caches), so +the comparison isolates the read transport: +- **OCI (full pull)** — baseline: download the gzip layer from the same + registry and fully extract it before starting the container (pipelined + `curl | gunzip | tar -x`, as containerd does), no lazy loading. +- **v2 FUSE** — optional column: the nydus v2 `nydusd` daemon serving the + same rootfs as a RAFS v6 (zstd) image, enabled by pointing + `NYDUSFS_BENCH_V2_NYDUSD` and `NYDUSFS_BENCH_V2_IMAGE_BIN` at v2 binaries. - **FUSE** — every read and metadata call is a userspace round trip through the `nydus fuse` daemon. - **NBD** — kernel EROFS over `/dev/nbdX`; cache misses reach the daemon through the NBD socket, metadata is served by the kernel EROFS driver. - **ublk** — like NBD, but block requests travel through `ublk_drv`'s io_uring SQE/CQE shared memory instead of a kernel socket (Linux 6.0+). +- **fileio** — kernel EROFS mounted file-backed over a FUSE export of the + flattened image; metadata is served by the kernel EROFS driver and only + backing-file reads reach the daemon (Linux 6.12+). - **fanotify** — kernel EROFS mount over the cache files; a `FAN_PRE_ACCESS` event fills missing ranges, warm reads never leave the kernel (Linux ≥ 6.15). -- **erofsfuse** — optional column: the C erofsfuse reference implementation - reading the blob directly. - -Methodology (per mode): wipe the nydus cache, drop the page cache, start the -daemon (recording **mount-ready** and **first-1MiB-read** latency), cold-read -the whole fio target (the end-to-end on-demand fetch path, reported as -**prewarm** throughput), then run every fio job and metadata benchmark with -the page cache dropped before each job — warm nydus cache, cold page cache. - - - -| Benchmark | Unit | FUSE | NBD | ublk | fanotify | -| --- | --- | ---: | ---: | ---: | ---: | -| Mount ready | s | 0.23 | 0.23 | 0.23 | 0.22 | -| First 1 MiB cold read | s | 0.024 | 0.006 | 0.006 | 0.025 | -| Prewarm (full-file cold fetch) | MiB/s | 902 | 1,345 | 1,239 | 804 | -| Sequential read 128K | MiB/s | 6,486 | 31,177 | 32,125 | 32,033 | -| Sequential read 4-job 128K | MiB/s | 24,991 | 109,374 | 58,732 | 55,039 | -| Random read 128K | MiB/s | 7,569 | 8,304 | 10,105 | 11,356 | -| Random read 4-job 128K | MiB/s | 35,271 | 44,448 | 41,836 | 44,726 | -| Random read 4K | IOPS | 1,225,676 | 1,269,728 | 1,318,339 | 1,283,015 | -| Random read 4K latency | µs | 0.7 | 0.7 | 0.7 | 0.7 | -| Stat | IOPS | 1,333,959 | 1,522,750 | 1,681,039 | 1,517,389 | -| Readdir | IOPS | 10,250 | 61,218 | 61,878 | 61,182 | -| Listxattr | IOPS | 15,449 | 2,216,639 | 2,248,846 | 2,246,635 | -| Getxattr | IOPS | 15,014 | 2,115,001 | 2,109,951 | 2,158,214 | -| Readdir + stat (`ls -l`) | IOPS | 93.1 | 95.1 | 126.0 | 118.5 | - -- Measured on Ubuntu 24.04 (arm64), Linux 7.0.0, ext4-backed blob store and - caches; corpus: 8 × 64 MiB + 256 × 1 MiB + 10,000 small files - (~850 MiB blob), 1 MiB chunk size. The erofsfuse column was not available - on this host. -- The unified suite replaces the earlier "Fanotify vs FUSE" (registry - backend) and "Block device vs FUSE" comparisons; their numbers were - produced by different setups and are not directly comparable. -- Cold-page `direct=0` fio jobs largely re-warm during each 20 s job (the - target file is 64 MiB), so sequential rows partly reflect page-cache and - readahead policy, not just protocol overhead. -- The kernel EROFS modes (NBD/ublk/fanotify) serve all metadata in-kernel, - which shows up as the readdir/xattr gap over FUSE; warm fanotify reads - never leave the kernel at all. + +Methodology: this is a REAL Next.js application end-to-end test, not a +synthetic fio run. The image is a production `next build` of +`create-next-app` on `node:20` (1.6 GiB rootfs, ~35k files), served from a +local OCI registry (`registry:2`); extra network latency is injected on the +registry traffic with `tc netem` to emulate remote registries. Per run the +nydus cache and the page cache are wiped, the image is mounted (nydus + +overlayfs), and the container is started with `runc` running `npm start`. +The reported time is mount start until the FIRST successful HTTP 200 from +the Next.js server — the moment the service is actually usable. Values are +the mean of 2 runs; run-to-run spread is < 5%. + + + +| Serving mode | Image size | RTT ≈ 0 | RTT +30 ms | RTT +50 ms | Data fetched | +| --- | ---: | ---: | ---: | ---: | ---: | +| v2 FUSE | 500 MiB | 1.72 s | 5.51 s | 8.64 s | 197 MiB | +| FUSE | 465 MiB | 1.44 s | 3.48 s | 4.90 s | 230 MiB | +| NBD | 465 MiB | 1.15 s | 3.28 s | 4.77 s | 231 MiB | +| ublk | 465 MiB | 1.04 s | 3.09 s | 4.62 s | 231 MiB | +| fileio | 465 MiB | 1.17 s | 3.21 s | 4.83 s | 230 MiB | +| fanotify | 465 MiB | 1.10 s | 3.20 s | 4.60 s | 230 MiB | +| FUSE + optimize | 521 MiB | 0.92 s | 1.56 s | 1.75 s | 230 MiB | +| NBD + optimize | 521 MiB | 0.72 s | 1.25 s | 1.67 s | 230 MiB | +| ublk + optimize | 521 MiB | 0.86 s | 1.14 s | 1.45 s | 230 MiB | +| fileio + optimize | 521 MiB | 0.72 s | 1.18 s | 1.51 s | 230 MiB | +| fanotify + optimize | 521 MiB | **0.59 s** | **1.08 s** | **1.38 s** | 230 MiB | + +- Measured on Ubuntu 24.04 (arm64), Linux 7.0.0; image built with 1 MiB + chunks, 4 MiB block groups, zstd. The v2 row is the same rootfs as a + RAFS v6 zstd image served by the v2 `nydusd` from the same registry. + All rows are 2-run means from one session. +- The "+ optimize" rows mount the same image after `nydus optimize` rewrote + it from a recorded boot trace: mounts stream the 56 MiB hot-data + "ondemand" blob over ONE connection (prefetch scope `ondemand`) instead + of paying a round trip per cache miss, which makes ready time nearly + RTT-independent. The stored image grows by that ondemand blob; the rest + of the working set still loads on demand. +- Image size counts what a registry transfer needs: the OCI row is the + gzip tar layer; the nydus rows are the full blob (zstd data plus the + zstd-compressed embedded bootstrap) and the gzipped merged bootstrap + (v2 bootstrap 7.5 MiB → 2.6 MiB gzipped, v3 bootstrap 21.7 MiB → + 0.8 MiB gzipped). The v3 image is 10% below the OCI layer and 7% + below v2 for the same rootfs. +- Every v3 mode beats v2 at every latency point: 1.6× at zero RTT and + 1.8× at +50 ms RTT even without optimize. With optimize, v3 reaches + ready 6× faster than v2 and the OCI full pull at +50 ms (1.4 s vs + 8.6 s) and 14× faster than OCI at zero RTT. +- The OCI row barely moves with RTT because a full pull is a single + bandwidth-bound stream dominated by gunzip + untar of the whole layer; + it boots fast once extracted but pays all 517 MiB on every cold start. + Notably, v2 FUSE at +50 ms (8.64 s) is already no faster than the full + pull — its per-chunk round trips eat the entire lazy-loading win, while + v3's grouped fetches (and the optimize stream) keep it well ahead. +- The gap grows with registry latency because v3 fetches data in 4 MiB + block groups — roughly a third of the HTTP round trips v2 needs — even + though it transfers more bytes (230 vs 197 MiB); on latency-bound paths + request count dominates bytes. +- The kernel-EROFS modes (NBD/ublk/fileio/fanotify) serve all metadata + in-kernel; on metadata-heavy workloads (full-tree scans) they extend the + lead over v2 FUSE to 2–3×. + +### Image build performance (v2 vs v3) + +Building the Linux kernel source tree (1.8 GiB, ~95.8k files, warm page +cache) into an image, measured with `/usr/bin/time -v`, 3 runs each: + +| Builder | Wall time | Peak RSS | Data blob | Bootstrap (gzip) | Total transfer | +| --- | ---: | ---: | ---: | ---: | ---: | +| v2 `nydus-image create` (RAFS v6, zstd) | 5.6–6.1 s | 170 MiB | 298 MiB | 7.5 MiB | 306 MiB | +| v3 `nydus build` (zstd) | **1.7–1.8 s** | **82 MiB** | **236 MiB** | **1.9 MiB** | **238 MiB** | + +- v3 builds 3.3× faster than v2: source reads stay on the produce thread + while per-block-group crc32 + zstd run on a small background pipeline + drained in submission order, so the output remains byte-for-byte + deterministic. +- v3 output is 22% smaller end to end — the full blob stores its embedded + bootstrap as one zstd frame — and the metadata bootstrap compresses 4× + smaller than v2's (1.9 vs 7.5 MiB gzipped). +- Peak memory is 52% lower than v2. The build is streaming end to end: + read buffers are recycled, the encode pipeline is bounded to a couple of + in-flight block groups, the bootstrap is rendered in place inside the + layout buffer (no assembly copy), the standalone bootstrap is patched + from the embedded one instead of re-rendered, and the inode tree is + freed as soon as rendering finishes. + +### Whole-image conversion (`nydusify convert`, v2 vs v3) + +Converting `gitlab/gitlab-ce` (5.38 GB docker size, 9 layers, ~440k +files) between two local registries (pull from one, push to the other, +the push registry recreated before every run), cold page cache. Memory is +the peak of the summed RSS of the whole process tree: + +| Converter | Wall time | Peak memory (tree) | Output image | +| --- | ---: | ---: | ---: | +| v2 nydusify (RAFS v6, zstd) | 34.1–34.3 s | ~330 MiB | 1319 MiB | +| v3 nydusify (zstd) | **29.7–34.6 s** | **~130 MiB** | **1275 MiB** | + +- The source OCI image is 1333 MiB of gzip layers; v3's output is 4% + smaller than the source and 3% smaller than v2's. Each layer is a + self-contained full blob whose embedded bootstrap is stored as one + zstd frame — only merge, `check`, and single-blob mounts decode it, + so the runtime read path (merged bootstrap + blob meta sidecar) is + untouched. +- v3 converts layers with a bounded worker pool and one shared builder, + so its memory stays flat as images grow layers; v2 spawns one + `nydus-image` per layer in parallel, so its peak scales with the layer + count. Layer blobs are uploaded as soon as each is built, overlapping + the remaining conversions. +- `nydus merge` (the bootstrap-merging step) never materialises the + merged tree: each directory's entries are k-way merged across the + layer bootstraps on demand while flattening, and the bootstrap is + stream-rendered; merging the 440k-inode gitlab tree peaks at ~107 MiB + in 0.1 s. + +Output image size across payload shapes (manifest layer totals): + +| Image | Source (OCI gzip) | v2 output | v3 output | +| --- | ---: | ---: | ---: | +| `continuumio/anaconda3` (2 layers, Python distro) | 1067 MiB | 1123 MiB | **965 MiB** (-10% / -14%) | +| `n8nio/n8n` (12 layers, node_modules-dense) | 359 MiB | 419 MiB | **320 MiB** (-11% / -24%) | + +- The gap widens on small-file-heavy payloads: v3's 4 MiB block groups + compress the small-file stream far better than v2's 1 MiB chunks, and + the per-layer embedded bootstraps (108 MiB of raw EROFS metadata on + n8n's largest layer) shrink ~20× as zstd frames. v2 comes out larger + than the OCI source on both images; v3 beats the source on both. v3 + also converts with a fraction of v2's peak process-tree memory + (anaconda3: 199 vs 449 MiB, n8n: 227 vs 523 MiB). ## Components diff --git a/config/registry.example.yaml b/config/registry.example.yaml index 6f581a09488..ac4e30929c0 100644 --- a/config/registry.example.yaml +++ b/config/registry.example.yaml @@ -79,6 +79,10 @@ storage: # written to disk. Kernel-served modes (fanotify/nbd/ublk/uffd) and # `nydus optimize` require a directory. dir: /var/lib/nydus/cache + # Skip verifying decoded block groups against their stored checksums + # before they are served. Default: true. Set to false to verify every + # decoded block group when the transport is not trusted end to end. + # skip_verify_checksums: true prefetch: # Number of concurrently prefetched blobs. diff --git a/docs/nydus.md b/docs/nydus.md index e8ef1383b6c..388a44e3a81 100644 --- a/docs/nydus.md +++ b/docs/nydus.md @@ -714,6 +714,10 @@ Fields: layer. Diskless mode applies to `nydus fuse` and `nydus check`; the modes that hand the cache file to the kernel (`fanotify`, `nbd`, `ublk`, `uffd`) and `nydus optimize` require a directory and reject its absence at startup. +- `storage.skip_verify_checksums` (default `true`) skips verifying decoded + block groups against their stored checksums before they are served. Set it + to `false` to verify every decoded block group when the transport is not + trusted end to end. - `prefetch.concurrent_blob_count` (default `10`) caps how many blobs are prefetched concurrently. - `prefetch.timeout` (default `1h`) bounds how long prefetching one whole @@ -993,7 +997,8 @@ full blob file: +-------------------------------+ byte = footer.compressed_data_offset + footer.compressed_data_size | padding to 4 KiB alignment | +-------------------------------+ byte = footer.bootstrap_offset -| bootstrap | +| bootstrap (zstd frame) | +| decodes to the EROFS image: | | block 0 | | +-------------------------+ | | | 0x0000..0x03ff zeros | | @@ -1042,7 +1047,9 @@ u64 blob_meta_offset u64 compressed_data_size u32 bootstrap_blocks u32 blob_meta_blocks -u8 reserved1[4032] compat area: writers zero, readers ignore +u64 bootstrap_compressed_size exact zstd frame bytes when the + BOOTSTRAP_ZSTD flag is set, else 0 +u8 reserved1[4024] compat area: writers zero, readers ignore ``` The `magic + version + flags` header prefix matches the blob meta @@ -1060,9 +1067,13 @@ The inequalities allow alignment padding between regions. Offsets and the footer offset must be 4 KiB aligned. The bootstrap and blob meta region lengths are stored as 4 KiB block counts in the footer. -The bootstrap region is a valid metadata-only EROFS image by itself. When -`--bootstrap` is specified, the standalone bootstrap is byte-for-byte identical -to this embedded region. +The bootstrap region stores the metadata-only EROFS image as a single zstd +frame (footer incompat flag `BOOTSTRAP_ZSTD = 1 << 0`), padded with zeros to +the 4 KiB region boundary; `bootstrap_compressed_size` carries the exact frame +length so readers decode without trusting the zero tail. An empty bootstrap +(ondemand blobs) keeps the flag clear and the size zero. When `--bootstrap` is +specified, the standalone bootstrap file is byte-for-byte identical to the +decoded region. ### Bootstrap region details diff --git a/nydus-config/src/lib.rs b/nydus-config/src/lib.rs index b71c8048888..e3d52d3dd5b 100644 --- a/nydus-config/src/lib.rs +++ b/nydus-config/src/lib.rs @@ -108,6 +108,13 @@ fn default_dragonfly_fallback_interval() -> Duration { Duration::from_secs(1) } +/// Returns the default for skipping decoded block group checksum +/// verification. +#[inline] +fn default_storage_skip_verify_checksums() -> bool { + true +} + /// The local backend configuration, serving blobs from a directory. #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] @@ -309,7 +316,7 @@ pub struct DragonflyConfig { } /// The storage configuration: where downloaded blob data is kept. -#[derive(Debug, Clone, Default, Deserialize)] +#[derive(Debug, Clone, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct StorageConfig { /// The directory storing each blob's decoded chunk cache file. When @@ -320,6 +327,22 @@ pub struct StorageConfig { /// require a directory. #[serde(default)] pub dir: Option, + + /// Skip verifying decoded block groups against their stored checksums + /// before they are served (the default). Set to `false` to verify every + /// decoded block group when the transport is not trusted end to end. + #[serde(default = "default_storage_skip_verify_checksums")] + pub skip_verify_checksums: bool, +} + +/// Implement Default for StorageConfig. +impl Default for StorageConfig { + fn default() -> Self { + Self { + dir: None, + skip_verify_checksums: default_storage_skip_verify_checksums(), + } + } } /// The prefetch configuration, controlling background blob prefetch. @@ -519,6 +542,7 @@ mod tests { config.storage.dir.as_deref(), Some(Path::new("/var/lib/nydus/cache")) ); + assert!(!config.storage.skip_verify_checksums); assert_eq!(config.prefetch.concurrent_blob_count, 4); assert_eq!(config.prefetch.timeout, Duration::from_secs(120)); @@ -680,6 +704,11 @@ config: storage.dir.as_deref(), Some(Path::new("/var/lib/nydus/cache")) ); + assert!(storage.skip_verify_checksums); + + let storage: StorageConfig = + serde_yaml::from_str("dir: /cache\nskip_verify_checksums: false\n").unwrap(); + assert!(!storage.skip_verify_checksums); } #[test] diff --git a/nydus-config/testdata/config.yaml b/nydus-config/testdata/config.yaml index 00811ff8e49..f75a0f26065 100644 --- a/nydus-config/testdata/config.yaml +++ b/nydus-config/testdata/config.yaml @@ -20,6 +20,7 @@ backend: storage: dir: /var/lib/nydus/cache + skip_verify_checksums: false prefetch: concurrent_blob_count: 4 diff --git a/nydus-core/Cargo.toml b/nydus-core/Cargo.toml index bc1faeca4a9..2902b21c3dd 100644 --- a/nydus-core/Cargo.toml +++ b/nydus-core/Cargo.toml @@ -9,6 +9,7 @@ repository = "https://github.com/dragonflyoss/nydus" readme = "README.md" [dependencies] +zstd = "0.13" libc = { workspace = true } memmap2 = { workspace = true } nydus-backend = { version = "0.1.0", path = "../nydus-backend" } diff --git a/nydus-core/src/blob.rs b/nydus-core/src/blob.rs index d0f69f82bab..896f7ec339f 100644 --- a/nydus-core/src/blob.rs +++ b/nydus-core/src/blob.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::fmt; use std::path::PathBuf; use std::str::FromStr; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, Mutex, OnceLock}; use nydus_error::{Context, Error, Result}; use nydus_format::erofs::EROFS_BLOCK_SIZE; @@ -95,6 +95,9 @@ pub struct Blobs { pub(crate) index_by_blob_id: HashMap, /// Memoised result of [`Blobs::flat_layout`]. pub(crate) flat_layout: OnceLock>, + /// Serialises the first [`Blobs::flat_layout`] computation so a + /// background warm-up and an early I/O do not both download blob meta. + pub(crate) flat_layout_init: Mutex<()>, } impl Blobs { @@ -148,11 +151,19 @@ impl Blobs { if let Some(layout) = self.flat_layout.get() { return Ok(layout); } + // Single-flight: the winner prepares the blobs while latecomers block + // here and then read the memoised result. A failed attempt leaves the + // cell empty so the next caller retries. + let _init = self + .flat_layout_init + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if let Some(layout) = self.flat_layout.get() { + return Ok(layout); + } let mut blobs = self.prepare_all()?; blobs.retain(|blob| !blob.is_redirect); blobs.sort_by_key(|blob| blob.mapped_offset); - // A racing caller may have won the initialisation; either value is - // equally valid because the layout is deterministic. let _ = self.flat_layout.set(blobs); Ok(self .flat_layout diff --git a/nydus-core/src/flat.rs b/nydus-core/src/flat.rs new file mode 100644 index 00000000000..3f101622cca --- /dev/null +++ b/nydus-core/src/flat.rs @@ -0,0 +1,175 @@ +// Copyright (C) 2026 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +//! The flattened image view shared by the block-shaped mount services. +//! +//! The crate already supplies the primitives — [`NydusCore::fetch_flat_ranges`] +//! resolves a window of the flattened address space into mmap-ready extents, +//! and [`MmapCache`] copies them out — but every service that presents the +//! image as a linear byte range needs the same glue on top: an image size +//! rounded up to whatever the transport addresses in, and a read that fetches +//! the covering ranges, copies the resident bytes, and reports the rest as +//! zeros. +//! +//! This module is that glue. It lives here rather than in the service crate +//! so an embedder wiring the read path into a hypervisor (virtio-pmem, a +//! block target) gets an addressable image without pulling in FUSE, the CLI, +//! or any mount service. +//! +//! What stays in each service is what is genuinely protocol-specific: NBD +//! rejects out-of-range requests because its protocol guarantees they never +//! occur, ublk converts failures back into an `errno` for its completion +//! path, and uffd resolves ranges without copying because the kernel hands it +//! the destination pages. + +use std::path::Path; +use std::sync::Arc; + +use nydus_config::Config; +use nydus_error::{Context, Error, Result}; +use nydus_format::erofs::EROFS_BLOCK_SIZE; +use nydus_format::utils::align_up_u64; + +use crate::extent::{Extent, MmapCache}; +use crate::NydusCore; + +/// EROFS block size as u64 — the granularity every flattened read is aligned +/// to. +pub const BLOCK_SIZE: u64 = EROFS_BLOCK_SIZE as u64; + +/// A nydus image presented as one linear, read-only byte range. +/// +/// The layout is the one `nydus-core` defines: the bootstrap at the head, +/// then each blob at its mapped offset, with gaps and redirect blobs reading +/// as zeros. +pub struct FlatImage { + core: Arc, + size: u64, + maps: MmapCache, +} + +impl FlatImage { + /// Open `bootstrap` with `config` and expose it as a flattened image + /// whose size is rounded up to `align` bytes. + /// + /// Pass [`BLOCK_SIZE`] for transports that address in EROFS blocks; a + /// larger alignment (a ublk logical block, a uffd region) rounds up and + /// the padding reads as zeros. No blob meta is downloaded and no cache + /// file is created here, so this returns quickly even for large images. + pub fn open(bootstrap: &Path, config: Config, align: u64) -> Result { + let core = Arc::new(NydusCore::new(bootstrap, config)?); + Self::with_core(core, align) + } + + /// Wrap an already-open core, e.g. to share one image between a service + /// and its metrics or prefetch machinery. + pub fn with_core(core: Arc, align: u64) -> Result { + let flat_size = core.flat_size(); + if flat_size == 0 { + return Err(Error::InvalidImage( + "flattened image size is zero".to_string(), + )); + } + if flat_size % BLOCK_SIZE != 0 { + return Err(Error::InvalidImage(format!( + "flattened image size {flat_size} is not a multiple of the {BLOCK_SIZE} B EROFS block size" + ))); + } + let size = align_up_u64(flat_size, align) + .ok_or_else(|| Error::Overflow("flattened image size overflow".to_string()))?; + + Ok(Self { + core, + size, + maps: MmapCache::default(), + }) + } + + /// Size in bytes of the flattened view, rounded up to the alignment given + /// at construction. + pub fn size(&self) -> u64 { + self.size + } + + /// Size in [`BLOCK_SIZE`] units. + pub fn block_count(&self) -> u64 { + self.size / BLOCK_SIZE + } + + /// Borrow the underlying core, e.g. to snapshot metrics, start prefetch, + /// or locate the bootstrap region inside the view. + pub fn core(&self) -> &Arc { + &self.core + } + + /// The core-owned `/dev/zero` fd used to serve holes. + pub fn zero_fd(&self) -> std::os::fd::RawFd { + self.core.zero_fd() + } + + /// Fetch `[offset, offset + buf.len())` and copy the resident bytes into + /// `buf`, serving holes, redirect slots, gaps and any range past the end + /// of the image as zeros. + /// + /// `offset` and `buf.len()` must both be [`BLOCK_SIZE`]-aligned: the fetch + /// path rounds outward to whole block groups, so an unaligned window + /// would silently pull in neighbouring data. On success every byte of + /// `buf` has been written. + pub fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result<()> { + if buf.is_empty() { + return Ok(()); + } + debug_assert!(offset % BLOCK_SIZE == 0); + debug_assert!((buf.len() as u64) % BLOCK_SIZE == 0); + + // Alignment padding lives past the flattened extent, so a request can + // legitimately start there; it reads as zeros like any other hole. + if offset >= self.size { + buf.fill(0); + return Ok(()); + } + let len = (buf.len() as u64).min(self.size - offset); + if len < buf.len() as u64 { + buf[len as usize..].fill(0); + } + + let ranges = self + .core + .fetch_flat_ranges(offset, len) + .context("failed to fetch flat ranges")?; + + // The shared copy is gap-tolerant, so a drifted fetch contract would + // show up as silently misplaced bytes rather than an error. + self.validate_contiguous_ranges(&ranges, offset, len)?; + self.maps + .copy_ranges( + &ranges, + offset, + self.core.zero_fd(), + &mut buf[..len as usize], + ) + .context("failed to copy flat ranges")?; + Ok(()) + } + + fn validate_contiguous_ranges(&self, ranges: &[Extent], offset: u64, len: u64) -> Result<()> { + let mut written = 0u64; + for range in ranges { + if range.source_offset != offset + written { + return Err(Error::Runtime(format!( + "flat ranges are not contiguous: expected source offset {}, got {}", + offset + written, + range.source_offset + ))); + } + written += range.len; + if written > len { + return Err(Error::Runtime(format!( + "flat range segments overflow the read window: covered={written} len={len}" + ))); + } + } + Ok(()) + } +} diff --git a/nydus-core/src/lib.rs b/nydus-core/src/lib.rs index 60d35ddbef5..fa86711824e 100644 --- a/nydus-core/src/lib.rs +++ b/nydus-core/src/lib.rs @@ -30,18 +30,20 @@ pub mod blob; pub mod entry; pub mod extent; +pub mod flat; pub mod reader; pub use blob::{BlobId, BlobInfo, Blobs}; pub use entry::FileType; pub use extent::{Extent, ResolveMode}; +pub use flat::FlatImage; pub use reader::ErofsReader; use std::fs::{File, OpenOptions}; use std::os::fd::{AsRawFd, RawFd}; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, Mutex, OnceLock}; use nydus_config::Config; use nydus_error::{Context, Error, Result}; @@ -120,6 +122,7 @@ impl NydusCore { let prefetch_timeout = config.prefetch.timeout; let prefetch_retry_delay_min = config.prefetch.retry_delay_min; let prefetch_retry_delay_max = config.prefetch.retry_delay_max; + nydus_storage::cache::set_skip_verify_checksums(config.storage.skip_verify_checksums); let backend = build_backend(&config.backend).context("failed to build blob backend")?; // The multi-device model hands each blob's cache file to the kernel // (as an EROFS device or fill target), so diskless mode cannot apply. @@ -215,6 +218,7 @@ impl NydusCore { raw_blob_infos, index_by_blob_id, flat_layout: OnceLock::new(), + flat_layout_init: Mutex::new(()), }, fs: ImageFs::new(reader, zero_file.clone()), bootstrap: bootstrap_file, diff --git a/nydus-core/src/reader/metadata.rs b/nydus-core/src/reader/metadata.rs index 699026905fd..ead88f9b2ec 100644 --- a/nydus-core/src/reader/metadata.rs +++ b/nydus-core/src/reader/metadata.rs @@ -68,6 +68,106 @@ impl ErofsReader { Ok(entries) } + /// Look up `name` in directory `nid` by binary search over the sorted + /// EROFS dirents (across blocks, then within the block), the same + /// algorithm the kernel driver uses. Returns the child nid, or `None` + /// when the name is absent or the directory data is malformed. + pub fn lookup_dir_entry( + &self, + nid: u64, + inode: &ErofsInode<'_>, + name: &[u8], + ) -> io::Result> { + let dir_size = inode.size() as usize; + if dir_size == 0 { + return Ok(None); + } + match self.read_flat_data(nid, inode, 0, dir_size) { + Ok(data) => Ok(Self::find_dir_entry(data, dir_size, name)), + Err(_) => { + let data = self.read_flat_data_vec(nid, inode, 0, dir_size)?; + Ok(Self::find_dir_entry(&data, dir_size, name)) + } + } + } + + /// Entry `index` of one directory block as `(nid, name)`. `dirent_count` + /// must come from [`dir_block`], which bounds it by the bytes that + /// physically fit, so the dirent casts below cannot go out of range; only + /// the untrusted name offsets still need checking. + fn dir_block_entry( + block_data: &[u8], + dirent_count: usize, + index: usize, + ) -> Option<(u64, &[u8])> { + let block_len = block_data.len(); + let de_off = index * EROFS_DIRENT_SIZE; + let de: &ErofsDirent = cast_ref(&block_data[de_off..de_off + EROFS_DIRENT_SIZE]); + let nameoff = de.nameoff() as usize; + let name_end = if index + 1 < dirent_count { + let next: &ErofsDirent = cast_ref(&block_data[(index + 1) * EROFS_DIRENT_SIZE..]); + next.nameoff() as usize + } else { + let mut end = nameoff.min(block_len); + while end < block_len && block_data[end] != 0 { + end += 1; + } + end + }; + if nameoff >= block_len || name_end > block_len || name_end < nameoff { + return None; + } + Some((de.nid(), &block_data[nameoff..name_end])) + } + + /// Directory block `index` of a directory of `dir_size` bytes, with its + /// entry count capped by what physically fits in the block. + fn dir_block(data: &[u8], dir_size: usize, index: usize) -> Option<(&[u8], usize)> { + let block_size = EROFS_BLOCK_SIZE as usize; + let start = index * block_size; + let end = (start + block_size).min(dir_size); + let block_data = &data[start..end]; + if block_data.len() < EROFS_DIRENT_SIZE { + return None; + } + let first: &ErofsDirent = cast_ref(&block_data[..EROFS_DIRENT_SIZE]); + let count = (first.nameoff() as usize / EROFS_DIRENT_SIZE) + .min(block_data.len() / EROFS_DIRENT_SIZE); + Some((block_data, count)) + } + + fn find_dir_entry(data: &[u8], dir_size: usize, target: &[u8]) -> Option { + let block_size = EROFS_BLOCK_SIZE as usize; + let nblocks = dir_size.div_ceil(block_size); + + // Rightmost block whose first entry name is <= target. + let (mut lo, mut hi) = (0usize, nblocks); + while hi - lo > 1 { + let mid = (lo + hi) / 2; + let first_name = Self::dir_block(data, dir_size, mid) + .and_then(|(bd, count)| Self::dir_block_entry(bd, count, 0)) + .map(|(_, name)| name)?; + if first_name <= target { + lo = mid; + } else { + hi = mid; + } + } + + let (block_data, count) = Self::dir_block(data, dir_size, lo)?; + let (mut left, mut right) = (0usize, count); + while left < right { + let mid = left + (right - left) / 2; + let (entry_nid, entry_name) = Self::dir_block_entry(block_data, count, mid)?; + match entry_name.cmp(target) { + std::cmp::Ordering::Equal => return Some(entry_nid), + std::cmp::Ordering::Less => left = mid + 1, + std::cmp::Ordering::Greater => right = mid, + } + } + None + } + fn parse_dir_entries(data: &[u8], dir_size: usize, cb: &mut F) -> io::Result<()> where F: FnMut(u64, u8, &[u8]) -> io::Result, diff --git a/nydus-core/src/reader/mod.rs b/nydus-core/src/reader/mod.rs index 6a101ba841f..a3830b87d10 100644 --- a/nydus-core/src/reader/mod.rs +++ b/nydus-core/src/reader/mod.rs @@ -72,8 +72,11 @@ pub struct ErofsReader { impl ErofsReader { /// Open a nydus blob / bootstrap file for metadata-only inspection. pub fn open_metadata_only(path: &Path) -> io::Result { - let mmap = Self::mmap_file(path)?; - let image_offset = Self::image_offset_from_footer(&mmap)?.unwrap_or(0); + let mmap = Self::mmap_file(path, false)?; + let (mmap, image_offset) = match Self::unpack_embedded_image(mmap)? { + (mmap, Some(image_offset)) => (mmap, image_offset), + (mmap, None) => (mmap, 0), + }; let sb_offset = image_offset .checked_add(EROFS_SUPER_OFFSET as usize) .ok_or_else(|| { @@ -92,13 +95,20 @@ impl ErofsReader { } /// Open a self-contained full blob (`payload + bootstrap + blob meta + - /// footer`): everything is served from the file itself, no backend or - /// cache is involved. - pub fn open_blob(blob_path: &Path) -> io::Result { - let mmap = Self::mmap_file(blob_path)?; - let image_offset = Self::image_offset_from_footer(&mmap)?.ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "nydus blob footer not found") - })?; + /// footer`): everything is served from the file itself, no remote + /// backend is involved. `cache_dir`, when given, caches the decoded + /// block groups so repeat reads skip re-decoding from the blob. + pub fn open_blob(blob_path: &Path, cache_dir: Option<&Path>) -> io::Result { + let mmap = Self::mmap_file(blob_path, false)?; + let (mmap, image_offset) = match Self::unpack_embedded_image(mmap)? { + (mmap, Some(image_offset)) => (mmap, image_offset), + (_, None) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "nydus blob footer not found", + )) + } + }; let sb_offset = image_offset .checked_add(EROFS_SUPER_OFFSET as usize) .ok_or_else(|| { @@ -122,7 +132,7 @@ impl ErofsReader { .iter() .map(|info| (info.blob_index, info.blob_id)), nydus_backend::metered(backend), - None, + cache_dir, None, )?; @@ -145,7 +155,7 @@ impl ErofsReader { cache_dir: Option<&Path>, trace_recorder: Option>, ) -> io::Result { - let mmap = Self::mmap_file(bootstrap_path)?; + let mmap = Self::mmap_file(bootstrap_path, true)?; let sb_offset = EROFS_SUPER_OFFSET as usize; let sb = Self::superblock_from(&mmap, sb_offset)?; Self::validate_superblock(sb)?; @@ -170,19 +180,59 @@ impl ErofsReader { }) } - fn mmap_file(path: &Path) -> io::Result { + fn mmap_file(path: &Path, populate: bool) -> io::Result { let file = fs::File::open(path)?; - unsafe { Mmap::map(&file) } + // Populate is only for standalone bootstraps: a few MiB that every + // metadata operation resolves against, so paying the read up front + // (milliseconds) removes a page fault per cold folio. Full blobs must + // NOT be populated — they carry the entire data region, and faulting + // in a multi-GiB blob just to read its metadata tail multiplies RSS + // by the blob size (as `nydus merge` over large layers showed). + let mut options = memmap2::MmapOptions::new(); + if populate { + options.populate(); + } + unsafe { options.map(&file) } } - fn image_offset_from_footer(mmap: &[u8]) -> io::Result> { - let Some(footer) = BlobFooter::from_blob_bytes(mmap).map_err(io::Error::other)? else { - return Ok(None); + /// Resolve the EROFS image inside `mmap`: `(mmap, None)` for a bare + /// bootstrap without a footer, `(mmap, Some(offset))` for a full blob + /// with a raw embedded bootstrap, and a fresh anonymous mapping holding + /// the decompressed bytes (offset 0) when the footer declares the + /// bootstrap region zstd-compressed. + fn unpack_embedded_image(mmap: Mmap) -> io::Result<(Mmap, Option)> { + let Some(footer) = BlobFooter::from_blob_bytes(&mmap).map_err(io::Error::other)? else { + return Ok((mmap, None)); }; - usize::try_from(footer.bootstrap_offset()) - .map(Some) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "bootstrap offset too large")) + let bootstrap_offset = usize::try_from(footer.bootstrap_offset()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "bootstrap offset too large") + })?; + + let Some(compressed_size) = footer.bootstrap_compressed_size() else { + return Ok((mmap, Some(bootstrap_offset))); + }; + let compressed_size = usize::try_from(compressed_size) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "bootstrap frame too large"))?; + let end = bootstrap_offset + .checked_add(compressed_size) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "bootstrap region overflow") + })?; + if end > mmap.len() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "compressed bootstrap region beyond blob end", + )); + } + + let decoded = zstd::stream::decode_all(&mmap[bootstrap_offset..end]) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + // An anonymous mapping keeps the field type (and every downstream + // zero-copy cast) unchanged; the file mapping is dropped here. + let mut anon = memmap2::MmapOptions::new().len(decoded.len()).map_anon()?; + anon.copy_from_slice(&decoded); + Ok((anon.make_read_only()?, Some(0))) } fn superblock_from(mmap: &[u8], sb_offset: usize) -> io::Result<&ErofsSuperblock> { @@ -398,9 +448,7 @@ impl ErofsReader { let absolute_offset = blob_offset.checked_add(chunk_off).ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidInput, "blob write offset overflow") })?; - let mut buf = vec![0u8; len]; - cache.read_at(absolute_offset, &mut buf)?; - writer.write_all(&buf) + cache.write_data_to(absolute_offset, len, writer) } pub(crate) fn nid_to_offset(&self, nid: u64) -> usize { diff --git a/nydus-format/Cargo.toml b/nydus-format/Cargo.toml index 16c25baa244..7eaec5b51c2 100644 --- a/nydus-format/Cargo.toml +++ b/nydus-format/Cargo.toml @@ -9,6 +9,7 @@ repository = "https://github.com/dragonflyoss/nydus" readme = "README.md" [dependencies] +zstd = "0.13" bitflags = "2" crc32c = "0.6" libc = { workspace = true } diff --git a/nydus-format/src/blob/algorithm.rs b/nydus-format/src/blob/algorithm.rs index 4582d036a1a..4a3cbec6322 100644 --- a/nydus-format/src/blob/algorithm.rs +++ b/nydus-format/src/blob/algorithm.rs @@ -12,6 +12,7 @@ use std::fmt; pub enum BlobMetadataCompressor { None, Zstd, + Lz4Block, } impl BlobMetadataCompressor { @@ -20,6 +21,7 @@ impl BlobMetadataCompressor { match self { Self::None => BlobMetadataFlags::empty(), Self::Zstd => BlobMetadataFlags::COMPRESSOR_ZSTD, + Self::Lz4Block => BlobMetadataFlags::COMPRESSOR_LZ4, } } } @@ -31,6 +33,7 @@ impl fmt::Display for BlobMetadataCompressor { f.write_str(match self { Self::None => "none", Self::Zstd => "zstd", + Self::Lz4Block => "lz4-block", }) } } @@ -39,7 +42,9 @@ impl fmt::Display for BlobMetadataCompressor { /// and `BlobMetadataFlags` can only hold defined bits. impl From for BlobMetadataCompressor { fn from(value: BlobMetadataFlags) -> Self { - if value.contains(BlobMetadataFlags::COMPRESSOR_ZSTD) { + if value.contains(BlobMetadataFlags::COMPRESSOR_LZ4) { + Self::Lz4Block + } else if value.contains(BlobMetadataFlags::COMPRESSOR_ZSTD) { Self::Zstd } else { Self::None diff --git a/nydus-format/src/blob/flag.rs b/nydus-format/src/blob/flag.rs index 8cf4f25ae0e..902e363dc22 100644 --- a/nydus-format/src/blob/flag.rs +++ b/nydus-format/src/blob/flag.rs @@ -7,17 +7,127 @@ use crate::error::{Error, Result}; -/// The incompatible (reject-when-unknown) half of a format `flags` word. -pub const INCOMPAT_MASK: u32 = 0x0000_FFFF; +/// A format `flags` word, split EROFS-style into the incompatible low half +/// (unknown bits reject the file) and the compatible high half (unknown bits +/// are ignored). Wraps the raw on-disk word verbatim, so compat bits written +/// by a newer writer survive a round trip through this reader. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct FeatureFlags(u32); -/// Reject `flags` whose incompat half carries bits outside `supported`. -pub fn validate_incompat_flags(flags: u32, supported: u32) -> Result<()> { - let unknown_incompat = flags & INCOMPAT_MASK & !supported; - if unknown_incompat != 0 { - return Err(Error::Unsupported(format!( - "unsupported incompat flags {unknown_incompat:#x} (image is newer than this reader)" - ))); +impl FeatureFlags { + /// The incompatible (reject-when-unknown) half of the word. + pub const INCOMPAT_MASK: u32 = 0x0000_FFFF; + + /// A word with no feature bits set. + pub const fn empty() -> Self { + Self(0) + } + + /// Wrap a raw on-disk word. Nothing is rejected here, unknown incompat + /// bits are caught by [`Self::validate_incompat`]. + pub const fn from_bits(bits: u32) -> Self { + Self(bits) + } + + /// The raw on-disk word. + pub const fn bits(self) -> u32 { + self.0 + } + + /// Whether every bit of `bits` is set. + pub const fn contains(self, bits: u32) -> bool { + self.0 & bits == bits } - Ok(()) + /// Set or clear every bit of `bits` per `value` (`bitflags::Flags::set` + /// semantics). + pub fn set(&mut self, bits: u32, value: bool) { + if value { + self.0 |= bits; + } else { + self.0 &= !bits; + } + } + + /// Reject a word whose incompat half carries bits outside `supported`. + pub fn validate_incompat(self, supported: u32) -> Result<()> { + let unknown_incompat = self.0 & Self::INCOMPAT_MASK & !supported; + if unknown_incompat != 0 { + return Err(Error::Unsupported(format!( + "unsupported incompat flags {unknown_incompat:#x} (image is newer than this reader)" + ))); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn raw_bits_round_trip_verbatim() { + assert_eq!(FeatureFlags::from_bits(0xdead_beef).bits(), 0xdead_beef); + } + + #[test] + fn contains_requires_every_bit() { + let flags = FeatureFlags::from_bits(0b011); + + assert!(flags.contains(0b001)); + assert!(flags.contains(0b011)); + assert!(!flags.contains(0b100)); + assert!(!flags.contains(0b101)); + } + + #[test] + fn set_sets_or_clears_only_the_given_bits() { + let mut flags = FeatureFlags::from_bits(0x8000_0000); + + flags.set(0b001, true); + flags.set(0b110, true); + assert_eq!(flags.bits(), 0x8000_0007); + + flags.set(0b010, false); + assert_eq!(flags.bits(), 0x8000_0005); + + flags.set(0b010, false); + assert_eq!(flags.bits(), 0x8000_0005); + } + + #[test] + fn incompat_validation_follows_the_split_word_rules() { + let supported = 0b1; + let cases: [(&str, u32, Option<&str>); 4] = [ + ("empty word", 0, None), + ("supported incompat bit", 0b1, None), + ("unknown compat bits are ignored", 0xffff_0001, None), + ( + "unknown incompat bit", + 0b10, + Some("unsupported incompat flags"), + ), + ]; + + for (case, bits, expected) in cases { + let result = FeatureFlags::from_bits(bits).validate_incompat(supported); + match expected { + None => assert!(result.is_ok(), "{case}"), + Some(message) => { + let err = result.unwrap_err(); + assert!(err.to_string().contains(message), "{case}: {err}"); + } + } + } + } + + #[test] + fn the_rejection_names_only_the_unknown_bits() { + let err = FeatureFlags::from_bits(0b111) + .validate_incompat(0b001) + .unwrap_err(); + + assert!(err.to_string().contains("0x6"), "{err}"); + } } diff --git a/nydus-format/src/blob/footer.rs b/nydus-format/src/blob/footer.rs index 7b43c8e5ed8..4ea09aae727 100644 --- a/nydus-format/src/blob/footer.rs +++ b/nydus-format/src/blob/footer.rs @@ -1,4 +1,4 @@ -use crate::blob::flag::validate_incompat_flags; +use crate::blob::flag::FeatureFlags; use crate::erofs::{blocks_to_bytes, EROFS_BLOCK_SIZE}; use crate::error::{Context, Error, Result}; use crate::utils::le::{read_u32_at, read_u64_at, write_u32_at, write_u64_at}; @@ -27,29 +27,24 @@ pub const NYDUS_BLOB_FOOTER_SIZE: usize = 4096; /// this boundary (the EROFS block size). pub const NYDUS_BLOB_FOOTER_ALIGNMENT: u64 = EROFS_BLOCK_SIZE as u64; -/// `flags` is split EROFS-style (see [`crate::blob::flag`]): the low 16 -/// bits are incompatible features (unknown bits reject), the high 16 bits -/// are compatible features (unknown bits are ignored). No bits are defined -/// yet. -const NYDUS_BLOB_FOOTER_SUPPORTED_INCOMPAT: u32 = 0; +/// Incompat flag: the embedded bootstrap region holds one zstd frame +/// instead of raw EROFS bytes. `bootstrap_compressed_size` then carries the +/// frame's exact byte length within the block-aligned region. The merged +/// bootstrap and the blob meta sidecar the runtime mounts are unaffected, +/// only merge, `check`, and single-blob mounts decode this region. +pub const NYDUS_BLOB_FOOTER_INCOMPAT_BOOTSTRAP_ZSTD: u32 = 1 << 0; + +/// The incompat bits this reader understands, enforced by +/// [`FeatureFlags::validate_incompat`]. +const NYDUS_BLOB_FOOTER_SUPPORTED_INCOMPAT: u32 = NYDUS_BLOB_FOOTER_INCOMPAT_BOOTSTRAP_ZSTD; /// Byte range of the crc32 field within the footer. const NYDUS_BLOB_FOOTER_CRC32_FIELD: Range = 16..20; /// The trailing footer of a nydus full blob: the blob's self-describing map, -/// recording where each region lives, sealed with a crc32c. -/// -/// A full blob lays its regions out back to back (alignment gaps allowed), -/// every offset 4KiB aligned, with the footer as the fixed-size tail: -/// -/// ```text -/// ┌─────────────────┬───────────────────┬───────────┬────────┐ -/// │ compressed data │ bootstrap (EROFS) │ blob meta │ footer │ -/// └─────────────────┴───────────────────┴───────────┴────────┘ -/// 0 ▲ EOF -/// blob meta ends exactly at the -/// footer offset (EOF - 4096) -/// ``` +/// recording where each region lives, sealed with a crc32c. The whole-blob +/// layout the fields describe is drawn at +/// [`finish_full_blob`](crate::blob::finish_full_blob). /// /// The footer's own 4096 bytes (integers little-endian): /// @@ -68,13 +63,15 @@ const NYDUS_BLOB_FOOTER_CRC32_FIELD: Range = 16..20; /// 56 4 bootstrap_blocks 4KiB blocks, zero for an ondemand /// redirect blob without a bootstrap /// 60 4 blob_metadata_blocks 4KiB blocks, never zero -/// 64 4032 reserved writers zero it, readers ignore it +/// 64 8 bootstrap_compressed_size exact zstd frame bytes when the +/// BOOTSTRAP_ZSTD flag is set, else 0 +/// 72 4024 reserved writers zero it, readers ignore it /// ``` #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct BlobFooter { magic: [u8; 8], version: u32, - flags: u32, + flags: FeatureFlags, crc32: u32, reserved0: u32, compressed_data_offset: u64, @@ -83,12 +80,18 @@ pub struct BlobFooter { compressed_data_size: u64, bootstrap_blocks: u32, blob_metadata_blocks: u32, + bootstrap_compressed_size: u64, } impl BlobFooter { /// Creates a validated, sealed footer for the given region layout: the /// fields and the layout are checked first, so a constructed footer is /// valid by definition, then the crc32 is computed over the final bytes. + /// + /// `Some(n)` declares the bootstrap region stores one zstd frame of + /// exactly `n` bytes and sets the BOOTSTRAP_ZSTD incompat flag, `None` + /// keeps the region raw. [`Self::bootstrap_compressed_size`] reads the + /// same value back. pub fn new( compressed_data_offset: u64, compressed_data_size: u64, @@ -96,11 +99,18 @@ impl BlobFooter { bootstrap_blocks: u32, blob_metadata_offset: u64, blob_metadata_blocks: u32, + bootstrap_compressed_size: Option, ) -> Result { + let mut flags = FeatureFlags::empty(); + flags.set( + NYDUS_BLOB_FOOTER_INCOMPAT_BOOTSTRAP_ZSTD, + bootstrap_compressed_size.is_some(), + ); + let mut footer = Self { magic: NYDUS_BLOB_FOOTER_MAGIC, version: NYDUS_BLOB_FOOTER_VERSION, - flags: 0, + flags, crc32: 0, reserved0: 0, compressed_data_offset, @@ -109,6 +119,7 @@ impl BlobFooter { compressed_data_size, bootstrap_blocks, blob_metadata_blocks, + bootstrap_compressed_size: bootstrap_compressed_size.unwrap_or(0), }; footer.validate()?; @@ -129,7 +140,7 @@ impl BlobFooter { let footer = Self { magic: bytes[0..8].try_into().unwrap(), version: read_u32_at(bytes, 8), - flags: read_u32_at(bytes, 12), + flags: FeatureFlags::from_bits(read_u32_at(bytes, 12)), crc32: read_u32_at(bytes, 16), reserved0: read_u32_at(bytes, 20), compressed_data_offset: read_u64_at(bytes, 24), @@ -138,6 +149,7 @@ impl BlobFooter { compressed_data_size: read_u64_at(bytes, 48), bootstrap_blocks: read_u32_at(bytes, 56), blob_metadata_blocks: read_u32_at(bytes, 60), + bootstrap_compressed_size: read_u64_at(bytes, 64), }; footer.validate()?; @@ -161,7 +173,7 @@ impl BlobFooter { let mut data = [0u8; NYDUS_BLOB_FOOTER_SIZE]; data[0..8].copy_from_slice(&self.magic); write_u32_at(&mut data, 8, self.version); - write_u32_at(&mut data, 12, self.flags); + write_u32_at(&mut data, 12, self.flags.bits()); write_u32_at(&mut data, 16, self.crc32); write_u32_at(&mut data, 20, self.reserved0); write_u64_at(&mut data, 24, self.compressed_data_offset); @@ -170,6 +182,7 @@ impl BlobFooter { write_u64_at(&mut data, 48, self.compressed_data_size); write_u32_at(&mut data, 56, self.bootstrap_blocks); write_u32_at(&mut data, 60, self.blob_metadata_blocks); + write_u64_at(&mut data, 64, self.bootstrap_compressed_size); data } @@ -244,7 +257,28 @@ impl BlobFooter { )); } - validate_incompat_flags(self.flags, NYDUS_BLOB_FOOTER_SUPPORTED_INCOMPAT)?; + let compressed = self + .flags + .contains(NYDUS_BLOB_FOOTER_INCOMPAT_BOOTSTRAP_ZSTD); + if compressed + && (self.bootstrap_compressed_size == 0 + || self.bootstrap_compressed_size > self.bootstrap_size()) + { + return Err(Error::InvalidImage(format!( + "nydus footer compressed bootstrap size {} outside its region of {} bytes", + self.bootstrap_compressed_size, + self.bootstrap_size() + ))); + } + if !compressed && self.bootstrap_compressed_size != 0 { + return Err(Error::InvalidImage( + "nydus footer compressed bootstrap size requires the BOOTSTRAP_ZSTD flag" + .to_string(), + )); + } + + self.flags + .validate_incompat(NYDUS_BLOB_FOOTER_SUPPORTED_INCOMPAT)?; Ok(()) } @@ -365,6 +399,14 @@ impl BlobFooter { blocks_to_bytes(self.bootstrap_blocks) } + /// Exact byte length of the zstd frame in the bootstrap region, or + /// `None` when the bootstrap is stored raw. + pub fn bootstrap_compressed_size(&self) -> Option { + self.flags + .contains(NYDUS_BLOB_FOOTER_INCOMPAT_BOOTSTRAP_ZSTD) + .then_some(self.bootstrap_compressed_size) + } + /// Size of the blob meta region in bytes. pub fn blob_metadata_size(&self) -> u64 { blocks_to_bytes(self.blob_metadata_blocks) @@ -385,7 +427,7 @@ mod tests { use super::*; fn footer() -> BlobFooter { - BlobFooter::new(0, 17, 4096, 1, 8192, 1).unwrap() + BlobFooter::new(0, 17, 4096, 1, 8192, 1, None).unwrap() } fn reseal(mut bytes: [u8; NYDUS_BLOB_FOOTER_SIZE]) -> [u8; NYDUS_BLOB_FOOTER_SIZE] { @@ -524,7 +566,7 @@ mod tests { #[test] fn zero_bootstrap_blocks_are_valid() { - BlobFooter::new(0, 17, 4096, 0, 4096, 1).unwrap(); + BlobFooter::new(0, 17, 4096, 0, 4096, 1, None).unwrap(); } #[test] @@ -558,6 +600,7 @@ mod tests { boot_blocks, meta_off, meta_blocks, + None, ) .unwrap_err(); assert!(err.to_string().contains(expected), "{case}: {err}"); diff --git a/nydus-format/src/blob/metadata.rs b/nydus-format/src/blob/metadata.rs index 06354c98e4b..38299d66b6f 100644 --- a/nydus-format/src/blob/metadata.rs +++ b/nydus-format/src/blob/metadata.rs @@ -1,5 +1,5 @@ use crate::blob::algorithm::{BlobMetadataCompressor, BlobMetadataDigester}; -use crate::blob::flag::validate_incompat_flags; +use crate::blob::flag::FeatureFlags; use crate::erofs::EROFS_BLOCK_SIZE; use crate::error::{Context, Error, Result}; use crate::utils::le::{ @@ -85,6 +85,7 @@ bitflags! { pub struct BlobMetadataFlags: u32 { const COMPRESSOR_ZSTD = 1 << 0; const DIGESTER_BLAKE3 = 1 << 1; + const COMPRESSOR_LZ4 = 1 << 2; } } @@ -240,7 +241,8 @@ impl BlobMetadataHeader { let flags = BlobMetadataFlags::from_bits_truncate(self.flags); BlobMetadataDigester::try_from(flags)?; - validate_incompat_flags(self.flags, NYDUS_BLOB_METADATA_SUPPORTED_INCOMPAT)?; + FeatureFlags::from_bits(self.flags) + .validate_incompat(NYDUS_BLOB_METADATA_SUPPORTED_INCOMPAT)?; Ok(()) } diff --git a/nydus-format/src/blob/mod.rs b/nydus-format/src/blob/mod.rs index fbc699df484..6572b10928e 100644 --- a/nydus-format/src/blob/mod.rs +++ b/nydus-format/src/blob/mod.rs @@ -13,7 +13,7 @@ pub mod algorithm; pub mod flag; pub mod footer; pub mod metadata; -pub use algorithm::{BlobMetadataCompressor, BlobMetadataDigester}; +pub use algorithm::BlobMetadataCompressor; pub use footer::NYDUS_BLOB_FOOTER_ALIGNMENT; pub use footer::{BlobFooter, NYDUS_BLOB_FOOTER_SIZE}; pub use metadata::{ @@ -23,49 +23,110 @@ pub use metadata::{ DEFAULT_NYDUS_BLOB_METADATA_CHUNK_SIZE, NYDUS_BLOB_METADATA_SUFFIX, }; -/// Finish a full blob: append the trailing regions of the layout -/// `[data][pad][bootstrap][pad][blob meta][footer]` to `writer`, which must -/// already hold the `compressed_data_size` bytes of blob data. An empty -/// `bootstrap` yields the ondemand layout (no bootstrap region, zero -/// bootstrap blocks). Returns the footer describing the finished blob. +/// Finish a full blob: append everything behind the data region to `writer`, +/// which must already hold the `compressed_data_size` bytes of blob data. +/// Returns the sealed footer describing the finished blob. +/// +/// The finished blob, every region offset 4 KiB aligned: +/// +/// ```text +/// ┌─────────────────┬───┬───────────────────────┬──────────────────┬────────┐ +/// │ compressed data │pad│ bootstrap │ blob meta │ footer │ +/// └─────────────────┴───┴───────────────────────┴──────────────────┴────────┘ +/// 0 bootstrap_offset blob_metadata_offset EOF +/// +/// compressed data the block group payloads, packed back to back and +/// byte-exact (compressed_data_size bytes), mapped by the +/// blob meta block group table +/// pad zeros up to the 4 KiB aligned bootstrap_offset +/// bootstrap one zstd frame of the metadata-only EROFS image +/// (bootstrap_compressed_size bytes), zero tail up to +/// bootstrap_blocks × 4 KiB, absent for an ondemand blob +/// blob meta the LPBLMETA bytes (header, chunk table, block group +/// table), already block-padded, ending exactly at the +/// footer offset +/// footer the sealed LPFOOTER block, fixed 4 KiB at the tail +/// ``` +/// +/// An empty `bootstrap` yields the ondemand layout (no bootstrap region, +/// zero bootstrap blocks). pub fn finish_full_blob( writer: &mut dyn Write, compressed_data_size: u64, bootstrap: &[u8], blob_metadata: &BlobMetadata, ) -> Result { - let bootstrap_size = bootstrap.len() as u64; + let compressed_bootstrap = compress_bootstrap(bootstrap)?; + let blob_footer = new_blob_footer(compressed_data_size, &compressed_bootstrap, blob_metadata)?; + write_blob_tail(writer, &blob_footer, &compressed_bootstrap, blob_metadata)?; + Ok(blob_footer) +} + +/// Compress the embedded bootstrap. An empty bootstrap (the ondemand layout) +/// stores no bytes at all, since even an empty zstd frame would occupy a +/// whole block-aligned region. +fn compress_bootstrap(bootstrap: &[u8]) -> Result> { + if bootstrap.is_empty() { + return Ok(Vec::new()); + } + + zstd::stream::encode_all(bootstrap, zstd::DEFAULT_COMPRESSION_LEVEL) + .context("failed to compress bootstrap") +} + +/// Lay the trailing regions out behind the data region, each 4 KiB aligned, +/// and seal the footer describing them. Construction validates the layout, +/// so the sealed footer is the single source of truth the write pass +/// follows. +fn new_blob_footer( + compressed_data_size: u64, + compressed_bootstrap: &[u8], + blob_metadata: &BlobMetadata, +) -> Result { + let bootstrap_compressed_size = compressed_bootstrap.len() as u64; + let bootstrap_size = align_up_u64(bootstrap_compressed_size, NYDUS_BLOB_FOOTER_ALIGNMENT) + .ok_or_else(|| Error::Overflow("bootstrap region overflow".to_string()))?; let bootstrap_offset = align_up_u64(compressed_data_size, NYDUS_BLOB_FOOTER_ALIGNMENT) .ok_or_else(|| Error::Overflow("bootstrap offset overflow".to_string()))?; let blob_metadata_offset = bootstrap_offset .checked_add(bootstrap_size) - .and_then(|bootstrap_end| align_up_u64(bootstrap_end, NYDUS_BLOB_FOOTER_ALIGNMENT)) .ok_or_else(|| Error::Overflow("blob meta offset overflow".to_string()))?; - write_zeros(writer, bootstrap_offset - compressed_data_size)?; - writer - .write_all(bootstrap) - .context("failed to write blob bootstrap")?; + BlobFooter::new( + 0, + compressed_data_size, + bootstrap_offset, + bytes_to_blocks(bootstrap_size)?, + blob_metadata_offset, + bytes_to_blocks(blob_metadata.padded_size())?, + (!compressed_bootstrap.is_empty()).then_some(bootstrap_compressed_size), + ) +} +/// Stream everything behind the data region in offset order — bootstrap, +/// blob meta, then the footer itself — zero-padding the alignment gaps the +/// footer declares. +fn write_blob_tail( + writer: &mut dyn Write, + footer: &BlobFooter, + compressed_bootstrap: &[u8], + blob_metadata: &BlobMetadata, +) -> Result<()> { write_zeros( writer, - blob_metadata_offset - bootstrap_offset - bootstrap_size, + footer.bootstrap_offset() - footer.compressed_data_size(), )?; + writer + .write_all(compressed_bootstrap) + .context("failed to write blob bootstrap")?; + + let bootstrap_end = footer.bootstrap_offset() + compressed_bootstrap.len() as u64; + write_zeros(writer, footer.blob_metadata_offset() - bootstrap_end)?; blob_metadata .write_to(writer) .context("failed to write blob meta")?; - let footer = BlobFooter::new( - 0, - compressed_data_size, - bootstrap_offset, - bytes_to_blocks(bootstrap_size, "bootstrap")?, - blob_metadata_offset, - bytes_to_blocks(blob_metadata.padded_size(), "blob meta")?, - )?; footer .write_to(writer) - .context("failed to write blob footer")?; - - Ok(footer) + .context("failed to write blob footer") } diff --git a/nydus-format/src/erofs/block.rs b/nydus-format/src/erofs/block.rs index ca70a27a5b7..c0f1a8193d2 100644 --- a/nydus-format/src/erofs/block.rs +++ b/nydus-format/src/erofs/block.rs @@ -4,17 +4,16 @@ use crate::error::{Error, Result}; use super::EROFS_BLOCK_SIZE; -/// Convert a byte size to a 4 KiB block count. `name` labels the region in -/// error messages. -pub fn bytes_to_blocks(size: u64, name: &str) -> Result { +/// Convert a byte size to a 4 KiB block count. +pub fn bytes_to_blocks(size: u64) -> Result { if size % EROFS_BLOCK_SIZE as u64 != 0 { return Err(Error::InvalidImage(format!( - "{name} size is not block aligned: {size}" + "size is not block aligned: {size}" ))); } u32::try_from(size / EROFS_BLOCK_SIZE as u64) - .map_err(|err| Error::Overflow(format!("{name} exceeds u32 block count: {err}"))) + .map_err(|err| Error::Overflow(format!("block count exceeds u32: {err}"))) } /// Convert a 4 KiB block count to a byte size. diff --git a/nydus-format/src/erofs/mod.rs b/nydus-format/src/erofs/mod.rs index 2a254510651..f545c484de3 100644 --- a/nydus-format/src/erofs/mod.rs +++ b/nydus-format/src/erofs/mod.rs @@ -43,6 +43,10 @@ pub const EROFS_SLOTSIZE: u32 = 1 << EROFS_ISLOTBITS; // Feature flags. pub const EROFS_FEATURE_COMPAT_SB_CHKSUM: u32 = 0x0000_0001; pub const EROFS_FEATURE_COMPAT_MTIME: u32 = 0x0000_0002; +/// Nydus-private compat bit: no inode in this image carries any xattr, so a +/// userspace server can answer xattr requests with ENOSYS (which makes the +/// kernel stop sending them). Compat bits are ignored by kernel EROFS. +pub const EROFS_FEATURE_COMPAT_NYDUS_NO_XATTR: u32 = 0x2000_0000; /// RAFS v6 marker: RAFS v6 bootstraps embed a private extension superblock /// and always set this compat bit; pure-EROFS nydus (rafs v7) bootstraps /// never do. This crate does not read RAFS v6 images — the bit exists only diff --git a/nydus-format/src/utils/le.rs b/nydus-format/src/utils/le.rs index 6e679d1a24f..4d8a2a80f14 100644 --- a/nydus-format/src/utils/le.rs +++ b/nydus-format/src/utils/le.rs @@ -57,6 +57,15 @@ pub fn read_u64_at(data: &[u8], offset: usize) -> u64 { u64::from_le_bytes(data[offset..offset + 8].try_into().unwrap()) } +/// Read a little-endian 40-bit unsigned integer (5 bytes) from `data` at +/// `offset`, zero-extended to a `u64`. +#[inline] +pub fn read_u40_at(data: &[u8], offset: usize) -> u64 { + let mut bytes = [0u8; 8]; + bytes[..5].copy_from_slice(&data[offset..offset + 5]); + u64::from_le_bytes(bytes) +} + #[inline] pub fn write_u8_at(data: &mut [u8], offset: usize, value: u8) { data[offset] = value; @@ -76,3 +85,11 @@ pub fn write_u32_at(data: &mut [u8], offset: usize, value: u32) { pub fn write_u64_at(data: &mut [u8], offset: usize, value: u64) { data[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); } + +/// Write the low 40 bits of `value` little-endian (5 bytes) into `data` at +/// `offset`. The value must fit in 40 bits. +#[inline] +pub fn write_u40_at(data: &mut [u8], offset: usize, value: u64) { + debug_assert!(value < 1 << 40); + data[offset..offset + 5].copy_from_slice(&value.to_le_bytes()[..5]); +} diff --git a/nydus-storage/Cargo.toml b/nydus-storage/Cargo.toml index 2e140802319..51039848673 100644 --- a/nydus-storage/Cargo.toml +++ b/nydus-storage/Cargo.toml @@ -22,6 +22,7 @@ tempfile = { workspace = true } tracing = { workspace = true } xattr = { workspace = true } zstd = "0.13" +lz4_flex = "0.11" [dev-dependencies] blake3 = "1" diff --git a/nydus-storage/src/cache/local.rs b/nydus-storage/src/cache/local.rs index 8ae7121d349..18c7e06da87 100644 --- a/nydus-storage/src/cache/local.rs +++ b/nydus-storage/src/cache/local.rs @@ -6,7 +6,7 @@ use std::os::fd::{AsRawFd, RawFd}; use std::os::unix::fs::{FileExt, MetadataExt}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Condvar, Mutex, RwLock}; +use std::sync::{Arc, Condvar, Mutex, OnceLock, RwLock}; use std::time::{Duration, Instant}; use tracing::{info, warn}; @@ -98,6 +98,11 @@ pub struct LocalBlobCache { backend: Arc, trace_recorder: Option>, inflight_block_groups: Mutex>>, + /// Read-only mapping of the cache data file, created once every block + /// group is ready. It serves reads by memcpy from the page cache, without + /// the pread round-trip per request. Bytes never change after ALL_READY + /// latches (rewrites by racing processes are byte-identical). + cache_mmap: OnceLock, /// Keeps the processes sharing this cache from each fetching the same /// cold block group. block_group_locks: BlockGroupLocks, @@ -135,15 +140,14 @@ impl LocalBlobCache { let cache_data_path = cache_dir.join(format!("{cache_key_hex}.blob.data")); - let block_block_group_map_path = cache_dir.join(format!("{cache_key_hex}.group.map")); + let block_group_map_path = cache_dir.join(format!("{cache_key_hex}.group.map")); // The block_group_map is only meaningful together with the cache data file it // describes: a leftover block_group_map whose data file has been removed // would claim block groups are ready while reads hit sparse zeros. Note this // before creating the data file below, which would otherwise mask it. // (Removing the map while keeping the data is the safe direction and // needs no handling.) - let stale_block_block_group_map = - block_block_group_map_path.exists() && !cache_data_path.exists(); + let stale_block_group_map = block_group_map_path.exists() && !cache_data_path.exists(); // Create the cache data file eagerly, before the block_group_map, so that // "block_group_map file exists => data file exists" holds and the check above @@ -157,18 +161,16 @@ impl LocalBlobCache { data_file.set_len(blob_metadata.uncompressed_size())?; drop(data_file); - let block_group_map = BlockGroupMap::open( - &block_block_group_map_path, - blob_metadata.block_group_count(), - )?; - if stale_block_block_group_map { + let block_group_map = + BlockGroupMap::open(&block_group_map_path, blob_metadata.block_group_count())?; + if stale_block_group_map { // Reset in place rather than unlinking: handles already mapping // this file observe the reset, whereas a replacement inode would // split them off with their readiness invisible to each other. block_group_map.reset()?; warn!( "stale block_group_map without cache data file, reset: {}", - block_block_group_map_path.display() + block_group_map_path.display() ); } @@ -188,6 +190,7 @@ impl LocalBlobCache { backend, trace_recorder, inflight_block_groups: Mutex::new(HashMap::new()), + cache_mmap: OnceLock::new(), block_group_locks, }) } @@ -197,6 +200,32 @@ impl LocalBlobCache { &self.blob_metadata } + /// Fetch, decode and validate one block group's bytes directly from the + /// backend, without touching the cache data file or block_group_map. This + /// is the block-group-granular read used by `nydus optimize` to re-encode + /// accessed block groups into an ondemand artifact. + pub fn fetch_block_group(&self, block_group_index: usize) -> io::Result> { + let block_group = *self + .blob_metadata + .block_group(block_group_index) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "block group index out of range", + ) + })?; + let mut buffers = BlockGroupBuffers::default(); + let decoded = fetch_decode_validate_block_group_into( + &self.blob_id, + &self.blob_metadata, + &self.backend, + &block_group, + &mut buffers, + ReadKind::OnDemand, + )?; + Ok(decoded.to_vec()) + } + fn cache_file(&self) -> io::Result> { if let Some(file) = self.cache_file.read().unwrap().as_ref() { return Ok(file.clone()); @@ -240,6 +269,37 @@ impl LocalBlobCache { Ok(()) } + /// The `[offset, offset+len)` slice of the cache-file mapping when every + /// block group is ready, `None` when the blob is still filling (callers + /// then take the ensure + pread path). Redirect blobs never latch + /// ALL_READY through this path, so the mapping is only built for dense + /// blobs. + fn all_ready_slice(&self, offset: u64, len: usize) -> io::Result> { + if !self.block_group_map.is_all_ready() { + return Ok(None); + } + let mmap = if let Some(mmap) = self.cache_mmap.get() { + mmap + } else { + let file = self.cache_file()?; + // SAFETY: the mapping is read-only and its bytes are final once + // ALL_READY latches; concurrent identical rewrites are benign. + let mmap = unsafe { memmap2::MmapOptions::new().map(file.as_ref())? }; + self.cache_mmap.get_or_init(|| mmap) + }; + let end = offset + .checked_add(len as u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "blob read overflow"))?; + if end > mmap.len() as u64 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "blob read beyond cache data file", + )); + } + nydus_telemetry::metrics::inc_cache_hit_block_group(); + Ok(Some(&mmap[offset as usize..end as usize])) + } + fn ensure_block_group( &self, block_group_index: usize, @@ -500,6 +560,7 @@ impl BlobCache for LocalBlobCache { // Prefetch writes the bulk of the cache, so it is worth one stat to // make sure the file it fills is still the one other processes read. self.ensure_data_file_linked(&cache_file)?; + // Prefetch owns its decode buffers and does not take `fetch_lock`, so it // never blocks on-demand FUSE reads. The block_group_map is internally locked // and `set_ready` is idempotent, so racing with a read at worst decodes @@ -566,6 +627,11 @@ impl BlobCache for LocalBlobCache { return Ok(()); } + if let Some(mapped) = self.all_ready_slice(offset, dst.len())? { + dst.copy_from_slice(mapped); + return Ok(()); + } + let cache_file = self.cache_file()?; self.ensure_byte_range(offset, dst.len() as u64, cache_file.as_ref())?; @@ -575,6 +641,16 @@ impl BlobCache for LocalBlobCache { cache_file.as_ref().read_exact_at(dst, offset) } + fn write_data_to(&self, offset: u64, len: usize, writer: &mut dyn io::Write) -> io::Result<()> { + if len == 0 { + return Ok(()); + } + if let Some(mapped) = self.all_ready_slice(offset, len)? { + return writer.write_all(mapped); + } + super::write_data_via_scratch(self, offset, len, writer) + } + fn prepare(&self) -> io::Result { // Opening the cache file creates it (sparse) and sizes it to the dense // uncompressed address space. @@ -601,6 +677,7 @@ impl BlobCache for LocalBlobCache { let end = offset.checked_add(len).ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidInput, "blob probe range overflow") })?; + let (first, last) = self.block_group_span(offset, end)?.into_inner(); self.block_group_map @@ -819,14 +896,14 @@ impl BlobCache for LocalBlobCache { "redirect fill block_group index out of range", ) })?; - if self.block_group_map.is_ready(block_group_index)? { - nydus_telemetry::metrics::inc_cache_hit_block_group(); - return Ok(()); - } // Cross-check against this blob's own block group metadata: the redirect // block group's crc32 was copied from this source block group at optimize time, so // any divergence (stale optimize artifact, corrupted transfer) is // caught here before it can poison the cache. + if self.block_group_map.is_ready(block_group_index)? { + nydus_telemetry::metrics::inc_cache_hit_block_group(); + return Ok(()); + } super::validate_block_group_with_metrics(&self.backend, block_group, decoded)?; let cache_file = self.cache_file()?; write_all_at( @@ -1269,6 +1346,7 @@ mod tests { #[test] fn local_blob_cache_rejects_bad_crc32_before_marking_chunk_ready() { + super::super::set_skip_verify_checksums(false); let backend_dir = tempdir().unwrap(); let cache_dir = tempdir().unwrap(); let payload = vec![0xacu8; 4096]; @@ -1323,6 +1401,7 @@ mod tests { #[test] fn fill_block_group_from_redirect_validates_then_caches() { + super::super::set_skip_verify_checksums(false); let backend_dir = tempdir().unwrap(); let cache_dir = tempdir().unwrap(); let payload = vec![0x6eu8; 4096]; diff --git a/nydus-storage/src/cache/mod.rs b/nydus-storage/src/cache/mod.rs index 73ad59927c4..6caea4458f2 100644 --- a/nydus-storage/src/cache/mod.rs +++ b/nydus-storage/src/cache/mod.rs @@ -35,6 +35,13 @@ pub use remote::RemoteBlobCache; pub trait BlobCache: Send + Sync { fn read_at(&self, offset: u64, dst: &mut [u8]) -> io::Result<()>; + /// Stream `len` bytes at `offset` into `writer`. The default bounces + /// through a per-thread buffer; implementations that can serve reads from + /// a mapping should override it to skip the intermediate copy. + fn write_data_to(&self, offset: u64, len: usize, writer: &mut dyn io::Write) -> io::Result<()> { + write_data_via_scratch(self, offset, len, writer) + } + /// Return the raw fd of the cache data file for mmap use. /// /// The caller must not close the fd; it remains owned by this cache. @@ -255,9 +262,7 @@ pub fn decode_block_group_from_window( if is_stored_plain_block_group(blob_metadata, block_group) { decoded.extend_from_slice(encoded); } else { - decoded.reserve(decoded_len); - zstd::stream::copy_decode(&mut Cursor::new(encoded), &mut *decoded) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + decode_block_group(blob_metadata, encoded, decoded_len, decoded)?; } validate_block_group_with_metrics(backend, block_group, decoded) @@ -311,14 +316,76 @@ pub fn fetch_decode_validate_block_group_into<'a>( )?; buffers.decoded.clear(); - buffers.decoded.reserve(decoded_len); - zstd::stream::copy_decode(&mut Cursor::new(&buffers.encoded), &mut buffers.decoded) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + decode_block_group( + blob_metadata, + &buffers.encoded, + decoded_len, + &mut buffers.decoded, + )?; validate_block_group_with_metrics(backend, block_group, &buffers.decoded)?; Ok(&buffers.decoded) } +/// Read `[offset, offset + len)` through `cache.read_at` into a per-thread +/// scratch buffer and copy it into `writer`: the fallback for caches that +/// cannot serve reads from a mapping. +fn write_data_via_scratch( + cache: &C, + offset: u64, + len: usize, + writer: &mut dyn io::Write, +) -> io::Result<()> { + thread_local! { + static SCRATCH: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; + } + SCRATCH.with(|cell| { + let mut buf = cell.borrow_mut(); + if buf.len() < len { + buf.resize(len, 0); + } + let buf = &mut buf[..len]; + cache.read_at(offset, buf)?; + writer.write_all(buf) + }) +} + +/// Decompress one encoded block group into `decoded` (cleared by the caller) +/// according to the compressor the blob meta header declares. +fn decode_block_group( + blob_metadata: &BlobMetadata, + encoded: &[u8], + decoded_len: usize, + decoded: &mut Vec, +) -> io::Result<()> { + match blob_metadata.compressor() { + BlobMetadataCompressor::Zstd => { + decoded.reserve(decoded_len); + zstd::stream::copy_decode(&mut Cursor::new(encoded), &mut *decoded) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + } + BlobMetadataCompressor::Lz4Block => { + decoded.resize(decoded_len, 0); + let written = lz4_flex::block::decompress_into(encoded, decoded) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + if written != decoded_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "lz4 block group decompressed to an unexpected size", + )); + } + } + BlobMetadataCompressor::None => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "blob meta declares no compressor but the block group is stored compressed", + )); + } + } + Ok(()) +} + /// Validate a decoded block group and, on CRC failure, attribute a CRC error metric to /// the backend that served the bytes. A read diverted from the backend's /// static target (e.g. a Dragonfly fallback to the origin) is attributed to @@ -363,6 +430,9 @@ pub fn validate_decoded_block_group( )); } + if skip_verify_checksums() { + return Ok(()); + } let crc32 = crc32c::crc32c(decoded); if crc32 != block_group.crc32() { return Err(io::Error::new( @@ -374,6 +444,22 @@ pub fn validate_decoded_block_group( Ok(()) } +/// Process-wide switch skipping block group checksum verification (the +/// default), set at service startup from `storage.skip_verify_checksums`. A +/// process serves one mount, so a per-cache flag would only thread the same +/// value through every call site. +static SKIP_VERIFY_CHECKSUMS: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(true); + +/// Skip (or re-enable) block group checksum verification for this process. +pub fn set_skip_verify_checksums(skip: bool) { + SKIP_VERIFY_CHECKSUMS.store(skip, std::sync::atomic::Ordering::Relaxed); +} + +fn skip_verify_checksums() -> bool { + SKIP_VERIFY_CHECKSUMS.load(std::sync::atomic::Ordering::Relaxed) +} + /// Marker error wrapped in an [`io::Error`] when a decoded block group fails CRC /// validation, so callers with backend context can attribute the failure to the /// origin or a proxy via [`is_block_group_crc_mismatch`]. @@ -473,6 +559,7 @@ mod tests { #[test] fn crc_failure_is_attributed_to_the_static_target_without_an_override() { + set_skip_verify_checksums(false); use nydus_telemetry::metrics::BackendTarget; let backend: Arc = Arc::new(StaticTargetBackend); diff --git a/nydus-storage/src/cache/remote.rs b/nydus-storage/src/cache/remote.rs index f73684a8768..efbf56b14e9 100644 --- a/nydus-storage/src/cache/remote.rs +++ b/nydus-storage/src/cache/remote.rs @@ -59,6 +59,7 @@ impl BlobCache for RemoteBlobCache { let end = offset.checked_add(dst.len() as u64).ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidInput, "blob read range overflow") })?; + let first = self .blob_metadata .block_group_index_from_uncompressed_offset(offset) diff --git a/nydus/Cargo.toml b/nydus/Cargo.toml index 18e762ade33..29e537233d5 100644 --- a/nydus/Cargo.toml +++ b/nydus/Cargo.toml @@ -43,6 +43,10 @@ uffd = [ fanotify = ["dep:signal-hook"] # Export a nydus image as a block device through the NBD protocol. nbd = ["dep:signal-hook"] +# File-backed EROFS: export the flattened image as one file over FUSE and let +# the kernel EROFS driver mount it. Requires Linux >= 6.12 +# (CONFIG_EROFS_FS_BACKED_BY_FILE). +fileio = ["fuse", "dep:signal-hook"] # Container image registry backend (OCI distribution). backend-registry = ["nydus-backend/backend-registry"] # Dragonfly P2P SDK proxy support for the registry backend. @@ -55,9 +59,9 @@ backend-dragonfly-proxy = [ ublk = ["dep:libublk", "dep:signal-hook"] [dependencies] -blake3 = "1" anstream = { version = "0.6", optional = true } anstyle = { version = "1", optional = true } +blake3 = "1" bytesize = { version = "1", optional = true } clap = { workspace = true, optional = true } crc32c = "0.6" @@ -85,6 +89,7 @@ signal-hook = { version = "0.3", optional = true } tar = "0.4" tabled = { version = "0.21.0", optional = true } zstd = "0.13" +lz4_flex = "0.11" thiserror = { workspace = true } tokio = { workspace = true, features = [ "fs", diff --git a/nydus/src/bin/nydus/build.rs b/nydus/src/bin/nydus/build.rs index 826167fcb5f..1f720f81020 100644 --- a/nydus/src/bin/nydus/build.rs +++ b/nydus/src/bin/nydus/build.rs @@ -64,7 +64,7 @@ pub struct BuildCommand { DEFAULT_NYDUS_BLOB_METADATA_BLOCK_GROUP_SIZE as u64 / bytesize::MIB ), env = "NYDUS_BUILD_BLOCK_GROUP_SIZE", - help = "Specify the uncompressed size of each block group, the unit of compression and of a single backend read (must be a power of two, >= 1MiB, and >= the chunk size). The value needs to be set with human readable format, for example: 4mib, 16mib" + help = "Specify the uncompressed size of each block group, the unit of compression and of a single backend read (must be a power of two, >= 512KiB, and >= the chunk size). The value needs to be set with human readable format, for example: 4mib, 16mib" )] block_group_size: ByteSize, @@ -107,6 +107,7 @@ pub struct BuildCommand { pub enum Compressor { None, Zstd, + Lz4Block, } /// Implement the conversion from Compressor to BlobMetadataCompressor. @@ -115,6 +116,7 @@ impl From for BlobMetadataCompressor { match value { Compressor::None => Self::None, Compressor::Zstd => Self::Zstd, + Compressor::Lz4Block => Self::Lz4Block, } } } @@ -369,12 +371,8 @@ fn print_blob_build_summary(summary: BlobBuildSummary<'_>) { full_blob_digest: String, #[tabled(rename = "CHUNK SIZE")] chunk_size: String, - #[tabled(rename = "CHUNK COUNT")] - chunk_count: String, #[tabled(rename = "BLOCK GROUP COUNT")] block_group_count: String, - #[tabled(rename = "CHUNK DIGESTER")] - chunk_digester: String, #[tabled(rename = "CHUNK COMPRESSOR")] chunk_compressor: String, #[tabled(rename = "BLOB COMPRESSED SIZE")] @@ -406,9 +404,7 @@ fn print_blob_build_summary(summary: BlobBuildSummary<'_>) { data_blob_digest: hex_string(summary.data_blob_digest), full_blob_digest: hex_string(summary.full_blob_digest), chunk_size: summary.blob_metadata.chunk_size().to_string(), - chunk_count: summary.blob_metadata.chunk_count().to_string(), block_group_count: summary.blob_metadata.block_group_count().to_string(), - chunk_digester: summary.blob_metadata.digester().to_string(), chunk_compressor: summary.blob_metadata.compressor().to_string(), blob_compressed_size: summary.blob_metadata.compressed_end().to_string(), blob_uncompressed_size: summary.blob_metadata.uncompressed_size().to_string(), diff --git a/nydus/src/bin/nydus/check.rs b/nydus/src/bin/nydus/check.rs index 73a40efb298..6d1fd87eabb 100644 --- a/nydus/src/bin/nydus/check.rs +++ b/nydus/src/bin/nydus/check.rs @@ -438,12 +438,8 @@ fn print_blobs(blobs: &BTreeMap) { full_blob_digest: String, #[tabled(rename = "CHUNK SIZE")] chunk_size: String, - #[tabled(rename = "CHUNK COUNT")] - chunk_count: String, #[tabled(rename = "BLOCK GROUP COUNT")] block_group_count: String, - #[tabled(rename = "CHUNK DIGESTER")] - chunk_digester: String, #[tabled(rename = "CHUNK COMPRESSOR")] chunk_compressor: String, #[tabled(rename = "BLOB COMPRESSED SIZE")] @@ -473,9 +469,7 @@ fn print_blobs(blobs: &BTreeMap) { data_blob_digest: data_blob_digest(blob), full_blob_digest: optional_digest(blob.blob_sha256), chunk_size: blob_metadata_field(blob, |meta| meta.chunk_size), - chunk_count: blob_metadata_field(blob, |meta| meta.chunk_count), block_group_count: blob_metadata_field(blob, |meta| meta.block_group_count), - chunk_digester: blob_metadata_field(blob, |meta| meta.digester), chunk_compressor: blob_metadata_field(blob, |meta| meta.compressor), blob_compressed_size: blob_metadata_field_or( blob, diff --git a/nydus/src/bin/nydus/export.rs b/nydus/src/bin/nydus/export.rs index 589dd0e262b..581615576b5 100644 --- a/nydus/src/bin/nydus/export.rs +++ b/nydus/src/bin/nydus/export.rs @@ -71,7 +71,7 @@ impl ExportCommand { /// Runs the export: opens the source blob and streams its OCI layer tar /// to the output file, or to stdout when no output is given. fn run(&self) -> Result<()> { - let reader = ErofsReader::open_blob(&self.source) + let reader = ErofsReader::open_blob(&self.source, None) .with_context(|| format!("failed to open nydus blob: {}", self.source.display()))?; match &self.output { @@ -114,7 +114,7 @@ mod tests { .unwrap(); build_image(&options, File::create(blob).unwrap()).unwrap(); - let reader = ErofsReader::open_blob(blob).unwrap(); + let reader = ErofsReader::open_blob(blob, None).unwrap(); let tar_file = File::create(tar_path).unwrap(); write_tar(&reader, BufWriter::new(tar_file)).unwrap(); } diff --git a/nydus/src/bin/nydus/fileio.rs b/nydus/src/bin/nydus/fileio.rs new file mode 100644 index 00000000000..be8baaff5b0 --- /dev/null +++ b/nydus/src/bin/nydus/fileio.rs @@ -0,0 +1,231 @@ +use clap::Parser; +use nydus::error::{Context, Result}; +use nydus::fileio::{image_path, mount_image_file, warm_bootstrap, FileioService, FlatImageFs}; +use nydus::signal; +use nydus_config::Config; +use nydus_core::flat::{FlatImage, BLOCK_SIZE}; +use nydus_telemetry::logging::init_tracing; +use std::num::NonZeroUsize; +use std::path::PathBuf; +use std::sync::Arc; +use tracing::{info, warn, Level}; + +use super::*; + +/// Default worker-thread cap for the export. The kernel drives cold reads +/// through EROFS's async pipeline, so a handful of threads keeps concurrent +/// backend fetches overlapping without oversubscribing small guests. +const DEFAULT_MAX_THREADS: usize = 8; + +#[derive(Debug, Clone, Parser)] +pub struct FileioCommand { + #[arg( + long, + env = "NYDUS_FILEIO_BOOTSTRAP", + help = "Specify the file path to nydus bootstrap" + )] + bootstrap: PathBuf, + + #[arg( + long, + env = "NYDUS_FILEIO_CONFIG", + help = "Specify the file path to a YAML storage config providing backend/cache directories" + )] + config: PathBuf, + + #[arg( + long, + env = "NYDUS_FILEIO_EXPORT_DIR", + help = "Specify the directory where the flattened image file is exported over FUSE" + )] + export_dir: PathBuf, + + #[arg( + long, + env = "NYDUS_FILEIO_MOUNTPOINT", + help = "Specify the mountpoint for the EROFS filesystem. When given, the daemon mounts the exported image file once the export is live and unmounts it on shutdown; when omitted, only the file is exported and the caller mounts it" + )] + mountpoint: Option, + + #[arg( + long, + env = "NYDUS_FILEIO_THREADS", + help = "Specify the number of FUSE worker threads serving the exported image file. Defaults to the available CPU count, capped at 8" + )] + threads: Option, + + #[arg( + long, + default_value_t = true, + env = "NYDUS_FILEIO_WARM_BOOTSTRAP", + help = "Push the bootstrap region into the export's page cache at startup, so the kernel can resolve metadata without faulting it in folio by folio. Measured no difference against a local backend; it can only pay off when fetching the bootstrap is slow", + action = clap::ArgAction::Set + )] + warm_bootstrap: bool, + + #[arg( + long, + default_value_t = true, + env = "NYDUS_FILEIO_DIRECT_IO", + help = "Mount the image with EROFS 'directio' so the export's page cache stays empty; without it the image data is held both there and against the inode the application reads (measured ~40% more page cache). Warm reads are unaffected; the cost is roughly 80ms on the first read of a cold file, from losing readahead on the export", + action = clap::ArgAction::Set + )] + direct_io: bool, + #[arg( + long, + env = "NYDUS_FILEIO_APISERVER", + help = "Specify the address to serve Prometheus metrics over a Unix socket, e.g. `unix:///run/nydus/api.sock`. The metrics are exposed at `/metrics`" + )] + apiserver: Option, + #[arg( + short = 'l', + long, + default_value = "info", + env = "NYDUS_FILEIO_LOG_LEVEL", + help = "Specify the logging level [trace, debug, info, warn, error]" + )] + log_level: Level, + + #[arg( + long, + default_value_os_t = default_log_dir(), + env = "NYDUS_FILEIO_LOG_DIR", + help = "Specify the log directory" + )] + log_dir: PathBuf, + + #[arg( + long, + default_value_t = 6, + env = "NYDUS_FILEIO_LOG_MAX_FILES", + help = "Specify the max number of log files" + )] + log_max_files: usize, + + #[arg( + long, + hide = true, + default_value_t = true, + env = "NYDUS_FILEIO_CONSOLE", + help = "Specify whether to print log" + )] + console: bool, +} + +fn default_fileio_threads() -> NonZeroUsize { + NonZeroUsize::new(default_parallelism(1, DEFAULT_MAX_THREADS)).unwrap() +} + +impl FileioCommand { + /// Export the flattened image over FUSE and, when a mountpoint is given, + /// mount it as file-backed EROFS until a termination signal arrives. + pub fn execute(&self) -> Result<()> { + let signals = signal::register_termination_signals()?; + + // The returned guards must stay alive for the daemon's lifetime or + // file logging stops. + let _guards = init_tracing( + NAME, + self.log_dir.clone(), + self.log_level, + self.log_max_files, + self.console, + ); + + let config = Config::load(&self.config)?; + self.run(signals, config) + } + + fn run(&self, signals: signal::Signals, config: Config) -> Result<()> { + // The kernel EROFS driver reads the backing file in whole blocks. + let flat = Arc::new( + FlatImage::open(&self.bootstrap, config, BLOCK_SIZE) + .context("failed to build the flattened view")?, + ); + // Warm the blob preparation (meta download + cache file sizing) in + // the background so the FUSE export and EROFS mount come up without + // waiting on backend round trips. flat_layout() is single-flight: a + // first read arriving early joins the same preparation. + let warm = flat.core().clone(); + std::thread::Builder::new() + .name("fileio-blob-warmup".to_string()) + .spawn(move || { + if let Err(err) = warm.blobs.flat_layout() { + tracing::warn!("background blob preparation failed: {err:#}"); + } + }) + .context("failed to spawn the blob warm-up thread")?; + let bootstrap_size = flat.core().bootstrap_size; + let fs = FlatImageFs::new(flat.clone()); + let image_size = fs.image_size(); + + let threads = self.threads.unwrap_or_else(default_fileio_threads); + let mut fuse_config = fuser::Config::default(); + // A container rootfs is read by uids other than the daemon's, and + // fuser's default ACL rejects those in userspace before the kernel's + // own permission check is reached. The export itself is read-only. + fuse_config.acl = fuser::SessionACL::All; + fuse_config.mount_options = vec![ + fuser::MountOption::RO, + fuser::MountOption::FSName("nydus-fileio".to_string()), + fuser::MountOption::NoAtime, + ]; + fuse_config.n_threads = Some(threads.get()); + fuse_config.clone_fd = true; + + let mut service = FileioService::mount(fs, &self.export_dir, &fuse_config)?; + let image = image_path(&self.export_dir); + info!( + "exported {} as {} ({} bytes, {} worker thread(s))", + self.bootstrap.display(), + image.display(), + image_size, + threads + ); + + if self.warm_bootstrap { + // A second handle onto the same shared view; warming reads through + // exactly the path the export serves. + let warm_fs = FlatImageFs::new(flat.clone()); + warm_bootstrap(&warm_fs, &service.notifier(), bootstrap_size); + } + + if let Some(mountpoint) = &self.mountpoint { + if let Err(err) = mount_image_file(&image, mountpoint, self.direct_io) { + // Nothing is mounted on top, so ending the session here leaves + // no stale EROFS mount behind. + service.shutdown(); + return Err(err); + } + service.set_erofs_mountpoint(mountpoint); + info!("mounted {} at {}", image.display(), mountpoint.display()); + } + + // Non-fatal like the fuse service: the export keeps serving without metrics. + let api_server = match self.apiserver.as_deref() { + Some(address) => match crate::api_server::ApiServer::start(address) { + Ok(server) => Some(server), + Err(err) => { + warn!("failed to start metrics apiserver: {}", err.report()); + None + } + }, + None => None, + }; + + // The export is served on background threads, so the daemon parks + // here until the signal thread reports a termination signal. + let (shutdown_tx, shutdown_rx) = std::sync::mpsc::channel::<()>(); + let signal_thread = signal::spawn_signal_thread("fileio", signals, move || { + let _ = shutdown_tx.send(()); + })?; + let _ = shutdown_rx.recv(); + + service.shutdown(); + if let Some(server) = api_server { + server.stop(); + } + signal_thread.shutdown()?; + Ok(()) + } +} diff --git a/nydus/src/bin/nydus/fuse.rs b/nydus/src/bin/nydus/fuse.rs index 5249e174047..3b124e055db 100644 --- a/nydus/src/bin/nydus/fuse.rs +++ b/nydus/src/bin/nydus/fuse.rs @@ -162,6 +162,9 @@ impl FuseCommand { Some(path) => Some(Config::load(path)?), None => None, }; + if let Some(config) = storage_config.as_ref() { + nydus_storage::cache::set_skip_verify_checksums(config.storage.skip_verify_checksums); + } // Runs the FUSE service until shutdown. self.run(storage_config) @@ -257,7 +260,9 @@ impl FuseCommand { } let reader = match (&self.blob, &self.bootstrap, backend) { - (Some(blob), None, _) => ErofsReader::open_blob(blob), + // A self-contained full blob still wants the decoded-block-group + // cache: without it every read decodes from the blob in place. + (Some(blob), None, _) => ErofsReader::open_blob(blob, cache_dir.as_deref()), (None, Some(bootstrap), Some(backend)) => { ErofsReader::open_bootstrap(bootstrap, backend, cache_dir.as_deref(), None) } diff --git a/nydus/src/bin/nydus/main.rs b/nydus/src/bin/nydus/main.rs index 1f33b45cfb8..3bca7e1bbf6 100644 --- a/nydus/src/bin/nydus/main.rs +++ b/nydus/src/bin/nydus/main.rs @@ -8,6 +8,8 @@ pub mod check; pub mod export; #[cfg(feature = "fanotify")] pub mod fanotify; +#[cfg(feature = "fileio")] +pub mod fileio; pub mod fuse; pub mod merge; #[cfg(feature = "nbd")] @@ -141,6 +143,16 @@ pub enum Command { long_about = "Export a nydus image as a read-only block device through the NBD protocol, optionally mounting the device as EROFS once the session is live." )] Nbd(nbd::NbdCommand), + + #[cfg(feature = "fileio")] + #[command( + name = "fileio", + author, + version, + about = "Serve a flattened nydus image as a file-backed EROFS mount", + long_about = "Export a flattened nydus image as a single file over FUSE and let the kernel EROFS driver mount that file directly (CONFIG_EROFS_FS_BACKED_BY_FILE, Linux >= 6.12). Lookup, readdir, stat and xattr are resolved in-kernel; only cold byte ranges reach the daemon." + )] + Fileio(fileio::FileioCommand), } /// Implement the execute for Command. @@ -161,6 +173,8 @@ impl Command { Self::Fanotify(cmd) => cmd.execute(), #[cfg(feature = "nbd")] Self::Nbd(cmd) => cmd.execute(), + #[cfg(feature = "fileio")] + Self::Fileio(cmd) => cmd.execute(), } } } diff --git a/nydus/src/bin/nydus/merge.rs b/nydus/src/bin/nydus/merge.rs index fc6470cbbcd..2646ce9526e 100644 --- a/nydus/src/bin/nydus/merge.rs +++ b/nydus/src/bin/nydus/merge.rs @@ -1,5 +1,5 @@ use clap::{Parser, ValueEnum}; -use nydus::build::merge::{merge_sources_to_bootstrap_bytes, WhiteoutSpec as MergeWhiteoutSpec}; +use nydus::build::merge::{merge_sources_to_bootstrap_writer, WhiteoutSpec as MergeWhiteoutSpec}; use nydus::error::{Context, Result}; use nydus_telemetry::logging::init_command_tracing; use std::fs::File; @@ -70,20 +70,17 @@ impl MergeCommand { self.run() } - /// Runs the merge: overlays the source layers in order and persists the - /// merged bootstrap. + /// Runs the merge: overlays the source layers in order and streams the + /// merged bootstrap straight to the output file. fn run(&self) -> Result<()> { let whiteout_spec = match self.whiteout_spec { WhiteoutSpec::Oci => MergeWhiteoutSpec::Oci, }; - let bootstrap_bytes = merge_sources_to_bootstrap_bytes(&self.sources, whiteout_spec)?; let output = File::create(&self.bootstrap) .with_context(|| format!("failed to create bootstrap: {}", self.bootstrap.display()))?; let mut writer = BufWriter::new(output); - writer - .write_all(&bootstrap_bytes) - .with_context(|| format!("failed to write bootstrap: {}", self.bootstrap.display()))?; + merge_sources_to_bootstrap_writer(&self.sources, whiteout_spec, &mut writer)?; writer .flush() .with_context(|| format!("failed to flush bootstrap: {}", self.bootstrap.display()))?; diff --git a/nydus/src/bin/nydus/ublk.rs b/nydus/src/bin/nydus/ublk.rs index 1e7314d41bd..45fce59b1e8 100644 --- a/nydus/src/bin/nydus/ublk.rs +++ b/nydus/src/bin/nydus/ublk.rs @@ -8,7 +8,7 @@ use nydus_config::Config; use nydus_telemetry::logging::init_tracing; use std::path::PathBuf; use std::sync::Arc; -use tracing::{info, Level}; +use tracing::{info, warn, Level}; use super::*; @@ -68,6 +68,13 @@ pub struct UblkCommand { )] unprivileged: bool, + #[arg( + long, + env = "NYDUS_UBLK_APISERVER", + help = "Specify the address to serve Prometheus metrics over a Unix socket, e.g. `unix:///run/nydus/api.sock`. The metrics are exposed at `/metrics`" + )] + apiserver: Option, + #[arg( short = 'l', long, @@ -155,6 +162,18 @@ impl UblkCommand { let service = UblkService::new(core, &options)?; println!("{}", service.dev_path()); + // Non-fatal like the fuse service: the device keeps serving without metrics. + let api_server = match self.apiserver.as_deref() { + Some(address) => match crate::api_server::ApiServer::start(address) { + Ok(server) => Some(server), + Err(err) => { + warn!("failed to start metrics apiserver: {}", err.report()); + None + } + }, + None => None, + }; + let handle = service.handle(); let signal_thread = signal::spawn_signal_thread("ublk", signals, move || { handle.stop(); @@ -162,6 +181,9 @@ impl UblkCommand { let result = service.run(); service.delete(); + if let Some(server) = api_server { + server.stop(); + } signal_thread.shutdown()?; result diff --git a/nydus/src/build/blob_chunk.rs b/nydus/src/build/blob_chunk.rs index e040037ad8c..ab09b1f5f92 100644 --- a/nydus/src/build/blob_chunk.rs +++ b/nydus/src/build/blob_chunk.rs @@ -7,12 +7,19 @@ use nydus_format::blob::{ use nydus_format::erofs::{ErofsChunkAddr, EROFS_BLOB_ID_SIZE, EROFS_BLOCK_SIZE, EROFS_NULL_ADDR}; use nydus_format::utils::align_up_usize; use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; use std::fs::File; use std::io::{Read, Write}; use std::mem; use std::path::Path; - -/// Manages writing chunk data to a separate blob device. +use std::sync::mpsc; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +/// Manages writing chunk data to a separate blob device. Chunk bytes +/// (tail-block padding included) stream straight into block groups, so the +/// group stream equals the logical space and blob meta carries only the +/// block group table. pub struct BlobWriter { writer: W, file_chunk_size: u32, @@ -23,12 +30,155 @@ pub struct BlobWriter { data_hasher: Sha256, block_group_block_offset: u64, block_group_buffer: Vec, - blob_metadata_block_groups: Vec, blob_metadata_chunks: Vec, + blob_metadata_block_groups: Vec, + // Reused per-file read buffer: a fresh 1 MiB Vec per file costs an + // mmap/munmap plus page faults for every source file. + chunk_buf: Vec, + // Lazily started background crc32+zstd pipeline for block groups. + encoder: Option, } const MAX_COMPRESSED_SIZE_PERCENT: u128 = 70; +/// One block of zeros for hashing and storing tail-block padding without +/// allocating; padding never exceeds a single EROFS block. +const ZERO_BLOCK: [u8; EROFS_BLOCK_SIZE as usize] = [0u8; EROFS_BLOCK_SIZE as usize]; + +/// Number of background block-group encoder threads. Encoding (crc32 + zstd) +/// runs well ahead of the single-threaded produce side, so two workers fully +/// hide it; more would only grow the in-flight memory. +const ENCODE_WORKERS: usize = 2; +/// Maximum encode jobs in flight before the producer drains one; bounds the +/// extra peak memory to a couple of block groups (in + encoded out each). +const ENCODE_MAX_IN_FLIGHT: usize = 2; + +struct EncodeJob { + seq: u64, + data: Vec, +} + +struct EncodedBlockGroup { + data: Vec, + crc32: u32, + /// `Some` when compression met the format's worthwhile threshold. + compressed: Option>, +} + +/// Offloads per-block-group crc32 + zstd to background threads while the +/// caller keeps producing. Results are drained strictly in submission order +/// so the written stream and metadata tables stay deterministic; input +/// buffers circulate back for reuse. +struct BlockGroupEncoder { + tx: Option>, + encoded_rx: mpsc::Receiver<(u64, EncodedBlockGroup)>, + pending: BTreeMap, + next_seq_in: u64, + next_seq_out: u64, + free_buffers: Vec>, + workers: Vec>, +} + +impl BlockGroupEncoder { + fn new(compressor: BlobMetadataCompressor) -> Self { + let (tx, rx) = mpsc::channel::(); + let (encoded_tx, encoded_rx) = mpsc::channel(); + let rx = Arc::new(Mutex::new(rx)); + let workers = (0..ENCODE_WORKERS) + .map(|_| { + let rx = Arc::clone(&rx); + let encoded_tx = encoded_tx.clone(); + std::thread::spawn(move || loop { + let job = match rx.lock().unwrap_or_else(|p| p.into_inner()).recv() { + Ok(job) => job, + Err(_) => break, + }; + let crc32 = crc32c(&job.data); + let compressed = match compressor { + BlobMetadataCompressor::None => None, + BlobMetadataCompressor::Zstd => zstd::bulk::compress(&job.data, 0) + .ok() + .filter(|c| compression_is_worthwhile(c.len(), job.data.len())), + BlobMetadataCompressor::Lz4Block => { + let compressed = lz4_flex::block::compress(&job.data); + compression_is_worthwhile(compressed.len(), job.data.len()) + .then_some(compressed) + } + }; + let encoded = EncodedBlockGroup { + data: job.data, + crc32, + compressed, + }; + if encoded_tx.send((job.seq, encoded)).is_err() { + break; + } + }) + }) + .collect(); + Self { + tx: Some(tx), + encoded_rx, + pending: BTreeMap::new(), + next_seq_in: 0, + next_seq_out: 0, + free_buffers: Vec::new(), + workers, + } + } + + fn submit(&mut self, data: Vec) -> Result<()> { + let job = EncodeJob { + seq: self.next_seq_in, + data, + }; + self.next_seq_in += 1; + self.tx + .as_ref() + .expect("encoder is alive until finish") + .send(job) + .map_err(|_| Error::Runtime("block group encoder threads exited early".to_string())) + } + + fn in_flight(&self) -> usize { + (self.next_seq_in - self.next_seq_out) as usize + } + + /// Receive the next completed group in submission order. + fn recv_next(&mut self) -> Result { + loop { + if let Some(encoded) = self.pending.remove(&self.next_seq_out) { + self.next_seq_out += 1; + return Ok(encoded); + } + let (seq, encoded) = self.encoded_rx.recv().map_err(|_| { + Error::Runtime("block group encoder threads exited early".to_string()) + })?; + self.pending.insert(seq, encoded); + } + } + + fn take_buffer(&mut self) -> Option> { + self.free_buffers.pop() + } + + fn recycle_buffer(&mut self, mut buf: Vec) { + if self.free_buffers.len() < ENCODE_MAX_IN_FLIGHT { + buf.clear(); + self.free_buffers.push(buf); + } + } +} + +impl Drop for BlockGroupEncoder { + fn drop(&mut self) { + self.tx.take(); + for worker in self.workers.drain(..) { + let _ = worker.join(); + } + } +} + impl BlobWriter { pub fn new(path: &Path, chunk_size: u32) -> Result { Self::new_with_compressor(path, chunk_size, BlobMetadataCompressor::None) @@ -99,8 +249,10 @@ impl BlobWriter { data_hasher: Sha256::new(), block_group_block_offset: 0, block_group_buffer: Vec::with_capacity(block_group_size as usize), - blob_metadata_block_groups: Vec::new(), blob_metadata_chunks: Vec::new(), + blob_metadata_block_groups: Vec::new(), + chunk_buf: vec![0u8; file_chunk_size as usize], + encoder: None, }) } @@ -146,13 +298,22 @@ impl BlobWriter { } pub fn finish(&mut self) -> Result<()> { + // The data stream is byte granular, so the tail block group must be + // zero padded to a whole block before it is flushed (block groups + // always describe whole uncompressed blocks). + if !self.block_group_buffer.is_empty() { + let padded = + align_up_usize(self.block_group_buffer.len(), EROFS_BLOCK_SIZE as usize) + .ok_or_else(|| Error::Overflow("block group padding overflow".to_string()))?; + self.block_group_buffer.resize(padded, 0); + } self.flush_block_group()?; + self.drain_all_encoded()?; self.writer.flush().context("failed to flush blob device") } /// Process a regular file: read it in chunk-sized pieces and append every - /// chunk to the blob device. Chunk-level digests are recorded in blob meta; - /// deduplication is intentionally disabled for now. + /// chunk to the blob device. pub fn write_file_chunks( &mut self, path: &Path, @@ -168,13 +329,21 @@ impl BlobWriter { let chunk_size = self.file_chunk_size as u64; let chunk_count = file_size.div_ceil(chunk_size); let mut indexes = Vec::with_capacity(chunk_count as usize); - let mut chunk_buf = vec![0u8; self.file_chunk_size as usize]; + let mut chunk_buf = mem::take(&mut self.chunk_buf); + if chunk_buf.len() < self.file_chunk_size as usize { + chunk_buf = vec![0u8; self.file_chunk_size as usize]; + } for i in 0..chunk_count { let remaining = file_size - i * chunk_size; let to_read = remaining.min(chunk_size) as usize; - f.read_exact(&mut chunk_buf[..to_read]) - .with_context(|| format!("failed to read source file: {}", path.display()))?; + if let Err(err) = f + .read_exact(&mut chunk_buf[..to_read]) + .with_context(|| format!("failed to read source file: {}", path.display())) + { + self.chunk_buf = chunk_buf; + return Err(err); + } // A fully-zero chunk (a real filesystem hole reads back as zeros, // and so does zero-filled data) is not stored at all: it gets a @@ -197,13 +366,20 @@ impl BlobWriter { // block so block groups pack dense real blocks instead of large zero runs. let write_len = align_up_usize(to_read, EROFS_BLOCK_SIZE as usize).expect("alignment overflowed"); - let blkaddr = self.append_chunk(&chunk_buf[..to_read], write_len)?; + let blkaddr = match self.append_chunk(&chunk_buf[..to_read], write_len) { + Ok(blkaddr) => blkaddr, + Err(err) => { + self.chunk_buf = chunk_buf; + return Err(err); + } + }; indexes.push(ErofsChunkAddr { blkaddr, device_id: 1, }); } + self.chunk_buf = chunk_buf; Ok(indexes) } @@ -214,28 +390,34 @@ impl BlobWriter { Error::Overflow(format!("blob meta chunk block count exceeds u32: {err}")) })?; - // Block-aligned chunk payload: real bytes followed by zero padding only - // in its final block. - let mut uncompressed = vec![0u8; write_len]; - uncompressed[..data.len()].copy_from_slice(data); + // The chunk occupies `block_count` logical blocks (EROFS chunk indexes + // address the dense logical space). self.next_blkaddr += block_count as u64; // Record the chunk by its absolute block position; chunks are tracked - // independently of block groups as a digest index only. - let digest = *blake3::hash(&uncompressed).as_bytes(); - let chunk = BlobMetadataChunk::new(digest, addr, block_count)?; + // independently of block groups as a digest index only. The digest + // covers the block-aligned payload (real bytes plus tail-block zero + // padding), hashed in place to avoid materialising a padded copy. + let mut hasher = blake3::Hasher::new(); + hasher.update(data); + if write_len > data.len() { + hasher.update(&ZERO_BLOCK[..write_len - data.len()]); + } + let chunk = BlobMetadataChunk::new(*hasher.finalize().as_bytes(), addr, block_count)?; self.blob_metadata_chunks.push(chunk); - // Feed the bytes into the block group stream, which packs whole blocks up to - // the block group size regardless of chunk boundaries. - self.append_to_block_group_stream(&uncompressed)?; - + // The group stream mirrors the logical space one-to-one, so the + // tail-block padding must be stored physically. + self.append_to_block_group_stream(data)?; + if write_len > data.len() { + self.append_to_block_group_stream(&ZERO_BLOCK[..write_len - data.len()])?; + } Ok(addr) } - /// Append block-aligned data to the current block group, flushing whenever it - /// fills to the block group size. A chunk may straddle a block group boundary, so block groups - /// are pure block runs of exactly `block_group_size` (except the last). + /// Append data to the current block group, flushing whenever it fills to + /// the block group size, so block groups are pure block runs of exactly + /// `block_group_size` (except the last). fn append_to_block_group_stream(&mut self, mut data: &[u8]) -> Result<()> { let block_group_size = self.block_group_size as usize; while !data.is_empty() { @@ -255,22 +437,36 @@ impl BlobWriter { return Ok(()); } - let uncompressed = mem::take(&mut self.block_group_buffer); - self.block_group_buffer = Vec::with_capacity(self.block_group_size as usize); - let crc32 = crc32c(&uncompressed); - let compressed = match self.compressor { - BlobMetadataCompressor::None => None, - BlobMetadataCompressor::Zstd => { - let compressed = zstd::bulk::compress(&uncompressed, 0) - .context("failed to compress blob meta block group with zstd")?; - if compression_is_worthwhile(compressed.len(), uncompressed.len()) { - Some(compressed) - } else { - None - } - } - }; - let encoded = compressed.as_deref().unwrap_or(&uncompressed); + if self.encoder.is_none() { + self.encoder = Some(BlockGroupEncoder::new(self.compressor)); + } + let encoder = self.encoder.as_mut().expect("encoder initialised above"); + let replacement = encoder + .take_buffer() + .unwrap_or_else(|| Vec::with_capacity(self.block_group_size as usize)); + let uncompressed = mem::replace(&mut self.block_group_buffer, replacement); + encoder.submit(uncompressed)?; + while self + .encoder + .as_ref() + .expect("encoder initialised above") + .in_flight() + > ENCODE_MAX_IN_FLIGHT + { + self.drain_one_encoded()?; + } + Ok(()) + } + + /// Write out the next completed block group, in submission order. + fn drain_one_encoded(&mut self) -> Result<()> { + let group = self + .encoder + .as_mut() + .expect("drain is only called with a live encoder") + .recv_next()?; + let uncompressed_len = group.data.len(); + let encoded: &[u8] = group.compressed.as_deref().unwrap_or(&group.data); // Encoded block group payloads are packed back-to-back in the data region. // No block padding is inserted between compressed block groups; they are read @@ -284,7 +480,7 @@ impl BlobWriter { self.next_compressed_offset = compressed_offset + encoded.len() as u64; let block_count = - u32::try_from(uncompressed.len() / EROFS_BLOCK_SIZE as usize).map_err(|err| { + u32::try_from(uncompressed_len / EROFS_BLOCK_SIZE as usize).map_err(|err| { Error::Overflow(format!( "blob meta block group uncompressed block count exceeds u32: {err}" )) @@ -294,10 +490,26 @@ impl BlobWriter { block_count, compressed_offset, encoded.len() as u32, - crc32, + group.crc32, )?; self.blob_metadata_block_groups.push(entry); self.block_group_block_offset += block_count as u64; + + self.encoder + .as_mut() + .expect("drain is only called with a live encoder") + .recycle_buffer(group.data); + Ok(()) + } + + fn drain_all_encoded(&mut self) -> Result<()> { + while self + .encoder + .as_ref() + .is_some_and(|encoder| encoder.in_flight() > 0) + { + self.drain_one_encoded()?; + } Ok(()) } } @@ -315,7 +527,6 @@ mod tests { use nydus_format::blob::DEFAULT_NYDUS_BLOB_METADATA_CHUNK_SIZE; use std::fs; use tempfile::tempdir; - #[test] fn blob_metadata_block_group_round_trips_minimal_fields() { let payload = vec![0u8; 0x3000]; @@ -368,8 +579,9 @@ mod tests { assert_eq!(indexes_b.len(), 1); assert_eq!(indexes_a[0].blkaddr, 0); assert_eq!(indexes_a[1].blkaddr, 256); - // Dense packing: file_a's 4KiB tail chunk occupies a single block, so - // file_b starts right after it instead of being padded to a full chunk. + // Dense logical packing: file_a's 4KiB tail chunk occupies a single + // block, so file_b starts right after it instead of being padded to a + // full chunk. assert_eq!(indexes_b[0].blkaddr, 257); assert_eq!(writer.total_blocks(), 513); @@ -434,11 +646,7 @@ mod tests { assert_eq!(indexes[0].blkaddr, 0); assert_eq!(indexes[1].blkaddr, 1); assert_eq!(blob_metadata.header().chunk_size(), EROFS_BLOCK_SIZE); - assert_eq!(blob_metadata.chunks().len(), 2); assert_eq!(blob_metadata.block_groups().len(), 1); - assert_eq!(blob_metadata.chunks()[0].uncompressed_block_count(), 1); - assert_eq!(blob_metadata.chunks()[0].uncompressed_size(), 4096); - assert_eq!(blob_metadata.chunks()[1].uncompressed_block_offset(), 1); assert_eq!(blob_metadata.block_groups()[0].uncompressed_size(), 8192); } @@ -458,7 +666,6 @@ mod tests { .write_file_chunks(&input_path, content.len() as u64) .unwrap(); writer.finish().unwrap(); - let blob_metadata = writer.blob_metadata(0).unwrap(); // The all-zero chunk becomes a hole: a null chunk index with no blob // reference, no blob-meta chunk entry, and no bytes in the data region. @@ -466,8 +673,6 @@ mod tests { assert_eq!(indexes[0].blkaddr, 0); assert_eq!(indexes[1].blkaddr, EROFS_NULL_ADDR); assert_eq!(indexes[2].blkaddr, 1); - assert_eq!(blob_metadata.chunks().len(), 2); - assert_eq!(blob_metadata.chunks()[1].uncompressed_block_offset(), 1); assert_eq!(writer.total_blocks(), 2); let data = fs::read(&blob_path).unwrap(); assert_eq!(data.len(), 2 * EROFS_BLOCK_SIZE as usize); @@ -496,7 +701,6 @@ mod tests { // Every chunk is a hole: nothing lands in the blob at all. assert_eq!(indexes.len(), 2); assert!(indexes.iter().all(|ci| ci.blkaddr == EROFS_NULL_ADDR)); - assert!(writer.blob_metadata_chunks().is_empty()); assert!(writer.blob_metadata_block_groups().is_empty()); assert_eq!(writer.total_blocks(), 0); assert_eq!(fs::read(&blob_path).unwrap().len(), 0); @@ -522,7 +726,6 @@ mod tests { writer.finish().unwrap(); let block_groups = writer.blob_metadata_block_groups(); - assert_eq!(writer.blob_metadata_chunks().len(), 1); assert_eq!(block_groups.len(), 1); assert_eq!(block_groups[0].uncompressed_block_count(), 256); assert_eq!( @@ -552,7 +755,7 @@ mod tests { .unwrap(); let raw = fs::read(&blob_metadata_path).unwrap(); - // 4 KiB header block + one chunk + one block group, padded to a block. + // 4 KiB header block + one block group, padded to a block. assert_eq!(raw.len(), 8192); let blob_metadata = BlobMetadata::from_path(&blob_metadata_path, false).unwrap(); @@ -564,6 +767,78 @@ mod tests { assert_eq!(blob_metadata.chunks()[0].uncompressed_block_offset(), 0); assert_eq!(blob_metadata.block_groups()[0].compressed_offset(), 8192); } + #[test] + fn blob_writer_stores_duplicate_content_verbatim() { + let dir = tempdir().unwrap(); + let blob_path = dir.path().join("blob.data"); + let file_a = dir.path().join("a.bin"); + let file_b = dir.path().join("b.bin"); + let body = pseudo_random_bytes((1 << 20) + 100); + fs::write(&file_a, &body).unwrap(); + fs::write(&file_b, &body).unwrap(); + + let mut writer = BlobWriter::new_with_compressor( + &blob_path, + DEFAULT_NYDUS_BLOB_METADATA_CHUNK_SIZE, + BlobMetadataCompressor::None, + ) + .unwrap(); + writer + .write_file_chunks(&file_a, body.len() as u64) + .unwrap(); + writer + .write_file_chunks(&file_b, body.len() as u64) + .unwrap(); + writer.finish().unwrap(); + + let padded = align_up_usize(body.len(), EROFS_BLOCK_SIZE as usize) + .expect("alignment overflowed") as u64; + assert_eq!(writer.data_size(), 2 * padded); + assert_eq!(writer.total_blocks() * EROFS_BLOCK_SIZE as u64, 2 * padded); + + let blob_metadata = writer.blob_metadata(0).unwrap(); + assert_eq!(blob_metadata.header().chunk_count(), 4); + assert_eq!(blob_metadata.uncompressed_size(), 2 * padded); + + let data = fs::read(&blob_path).unwrap(); + assert_eq!(data.len() as u64, 2 * padded); + assert_eq!(&data[..body.len()], &body[..]); + assert!(data[body.len()..padded as usize].iter().all(|b| *b == 0)); + assert_eq!( + &data[padded as usize..padded as usize + body.len()], + &body[..] + ); + } + + #[test] + fn blob_writer_keeps_zero_chunk_elision_and_stores_tail_padding() { + let dir = tempdir().unwrap(); + let blob_path = dir.path().join("blob.data"); + let input_path = dir.path().join("input.bin"); + let mut content = vec![b'a'; EROFS_BLOCK_SIZE as usize]; + content.extend(vec![0u8; EROFS_BLOCK_SIZE as usize]); + content.extend(vec![b'c'; 100]); + fs::write(&input_path, &content).unwrap(); + + let mut writer = BlobWriter::new(&blob_path, EROFS_BLOCK_SIZE).unwrap(); + let indexes = writer + .write_file_chunks(&input_path, content.len() as u64) + .unwrap(); + writer.finish().unwrap(); + + assert_eq!(indexes.len(), 3); + assert_eq!(indexes[0].blkaddr, 0); + assert_eq!(indexes[1].blkaddr, EROFS_NULL_ADDR); + assert_eq!(indexes[2].blkaddr, 1); + assert_eq!(writer.total_blocks(), 2); + + let data = fs::read(&blob_path).unwrap(); + assert_eq!(data.len(), 2 * EROFS_BLOCK_SIZE as usize); + assert!(data[..EROFS_BLOCK_SIZE as usize].iter().all(|b| *b == b'a')); + let tail = &data[EROFS_BLOCK_SIZE as usize..]; + assert!(tail[..100].iter().all(|b| *b == b'c')); + assert!(tail[100..].iter().all(|b| *b == 0)); + } fn pseudo_random_bytes(len: usize) -> Vec { let mut value = 0x1234_5678_9abc_def0u64; diff --git a/nydus/src/build/bootstrap.rs b/nydus/src/build/bootstrap.rs index 4192f88ce9a..5fe835cea92 100644 --- a/nydus/src/build/bootstrap.rs +++ b/nydus/src/build/bootstrap.rs @@ -1,17 +1,18 @@ use super::layout::MetadataLayout; use crate::build::dir::{serialize_directory, DirChild}; use crate::build::image::{ - device_table_meta_blkaddr, write_erofs_superblock_checksum, write_image, + device_table_meta_blkaddr, fill_image_head, write_erofs_superblock_checksum, }; use crate::build::inode::{ erofs_inode_size, serialize_inode, symlink_is_inline, InodeData, InodeInfo, }; -use nydus_error::{Error, Result}; +use nydus_error::{Context, Error, Result}; use nydus_format::erofs::{ ErofsDeviceSlot, EROFS_BLOCK_SIZE, EROFS_DEVICESLOT_SIZE, EROFS_FT_DIR, EROFS_SB_BASE_SIZE, - EROFS_SUPER_OFFSET, + EROFS_SUPER_OFFSET, EROFS_XATTR_INDEX_TRUSTED, }; use nydus_format::utils::align_up_usize; +use std::io::Write; pub const FLATTENED_BLOB_ALIGNMENT: u64 = 0x8_0000; @@ -32,17 +33,189 @@ pub fn render_flattened_bootstrap( device_slots: &[ErofsDeviceSlot], uuid: &[u8; 16], ) -> Result> { + let mut bootstrap = Vec::new(); + render_flattened_bootstrap_to(&mut bootstrap, inodes, epoch, device_slots, uuid)?; + Ok(bootstrap) +} + +/// Stream-render a flattened bootstrap into `writer`: a sizing pass assigns +/// every offset without materialising a buffer, then the head, the inode +/// region and the directory/symlink data are written strictly in offset +/// order. Peak memory is O(1) in the bootstrap size (directory data is +/// serialized twice: once for its size, once for the write). Returns the +/// bootstrap size in bytes. +pub fn render_flattened_bootstrap_to( + writer: &mut impl Write, + inodes: &mut [InodeInfo], + epoch: u64, + device_slots: &[ErofsDeviceSlot], + uuid: &[u8; 16], +) -> Result { + if inodes.is_empty() { + return Err(Error::InvalidParameter( + "cannot render bootstrap for empty inode set".to_string(), + )); + } + + let meta_blkaddr = device_table_meta_blkaddr(device_slots.len())?; + let head_size = meta_blkaddr as usize * EROFS_BLOCK_SIZE as usize; + let mut layout = MetadataLayout::size_only(meta_blkaddr); + + // --- Sizing pass: identical allocation order to the buffered renderer --- + alloc_inodes(&mut layout, inodes, epoch); + set_parent_nids(inodes); + layout.pad_to_block(); + + // Data-region entries in allocation (= write) order, identified by inode + // index: every directory's data, then every long symlink's target. + let mut data_entries: Vec<(usize, usize)> = Vec::new(); + for index in 0..inodes.len() { + if !matches!(inodes[index].data, InodeData::Directory { .. }) { + continue; + } + let dir_data_len = serialize_dir_data(inodes, index).len(); + let (data_offset, data_startblk) = layout.alloc_dir_data(dir_data_len); + if let InodeData::Directory { + ref mut startblk, + ref mut data_size, + .. + } = inodes[index].data + { + *startblk = data_startblk; + *data_size = dir_data_len; + } + inodes[index].size = dir_data_len as u64; + data_entries.push((index, data_offset)); + } + for (index, inode) in inodes.iter_mut().enumerate() { + if symlink_is_inline(inode) { + continue; + } + let InodeData::Symlink { ref target, .. } = inode.data else { + continue; + }; + let (data_offset, data_startblk) = layout.alloc_dir_data(target.len()); + if let InodeData::Symlink { + ref mut startblk, .. + } = inode.data + { + *startblk = data_startblk; + } + data_entries.push((index, data_offset)); + } + + let metadata_len = layout.pad_to_block(); + let bootstrap_size = (head_size + metadata_len) as u64; + + // The head can be written up front: the flattened device addresses only + // need the total size, and the superblock checksum covers block 0 alone. + let mut flattened_slots = device_slots.to_vec(); + set_flattened_mapped_blkaddrs( + &mut flattened_slots, + bootstrap_size, + FLATTENED_BLOB_ALIGNMENT, + )?; + + let root_nid = inodes[0].nid; + if root_nid > u16::MAX as u64 { + return Err(Error::Overflow("root nid exceeds 16-bit range".to_string())); + } + let mut head = vec![0u8; head_size]; + fill_image_head( + &mut head, + metadata_len, + root_nid as u16, + inodes.len() as u64, + epoch, + &flattened_slots, + uuid, + has_visible_xattrs(inodes), + )?; + writer + .write_all(&head) + .context("failed to write bootstrap head")?; + + // --- Write pass: inode region, then data region, in offset order --- + let mut cursor = 0usize; + for inode in inodes.iter() { + debug_assert!(inode.meta_offset >= cursor); + write_zeros(writer, inode.meta_offset - cursor)?; + let bytes = serialize_inode(inode, epoch); + writer + .write_all(&bytes) + .context("failed to write bootstrap inode")?; + cursor = inode.meta_offset + bytes.len(); + } + + for (index, data_offset) in data_entries { + debug_assert!(data_offset >= cursor); + write_zeros(writer, data_offset - cursor)?; + match &inodes[index].data { + InodeData::Directory { .. } => { + let dir_data = serialize_dir_data(inodes, index); + writer + .write_all(&dir_data) + .context("failed to write bootstrap directory data")?; + cursor = data_offset + dir_data.len(); + } + InodeData::Symlink { target, .. } => { + writer + .write_all(target) + .context("failed to write bootstrap symlink target")?; + cursor = data_offset + target.len(); + } + _ => unreachable!("data_entries only holds directories and long symlinks"), + } + } + + debug_assert!(metadata_len >= cursor); + write_zeros(writer, metadata_len - cursor)?; + Ok(bootstrap_size) +} + +/// Serialize the directory data of `inodes[index]` from its child refs, +/// resolving child nids through the shared inode table. +fn serialize_dir_data(inodes: &[InodeInfo], index: usize) -> Vec { + let InodeData::Directory { + ref children, + parent_nid, + .. + } = inodes[index].data + else { + unreachable!("serialize_dir_data is only called for directories"); + }; + let dir_children: Vec = children + .iter() + .map(|de| DirChild { + name: de.name.clone(), + nid: inodes[de.inode_index].nid, + file_type: de.file_type, + }) + .collect(); + serialize_directory(&dir_children, inodes[index].nid, parent_nid) +} + +fn write_zeros(writer: &mut impl Write, n: usize) -> Result<()> { + nydus_format::utils::write_zeros(writer, n as u64).context("failed to write bootstrap padding") +} + +/// Rewrite a rendered bootstrap's device table with flattened mapped block +/// addresses for the given slots and refresh the superblock checksum. The +/// metadata region is device-slot independent, so a bootstrap rendered for +/// one slot set can be retargeted in place instead of re-rendered. +pub(crate) fn flatten_bootstrap_in_place( + bootstrap: &mut [u8], + device_slots: &[ErofsDeviceSlot], +) -> Result<()> { let mut device_slots = device_slots.to_vec(); - let mut bootstrap = render_bootstrap_inner(inodes, epoch, &device_slots, uuid)?; - debug_assert_eq!(bootstrap.len() % EROFS_BLOCK_SIZE as usize, 0); set_flattened_mapped_blkaddrs( &mut device_slots, bootstrap.len() as u64, FLATTENED_BLOB_ALIGNMENT, )?; - patch_device_slots(&mut bootstrap, &device_slots)?; + patch_device_slots(bootstrap, &device_slots)?; debug_assert_eq!(bootstrap.len() % EROFS_BLOCK_SIZE as usize, 0); - Ok(bootstrap) + Ok(()) } fn set_flattened_mapped_blkaddrs( @@ -124,66 +297,57 @@ fn render_bootstrap_inner( let mut layout = MetadataLayout::with_meta_blkaddr(device_table_meta_blkaddr(device_slots.len())?); - for inode in inodes.iter_mut() { - if !symlink_is_inline(inode) && matches!(inode.data, InodeData::Symlink { .. }) { - inode.is_extended = true; - } - // A compact inode stores mtime as a 32-bit delta from the epoch, so a - // timestamp further out than that has to move to the extended layout - // rather than wrap. - if inode.mtime.wrapping_sub(epoch) > u32::MAX as u64 { - inode.is_extended = true; - } - let inode_size = erofs_inode_size(inode); - let has_inline = symlink_is_inline(inode); - let (offset, nid) = layout.alloc_inode(inode_size, has_inline); - inode.meta_offset = offset; - inode.nid = nid; - } - + alloc_inodes(&mut layout, inodes, epoch); set_parent_nids(inodes); layout.pad_to_block(); + // The directory-data region that follows is in the same order of + // magnitude as the inode region; one generous reservation avoids every + // doubling realloc (each one transiently duplicates the buffer in RSS). + layout.reserve(layout.buf().len() * 2); - let dir_infos: Vec<(usize, Vec, u64, u64)> = inodes + let dir_indexes: Vec = inodes .iter() .enumerate() .filter_map(|(index, inode)| { - if let InodeData::Directory { - ref children, - parent_nid, - .. - } = inode.data - { - let self_nid = inode.nid; - let dir_children: Vec = children - .iter() - .map(|de| DirChild { - name: de.name.clone(), - nid: inodes[de.inode_index].nid, - file_type: de.file_type, - }) - .collect(); - Some((index, dir_children, self_nid, parent_nid)) - } else { - None - } + matches!(inode.data, InodeData::Directory { .. }).then_some(index) }) .collect(); - for (index, dir_children, self_nid, parent_nid) in dir_infos { + // Directories are processed one at a time: cloning every directory's + // child names up front would keep a second copy of all file names + // resident at once. + for index in dir_indexes { + let InodeData::Directory { + ref children, + parent_nid, + .. + } = inodes[index].data + else { + unreachable!("dir_indexes only collects directory inodes"); + }; + let self_nid = inodes[index].nid; + let dir_children: Vec = children + .iter() + .map(|de| DirChild { + name: de.name.clone(), + nid: inodes[de.inode_index].nid, + file_type: de.file_type, + }) + .collect(); let dir_data = serialize_directory(&dir_children, self_nid, parent_nid); + drop(dir_children); let dir_data_len = dir_data.len(); let (data_offset, startblk) = layout.alloc_dir_data(dir_data_len); layout.write_at(data_offset, &dir_data); if let InodeData::Directory { - startblk: ref mut sb, - data_size: ref mut dds, + startblk: ref mut slot_startblk, + data_size: ref mut slot_data_size, .. } = inodes[index].data { - *sb = startblk; - *dds = dir_data_len; + *slot_startblk = startblk; + *slot_data_size = dir_data_len; } inodes[index].size = dir_data_len as u64; } @@ -207,11 +371,11 @@ fn render_bootstrap_inner( let (data_offset, startblk) = layout.alloc_dir_data(target.len()); layout.write_at(data_offset, &target); if let InodeData::Symlink { - startblk: ref mut sb, + startblk: ref mut slot_startblk, .. } = inodes[index].data { - *sb = startblk; + *slot_startblk = startblk; } } @@ -226,20 +390,60 @@ fn render_bootstrap_inner( return Err(Error::Overflow("root nid exceeds 16-bit range".to_string())); } - let mut bootstrap = Vec::new(); - write_image( + // The layout buffer already holds the head region followed by the padded + // metadata area; fill the head in place so the buffer IS the bootstrap + // and the tens-of-MiB metadata copy of the old write_image path is gone. + let head_size = + device_table_meta_blkaddr(device_slots.len())? as usize * EROFS_BLOCK_SIZE as usize; + let mut bootstrap = layout.into_image_buf(); + let metadata_len = bootstrap.len() - head_size; + fill_image_head( &mut bootstrap, - layout.buf(), + metadata_len, root_nid as u16, inodes.len() as u64, epoch, device_slots, uuid, + has_visible_xattrs(inodes), )?; Ok(bootstrap) } +/// Assign every inode's on-disk slot in table order: promote inodes that +/// cannot stay compact to the extended layout, then allocate and stamp each +/// one's metadata offset and nid. +fn alloc_inodes(layout: &mut MetadataLayout, inodes: &mut [InodeInfo], epoch: u64) { + for inode in inodes.iter_mut() { + if !symlink_is_inline(inode) && matches!(inode.data, InodeData::Symlink { .. }) { + inode.is_extended = true; + } + // A compact inode stores mtime as a 32-bit delta from the epoch, so a + // timestamp further out than that has to move to the extended layout + // rather than wrap. + if inode.mtime.wrapping_sub(epoch) > u32::MAX as u64 { + inode.is_extended = true; + } + let inode_size = erofs_inode_size(inode); + let has_inline = symlink_is_inline(inode); + let (offset, nid) = layout.alloc_inode(inode_size, has_inline); + inode.meta_offset = offset; + inode.nid = nid; + } +} + +/// Whether any inode carries a user-visible xattr. Nydus-internal xattrs +/// (trusted.nydus.*) are hidden from readers, so they alone do not +/// disqualify the image-wide no-xattr shortcut. +fn has_visible_xattrs(inodes: &[InodeInfo]) -> bool { + inodes.iter().any(|inode| { + inode.xattrs.iter().any(|entry| { + !(entry.name_index == EROFS_XATTR_INDEX_TRUSTED && entry.suffix.starts_with(b"nydus.")) + }) + }) +} + pub(crate) fn set_parent_nids(inodes: &mut [InodeInfo]) { let root_nid = inodes[0].nid; if let InodeData::Directory { diff --git a/nydus/src/build/image.rs b/nydus/src/build/image.rs index 75c2e1ffc8e..85322763361 100644 --- a/nydus/src/build/image.rs +++ b/nydus/src/build/image.rs @@ -1,3 +1,4 @@ +#[cfg(test)] use std::io::Write; use crc32c::crc32c_append; @@ -5,7 +6,8 @@ use crc32c::crc32c_append; use nydus_error::{Error, Result}; use nydus_format::erofs::{ ErofsDeviceSlot, ErofsSuperblock, EROFS_BLOCK_SIZE, EROFS_DEVICESLOT_SIZE, - EROFS_FEATURE_COMPAT_MTIME, EROFS_FEATURE_COMPAT_SB_CHKSUM, EROFS_FEATURE_INCOMPAT_48BIT, + EROFS_FEATURE_COMPAT_MTIME, EROFS_FEATURE_COMPAT_NYDUS_NO_XATTR, + EROFS_FEATURE_COMPAT_SB_CHKSUM, EROFS_FEATURE_INCOMPAT_48BIT, EROFS_FEATURE_INCOMPAT_CHUNKED_FILE, EROFS_FEATURE_INCOMPAT_DEVICE_TABLE, EROFS_SB_BASE_SIZE, EROFS_SUPER_OFFSET, }; @@ -23,6 +25,9 @@ use nydus_format::erofs::{ /// regions never overlap. For images with up to 23 device slots the table fits /// in block 0 and `meta_blkaddr` stays 1, matching the previous layout. #[allow(clippy::too_many_arguments)] +/// Streaming variant of [`fill_image_head`], kept for tests: writes the head +/// region followed by the block-padded metadata area. +#[cfg(test)] pub(crate) fn write_image( image: &mut impl Write, metadata_buf: &[u8], @@ -31,14 +36,60 @@ pub(crate) fn write_image( epoch: u64, device_slots: &[ErofsDeviceSlot], uuid: &[u8; 16], + has_xattrs: bool, ) -> Result<()> { let block_size = EROFS_BLOCK_SIZE as usize; let meta_blkaddr = device_table_meta_blkaddr(device_slots.len())?; let head_size = meta_blkaddr as usize * block_size; - let meta_blocks = metadata_buf.len().div_ceil(block_size); + + let mut head = vec![0u8; head_size]; + fill_image_head( + &mut head, + metadata_buf.len(), + root_nid, + total_inodes, + epoch, + device_slots, + uuid, + has_xattrs, + )?; + image.write_all(&head)?; + + // --- Metadata blocks --- + image.write_all(metadata_buf)?; + + let remainder = metadata_buf.len() % block_size; + if remainder != 0 { + let pad = vec![0u8; block_size - remainder]; + image.write_all(&pad)?; + } + + Ok(()) +} + +/// Fill the image head region (superblock, device table, checksum) in place +/// at the start of `image_buf`, which must hold at least the head region. +/// `metadata_len` is the size of the metadata area that follows the head. +#[allow(clippy::too_many_arguments)] +pub(crate) fn fill_image_head( + image_buf: &mut [u8], + metadata_len: usize, + root_nid: u16, + total_inodes: u64, + epoch: u64, + device_slots: &[ErofsDeviceSlot], + uuid: &[u8; 16], + has_xattrs: bool, +) -> Result<()> { + let block_size = EROFS_BLOCK_SIZE as usize; + let meta_blkaddr = device_table_meta_blkaddr(device_slots.len())?; + let meta_blocks = metadata_len.div_ceil(block_size); let total_blocks = meta_blkaddr as u64 + meta_blocks as u64; - let feature_compat = EROFS_FEATURE_COMPAT_MTIME | EROFS_FEATURE_COMPAT_SB_CHKSUM; + let mut feature_compat = EROFS_FEATURE_COMPAT_MTIME | EROFS_FEATURE_COMPAT_SB_CHKSUM; + if !has_xattrs { + feature_compat |= EROFS_FEATURE_COMPAT_NYDUS_NO_XATTR; + } let mut feature_incompat = EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | EROFS_FEATURE_INCOMPAT_DEVICE_TABLE; // The `*_hi` halves of chunk index and device slot addresses are only @@ -64,9 +115,6 @@ pub(crate) fn write_image( (EROFS_SUPER_OFFSET as usize + EROFS_SB_BASE_SIZE) as u16 / EROFS_DEVICESLOT_SIZE as u16 }; - // --- Head region (block 0 .. meta_blkaddr) --- - let mut head = vec![0u8; head_size]; - let sb = ErofsSuperblock::new( feature_compat, feature_incompat, @@ -80,11 +128,11 @@ pub(crate) fn write_image( uuid, ); let sb_offset = EROFS_SUPER_OFFSET as usize; - head[sb_offset..sb_offset + EROFS_SB_BASE_SIZE].copy_from_slice(sb.as_bytes()); + image_buf[sb_offset..sb_offset + EROFS_SB_BASE_SIZE].copy_from_slice(sb.as_bytes()); let devslot_offset = sb_offset + EROFS_SB_BASE_SIZE; let device_table_end = devslot_offset + device_slots.len() * EROFS_DEVICESLOT_SIZE; - if device_table_end > head.len() { + if device_table_end > meta_blkaddr as usize * block_size { return Err(Error::InvalidImage( "device table does not fit in the reserved metadata head region".to_string(), )); @@ -93,23 +141,10 @@ pub(crate) fn write_image( for (index, devslot) in device_slots.iter().enumerate() { let start = devslot_offset + index * EROFS_DEVICESLOT_SIZE; let end = start + EROFS_DEVICESLOT_SIZE; - head[start..end].copy_from_slice(devslot.as_bytes()); + image_buf[start..end].copy_from_slice(devslot.as_bytes()); } - write_erofs_superblock_checksum(&mut head)?; - - image.write_all(&head)?; - - // --- Metadata blocks --- - image.write_all(metadata_buf)?; - - let remainder = metadata_buf.len() % block_size; - if remainder != 0 { - let pad = vec![0u8; block_size - remainder]; - image.write_all(&pad)?; - } - - Ok(()) + write_erofs_superblock_checksum(image_buf) } /// Compute the block address at which the inode metadata region starts. @@ -166,7 +201,7 @@ mod tests { #[test] fn write_image_sets_erofs_superblock_checksum() { let mut image = Vec::new(); - write_image(&mut image, &[], 0, 1, 0, &[], &[0u8; 16]).unwrap(); + write_image(&mut image, &[], 0, 1, 0, &[], &[0u8; 16], false).unwrap(); let sb_offset = EROFS_SUPER_OFFSET as usize; let feature_compat = @@ -202,7 +237,17 @@ mod tests { .collect(); let mut image = Vec::new(); - write_image(&mut image, &[0u8; 64], 0, 1, 0, &device_slots, &[0u8; 16]).unwrap(); + write_image( + &mut image, + &[0u8; 64], + 0, + 1, + 0, + &device_slots, + &[0u8; 16], + false, + ) + .unwrap(); let sb_offset = EROFS_SUPER_OFFSET as usize; let meta_blkaddr = diff --git a/nydus/src/build/inode.rs b/nydus/src/build/inode.rs index 1f5e685dc12..e3e59510297 100644 --- a/nydus/src/build/inode.rs +++ b/nydus/src/build/inode.rs @@ -278,20 +278,23 @@ pub(crate) trait TreeNode: Sized { /// `(dev, ino)`. type LinkKey: Copy + Eq + std::hash::Hash; - /// Common inode attributes. - fn attrs(&self) -> NodeAttrs; + /// Common inode attributes. Takes `&mut self` so owned fields (xattrs) + /// can be moved out instead of cloned; called exactly once per node. + /// Fallible because lazily-expanded sources read them from disk. + fn attrs(&mut self) -> Result; /// Hardlink-group key; `Some` only for non-directories that may share - /// their inode with other links. - fn link_key(&self) -> Option; + /// their inode with other links. Fallible for lazily-expanded sources. + fn link_key(&mut self) -> Result>; /// `Some(children)` sorted by name when the node is a directory, `None` - /// otherwise. - fn children(&self, ctx: &mut C) -> Result>>; + /// otherwise. Owned children are moved out so each subtree can be freed + /// as soon as it has been flattened. + fn children(&mut self, ctx: &mut C) -> Result>>; /// Type-specific data for a non-directory node; called exactly once per /// inode (regular-file contents are chunked into the blob here). - fn leaf_data(&self, ctx: &mut C) -> Result; + fn leaf_data(&mut self, ctx: &mut C) -> Result; } /// Flatten a source tree into one [`InodeInfo`] per filesystem object, as a @@ -309,28 +312,30 @@ pub(crate) fn flatten_tree>(root: N, ctx: &mut C) -> Result>( - node: &N, + mut node: N, ctx: &mut C, inodes: &mut Vec, ino_counter: &mut u32, hardlink_map: &mut HashMap, ) -> Result { - let link_key = node.link_key(); + let link_key = node.link_key()?; if let Some(key) = link_key { if let Some(existing_index) = hardlink_map.get(&key) { return Ok(*existing_index); } } - let attrs = node.attrs(); + let attrs = node.attrs()?; *ino_counter += 1; let ino = *ino_counter; let inode_index = inodes.len(); @@ -362,7 +367,7 @@ fn flatten_tree_node>( let mut child_entries = Vec::with_capacity(children.len()); let mut subdir_count = 0u32; for (name, child) in children { - let child_index = flatten_tree_node(&child, ctx, inodes, ino_counter, hardlink_map)?; + let child_index = flatten_tree_node(child, ctx, inodes, ino_counter, hardlink_map)?; let file_type = mode_to_erofs_file_type(inodes[child_index].mode); if file_type == EROFS_FT_DIR { subdir_count += 1; @@ -450,8 +455,8 @@ impl FsTreeNode { impl<'a, W: Write> TreeNode> for FsTreeNode { type LinkKey = (u64, u64); - fn attrs(&self) -> NodeAttrs { - NodeAttrs { + fn attrs(&mut self) -> Result { + Ok(NodeAttrs { mode: self.meta.mode() as u16, uid: self.meta.uid(), gid: self.meta.gid(), @@ -460,15 +465,15 @@ impl<'a, W: Write> TreeNode> for FsTreeNode { mtime_nsec: self.meta.mtime_nsec() as u32, nlink: self.meta.nlink() as u32, xattrs: read_xattrs_from_path(&self.path), - } + }) } - fn link_key(&self) -> Option<(u64, u64)> { - (!self.meta.file_type().is_dir() && self.meta.nlink() > 1) - .then(|| (self.meta.dev(), self.meta.ino())) + fn link_key(&mut self) -> Result> { + Ok((!self.meta.file_type().is_dir() && self.meta.nlink() > 1) + .then(|| (self.meta.dev(), self.meta.ino()))) } - fn children(&self, ctx: &mut FsBuildContext<'a, W>) -> Result>> { + fn children(&mut self, ctx: &mut FsBuildContext<'a, W>) -> Result>> { if !self.meta.file_type().is_dir() { return Ok(None); } @@ -493,7 +498,7 @@ impl<'a, W: Write> TreeNode> for FsTreeNode { Ok(Some(children)) } - fn leaf_data(&self, ctx: &mut FsBuildContext<'a, W>) -> Result { + fn leaf_data(&mut self, ctx: &mut FsBuildContext<'a, W>) -> Result { let ft = self.meta.file_type(); if ft.is_file() { let chunk_index_entries = ctx diff --git a/nydus/src/build/layout.rs b/nydus/src/build/layout.rs index bda17f8a1f1..e988dd00be8 100644 --- a/nydus/src/build/layout.rs +++ b/nydus/src/build/layout.rs @@ -3,8 +3,10 @@ use nydus_format::utils::align_up_usize; /// Metadata layout allocator. /// -/// Manages a contiguous byte buffer representing the metadata area -/// starting at `meta_blkaddr` (block 1 = byte 4096 in the image). +/// Manages a contiguous byte buffer holding the image head region (blocks +/// `0..meta_blkaddr`, filled in later) followed by the metadata area, so the +/// finished buffer IS the bootstrap and no assembly copy is needed. All +/// offsets exposed to callers are metadata-relative. /// /// Two-phase usage: /// 1. Allocate inode slots with `alloc_inode()` — returns (offset, nid). @@ -12,14 +14,20 @@ use nydus_format::utils::align_up_usize; /// 3. Allocate directory data blocks with `alloc_dir_data()`. /// 4. Write serialized data at the reserved offsets with `write_at()`. pub(crate) struct MetadataLayout { - /// The metadata byte buffer. + /// Head region bytes followed by the metadata byte buffer. buf: Vec, - /// Current allocation cursor. + /// Size of the head region at the start of `buf`. + head_size: usize, + + /// Current allocation cursor, relative to the metadata area. cursor: usize, /// Starting block address of the metadata area in the image. meta_blkaddr: u32, + + /// When set, the buffer is never grown: only offsets are tracked. + size_only: bool, } impl Default for MetadataLayout { @@ -37,16 +45,47 @@ impl MetadataLayout { /// default block 1, for images whose device table pushes the metadata /// region past block 0. pub(crate) fn with_meta_blkaddr(meta_blkaddr: u32) -> Self { + let head_size = meta_blkaddr as usize * EROFS_BLOCK_SIZE as usize; + Self { + buf: vec![0u8; head_size], + head_size, + cursor: 0, + meta_blkaddr, + size_only: false, + } + } + + /// A layout that only tracks offsets and never materialises the buffer, + /// for the sizing pass of streaming rendering: allocations return the + /// same offsets/nids as a real layout but cost no memory. + pub(crate) fn size_only(meta_blkaddr: u32) -> Self { Self { buf: Vec::new(), + head_size: meta_blkaddr as usize * EROFS_BLOCK_SIZE as usize, cursor: 0, meta_blkaddr, + size_only: true, } } /// The serialized metadata area. pub(crate) fn buf(&self) -> &[u8] { - &self.buf + &self.buf[self.head_size..] + } + + /// Reserve capacity up front. Untouched capacity costs no resident + /// memory, while growing a tens-of-MiB buffer by doubling pays a full + /// copy at every realloc — a transient RSS spike of the buffer size. + pub(crate) fn reserve(&mut self, additional: usize) { + self.buf.reserve(additional); + } + + /// Consume the layout and return the full image buffer: the zeroed head + /// region followed by the block-padded metadata area. The head is filled + /// in by the caller; no copy of the metadata is made. + pub(crate) fn into_image_buf(mut self) -> Vec { + self.pad_to_block(); + self.buf } /// Allocate space for an inode. Returns `(offset_in_buf, nid)`. @@ -64,8 +103,8 @@ impl MetadataLayout { let aligned = align_up_usize(size, EROFS_SLOTSIZE as usize).expect("alignment overflowed"); let offset = self.cursor; self.cursor += aligned; - if self.buf.len() < self.cursor { - self.buf.resize(self.cursor, 0); + if !self.size_only && self.buf.len() < self.head_size + self.cursor { + self.buf.resize(self.head_size + self.cursor, 0); } let nid = (offset / EROFS_SLOTSIZE as usize) as u64; @@ -77,8 +116,8 @@ impl MetadataLayout { let aligned = align_up_usize(self.cursor, EROFS_BLOCK_SIZE as usize).expect("alignment overflowed"); self.cursor = aligned; - if self.buf.len() < self.cursor { - self.buf.resize(self.cursor, 0); + if !self.size_only && self.buf.len() < self.head_size + self.cursor { + self.buf.resize(self.head_size + self.cursor, 0); } self.cursor @@ -93,8 +132,8 @@ impl MetadataLayout { let aligned_size = align_up_usize(size, EROFS_BLOCK_SIZE as usize).expect("alignment overflowed"); self.cursor += aligned_size; - if self.buf.len() < self.cursor { - self.buf.resize(self.cursor, 0); + if !self.size_only && self.buf.len() < self.head_size + self.cursor { + self.buf.resize(self.head_size + self.cursor, 0); } let startblk = self.meta_blkaddr as u64 + (offset / EROFS_BLOCK_SIZE as usize) as u64; @@ -103,7 +142,8 @@ impl MetadataLayout { /// Write data at a previously allocated offset. pub(crate) fn write_at(&mut self, offset: usize, data: &[u8]) { - self.buf[offset..offset + data.len()].copy_from_slice(data); + let start = self.head_size + offset; + self.buf[start..start + data.len()].copy_from_slice(data); } } diff --git a/nydus/src/build/merge.rs b/nydus/src/build/merge.rs index cf42dbe06d8..e11f80457f1 100644 --- a/nydus/src/build/merge.rs +++ b/nydus/src/build/merge.rs @@ -2,10 +2,11 @@ use std::collections::{BTreeMap, HashMap}; use std::fs; use std::path::{Path, PathBuf}; -use crate::build::bootstrap::render_flattened_bootstrap; +use crate::build::bootstrap::{render_flattened_bootstrap, render_flattened_bootstrap_to}; use crate::build::inode::{ flatten_tree, set_root_prefetch_blobs_xattr, InodeData, NamedChildren, NodeAttrs, TreeNode, }; +use nydus_core::reader::RawDirEntry; use nydus_core::ErofsReader; use nydus_error::{Context, Error, Result}; use nydus_format::erofs::{ @@ -19,86 +20,337 @@ use nydus_format::utils::parse_sha256_hex; const OCI_WHITEOUT_PREFIX: &[u8] = b".wh."; const OCI_OPAQUE_MARKER: &[u8] = b".wh..wh..opq"; +/// Return freed glibc heap pages to the OS. The consumed merge tree leaves +/// ~100 MiB of freed small allocations that glibc keeps in its arenas; the +/// buffers allocated afterwards (inode table growth, render buffer) are +/// large mmap'd blocks that cannot reuse them, so without trimming the peak +/// RSS stacks both. +fn release_freed_heap() { + // malloc_trim is glibc-only; musl has no equivalent (and no arena bloat). + #[cfg(all(target_os = "linux", target_env = "gnu"))] + unsafe { + libc::malloc_trim(0); + } +} + #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum WhiteoutSpec { Oci, } -#[derive(Clone)] -struct MergeNode { - link_id: Option, - mode: u16, - uid: u32, - gid: u32, - size: u64, - mtime: u64, - mtime_nsec: u32, - nlink: u32, - xattrs: Vec, - data: MergeNodeData, -} - +/// Identifies a hardlink group across layers: the inode's home layer and nid. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] struct MergeLinkId { layer_id: u32, nid: u64, } -#[derive(Clone)] -enum MergeNodeData { - RegularFile { - chunk_index_entries: Vec, - chunk_size_bits: u32, - }, - Directory { - children: BTreeMap, MergeNode>, - }, - Symlink { - target: Vec, - }, - SpecialDev { - rdev: u32, - }, - SpecialNoData, +/// One source layer participating in the k-way merge: its metadata reader +/// and the mapping from its local blob indexes to the merged device table. +struct MergeLayer { + layer_id: u32, + reader: ErofsReader, + epoch: u64, + fixed_nsec: u32, + local_to_global: HashMap, +} + +/// The (layer, nid) variants of one merged path, in lower..upper order. +/// Directories keep every stacked directory variant so their children merge; +/// for any other kind only the topmost variant exists (upper shadows lower). +struct KWayVariants { + /// Indexes into the layer slice paired with the nid in that layer. + variants: Vec<(usize, u64)>, + is_dir: bool, +} + +/// A lazily expanded node of the merged tree: children are produced by +/// k-way merging the variant directories' entries on demand, so no merged +/// tree is ever materialised — peak memory is one directory's entry list +/// plus the DFS path. +struct KWayNode<'a> { + layers: &'a [MergeLayer], + whiteout_spec: WhiteoutSpec, + variants: KWayVariants, +} + +impl KWayVariants { + fn top(&self) -> (usize, u64) { + *self + .variants + .last() + .expect("a merged path always has at least one variant") + } +} + +impl<'a> KWayNode<'a> { + fn top_layer_and_inode(&self) -> Result<(&'a MergeLayer, u64)> { + let (layer_index, nid) = self.variants.top(); + Ok((&self.layers[layer_index], nid)) + } +} + +impl TreeNode<()> for KWayNode<'_> { + type LinkKey = MergeLinkId; + + fn attrs(&mut self) -> Result { + let (layer, nid) = self.top_layer_and_inode()?; + let inode = layer + .reader + .inode(nid) + .with_context(|| format!("failed to read inode: {nid}"))?; + let mut xattrs: Vec = layer + .reader + .read_xattrs(nid, &inode)? + .into_iter() + .filter_map(|(name, value)| { + erofs_xattr_name_split(&name).map(|(index, suffix)| XattrEntry { + name_index: index, + suffix: suffix.to_vec(), + value, + }) + }) + .collect(); + xattrs.sort_by(|a, b| (a.name_index, &a.suffix).cmp(&(b.name_index, &b.suffix))); + Ok(NodeAttrs { + mode: inode.mode(), + uid: inode.uid(), + gid: inode.gid(), + size: inode.size(), + mtime: inode.mtime(layer.epoch), + mtime_nsec: inode.effective_mtime_nsec(layer.fixed_nsec), + nlink: inode.nlink(), + xattrs, + }) + } + + fn link_key(&mut self) -> Result> { + if self.variants.is_dir { + return Ok(None); + } + let (layer, nid) = self.top_layer_and_inode()?; + let inode = layer + .reader + .inode(nid) + .with_context(|| format!("failed to read inode: {nid}"))?; + Ok( + (mode_to_erofs_file_type(inode.mode()) == EROFS_FT_REG_FILE && inode.nlink() > 1) + .then_some(MergeLinkId { + layer_id: layer.layer_id, + nid, + }), + ) + } + + fn children(&mut self, _ctx: &mut ()) -> Result>> { + if !self.variants.is_dir { + return Ok(None); + } + let mut merged: BTreeMap, KWayVariants> = BTreeMap::new(); + for &(layer_index, nid) in &self.variants.variants { + let layer = &self.layers[layer_index]; + let inode = layer + .reader + .inode(nid) + .with_context(|| format!("failed to read inode: {nid}"))?; + let entries = layer.reader.read_dir(nid, &inode)?; + merge_layer_entries(&mut merged, entries, layer_index, self.whiteout_spec); + } + Ok(Some( + merged + .into_iter() + .map(|(name, variants)| { + ( + name, + KWayNode { + layers: self.layers, + whiteout_spec: self.whiteout_spec, + variants, + }, + ) + }) + .collect(), + )) + } + + fn leaf_data(&mut self, _ctx: &mut ()) -> Result { + let (layer, nid) = self.top_layer_and_inode()?; + let inode = layer + .reader + .inode(nid) + .with_context(|| format!("failed to read inode: {nid}"))?; + Ok(match mode_to_erofs_file_type(inode.mode()) { + EROFS_FT_REG_FILE => { + if inode.data_layout() != EROFS_INODE_CHUNK_BASED { + return Err(Error::Unsupported( + "merge currently only supports chunk-based regular files".to_string(), + )); + } + let chunk_size_bits = layer.reader.chunk_bits(&inode); + let chunk_index_entries = layer + .reader + .read_chunk_index_entries(nid, &inode)? + .into_iter() + .map(|index| { + // A hole chunk carries no blob reference at all (its + // on-disk device_id bits are part of the null sentinel), + // so it passes through unchanged instead of being device + // remapped. + if index.blkaddr == EROFS_NULL_ADDR || index.device_id == 0 { + Ok(index) + } else { + let mapped = layer + .local_to_global + .get(&index.device_id) + .copied() + .ok_or_else(|| { + Error::InvalidImage(format!( + "missing global blob index mapping for source blob {}", + index.device_id + )) + })?; + Ok(ErofsChunkAddr { + blkaddr: index.blkaddr, + device_id: mapped, + }) + } + }) + .collect::>>()?; + InodeData::RegularFile { + chunk_index_entries, + chunk_size_bits, + } + } + EROFS_FT_SYMLINK => InodeData::Symlink { + target: layer.reader.read_symlink(nid, &inode)?, + startblk: 0, + }, + EROFS_FT_CHRDEV | EROFS_FT_BLKDEV => InodeData::Device { rdev: inode.rdev() }, + EROFS_FT_FIFO | EROFS_FT_SOCK => InodeData::FifoOrSocket, + other => { + return Err(Error::Unsupported(format!( + "unsupported inode file type {other} while loading layer" + ))) + } + }) + } +} + +/// Merge one layer's directory entries (upper) into the accumulated view, +/// applying whiteout semantics: an opaque marker discards everything below, +/// `.wh.` removes `` from below, upper non-directories shadow +/// whatever is below, and stacked directories merge. Whiteout entries +/// themselves never appear in the result. +fn merge_layer_entries( + merged: &mut BTreeMap, KWayVariants>, + entries: Vec, + layer_index: usize, + whiteout_spec: WhiteoutSpec, +) { + if entries + .iter() + .any(|entry| is_opaque_marker(&entry.name, whiteout_spec)) + { + merged.clear(); + } + for entry in &entries { + if let Some(target) = whiteout_target(&entry.name, whiteout_spec) { + merged.remove(target); + } + } + for entry in entries { + if entry.name == b"." + || entry.name == b".." + || is_opaque_marker(&entry.name, whiteout_spec) + || whiteout_target(&entry.name, whiteout_spec).is_some() + { + continue; + } + let is_dir = entry.file_type == EROFS_FT_DIR; + match merged.get_mut(&entry.name) { + Some(existing) if is_dir && existing.is_dir => { + existing.variants.push((layer_index, entry.nid)); + } + _ => { + merged.insert( + entry.name, + KWayVariants { + variants: vec![(layer_index, entry.nid)], + is_dir, + }, + ); + } + } + } } pub fn merge_sources_to_bootstrap_bytes( sources: &[PathBuf], whiteout_spec: WhiteoutSpec, ) -> Result> { + let mut bootstrap = Vec::new(); + merge_sources_to_bootstrap_writer(sources, whiteout_spec, &mut bootstrap)?; + Ok(bootstrap) +} + +/// Merge the sources and stream the flattened bootstrap into `writer`. The +/// merged tree is never materialised (children are k-way merged on demand +/// during flattening) and the bootstrap is stream-rendered, so peak memory +/// is the flat inode table plus one directory's entries. +pub fn merge_sources_to_bootstrap_writer( + sources: &[PathBuf], + whiteout_spec: WhiteoutSpec, + writer: &mut impl std::io::Write, +) -> Result<()> { if sources.is_empty() { return Err(Error::InvalidParameter( "merge requires at least one source".to_string(), )); } - let mut merged_root: Option = None; let mut device_slots = Vec::new(); let mut blob_indexes = HashMap::new(); + let mut layers = Vec::with_capacity(sources.len()); for (layer_id, source) in sources.iter().enumerate() { let source_blob_id = parse_source_blob_id(source) .with_context(|| format!("invalid merge source: {}", source.display()))?; - let layer = load_layer( - layer_id as u32, - source, + let reader = ErofsReader::open_metadata_only(source) + .with_context(|| format!("failed to load layer: {}", source.display()))?; + validate_single_layer_blob_source(source, &reader)?; + let local_to_global = register_blobs( + &reader, source_blob_id, &mut device_slots, &mut blob_indexes, - ) - .with_context(|| format!("failed to load layer: {}", source.display()))?; - merged_root = Some(match merged_root { - Some(existing) => overlay_nodes(existing, layer, whiteout_spec)?, - None => layer, + )?; + layers.push(MergeLayer { + layer_id: layer_id as u32, + epoch: reader.superblock().epoch(), + fixed_nsec: reader.superblock().fixed_nsec(), + local_to_global, + reader, }); } - let mut merged_root = merged_root - .ok_or_else(|| Error::InvalidImage("merge produced no root node".to_string()))?; - strip_whiteout_entries(&mut merged_root, whiteout_spec); + let root = KWayNode { + layers: &layers, + whiteout_spec, + variants: KWayVariants { + variants: layers + .iter() + .enumerate() + .map(|(index, layer)| (index, layer.reader.superblock().root_nid())) + .collect(), + is_dir: true, + }, + }; - // `flatten_tree` always yields at least the root inode. - let mut inodes = flatten_tree(&merged_root, &mut ())?; + // `flatten_tree` always yields at least the root inode; children are + // k-way merged on demand while flattening. + let mut inodes = flatten_tree(root, &mut ())?; + drop(layers); + release_freed_heap(); // `build_tree` zeroes the root mtime for reproducibility, so the minimum // inode mtime read back from any layer is always 0. @@ -109,7 +361,8 @@ pub fn merge_sources_to_bootstrap_bytes( let prefetch_blob_indexes = (1..=blob_count).collect::>(); set_root_prefetch_blobs_xattr(&mut inodes[0], &prefetch_blob_indexes)?; - render_flattened_bootstrap(&mut inodes, epoch, &device_slots, &uuid) + render_flattened_bootstrap_to(writer, &mut inodes, epoch, &device_slots, &uuid)?; + Ok(()) } /// Rewrite an existing merged bootstrap for the `optimize` flow: append an @@ -123,7 +376,7 @@ pub(crate) fn rewrite_bootstrap_with_ondemand_blob( ) -> Result> { let reader = ErofsReader::open_metadata_only(parent_bootstrap) .with_context(|| format!("failed to open bootstrap: {}", parent_bootstrap.display()))?; - let blob_infos = reader.blob_infos()?; + let blob_infos = reader.blob_infos()?.to_vec(); if blob_infos.is_empty() { return Err(Error::InvalidImage( "parent bootstrap contains no blobs".to_string(), @@ -143,22 +396,33 @@ pub(crate) fn rewrite_bootstrap_with_ondemand_blob( .iter() .map(|info| (info.blob_index, info.blob_index)) .collect(); - let root = load_node( - &reader, - 0, - reader.superblock().root_nid(), - reader.superblock().epoch(), - &identity, - ) - .with_context(|| { + let layers = [MergeLayer { + layer_id: 0, + epoch: reader.superblock().epoch(), + fixed_nsec: reader.superblock().fixed_nsec(), + local_to_global: identity, + reader, + }]; + let root = KWayNode { + layers: &layers, + // A merged bootstrap carries no whiteout entries; the spec is inert. + whiteout_spec: WhiteoutSpec::Oci, + variants: KWayVariants { + variants: vec![(0, layers[0].reader.superblock().root_nid())], + is_dir: true, + }, + }; + + // `flatten_tree` always yields at least the root inode; children are + // expanded lazily from the bootstrap. + let mut inodes = flatten_tree(root, &mut ()).with_context(|| { format!( "failed to load bootstrap inode tree: {}", parent_bootstrap.display() ) })?; - - // `flatten_tree` always yields at least the root inode. - let mut inodes = flatten_tree(&root, &mut ())?; + let [MergeLayer { reader, .. }] = layers; + release_freed_heap(); let mut device_slots: Vec = blob_infos .iter() @@ -191,26 +455,6 @@ pub(crate) fn rewrite_bootstrap_with_ondemand_blob( render_flattened_bootstrap(&mut inodes, epoch, &device_slots, &uuid) } -fn load_layer( - layer_id: u32, - source: &Path, - source_blob_id: [u8; EROFS_BLOB_ID_SIZE], - device_slots: &mut Vec, - blob_indexes: &mut HashMap<[u8; EROFS_BLOB_ID_SIZE], u16>, -) -> Result { - let reader = ErofsReader::open_metadata_only(source)?; - validate_single_layer_blob_source(source, &reader)?; - let layer_epoch = reader.superblock().epoch(); - let local_to_global = register_blobs(&reader, source_blob_id, device_slots, blob_indexes)?; - load_node( - &reader, - layer_id, - reader.superblock().root_nid(), - layer_epoch, - &local_to_global, - ) -} - fn register_blobs( reader: &ErofsReader, source_blob_id: [u8; EROFS_BLOB_ID_SIZE], @@ -274,186 +518,6 @@ fn validate_single_layer_blob_source(path: &Path, reader: &ErofsReader) -> Resul Ok(()) } -fn load_node( - reader: &ErofsReader, - layer_id: u32, - nid: u64, - epoch: u64, - local_to_global: &HashMap, -) -> Result { - let inode = reader - .inode(nid) - .with_context(|| format!("failed to read inode: {nid}"))?; - let mode = inode.mode(); - let mut xattrs: Vec = reader - .read_xattrs(nid, &inode)? - .into_iter() - .filter_map(|(name, value)| { - erofs_xattr_name_split(&name).map(|(index, suffix)| XattrEntry { - name_index: index, - suffix: suffix.to_vec(), - value, - }) - }) - .collect(); - xattrs.sort_by(|a, b| (a.name_index, &a.suffix).cmp(&(b.name_index, &b.suffix))); - - let data = - match mode_to_erofs_file_type(mode) { - EROFS_FT_DIR => { - let mut children = BTreeMap::new(); - for entry in reader.read_dir(nid, &inode)? { - if entry.name == b"." || entry.name == b".." { - continue; - } - children.insert( - entry.name.clone(), - load_node(reader, layer_id, entry.nid, epoch, local_to_global) - .with_context(|| { - format!( - "failed to load child: {}", - String::from_utf8_lossy(&entry.name) - ) - })?, - ); - } - MergeNodeData::Directory { children } - } - EROFS_FT_REG_FILE => { - if inode.data_layout() != EROFS_INODE_CHUNK_BASED { - return Err(Error::Unsupported( - "merge currently only supports chunk-based regular files".to_string(), - )); - } - let chunk_size_bits = reader.chunk_bits(&inode); - let chunk_index_entries = reader - .read_chunk_index_entries(nid, &inode)? - .into_iter() - .map(|index| { - // A hole chunk carries no blob reference at all (its - // on-disk device_id bits are part of the null sentinel), - // so it passes through unchanged instead of being device - // remapped. - if index.blkaddr == EROFS_NULL_ADDR || index.device_id == 0 { - Ok(index) - } else { - let mapped = local_to_global - .get(&index.device_id) - .copied() - .ok_or_else(|| { - Error::InvalidImage(format!( - "missing global blob index mapping for source blob {}", - index.device_id - )) - })?; - Ok(ErofsChunkAddr { - blkaddr: index.blkaddr, - device_id: mapped, - }) - } - }) - .collect::>>()?; - MergeNodeData::RegularFile { - chunk_index_entries, - chunk_size_bits, - } - } - EROFS_FT_SYMLINK => MergeNodeData::Symlink { - target: reader.read_symlink(nid, &inode)?, - }, - EROFS_FT_CHRDEV | EROFS_FT_BLKDEV => MergeNodeData::SpecialDev { rdev: inode.rdev() }, - EROFS_FT_FIFO | EROFS_FT_SOCK => MergeNodeData::SpecialNoData, - other => { - return Err(Error::Unsupported(format!( - "unsupported inode file type {other} while loading layer" - ))) - } - }; - - Ok(MergeNode { - link_id: if mode_to_erofs_file_type(mode) == EROFS_FT_REG_FILE && inode.nlink() > 1 { - Some(MergeLinkId { layer_id, nid }) - } else { - None - }, - mode, - uid: inode.uid(), - gid: inode.gid(), - size: inode.size(), - mtime: inode.mtime(epoch), - mtime_nsec: inode.effective_mtime_nsec(reader.superblock().fixed_nsec()), - nlink: inode.nlink(), - xattrs, - data, - }) -} - -fn overlay_nodes( - lower: MergeNode, - upper: MergeNode, - whiteout_spec: WhiteoutSpec, -) -> Result { - if let ( - MergeNodeData::Directory { - children: lower_children, - }, - MergeNodeData::Directory { - children: upper_children, - }, - ) = (&lower.data, &upper.data) - { - let lower_children = lower_children.clone(); - let upper_children = upper_children.clone(); - return overlay_directories(lower_children, upper, upper_children, whiteout_spec); - } - - Ok(upper) -} - -fn overlay_directories( - lower_children: BTreeMap, MergeNode>, - upper_meta: MergeNode, - upper_children: BTreeMap, MergeNode>, - whiteout_spec: WhiteoutSpec, -) -> Result { - let mut merged_children = lower_children; - let opaque = upper_children - .keys() - .any(|name| is_opaque_marker(name, whiteout_spec)); - if opaque { - merged_children.clear(); - } - - for name in upper_children.keys() { - if let Some(target) = whiteout_target(name, whiteout_spec) { - merged_children.remove(target); - } - } - - for (name, child) in upper_children { - if is_opaque_marker(&name, whiteout_spec) || whiteout_target(&name, whiteout_spec).is_some() - { - continue; - } - - match merged_children.remove(&name) { - Some(existing) => { - merged_children.insert(name, overlay_nodes(existing, child, whiteout_spec)?); - } - None => { - merged_children.insert(name, child); - } - } - } - - Ok(MergeNode { - data: MergeNodeData::Directory { - children: merged_children, - }, - ..upper_meta - }) -} - fn is_opaque_marker(name: &[u8], whiteout_spec: WhiteoutSpec) -> bool { match whiteout_spec { WhiteoutSpec::Oci => name == OCI_OPAQUE_MARKER, @@ -472,247 +536,144 @@ fn whiteout_target(name: &[u8], whiteout_spec: WhiteoutSpec) -> Option<&[u8]> { } } -fn strip_whiteout_entries(node: &mut MergeNode, whiteout_spec: WhiteoutSpec) { - let MergeNodeData::Directory { children } = &mut node.data else { - return; - }; - - children.retain(|name, _| { - !is_opaque_marker(name, whiteout_spec) && whiteout_target(name, whiteout_spec).is_none() - }); - for child in children.values_mut() { - strip_whiteout_entries(child, whiteout_spec); - } -} - -/// [`TreeNode`] over the in-memory merge tree, so [`flatten_tree`] produces -/// exactly the same inodes for a merged layer as for a directory build. -impl<'a> TreeNode<()> for &'a MergeNode { - type LinkKey = MergeLinkId; - - fn attrs(&self) -> NodeAttrs { - NodeAttrs { - mode: self.mode, - uid: self.uid, - gid: self.gid, - size: self.size, - mtime: self.mtime, - mtime_nsec: self.mtime_nsec, - nlink: self.nlink, - xattrs: self.xattrs.clone(), - } - } - - fn link_key(&self) -> Option { - self.link_id - } - - fn children(&self, _ctx: &mut ()) -> Result>> { - let node: &'a MergeNode = self; - match &node.data { - MergeNodeData::Directory { children } => Ok(Some( - children - .iter() - .map(|(name, child)| (name.clone(), child)) - .collect(), - )), - _ => Ok(None), - } - } - - fn leaf_data(&self, _ctx: &mut ()) -> Result { - Ok(match &self.data { - MergeNodeData::RegularFile { - chunk_index_entries, - chunk_size_bits, - } => InodeData::RegularFile { - chunk_index_entries: chunk_index_entries.clone(), - chunk_size_bits: *chunk_size_bits, - }, - MergeNodeData::Symlink { target } => InodeData::Symlink { - target: target.clone(), - startblk: 0, - }, - MergeNodeData::SpecialDev { rdev } => InodeData::Device { rdev: *rdev }, - MergeNodeData::SpecialNoData => InodeData::FifoOrSocket, - MergeNodeData::Directory { .. } => { - unreachable!("leaf_data is only called for non-directories") - } - }) - } -} - #[cfg(test)] mod tests { use super::*; use nydus_format::erofs::{ - needs_erofs_extended_inode, EROFS_BLKSZBITS, EROFS_XATTR_INDEX_TRUSTED, - NYDUS_XATTR_SUFFIX_PREFETCH_BLOBS, + needs_erofs_extended_inode, EROFS_XATTR_INDEX_TRUSTED, NYDUS_XATTR_SUFFIX_PREFETCH_BLOBS, }; const OPAQUE: &str = ".wh..wh..opq"; - fn directory(entries: Vec<(&str, MergeNode)>) -> MergeNode { - let children = entries - .into_iter() - .map(|(name, node)| (name.as_bytes().to_vec(), node)) - .collect(); - merge_node(MergeNodeData::Directory { children }) - } - - fn regular_file() -> MergeNode { - merge_node(MergeNodeData::RegularFile { - chunk_index_entries: Vec::new(), - chunk_size_bits: EROFS_BLKSZBITS as u32, - }) - } - - fn merge_node(data: MergeNodeData) -> MergeNode { - let mode = match data { - MergeNodeData::Directory { .. } => libc::S_IFDIR as u16 | 0o755, - MergeNodeData::RegularFile { .. } => libc::S_IFREG as u16 | 0o644, - _ => libc::S_IFREG as u16 | 0o644, - }; - MergeNode { - link_id: None, - mode, - uid: 0, - gid: 0, - size: 0, - mtime: 0, - mtime_nsec: 0, - nlink: 1, - xattrs: Vec::new(), - data, - } + fn entries(items: &[(&str, u8)]) -> Vec { + items + .iter() + .enumerate() + .map(|(i, (name, file_type))| RawDirEntry { + nid: i as u64 + 1, + file_type: *file_type, + name: name.as_bytes().to_vec(), + }) + .collect() } - fn child_names(node: &MergeNode) -> Vec { - let MergeNodeData::Directory { children } = &node.data else { - panic!("not a directory") - }; - children + fn merged_names(merged: &BTreeMap, KWayVariants>) -> Vec { + merged .keys() .map(|k| String::from_utf8_lossy(k).into_owned()) .collect() } #[test] - fn strip_whiteout_entries_removes_opaque_marker_from_inserted_directory() { - let mut root = directory(vec![( - "opt", - directory(vec![( - "yarn-v1.22.19", - directory(vec![(OPAQUE, regular_file())]), - )]), - )]); - - strip_whiteout_entries(&mut root, WhiteoutSpec::Oci); - - let MergeNodeData::Directory { children } = &root.data else { - panic!("root should be a directory") - }; - let opt = children.get(b"opt".as_slice()).unwrap(); - let MergeNodeData::Directory { children } = &opt.data else { - panic!("opt should be a directory") - }; - let yarn = children.get(b"yarn-v1.22.19".as_slice()).unwrap(); - assert!(child_names(yarn).is_empty()); - } - - #[test] - fn strip_whiteout_entries_removes_plain_whiteout_marker() { - let mut root = directory(vec![ - (".wh.removed", regular_file()), - ("kept", regular_file()), - ]); - - strip_whiteout_entries(&mut root, WhiteoutSpec::Oci); + fn whiteout_semantics_follow_the_oci_rules() { + type Layers = &'static [&'static [(&'static str, u8)]]; + let cases: [(&str, Layers, &[&str]); 4] = [ + ( + "opaque marker clears lower entries and is dropped", + &[ + &[("old.txt", EROFS_FT_REG_FILE), ("subdir", EROFS_FT_DIR)], + &[(OPAQUE, EROFS_FT_REG_FILE), ("new.txt", EROFS_FT_REG_FILE)], + ], + &["new.txt"], + ), + ( + "plain whiteout removes the lower entry and the marker", + &[ + &[("kept", EROFS_FT_REG_FILE), ("removed", EROFS_FT_REG_FILE)], + &[(".wh.removed", EROFS_FT_REG_FILE)], + ], + &["kept"], + ), + ( + "bottom layer whiteout markers are never emitted", + &[&[ + (".wh.lower-only", EROFS_FT_REG_FILE), + (OPAQUE, EROFS_FT_REG_FILE), + ("fresh", EROFS_FT_REG_FILE), + ]], + &["fresh"], + ), + ( + "lower whiteout marker does not delete a later upper entry", + &[ + &[(".wh.recreated", EROFS_FT_REG_FILE)], + &[("recreated", EROFS_FT_REG_FILE)], + ], + &["recreated"], + ), + ]; - assert_eq!(child_names(&root), vec!["kept"]); + for (case, layers, expected) in cases { + let mut merged = BTreeMap::new(); + for (layer_index, layer) in layers.iter().enumerate() { + merge_layer_entries(&mut merged, entries(layer), layer_index, WhiteoutSpec::Oci); + } + assert_eq!(merged_names(&merged), expected, "{case}"); + } } #[test] - fn overlay_opaque_directory_keeps_upper_entries_and_drops_marker() { - let lower = directory(vec![( - "opq", - directory(vec![ - ("old.txt", regular_file()), - ("subdir", directory(Vec::new())), - ]), - )]); - let upper = directory(vec![( - "opq", - directory(vec![(OPAQUE, regular_file()), ("new.txt", regular_file())]), - )]); - - let mut merged = overlay_nodes(lower, upper, WhiteoutSpec::Oci).unwrap(); - strip_whiteout_entries(&mut merged, WhiteoutSpec::Oci); - - let MergeNodeData::Directory { children } = &merged.data else { - panic!("root should be a directory") - }; - assert_eq!( - child_names(children.get(b"opq".as_slice()).unwrap()), - vec!["new.txt"] + fn upper_whiteout_does_not_delete_same_layer_dotfile() { + let mut merged = BTreeMap::new(); + merge_layer_entries( + &mut merged, + entries(&[(".dotfile", EROFS_FT_REG_FILE)]), + 0, + WhiteoutSpec::Oci, ); - } - - #[test] - fn overlay_plain_whiteout_removes_lower_entry_and_marker() { - let lower = directory(vec![("kept", regular_file()), ("removed", regular_file())]); - let upper = directory(vec![(".wh.removed", regular_file())]); - - let mut merged = overlay_nodes(lower, upper, WhiteoutSpec::Oci).unwrap(); - strip_whiteout_entries(&mut merged, WhiteoutSpec::Oci); - - assert_eq!(child_names(&merged), vec!["kept"]); - } - - #[test] - fn strip_whiteout_entries_removes_marker_inside_inserted_directory() { - let mut root = directory(vec![( - "newdir", - directory(vec![ - (".wh.lower-only", regular_file()), - ("fresh", regular_file()), + merge_layer_entries( + &mut merged, + entries(&[ + (".dotfile", EROFS_FT_REG_FILE), + (".wh..dotfile", EROFS_FT_REG_FILE), ]), - )]); - - strip_whiteout_entries(&mut root, WhiteoutSpec::Oci); - - let MergeNodeData::Directory { children } = &root.data else { - panic!("root should be a directory") - }; - assert_eq!( - child_names(children.get(b"newdir".as_slice()).unwrap()), - vec!["fresh"] + 1, + WhiteoutSpec::Oci, ); + assert_eq!(merged_names(&merged), vec![".dotfile"]); + assert_eq!(merged[b".dotfile".as_slice()].top(), (1, 1)); } #[test] - fn lower_whiteout_marker_does_not_delete_later_upper_entry() { - let lower = directory(vec![(".wh.recreated", regular_file())]); - let upper = directory(vec![("recreated", regular_file())]); - - let mut merged = overlay_nodes(lower, upper, WhiteoutSpec::Oci).unwrap(); - strip_whiteout_entries(&mut merged, WhiteoutSpec::Oci); - - assert_eq!(child_names(&merged), vec!["recreated"]); + fn directories_stack_variants_and_files_shadow() { + let mut merged = BTreeMap::new(); + merge_layer_entries( + &mut merged, + entries(&[("dir", EROFS_FT_DIR), ("file", EROFS_FT_REG_FILE)]), + 0, + WhiteoutSpec::Oci, + ); + merge_layer_entries( + &mut merged, + entries(&[("dir", EROFS_FT_DIR), ("file", EROFS_FT_REG_FILE)]), + 1, + WhiteoutSpec::Oci, + ); + let dir = &merged[b"dir".as_slice()]; + assert!(dir.is_dir); + assert_eq!(dir.variants, vec![(0, 1), (1, 1)]); + let file = &merged[b"file".as_slice()]; + assert_eq!(file.variants, vec![(1, 2)]); } #[test] - fn upper_whiteout_does_not_delete_same_layer_dotfile() { - let lower = directory(vec![(".dotfile", regular_file())]); - let upper = directory(vec![ - (".dotfile", regular_file()), - (".wh..dotfile", regular_file()), - ]); - - let mut merged = overlay_nodes(lower, upper, WhiteoutSpec::Oci).unwrap(); - strip_whiteout_entries(&mut merged, WhiteoutSpec::Oci); - - assert_eq!(child_names(&merged), vec![".dotfile"]); + fn upper_file_replaces_lower_directory() { + let mut merged = BTreeMap::new(); + merge_layer_entries( + &mut merged, + entries(&[("path", EROFS_FT_DIR)]), + 0, + WhiteoutSpec::Oci, + ); + merge_layer_entries( + &mut merged, + entries(&[("path", EROFS_FT_REG_FILE)]), + 1, + WhiteoutSpec::Oci, + ); + let node = &merged[b"path".as_slice()]; + assert!(!node.is_dir); + assert_eq!(node.variants, vec![(1, 1)]); } /// Set a path's mtime to whole seconds (no nanoseconds), without @@ -812,15 +773,33 @@ mod tests { let source_blob_id = parse_source_blob_id(&merge_source).unwrap(); let mut device_slots = Vec::new(); let mut blob_indexes = HashMap::new(); - let root = load_layer( - 0, - &merge_source, + let reader = ErofsReader::open_metadata_only(&merge_source).unwrap(); + validate_single_layer_blob_source(&merge_source, &reader).unwrap(); + let local_to_global = register_blobs( + &reader, source_blob_id, &mut device_slots, &mut blob_indexes, ) .unwrap(); - let mut merged = flatten_tree(&root, &mut ()).unwrap(); + let epoch = reader.superblock().epoch(); + let fixed_nsec = reader.superblock().fixed_nsec(); + let layers = [MergeLayer { + layer_id: 0, + epoch, + fixed_nsec, + local_to_global, + reader, + }]; + let root = KWayNode { + layers: &layers, + whiteout_spec: WhiteoutSpec::Oci, + variants: KWayVariants { + variants: vec![(0, layers[0].reader.superblock().root_nid())], + is_dir: true, + }, + }; + let mut merged = flatten_tree(root, &mut ()).unwrap(); // The layer bootstrap carries the prefetch xattr the build stamps on // its root after flattening; drop it so the roots compare equal. diff --git a/nydus/src/build/mod.rs b/nydus/src/build/mod.rs index 75b46e4c96c..98505b88d17 100644 --- a/nydus/src/build/mod.rs +++ b/nydus/src/build/mod.rs @@ -20,7 +20,7 @@ use std::path::PathBuf; use sha2::{Digest, Sha256}; use blob_chunk::BlobWriter; -use bootstrap::{render_bootstrap, render_flattened_bootstrap}; +use bootstrap::render_bootstrap; use inode::{build_tree, set_root_prefetch_blobs_xattr}; use nydus_error::{Context, Error, Result}; use nydus_format::blob::{ @@ -30,7 +30,7 @@ use nydus_format::erofs::{ErofsDeviceSlot, EROFS_BLOB_ID_SIZE, EROFS_BLOCK_SIZE} use nydus_format::utils::sha256_bytes; /// The minimum block group uncompressed size. -pub const MIN_BLOCK_GROUP_SIZE: u32 = 1024 * 1024; +pub const MIN_BLOCK_GROUP_SIZE: u32 = 512 * 1024; /// Options for [`build_image`]. #[derive(Debug)] @@ -106,7 +106,7 @@ impl BuildImageOptions { // file chunk size so a chunk always fits in a block group. if !block_group_size.is_power_of_two() || block_group_size < MIN_BLOCK_GROUP_SIZE { return Err(Error::InvalidParameter(format!( - "block group size {block_group_size} must be a power of two and at least 1MiB" + "block group size {block_group_size} must be a power of two and at least 512KiB" ))); } @@ -161,6 +161,10 @@ pub fn build_image(options: &BuildImageOptions, writer: impl Write) -> Result Result Result> { fn blob_metadata_summary_from_bytes(data: &[u8]) -> Result { let blob_metadata = BlobMetadata::from_bytes(data, false)?; Ok(BlobMetadataSummary { - chunk_count: blob_metadata.chunk_count(), block_group_count: blob_metadata.block_group_count(), chunk_size: blob_metadata.chunk_size(), - digester: blob_metadata.digester(), compressor: blob_metadata.compressor(), total_uncompressed_size: blob_metadata.uncompressed_size(), total_compressed_size: blob_metadata.compressed_end(), @@ -570,6 +566,7 @@ mod tests { 0, data.len() as u64, 1, + None, ) .unwrap(); let mut blob = Vec::new(); diff --git a/nydus/src/fanotify/mount.rs b/nydus/src/fanotify/mount.rs index e5f6b46b485..d8a1991d05e 100644 --- a/nydus/src/fanotify/mount.rs +++ b/nydus/src/fanotify/mount.rs @@ -55,6 +55,11 @@ pub fn mount_erofs(bootstrap: &Path, devices: &[BlobDevice], mountpoint: &Path) /// Build the binary mount data without lossy UTF-8 conversion. A comma in a /// device path is rejected because the kernel option grammar cannot distinguish /// it from the next mount option. +/// +/// `directio` is deliberately not passed. Measured on this corpus it cut cold +/// random reads by 72% and cold sequential reads by half, while saving almost +/// nothing: the backing files' page cache is what EROFS resolves metadata +/// from and what absorbs sub-block reads, so it is not a redundant copy here. fn mount_options(devices: &[BlobDevice]) -> Result> { let mut options = b"ro".to_vec(); let mut expected_index = 1u16; diff --git a/nydus/src/fileio/fs.rs b/nydus/src/fileio/fs.rs new file mode 100644 index 00000000000..f2dab131fc4 --- /dev/null +++ b/nydus/src/fileio/fs.rs @@ -0,0 +1,277 @@ +// Copyright (C) 2026 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +//! Single-file FUSE export of the flattened image. +//! +//! The mount holds exactly one regular file whose contents are the flattened +//! device view built by [`FlatImage`]: the bootstrap at the head, then each +//! blob at its mapped offset. The kernel EROFS driver mounts that file +//! directly (`CONFIG_EROFS_FS_BACKED_BY_FILE`), so every filesystem +//! operation — lookup, readdir, stat, xattr — is resolved in-kernel against +//! metadata bytes that live in this file's page cache. Only cold byte ranges +//! come back here as FUSE reads. +//! +//! Directory contents are fixed at mount time, so there is no inode table: +//! the root is [`ROOT_INO`] and the image file is [`IMAGE_INO`]. + +use std::ffi::OsStr; +use std::io; +use std::os::unix::ffi::OsStrExt; +use std::sync::Arc; +use std::time::{Duration, UNIX_EPOCH}; + +use fuser::{ + Errno, FileAttr, FileHandle, FileType, Filesystem, FopenFlags, Generation, INodeNo, + KernelConfig, LockOwner, OpenFlags, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, + ReplyOpen, ReplyStatfs, ReplyXattr, Request, +}; +use nydus_core::flat::{FlatImage, BLOCK_SIZE}; +use nydus_format::erofs::EROFS_BLOCK_SIZE; + +/// Inode of the root directory, fixed by the FUSE protocol. +pub const ROOT_INO: u64 = 1; +/// Inode of the exported flattened image file. +pub const IMAGE_INO: u64 = 2; +/// Name of the exported flattened image file inside the mount. +pub const IMAGE_NAME: &str = "image"; + +/// The export is immutable for the mount's lifetime, so the kernel can cache +/// attributes and dentries indefinitely. +const ENTRY_TIMEOUT: Duration = Duration::from_secs(86400 * 365 * 10); + +/// Single-file view of the flattened image. +pub struct FlatImageFs { + image: Arc, +} + +impl FlatImageFs { + /// Export `image`'s flattened view as one file named [`IMAGE_NAME`]. + pub fn new(image: Arc) -> Self { + Self { image } + } + + /// Size in bytes of the exported image file. + pub fn image_size(&self) -> u64 { + self.image.size() + } + + fn attr(&self, ino: u64) -> Option { + let (kind, perm, size, nlink) = match ino { + ROOT_INO => (FileType::Directory, 0o555, 0, 2), + IMAGE_INO => (FileType::RegularFile, 0o444, self.image_size(), 1), + _ => return None, + }; + Some(FileAttr { + ino: INodeNo(ino), + size, + // Reported in 512 B units like stat(2), rounded up. + blocks: size.div_ceil(512), + atime: UNIX_EPOCH, + mtime: UNIX_EPOCH, + ctime: UNIX_EPOCH, + crtime: UNIX_EPOCH, + kind, + perm, + nlink, + uid: 0, + gid: 0, + rdev: 0, + blksize: EROFS_BLOCK_SIZE, + flags: 0, + }) + } + + /// Read `[offset, offset + size)` of the flattened view into a fresh + /// buffer, clamped at EOF. + /// + /// [`FlatImage::read_at`] only accepts block-aligned windows, so an + /// unaligned request is widened to block boundaries and the interesting + /// slice is copied out. The kernel issues page-aligned reads in practice; + /// this keeps a hand-rolled `read(2)` on the export correct as well. + pub fn read_image(&self, offset: u64, size: u32) -> io::Result> { + let image_size = self.image_size(); + if offset >= image_size { + return Ok(Vec::new()); + } + let want = (size as u64).min(image_size - offset); + if want == 0 { + return Ok(Vec::new()); + } + + let start = offset - offset % BLOCK_SIZE; + let end = (offset + want) + .div_ceil(BLOCK_SIZE) + .checked_mul(BLOCK_SIZE) + .ok_or_else(|| io::Error::other("flat read range overflow"))? + .min(image_size); + let mut aligned = vec![0u8; (end - start) as usize]; + self.image + .read_at(start, &mut aligned) + .map_err(|err| io::Error::from_raw_os_error(error_to_errno(&err)))?; + + let from = (offset - start) as usize; + aligned.drain(..from); + aligned.truncate(want as usize); + Ok(aligned) + } +} + +/// Map a core error onto the errno reported to the kernel. Anything that is +/// not a plain I/O failure is still an I/O failure from the mount's point of +/// view, so EROFS surfaces it as a read error rather than something a caller +/// might retry differently. +fn error_to_errno(err: &nydus_error::Error) -> i32 { + match err { + nydus_error::Error::Io(err) => err.raw_os_error().unwrap_or(libc::EIO), + _ => libc::EIO, + } +} + +impl Filesystem for FlatImageFs { + fn init(&mut self, _req: &Request, config: &mut KernelConfig) -> io::Result<()> { + // EROFS reads metadata one folio at a time through `read_folio`, so + // the depth of the kernel's async pipeline is what keeps cold data + // reads from serializing behind each other. + let _ = config.set_max_background(64); + Ok(()) + } + + fn lookup(&self, _req: &Request, parent: INodeNo, name: &OsStr, reply: ReplyEntry) { + if parent.0 != ROOT_INO || name.as_bytes() != IMAGE_NAME.as_bytes() { + reply.error(Errno::ENOENT); + return; + } + match self.attr(IMAGE_INO) { + Some(attr) => reply.entry(&ENTRY_TIMEOUT, &attr, Generation(0)), + None => reply.error(Errno::ENOENT), + } + } + + fn getattr(&self, _req: &Request, ino: INodeNo, _fh: Option, reply: ReplyAttr) { + match self.attr(ino.0) { + Some(attr) => reply.attr(&ENTRY_TIMEOUT, &attr), + None => reply.error(Errno::ENOENT), + } + } + + fn open(&self, _req: &Request, ino: INodeNo, _flags: OpenFlags, reply: ReplyOpen) { + if ino.0 != IMAGE_INO { + reply.error(Errno::ENOENT); + return; + } + // KEEP_CACHE is what makes this mode worthwhile: the image is + // immutable, so metadata folios EROFS already faulted in stay valid + // for the lifetime of the mount instead of being dropped on open. + reply.opened(FileHandle(0), FopenFlags::FOPEN_KEEP_CACHE); + } + + fn read( + &self, + _req: &Request, + ino: INodeNo, + _fh: FileHandle, + offset: u64, + size: u32, + _flags: OpenFlags, + _lock_owner: Option, + reply: ReplyData, + ) { + if ino.0 != IMAGE_INO { + reply.error(Errno::ENOENT); + return; + } + match self.read_image(offset, size) { + Ok(data) => reply.data(&data), + Err(err) => reply.error(Errno::from_i32(err.raw_os_error().unwrap_or(libc::EIO))), + } + } + + fn opendir(&self, _req: &Request, ino: INodeNo, _flags: OpenFlags, reply: ReplyOpen) { + if ino.0 != ROOT_INO { + reply.error(Errno::ENOTDIR); + return; + } + reply.opened(FileHandle(0), FopenFlags::empty()); + } + + fn readdir( + &self, + _req: &Request, + ino: INodeNo, + _fh: FileHandle, + offset: u64, + mut reply: ReplyDirectory, + ) { + if ino.0 != ROOT_INO { + reply.error(Errno::ENOTDIR); + return; + } + let entries = [ + (ROOT_INO, FileType::Directory, "."), + (ROOT_INO, FileType::Directory, ".."), + (IMAGE_INO, FileType::RegularFile, IMAGE_NAME), + ]; + for (index, (entry_ino, kind, name)) in entries.iter().enumerate().skip(offset as usize) { + // The offset handed back is where a resumed readdir continues, so + // it must point past the entry just added. + if reply.add( + INodeNo(*entry_ino), + index as u64 + 1, + *kind, + OsStr::new(*name), + ) { + break; + } + } + reply.ok(); + } + + fn statfs(&self, _req: &Request, _ino: INodeNo, reply: ReplyStatfs) { + let blocks = self.image_size().div_ceil(BLOCK_SIZE); + reply.statfs( + blocks, + 0, + 0, + 2, + 0, + EROFS_BLOCK_SIZE, + IMAGE_NAME.len() as u32, + EROFS_BLOCK_SIZE, + ); + } + + /// Neither the root nor the image file carries extended attributes: the + /// ones a container cares about live inside the image and are resolved by + /// the EROFS driver mounted on top, never through this export. + /// + /// Answering with an empty list rather than letting the default `ENOSYS` + /// through keeps tooling that walks the export (`ls -l`, `cp -a`, `tar`) + /// from reporting an error on a mount that simply has no xattrs. + fn listxattr(&self, _req: &Request, ino: INodeNo, size: u32, reply: ReplyXattr) { + if self.attr(ino.0).is_none() { + reply.error(Errno::from_i32(libc::ENOENT)); + return; + } + // A zero `size` is the caller probing for the buffer it must allocate. + if size == 0 { + reply.size(0); + } else { + reply.data(&[]); + } + } + + /// Counterpart to [`Self::listxattr`]: every name misses. `ENODATA` says + /// "this file has no such attribute", which is what callers probing for + /// `security.*` or `system.posix_acl_access` expect, whereas the default + /// `ENOSYS` would claim the whole mount lacks xattr support and can make + /// the kernel stop asking on other inodes too. + fn getxattr(&self, _req: &Request, ino: INodeNo, _name: &OsStr, _size: u32, reply: ReplyXattr) { + let errno = if self.attr(ino.0).is_none() { + libc::ENOENT + } else { + libc::ENODATA + }; + reply.error(Errno::from_i32(errno)); + } +} diff --git a/nydus/src/fileio/mod.rs b/nydus/src/fileio/mod.rs new file mode 100644 index 00000000000..67879571981 --- /dev/null +++ b/nydus/src/fileio/mod.rs @@ -0,0 +1,25 @@ +// Copyright (C) 2026 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +//! File-backed EROFS service for nydus images. +//! +//! The daemon exports the flattened image (bootstrap at the head, then each +//! blob at its mapped offset) as a single file over FUSE, and the kernel +//! EROFS driver mounts that file directly. Unlike [`crate::fuse`], which +//! answers every filesystem operation from userspace, here the kernel parses +//! the EROFS metadata itself: lookup, readdir, stat and xattr never leave the +//! kernel, and only cold byte ranges come back as FUSE reads. Unlike +//! [`crate::nbd`] and [`crate::ublk`], no block device is involved, so the +//! guest needs neither `nbd` nor `ublk_drv`. The flattened view itself comes +//! from [`nydus_core::flat`], shared with those services. +//! +//! Requires Linux >= 6.12 (`CONFIG_EROFS_FS_BACKED_BY_FILE`). + +pub mod fs; +pub mod mount; +pub mod service; + +pub use fs::{FlatImageFs, IMAGE_NAME}; +pub use mount::mount_image_file; +pub use service::{image_path, warm_bootstrap, FileioService}; diff --git a/nydus/src/fileio/mount.rs b/nydus/src/fileio/mount.rs new file mode 100644 index 00000000000..ad83df4a83b --- /dev/null +++ b/nydus/src/fileio/mount.rs @@ -0,0 +1,57 @@ +// Copyright (C) 2026 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +//! EROFS mount of the exported flattened image file. + +use std::ffi::CString; +use std::path::Path; + +use nydus_error::{Context, Result}; + +use crate::mount::path_cstring; + +/// Mount the exported image file at `mountpoint` as read-only EROFS. +/// +/// The source is a regular file, so `get_tree_bdev` fails with `-ENOTBLK` and +/// the kernel falls back to file-backed mode (`CONFIG_EROFS_FS_BACKED_BY_FILE`, +/// Linux >= 6.12), reading it through the VFS — which is what routes cold +/// ranges back to the FUSE export. +/// +/// `direct_io` adds the `directio` option, which makes EROFS submit data reads +/// with `IOCB_DIRECT` so the export's own page cache stays empty. Every byte +/// EROFS reads is cached again against the inode the application reads, so +/// without it the image is held twice; measured on a 16 MiB read, the export +/// accumulated 16.2 MiB that `directio` reduces to zero. It costs roughly 30% +/// on cold reads, because each read becomes a FUSE round trip with no +/// intervening readahead, and nothing on warm reads. Metadata is unaffected +/// either way: `erofs_bread` resolves it through `read_mapping_folio`, which +/// the option does not touch. +pub fn mount_image_file(image: &Path, mountpoint: &Path, direct_io: bool) -> Result<()> { + let source = path_cstring(image, "image file")?; + let target = path_cstring(mountpoint, "mountpoint")?; + let fs_type = CString::new("erofs").expect("erofs has no interior NUL"); + let options = direct_io.then(|| CString::new("directio").expect("static option has no NUL")); + + let ret = unsafe { + libc::mount( + source.as_ptr(), + target.as_ptr(), + fs_type.as_ptr(), + libc::MS_RDONLY | libc::MS_NODEV | libc::MS_NOSUID, + options + .as_ref() + .map_or(std::ptr::null(), |opts| opts.as_ptr().cast()), + ) + }; + if ret < 0 { + return Err(std::io::Error::last_os_error()).with_context(|| { + format!( + "failed to mount {} at {} as erofs (needs Linux >= 6.12 with CONFIG_EROFS_FS_BACKED_BY_FILE)", + image.display(), + mountpoint.display() + ) + }); + } + Ok(()) +} diff --git a/nydus/src/fileio/service.rs b/nydus/src/fileio/service.rs new file mode 100644 index 00000000000..9f5c6da2123 --- /dev/null +++ b/nydus/src/fileio/service.rs @@ -0,0 +1,129 @@ +// Copyright (C) 2026 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +//! Mount lifecycle for the file-backed EROFS service. +//! +//! Two mounts stack here: the FUSE export holding the flattened image file, +//! and the kernel EROFS mount that consumes it. They must come down in the +//! opposite order they went up — EROFS keeps the backing file open, so +//! ending the FUSE session first leaves the EROFS mount reading a dead +//! session and the export busy. + +use std::path::{Path, PathBuf}; + +use fuser::Notifier; +use nydus_error::{Context, Result}; +use tracing::{debug, info, warn}; + +use super::fs::{FlatImageFs, IMAGE_INO, IMAGE_NAME}; +use crate::mount::unmount; + +/// Chunk size for warming the bootstrap into the export's page cache. +/// +/// `FUSE_NOTIFY_STORE` carries its payload inline in a single notification, +/// so the bootstrap is pushed in slices rather than as one multi-MiB message. +const WARM_CHUNK_SIZE: u64 = 1 << 20; + +/// Path of the exported image file inside the FUSE mount. +pub fn image_path(fuse_mountpoint: &Path) -> PathBuf { + fuse_mountpoint.join(IMAGE_NAME) +} + +/// Push the bootstrap region of the flattened image into the kernel page +/// cache of the exported file. +/// +/// EROFS resolves every lookup, readdir, stat and xattr by reading metadata +/// bytes out of this file through `read_folio`, which faults one folio at a +/// time with no readahead. Without warming, walking a large image costs one +/// FUSE round trip per 4 KiB of metadata; afterwards metadata operations stay +/// entirely in-kernel. +/// +/// Best effort by design: the kernel may drop the stored pages under memory +/// pressure and reads then fall back to the FUSE path, so a failure here is +/// logged rather than propagated. +pub fn warm_bootstrap(fs: &FlatImageFs, notifier: &Notifier, bootstrap_size: u64) { + let end = bootstrap_size.min(fs.image_size()); + if end == 0 { + return; + } + + let mut offset = 0u64; + while offset < end { + let len = WARM_CHUNK_SIZE.min(end - offset); + let data = match fs.read_image(offset, len as u32) { + Ok(data) if !data.is_empty() => data, + Ok(_) => break, + Err(err) => { + warn!("failed to read bootstrap at offset {offset} for warming: {err}"); + return; + } + }; + if let Err(err) = notifier.store(fuser::INodeNo(IMAGE_INO), offset, &data) { + // A kernel that rejects the notification leaves the cold read path + // intact, so stop warming instead of failing the mount. + debug!("fuse notify store at offset {offset} rejected: {err}"); + return; + } + offset += data.len() as u64; + } + info!("warmed {offset} bytes of bootstrap metadata into the export page cache"); +} + +/// The FUSE export plus the EROFS mount stacked on it, torn down in +/// dependency order. +pub struct FileioService { + session: fuser::BackgroundSession, + fuse_mountpoint: PathBuf, + erofs_mountpoint: Option, +} + +impl FileioService { + /// Mount `fs` at `fuse_mountpoint` and serve it on background threads. + pub fn mount(fs: FlatImageFs, fuse_mountpoint: &Path, config: &fuser::Config) -> Result { + let session = fuser::Session::new(fs, fuse_mountpoint, config).with_context(|| { + format!( + "failed to mount fuse export at {}", + fuse_mountpoint.display() + ) + })?; + let session = session + .spawn() + .context("failed to spawn fuse export session")?; + Ok(Self { + session, + fuse_mountpoint: fuse_mountpoint.to_path_buf(), + erofs_mountpoint: None, + }) + } + + /// Notifier for pushing data into the export's page cache. + pub fn notifier(&self) -> Notifier { + self.session.notifier() + } + + /// Record the EROFS mountpoint stacked on the export so shutdown unmounts + /// it before ending the session. + pub fn set_erofs_mountpoint(&mut self, mountpoint: &Path) { + self.erofs_mountpoint = Some(mountpoint.to_path_buf()); + } + + /// Unmount EROFS first, then unmount the export and end the session. + pub fn shutdown(self) { + if let Some(mountpoint) = &self.erofs_mountpoint { + if let Err(err) = unmount(mountpoint, || {}) { + warn!("failed to unmount {}: {}", mountpoint.display(), err); + } + } + // Unmounting is what makes the session threads return; joining alone + // would block until something else tore the export down. + if let Err(err) = self.session.umount_and_join() { + warn!( + "fuse export at {} ended with an error: {}", + self.fuse_mountpoint.display(), + err + ); + } + info!("stopped fuse export at {}", self.fuse_mountpoint.display()); + } +} diff --git a/nydus/src/fuse/fs.rs b/nydus/src/fuse/fs.rs index 5488941dfb7..2977ac0c94f 100644 --- a/nydus/src/fuse/fs.rs +++ b/nydus/src/fuse/fs.rs @@ -1,21 +1,23 @@ +use std::cell::RefCell; use std::collections::HashMap; use std::ffi::OsStr; use std::io; use std::os::unix::ffi::OsStrExt; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, UNIX_EPOCH}; use fuser::{ AccessFlags, Errno, FileAttr, FileHandle, FileType, Filesystem, FopenFlags, Generation, - INodeNo, LockOwner, OpenFlags, PollEvents, PollFlags, PollNotifier, ReplyAttr, ReplyData, - ReplyDirectory, ReplyDirectoryPlus, ReplyEmpty, ReplyEntry, ReplyOpen, ReplyPoll, ReplyStatfs, - ReplyXattr, Request, + INodeNo, InitFlags, KernelConfig, LockOwner, OpenFlags, PollEvents, PollFlags, PollNotifier, + ReplyAttr, ReplyData, ReplyDirectory, ReplyDirectoryPlus, ReplyEmpty, ReplyEntry, ReplyOpen, + ReplyPoll, ReplyStatfs, ReplyXattr, Request, }; use nydus_format::erofs::{ - is_nydus_xattr, ErofsInode, EROFS_FT_BLKDEV, EROFS_FT_CHRDEV, EROFS_FT_DIR, EROFS_FT_FIFO, - EROFS_FT_REG_FILE, EROFS_FT_SOCK, EROFS_FT_SYMLINK, + is_nydus_xattr, ErofsInode, EROFS_FEATURE_COMPAT_NYDUS_NO_XATTR, EROFS_FT_BLKDEV, + EROFS_FT_CHRDEV, EROFS_FT_DIR, EROFS_FT_FIFO, EROFS_FT_REG_FILE, EROFS_FT_SOCK, + EROFS_FT_SYMLINK, }; use nydus_telemetry::metrics; @@ -35,18 +37,49 @@ pub struct ErofsFs { reader: Arc, dir_handles: Mutex>>, next_dir_handle: AtomicU64, + /// Kernel accepts ENOSYS from open/opendir as "stop sending them": file + /// and directory opens then cost no FUSE round-trip at all and the dummy + /// handles keep the page cache (KEEP_CACHE, and CACHE_DIR for dirs). + no_open: AtomicBool, + no_opendir: AtomicBool, + /// Image-wide "no inode has xattrs" declaration from the builder: xattr + /// requests answer ENOSYS so the kernel stops sending them entirely. + no_xattr: bool, } +/// An opened directory. Entries materialize on the first readdir; with +/// FOPEN_CACHE_DIR the kernel usually serves repeat listings from the page +/// cache and never sends that readdir, so opendir must not pay for one. struct DirHandle { - entries: Vec, + ino: u64, + entries: Mutex>>>, +} + +impl DirHandle { + fn entries(&self, fs: &ErofsFs) -> io::Result>> { + let mut guard = self.entries.lock().unwrap(); + if let Some(entries) = guard.as_ref() { + return Ok(entries.clone()); + } + let nid = fs.ino_to_nid(self.ino); + let vi = fs.reader.inode(nid)?; + let entries = Arc::new(fs.reader.read_dir(nid, &vi)?); + *guard = Some(entries.clone()); + Ok(entries) + } } impl ErofsFs { pub fn new(reader: Arc) -> Self { + let no_xattr = + reader.superblock().feature_compat() & EROFS_FEATURE_COMPAT_NYDUS_NO_XATTR != 0; Self { reader, dir_handles: Mutex::new(HashMap::new()), next_dir_handle: AtomicU64::new(1), + no_open: AtomicBool::new(false), + no_opendir: AtomicBool::new(false), + no_xattr, } } @@ -115,24 +148,12 @@ impl ErofsFs { } } - fn iterate_dir(&self, ino: u64, mut cb: F) -> io::Result<()> - where - F: FnMut(u64, u8, &[u8]) -> io::Result, - { - let nid = self.ino_to_nid(ino); - let vi = self.reader.inode(nid)?; - self.reader - .for_each_dir_entry(nid, &vi, |entry_nid, file_type, name| { - cb(entry_nid, file_type, name) - }) - } - fn create_dir_handle(&self, ino: u64) -> io::Result { - let nid = self.ino_to_nid(ino); - let vi = self.reader.inode(nid)?; - let entries = self.reader.read_dir(nid, &vi)?; let handle = self.next_dir_handle.fetch_add(1, Ordering::Relaxed); - let dir_handle = Arc::new(DirHandle { entries }); + let dir_handle = Arc::new(DirHandle { + ino, + entries: Mutex::new(None), + }); self.dir_handles.lock().unwrap().insert(handle, dir_handle); Ok(handle) } @@ -145,12 +166,46 @@ impl ErofsFs { .cloned() .ok_or_else(|| io::Error::from_raw_os_error(libc::EBADF)) } + + /// Directory entries for a readdir(plus): through the handle when opendir + /// issued one, or straight from the inode for the kernel's no-opendir + /// dummy handle (fh 0). + fn dir_entries(&self, ino: INodeNo, fh: FileHandle) -> io::Result>> { + if fh.0 != 0 { + return self.dir_handle(fh.0)?.entries(self); + } + let nid = self.ino_to_nid(ino.0); + let vi = self.reader.inode(nid)?; + Ok(Arc::new(self.reader.read_dir(nid, &vi)?)) + } } fn io_errno(e: &io::Error) -> Errno { Errno::from_i32(e.raw_os_error().unwrap_or(libc::EIO)) } +/// The reply body for a cached negative lookup: ino 0 tells the kernel "no +/// such entry, remember that for the ttl". Every other field is ignored. +fn negative_attr() -> FileAttr { + FileAttr { + ino: INodeNo(0), + size: 0, + blocks: 0, + atime: UNIX_EPOCH, + mtime: UNIX_EPOCH, + ctime: UNIX_EPOCH, + crtime: UNIX_EPOCH, + kind: FileType::RegularFile, + perm: 0, + nlink: 0, + uid: 0, + gid: 0, + rdev: 0, + blksize: 0, + flags: 0, + } +} + /// RAII guard that records a FUSE operation's outcome and latency on drop. /// It assumes success unless [`fail`](FsOpMetric::fail) is called before the /// op replies with an error. @@ -211,6 +266,34 @@ fn should_hide_xattr(ino: u64, name: &[u8]) -> bool { } impl Filesystem for ErofsFs { + fn init(&mut self, _req: &Request, config: &mut KernelConfig) -> io::Result<()> { + // fuser only requests ASYNC_READ|BIG_WRITES|MAX_PAGES by default, so + // without these the kernel never issues READDIRPLUS (leaving our + // readdirplus implementation dead code), serializes lookups within one + // directory, and drops cached symlink targets. + let _ = config.add_capabilities(InitFlags::FUSE_DO_READDIRPLUS); + // Deliberately NOT FUSE_READDIRPLUS_AUTO: under AUTO the kernel's + // heuristic falls back to plain READDIR for large directories, and a + // following stat of every entry becomes one LOOKUP round trip each. + let _ = config.add_capabilities(InitFlags::FUSE_PARALLEL_DIROPS); + let _ = config.add_capabilities(InitFlags::FUSE_CACHE_SYMLINKS); + if config + .add_capabilities(InitFlags::FUSE_NO_OPEN_SUPPORT) + .is_ok() + { + self.no_open.store(true, Ordering::Relaxed); + } + if config + .add_capabilities(InitFlags::FUSE_NO_OPENDIR_SUPPORT) + .is_ok() + { + self.no_opendir.store(true, Ordering::Relaxed); + } + // Default of 16 throttles the kernel's async readahead pipeline. + let _ = config.set_max_background(64); + Ok(()) + } + fn lookup(&self, _req: &Request, parent: INodeNo, name: &OsStr, reply: ReplyEntry) { let mut m = FsOpMetric::new(metrics::FsOp::Lookup); let target = name.as_bytes(); @@ -219,19 +302,19 @@ impl Filesystem for ErofsFs { reply.error(Errno::ENAMETOOLONG); return; } - let mut found = None; - let res = self.iterate_dir(parent.0, |entry_nid, _file_type, entry_name| { - if entry_name == target { - found = Some(entry_nid); - return Ok(false); + let parent_nid = self.ino_to_nid(parent.0); + let found = match self + .reader + .inode(parent_nid) + .and_then(|vi| self.reader.lookup_dir_entry(parent_nid, &vi, target)) + { + Ok(found) => found, + Err(err) => { + m.fail(); + reply.error(io_errno(&err)); + return; } - Ok(true) - }); - if let Err(err) = res { - m.fail(); - reply.error(io_errno(&err)); - return; - } + }; if let Some(child_nid) = found { match self.reader.inode(child_nid) { @@ -247,8 +330,13 @@ impl Filesystem for ErofsFs { return; } + // Cache the miss: an entry with ino 0 is a negative dentry the kernel + // keeps for the ttl, so repeats resolve in the dcache instead of one + // FUSE round trip each. The image is immutable, so a miss holds + // forever — and module resolution (Node's require walk, Python's + // sys.path probing) retries the same missing names constantly. m.fail(); - reply.error(Errno::ENOENT); + reply.entry(&EROFS_FUSE_TIMEOUT, &negative_attr(), Generation(0)); } fn forget(&self, _req: &Request, _ino: INodeNo, _nlookup: u64) { @@ -272,6 +360,13 @@ impl Filesystem for ErofsFs { fn open(&self, _req: &Request, ino: INodeNo, flags: OpenFlags, reply: ReplyOpen) { let mut m = FsOpMetric::new(metrics::FsOp::Open); + // ENOSYS makes the kernel treat this and every later open as success + // without a handle, with KEEP_CACHE semantics; the read-only mount + // already rejects write opens before they reach us. + if self.no_open.load(Ordering::Relaxed) { + reply.error(Errno::ENOSYS); + return; + } if flags.0 & (libc::O_WRONLY | libc::O_RDWR) != 0 { m.fail(); reply.error(Errno::EROFS); @@ -346,18 +441,26 @@ impl Filesystem for ErofsFs { } }; - // Use write_file_data_to to fill a Vec zero-copy from mmap. - let mut buf: Vec = Vec::with_capacity(size as usize); - match self - .reader - .write_file_data_to(nid, &vi, offset, size, &mut buf) - { - Ok(_) => reply.data(&buf), - Err(err) => { - m.fail(); - reply.error(io_errno(&err)); - } + // Reuse a per-worker buffer: a fresh Vec per request costs an mmap + // round-trip plus page faults for every large read. + thread_local! { + static READ_BUF: RefCell> = const { RefCell::new(Vec::new()) }; } + READ_BUF.with(|cell| { + let mut buf = cell.borrow_mut(); + buf.clear(); + buf.reserve(size as usize); + match self + .reader + .write_file_data_to(nid, &vi, offset, size, &mut *buf) + { + Ok(_) => reply.data(&buf), + Err(err) => { + m.fail(); + reply.error(io_errno(&err)); + } + } + }); } fn readlink(&self, _req: &Request, ino: INodeNo, reply: ReplyData) { @@ -382,6 +485,13 @@ impl Filesystem for ErofsFs { fn opendir(&self, _req: &Request, ino: INodeNo, _flags: OpenFlags, reply: ReplyOpen) { let mut m = FsOpMetric::new(metrics::FsOp::Opendir); + // See open(): dropping opendir/releasedir round-trips also gives the + // kernel-side dummy handle FOPEN_CACHE_DIR, so repeat listings are + // served from the page cache without any FUSE traffic. + if self.no_opendir.load(Ordering::Relaxed) { + reply.error(Errno::ENOSYS); + return; + } let nid = self.ino_to_nid(ino.0); let vi = match self.reader.inode(nid) { Ok(vi) => vi, @@ -412,14 +522,14 @@ impl Filesystem for ErofsFs { fn readdir( &self, _req: &Request, - _ino: INodeNo, + ino: INodeNo, fh: FileHandle, offset: u64, mut reply: ReplyDirectory, ) { let mut m = FsOpMetric::new(metrics::FsOp::Readdir); - let dir_handle = match self.dir_handle(fh.0) { - Ok(h) => h, + let entries = match self.dir_entries(ino, fh) { + Ok(entries) => entries, Err(err) => { m.fail(); reply.error(io_errno(&err)); @@ -427,7 +537,7 @@ impl Filesystem for ErofsFs { } }; let start = usize::try_from(offset).unwrap_or(usize::MAX); - for (index, entry) in dir_handle.entries.iter().enumerate().skip(start) { + for (index, entry) in entries.iter().enumerate().skip(start) { let ino = self.nid_to_ino(entry.nid); let kind = erofs_ft_to_kind(entry.file_type); let name = OsStr::from_bytes(&entry.name); @@ -441,14 +551,14 @@ impl Filesystem for ErofsFs { fn readdirplus( &self, _req: &Request, - _ino: INodeNo, + ino: INodeNo, fh: FileHandle, offset: u64, mut reply: ReplyDirectoryPlus, ) { let mut m = FsOpMetric::new(metrics::FsOp::Readdirplus); - let dir_handle = match self.dir_handle(fh.0) { - Ok(h) => h, + let entries = match self.dir_entries(ino, fh) { + Ok(entries) => entries, Err(err) => { m.fail(); reply.error(io_errno(&err)); @@ -456,7 +566,7 @@ impl Filesystem for ErofsFs { } }; let start = usize::try_from(offset).unwrap_or(usize::MAX); - for (index, entry) in dir_handle.entries.iter().enumerate().skip(start) { + for (index, entry) in entries.iter().enumerate().skip(start) { let child_inode = match self.reader.inode(entry.nid) { Ok(vi) => vi, Err(err) => { @@ -533,6 +643,10 @@ impl Filesystem for ErofsFs { fn getxattr(&self, _req: &Request, ino: INodeNo, name: &OsStr, size: u32, reply: ReplyXattr) { let mut m = FsOpMetric::new(metrics::FsOp::Getxattr); + if self.no_xattr { + reply.error(Errno::ENOSYS); + return; + } let nid = self.ino_to_nid(ino.0); let vi = match self.reader.inode(nid) { Ok(vi) => vi, @@ -579,6 +693,10 @@ impl Filesystem for ErofsFs { fn listxattr(&self, _req: &Request, ino: INodeNo, size: u32, reply: ReplyXattr) { let mut m = FsOpMetric::new(metrics::FsOp::Listxattr); + if self.no_xattr { + reply.error(Errno::ENOSYS); + return; + } let nid = self.ino_to_nid(ino.0); let vi = match self.reader.inode(nid) { Ok(vi) => vi, diff --git a/nydus/src/lib.rs b/nydus/src/lib.rs index 7dbef66c97d..e46c8baa1a4 100644 --- a/nydus/src/lib.rs +++ b/nydus/src/lib.rs @@ -1,6 +1,6 @@ //! EROFS-oriented image tooling on top of [`nydus_core`]: image inspection -//! and export, plus optional FUSE / NBD / ublk / fanotify / userfaultfd -//! frontends. +//! and export, plus optional FUSE / file-backed EROFS / NBD / ublk / fanotify +//! / userfaultfd frontends. #![warn(unreachable_pub)] @@ -11,15 +11,18 @@ pub mod check; pub mod export; #[cfg(feature = "fanotify")] pub mod fanotify; +#[cfg(feature = "fileio")] +pub mod fileio; #[cfg(feature = "fuse")] pub mod fuse; -#[cfg(any(feature = "fanotify", feature = "nbd"))] +#[cfg(any(feature = "fanotify", feature = "fileio", feature = "nbd"))] pub mod mount; #[cfg(feature = "nbd")] pub mod nbd; pub mod optimize; #[cfg(any( feature = "fanotify", + feature = "fileio", feature = "nbd", feature = "ublk", feature = "uffd" diff --git a/nydus/src/nbd/core.rs b/nydus/src/nbd/core.rs index 245b9741e2c..c0222d21669 100644 --- a/nydus/src/nbd/core.rs +++ b/nydus/src/nbd/core.rs @@ -24,24 +24,17 @@ use std::path::Path; use std::sync::Arc; use nydus_config::Config; -use nydus_core::extent::MmapCache; +use nydus_core::flat::{FlatImage, BLOCK_SIZE}; use nydus_core::NydusCore; -use nydus_error::{Context, Error, Result}; -use nydus_format::erofs::EROFS_BLOCK_SIZE; - -/// EROFS block size as u64 — reuses the canonical constant from the core. -const BLOCK_SIZE: u64 = EROFS_BLOCK_SIZE as u64; +use nydus_error::{Error, Result}; /// Read-side handle for the NBD on-demand service. /// -/// Holds a shared core (so background prefetch keeps running) and the -/// flattened device size computed at construction. All read paths are -/// block-aligned by the NBD protocol, which matches the core's fetch -/// precondition. +/// A thin protocol adapter over the shared [`FlatImage`]: it adds the range +/// check the NBD protocol guarantees (so a violated guarantee surfaces as an +/// error instead of silent zeros) and reports the geometry the driver needs. pub struct NbdCore { - core: Arc, - device_size: u64, - maps: MmapCache, + image: FlatImage, } impl NbdCore { @@ -49,34 +42,24 @@ impl NbdCore { /// No blob meta is downloaded and no cache file is created until a read /// first touches a blob, so this returns quickly even for large images. pub fn new(bootstrap: &Path, config: Config) -> Result { - let core = Arc::new(NydusCore::new(bootstrap, config)?); - - let flat_size = core.flat_size(); - if flat_size == 0 { - return Err(Error::InvalidImage( - "flattened image size is zero".to_string(), - )); - } - if flat_size % BLOCK_SIZE != 0 { - return Err(Error::InvalidImage(format!( - "flattened image size {flat_size} is not a multiple of the {BLOCK_SIZE} B EROFS block size" - ))); - } - Ok(Self { - core, - device_size: flat_size, - maps: MmapCache::default(), - }) + // The NBD device addresses in EROFS blocks, so no extra rounding. + let image = FlatImage::open(bootstrap, config, BLOCK_SIZE)?; + Ok(Self { image }) } /// Total size in bytes of the flattened block device exposed to the kernel. pub fn device_size(&self) -> u64 { - self.device_size + self.image.size() } /// Total block count (4 KiB units) reported to the NBD driver. pub fn block_count(&self) -> u64 { - self.device_size / BLOCK_SIZE + self.image.block_count() + } + + /// Borrow the underlying core, e.g. to snapshot metrics. + pub fn core(&self) -> &Arc { + self.image.core() } /// Fetch `[offset, offset + buf.len())` of the flattened device view and @@ -88,50 +71,18 @@ impl NbdCore { if buf.is_empty() { return Ok(()); } - debug_assert!(offset % BLOCK_SIZE == 0); - debug_assert!((buf.len() as u64) % BLOCK_SIZE == 0); let len = buf.len() as u64; let end = offset .checked_add(len) .ok_or_else(|| Error::Overflow("nbd read range overflow".to_string()))?; - if end > self.device_size { + // The shared view would zero-fill past the end; for NBD that can only + // mean a malformed request, so reject it instead. + if end > self.device_size() { return Err(Error::InvalidParameter(format!( "nbd read [{offset}, +{len}) past flattened device size {}", - self.device_size + self.device_size() ))); } - - let ranges = self - .core - .fetch_flat_ranges(offset, len) - .context("failed to fetch flat ranges for nbd read")?; - // The fetch contract is a gapless cover of the request window; check it - // explicitly so a contract drift surfaces as an error instead of - // silently misplacing bytes (the shared copy below is gap-tolerant). - let mut written = 0usize; - for range in &ranges { - if range.source_offset != offset + written as u64 { - return Err(Error::Runtime(format!( - "flat ranges are not contiguous: expected source offset {}, got {}", - offset + written as u64, - range.source_offset - ))); - } - let seg_len = range.len as usize; - if written + seg_len > buf.len() { - return Err(Error::Runtime(format!( - "flat range segment overflows read buffer: written={written} seg_len={seg_len} buf={}", - buf.len() - ))); - } - written += seg_len; - } - // Copy through the shared engine: zero fds serve zeros, everything else - // is copied out of a shared mapping when possible (pread fallback), and - // any uncovered tail is zero-filled defensively. - self.maps - .copy_ranges(&ranges, offset, self.core.zero_fd(), buf) - .context("failed to copy flat ranges for nbd read")?; - Ok(()) + self.image.read_at(offset, buf) } } diff --git a/nydus/src/optimize/mod.rs b/nydus/src/optimize/mod.rs index bc6815b11c4..4fe32dd2c7b 100644 --- a/nydus/src/optimize/mod.rs +++ b/nydus/src/optimize/mod.rs @@ -33,7 +33,7 @@ use nydus_format::blob::{ }; use nydus_format::erofs::EROFS_BLOB_ID_SIZE; use nydus_storage::access_trace::{TraceDocument, TraceEntry, TRACE_DOCUMENT_VERSION}; -use nydus_storage::cache::{BlobCache, LocalBlobCache}; +use nydus_storage::cache::LocalBlobCache; /// The result of [`build_ondemand_blob`]: the assembled ondemand artifact and /// the rewritten bootstrap, ready to be written out by the caller. @@ -90,7 +90,6 @@ pub fn build_ondemand_blob( let mut ondemand_data = Vec::new(); let mut ondemand_block_groups = Vec::new(); let mut next_block_offset = 0u64; - let mut decoded = Vec::new(); for BlockGroupRef { blob_index, @@ -107,6 +106,11 @@ pub fn build_ondemand_blob( .with_context(|| format!("failed to open source blob: {blob_index}"))?, ), }; + if cache.blob_metadata().is_redirect() { + return Err(Error::InvalidImage(format!( + "source blob {blob_index} is already an ondemand blob; refusing to optimize" + ))); + } let block_group = *cache .blob_metadata() @@ -116,22 +120,14 @@ pub fn build_ondemand_blob( "pattern references block group {block_group_index} out of range for blob {blob_index}" )) })?; - if block_group.is_redirect() { - return Err(Error::InvalidImage(format!( - "source blob {blob_index} is already an ondemand blob (refusing to optimize)" - ))); - } - let decoded_len = usize::try_from(block_group.uncompressed_size()).map_err(|err| { - Error::Overflow(format!( - "block group uncompressed size exceeds usize: {err}" - )) - })?; - decoded.resize(decoded_len, 0); - cache - .read_at(block_group.uncompressed_offset(), &mut decoded) + // Fetch the block group's decoded bytes straight from the backend at + // block group granularity: the redirect fill on the runtime side works + // per source block group. + let decoded = cache + .fetch_block_group(*block_group_index as usize) .with_context(|| { - format!("failed to read block group {block_group_index} of blob {blob_index}") + format!("failed to fetch block group {block_group_index} of blob {blob_index}") })?; // Recompress the decoded bytes for the ondemand artifact, storing them diff --git a/nydus/src/ublk/core.rs b/nydus/src/ublk/core.rs index babb40b0903..133bb330574 100644 --- a/nydus/src/ublk/core.rs +++ b/nydus/src/ublk/core.rs @@ -8,15 +8,14 @@ //! the resulting device can be mounted with `mount -t erofs /dev/ublkbN`. use std::io; -use std::os::fd::RawFd; use std::path::Path; +use std::sync::Arc; use nydus_config::Config; -use nydus_core::extent::MmapCache; +use nydus_core::flat::FlatImage; use nydus_core::NydusCore; -use nydus_error::{Context, Error, Result}; +use nydus_error::{Context, Result}; use nydus_format::erofs::EROFS_BLOCK_SIZE; -use nydus_format::utils::align_up_u64; use tracing::warn; /// Logical block size exposed by the ublk device. Matching the EROFS block size @@ -25,10 +24,7 @@ pub const UBLK_LOGICAL_BLOCK_SIZE: u64 = EROFS_BLOCK_SIZE as u64; /// Read-only block device backed by a nydus image. pub struct UblkCore { - core: NydusCore, - zero_fd: RawFd, - device_size: u64, - maps: MmapCache, + image: FlatImage, } impl UblkCore { @@ -37,38 +33,35 @@ impl UblkCore { /// table are prepared up front, and background prefetch follows /// `config.prefetch`. pub fn new(bootstrap: &Path, config: Config) -> Result { - let core = NydusCore::new(bootstrap, config) - .context("failed to open nydus image for the ublk device")?; - let zero_fd = core.zero_fd(); - // Round the device size up to a whole block: the kernel always reads in - // block units, and the tail block of the last blob may be partial. - let device_size = align_up_u64(core.flat_size(), UBLK_LOGICAL_BLOCK_SIZE) - .ok_or_else(|| Error::Overflow("flattened device size overflow".to_string()))?; - // Preparing a blob loads and validates its meta and sizes its cache - // file. Doing it up front keeps the first block read (typically - // `mount`) from paying for it, and surfaces preparation errors at - // startup instead of as I/O errors. - core.blobs - .flat_layout() - .context("failed to prepare the blobs backing the device")?; + // The kernel always reads in whole blocks, and the tail block of the + // last blob may be partial, so the device is rounded up. + let image = FlatImage::open(bootstrap, config, UBLK_LOGICAL_BLOCK_SIZE)?; + // Warm the blob preparation (meta download + cache file sizing) in + // the background so device creation and the EROFS mount are not + // blocked on backend round trips. flat_layout() is single-flight: an + // I/O arriving first simply joins the same preparation. + let warm = image.core().clone(); + std::thread::Builder::new() + .name("ublk-blob-warmup".to_string()) + .spawn(move || { + if let Err(err) = warm.blobs.flat_layout() { + tracing::warn!("background blob preparation failed: {err:#}"); + } + }) + .context("failed to spawn the blob warm-up thread")?; - Ok(Self { - core, - zero_fd, - device_size, - maps: MmapCache::default(), - }) + Ok(Self { image }) } /// Size of the block device in bytes, always a multiple of /// [`UBLK_LOGICAL_BLOCK_SIZE`]. pub fn device_size(&self) -> u64 { - self.device_size + self.image.size() } /// Borrow the underlying core, e.g. to snapshot metrics. - pub fn core(&self) -> &NydusCore { - &self.core + pub fn core(&self) -> &Arc { + self.image.core() } /// Read `buf.len()` bytes at `offset` of the flattened device. @@ -78,29 +71,22 @@ impl UblkCore { /// device with sparse backing would return. Missing blob data is fetched /// from the backend on demand before the copy. pub fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { - if buf.is_empty() { - return Ok(()); - } - if offset >= self.device_size { - buf.fill(0); - return Ok(()); - } - - let len = (buf.len() as u64).min(self.device_size - offset); - let ranges = self.core.fetch_flat_ranges(offset, len).map_err(|err| { + self.image.read_at(offset, buf).map_err(|err| { // Recover the OS errno when the fetch failed on IO, so the ublk // reply carries the real code instead of collapsing to EIO. A // bare `from_raw_os_error` drops the context chain, so log it // here before converting. match err.io_error().and_then(|io_err| io_err.raw_os_error()) { Some(errno) => { - warn!("ublk fetch at {offset} (+{len}) failed: {}", err.report()); + warn!( + "ublk read at {offset} (+{}) failed: {}", + buf.len(), + err.report() + ); io::Error::from_raw_os_error(errno) } None => io::Error::other(err), } - })?; - - self.maps.copy_ranges(&ranges, offset, self.zero_fd, buf) + }) } } diff --git a/nydus/tests/testsuite/nydus_core.rs b/nydus/tests/testsuite/nydus_core.rs index 7718944efeb..3bf33ed0129 100644 --- a/nydus/tests/testsuite/nydus_core.rs +++ b/nydus/tests/testsuite/nydus_core.rs @@ -39,6 +39,17 @@ fn build_test_image( build_test_image_with_layout(root, false) } +fn build_duplicate_corpus_test_image( + root: &Path, +) -> ( + PathBuf, + Config, + [u8; EROFS_BLOB_ID_SIZE], + HashMap>, +) { + build_test_image_full(root, false, true) +} + fn build_flattened_test_image( root: &Path, ) -> ( @@ -58,6 +69,19 @@ fn build_test_image_with_layout( Config, [u8; EROFS_BLOB_ID_SIZE], HashMap>, +) { + build_test_image_full(root, flattened, false) +} + +fn build_test_image_full( + root: &Path, + flattened: bool, + dedup_corpus: bool, +) -> ( + PathBuf, + Config, + [u8; EROFS_BLOB_ID_SIZE], + HashMap>, ) { let corpus_dir = root.join("corpus"); fs::create_dir_all(&corpus_dir).unwrap(); @@ -85,6 +109,20 @@ fn build_test_image_with_layout( corpus.insert("empty.txt".to_string(), Vec::new()); symlink("file1", corpus_dir.join("link_to_file1")).unwrap(); + if dedup_corpus { + let mut shifted = b"shifted-header:".to_vec(); + shifted.extend_from_slice(&corpus["file1"]); + fs::write(corpus_dir.join("file1_shifted"), &shifted).unwrap(); + corpus.insert("file1_shifted".to_string(), shifted); + fs::write(corpus_dir.join("file1_copy"), &corpus["file1"]).unwrap(); + corpus.insert("file1_copy".to_string(), corpus["file1"].clone()); + let mut holey = vec![0u8; 3 << 20]; + holey[..4096].copy_from_slice(&corpus["file2"][..4096]); + holey[(2 << 20) + 5..(2 << 20) + 4101].copy_from_slice(&corpus["file2"][..4096]); + fs::write(corpus_dir.join("holey"), &holey).unwrap(); + corpus.insert("holey".to_string(), holey); + } + let blob_dir = root.join("blobs"); fs::create_dir_all(&blob_dir).unwrap(); let staging = blob_dir.join("staging"); @@ -202,12 +240,13 @@ fn core_describes_devices_and_fetches_aligned_ranges() { assert_eq!(bootstrap_ranges[0].source_offset, 0); assert_eq!(bootstrap_ranges[0].len, EROFS_BLOCK_SIZE as u64); - // Fetch a block-aligned range in the middle; the cache file should be - // populated for that range and a second fetch is idempotent. The dense - // blob address space is independent of path order, so exact file - // content is covered by the static read API test below. + // Fetch a block-aligned range spanning more than one block group's worth + // of data; the cache file should be populated for that range and a second + // fetch is idempotent. The dense blob address space is independent of + // path order, so exact file content is covered by the static read API + // test below. let block = EROFS_BLOCK_SIZE as u64; - let (blob_offset, len) = (256 * block, 16 * block); + let (blob_offset, len) = (block, 272 * block); let offset = descriptor.mapped_offset + blob_offset; assert!(core.probe_flat_ranges(offset, len).unwrap().is_empty()); let fd_ranges = core.fetch_flat_ranges(offset, len).unwrap(); @@ -228,12 +267,13 @@ fn core_describes_devices_and_fetches_aligned_ranges() { core.blobs.fetch(&blob_id, 0, 0).unwrap(); let trace = core.trace_snapshot(); - assert_eq!(trace.entries.len(), 1); - assert_eq!(trace.entries[0].blob_index, 1); - assert_eq!(trace.entries[0].block_group_index, 1); + assert_eq!(trace.entries.len(), 2); + assert!(trace.entries.iter().all(|entry| entry.blob_index == 1)); + assert_eq!(trace.entries[0].block_group_index, 0); + assert_eq!(trace.entries[1].block_group_index, 1); assert_eq!( core.trace_json(), - "{\"version\":1,\"patterns\":[{\"blob_index\":1,\"block_group_index\":1}]}" + "{\"version\":1,\"patterns\":[{\"blob_index\":1,\"block_group_index\":0},{\"blob_index\":1,\"block_group_index\":1}]}" ); // Unaligned ranges and unknown blobs are rejected. @@ -425,3 +465,35 @@ fn node_fetch_populates_blob_cache_without_reading_data() { file1_entry.fetch(0, 0).unwrap(); core.fs.open("/").unwrap().fetch(0, 4096).unwrap_err(); } + +#[test] +fn core_reads_back_duplicate_corpus_image() { + let dir = tempdir().unwrap(); + let (bootstrap, config, _blob_id, corpus) = build_duplicate_corpus_test_image(dir.path()); + + let core = NydusCore::new(&bootstrap, config).unwrap(); + + for (name, expected) in &corpus { + let entry = core.fs.open(name).unwrap(); + let all = entry.read().unwrap(); + assert_eq!( + &all[..expected.len()], + expected.as_slice(), + "content mismatch for {name}" + ); + assert!( + all[expected.len()..].iter().all(|byte| *byte == 0), + "tail padding not zero for {name}" + ); + } + + let entry = core.fs.open("file1_shifted").unwrap(); + let mut buf = vec![0u8; 100_000]; + let read = entry.read_at(123_457, &mut buf).unwrap(); + assert_eq!(read, buf.len()); + assert_eq!(&buf, &corpus["file1_shifted"][123_457..123_457 + read]); + + let file1_entry = core.fs.open("file1").unwrap(); + file1_entry.fetch(12345, 4097).unwrap(); + assert!(!file1_entry.probe_ranges(12345, 4097).unwrap().is_empty()); +} diff --git a/nydusify/go.sum b/nydusify/go.sum index d406e4c6b13..c0feb8b43fc 100644 --- a/nydusify/go.sum +++ b/nydusify/go.sum @@ -31,7 +31,6 @@ github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsx github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= @@ -95,7 +94,6 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -110,7 +108,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= diff --git a/nydusify/internal/pipeline/convert.go b/nydusify/internal/pipeline/convert.go index b76f0a8729f..696412f2567 100644 --- a/nydusify/internal/pipeline/convert.go +++ b/nydusify/internal/pipeline/convert.go @@ -24,6 +24,10 @@ type Option struct { BuilderPath string // WorkDir is a scratch directory for layer extraction, FIFOs and staging. WorkDir string + // OnBlobConverted, when set, is called once per converted layer blob as + // soon as it is committed to the content store, so uploads can overlap + // with the remaining conversion work. Must be safe for concurrent calls. + OnBlobConverted func(desc ocispec.Descriptor) // ChunkSize is the nydus file chunk size in bytes. ChunkSize uint32 // BlockGroupSize is the nydus block group uncompressed size in bytes (a multiple of @@ -64,6 +68,16 @@ func Convert(ctx context.Context, cs content.Store, srcDesc ocispec.Descriptor, Compressor: opt.Compressor, LogLevel: opt.LogLevel, }) + if opt.OnBlobConverted != nil { + inner := layerFn + layerFn = func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { + newDesc, err := inner(ctx, cs, desc) + if err == nil && newDesc != nil && nydus.IsBlob(*newDesc) { + opt.OnBlobConverted(*newDesc) + } + return newDesc, err + } + } hookFn := ConvertHookFunc(nydus.MergeOption{ BuilderPath: opt.BuilderPath, WorkDir: opt.WorkDir, @@ -83,6 +97,14 @@ func Convert(ctx context.Context, cs content.Store, srcDesc ocispec.Descriptor, if err := labelNydusBlobDiffIDs(ctx, cs, srcDesc, platformMC); err != nil { return nil, errors.Wrap(err, "label nydus blob diff ids") } + // Label every source OCI layer with its diff id from the image config: + // the converter calls images.GetDiffID per layer BEFORE spawning the + // parallel layer conversions, and without the label that decompresses + // every gzip layer serially — for a large image several seconds on the + // critical path for digests the config already carries. + if err := labelSourceLayerDiffIDs(ctx, cs, srcDesc, platformMC); err != nil { + return nil, errors.Wrap(err, "label source layer diff ids") + } newDesc, err := indexConvertFn(ctx, cs, srcDesc) if err != nil { diff --git a/nydusify/internal/pipeline/layer.go b/nydusify/internal/pipeline/layer.go index 82584be9321..d9c46da179f 100644 --- a/nydusify/internal/pipeline/layer.go +++ b/nydusify/internal/pipeline/layer.go @@ -11,6 +11,8 @@ import ( "io" "os" "path/filepath" + "runtime" + "strconv" "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/core/images" @@ -29,6 +31,10 @@ import ( // (preserving OCI whiteouts), then `nydus build` streams the resulting full // blob through a FIFO directly into the content store. func LayerConvertFunc(opt nydus.PackOption) converter.ConvertFunc { + // The containerd converter spawns every layer conversion at once; each + // one runs an extraction plus a `nydus build`, so an unbounded fan-out + // multiplies peak memory and thrashes the CPU on many-layer images. + slots := make(chan struct{}, layerConvertConcurrency()) return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) { if ctx.Err() != nil { return nil, ctx.Err() @@ -41,6 +47,13 @@ func LayerConvertFunc(opt nydus.PackOption) converter.ConvertFunc { return nil, nil } + select { + case slots <- struct{}{}: + defer func() { <-slots }() + case <-ctx.Done(): + return nil, ctx.Err() + } + newDesc, err := convertLayer(ctx, cs, desc, opt) if err != nil { return nil, errors.Wrapf(err, "convert layer %s", desc.Digest) @@ -49,6 +62,27 @@ func LayerConvertFunc(opt nydus.PackOption) converter.ConvertFunc { } } +// layerConvertConcurrency bounds parallel layer conversions: enough to keep +// the cores busy, small enough to bound the sum of concurrent extract dirs +// and builder RSS. NYDUSIFY_LAYER_CONCURRENCY overrides the default (large +// layers dominate the critical path, so lowering it trades little wall time +// for a proportional peak-memory cut). +func layerConvertConcurrency() int { + if env := os.Getenv("NYDUSIFY_LAYER_CONCURRENCY"); env != "" { + if n, err := strconv.Atoi(env); err == nil && n > 0 { + return n + } + } + n := runtime.NumCPU() / 2 + if n < 2 { + n = 2 + } + if n > 4 { + n = 4 + } + return n +} + func convertLayer(ctx context.Context, cs content.Store, desc ocispec.Descriptor, opt nydus.PackOption) (*ocispec.Descriptor, error) { // Prepare a unique scratch area for this layer. layerDir, err := os.MkdirTemp(opt.WorkDir, "layer-") diff --git a/nydusify/internal/pipeline/multi.go b/nydusify/internal/pipeline/multi.go index e7cf1c75200..0955a46c3b5 100644 --- a/nydusify/internal/pipeline/multi.go +++ b/nydusify/internal/pipeline/multi.go @@ -412,3 +412,65 @@ func labelNydusBlobDiffIDs(ctx context.Context, cs content.Store, rootDesc ocisp } return nil } + +// labelSourceLayerDiffIDs sets the containerd.io/uncompressed content-store +// label on every source layer, taking each diff id from the image config's +// rootfs.diff_ids (1:1 with manifest layers). Without the label +// images.GetDiffID decompresses every compressed layer just to recompute a +// digest the config already carries. +func labelSourceLayerDiffIDs(ctx context.Context, cs content.Store, rootDesc ocispec.Descriptor, platformMC platforms.MatchComparer) error { + var manifestDescs []ocispec.Descriptor + switch { + case images.IsManifestType(rootDesc.MediaType): + manifestDescs = []ocispec.Descriptor{rootDesc} + case images.IsIndexType(rootDesc.MediaType): + var index ocispec.Index + if err := oci.ReadJSON(ctx, cs, rootDesc, &index); err != nil { + return errors.Wrap(err, "read index json") + } + for _, m := range index.Manifests { + if m.Platform == nil || platformMC.Match(*m.Platform) { + manifestDescs = append(manifestDescs, m) + } + } + default: + return nil + } + + for _, manifestDesc := range manifestDescs { + var manifest ocispec.Manifest + if err := oci.ReadJSON(ctx, cs, manifestDesc, &manifest); err != nil { + return errors.Wrap(err, "read manifest json") + } + var config ocispec.Image + if err := oci.ReadJSON(ctx, cs, manifest.Config, &config); err != nil { + return errors.Wrap(err, "read image config json") + } + if len(config.RootFS.DiffIDs) != len(manifest.Layers) { + // Layer/diff-id mismatch (e.g. foreign layers); leave GetDiffID + // to compute the truth. + continue + } + for i, layer := range manifest.Layers { + diffID := config.RootFS.DiffIDs[i].String() + info, err := cs.Info(ctx, layer.Digest) + if err != nil { + if errdefs.IsNotFound(err) { + continue + } + return errors.Wrapf(err, "stat layer %s", layer.Digest) + } + if info.Labels[nydus.LayerAnnotationUncompressed] == diffID { + continue + } + if info.Labels == nil { + info.Labels = map[string]string{} + } + info.Labels[nydus.LayerAnnotationUncompressed] = diffID + if _, err := cs.Update(ctx, info, "labels"); err != nil { + return errors.Wrapf(err, "label layer %s", layer.Digest) + } + } + } + return nil +} diff --git a/nydusify/internal/pipeline/tooci.go b/nydusify/internal/pipeline/tooci.go index bab64a14fdf..f69d69c4524 100644 --- a/nydusify/internal/pipeline/tooci.go +++ b/nydusify/internal/pipeline/tooci.go @@ -73,6 +73,17 @@ func convertIndexToOCI(ctx context.Context, cs content.Store, desc ocispec.Descr } for i, manifest := range index.Manifests { + // Non-nydus manifests carried by the index (e.g. buildkit + // attestation manifests with in-toto JSON layers) are passed + // through unchanged, mirroring the forward conversion. + nydusManifest, err := isNydusManifest(ctx, cs, manifest) + if err != nil { + return nil, errors.Wrapf(err, "inspect manifest %s", manifest.Digest) + } + if !nydusManifest { + labels[fmt.Sprintf("containerd.io/gc.ref.content.m.%d", i)] = manifest.Digest.String() + continue + } newDesc, err := convertManifestToOCI(ctx, cs, manifest, opt) if err != nil { return nil, errors.Wrapf(err, "convert manifest %s", manifest.Digest) @@ -85,6 +96,21 @@ func convertIndexToOCI(ctx context.Context, cs content.Store, desc ocispec.Descr return oci.WriteJSON(ctx, cs, index, desc, labels) } +// isNydusManifest reports whether the manifest holds converted nydus layers +// (data blobs plus a bootstrap), i.e. it is subject to the reverse +// conversion rather than passed through. +func isNydusManifest(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (bool, error) { + if !images.IsManifestType(desc.MediaType) { + return false, nil + } + var manifest ocispec.Manifest + if err := oci.ReadJSON(ctx, cs, desc, &manifest); err != nil { + return false, errors.Wrap(err, "read manifest json") + } + blobs, bootstrap, _, err := nydus.SplitLayers(manifest.Layers) + return err == nil && bootstrap != nil && len(blobs) > 0, nil +} + func convertManifestToOCI(ctx context.Context, cs content.Store, desc ocispec.Descriptor, opt ToOCIOption) (*ocispec.Descriptor, error) { var manifest ocispec.Manifest manifestLabels, err := oci.Labels(ctx, cs, desc.Digest) diff --git a/nydusify/internal/remote/provider.go b/nydusify/internal/remote/provider.go index 1a741558c4e..4cda13ec1fa 100644 --- a/nydusify/internal/remote/provider.go +++ b/nydusify/internal/remote/provider.go @@ -157,6 +157,18 @@ func (p *Provider) Push(ctx context.Context, desc ocispec.Descriptor, ref string return push(ctx, p.store, p.resolver(Target), desc, normalized, p.platformMC) } +// PushBlob uploads a single blob (no children walked) from the local store to +// ref, using the target registry settings. Used to overlap per-layer blob +// uploads with the remaining conversion work; the final Push then skips +// already-uploaded blobs. +func (p *Provider) PushBlob(ctx context.Context, desc ocispec.Descriptor, ref string) error { + normalized, err := normalizeRef(ref) + if err != nil { + return err + } + return push(ctx, p.store, p.resolver(Target), desc, normalized, p.platformMC) +} + // normalizeRef expands shorthand image references to fully-qualified names so // that Docker Hub shortcuts ("mariadb") and untagged references ("repo/img") // resolve correctly. For example: diff --git a/nydusify/main.go b/nydusify/main.go index 996a621f9e0..6638eed2568 100644 --- a/nydusify/main.go +++ b/nydusify/main.go @@ -20,6 +20,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + "golang.org/x/sync/errgroup" "golang.org/x/sys/unix" "github.com/dragonflyoss/nydus/nydusify/internal/checker" @@ -304,6 +305,10 @@ func runConvert(c *cli.Context) error { } logrus.Infof("converting image to nydus format") + // Converted layer blobs are uploaded as soon as they are built, so the + // registry upload overlaps the remaining layer conversions; the final + // push then skips the blobs that already landed. + var eagerPush errgroup.Group newDesc, err = pipeline.Convert(ctx, provider.ContentStore(), srcDesc, pipeline.Option{ BuilderPath: c.String("builder"), WorkDir: scratchDir, @@ -312,7 +317,19 @@ func runConvert(c *cli.Context) error { Compressor: c.String("compressor"), LogLevel: c.String("log-level"), PlatformMC: platformMC, + OnBlobConverted: func(desc ocispec.Descriptor) { + eagerPush.Go(func() error { + if err := provider.PushBlob(ctx, desc, target); err != nil { + // Non-fatal: the final push uploads it again. + logrus.Warnf("eager blob push %s: %v", desc.Digest, err) + } + return nil + }) + }, }) + // Eager pushes never return errors (failures fall back to the final + // push); waiting only bounds their lifetime. + _ = eagerPush.Wait() if err != nil { return errors.Wrap(err, "convert") } diff --git a/nydusify/pkg/nydus/footer.go b/nydusify/pkg/nydus/footer.go index 1a87ae22779..7bd8be24315 100644 --- a/nydusify/pkg/nydus/footer.go +++ b/nydusify/pkg/nydus/footer.go @@ -28,6 +28,13 @@ const ( // at offset 12: unknown incompat bits mean the footer cannot be parsed. // The version at offset 8 is informational and not gated on. footerIncompatMask = 0x0000FFFF + // footerIncompatBootstrapZstd marks the embedded bootstrap region as one zstd + // frame. Staging copies the region verbatim (nydus merge decodes it), so + // the flag is understood, not acted on, here. + footerIncompatBootstrapZstd = 1 << 0 + // footerSupportedIncompat is the set of incompat flag bits this staging + // code can pass through. + footerSupportedIncompat = footerIncompatBootstrapZstd // bootstrapOffsetField is the byte offset of the u64 bootstrap_offset field // within the footer. bootstrapOffsetField = 32 @@ -70,7 +77,7 @@ func readFooter(ra io.ReaderAt, size int64) ([]byte, error) { if string(footer[0:8]) != NydusBlobFooterMagic { return nil, errors.Errorf("not a nydus blob: bad footer magic %q", footer[0:8]) } - if incompat := binary.LittleEndian.Uint32(footer[12:16]) & footerIncompatMask; incompat != 0 { + if incompat := binary.LittleEndian.Uint32(footer[12:16]) & footerIncompatMask; incompat&^footerSupportedIncompat != 0 { return nil, errors.Errorf("unsupported nydus footer incompat flags %#x", incompat) } return footer, nil diff --git a/tests/e2e/bench_test.go b/tests/e2e/bench_test.go index 19995204204..d5b739f6af2 100644 --- a/tests/e2e/bench_test.go +++ b/tests/e2e/bench_test.go @@ -17,16 +17,39 @@ package e2e // fanotify: kernel EROFS mount over the cache files; a FAN_PRE_ACCESS // event fills missing ranges, warm reads never leave the // kernel (Linux >= 6.15). +// fileio: kernel EROFS mounted on a single FUSE-exported file +// (CONFIG_EROFS_FS_BACKED_BY_FILE); metadata resolves in-kernel +// and only cold byte ranges become FUSE reads. // erofsfuse: the C erofsfuse reference implementation reading the blob // directly (no daemon, no cache). // // Methodology (per mode): wipe the nydus cache, drop the page cache, start -// the daemon (recording mount-ready and first-1MiB-read latency), cold-read -// the whole fio target (the end-to-end on-demand fetch path — recorded as -// prewarm throughput), then run every fio job and metadata benchmark with -// the page cache dropped before each job (warm nydus cache, cold page -// cache). Unavailable modes (missing kernel module, old kernel, feature not -// compiled in) are skipped individually and their column omitted. +// the daemon, then take a sequence of measurements in which each one touches +// data or metadata this mode has not touched yet: +// +// mount ready daemon up and the tree mountable +// cold metadata walk one readdir+lstat pass over the whole tree, taken +// first so nothing has warmed it — the only row a +// bootstrap prewarm can move +// cold walk + xattr the same pass with a listxattr per entry; the delta +// is what xattrs cost on first access +// first 1MiB first-byte latency on an untouched file +// cold seq read one pass over that file, backend fetch included +// data fetched bytes pulled from the backend for exactly that file, +// snapshotted before anything else fetches +// cold rand read a non-repeating 4K random pass over a second file +// this mode has never opened +// +// Nothing here is a steady-state loop. Container start reads a rootfs once; +// looping over the same inodes for twenty seconds would report the speed of +// whichever cache ended up holding them, which after the first pass is the +// kernel's page cache in every mode — nydus's FUSE mode hands out effectively +// infinite attr/entry timeouts plus FOPEN_CACHE_DIR, so even its warm stat +// and readdir never leave the kernel. Set NYDUSFS_BENCH_WARM=1 to append the +// old steady-state fio and metadata rows for regression tracking. +// +// Unavailable modes (missing kernel module, old kernel, feature not compiled +// in) are skipped individually and their column omitted. // // Activation: NYDUSFS_RUN_BENCH=1 (set by `make test-bench`). Requires root // and fio; the NBD/ublk modes additionally need their kernel modules and @@ -40,23 +63,34 @@ package e2e // NYDUSFS_PERF_MEDIUM_FILE_COUNT Number of medium files in the corpus (default 256). // NYDUSFS_PERF_MEDIUM_FILE Size of each medium file (default 1MiB). // NYDUSFS_PERF_SMALL_FILE_COUNT Number of small files for the stat benchmark (default 10000). -// NYDUSFS_PERF_FIO_RUNTIME Fio benchmark duration in seconds (default 20). +// NYDUSFS_PERF_FIO_RUNTIME Fio benchmark duration in seconds (default 8, after a 2s ramp). // NYDUSFS_PERF_FIO_SEQ_NUMJOBS Sequential-read multi-thread job count (default 4). // NYDUSFS_PERF_FIO_RAND_NUMJOBS Random-read multi-thread job count (default 4). // NYDUSFS_PERF_READDIR_DIRS Number of directories for the readdir corpus (default 128). // NYDUSFS_PERF_READDIR_FILES_PER_DIR Files per directory for the readdir corpus (default 256). -// NYDUSFS_PERF_META_SECS Metadata benchmark duration in seconds (default 5). -// NYDUSFS_PERF_READDIR_META_SECS Readdir benchmark duration in seconds (default 5). +// NYDUSFS_PERF_META_SECS Metadata benchmark duration in seconds (default 3). +// NYDUSFS_PERF_READDIR_META_SECS Readdir benchmark duration in seconds (default 3). // NYDUSFS_PERF_READDIR_PASSES_PER_DIR Repeated os.ReadDir calls per directory per iteration (default 8). +// NYDUSFS_PERF_COLD_RAND_IO_SIZE Bytes read by the cold random-read pass (default 16MiB). +// NYDUSFS_BENCH_MODES Comma-separated mode names to run; others are skipped. +// NYDUSFS_BENCH_WARM Also run and print the steady-state fio/metadata rows. +// NYDUSFS_BENCH_NOCDC_FUSE Add the fuse-nocdc column (image built with --deduplicator none). +// NYDUSFS_BENCH_FILEIO_NOWARM Add the fileio-nowarm column (fileio without the FUSE_NOTIFY_STORE prewarm). +// NYDUSFS_BENCH_V2_NYDUSD, NYDUSFS_BENCH_V2_IMAGE_BIN Paths to nydus v2 binaries; both set adds the v2-fuse column. import ( "bufio" + "context" "encoding/json" "fmt" + "io" "io/fs" + "net" + "net/http" "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "testing" @@ -66,11 +100,13 @@ import ( "github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/text" "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" ) // Fixed corpus layout produced by corpus.MakePerfCorpus. const ( benchTargetRel = "large/file_0.bin" + benchRandRel = "large/file_1.bin" benchStatRel = "small" benchReaddirRel = "dirs" benchXattrRel = "xattrs" @@ -89,6 +125,18 @@ type benchEnv struct { bootstrap string blobPath string // single blob produced by the build, for erofsfuse --device nbdDev string // free /dev/nbdX picked for the NBD mode + + // Optional fuse-nocdc comparison column (--deduplicator none image), + // enabled by the NYDUSFS_BENCH_NOCDC_FUSE environment variable. + nocdcBootstrap string + nocdcBlobDir string + + // Optional nydus v2 (rafs v6) comparison column, enabled by the + // NYDUSFS_BENCH_V2_NYDUSD and NYDUSFS_BENCH_V2_IMAGE_BIN environment variables. + v2Nydusd string + v2ImageBin string + v2Bootstrap string + v2BlobDir string } // benchMode is one serving mode: how to detect availability and how to @@ -103,11 +151,40 @@ type benchMode struct { // benchModeResult collects every measurement for one mode. type benchModeResult struct { - mountSec float64 - firstReadSec float64 - prewarmMiBps float64 - fetchedMiB float64 // < 0 when the mode has no cache - bench map[string]*benchResult + mountSec float64 + coldWalkSec float64 + coldWalkEntries int + coldXattrWalkSec float64 + negativeLookupOps float64 + firstReadSec float64 + coldSeqMiBps float64 + coldSeqCachedMiB float64 // page cache growth across the cold read + fetchedMiB float64 // < 0 when the mode has no cache + coldRandIOPS float64 + coldRandLatUs float64 + bench map[string]*benchResult // nil unless NYDUSFS_BENCH_WARM is set +} + +// cachedMiB reports the kernel's total page cache from /proc/meminfo. +// +// Serving one byte can cache it more than once: the daemon's own copy of the +// blob cache file, an intermediate object (the FUSE export, a block device), +// and finally the inode the application reads. Growth across a cold read of a +// known size is the cheapest way to see how many of those copies survive. +func cachedMiB(t *testing.T) float64 { + t.Helper() + data, err := os.ReadFile("/proc/meminfo") + require.NoError(t, err) + for _, line := range strings.Split(string(data), "\n") { + if !strings.HasPrefix(line, "Cached:") { + continue + } + var kb float64 + if _, err := fmt.Sscanf(strings.TrimPrefix(line, "Cached:"), "%f kB", &kb); err == nil { + return kb / 1024 + } + } + return -1 } func TestBench(t *testing.T) { @@ -132,7 +209,46 @@ func TestBench(t *testing.T) { t.Log("Building NydusFS image (chunksize=1MiB)...") e.blobPath = buildNydusFSImageToDir(t, e.nydusBin, e.bootstrap, e.blobDir, corpusDir, 1024*1024) + if os.Getenv("NYDUSFS_BENCH_NOCDC_FUSE") != "" { + t.Log("Building NydusFS image with --deduplicator none...") + e.nocdcBlobDir = filepath.Join(e.workDir, "nocdc-blobs") + e.nocdcBootstrap = filepath.Join(e.workDir, "nocdc.boot") + buildNydusFSImageToDir(t, e.nydusBin, e.nocdcBootstrap, e.nocdcBlobDir, corpusDir, 1024*1024, + "--deduplicator", "none") + } + + e.v2Nydusd = os.Getenv("NYDUSFS_BENCH_V2_NYDUSD") + e.v2ImageBin = os.Getenv("NYDUSFS_BENCH_V2_IMAGE_BIN") + if e.v2Nydusd != "" && e.v2ImageBin != "" { + t.Log("Building nydus v2 (rafs v6) image...") + e.v2BlobDir = filepath.Join(e.workDir, "v2-blobs") + e.v2Bootstrap = filepath.Join(e.workDir, "v2.boot") + require.NoError(t, os.MkdirAll(e.v2BlobDir, 0755)) + out, err := exec.Command(e.v2ImageBin, "create", + "--fs-version", "6", "--compressor", "zstd", + "--blob-id", "v2blob", + "--blob", filepath.Join(e.v2BlobDir, "v2blob"), + "--external-blob", filepath.Join(e.workDir, "v2.ext"), + "--bootstrap", e.v2Bootstrap, + corpusDir).CombinedOutput() + require.NoError(t, err, "nydus-image create failed: %s", string(out)) + } + modes := e.buildModes(t) + // NYDUSFS_BENCH_MODES=fileio,fileio-directio runs just those columns — + // full runs cost minutes per mode, so comparing two variants should not + // pay for the other five. + if filter := os.Getenv("NYDUSFS_BENCH_MODES"); filter != "" { + wanted := make(map[string]bool) + for _, name := range strings.Split(filter, ",") { + wanted[strings.TrimSpace(name)] = true + } + for _, m := range modes { + if m.skip == "" && !wanted[m.name] { + m.skip = "filtered out by NYDUSFS_BENCH_MODES" + } + } + } results := make(map[string]*benchModeResult, len(modes)) for _, m := range modes { if m.skip != "" { @@ -147,8 +263,9 @@ func TestBench(t *testing.T) { printBenchTable(t, modes, results) } -// runMode measures one mode end to end: cold start, prewarm, then the cold -// page-cache benchmark suite. +// runMode measures one mode end to end. Every measurement below touches data +// or metadata this mode has not touched yet, with the page cache dropped +// beforehand, because first access is what decides container start time. func (e *benchEnv) runMode(t *testing.T, m *benchMode) *benchModeResult { t.Helper() if m.cache != "" { @@ -157,38 +274,160 @@ func (e *benchEnv) runMode(t *testing.T, m *benchMode) *benchModeResult { dropCaches(t) res := &benchModeResult{fetchedMiB: -1} - mountStart := time.Now() - stop := m.start(t) - defer stop() - res.mountSec = time.Since(mountStart).Seconds() - t.Logf(" mount ready: %.2fs", res.mountSec) - - target := filepath.Join(m.mnt, benchTargetRel) - readStart := time.Now() - _, err := readSlice(target, 0, 1) - require.NoError(t, err, "first cold read failed") - res.firstReadSec = time.Since(readStart).Seconds() - t.Logf(" first 1MiB cold read: %.3fs", res.firstReadSec) - - // Cold-read the whole fio target: fills the nydus cache and block group map for - // every byte fio will touch, and IS the end-to-end on-demand fetch path. - prewarmStart := time.Now() - n := readWhole(t, target) - res.prewarmMiBps = float64(n) / (1 << 20) / time.Since(prewarmStart).Seconds() - t.Logf(" prewarm: %.1f MiB cold-read at %.1f MiB/s", float64(n)/(1<<20), res.prewarmMiBps) - - statDir := filepath.Join(m.mnt, benchStatRel) - res.bench = runBenchmarks(t, e.fioBin, target, statDir, filepath.Join(m.mnt, benchReaddirRel)) - dropCaches(t) - addMetaBenchmarks(t, res.bench, filepath.Join(m.mnt, benchXattrRel), benchXattrName, statDir) + func() { + mountStart := time.Now() + stop := m.start(t) + defer stop() + res.mountSec = time.Since(mountStart).Seconds() + t.Logf(" mount ready: %.2fs", res.mountSec) + + // First, before anything else touches the tree: this is the only + // measurement a bootstrap prewarm can move. + res.coldWalkSec, res.coldWalkEntries = coldMetadataWalk(t, m.mnt, false) + t.Logf(" cold metadata walk: %d entries in %.3fs (%.0f entries/s)", + res.coldWalkEntries, res.coldWalkSec, float64(res.coldWalkEntries)/res.coldWalkSec) + + res.negativeLookupOps = benchNegativeLookup(t, filepath.Join(m.mnt, benchStatRel)) + t.Logf(" negative lookups: %.0f ops/s", res.negativeLookupOps) + + dropCaches(t) + target := filepath.Join(m.mnt, benchTargetRel) + readStart := time.Now() + _, err := readSlice(target, 0, 1) + require.NoError(t, err, "first cold read failed") + res.firstReadSec = time.Since(readStart).Seconds() + t.Logf(" first 1MiB cold read: %.3fs", res.firstReadSec) + + // Single cold pass over a file nothing has read: the end-to-end + // on-demand fetch path, backend included. + cachedBefore := cachedMiB(t) + seqStart := time.Now() + n := readWhole(t, target) + res.coldSeqMiBps = float64(n) / (1 << 20) / time.Since(seqStart).Seconds() + res.coldSeqCachedMiB = cachedMiB(t) - cachedBefore + t.Logf(" cold seq read: %.1f MiB at %.1f MiB/s, page cache +%.1f MiB", + float64(n)/(1<<20), res.coldSeqMiBps, res.coldSeqCachedMiB) + + // Snapshot here, while the only data ever fetched is that one file: + // anything measured later would fold its fetches into the number. + // Backend counters from the daemon's metrics endpoint where available + // (v3 daemons); cache-file disk allocation otherwise. + if fetched := backendReadMiB(e.apiSocket(m.name)); fetched >= 0 { + res.fetchedMiB = fetched + t.Logf(" backend bytes for %.1f MiB of file: %.1f MiB", float64(n)/(1<<20), res.fetchedMiB) + } else if m.cache != "" { + res.fetchedMiB = cacheDirUsedBytes(m.cache) + t.Logf(" cache allocation for %.1f MiB of file: %.1f MiB", float64(n)/(1<<20), res.fetchedMiB) + } + + dropCaches(t) + res.coldRandIOPS, res.coldRandLatUs = coldRandRead(t, e.fioBin, filepath.Join(m.mnt, benchRandRel)) + t.Logf(" cold rand read 4K: %.0f IOPS, %.0f µs", res.coldRandIOPS, res.coldRandLatUs) + + if os.Getenv("NYDUSFS_BENCH_WARM") != "" { + statDir := filepath.Join(m.mnt, benchStatRel) + res.bench = runBenchmarks(t, e.fioBin, target, statDir, filepath.Join(m.mnt, benchReaddirRel)) + dropCaches(t) + addMetaBenchmarks(t, res.bench, filepath.Join(m.mnt, benchXattrRel), benchXattrName, statDir) + } + }() + + // The xattr pass needs its own mount to be comparable with the plain + // walk above: a second walk inside one session finds the daemon's own + // mappings already built, and dropCaches cannot evict those. if m.cache != "" { - res.fetchedMiB = cacheDirUsedBytes(m.cache) + wipeCacheDir(m.cache) } + dropCaches(t) + func() { + stop := m.start(t) + defer stop() + res.coldXattrWalkSec, _ = coldMetadataWalk(t, m.mnt, true) + t.Logf(" cold walk + xattr: %.3fs (xattr delta %+.3fs)", + res.coldXattrWalkSec, res.coldXattrWalkSec-res.coldWalkSec) + }() return res } // ------------------------------------------------------------------ modes ---- +// coldMetadataWalk enumerates the whole mount once with the page cache cold: +// readdir every directory and lstat every entry, optionally adding a +// listxattr per entry. +// +// Container start enumerates a rootfs once; it does not stat the same inode a +// million times. A steady-state loop would instead report whichever cache +// ended up holding the metadata, which for every mode here is the kernel's +// — nydus's FUSE mode hands out effectively infinite attr/entry timeouts and +// FOPEN_CACHE_DIR, so its warm stat and readdir are in-kernel too. +func coldMetadataWalk(t *testing.T, root string, withXattr bool) (float64, int) { + t.Helper() + entries := 0 + buf := make([]byte, 4096) + start := time.Now() + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + // WalkDir fills DirEntry from getdents alone; Info forces the lstat. + if _, err := d.Info(); err != nil { + return err + } + if withXattr { + _, _ = unix.Llistxattr(path, buf) + } + entries++ + return nil + }) + elapsed := time.Since(start).Seconds() + require.NoError(t, err) + require.NotZero(t, entries) + return elapsed, entries +} + +// benchNegativeLookup stats the same set of names that do not exist, over and +// over, for one second. This is what module resolution does to a rootfs — +// Node's require walk and Python's sys.path scan probe far more missing paths +// than existing ones, and they retry the same misses on every import. A +// filesystem that caches negative dentries answers repeats from the dcache; one +// that does not pays a round trip per probe, forever. +func benchNegativeLookup(t *testing.T, dir string) float64 { + t.Helper() + names := make([]string, 64) + for i := range names { + names[i] = filepath.Join(dir, fmt.Sprintf("no_such_module_%d.js", i)) + } + ops := 0 + start := time.Now() + deadline := start.Add(time.Second) + for time.Now().Before(deadline) { + for _, name := range names { + if _, err := os.Lstat(name); err == nil { + t.Fatalf("%s unexpectedly exists", name) + } + } + ops += len(names) + } + return float64(ops) / time.Since(start).Seconds() +} + +// coldRandRead does a single non-repeating pass of random 4K reads over a +// file this mode has never touched, so every I/O is an on-demand fetch. +// fio's random map keeps blocks from repeating, which keeps the page cache +// out of the measurement. +func coldRandRead(t *testing.T, fioBin, file string) (float64, float64) { + t.Helper() + require.FileExists(t, file) + ioSize := corpus.EnvInt("NYDUSFS_PERF_COLD_RAND_IO_SIZE", 16*1024*1024) + res := runFio(t, fioBin, []string{ + "--name=cold_rand", "--filename=" + file, + "--rw=randread", "--bs=4k", "--direct=0", "--invalidate=0", + fmt.Sprintf("--io_size=%d", ioSize), + "--numjobs=1", "--readonly", + }) + return res.ReadIOPS, res.ReadLat +} + // buildModes prepares the per-mode dirs/configs and probes availability. // A mode that cannot run on this host gets a skip reason instead of failing // the whole benchmark. @@ -210,13 +449,23 @@ func (e *benchEnv) buildModes(t *testing.T) []*benchMode { } require.NoError(t, os.MkdirAll(m.mnt, 0755)) require.NoError(t, os.MkdirAll(m.cache, 0755)) - e.writeConfig(t, name, m.cache) + e.writeConfig(t, name, e.blobDir, m.cache) return m } fuse := newMode("fuse", e.startFuse) fuse.skip = subOK("fuse") + // Same fuse serving path, but the image was built with --deduplicator + // none (chunkless blob meta): isolates the CDC lookup/dedup cost. + nocdc := newMode("fuse-nocdc", e.startFuseNocdc) + if nocdc.skip = subOK("fuse"); nocdc.skip == "" && e.nocdcBootstrap == "" { + nocdc.skip = "set NYDUSFS_BENCH_NOCDC_FUSE=1 to enable the fuse-nocdc column" + } + if nocdc.skip == "" { + e.writeConfig(t, "fuse-nocdc", e.nocdcBlobDir, nocdc.cache) + } + nbd := newMode("nbd", e.startNbd) if nbd.skip = subOK("nbd"); nbd.skip == "" { if !erofsSupported() { @@ -244,6 +493,28 @@ func (e *benchEnv) buildModes(t *testing.T) []*benchMode { } } + fileio := newMode("fileio", e.startFileio) + if fileio.skip = subOK("fileio"); fileio.skip == "" { + if !erofsSupported() { + fileio.skip = "erofs not in /proc/filesystems — kernel lacks EROFS support" + } else if maj, min := kernelVersion(t); maj < 6 || (maj == 6 && min < 12) { + fileio.skip = fmt.Sprintf("kernel %d.%d < 6.12: CONFIG_EROFS_FS_BACKED_BY_FILE unavailable", maj, min) + } + } + require.NoError(t, os.MkdirAll(filepath.Join(e.workDir, "fileio-export"), 0755)) + + // Same serving path with the bootstrap prewarm off: the delta against the + // fileio column is what FUSE_NOTIFY_STORE is worth. + fileioNoWarm := newMode("fileio-nowarm", e.startFileioNoWarm) + if fileioNoWarm.skip = fileio.skip; fileioNoWarm.skip == "" && os.Getenv("NYDUSFS_BENCH_FILEIO_NOWARM") == "" { + fileioNoWarm.skip = "set NYDUSFS_BENCH_FILEIO_NOWARM=1 to enable the fileio-nowarm column" + } + require.NoError(t, os.MkdirAll(filepath.Join(e.workDir, "fileio-nowarm-export"), 0755)) + + fileioBuffered := newMode("fileio-buffered", e.startFileioBuffered) + fileioBuffered.skip = fileio.skip + require.NoError(t, os.MkdirAll(filepath.Join(e.workDir, "fileio-buffered-export"), 0755)) + cerofs := &benchMode{ name: benchErofsfuse, mnt: filepath.Join(e.workDir, "erofsfuse-mnt"), @@ -254,17 +525,67 @@ func (e *benchEnv) buildModes(t *testing.T) []*benchMode { cerofs.skip = err.Error() } - return []*benchMode{fuse, nbd, ublk, fan, cerofs} + v2 := &benchMode{ + name: "v2-fuse", + mnt: filepath.Join(e.workDir, "v2-mnt"), + cache: filepath.Join(e.workDir, "v2-cache"), + start: e.startV2Fuse, + } + require.NoError(t, os.MkdirAll(v2.mnt, 0755)) + require.NoError(t, os.MkdirAll(v2.cache, 0755)) + if e.v2Bootstrap == "" { + v2.skip = "set NYDUSFS_BENCH_V2_NYDUSD and NYDUSFS_BENCH_V2_IMAGE_BIN to enable the v2 column" + } + + return []*benchMode{v2, fuse, nocdc, nbd, ublk, fan, fileio, fileioNoWarm, fileioBuffered, cerofs} } -// writeConfig writes one local-backend storage config per mode: shared blob -// dir, separate cache dirs. Prefetch stays disabled — a background prefetch +func (e *benchEnv) startV2Fuse(t *testing.T) func() { + t.Helper() + mnt := filepath.Join(e.workDir, "v2-mnt") + config := fmt.Sprintf(`{ + "device": { + "backend": { "type": "localfs", "config": { "dir": %q } }, + "cache": { "type": "blobcache", "config": { "work_dir": %q } } + }, + "mode": "direct", + "digest_validate": false, + "enable_xattr": true, + "iostats_files": false +}`, e.v2BlobDir, filepath.Join(e.workDir, "v2-cache")) + configPath := filepath.Join(e.workDir, "v2.json") + require.NoError(t, os.WriteFile(configPath, []byte(config), 0644)) + cmd := exec.Command(e.v2Nydusd, + "--config", configPath, + "--bootstrap", e.v2Bootstrap, + "--mountpoint", mnt, + "--log-level", "error", + ) + exited := spawnDaemon(t, cmd, e.logPath("v2-fuse")) + require.Eventually(t, func() bool { + select { + case <-exited: + require.FailNowf(t, "v2 nydusd exited during startup", "%s", readFileOrEmpty(e.logPath("v2-fuse"))) + default: + } + return isMountpoint(mnt) + }, 60*time.Second, 200*time.Millisecond, "v2 nydusd did not mount:\n%s", readFileOrEmpty(e.logPath("v2-fuse"))) + return func() { + terminateDaemon(cmd, exited) + if isMountpoint(mnt) { + unmountFuse(mnt) + } + } +} + +// writeConfig writes one local-backend storage config per mode: per-mode blob +// dir and separate cache dirs. Prefetch stays disabled — a background prefetch // would warm the cache mid-run and corrupt the cold measurements. -func (e *benchEnv) writeConfig(t *testing.T, name, cacheDir string) { +func (e *benchEnv) writeConfig(t *testing.T, name, blobDir, cacheDir string) { t.Helper() config := fmt.Sprintf( "backend:\n type: local\n config:\n dir: %s\nstorage:\n dir: %s\nprefetch:\n scope: none\n", - e.blobDir, cacheDir, + blobDir, cacheDir, ) require.NoError(t, os.WriteFile(e.configPath(name), []byte(config), 0644)) } @@ -273,6 +594,60 @@ func (e *benchEnv) configPath(name string) string { return filepath.Join(e.workDir, name+".yaml") } +// apiSocket is where mode `name`'s daemon serves Prometheus /metrics. +func (e *benchEnv) apiSocket(name string) string { + return filepath.Join(e.workDir, name+"-api.sock") +} + +// backendReadMiB sums the backend_*_read_bytes counters from the daemon's +// metrics endpoint: bytes actually pulled from the backend, as opposed to +// cache-file disk allocation, which a 4 KiB block per tiny record inflates +// (a decoded block group publishes its records at block-aligned logical +// offsets, so `du` reports about 3x the real transfer on this corpus). +// Returns -1 when the socket is absent or unreadable. +func backendReadMiB(socket string) float64 { + client := http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socket) + }, + }, + Timeout: 5 * time.Second, + } + resp, err := client.Get("http://unix/metrics") + if err != nil { + return -1 + } + defer func() { _ = resp.Body.Close() }() + data, err := io.ReadAll(resp.Body) + if err != nil { + return -1 + } + var total float64 + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + continue + } + // Only the by-kind counters: origin/proxy count the same bytes + // again by source, so including them doubles the total. + switch fields[0] { + case "backend_ondemand_read_bytes", "backend_prefetch_read_bytes", + "backend_redirect_read_bytes": + default: + continue + } + if v, err := strconv.ParseFloat(fields[1], 64); err == nil { + total += v + } + } + return total / (1024 * 1024) +} + func (e *benchEnv) logPath(name string) string { return filepath.Join(e.workDir, name+".log") } @@ -321,22 +696,31 @@ func terminateDaemon(cmd *exec.Cmd, exited chan struct{}) { } func (e *benchEnv) startFuse(t *testing.T) func() { + return e.startFuseNamed(t, "fuse", e.bootstrap) +} + +func (e *benchEnv) startFuseNocdc(t *testing.T) func() { + return e.startFuseNamed(t, "fuse-nocdc", e.nocdcBootstrap) +} + +func (e *benchEnv) startFuseNamed(t *testing.T, name, bootstrap string) func() { t.Helper() - mnt := filepath.Join(e.workDir, "fuse-mnt") + mnt := filepath.Join(e.workDir, name+"-mnt") cmd := exec.Command(e.nydusBin, "fuse", - "--bootstrap", e.bootstrap, - "--config", e.configPath("fuse"), + "--bootstrap", bootstrap, + "--config", e.configPath(name), "--mountpoint", mnt, + "--apiserver", "unix://"+e.apiSocket(name), ) - exited := spawnDaemon(t, cmd, e.logPath("fuse")) + exited := spawnDaemon(t, cmd, e.logPath(name)) require.Eventually(t, func() bool { select { case <-exited: - require.FailNowf(t, "fuse daemon exited during startup", "%s", readFileOrEmpty(e.logPath("fuse"))) + require.FailNowf(t, "fuse daemon exited during startup", "%s", readFileOrEmpty(e.logPath(name))) default: } return isMountpoint(mnt) - }, 60*time.Second, 200*time.Millisecond, "fuse daemon did not mount:\n%s", readFileOrEmpty(e.logPath("fuse"))) + }, 60*time.Second, 200*time.Millisecond, "fuse daemon did not mount:\n%s", readFileOrEmpty(e.logPath(name))) return func() { terminateDaemon(cmd, exited) if isMountpoint(mnt) { @@ -385,6 +769,7 @@ func (e *benchEnv) startUblk(t *testing.T) func() { "--config", e.configPath("ublk"), "--log-level", "info", "--log-dir", e.logDir("ublk"), + "--apiserver", "unix://"+e.apiSocket("ublk"), ) stdoutPipe, err := cmd.StdoutPipe() require.NoError(t, err) @@ -473,6 +858,66 @@ func (e *benchEnv) startFanotify(t *testing.T) func() { } } +// startFileio exports the flattened image over FUSE and lets the kernel EROFS +// driver mount that file directly, so the daemon owns both mounts and tears +// them down in dependency order on shutdown. +func (e *benchEnv) startFileio(t *testing.T) func() { + return e.startFileioMode(t, "fileio", true, true) +} + +// startFileioNoWarm serves the same image with the FUSE_NOTIFY_STORE bootstrap +// push disabled, so the pair of columns isolates what that prewarm buys. +func (e *benchEnv) startFileioNoWarm(t *testing.T) func() { + return e.startFileioMode(t, "fileio-nowarm", false, true) +} + +// startFileioBuffered mounts without EROFS 'directio' (the daemon's default +// is on); the delta against the fileio column is what holding the image in +// the export's page cache as well buys and costs. +func (e *benchEnv) startFileioBuffered(t *testing.T) func() { + return e.startFileioMode(t, "fileio-buffered", true, false) +} + +func (e *benchEnv) startFileioMode(t *testing.T, name string, warmBootstrap, directIO bool) func() { + t.Helper() + mnt := filepath.Join(e.workDir, name+"-mnt") + export := filepath.Join(e.workDir, name+"-export") + cmd := exec.Command(e.nydusBin, "fileio", + "--bootstrap", e.bootstrap, + "--config", e.configPath(name), + "--export-dir", export, + "--mountpoint", mnt, + "--warm-bootstrap", fmt.Sprintf("%t", warmBootstrap), + "--direct-io", fmt.Sprintf("%t", directIO), + "--log-level", "info", + "--log-dir", e.logDir(name), + "--apiserver", "unix://"+e.apiSocket(name), + ) + exited := spawnDaemon(t, cmd, e.logPath(name)) + require.Eventually(t, func() bool { + select { + case <-exited: + require.FailNowf(t, "fileio daemon exited during startup", + "%s", readFileOrEmpty(e.logPath(name))) + default: + } + return isMountpoint(mnt) + }, 60*time.Second, 200*time.Millisecond, "fileio daemon did not mount:\n%s", + readFileOrEmpty(e.logPath(name))) + return func() { + // The daemon unmounts EROFS before ending the export; the fallbacks + // only matter if it died without running its shutdown path. + terminateDaemon(cmd, exited) + if isMountpoint(mnt) { + _ = exec.Command("umount", mnt).Run() + _ = exec.Command("umount", "-l", mnt).Run() + } + if isMountpoint(export) { + _ = exec.Command("umount", "-l", export).Run() + } + } +} + func (e *benchEnv) startErofsfuse(t *testing.T) func() { t.Helper() bin := mustLookupCErofsFuse(t) @@ -554,20 +999,49 @@ func printBenchTable(t *testing.T, modes []*benchMode, results map[string]*bench appendRow("Mount ready", func(res *benchModeResult) string { return fmt.Sprintf("%.2f s", res.mountSec) }) + appendRow("Cold metadata walk", func(res *benchModeResult) string { + return fmt.Sprintf("%.3f s", res.coldWalkSec) + }) + appendRow("Cold walk rate", func(res *benchModeResult) string { + if res.coldWalkSec <= 0 { + return "—" + } + return fmt.Sprintf("%.0f ent/s", float64(res.coldWalkEntries)/res.coldWalkSec) + }) + appendRow("Cold walk + xattr", func(res *benchModeResult) string { + return fmt.Sprintf("%.3f s", res.coldXattrWalkSec) + }) + appendRow(" xattr delta", func(res *benchModeResult) string { + return fmt.Sprintf("%+.3f s", res.coldXattrWalkSec-res.coldWalkSec) + }) + appendRow("Negative lookup ops/s", func(res *benchModeResult) string { + return fmt.Sprintf("%.0f op/s", res.negativeLookupOps) + }) appendRow("First 1MiB cold read", func(res *benchModeResult) string { return fmt.Sprintf("%.3f s", res.firstReadSec) }) - appendRow("Prewarm (full-file cold fetch)", func(res *benchModeResult) string { - return fmt.Sprintf("%.1f MiB/s", res.prewarmMiBps) + appendRow("Cold seq read (64MiB)", func(res *benchModeResult) string { + return fmt.Sprintf("%.1f MiB/s", res.coldSeqMiBps) }) - appendRow("Data fetched (prewarm)", func(res *benchModeResult) string { + appendRow(" page cache used", func(res *benchModeResult) string { + return fmt.Sprintf("%.1f MiB", res.coldSeqCachedMiB) + }) + appendRow("Backend read (that 64MiB)", func(res *benchModeResult) string { if res.fetchedMiB < 0 { return "—" } return fmt.Sprintf("%.1f MiB", res.fetchedMiB) }) + appendRow("Cold rand read IOPS (4K)", func(res *benchModeResult) string { + return fmt.Sprintf("%.0f IOPS", res.coldRandIOPS) + }) + appendRow("Cold rand read Lat (4K)", func(res *benchModeResult) string { + return fmt.Sprintf("%.0f µs", res.coldRandLatUs) + }) - tw.AppendSeparator() + // Steady-state rows: only populated under NYDUSFS_BENCH_WARM. They say + // little about container start, and once every mode's metadata is in the + // kernel's caches they largely measure the same page-cache path. rows := []row{ {"Seq Read BW (128K)", "seq_read_128k", "MiB/s", bw}, {"Rand Read BW (128K)", "rand_read_128k", "MiB/s", bw}, @@ -589,13 +1063,16 @@ func printBenchTable(t *testing.T, modes []*benchMode, results map[string]*bench {"Readdir+stat (ls -l) IOPS", "readdir_stat", "IOPS", iops}, {"Readdir+stat (ls -l) Latency", "readdir_stat", "µs", lat}, } - for _, r := range rows { - appendRow(r.label, func(res *benchModeResult) string { - if br, ok := res.bench[r.key]; ok && br != nil { - return fmt.Sprintf("%.1f %s", r.get(br), r.unit) - } - return "—" - }) + if os.Getenv("NYDUSFS_BENCH_WARM") != "" { + tw.AppendSeparator() + for _, r := range rows { + appendRow(r.label, func(res *benchModeResult) string { + if br, ok := res.bench[r.key]; ok && br != nil { + return fmt.Sprintf("%.1f %s", r.get(br), r.unit) + } + return "—" + }) + } } var skipped []string @@ -622,7 +1099,7 @@ func printBenchTable(t *testing.T, modes []*benchMode, results map[string]*bench func runBenchmarks(t *testing.T, fioBin, targetFile, statDir, readdirDir string) map[string]*benchResult { require.FileExists(t, targetFile) - fioRuntime := corpus.EnvInt("NYDUSFS_PERF_FIO_RUNTIME", 20) + fioRuntime := corpus.EnvInt("NYDUSFS_PERF_FIO_RUNTIME", 8) fioSeqNumjobs := corpus.EnvInt("NYDUSFS_PERF_FIO_SEQ_NUMJOBS", 4) fioRandNumjobs := corpus.EnvInt("NYDUSFS_PERF_FIO_RAND_NUMJOBS", 4) results := make(map[string]*benchResult) @@ -631,35 +1108,35 @@ func runBenchmarks(t *testing.T, fioBin, targetFile, statDir, readdirDir string) results["seq_read_128k"] = runFio(t, fioBin, []string{ "--name=seq_read", "--filename=" + targetFile, "--rw=read", "--bs=128k", "--direct=0", "--invalidate=0", - "--numjobs=1", fmt.Sprintf("--runtime=%d", fioRuntime), "--time_based", "--readonly", + "--numjobs=1", fmt.Sprintf("--runtime=%d", fioRuntime), "--ramp_time=2", "--time_based", "--readonly", }) dropCaches(t) results["rand_read_128k"] = runFio(t, fioBin, []string{ "--name=rand_read", "--filename=" + targetFile, "--rw=randread", "--bs=128k", "--direct=0", "--invalidate=0", - "--numjobs=1", fmt.Sprintf("--runtime=%d", fioRuntime), "--time_based", "--readonly", + "--numjobs=1", fmt.Sprintf("--runtime=%d", fioRuntime), "--ramp_time=2", "--time_based", "--readonly", }) dropCaches(t) results["seq_read_4k"] = runFio(t, fioBin, []string{ "--name=seq_read_4k", "--filename=" + targetFile, "--rw=read", "--bs=4k", "--direct=0", "--invalidate=0", - "--numjobs=1", fmt.Sprintf("--runtime=%d", fioRuntime), "--time_based", "--readonly", + "--numjobs=1", fmt.Sprintf("--runtime=%d", fioRuntime), "--ramp_time=2", "--time_based", "--readonly", }) dropCaches(t) results["rand_read_4k"] = runFio(t, fioBin, []string{ "--name=rand_read_4k", "--filename=" + targetFile, "--rw=randread", "--bs=4k", "--direct=0", "--invalidate=0", - "--numjobs=1", fmt.Sprintf("--runtime=%d", fioRuntime), "--time_based", "--readonly", + "--numjobs=1", fmt.Sprintf("--runtime=%d", fioRuntime), "--ramp_time=2", "--time_based", "--readonly", }) dropCaches(t) results["seq_read_4t_128k"] = runFio(t, fioBin, []string{ "--name=seq_read_4t", "--filename=" + targetFile, "--rw=read", "--bs=128k", "--direct=0", "--invalidate=0", - fmt.Sprintf("--numjobs=%d", fioSeqNumjobs), fmt.Sprintf("--runtime=%d", fioRuntime), "--time_based", + fmt.Sprintf("--numjobs=%d", fioSeqNumjobs), fmt.Sprintf("--runtime=%d", fioRuntime), "--ramp_time=2", "--time_based", "--readonly", "--group_reporting", }) @@ -667,7 +1144,7 @@ func runBenchmarks(t *testing.T, fioBin, targetFile, statDir, readdirDir string) results["rand_read_4t_128k"] = runFio(t, fioBin, []string{ "--name=rand_read_4t", "--filename=" + targetFile, "--rw=randread", "--bs=128k", "--direct=0", "--invalidate=0", - fmt.Sprintf("--numjobs=%d", fioRandNumjobs), fmt.Sprintf("--runtime=%d", fioRuntime), "--time_based", + fmt.Sprintf("--numjobs=%d", fioRandNumjobs), fmt.Sprintf("--runtime=%d", fioRuntime), "--ramp_time=2", "--time_based", "--readonly", "--group_reporting", }) @@ -675,7 +1152,7 @@ func runBenchmarks(t *testing.T, fioBin, targetFile, statDir, readdirDir string) results["seq_read_4t_4k"] = runFio(t, fioBin, []string{ "--name=seq_read_4t", "--filename=" + targetFile, "--rw=read", "--bs=4k", "--direct=0", "--invalidate=0", - fmt.Sprintf("--numjobs=%d", fioSeqNumjobs), fmt.Sprintf("--runtime=%d", fioRuntime), "--time_based", + fmt.Sprintf("--numjobs=%d", fioSeqNumjobs), fmt.Sprintf("--runtime=%d", fioRuntime), "--ramp_time=2", "--time_based", "--readonly", "--group_reporting", }) @@ -683,7 +1160,7 @@ func runBenchmarks(t *testing.T, fioBin, targetFile, statDir, readdirDir string) results["rand_read_4t_4k"] = runFio(t, fioBin, []string{ "--name=rand_read_4t", "--filename=" + targetFile, "--rw=randread", "--bs=4k", "--direct=0", "--invalidate=0", - fmt.Sprintf("--numjobs=%d", fioRandNumjobs), fmt.Sprintf("--runtime=%d", fioRuntime), "--time_based", + fmt.Sprintf("--numjobs=%d", fioRandNumjobs), fmt.Sprintf("--runtime=%d", fioRuntime), "--ramp_time=2", "--time_based", "--readonly", "--group_reporting", }) @@ -698,7 +1175,7 @@ func runBenchmarks(t *testing.T, fioBin, targetFile, statDir, readdirDir string) // benchStat repeatedly stats every file in dir for the configured metadata duration and // reports the achieved ops/s and latency. func benchStat(t *testing.T, dir string) *benchResult { - metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_META_SECS", 5)) * time.Second + metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_META_SECS", 3)) * time.Second var files []string _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { @@ -733,7 +1210,7 @@ func benchStat(t *testing.T, dir string) *benchResult { // benchReaddir repeatedly reads every subdirectory of dir for the configured // metadata duration and reports the achieved ops/s and latency. func benchReaddir(t *testing.T, dir string) *benchResult { - metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_READDIR_META_SECS", 5)) * time.Second + metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_READDIR_META_SECS", 3)) * time.Second passesPerDir := corpus.EnvInt("NYDUSFS_PERF_READDIR_PASSES_PER_DIR", 8) entries, err := os.ReadDir(dir) @@ -786,7 +1263,7 @@ func addMetaBenchmarks(t *testing.T, results map[string]*benchResult, xattrDir, // the configured metadata duration and reports the achieved ops/s and latency. // Returns nil when no regular files are found. func benchListxattr(t *testing.T, dir string) *benchResult { - metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_META_SECS", 5)) * time.Second + metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_META_SECS", 3)) * time.Second var files []string _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { @@ -823,7 +1300,7 @@ func benchListxattr(t *testing.T, dir string) *benchResult { // for the configured metadata duration and reports the achieved ops/s and // latency. Returns nil when no regular files are found. func benchGetxattr(t *testing.T, dir, xattrName string) *benchResult { - metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_META_SECS", 5)) * time.Second + metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_META_SECS", 3)) * time.Second var files []string _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { @@ -860,7 +1337,7 @@ func benchGetxattr(t *testing.T, dir, xattrName string) *benchResult { // simulating an "ls -l" workload, for the configured metadata duration. // Returns nil when dir has no entries. func benchReaddirStat(t *testing.T, dir string) *benchResult { - metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_META_SECS", 5)) * time.Second + metaDuration := time.Duration(corpus.EnvInt("NYDUSFS_PERF_META_SECS", 3)) * time.Second // Verify the directory is non-empty. entries, err := os.ReadDir(dir) diff --git a/tests/e2e/harness.go b/tests/e2e/harness.go index b07c04b3a7f..5c5a094b693 100644 --- a/tests/e2e/harness.go +++ b/tests/e2e/harness.go @@ -379,15 +379,19 @@ func unmountFuse(mnt string) { _ = exec.Command("umount", "-l", mnt).Run() } -func buildNydusFSImageToDir(t *testing.T, nydusBin, imagePath, blobDir, srcDir string, chunkSize int) string { +func buildNydusFSImageToDir(t *testing.T, nydusBin, imagePath, blobDir, srcDir string, chunkSize int, extraArgs ...string) string { t.Helper() require.NoError(t, os.MkdirAll(blobDir, 0755)) before := listFilesInDir(t, blobDir) args := []string{"build", "--blob-dir", blobDir, "--chunk-size", fmt.Sprint(chunkSize), "--compressor", "zstd"} + if gs := os.Getenv("NYDUSFS_PERF_BLOCK_GROUP_SIZE"); gs != "" { + args = append(args, "--block-group-size", gs) + } if imagePath != "" { args = append(args, "--bootstrap", imagePath) } + args = append(args, extraArgs...) args = append(args, srcDir) out, err := exec.Command(nydusBin, args...).CombinedOutput()