diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 075a45f5..fa060cf3 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -31,6 +31,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f94fb4..16710d91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -109,6 +112,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -139,7 +145,7 @@ jobs: - name: Test native security and path equivalence env: FS_SAFE_PAX_REQUIRE_NATIVE: "1" - run: pnpm test test/native-integration.test.ts test/native-owned-tree.test.ts test/native-write-containment.test.ts test/native-staging-regression.test.ts test/staged-file.test.ts test/staged-file-failures.test.ts test/native-archive-equivalence.test.ts test/native-publish-equivalence.test.ts test/private-directory.test.ts test/archive-pax.test.ts test/archive-pax-security.test.ts test/archive-pax-compressed.test.ts test/archive-tar-strip.test.ts test/archive-tar-framing.test.ts test/archive-tar-framing-compressed.test.ts test/archive-gzip-integrity.test.ts test/temp-workspace-cleanup-ownership.test.ts test/temp-workspace-cleanup-admission.test.ts test/temp-workspace-cleanup-capability.test.ts + run: pnpm test test/native-integration.test.ts test/native-owned-tree.test.ts test/native-write-containment.test.ts test/native-staging-regression.test.ts test/staged-file.test.ts test/staged-file-failures.test.ts test/native-archive-equivalence.test.ts test/native-publish-equivalence.test.ts test/private-directory.test.ts test/archive-unified.test.ts test/archive-wasm-abi.test.ts test/archive-pax.test.ts test/archive-pax-security.test.ts test/archive-pax-compressed.test.ts test/archive-tar-strip.test.ts test/archive-tar-framing.test.ts test/archive-tar-framing-compressed.test.ts test/archive-gzip-integrity.test.ts test/archive-gzip-container.test.ts test/temp-workspace-cleanup-ownership.test.ts test/temp-workspace-cleanup-admission.test.ts test/temp-workspace-cleanup-capability.test.ts - name: Test native Root publication verification env: @@ -172,7 +178,9 @@ jobs: - name: Build and test in Alpine run: | docker run --rm -v "$GITHUB_WORKSPACE:/work" -w /work node:24-alpine sh -euxc ' - apk add --no-cache cargo rust musl-dev build-base python3 + apk add --no-cache cargo rust rust-wasm musl-dev build-base python3 tar + # Alpine installs tar in /bin; the bound producer uses /usr/bin/tar. + test -x /usr/bin/tar || ln -s /bin/tar /usr/bin/tar npm install --global pnpm@11.25.0 pnpm install --frozen-lockfile cargo test --workspace --locked @@ -185,7 +193,7 @@ jobs: node scripts/native-mode-smoke.mjs off node scripts/sidecar-contention-proof.mjs require FS_SAFE_NATIVE_MODE=require pnpm test test/root-create-only-preflight.test.ts test/sidecar-lock-root-admission.test.ts test/sidecar-lock-root-ancestry.test.ts test/sidecar-lock-root-budget.test.ts test/sidecar-lock-root-resolver.test.ts test/sidecar-lock-root-unlink.test.ts test/sidecar-lock-unlink-siblings.test.ts test/file-lock-sync-stale.test.ts test/file-lock-sync-release.test.ts - FS_SAFE_PAX_REQUIRE_NATIVE=1 pnpm test test/native-owned-tree.test.ts test/native-write-containment.test.ts test/native-staging-regression.test.ts test/staged-file.test.ts test/staged-file-failures.test.ts test/native-archive-equivalence.test.ts test/native-publish-equivalence.test.ts test/archive-pax.test.ts test/archive-pax-security.test.ts test/archive-pax-compressed.test.ts test/archive-tar-strip.test.ts test/archive-tar-framing.test.ts test/archive-tar-framing-compressed.test.ts test/archive-gzip-integrity.test.ts + FS_SAFE_PAX_REQUIRE_NATIVE=1 pnpm test test/native-owned-tree.test.ts test/native-write-containment.test.ts test/native-staging-regression.test.ts test/staged-file.test.ts test/staged-file-failures.test.ts test/native-archive-equivalence.test.ts test/native-publish-equivalence.test.ts test/archive-unified.test.ts test/archive-wasm-abi.test.ts test/archive-pax.test.ts test/archive-pax-security.test.ts test/archive-pax-compressed.test.ts test/archive-tar-strip.test.ts test/archive-tar-framing.test.ts test/archive-tar-framing-compressed.test.ts test/archive-gzip-integrity.test.ts test/archive-gzip-container.test.ts FS_SAFE_NATIVE_MODE=require pnpm test test/root-write-mode.test.ts test/root-write-verification.test.ts test/root-write-lifetime.test.ts test/root-write-exact-identity.test.ts test/secret-write-publication.test.ts test/native-write-mode-ownership.test.ts test/native-created-cleanup.test.ts test/file-mode-facades.test.ts FS_SAFE_NATIVE_MODE=require pnpm test test/archive-zip-admission.test.ts test/archive-zip-metadata.test.ts test/archive-zip-integrity.test.ts FS_SAFE_PAX_REQUIRE_NATIVE=1 pnpm test test/archive-filter-paths.test.ts test/archive-filter-compressed.test.ts test/archive-tar-gnu.test.ts test/archive-tar-gnu-meter.test.ts test/archive-tar-ignored.test.ts test/archive-tar-ignored-meter.test.ts test/archive-tar-admission.test.ts test/archive-tar-manifest.test.ts @@ -254,6 +262,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f30e9152..d605efda 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -44,6 +44,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/hash-identity-proof.yml b/.github/workflows/hash-identity-proof.yml index 7989fcca..ebd0e8a0 100644 --- a/.github/workflows/hash-identity-proof.yml +++ b/.github/workflows/hash-identity-proof.yml @@ -47,6 +47,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Create isolated proof directories run: | $root = Join-Path $env:RUNNER_TEMP ('hash-identity-proof-' + [guid]::NewGuid().ToString('N')) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 93dcfeac..6148317c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,6 +42,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -158,6 +161,9 @@ jobs: with: version: 0.15.2 + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -211,6 +217,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -301,6 +310,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -343,6 +355,9 @@ jobs: cache: pnpm cache-dependency-path: pnpm-lock.yaml + - name: Add portable TAR parser target + run: rustup target add wasm32-unknown-unknown + - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/CHANGELOG.md b/CHANGELOG.md index c0d59ea7..0146ce95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Accept bounded all-zero gzip container padding from system-tar stdout after validated member trailers, and reject nonzero data hidden after padding on both native and guarded JavaScript TAR routes. +- Unify native and guarded JavaScript TAR admission on one Rust core, bundle its WASM build, and accept strict UTF-8/newline PAX paths without a runtime `tar` dependency; retain bounded framing, raw-field validation, and guarded publication. +- Standardize malformed TAR mode fields on the native zero fallback while preserving ordinary octal, absent, zero, and safe GNU binary modes. - Fix `replaceFileAtomic({ dirMode })` rejecting a raw `fs.stat` mode: directory modes are masked to permission bits (`0o7777`) before application and verification, matching chmod semantics; 0.8.0 regressed this input tolerance with `directory final mode could not be verified`. ## 0.8.1 - 2026-09-04 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c9eb8fa..75349d2c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,7 @@ described in [SECURITY.md](SECURITY.md). Use the Node.js and pnpm versions declared by the repository. ```bash +rustup target add wasm32-unknown-unknown pnpm install --frozen-lockfile pnpm check ``` diff --git a/Cargo.lock b/Cargo.lock index 863cb0b5..b512d55f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,12 +152,28 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fs-safe-archive-core" +version = "0.1.0" +dependencies = [ + "tar", + "unicode-normalization", +] + +[[package]] +name = "fs-safe-archive-wasm" +version = "0.1.0" +dependencies = [ + "fs-safe-archive-core", +] + [[package]] name = "fs-safe-native" version = "0.8.1" dependencies = [ "bzip2", "flate2", + "fs-safe-archive-core", "libc", "napi", "napi-build", diff --git a/Cargo.toml b/Cargo.toml index 2180d48e..dacf569d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["native"] +members = ["native", "archive-core", "archive-wasm"] resolver = "2" [profile.release] diff --git a/README.md b/README.md index 14167021..99ec6b8e 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ This is a **library-level guardrail**, not OS-level isolation. It does not repla pnpm add @openclaw/fs-safe ``` -Node 22 or newer. Core root/path/json/temp helpers avoid framework dependencies. With all optional dependencies omitted, public subpaths remain safe to import and non-archive fallback-capable operations work in `auto` or `off`. Native-only features remain unavailable, and operations needing the binding in `require` mode fail with `helper-unavailable`. JavaScript ZIP/TAR fallback also needs the optional `jszip`/`tar` codecs. See the [0.6 migration guide](docs/migrating-to-0.6.md). +Node 22 or newer. Core root/path/json/temp helpers avoid framework dependencies. With all optional dependencies omitted, public subpaths remain safe to import and non-archive fallback-capable operations work in `auto` or `off`. Native-only features remain unavailable, and operations needing the binding in `require` mode fail with `helper-unavailable`. TAR/gzip fallback uses the bundled WASM build of the same Rust parser as native and works with optional dependencies omitted. ZIP fallback still needs optional `jszip`. See the [0.6 migration guide](docs/migrating-to-0.6.md). The package installs one prebuilt native binding for the current supported target. It supplies fd-relative and atomic no-replace primitives that Node does not expose diff --git a/archive-core/Cargo.toml b/archive-core/Cargo.toml new file mode 100644 index 00000000..1e816b87 --- /dev/null +++ b/archive-core/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "fs-safe-archive-core" +version = "0.1.0" +edition = "2024" +rust-version = "1.88" +publish = false +license = "MIT" + +[dependencies] +unicode-normalization = "0.1.25" + +[dev-dependencies] +tar = { version = "0.4.46", default-features = false } diff --git a/archive-core/src/lib.rs b/archive-core/src/lib.rs new file mode 100644 index 00000000..42eb8bef --- /dev/null +++ b/archive-core/src/lib.rs @@ -0,0 +1,6 @@ +pub mod tar_meter; +mod tar_pax; +pub mod tar_path; +mod tar_mode; + +pub use tar_meter::{TarMetadataMeter, TarMeterLimits, TarMember}; diff --git a/native/src/tar_meter.rs b/archive-core/src/tar_meter.rs similarity index 89% rename from native/src/tar_meter.rs rename to archive-core/src/tar_meter.rs index bd1b2e49..f0f494ab 100644 --- a/native/src/tar_meter.rs +++ b/archive-core/src/tar_meter.rs @@ -18,6 +18,7 @@ pub fn charge_manifest_path(total: &mut u64, path: &str, limit: u64) -> io::Resu #[derive(Clone, Copy)] pub struct TarMeterLimits { + pub windows_paths: bool, pub max_entries: usize, pub max_meta_entry_bytes: u64, pub max_decoded_bytes: u64, @@ -42,7 +43,29 @@ enum MeterState { }, } -// Keep raw framing aligned with src/archive-tar-meta.ts, before either parser. +/// One admitted identity and decoded payload range, independent of executor. +#[derive(Debug, Clone)] +pub struct TarMember { + pub path: String, + pub entry_type: u8, + pub size: u64, + pub mode: u32, + pub offset: u64, +} + +impl TarMember { + pub fn kind(&self) -> &'static str { + match self.entry_type { + 0 | b'0' | b'7' => "file", + b'5' | b'D' => "directory", + b'1' => "hardlink", b'2' => "symlink", + b'3' | b'4' | b'6' => "blocked", + _ => "other", + } + } +} + +// A push consumes at most one framing boundary and emits at most one member. pub struct TarMetadataMeter { inner: R, limits: TarMeterLimits, @@ -56,6 +79,8 @@ pub struct TarMetadataMeter { pending_gnu_path: Option, manifest_bytes: u64, zero_blocks: u8, + offset: u64, + member: Option, } impl TarMetadataMeter { @@ -73,6 +98,8 @@ impl TarMetadataMeter { pending_gnu_path: None, manifest_bytes: 0, zero_blocks: 0, + offset: 0, + member: None, } } @@ -87,7 +114,7 @@ impl TarMetadataMeter { io::Error::new(io::ErrorKind::InvalidData, META_LIMIT) } - fn validate_gnu_body(body: &[u8], kind: u8) -> io::Result<&str> { + fn validate_gnu_body(body: &[u8], kind: u8, windows: bool) -> io::Result<&str> { let value = body.strip_suffix(&[0]).unwrap_or(body); if value.is_empty() || value.contains(&0) { return Err(Self::invalid("empty GNU name or embedded NUL")); @@ -95,7 +122,7 @@ impl TarMetadataMeter { let name = std::str::from_utf8(value) .map_err(|_| Self::invalid("GNU name is not valid UTF-8"))?; if kind == b'L' { - crate::tar_path::validate_path(name) + crate::tar_path::validate_path(name, windows) .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, INVALID_GNU_PATH))?; } Ok(name) @@ -274,11 +301,16 @@ impl TarMetadataMeter { { return Err(Self::invalid("GNU effective non-directory path ends with a separator")); } - let raw_path = crate::tar_path::validate_member(&self.block)?; + let raw_path = crate::tar_path::validate_member(&self.block, self.limits.windows_paths)?; let path = self.pending_pax.as_ref().and_then(|pax| pax.path.as_deref()) .or(self.pending_gnu_path.as_deref()).unwrap_or(&raw_path); - crate::tar_path::validate_path(path)?; + crate::tar_path::validate_path(path, self.limits.windows_paths)?; charge_manifest_path(&mut self.manifest_bytes, path, self.limits.max_manifest_bytes)?; + self.member = Some(TarMember { + path: path.to_owned(), entry_type, size, + mode: crate::tar_mode::manifest_mode(&self.block, matches!(entry_type, b'5' | b'D')), + offset: self.offset, + }); self.pending_pax = None; self.pending_gnu = [false; 2]; self.pending_gnu_path = None; @@ -332,7 +364,7 @@ impl TarMetadataMeter { if kind == b'x' { self.pending_pax = Some(parse_local_pax(body)?); } else { - let name = Self::validate_gnu_body(body, kind)?; + let name = Self::validate_gnu_body(body, kind, self.limits.windows_paths)?; if kind == b'L' { self.pending_gnu_path = Some(name.to_owned()); } } self.state = if padding == 0 { @@ -380,7 +412,7 @@ impl TarMetadataMeter { Ok(()) } - fn check_eof(&self) -> io::Result<()> { + pub fn finish(&self) -> io::Result<()> { if self.pending_pax.is_some() { return Err(Self::invalid("dangling PAX metadata")); } @@ -398,35 +430,43 @@ impl TarMetadataMeter { } } -impl Read for TarMetadataMeter { - fn read(&mut self, output: &mut [u8]) -> io::Result { - if output.is_empty() { - return Ok(0); - } - // Never ask the decoder for body bytes until its header is admitted. +impl TarMetadataMeter { + pub fn take_member(&mut self) -> Option { self.member.take() } + + pub fn boundary(&self) -> usize { let boundary = match &self.state { MeterState::Header | MeterState::SparseHeader { .. } => 512 - self.block_len, MeterState::Metadata { body, used, .. } => body.len() - used, - MeterState::Data { remaining } => (*remaining).min(output.len() as u64) as usize, - MeterState::Eof => output.len(), + MeterState::Data { remaining } => (*remaining).min(65536) as usize, + MeterState::Eof => 65536, }; - // At the ceiling, read at most one byte to distinguish EOF from overflow. - let decoded_boundary = self.remaining_decoded_bytes.max(1).min(output.len() as u64) as usize; - let length = output.len().min(boundary).min(decoded_boundary); - let read = self.inner.read(&mut output[..length])?; - if read == 0 { - self.check_eof()?; - return Ok(0); - } - if read as u64 > self.remaining_decoded_bytes { - if matches!(self.state, MeterState::Eof) && output[0] != 0 { + boundary.min(65536).min(self.remaining_decoded_bytes.clamp(1, 65536) as usize) + } + + /// Consume only the next boundary. The caller drains the event before pushing again. + pub fn push(&mut self, bytes: &[u8]) -> io::Result { + let length = bytes.len().min(self.boundary()); + let bytes = &bytes[..length]; + if length as u64 > self.remaining_decoded_bytes { + if matches!(self.state, MeterState::Eof) && bytes[0] != 0 { return Err(Self::invalid("nonzero data after TAR EOF")); } return Err(io::Error::other(DECODED_LIMIT)); } - self.meter(&output[..read])?; - self.remaining_decoded_bytes -= read as u64; - Ok(read) + self.offset += length as u64; + self.meter(bytes)?; + self.remaining_decoded_bytes -= length as u64; + Ok(length) + } +} + +impl Read for TarMetadataMeter { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if output.is_empty() { return Ok(0); } + let length = output.len().min(self.boundary()); + let read = self.inner.read(&mut output[..length])?; + if read == 0 { self.finish()?; return Ok(0); } + self.push(&output[..read]) } } diff --git a/native/src/tar_meter_tests.rs b/archive-core/src/tar_meter_tests.rs similarity index 91% rename from native/src/tar_meter_tests.rs rename to archive-core/src/tar_meter_tests.rs index 96aa1030..3ace5746 100644 --- a/native/src/tar_meter_tests.rs +++ b/archive-core/src/tar_meter_tests.rs @@ -3,6 +3,7 @@ use std::io::Cursor; pub(super) fn test_limits(max_meta_entry_bytes: u64) -> TarMeterLimits { TarMeterLimits { + windows_paths: cfg!(windows), max_meta_entry_bytes, max_entries: 50_000, max_decoded_bytes: 768 * 1024 * 1024, @@ -58,7 +59,7 @@ fn reader(bytes: Vec, chunk: usize, limit: u64) -> TarMetadataMeter TarMetadataMeter::new(Chunked { inner: Cursor::new(bytes), chunk }, test_limits(limit)) } -// Mirrored from the exhaustive node-tar parser probe in the JS meter tests. +// Ordinary logical types retain their established classification. fn node_tar_hidden_flags() -> Vec { let flags: Vec<_> = (0..=255_u8).filter(|kind| !matches!(kind, 0 | b'0'..=b'7' | b'D' | b'g' | b'x' | b'K' | b'L' | b'N' | b'X' @@ -199,7 +200,7 @@ fn gnu_valid_utf8_terminators_pairs_and_state_reset_preserve_every_byte() { } #[test] -fn pax_framing_matches_tar_across_chunk_boundaries_and_size_directions() { +fn admitted_ranges_preserve_payloads_across_chunks_and_size_directions() { for (raw, size) in [(1, 700), (700, 1), (700, 0)] { let metadata = [ record("mtime", b"1787334189.823045922"), @@ -215,20 +216,20 @@ fn pax_framing_matches_tar_across_chunk_boundaries_and_size_directions() { let mut output = Vec::new(); reader(bytes.clone(), chunk, metadata.len() as u64).read_to_end(&mut output).unwrap(); assert_eq!(output, bytes, "meter must retain original bytes"); - let mut archive = tar::Archive::new(reader(bytes.clone(), chunk, metadata.len() as u64)); - let mut entries = archive.entries().unwrap(); - let mut first = entries.next().unwrap().unwrap(); - assert_eq!(first.path_bytes().as_ref(), b"package/value"); - assert_eq!(first.size(), size as u64); - let mut actual = Vec::new(); - first.read_to_end(&mut actual).unwrap(); - assert_eq!(actual, body); - let mut sentinel = entries.next().unwrap().unwrap(); - assert_eq!(sentinel.path_bytes().as_ref(), b"sentinel"); - actual.clear(); - sentinel.read_to_end(&mut actual).unwrap(); - assert_eq!(actual, b"end"); - assert!(entries.next().is_none()); + let mut parser = reader(bytes.clone(), chunk, metadata.len() as u64); + let mut scratch = [0; 65536]; + let mut members = Vec::new(); + while parser.read(&mut scratch).unwrap() != 0 { + if let Some(member) = parser.take_member() { members.push(member); } + } + assert_eq!(members.len(), 2); + assert_eq!(members[0].path, "package/value"); + assert_eq!(members[0].size, size as u64); + let offset = members[0].offset as usize; + assert_eq!(&bytes[offset..offset + size], body.as_slice()); + assert_eq!(members[1].path, "sentinel"); + let offset = members[1].offset as usize; + assert_eq!(&bytes[offset..offset + 3], b"end"); } } } @@ -246,10 +247,8 @@ fn pax_state_rejects_truncation_duplicates_mixed_and_dangling_metadata() { [extension.clone(), gnu.clone(), file.clone()].concat(), [gnu, extension, file.clone()].concat(), [pax(&[metadata.clone(), metadata].concat()), file.clone()].concat(), - [pax(&record("SCHILY.xattr.user.binary", b"a\nb")), file.clone()].concat(), [pax(&record("size", b"01")), file.clone()].concat(), [pax(&record("GNU.sparse.major", b"1")), file.clone()].concat(), - [pax(&record("path", "caf\u{e9}".as_bytes())), file.clone()].concat(), [pax(&record("size", b"9007199254740991")), file].concat(), ]; for bytes in invalid { @@ -588,3 +587,33 @@ fn manifest_budget_stops_repeated_long_paths_before_another_read() { assert_eq!(source.supplied, maximum); } } + +#[test] +fn unicode_and_newline_pax_values_preserve_admitted_ranges_at_every_chunk_size() { + for name in ["雪.txt", "line\n.txt", "\u{feff}name", "01"] { + for (raw_size, size) in [(1, 700), (700, 1), (700, 0)] { + let metadata = [record("path", name.as_bytes()), + record("SCHILY.xattr.binary", b"\xff\0\n\xfe"), + record("size", size.to_string().as_bytes())].concat(); + let body = vec![0xa7; size]; + let bytes = [pax(&metadata), member("raw", b'0', raw_size, &body), + member("sentinel", b'0', 3, b"end"), vec![0; 1024]].concat(); + for chunk in [1, 2, 3, 7, 511, 512, 513, 65536] { + let mut parser = reader(bytes.clone(), chunk, 1024); + let mut scratch = [0; 65536]; + let mut entries = Vec::new(); + while parser.read(&mut scratch).unwrap() != 0 { + if let Some(entry) = parser.take_member() { entries.push(entry); } + } + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].path, name); + assert_eq!(entries[0].size, size as u64); + let start = entries[0].offset as usize; + assert_eq!(&bytes[start..start + size], body); + assert_eq!(entries[1].path, "sentinel"); + let start = entries[1].offset as usize; + assert_eq!(&bytes[start..start + 3], b"end"); + } + } + } +} diff --git a/native/src/tar_mode.rs b/archive-core/src/tar_mode.rs similarity index 81% rename from native/src/tar_mode.rs rename to archive-core/src/tar_mode.rs index f474ce92..1f9503ef 100644 --- a/native/src/tar_mode.rs +++ b/archive-core/src/tar_mode.rs @@ -1,8 +1,8 @@ use crate::tar_meter::MAX_SAFE_INTEGER; /// Normalize absent and supported GNU modes without changing the native manifest ABI. -pub(crate) fn manifest_mode(header: &tar::Header, directory: bool) -> u32 { - let field = &header.as_old().mode; +pub(crate) fn manifest_mode(header: &[u8; 512], directory: bool) -> u32 { + let field: &[u8; 8] = header[100..108].try_into().unwrap(); if field.iter().all(|byte| matches!(*byte, 0 | b' ')) { return if directory { 0o755 } else { 0o644 }; } @@ -22,9 +22,11 @@ pub(crate) fn manifest_mode(header: &tar::Header, directory: bool) -> u32 { // Do not narrow to u32 before checking its JavaScript numeric domain. return (value & 0o7777) as u32; } - // Keep the existing octal decoder and its malformed/unsupported-mode fallback. - // In particular, an out-of-domain binary value must not wrap into rwx bits. - header.mode().unwrap_or(0) + // Malformed or unsupported fields have one common zero fallback. + let end = field.iter().position(|b| *b == 0).unwrap_or(field.len()); + let text = std::str::from_utf8(&field[..end]).unwrap_or("").trim_matches(' '); + if text.is_empty() || !text.bytes().all(|b| (b'0'..=b'7').contains(&b)) { return 0; } + u32::from_str_radix(text, 8).unwrap_or(0) } #[cfg(test)] @@ -34,7 +36,7 @@ mod tests { fn mode(field: [u8; 8], directory: bool) -> u32 { let mut header = tar::Header::new_ustar(); header.as_old_mut().mode = field; - manifest_mode(&header, directory) + manifest_mode(header.as_bytes(), directory) } #[test] diff --git a/native/src/tar_path.rs b/archive-core/src/tar_path.rs similarity index 52% rename from native/src/tar_path.rs rename to archive-core/src/tar_path.rs index 3babba95..d423c857 100644 --- a/native/src/tar_path.rs +++ b/archive-core/src/tar_path.rs @@ -7,14 +7,22 @@ fn invalid() -> io::Error { io::Error::new(io::ErrorKind::InvalidData, INVALID_PATH) } -pub fn validate_path(name: &str) -> io::Result<()> { - crate::validate_portable_relative_path(name, true).map_err(|_| invalid())?; +pub fn validate_path(name: &str, windows: bool) -> io::Result<()> { + if name.contains('\0') || name.starts_with(['/', '\\']) + || name.split(['/', '\\']).any(|part| part == "..") { + return Err(invalid()); + } if name.split(['/', '\\']).any(|part| { let bytes = part.as_bytes(); (bytes.len() >= 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':') - || (cfg!(windows) && part.contains(':')) - || part.nfc().map(char::len_utf8).sum::() > 255 - || part.nfd().map(char::len_utf8).sum::() > 255 + || (windows && part.contains(':')) + || if part.is_ascii() { + // NFC/NFD cannot change ASCII; avoid Unicode iteration on long metadata paths. + bytes.len() > 255 + } else { + part.nfc().map(char::len_utf8).sum::() > 255 + || part.nfd().map(char::len_utf8).sum::() > 255 + } }) { return Err(invalid()); } @@ -48,20 +56,46 @@ pub fn validate_header_fields(header: &[u8; 512]) -> io::Result<()> { } // Validate original components even when PAX/GNU replaces the member name. -// Keep fixed-field decoding aligned with src/archive-tar-admission.ts. -pub fn validate_member(header: &[u8; 512]) -> io::Result { +// Both executors consume this decoded identity. +pub fn validate_member(header: &[u8; 512], windows: bool) -> io::Result { let name = path_field(&header[..100])?; - validate_path(name)?; + validate_path(name, windows)?; if &header[257..265] == b"ustar\x0000" { // Match node-tar's star layout; atime/ctime are not prefix bytes. let prefix_end = if header[475] == 0 { 475 } else { 500 }; let prefix = path_field(&header[345..prefix_end])?; - validate_path(prefix)?; + validate_path(prefix, windows)?; if !prefix.is_empty() { let path = format!("{prefix}/{name}"); - validate_path(&path)?; + validate_path(&path, windows)?; return Ok(path); } } Ok(name.to_owned()) } + +#[cfg(test)] +mod tests { + use super::validate_path; + + #[test] + fn component_limits_preserve_ascii_and_unicode_normalization_boundaries() { + for windows in [false, true] { + for (name, accepted) in [ + ("a".repeat(255), true), + ("a".repeat(256), false), + (format!("{}\n", "a".repeat(254)), true), + ("é".repeat(85), true), + ("é".repeat(86), false), + ("각".repeat(28), true), + ("각".repeat(29), false), + ] { + assert_eq!(validate_path(&name, windows).is_ok(), accepted); + } + for name in ["../leaf", "pkg/C:leaf", "/absolute", "nul\0name"] { + assert!(validate_path(name, windows).is_err()); + } + assert_eq!(validate_path("name:stream", windows).is_ok(), !windows); + } + } +} diff --git a/native/src/tar_pax.rs b/archive-core/src/tar_pax.rs similarity index 90% rename from native/src/tar_pax.rs rename to archive-core/src/tar_pax.rs index 626c108a..3ece2819 100644 --- a/native/src/tar_pax.rs +++ b/archive-core/src/tar_pax.rs @@ -43,6 +43,11 @@ fn ascii(bytes: &[u8]) -> io::Result<&str> { std::str::from_utf8(bytes).map_err(|_| invalid()) } +fn structural_text(bytes: &[u8]) -> io::Result<&str> { + if bytes.is_empty() || bytes.contains(&0) { return Err(invalid()); } + std::str::from_utf8(bytes).map_err(|_| invalid()) +} + fn timestamp(bytes: &[u8]) -> io::Result<()> { let text = ascii(bytes)?; let unsigned = text.strip_prefix('-').unwrap_or(text); @@ -58,8 +63,7 @@ fn timestamp(bytes: &[u8]) -> io::Result<()> { Ok(()) } -// Keep this byte grammar aligned with src/archive-tar-pax.ts. In particular, -// Rust takes the first duplicate and JS takes the last, so neither is allowed. +// Records are byte-counted; embedded newlines belong to the value. pub fn parse_local_pax(body: &[u8]) -> io::Result { if body.is_empty() { return Err(invalid()); @@ -82,9 +86,6 @@ pub fn parse_local_pax(body: &[u8]) -> io::Result { return Err(invalid()); } let record = &body[offset + space + 1..end - 1]; - if record.contains(&b'\n') { - return Err(invalid()); - } let equals = record .iter() .position(|b| *b == b'=') @@ -98,11 +99,11 @@ pub fn parse_local_pax(body: &[u8]) -> io::Result { let value = &record[equals + 1..]; match key { "path" => { - result.path = Some(ascii(value)?.to_owned()); + result.path = Some(structural_text(value)?.to_owned()); result.path_trailing_separator = matches!(value.last(), Some(b'/' | b'\\')); } "linkpath" => { - ascii(value)?; + structural_text(value)?; result.linkpath = true; } "size" => result.size = Some(decimal(value)?), @@ -117,8 +118,7 @@ pub fn parse_local_pax(body: &[u8]) -> io::Result { .iter() .any(|prefix| key.starts_with(prefix) && key.len() > prefix.len()) => { - // Ignored binary values may contain NUL/non-UTF8, but not LF: - // tar's numeric lookup stops at any malformed preceding line. + // Opaque bounded bytes, never restored to the filesystem. } _ => return Err(invalid()), } @@ -168,6 +168,6 @@ fn raw_text(field: &[u8]) -> io::Result<&str> { if end == 0 { Ok("") } else { - ascii(&field[..end]) + structural_text(&field[..end]) } } diff --git a/archive-wasm/Cargo.toml b/archive-wasm/Cargo.toml new file mode 100644 index 00000000..7ae5cf15 --- /dev/null +++ b/archive-wasm/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "fs-safe-archive-wasm" +version = "0.1.0" +edition = "2024" +rust-version = "1.88" +publish = false +license = "MIT" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +fs-safe-archive-core = { path = "../archive-core" } diff --git a/archive-wasm/src/lib.rs b/archive-wasm/src/lib.rs new file mode 100644 index 00000000..139f1518 --- /dev/null +++ b/archive-wasm/src/lib.rs @@ -0,0 +1,105 @@ +//! Private ABI. Each JavaScript parser owns a separate, import-free instance. +//! No caller-supplied pointer is dereferenced; only lengths into our fixed inbox. +use fs_safe_archive_core::{TarMetadataMeter, TarMeterLimits, TarMember}; +use std::cell::RefCell; + +struct State { + input: [u8; 65536], + parser: Option>, + member: Option, + error: String, +} +thread_local! { + static STATE: RefCell = const { RefCell::new(State { + input: [0; 65536], parser: None, member: None, error: String::new(), + }) }; +} + +#[unsafe(no_mangle)] +pub extern "C" fn input_ptr() -> usize { + STATE.with_borrow(|s| s.input.as_ptr() as usize) +} + +fn limit(value: f64, max: u64) -> Option { + if !value.is_finite() || value < 0.0 { None } else { Some(value.min(max as f64) as u64) } +} + +#[unsafe(no_mangle)] +pub extern "C" fn init(entries: f64, metadata: f64, decoded: f64, manifest: f64, windows: u32) -> i32 { + STATE.with_borrow_mut(|s| { + s.parser = None; + s.member = None; + s.error.clear(); + if windows > 1 { return -1; } + let Some((((entries, metadata), decoded), manifest)) = limit(entries, u32::MAX as u64) + .zip(limit(metadata, 9_007_199_254_740_991)) + .zip(limit(decoded, 9_007_199_254_740_991)) + .zip(limit(manifest, 64 * 1024 * 1024)) else { return -1; }; + s.parser = Some(TarMetadataMeter::new((), TarMeterLimits { + windows_paths: windows == 1, max_entries: entries as usize, max_meta_entry_bytes: metadata, + max_decoded_bytes: decoded, max_manifest_bytes: manifest, + })); + s.member = None; + s.error.clear(); + 0 + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn push(length: usize) -> i32 { + STATE.with_borrow_mut(|s| { + if length == 0 || length > s.input.len() || !s.error.is_empty() { return -1; } + let Some(parser) = &mut s.parser else { return -1; }; + match parser.push(&s.input[..length]) { + Ok(used) => { s.member = parser.take_member(); used as i32 } + Err(error) => { s.error = error.to_string(); s.parser = None; -1 } + } + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn finish() -> i32 { + STATE.with_borrow_mut(|s| { + let Some(parser) = s.parser.take() else { return -1; }; + match parser.finish() { + Ok(()) => 0, + Err(error) => { s.error = error.to_string(); -1 } + } + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn dispose() { + STATE.with_borrow_mut(|s| { s.parser = None; s.member = None; s.error.clear(); }); +} + +#[unsafe(no_mangle)] +pub extern "C" fn text_ptr() -> usize { + STATE.with_borrow(|s| { + if !s.error.is_empty() { s.error.as_ptr() as usize } + else { s.member.as_ref().map_or(0, |m| m.path.as_ptr() as usize) } + }) +} +#[unsafe(no_mangle)] +pub extern "C" fn text_len() -> usize { + STATE.with_borrow(|s| { + if !s.error.is_empty() { s.error.len() } + else { s.member.as_ref().map_or(0, |m| m.path.len()) } + }) +} +#[unsafe(no_mangle)] +pub extern "C" fn member_type() -> i32 { + STATE.with_borrow(|s| s.member.as_ref().map_or(-1, |m| m.entry_type as i32)) +} +#[unsafe(no_mangle)] +pub extern "C" fn member_size() -> f64 { + STATE.with_borrow(|s| s.member.as_ref().map_or(0.0, |m| m.size as f64)) +} +#[unsafe(no_mangle)] +pub extern "C" fn member_offset() -> f64 { + STATE.with_borrow(|s| s.member.as_ref().map_or(0.0, |m| m.offset as f64)) +} +#[unsafe(no_mangle)] +pub extern "C" fn member_mode() -> u32 { + STATE.with_borrow(|s| s.member.as_ref().map_or(0, |m| m.mode)) +} diff --git a/docs/archive.md b/docs/archive.md index 14c91d4f..c508ff73 100644 --- a/docs/archive.md +++ b/docs/archive.md @@ -2,15 +2,12 @@ `@openclaw/fs-safe/archive` extracts ZIP and TAR archives behind one API, with traversal checks, blocked-link-type rejection, and entry-count and byte budgets. When the native binding is available for the current platform, Rust streams ZIP, TAR, gzip, zstd, and bzip2 while TypeScript remains the sole policy owner; every accepted output is created fd-relative in a private staging root. Extraction then merges through the same safe-open boundary used by direct writes — a symlinked entry can't trick the merge into following an out-of-tree path. -The guarded JavaScript fallback uses optional runtime dependencies: `jszip` for -ZIP and `tar` for TAR/gzip. The native path does not use those packages. Installs -that omit optional dependencies can still import this subpath and use the -native path or pure path/limit helpers. - -Some package managers and CI installs skip optional dependencies -(`--no-optional`, `--omit=optional`, or equivalent). If an archive helper throws -that an optional archive dependency is not installed, install `jszip` and/or -`tar` explicitly in the consuming package. +TAR admission uses one Rust core compiled into both the native binding and a +bundled, import-free WebAssembly module. The guarded JavaScript fallback uses +that module for TAR/gzip and optional `jszip` for ZIP. TAR needs no optional +parser dependency, runtime download, install script, or consumer Rust toolchain. +Installs omitting optional dependencies can import every public subpath and use +TAR/gzip in `auto` or `off`; ZIP fallback still requires `jszip`. ```ts import { extractArchive, resolveArchiveKind } from "@openclaw/fs-safe/archive"; @@ -68,12 +65,12 @@ directories; ZIP UNIX creator records with zero attributes are explicit zero, while non-UNIX ZIP records use the absent-metadata defaults. TAR mode fields containing only NUL/ASCII-space padding use absent defaults. -Native extraction also recognizes GNU binary modes, including signed values, -within node-tar's JavaScript safe-integer range before masking permission bits. -Malformed or unsupported mode representations retain existing decoder behavior: -native falls back to zero, while JavaScript may default, parse an octal prefix, -or reject. These representations are not newly admitted or standardized by the -mode repair; raw framing and other numeric-field checks remain unchanged. +Both backends recognize GNU binary modes, including signed values, +within JavaScript's safe-integer range before masking permission bits. +Malformed or unsupported mode fields consistently fall back to zero, matching +the former native behavior. This replaces JavaScript's decoder-dependent octal +prefix parsing, defaulting, or rejection for malformed fields. Ordinary octal, +absent, explicit zero, and supported GNU binary fields retain their behavior. Final modes remain separate from private working staging permissions: files stay `0o600` and directories `0o700` until publication. Files receive their final @@ -112,8 +109,8 @@ normalizing separators. For example, `./pkg/hello.txt` with `stripComponents: 1` extracts to `hello.txt` on both backends. Entries with no remaining components are skipped before the filter callback, but still count toward `maxEntries` and undergo traversal validation. JavaScript TAR extraction -passes node-tar this accepted output path with its own stripping disabled, so -depth checks, collision checks, writes, and mode application agree. +copies the admitted payload range to this accepted output path, so depth checks, +collision checks, writes, and mode application agree. An `entryFilter` sees the validated **canonical effective archive path before stripping**, entry kind, and declared size. On every JavaScript and native @@ -162,11 +159,10 @@ If skipping was not explicitly part of the restore contract, omit `onFiltered`; the first `"skip"` then rejects the complete archive with `ArchiveSecurityError("entry-filtered")`. -Both TAR implementations finish bounded admission before TypeScript policy -evaluation, so a rejected plan never starts extraction. The JavaScript path -owns the extraction file stream and aborts node-tar through a pipeline on -parser disagreement, validation, or timeout failure, destroying both ends -instead of leaving a paused parser to drain indefinitely. +Both TAR routes finish bounded admission before TypeScript policy evaluation, +so a rejected plan never starts extraction. The JavaScript path owns its input, +decoder, and WASM parser streams, joining their teardown on validation, write, +or timeout failure. TAR character devices, block devices, and FIFOs are presented to the filter as `kind: "other"`. Accepted entries of these types reject with @@ -183,8 +179,7 @@ and output collision checks in physical order. Each remaining record reaches `entryFilter` once with its canonical pre-strip path, `kind: "other"`, and declared effective size. A filter skip rejects with `"entry-filtered"` unless `onFiltered: "skip-entry"` is explicit. Accepted unsupported records are safely -omitted and do not consume output payload budgets. This applies even when the -underlying TAR parser suppresses the record. GNU long names describe one such +omitted and do not consume output payload budgets. The shared core admits these records explicitly. GNU long names describe one such record and are then cleared; local PAX on unsupported types and GNU sparse `S` records retain their existing fail-closed format policy. @@ -219,7 +214,7 @@ type ArchiveExtractLimits = { }; ``` -Defaults exist for each (`DEFAULT_MAX_ARCHIVE_BYTES_ZIP`, `DEFAULT_MAX_ENTRIES`, `DEFAULT_MAX_EXTRACTED_BYTES`, `DEFAULT_MAX_ENTRY_BYTES`, `DEFAULT_MAX_META_ENTRY_BYTES`, `DEFAULT_MAX_ENTRY_PATH_COMPONENTS`). An explicit zero remains zero rather than selecting the default. `maxEntries` counts every archive entry, including entries removed by `stripComponents` or an explicit filter. The path-component default is 256. It is evaluated after `stripComponents` and before TypeScript accepts an entry for either JavaScript or native extraction, so rejected entries cannot cause implicit parent-directory creation. The 1 MiB metadata default matches node-tar's `maxMetaEntrySize`; fs-safe passes the same resolved value to node-tar and the native TAR meter. +Defaults exist for each (`DEFAULT_MAX_ARCHIVE_BYTES_ZIP`, `DEFAULT_MAX_ENTRIES`, `DEFAULT_MAX_EXTRACTED_BYTES`, `DEFAULT_MAX_ENTRY_BYTES`, `DEFAULT_MAX_META_ENTRY_BYTES`, `DEFAULT_MAX_ENTRY_PATH_COMPONENTS`). An explicit zero remains zero rather than selecting the default. `maxEntries` counts every archive entry, including entries removed by `stripComponents` or an explicit filter. The path-component default is 256. It is evaluated after `stripComponents` and before TypeScript accepts an entry for either JavaScript or native extraction, so rejected entries cannot cause implicit parent-directory creation. The same resolved 1 MiB metadata default applies to the native and WASM core. A limit violation throws `ArchiveLimitError`. Its constant and string code are: @@ -262,17 +257,17 @@ codes remain `"destination-not-directory"`, `"destination-symlink"`, and - **TOCTOU during merge:** extraction first writes to a private temp dir, then merges into `destDir` using the same boundary checks as `root().write()`. Destination symlink swaps are checked with the selected platform mechanism; non-Linux routes retain the best-effort race window documented in the [security model](security-model.md#containment-guarantees-by-platform). - **Zip bombs:** `maxExtractedBytes` and `maxEntryBytes` apply to *post-decompression* bytes, so highly-compressed payloads hit the cap before they exhaust disk. - **Corrupt ZIP payloads:** streamed output must match both the central-directory CRC and declared uncompressed size before it can leave private staging. -- **Corrupt gzip streams:** truncated compressed bodies, missing trailers, and checksum failures reject before extraction publishes files or an entry read returns bytes, on both JavaScript and native backends. +- **Gzip container integrity:** every concatenated gzip member must have a complete valid header, body, CRC32, and ISIZE trailer. A completed member may be followed by all-zero compressed-container padding (including system-tar stdout padding), bounded by the original archive-byte limit. The padding must remain zero through physical EOF; nonzero bytes or another member after padding reject. Truncation and corruption reject before publication or selected bytes return on both backends. Compressed padding is separate from decoded TAR EOF and does not bypass its checks. - **Slow-loris archives:** `timeoutMs` is a hard wall-clock budget for non-mutating work. Extraction is aborted on overrun; if a destination mutation is already in flight, that mutation and rollback are joined before rejection so archive-controlled publication cannot continue afterward. -- **Metadata bombs:** a streaming pass-through reader rejects oversized PAX, GNU long-name, and GNU long-link bodies before either TAR implementation buffers them. It understands octal and base-256 fixed sizes and validates bounded local PAX bodies before using their size overrides for member framing. Original archive bytes remain unchanged. +- **Metadata bombs:** a streaming pass-through reader rejects oversized PAX, GNU long-name, and GNU long-link bodies before buffering their bodies. It understands octal and base-256 fixed sizes and validates bounded local PAX bodies before using their size overrides for member framing. Original archive bytes remain unchanged. ### Raw TAR framing Extraction and bounded reads admit the complete decoded TAR stream through the -raw meter before either backend's TAR parser runs. This applies to plain TAR, +shared Rust core. This applies to plain TAR, gzip, and native-supported zstd/bzip2, without changing native-mode availability -or fallback policy. The existing TypeScript and Rust meters enforce the same -framing rules before parser normalization: +or fallback policy. The native and WASM builds enforce the same +framing rules: - Every nonzero header must have a valid unsigned octal checksum, delimited within its field. Checksum validation precedes metadata allocation and member @@ -301,22 +296,28 @@ Missing linknames on links and nonempty linknames on non-links still use the format error. PAX `x` and GNU long-name/long-link `L`/`K` payloads retain their existing support and metadata limits; the zero-body rule is not applied to all non-regular types. -PAX effective sizes still determine regular-member framing. Admission preserves -the input bytes, and all entry/path/byte limits and extraction deadlines remain -in force. Native inspection now completes this admission pass before parsing, -requiring one additional streaming read/decompression pass. -JavaScript admission reports an ordered logical-member manifest from the raw -meter, bounded by entry-count, manifest, and decoded limits. Policy runs once -over that manifest; extraction checks parser-visible members against the -accepted decisions before writing. Original member names and USTAR prefixes -are validated even when overridden, and non-padding bytes after a fixed path -field's NUL terminator reject rather than hiding an unsafe suffix. -Both meters enforce the 255-byte component ceiling under NFC and NFD before -metadata replaces a raw path, including Hangul decomposition expansion. -Native extraction and entry reads also drain their metered readers through -physical EOF after parser traversal, before completing directory modes, -publishing staged files, or returning the requested bytes. Finding the requested -member or reaching the parser's logical EOF cannot bypass trailing validation. +PAX effective sizes determine regular-member framing. Admission preserves input +bytes and emits an ordered manifest with exact effective names, types, modes, +sizes, and decoded payload offsets. TypeScript owns filtering, stripping, +collisions, permissions, and accepted-output limits. Executors replay admitted +ranges from the immutable staged input; no second TAR parser interprets PAX, +GNU names, or payload lengths. Native writes remain descriptor-relative; +JavaScript writes use the shared guarded private staging and pinned-write helpers. + +Original member names and USTAR prefixes are validated even when overridden. +Non-padding bytes after a fixed path field's NUL terminator reject. The core +enforces the 255-byte component ceiling under NFC and NFD, including Hangul +expansion. Every replay drains and validates physical EOF before publication or +returning selected bytes. Unrequested, filtered, and stripped members cannot +bypass validation. Decompression remains streaming; no complete decoded archive +is retained in memory or written to a decoded spool. + +The WASM transport has a fixed 64 KiB input buffer, one pending member event, +and a 256 MiB maximum linear memory per isolated parser instance. Metadata is +bounded before allocation; allocation failure rejects. Stream backpressure +bounds queued chunks, and completion/error destroys the instance's parser +state. The manifest retains the existing charged budget below; linear memory +is an additional execution resource bound, not a new public limit option. The raw meter enforces `maxEntries` before consuming each logical member's body, including members later skipped by filtering or stripping. PAX/GNU metadata @@ -343,9 +344,7 @@ archive overhead with safe addition. Ordinary limits, including zero and the existing defaulting/rounding rules, retain their behavior. There is no new public option. This is an absolute decoded admission cap, not a decompression-ratio policy; bounded stream/codec read-ahead remains. -After this complete preflight, the JavaScript backend disables node-tar's -independent ratio threshold so it cannot reject data that the native backend -accepts within the same absolute limits. +There is no independent TAR parser decompression-ratio threshold. ### Bounded local PAX support @@ -359,13 +358,14 @@ permits link creation. Effective sizes drive framing, filters, and the existing budgets; `maxEntries` still counts members, not their metadata headers. Records must have exact byte lengths, ASCII keys, a final newline, and no -duplicate keys, embedded newlines, or unconsumed bytes. Structural `path` and -`linkpath` values and ownership names must be nonempty printable ASCII. A PAX -member's raw name, USTAR prefix, and raw link target must also be printable -ASCII; raw link targets must be present only on links, even when overridden. -Unicode -PAX structural text is deliberately unsupported because the underlying parsers -do not agree when UTF-8 is split across input chunks. `size`, `uid`, and `gid` +duplicate keys or unconsumed bytes. `path` and `linkpath` must be nonempty strict +UTF-8 without NUL. Unicode, a leading BOM, numeric-looking names, and embedded +newlines preserve their exact spelling; newlines inside a byte-counted value +are data. Windows filesystem filename restrictions still apply during creation. +Ownership names retain the existing nonempty printable-ASCII contract. Raw name, +USTAR prefix, and link fields still require strict UTF-8 and NUL padding even +when metadata overrides them. Raw link targets must be present only on links. +`size`, `uid`, and `gid` must be canonical unsigned decimal safe integers (zero is valid; signs, leading zeros, fractions, and exponents are not). Padded member sizes must also fit the safe integer range. Raw and effective directory/link sizes must both be zero; @@ -378,8 +378,7 @@ with optional fractional digits, within JavaScript's Date range), `uid`, `gid`, destination. `LIBARCHIVE.xattr.*` and `SCHILY.xattr.*` with nonempty ASCII alphanumeric/dot/underscore/hyphen suffixes are also accepted as inert metadata, never restored as extended attributes. Their values are byte-counted and may -contain NUL or non-UTF8 bytes, including macOS provenance metadata; embedded -newlines are rejected because they can disrupt downstream record parsing. +contain NUL, non-UTF8 bytes, or newlines, including macOS provenance metadata. Global `g`, old `X`, old GNU `N`, empty/dangling/repeated local headers, mixed PAX/GNU extension chains, unknown keys, charset declarations, ACL extensions, @@ -393,11 +392,11 @@ chains without introducing a new limit or changing defaults. ### Bounded GNU long names and links -Both raw meters buffer GNU long-name `L` and long-link `K` bodies within -`maxMetaEntryBytes` before either TAR parser runs. A body must contain a nonempty +The shared core buffers GNU long-name `L` and long-link `K` bodies within +`maxMetaEntryBytes`. A body must contain a nonempty UTF-8 name, with either no NUL or exactly one terminal NUL. Embedded NULs, additional terminal NULs, bytes after a NUL, and invalid UTF-8 reject with -`ArchiveFormatError("archive-header-invalid")`. The meters preserve original +`ArchiveFormatError("archive-header-invalid")`. The core preserves original archive bytes, including the optional terminator and block padding. One logical member may have at most one `L` and one `K`, in either order. diff --git a/docs/contributing.md b/docs/contributing.md index e3e1bc5d..eba93077 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -19,7 +19,16 @@ manager version declared in `package.json`. pnpm build ``` -Runs `tsc -p tsconfig.json`. Output lands in `dist/`. The package's `prepack` hook re-runs the build before publishing — manual `pnpm build` is only required when you want to inspect the output or run a freshly-built copy locally. +Runs TypeScript compilation and builds the portable Rust TAR parser for +`wasm32-unknown-unknown`. Contributors need Rust (the native crate's declared +minimum or newer) and `rustup target add wasm32-unknown-unknown`; Alpine's +packaged toolchain uses `rust-wasm`. `pnpm archive:wasm` rebuilds just the parser. +The import-free asset lands at `dist/archive-parser.wasm`; source tests and +compiled consumers both resolve that generated artifact. Run `pnpm build` +before source tests in a fresh checkout. Do not commit `dist/` or built WASM. +Consumers receive the asset in the npm package and need no compiler. + +Output lands in `dist/`. The package's `prepack` hook re-runs the build before publishing — manual `pnpm build` is only required when you want to inspect the output or run a freshly-built copy locally. ## Test @@ -59,6 +68,23 @@ pnpm check This runs the filesystem boundary checks, build, tests, and package tarball/import validation. +### Real TAR producers + +After installing the freshly packed root (and optionally its freshly built host +binding) in a disposable consumer, run: + +```bash +pnpm archive:producer-smoke ./consumer off +pnpm archive:producer-smoke ./consumer require +``` + +This uses a child bound to canonical cwd/device/inode running `/usr/bin/tar -czf - .` +with unchanged stdout, and npm tar, on synthetic Unicode/newline/long-name files, +then the installed package API for exact payload hashes and bounded reads. +It also rejects a valid PAX override attached to an invalid raw UTF-8 field. +The `require` command must resolve the freshly packed native binding; the +`off` command uses the installed WASM asset. No live user files are read. + ### Native consumer installs After `pnpm build` and a fresh `pnpm native:build`, run `pnpm package:smoke`. diff --git a/docs/install.md b/docs/install.md index 32387cbf..1b892d60 100644 --- a/docs/install.md +++ b/docs/install.md @@ -85,7 +85,7 @@ Use the main entry for the common surface, or the focused subpaths when you want ## Runtime dependencies -`@openclaw/fs-safe` lists `jszip` and `tar` as optional dependencies for JavaScript ZIP/TAR [archive extraction](archive.md). They are loaded lazily; the JavaScript archive fallback requires the corresponding codec and reports a missing-optional-dependency error without it. Public subpaths remain safe to import with all optional dependencies omitted, but imports do not prove native availability. +`@openclaw/fs-safe` bundles an import-free WASM build of its Rust TAR parser for guarded JavaScript TAR/gzip [archive extraction](archive.md), including installs with optional dependencies omitted. ZIP fallback uses lazily loaded optional `jszip` and reports a missing-dependency error without it. Public subpaths remain safe to import with all optional dependencies omitted, but imports do not prove native availability. There are no peer dependencies. Exact-version optional packages carry the seven native targets and npm-compatible OS, CPU, and Linux libc filters install only diff --git a/docs/native-helper.md b/docs/native-helper.md index 311eea63..3c2ee95a 100644 --- a/docs/native-helper.md +++ b/docs/native-helper.md @@ -29,6 +29,11 @@ The equivalent environment variables are `FS_SAFE_NATIVE_MODE` and `OPENCLAW_FS_ | `off` | Do not load a native package. Use the guarded JavaScript path deterministically. | | `require` | Throw `FsSafeError("helper-unavailable")` instead of falling back when an operation needs the native binding and it cannot load. | +TAR/gzip in the guarded JavaScript path uses a bundled, import-free WASM build +of the same Rust parser used by native. `off` still disables native filesystem +code; it does not disable this portable parser. ZIP fallback still requires +optional `jszip`, and zstd/bzip2 remain native-only. + Configure the mode once during startup. Loading is lazy and cached; changing from `auto` to `require` after a failed load changes failure policy but does not repeatedly probe the binary. [`tempWorkspace()` and its scoped/sync variants](temp.md#private-temp-workspaces) diff --git a/docs/native.md b/docs/native.md index ade04f57..eabfaa19 100644 --- a/docs/native.md +++ b/docs/native.md @@ -64,13 +64,15 @@ returns a bounded manifest. TypeScript applies the shared path, filter, strip, mode, and byte policies and returns an index-bound extraction plan. Rust then creates only those planned entries beneath a private staging descriptor. -A raw meter sits between decompression and the TAR crate, with matching -TypeScript admission before node-tar. It parses 512-byte headers and bounded -local PAX `x` metadata, using supported effective sizes to locate the following -member body. GNU long-name/link `L`/`K` payloads remain -supported. `maxMetaEntryBytes` bounds each metadata body before allocation; -unsupported global/old metadata and sparse forms fail closed rather than being -interpreted as ordinary members. See [bounded local PAX support](archive.md#bounded-local-pax-support). +The `fs-safe-archive-core` Rust workspace crate owns TAR framing, paths, types, +mode decoding, GNU metadata, and byte-counted local PAX records. The native +binding and bundled WASM module compile the same source. No `tar::Archive` or +Node TAR parser reinterprets admitted identities or sizes. Executors replay +admitted payload ranges after complete bounded admission; native writes retain +the platform's descriptor-relative primitives, while fallback writes retain +the guarded Node staging/publication boundary. ZIP behavior is unchanged. +`maxMetaEntryBytes` bounds bodies before allocation; unsupported global/old +metadata and sparse forms fail closed. See [bounded local PAX support](archive.md#bounded-local-pax-support). Every raw pass receives only TypeScript's resolved `maxEntries`, `maxMetaEntryBytes`, and `maxDecodedBytes`. Shared resolution caps metadata and @@ -88,11 +90,15 @@ integer maximum. Every native pass receives that same cap and charges headers, metadata, bodies, padding, EOF blocks, and trailing zeros. It rejects overflow with `archive-decoded-size-exceeds-limit`; no ratio policy is implied. Extraction and entry reads drain the metered reader through physical EOF after -TAR iteration. Trailing framing or decoded-limit failures propagate before +admitted-range replay. Trailing framing or decoded-limit failures propagate before directory modes are finalized, staging is published, or selected bytes return. +Native gzip uses the existing flate2 member decoder with an explicit bounded +member/padding transition; JavaScript retains Node gunzip and validates its +unconsumed compressed suffix. Only all-zero physical padding after a complete +validated trailer is accepted, still within the original archive-byte budget. Native reads stop at framing boundaries so a rejected header does not request its body from the decoder; codec buffering can still read ahead internally. -Inspection finishes the complete bounded framing pass before parsing. Directory +Inspection finishes the complete bounded framing pass before returning its manifest. Directory and link bodies, missing two-block EOF, and nonzero trailers reject on both backends, as detailed in [raw TAR framing](archive.md#raw-tar-framing). Raw and padded sizes above JavaScript's safe-integer maximum reject as invalid framing @@ -155,7 +161,7 @@ remain TypeScript-owned. What changes is the syscall strength or availability: | Capability | Native path | Guarded JavaScript path | |---|---|---| | Root-relative opens/mutations | Descriptor-relative beneath operations. Pinned writes create parents and publish both replacement and no-replace targets relative to open directory descriptors. Linux reports `kernel-atomic`; macOS and Windows report `best-effort`. macOS uses `O_RESOLVE_BENEATH` when available plus an `F_GETPATH` detector, while Windows rejects reparse traversal in the object-manager call. | Reports `best-effort`: component-wise alias checks, no-follow opens where Node exposes them, private temp/rename, and post-operation identity verification. A same-privilege peer can replace a writable parent after a guard assertion but before Node resolves the pathname mutation; the mutation may land outside the intended root before the post-check detects it. | -| ZIP/TAR/gzip | Rust streaming decode and fd-relative output creation. | JSZip/node-tar into a private stage, then the same guarded merge policy. | +| ZIP/TAR/gzip | Rust streaming decode and fd-relative output creation. | Optional JSZip or bundled WASM TAR into guarded private staging, then the same guarded merge policy. | | Zstd/bzip2 TAR | Supported. | Unsupported; typed `helper-unavailable`. | | Publication copy | Clone, Linux `copy_file_range`, async native SHA-256. | Exclusive `wx` byte loop and Node SHA-256 with the same content/identity fences. | | `rename-noreplace` | Atomic platform no-replace rename. | Unsupported; no emulation by check-then-rename. | diff --git a/native/Cargo.toml b/native/Cargo.toml index f7dddd94..bbf917ff 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -16,7 +16,7 @@ flate2 = "1.1.10" napi = { version = "3.12.2", default-features = false, features = ["dyn-symbols", "napi8"] } napi-derive = "3.6.3" sha2 = "0.11.0" -tar = { version = "0.4.46", default-features = false } +fs-safe-archive-core = { path = "../archive-core" } unicode-normalization = "0.1.25" zip = { version = "8.6.0", default-features = false, features = ["deflate-flate2"] } zstd = { version = "0.13.3", default-features = false } @@ -41,3 +41,6 @@ windows-sys = { version = "0.61.2", features = [ [build-dependencies] napi-build = "2.4.1" + +[dev-dependencies] +tar = { version = "0.4.46", default-features = false } diff --git a/native/src/archive.rs b/native/src/archive.rs index b969d613..76e6e11b 100644 --- a/native/src/archive.rs +++ b/native/src/archive.rs @@ -10,7 +10,7 @@ use napi::bindgen_prelude::{AbortSignal, AsyncTask, Buffer, Task}; use napi::{Env, Error, Result, Status}; use napi_derive::napi; -use crate::tar_meter::{TarMetadataMeter, TarMeterLimits, MAX_SAFE_INTEGER, MAX_MANIFEST_BYTES, charge_manifest_path}; +use crate::tar_meter::{TarMetadataMeter, TarMeterLimits, MAX_SAFE_INTEGER, MAX_MANIFEST_BYTES}; use crate::{NativeResult, native_error, platform, validate_portable_relative_path}; #[napi(object)] @@ -39,6 +39,7 @@ pub struct ArchiveEntryData { kind: String, size: u64, mode: u32, + offset: u64, } #[derive(Clone, Copy)] @@ -65,6 +66,7 @@ pub struct NativeTarLimits { impl NativeTarLimits { fn checked(self) -> Result { Ok(TarMeterLimits { + windows_paths: cfg!(windows), max_entries: checked_tar_limit(self.max_entries, "maxEntries", u32::MAX as u64)? as usize, max_meta_entry_bytes: checked_tar_limit(self.max_meta_entry_bytes, "maxMetaEntryBytes", MAX_SAFE_INTEGER)?, max_decoded_bytes: checked_tar_limit(self.max_decoded_bytes, "maxDecodedBytes", MAX_SAFE_INTEGER)?, @@ -119,7 +121,7 @@ fn open_tar_reader( format: ArchiveFormat, cancelled: Arc, limits: TarMeterLimits, -) -> Result> { +) -> Result>> { let mut file = File::open(path).map_err(|error| io_error("open archive", error))?; let decoded: Box = match format { ArchiveFormat::TarZstd => Box::new(CancellationReader { @@ -140,7 +142,10 @@ fn open_tar_reader( .map_err(|error| io_error("rewind archive", error))?; if read == 2 && magic == [0x1f, 0x8b] { Box::new(CancellationReader { - inner: flate2::read::MultiGzDecoder::new(file), + inner: crate::archive_gzip::GzipContainer::new( + CancellationReader { inner: file, cancelled: Arc::clone(&cancelled) }, + Arc::clone(&cancelled), + ), cancelled, }) } else { @@ -154,35 +159,7 @@ fn open_tar_reader( return Err(Error::new(Status::InvalidArg, "zip is not a tar stream")); } }; - Ok(Box::new(TarMetadataMeter::new( - decoded, - limits, - ))) -} - -fn tar_kind(entry_type: tar::EntryType) -> &'static str { - if entry_type.is_dir() || entry_type.as_byte() == b'D' { - "directory" - } else if entry_type.is_file() || entry_type.is_contiguous() { - "file" - } else if entry_type.is_gnu_sparse() { - "sparse" - } else if entry_type.is_symlink() { - "symlink" - } else if entry_type.is_hard_link() { - "hardlink" - } else if entry_type.is_character_special() || entry_type.is_block_special() || entry_type.is_fifo() { - "blocked" - } else { - "other" - } -} - -fn checked_path(path: std::borrow::Cow<'_, std::path::Path>) -> Result { - path.into_owned() - .into_os_string() - .into_string() - .map_err(|_| Error::new(Status::InvalidArg, "archive entry path is not valid UTF-8")) + Ok(TarMetadataMeter::new(decoded, limits)) } fn drain_tar_metadata(reader: &mut impl Read) -> std::io::Result<()> { @@ -200,46 +177,17 @@ fn inspect_tar( limits: InspectLimits, cancelled: Arc, ) -> Result> { - // Admit the complete decoded framing before tar can stop at an early zero - // block or normalize a non-file size. Reuse the meter for every TAR codec. - drain_tar_metadata(&mut open_tar_reader( - path, - format, - Arc::clone(&cancelled), - limits.tar, - )?) - .map_err(|error| io_error("preflight tar metadata", error))?; - let mut archive = tar::Archive::new(open_tar_reader( - path, - format, - Arc::clone(&cancelled), - limits.tar, - )?); - let entries = archive - .entries() - .map_err(|error| io_error("read tar entries", error))?; + let mut reader = open_tar_reader(path, format, cancelled, limits.tar)?; let mut result = Vec::new(); - let mut manifest_bytes = 0_u64; - for (index, entry) in entries.enumerate() { - check_cancelled(&cancelled)?; - let entry = entry.map_err(|error| io_error("read tar entry", error))?; - let header = entry.header(); - check_manifest_count(index + 1, limits)?; - let size = entry.size(); - let path = checked_path( - entry - .path() - .map_err(|error| io_error("read tar path", error))?, - )?; - add_manifest_path_bytes(&mut manifest_bytes, &path, limits)?; - result.push(ArchiveEntryData { - index: u32::try_from(index) - .map_err(|_| Error::new(Status::InvalidArg, "too many archive entries"))?, - path, - kind: tar_kind(header.entry_type()).to_owned(), - size, - mode: crate::tar_mode::manifest_mode(header, tar_kind(header.entry_type()) == "directory"), - }); + let mut buffer = [0; 65536]; + while reader.read(&mut buffer).map_err(|error| io_error("admit tar", error))? != 0 { + if let Some(member) = reader.take_member() { + let kind = member.kind().to_owned(); + result.push(ArchiveEntryData { + index: result.len() as u32, path: member.path, kind, + size: member.size, mode: member.mode, offset: member.offset, + }); + } } Ok(result) } @@ -288,6 +236,7 @@ fn inspect_zip( kind: zip_kind(&file).to_owned(), size: file.size(), mode: file.unix_mode().unwrap_or(0), + offset: 0, }); } Ok(result) @@ -309,17 +258,6 @@ fn limit_error(code: &'static str) -> Error { Error::new(Status::GenericFailure, code) } -fn check_manifest_count(count: usize, limits: InspectLimits) -> Result<()> { - if count > limits.tar.max_entries { - return Err(limit_error("archive-entry-count-exceeds-limit")); - } - Ok(()) -} - -fn add_manifest_path_bytes(total: &mut u64, path: &str, limits: InspectLimits) -> Result<()> { - charge_manifest_path(total, path, limits.tar.max_manifest_bytes) - .map_err(|error| io_error("admit manifest path", error)) -} fn zip_entry_count(path: &str, max_entries: usize) -> Result { let mut file = File::open(path).map_err(|error| io_error("open zip archive", error))?; @@ -544,55 +482,44 @@ fn extract_tar( cancelled: Arc, limits: TarMeterLimits, ) -> Result<()> { - let mut archive = tar::Archive::new(open_tar_reader( - path, - format, - Arc::clone(&cancelled), - limits, - )?); - let entries = archive - .entries() - .map_err(|error| io_error("read tar entries", error))?; + let manifest = inspect_tar(path, format, InspectLimits { tar: limits }, Arc::clone(&cancelled))?; + let mut reader = open_tar_reader(path, format, Arc::clone(&cancelled), limits)?; + let mut position = 0; let mut directories = Vec::new(); - for (index, entry) in entries.enumerate() { + for entry in manifest { check_cancelled(&cancelled)?; - let mut entry = entry.map_err(|error| io_error("read tar entry", error))?; - let Some(item) = plan.remove(&index) else { - continue; - }; - let actual_kind = tar_kind(entry.header().entry_type()); - if actual_kind != item.kind || entry.size() as f64 != item.size { - return Err(Error::new( - Status::InvalidArg, - "archive entry changed after policy evaluation", - )); + let Some(item) = plan.remove(&(entry.index as usize)) else { continue; }; + if entry.kind != item.kind || entry.size as f64 != item.size { + return Err(Error::new(Status::InvalidArg, "archive entry changed after policy evaluation")); } + skip_tar_to(&mut reader, &mut position, entry.offset)?; if item.kind == "directory" { platform::mkdir_beneath(root_fd, &item.path, 0o700) .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; directories.push((item.path, item.mode)); } else { ensure_parent(root_fd, &item.path)?; - let size = entry.size(); - let mut reader = CancellationReader { - inner: &mut entry, - cancelled: Arc::clone(&cancelled), - }; - platform::write_archive_file(root_fd, &item.path, &mut reader, size, item.mode) + let mut payload = (&mut reader).take(entry.size); + platform::write_archive_file(root_fd, &item.path, &mut payload, entry.size, item.mode) .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + if payload.limit() != 0 { return Err(Error::new(Status::InvalidArg, "truncated TAR payload")); } + position += entry.size; } } - // tar stops at logical EOF; the meter must admit the physical tail before - // completing directories or allowing the caller to publish the plan. - drain_tar_metadata(&mut archive.into_inner()) - .map_err(|error| io_error("finish tar metadata", error))?; - finish_directories(root_fd, directories)?; + drain_tar_metadata(&mut reader).map_err(|error| io_error("finish tar", error))?; if !plan.is_empty() { - return Err(Error::new( - Status::InvalidArg, - "archive entries disappeared after policy evaluation", - )); + return Err(Error::new(Status::InvalidArg, "archive entries disappeared after policy evaluation")); } + finish_directories(root_fd, directories) +} + +fn skip_tar_to(reader: &mut impl Read, position: &mut u64, offset: u64) -> Result<()> { + let length = offset.checked_sub(*position) + .ok_or_else(|| Error::new(Status::InvalidArg, "invalid admitted TAR range"))?; + let copied = std::io::copy(&mut reader.take(length), &mut std::io::sink()) + .map_err(|error| io_error("replay tar", error))?; + if copied != length { return Err(Error::new(Status::InvalidArg, "truncated TAR range")); } + *position = offset; Ok(()) } @@ -724,58 +651,21 @@ fn read_tar_entry( cancelled: Arc, limits: TarMeterLimits, ) -> Result> { - let mut archive = tar::Archive::new(open_tar_reader( - path, - format, - Arc::clone(&cancelled), - limits, - )?); - let mut selected = None; - for (index, entry) in archive - .entries() - .map_err(|error| io_error("read tar entries", error))? - .enumerate() - { - if index >= limits.max_entries { - return Err(limit_error("archive-entry-count-exceeds-limit")); - } - check_cancelled(&cancelled)?; - let mut entry = entry.map_err(|error| io_error("read tar entry", error))?; - if selected.is_some() { - continue; - } - if checked_path( - entry - .path() - .map_err(|error| io_error("read tar path", error))?, - )? != requested - { - continue; - } - if tar_kind(entry.header().entry_type()) != "file" { - if entry.header().entry_type().is_gnu_sparse() { - return Err(Error::new( - Status::InvalidArg, - "archive-header-invalid: GNU sparse entries are not supported", - )); - } - return Err(Error::new( - Status::InvalidArg, - format!("archive entry is not a file: {requested}"), - )); - } - selected = Some(read_bounded(&mut entry, max_bytes, Arc::clone(&cancelled))?); + let manifest = inspect_tar(path, format, InspectLimits { tar: limits }, Arc::clone(&cancelled))?; + let member = manifest.iter().find(|entry| entry.path == requested) + .ok_or_else(|| Error::new(Status::InvalidArg, format!("archive entry not found: {requested}")))?; + if member.kind != "file" { + return Err(Error::new(Status::InvalidArg, format!("archive entry is not a file: {requested}"))); } - // Keep the first bounded result private until traversal and the physical - // tail pass the same framing/decoded limits as native inspection. - drain_tar_metadata(&mut archive.into_inner()) - .map_err(|error| io_error("finish tar metadata", error))?; - selected.ok_or_else(|| { - Error::new( - Status::InvalidArg, - format!("archive entry not found: {requested}"), - ) - }) + if member.size > max_bytes { return Err(limit_error("archive-entry-extracted-size-exceeds-limit")); } + let mut reader = open_tar_reader(path, format, Arc::clone(&cancelled), limits)?; + skip_tar_to(&mut reader, &mut 0, member.offset)?; + let output = read_bounded(&mut (&mut reader).take(member.size), max_bytes, cancelled)?; + if output.len() as u64 != member.size { + return Err(Error::new(Status::InvalidArg, "archive-header-invalid: truncated TAR payload")); + } + drain_tar_metadata(&mut reader).map_err(|error| io_error("finish tar", error))?; + Ok(output) } fn read_zip_entry( @@ -913,14 +803,6 @@ mod tests { static TEMP_PATH_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - #[test] - fn tar_manifest_preserves_blocked_and_dump_directory_kinds() { - for kind in [b'3', b'4', b'6'] { - assert_eq!(tar_kind(tar::EntryType::new(kind)), "blocked"); - } - assert_eq!(tar_kind(tar::EntryType::new(b'D')), "directory"); - } - fn fixture_tar() -> Vec { let mut bytes = Vec::new(); { @@ -955,6 +837,7 @@ mod tests { fn limits(max_entries: usize) -> InspectLimits { InspectLimits { tar: TarMeterLimits { + windows_paths: cfg!(windows), max_entries, max_meta_entry_bytes: 1024 * 1024, max_decoded_bytes: 768 * 1024 * 1024, @@ -1012,7 +895,7 @@ mod tests { let result = inspect_tar(path.to_str().unwrap(), format, limits(10), Arc::new(AtomicBool::new(false))); std::fs::remove_file(path).unwrap(); let manifest = result.unwrap(); - let kind = tar_kind(tar::EntryType::new(entry_type)); + let kind = if matches!(entry_type, b'5' | b'D') { "directory" } else { "file" }; assert_eq!(manifest[0].kind, kind); assert_eq!(manifest[0].mode, expected.unwrap_or(if kind == "directory" { 0o755 } else { 0o644 }), "mode={mode:?}, type={entry_type}"); } @@ -1193,7 +1076,6 @@ mod tests { let path = temp_path("tar-framing"); std::fs::write(&path, bytes).unwrap(); let error = inspect_tar(path.to_str().unwrap(), format, limits(10), Arc::new(AtomicBool::new(false))).unwrap_err(); - assert!(error.reason.contains("preflight tar metadata"), "{error}"); assert!(error.reason.contains("checksum failure"), "{error}"); std::fs::remove_file(path).unwrap(); } @@ -1323,8 +1205,10 @@ mod tests { one_entry.max_entries = 1; assert!(read("one.txt", 3, one_entry).unwrap_err().reason.contains("archive-entry-count-exceeds-limit")); - std::fs::write(&path, [canonical.as_slice(), &[1]].concat()).unwrap(); assert!(read("one.txt", 2, limits.tar).unwrap_err().reason.contains("archive-entry-extracted-size-exceeds-limit")); + std::fs::write(&path, [canonical.as_slice(), &[1]].concat()).unwrap(); + // Complete admission now precedes selected-member policy, as in the public API. + assert!(read("one.txt", 2, limits.tar).unwrap_err().reason.contains("archive-header-invalid")); assert!(read("absent", 3, limits.tar).unwrap_err().reason.contains("archive-header-invalid")); let plan = HashMap::from([(99, NativeArchivePlanEntry { index: 99, path: "absent".to_owned(), kind: "file".to_owned(), size: 0.0, mode: 0o600, diff --git a/native/src/archive_gzip.rs b/native/src/archive_gzip.rs new file mode 100644 index 00000000..5888cc6a --- /dev/null +++ b/native/src/archive_gzip.rs @@ -0,0 +1,130 @@ +use std::io::{self, BufRead, BufReader, Read}; +use std::sync::{Arc, atomic::{AtomicBool, Ordering}}; + +use flate2::bufread::GzDecoder; + +/// Decode every gzip member, then admit only zero container padding to physical EOF. +/// BufRead preserves input following the validated member trailer. +pub(crate) struct GzipContainer { + member: Option>>, + cancelled: Arc, + failed: bool, +} + +impl GzipContainer { + pub(crate) fn new(reader: R, cancelled: Arc) -> Self { + Self { + member: Some(GzDecoder::new(BufReader::with_capacity(65536, reader))), + cancelled, + failed: false, + } + } + + fn check_cancelled(&self) -> io::Result<()> { + if self.cancelled.load(Ordering::Relaxed) { + Err(io::Error::other("archive operation aborted")) + } else { Ok(()) } + } + + fn read_member_or_padding(&mut self, output: &mut [u8]) -> io::Result { + loop { + self.check_cancelled()?; + let Some(member) = &mut self.member else { return Ok(0); }; + let read = member.read(output)?; + if read != 0 { return Ok(read); } + // GzDecoder reaches EOF only after checking the complete CRC32/ISIZE trailer. + let mut input = self.member.take().unwrap().into_inner(); + match input.fill_buf()?.first() { + None => return Ok(0), + Some(0) => { + loop { + self.check_cancelled()?; + let bytes = input.fill_buf()?; + if bytes.is_empty() { return Ok(0); } + if bytes.iter().any(|byte| *byte != 0) { + return Err(io::Error::other("archive-header-invalid: nonzero gzip container padding")); + } + let length = bytes.len(); + input.consume(length); + } + } + // A following member must validate normally; arbitrary junk is never suppressed. + Some(_) => self.member = Some(GzDecoder::new(input)), + } + } + } +} + +impl Read for GzipContainer { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if self.failed { return Err(io::Error::other("gzip container already failed")); } + if output.is_empty() { return Ok(0); } + let result = self.read_member_or_padding(output); + if result.is_err() { self.failed = true; self.member = None; } + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + // Python stdlib gzip.compress(b"abc", mtime=0), with known CRC32/ISIZE. + const MEMBER: &[u8] = &[31, 139, 8, 0, 0, 0, 0, 0, 2, 255, 75, 76, 74, 6, 0, 194, 65, 36, 53, 3, 0, 0, 0]; + struct Chunked { + source: Cursor>, + chunk: usize, + cancel_at: u64, + cancelled: Arc, + } + impl Read for Chunked { + fn read(&mut self, output: &mut [u8]) -> io::Result { + let length = output.len().min(self.chunk); + let read = self.source.read(&mut output[..length])?; + if self.source.position() >= self.cancel_at { self.cancelled.store(true, Ordering::Relaxed); } + Ok(read) + } + } + fn decoder(bytes: Vec, chunk: usize, cancel_at: u64) -> GzipContainer { + let cancelled = Arc::new(AtomicBool::new(false)); + let source = Chunked { source: Cursor::new(bytes), chunk, cancel_at, cancelled: Arc::clone(&cancelled) }; + GzipContainer::new(source, cancelled) + } + #[test] + fn preserves_member_boundaries_with_short_reads_and_complete_padding() { + for chunk in [1, 7, 511, 65536] { + for padding in [0, 1, 511, 512, 513, 10240 - 2 * MEMBER.len()] { + let bytes = [MEMBER, MEMBER, &vec![0; padding]].concat(); + let mut output = Vec::new(); + decoder(bytes, chunk, u64::MAX).read_to_end(&mut output).unwrap(); + assert_eq!(output, b"abcabc"); + } + } + } + #[test] + fn padding_never_masks_a_bad_trailer_following_member_or_nonzero_tail() { + let mut corrupt = MEMBER.to_vec(); + corrupt[MEMBER.len() - 8] ^= 1; + for bytes in [ + [corrupt.as_slice(), &[0; 10240]].concat(), + [MEMBER, &[0; 513], &[1]].concat(), + [MEMBER, &[31, 139]].concat(), + [MEMBER, corrupt.as_slice()].concat(), + [MEMBER, &[0], MEMBER].concat(), + ] { + for chunk in [1, 7, 65536] { + let mut reader = decoder(bytes.clone(), chunk, u64::MAX); + assert!(reader.read_to_end(&mut Vec::new()).is_err()); + assert!(reader.read(&mut [0; 1]).is_err()); + } + } + } + #[test] + fn cancellation_interrupts_physical_padding_drain() { + let bytes = [MEMBER, &[0; 10240]].concat(); + let mut reader = decoder(bytes, 1, MEMBER.len() as u64 + 4); + let error = reader.read_to_end(&mut Vec::new()).unwrap_err(); + assert!(error.to_string().contains("archive operation aborted")); + } +} diff --git a/native/src/lib.rs b/native/src/lib.rs index 68039330..48a7de27 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -4,14 +4,12 @@ use napi::bindgen_prelude::*; use napi_derive::napi; mod archive; +mod archive_gzip; mod fast_file; mod owned_tree; #[cfg(unix)] mod staged_file; -mod tar_meter; -mod tar_mode; -mod tar_path; -mod tar_pax; +use fs_safe_archive_core::tar_meter; #[cfg(unix)] mod unix; #[cfg(windows)] diff --git a/package.json b/package.json index 8588b7b3..68f5ff81 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "provenance": true }, "files": [ + "dist/archive-parser.wasm", "dist/**/*.js", "dist/**/*.d.ts", "dist/**/*.d.ts.map", @@ -126,7 +127,7 @@ "scripts": { "benchmark": "node scripts/benchmark.mjs", "benchmark:publish": "pnpm build && node scripts/bench-publish.mjs", - "build": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.json", + "build": "node scripts/prepack-build.mjs", "lint:file-size": "node scripts/check-file-size.mjs", "lint:fs-boundary": "node scripts/check-fs-boundary-primitives.mjs", "prepack": "node scripts/prepack-build.mjs", @@ -139,7 +140,7 @@ "docs:check": "node scripts/check-doc-examples.mjs", "docs:site": "node scripts/build-docs-site.mjs", "native:build": "pnpm --filter @openclaw/fs-safe-native-build build", - "native:test": "cargo test --manifest-path native/Cargo.toml", + "native:test": "cargo test --workspace", "pack:check": "pnpm build && node scripts/check-pack.mjs", "public-api:update": "pnpm build && node scripts/check-pack.mjs --update-public-api", "package:collect": "node scripts/check-release-packages.mjs --output release-artifacts", @@ -150,7 +151,9 @@ "crabbox:hydrate": "crabbox actions hydrate", "crabbox:run": "crabbox run", "crabbox:stop": "crabbox stop", - "crabbox:warmup": "crabbox warmup" + "crabbox:warmup": "crabbox warmup", + "archive:wasm": "node scripts/build-archive-wasm.mjs", + "archive:producer-smoke": "node scripts/archive-producer-smoke.mjs" }, "optionalDependencies": { "@openclaw/fs-safe-darwin-arm64": "0.8.1", @@ -160,8 +163,7 @@ "@openclaw/fs-safe-linux-x64-gnu": "0.8.1", "@openclaw/fs-safe-linux-x64-musl": "0.8.1", "@openclaw/fs-safe-win32-x64-msvc": "0.8.1", - "jszip": "^3.10.1", - "tar": "7.5.22" + "jszip": "^3.10.1" }, "devDependencies": { "@emnapi/runtime": "2.0.0-alpha.4", @@ -173,6 +175,7 @@ "istanbul-lib-report": "3.0.1", "istanbul-reports": "3.2.0", "sigstore": "5.0.0", + "tar": "7.5.22", "typescript": "^7.0.2", "vite": "8.2.2", "vitest": "^4.1.11" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index db4dd243..612d5c44 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,6 +35,9 @@ importers: sigstore: specifier: 5.0.0 version: 5.0.0(supports-color@7.2.0) + tar: + specifier: 7.5.22 + version: 7.5.22 typescript: specifier: ^7.0.2 version: 7.0.2 @@ -69,9 +72,6 @@ importers: jszip: specifier: ^3.10.1 version: 3.10.1 - tar: - specifier: 7.5.22 - version: 7.5.22 native: {} @@ -2033,7 +2033,6 @@ snapshots: '@isaacs/fs-minipass@4.0.1': dependencies: minipass: 7.1.3 - optional: true '@jridgewell/resolve-uri@3.1.2': {} @@ -2649,8 +2648,7 @@ snapshots: chardet@2.2.0: {} - chownr@3.0.0: - optional: true + chownr@3.0.0: {} cli-width@4.1.0: {} @@ -3084,7 +3082,6 @@ snapshots: minipass: 7.1.3 minizlib: 3.1.0 yallist: 5.0.0 - optional: true tinybench@2.9.0: {} @@ -3188,5 +3185,4 @@ snapshots: yallist@4.0.0: {} - yallist@5.0.0: - optional: true + yallist@5.0.0: {} diff --git a/scripts/archive-producer-smoke.mjs b/scripts/archive-producer-smoke.mjs new file mode 100644 index 00000000..5f6389bf --- /dev/null +++ b/scripts/archive-producer-smoke.mjs @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import * as tar from "tar"; + +// Run against an already installed packed consumer, never a workspace source import. +const consumer = process.argv[2]; +if (!consumer) throw new Error("usage: pnpm archive:producer-smoke [off|require]"); +const mode = process.argv[3] ?? "off"; +if (!["off", "require"].includes(mode)) throw new Error("mode must be off or require"); +const require = createRequire(path.resolve(consumer, "package.json")); +const manifest = require.resolve("@openclaw/fs-safe/package.json"); +const packageDir = path.dirname(manifest); +const { configureFsSafeNative } = await import(pathToFileURL(require.resolve("@openclaw/fs-safe/config"))); +const { extractArchive, readArchiveEntry } = await import(pathToFileURL(require.resolve("@openclaw/fs-safe/archive"))); +assert.equal(JSON.parse(await fs.readFile(manifest, "utf8")).optionalDependencies.tar, undefined); +await fs.access(path.join(packageDir, "dist/archive-parser.wasm")); +configureFsSafeNative({ mode }); +const scratch = await fs.mkdtemp(path.join(tmpdir(), "fs-safe-producers-")); +const digest = (value) => createHash("sha256").update(value).digest("hex"); +try { + const source = path.join(scratch, "source"); + await fs.mkdir(source); + const names = ["雪.txt", "café", "\ufeffBOM", "01", "long-" + "a".repeat(130), ...(process.platform === "win32" ? [] : ["line\n.txt"])]; + for (const name of names) await fs.writeFile(path.join(source, name), `synthetic:${name}`); + const producers = process.platform === "win32" ? ["npm"] : ["system", "npm"]; + const observations = []; + for (const producer of producers) { + const archivePath = path.join(scratch, `${producer}.tgz`); + if (producer === "system") { + const canonical = await fs.realpath(source), stat = await fs.stat(source, { bigint: true }); + const bytes = execFileSync(process.execPath, [fileURLToPath(new URL("./archive-system-tar-worker.cjs", import.meta.url)), + source, canonical, String(stat.dev), String(stat.ino)], { maxBuffer: 4 * 1024 * 1024, timeout: 30000 }); + await fs.writeFile(archivePath, bytes); + } + else await tar.c({ file: archivePath, cwd: source, portable: true, gzip: true }, names); + const encoded = await fs.readFile(archivePath); + observations.push({ producer, bytes: encoded.length, sha256: digest(encoded) }); + const destDir = path.join(scratch, `${producer}-out`); + await fs.mkdir(destDir); + await extractArchive({ archivePath, destDir, timeoutMs: 15000 }); + const actual = await fs.readdir(destDir); + assert.deepEqual(actual.filter((name) => !name.startsWith("._")).sort(), [...names].sort()); + const companions = producer === "system" && process.platform === "darwin" + ? new Set(["._.", ...names.map((name) => `._${name}`)]) : new Set(); + for (const name of actual.filter((name) => name.startsWith("._"))) { + assert.ok(companions.has(name), "unexpected producer companion name"); + const bytes = await fs.readFile(path.join(destDir, name)); + // AppleDouble signature (RFC 1740); companion values are producer metadata. + assert.equal(bytes.readUInt32BE(0), 0x00051607); + assert.equal(digest(await readArchiveEntry(archivePath, name, { maxBytes: bytes.length })), digest(bytes)); + } + for (const name of names) { + const original = await fs.readFile(path.join(source, name)); + assert.equal(digest(await fs.readFile(path.join(destDir, name))), digest(original)); + assert.equal(digest(await readArchiveEntry(archivePath, name, { maxBytes: original.length })), digest(original)); + } + } + // Valid effective PAX metadata never excuses an invalid raw fallback field. + const metadata = Buffer.from(new tar.Pax({ path: "safe" }).encodeBody()); + function header(name, type, size) { + const h = new tar.Header({ path: name, type, size, mode: 0o644 }); + h.encode(); + return Buffer.from(h.block); + } + const raw = header("raw", "File", 0); + raw[0] = 0xff; + raw.fill(32, 148, 156); + raw.write(`${raw.reduce((sum, byte) => sum + byte, 0).toString(8).padStart(6, "0")}\0 `, 148); + const invalid = path.join(scratch, "invalid.tar"); + await fs.writeFile(invalid, Buffer.concat([header("PaxHeader", "ExtendedHeader", metadata.length), metadata, + Buffer.alloc((512 - metadata.length % 512) % 512), raw, Buffer.alloc(1024)])); + const destDir = path.join(scratch, "invalid-out"); + await fs.mkdir(destDir); + await assert.rejects(extractArchive({ archivePath: invalid, destDir, timeoutMs: 10000 }), { code: "entry-path" }); + await assert.rejects(readArchiveEntry(invalid, "safe", { maxBytes: 0 }), { code: "entry-path" }); + assert.deepEqual(await fs.readdir(destDir), []); + console.log(JSON.stringify({ result: "pass", mode, producers, observations, systemRoute: process.platform === "win32" ? "skipped (POSIX only)" : "bound cwd /usr/bin/tar -czf - . stdout", filenames: names.length, invalidRawRejected: true })); +} finally { await fs.rm(scratch, { recursive: true, force: true }); } diff --git a/scripts/archive-system-tar-worker.cjs b/scripts/archive-system-tar-worker.cjs new file mode 100644 index 00000000..eee5a940 --- /dev/null +++ b/scripts/archive-system-tar-worker.cjs @@ -0,0 +1,10 @@ +const fs = require("node:fs"); +const { spawn } = require("node:child_process"); +const [directory, canonical, device, inode] = process.argv.slice(2); +try { process.chdir(directory); } catch { process.exit(1); } +if (fs.realpathSync(".") !== canonical) process.exit(78); +const bound = fs.statSync(".", { bigint: true }); +if (String(bound.dev) !== device || String(bound.ino) !== inode) process.exit(78); +const child = spawn("/usr/bin/tar", ["-czf", "-", "."], { stdio: ["ignore", "inherit", "inherit"] }); +child.once("error", () => process.exit(1)); +child.once("exit", (code) => process.exit(code ?? 1)); diff --git a/scripts/build-archive-wasm.mjs b/scripts/build-archive-wasm.mjs new file mode 100644 index 00000000..803e23eb --- /dev/null +++ b/scripts/build-archive-wasm.mjs @@ -0,0 +1,14 @@ +import { spawnSync } from "node:child_process"; +import { copyFileSync, mkdirSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const result = spawnSync("cargo", ["rustc", "--locked", "--release", "-p", "fs-safe-archive-wasm", + "--target", "wasm32-unknown-unknown", "--", "-C", "link-arg=--max-memory=268435456"], { + stdio: "inherit", env: process.env, +}); +if (result.status !== 0) throw new Error("TAR WASM build failed; install Rust and the wasm32-unknown-unknown target"); +const artifact = resolve(process.env.CARGO_TARGET_DIR ?? "target", "wasm32-unknown-unknown/release/fs_safe_archive_wasm.wasm"); +const module = new WebAssembly.Module(readFileSync(artifact)); +if (WebAssembly.Module.imports(module).length !== 0) throw new Error("TAR WASM must have no host imports"); +mkdirSync("dist", { recursive: true }); +copyFileSync(artifact, "dist/archive-parser.wasm"); diff --git a/scripts/check-pack.mjs b/scripts/check-pack.mjs index 9b9af802..777bd40b 100644 --- a/scripts/check-pack.mjs +++ b/scripts/check-pack.mjs @@ -80,6 +80,7 @@ try { const paths = new Set(files.map((file) => file.path)); const expected = new Set([ "CHANGELOG.md", + "dist/archive-parser.wasm", "docs/assets/readme-banner.jpg", "LICENSE", "README.md", diff --git a/scripts/consumer-install-probe.mjs b/scripts/consumer-install-probe.mjs index ae619f7a..7ef81b07 100644 --- a/scripts/consumer-install-probe.mjs +++ b/scripts/consumer-install-probe.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { readFileSync, readdirSync, realpathSync, writeFileSync } from "node:fs"; +import { readFileSync, readdirSync, realpathSync, writeFileSync, mkdirSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, isAbsolute, join, relative } from "node:path"; @@ -54,8 +54,9 @@ if (expected.omitted) { assert.throws(() => rootRequire.resolve(name), { code: "MODULE_NOT_FOUND" }); } } else { - for (const name of ["jszip", "tar"]) insideConsumer(rootRequire.resolve(name)); + insideConsumer(rootRequire.resolve("jszip")); } +assert.throws(() => rootRequire.resolve("tar"), { code: "MODULE_NOT_FOUND" }); for (const subpath of Object.keys(expected.rootPkg.exports)) { if (subpath !== "./package.json") { await import(subpath === "." ? expected.rootPkg.name : expected.rootPkg.name + subpath.slice(1)); @@ -65,3 +66,22 @@ writeFileSync("installed.json", JSON.stringify({ root: expected.rootPkg.name, version: expected.rootPkg.version, nativePackages: [...physical], binary, })); + +// The bundled TAR parser works even in an install with every optional omitted. +const { configureFsSafeNative } = await import("@openclaw/fs-safe/config"); +const { extractArchive, readArchiveEntry } = await import("@openclaw/fs-safe/archive"); +configureFsSafeNative({ mode: "off" }); +const header = Buffer.alloc(512); +header.write("雪.txt"); +header.write("0000644\0", 100); +header.write("00000000003\0", 124); +header[156] = 48; +header.fill(32, 148, 156); +header.write(`${header.reduce((sum, byte) => sum + byte, 0).toString(8).padStart(6, "0")}\0 `, 148); +const archivePath = join(consumer, "bundled.tar"); +const destDir = join(consumer, "bundled-out"); +writeFileSync(archivePath, Buffer.concat([header, Buffer.from("TAR"), Buffer.alloc(509 + 1024)])); +mkdirSync(destDir); +await extractArchive({ archivePath, destDir, timeoutMs: 10000 }); +assert.equal(readFileSync(join(destDir, "雪.txt"), "utf8"), "TAR"); +assert.equal((await readArchiveEntry(archivePath, "雪.txt", { maxBytes: 3 })).toString(), "TAR"); diff --git a/scripts/prepack-build.mjs b/scripts/prepack-build.mjs index 06476de0..6be55563 100644 --- a/scripts/prepack-build.mjs +++ b/scripts/prepack-build.mjs @@ -29,4 +29,5 @@ const result = spawnSync(process.execPath, [tscBin, "-p", "tsconfig.json"], { stdio: "inherit", env: process.env, }); -if (result.status !== 0) process.exitCode = result.status ?? 1; +if (result.status !== 0) process.exit(result.status ?? 1); +await import("./build-archive-wasm.mjs"); diff --git a/src/archive-gzip-tail.ts b/src/archive-gzip-tail.ts new file mode 100644 index 00000000..ffd98810 --- /dev/null +++ b/src/archive-gzip-tail.ts @@ -0,0 +1,70 @@ +import fs from "node:fs/promises"; +import { Writable } from "node:stream"; +import type { Gunzip } from "node:zlib"; +import { ArchiveFormatError } from "./archive-errors.js"; + +/** Track physical input, not the sum of bytes consumed across separate writes: + * gunzip can resume on a later chunk after leaving an earlier padding gap. */ +export class GzipInput extends Writable { + private position = 0; + private firstUnused: number | undefined; + + constructor(private readonly decoder: Gunzip) { + super({ highWaterMark: 65536 }); + } + + get tailOffset(): number { + return this.firstUnused ?? this.position; + } + + override _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void { + const before = this.decoder.bytesWritten; + this.decoder.write(chunk, (error) => { + if (error) { callback(error); return; } + const used = this.decoder.bytesWritten - before; + if (!Number.isSafeInteger(used) || used < 0 || used > chunk.length || + !Number.isSafeInteger(this.position + chunk.length)) { + callback(new ArchiveFormatError("invalid gzip consumed-input boundary")); + return; + } + if (used < chunk.length) this.firstUnused ??= this.position + used; + this.position += chunk.length; + callback(); + }); + } + + override _final(callback: (error?: Error | null) => void): void { + this.decoder.end(callback); + } + + override _destroy(error: Error | null, callback: (error: Error | null) => void): void { + if (error || !this.writableFinished) this.decoder.destroy(error ?? undefined); + callback(error); + } +} + +/** Check the immutable staged suffix from the first unused physical byte, + * including later chunks the decoder may have consumed, before returning. */ +export async function validateGzipContainerTail(filePath: string, consumed: number, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const handle = await fs.open(filePath, "r"); + try { + const { size } = await handle.stat(); + if (!Number.isSafeInteger(consumed) || consumed <= 0 || consumed > size) { + throw new ArchiveFormatError("invalid gzip consumed-input boundary"); + } + if (consumed === size) return; + const buffer = Buffer.allocUnsafe(65536); + let position = consumed; + while (position < size) { + signal?.throwIfAborted(); + const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, size - position), position); + signal?.throwIfAborted(); + if (bytesRead === 0) throw new ArchiveFormatError("truncated gzip container padding"); + if (buffer.subarray(0, bytesRead).some((byte) => byte !== 0)) { + throw new ArchiveFormatError("nonzero gzip container padding"); + } + position += bytesRead; + } + } finally { await handle.close(); } +} diff --git a/src/archive-read.ts b/src/archive-read.ts index f5ef9992..36b7d75c 100644 --- a/src/archive-read.ts +++ b/src/archive-read.ts @@ -15,13 +15,11 @@ import { import { resolveArchiveKind, type ArchiveKind } from "./archive-kind.js"; import { DEFAULT_MAX_ARCHIVE_BYTES_ZIP, - DEFAULT_MAX_META_ENTRY_BYTES, ArchiveLimitError, ARCHIVE_LIMIT_ERROR_CODE, } from "./archive-limits.js"; -import { readTarEntryInfo } from "./archive-tar.js"; -import { preflightTarMetadata } from "./archive-tar-meta.js"; -import { importOptionalTar, NODE_TAR_RATIO_DISABLED, normalizeTarParserError } from "./archive-tar-runtime.js"; +import { inspectTar, replayTar } from "./archive-tar-stream.js"; +import type { AdmittedTarMember } from "./archive-tar-wasm.js"; import { loadZipArchiveWithPreflight } from "./archive-zip-preflight.js"; import { createZipIntegrityTransform, @@ -169,72 +167,27 @@ async function readZipEntry(buffer: Buffer, entryPath: string, maxBytes: number) } async function readTarEntry(archivePath: string, entryPath: string, maxBytes: number): Promise { - const tar = await importOptionalTar(); const seenPaths = new Set(); - await preflightTarMetadata({ - archivePath, - limits: resolveTarMeterLimits(), - onMember(info) { - const normalized = canonicalEntryPath(info.path); - if (seenPaths.has(normalized)) { - throw new ArchiveSecurityError("entry-path", `archive contains duplicate entry path: ${formatErrorDetail(normalized)}`); - } - seenPaths.add(normalized); - if (normalized === entryPath && !["File", "OldFile", "ContiguousFile"].includes(info.type)) { - throw new Error(`archive entry is not a file: ${formatErrorDetail(entryPath)}`); - } - }, - }); - let matched: Promise | undefined; - let entryError: Error | undefined; - try { - await tar.t({ - file: archivePath, - strict: true, - maxMetaEntrySize: DEFAULT_MAX_META_ENTRY_BYTES, - maxDecompressionRatio: NODE_TAR_RATIO_DISABLED, - onReadEntry(entry) { - const info = readTarEntryInfo(entry); - let normalized: string; - try { - normalized = canonicalEntryPath(info.path); - } catch (error) { - // Throws escaping node-tar's callback do not reject its promise. - entryError ??= error instanceof Error ? error : new Error(String(error)); - entry.resume(); - return; - } - if (normalized !== entryPath) { - entry.resume(); - return; - } - if (info.type !== "File" && info.type !== "OldFile" && info.type !== "ContiguousFile") { - entryError ??= new Error( - `archive entry is not a file: ${formatErrorDetail(entryPath)}`, - ); - entry.resume(); - return; - } - if (info.size > maxBytes) { - entryError ??= new ArchiveLimitError( - ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT, - ); - entry.resume(); - return; - } - matched = readStreamBounded(entry, maxBytes); - }, - }); - } catch (error) { - throw normalizeTarParserError(error); - } - if (entryError) { - throw entryError; - } - if (!matched) { - throw new Error(`archive entry not found: ${formatErrorDetail(entryPath)}`); + let selected: AdmittedTarMember | undefined; + const limits = resolveTarMeterLimits(); + await inspectTar({ archivePath, limits, onMember(info) { + const normalized = canonicalEntryPath(info.path); + if (seenPaths.has(normalized)) { + throw new ArchiveSecurityError("entry-path", `archive contains duplicate entry path: ${formatErrorDetail(normalized)}`); + } + seenPaths.add(normalized); + if (normalized === entryPath) selected = info; + } }); + if (!selected) throw new Error(`archive entry not found: ${formatErrorDetail(entryPath)}`); + if (!["File", "OldFile", "ContiguousFile"].includes(selected.type)) { + throw new Error(`archive entry is not a file: ${formatErrorDetail(entryPath)}`); } - return await matched; + if (selected.size > maxBytes) throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT); + let result: Buffer | undefined; + await replayTar({ archivePath, limits, members: [selected], async consume(_member, payload) { + result = await readStreamBounded(payload, maxBytes); + } }); + return result!; } export async function readArchiveEntry( diff --git a/src/archive-tar-admission.ts b/src/archive-tar-admission.ts deleted file mode 100644 index 7f7d4632..00000000 --- a/src/archive-tar-admission.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { ArchiveFormatError } from "./archive-errors.js"; -import { stripArchivePath, validateArchiveEntryPath } from "./archive-entry.js"; -import type { TarEntryInfo } from "./archive-tar.js"; -import { readTarHeaderPaths } from "./archive-tar-header.js"; - -// node-tar's normalFsTypes/ReadEntry switch. All other non-metadata types -// bypass both its filter and entry event; they still belong to our manifest. -// Header.decode normalizes NUL to "0" before choosing the emitted File type. -const visibleTypes = new Map([ - [0, "File"], [0x30, "File"], [0x31, "Link"], [0x32, "SymbolicLink"], - [0x33, "CharacterDevice"], [0x34, "BlockDevice"], [0x35, "Directory"], - [0x36, "FIFO"], [0x37, "ContiguousFile"], [0x44, "GNUDumpDir"], -]); - -export function rawTarMember(header: Buffer, size: number, effectivePath?: string): TarEntryInfo { - const { name, prefix } = readTarHeaderPaths(header); - validateArchiveEntryPath(name); - validateArchiveEntryPath(prefix); - const rawPath = prefix ? `${prefix}/${name}` : name; - validateArchiveEntryPath(rawPath); - const path = effectivePath ?? rawPath; - validateArchiveEntryPath(path); - return { path, size, type: visibleTypes.get(header[156]!) ?? "Unsupported" }; -} - -export function createTarAdmissionPlan( - manifest: readonly TarEntryInfo[], - check: (entry: TarEntryInfo) => boolean, - strip: number, -): { consume(entry: TarEntryInfo): string | null; finish(): void } { - const visible: Array<{ entry: TarEntryInfo; output: string | null }> = []; - for (const entry of manifest) { - const accepted = check(entry); - if (entry.type !== "Unsupported") { - visible.push({ entry, output: accepted ? stripArchivePath(entry.path, strip) : null }); - } - } - let index = 0; - const mismatch = () => new ArchiveFormatError("invalid TAR header: parser disagrees with raw admission"); - return { - consume(actual) { - const expected = visible[index++]; - if (!expected || expected.entry.type !== actual.type || expected.entry.size !== actual.size || - stripArchivePath(expected.entry.path, 0) !== stripArchivePath(actual.path, 0)) throw mismatch(); - return expected.output; - }, - finish() { if (index !== visible.length) throw mismatch(); }, - }; -} diff --git a/src/archive-tar-extract.ts b/src/archive-tar-extract.ts new file mode 100644 index 00000000..fe7a4267 --- /dev/null +++ b/src/archive-tar-extract.ts @@ -0,0 +1,56 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { Readable } from "node:stream"; +import { stripArchivePath } from "./archive-entry.js"; +import { type ExtractionDeadline } from "./archive-deadline.js"; +import type { ResolvedArchiveExtractLimits, TarMeterLimits } from "./archive-limits.js"; +import { mergePlannedArchiveIntoDestination } from "./archive-merge.js"; +import type { ExtractArchiveOptions } from "./archive-options.js"; +import { resolveArchiveEntryMode } from "./archive-policy.js"; +import { prepareArchiveDestinationDir, preparePrivateArchiveOutputPath, withStagedArchiveDestination } from "./archive-staging.js"; +import { createTarEntryPreflightChecker } from "./archive-tar.js"; +import { inspectTar, replayTar } from "./archive-tar-stream.js"; +import type { AdmittedTarMember } from "./archive-tar-wasm.js"; +import { runPinnedWriteHelper } from "./pinned-write.js"; + +export async function extractWasmTar(params: { + archivePath: string; options: ExtractArchiveOptions; limits: ResolvedArchiveExtractLimits; + tarLimits: TarMeterLimits; deadline: ExtractionDeadline; +}): Promise { + const { options, deadline, tarLimits } = params; + const manifest: AdmittedTarMember[] = []; + await inspectTar({ archivePath: params.archivePath, limits: tarLimits, signal: deadline.signal, + onMember: (entry) => { manifest.push(entry); } }); + deadline.check(); + const destinationRealDir = await prepareArchiveDestinationDir(options.destDir); + await withStagedArchiveDestination({ destinationRealDir, run: async (stagingPath) => { + const stagingDir = await fs.realpath(stagingPath); + const check = createTarEntryPreflightChecker({ rootDir: destinationRealDir, + stripComponents: options.stripComponents, limits: params.limits, + entryFilter: options.entryFilter, onFiltered: options.onFiltered }); + const strip = Math.max(0, Math.floor(options.stripComponents ?? 0)); + const accepted = manifest.filter((entry) => { deadline.check(); return check(entry); }).map((entry) => { + const kind = entry.type === "Directory" || entry.type === "GNUDumpDir" ? "directory" as const : "file" as const; + return { ...entry, path: stripArchivePath(entry.path, strip)!, kind, + mode: resolveArchiveEntryMode({ kind, archivedMode: entry.mode, policy: options.entryModes }) }; + }); + await replayTar({ archivePath: params.archivePath, limits: tarLimits, signal: deadline.signal, members: accepted, + async consume(member, payload) { + deadline.check(); + await preparePrivateArchiveOutputPath({ destinationDir: stagingDir, destinationRealDir: stagingDir, + relPath: member.path, outPath: path.join(stagingDir, member.path), originalPath: member.path, + isDirectory: member.kind === "directory", deadline }); + if (member.kind === "file") { + await runPinnedWriteHelper({ rootPath: stagingDir, relativeParentPath: path.posix.dirname(member.path), + basename: path.posix.basename(member.path), mkdir: false, mode: 0o600, overwrite: false, + maxBytes: member.size, input: { kind: "stream", stream: Readable.from(payload) } }); + } + deadline.check(); + }, + }); + deadline.check(); + await mergePlannedArchiveIntoDestination({ entries: accepted, sourceDir: stagingDir, + destinationDir: options.destDir, destinationRealDir, deadline }); + deadline.check(); + } }); +} diff --git a/src/archive-tar-gnu.ts b/src/archive-tar-gnu.ts deleted file mode 100644 index 233b7bd6..00000000 --- a/src/archive-tar-gnu.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { ArchiveFormatError } from "./archive-errors.js"; -import { validateArchiveEntryPath } from "./archive-entry.js"; - -// Keep the byte grammar aligned with native/src/tar_meter.rs. Parsers differ -// on embedded NULs and malformed UTF-8, so validate before either sees a name. -export function validateGnuMetadata(body: Buffer, type: "L" | "K"): string { - const value = body.at(-1) === 0 ? body.subarray(0, -1) : body; - if (!value.length || value.includes(0)) { - throw new ArchiveFormatError("invalid GNU metadata: empty name or embedded NUL"); - } - let name: string; - try { - name = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(value); - } catch { - throw new ArchiveFormatError("invalid GNU metadata: name is not valid UTF-8"); - } - if (type === "L") validateArchiveEntryPath(name); - return name; -} diff --git a/src/archive-tar-header.ts b/src/archive-tar-header.ts deleted file mode 100644 index 1af4e7f8..00000000 --- a/src/archive-tar-header.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { ArchiveFormatError, ArchiveSecurityError } from "./archive-errors.js"; -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); - -export function validateTarChecksum(header: Buffer): void { - const field = header.subarray(148, 156); - const end = field.indexOf(0); - const digits = field.subarray(0, end < 0 ? field.length : end).toString("latin1").replace(/^ +| +$/g, ""); - const sum = header.reduce((total, byte, index) => total + (index >= 148 && index < 156 ? 32 : byte), 0); - // Require an in-field delimiter: node-tar otherwise reads into the typeflag. - if (!/^[0-7]+$/.test(digits) || ![0, 32].includes(field[7]!) || - (end >= 0 && field.subarray(end).some((byte) => byte !== 0 && byte !== 32)) || - Number.parseInt(digits, 8) !== sum) { - throw new ArchiveFormatError("invalid TAR header: checksum failure"); - } -} - -function fixedField(field: Buffer): string { - const zero = field.indexOf(0); - if (zero >= 0 && field.subarray(zero).some((byte) => byte !== 0)) { - throw new ArchiveSecurityError("entry-path", "tar entry path contains bytes after NUL"); - } - try { - return utf8.decode(field.subarray(0, zero < 0 ? field.length : zero)); - } catch { - throw new ArchiveSecurityError("entry-path", "tar entry path is not valid UTF-8"); - } -} - -export function readTarHeaderPaths(header: Buffer): { name: string; prefix: string; linkname: string } { - // node-tar's star layout uses a 130-byte prefix followed by atime/ctime. - const prefixEnd = header[475] === 0 ? 475 : 500; - return { - name: fixedField(header.subarray(0, 100)), - prefix: header.subarray(257, 265).equals(Buffer.from("ustar\0" + "00")) - ? fixedField(header.subarray(345, prefixEnd)) : "", - linkname: fixedField(header.subarray(157, 257)), - }; -} - -export function validateTarHeader(header: Buffer): void { - validateTarChecksum(header); - const { linkname } = readTarHeaderPaths(header); - const isLink = header[156] === 0x31 || header[156] === 0x32; - if (isLink && !linkname) { - throw new ArchiveFormatError("invalid TAR header: linkname required on a link header"); - } - if (!isLink && linkname) { - throw new ArchiveFormatError("invalid TAR header: linkname forbidden on a non-link header"); - } -} diff --git a/src/archive-tar-meta.ts b/src/archive-tar-meta.ts deleted file mode 100644 index 95e84dea..00000000 --- a/src/archive-tar-meta.ts +++ /dev/null @@ -1,274 +0,0 @@ -import fs from "node:fs"; -import { Transform, Writable } from "node:stream"; -import { pipeline } from "node:stream/promises"; -import { createGunzip } from "node:zlib"; -import { ArchiveFormatError } from "./archive-errors.js"; -import { ARCHIVE_LIMIT_ERROR_CODE, ArchiveLimitError, tarManifestEntryCost, type TarMeterLimits } from "./archive-limits.js"; -import { parseLocalPax, paxMemberSize, type LocalPax } from "./archive-tar-pax.js"; -import { validateGnuMetadata } from "./archive-tar-gnu.js"; -import { rawTarMember } from "./archive-tar-admission.js"; -import type { TarEntryInfo } from "./archive-tar.js"; -import { validateTarHeader } from "./archive-tar-header.js"; - -type MeterState = - | { kind: "header" } - | { kind: "eof" } - | { kind: "data"; remaining: number } - | { kind: "metadata"; type: "pax" | "L" | "K"; body: Buffer; used: number; padding: number } - | { kind: "sparse"; dataRemaining: number; metaBytes: number }; - -// Keep raw framing aligned with native/src/tar_meter.rs, before either parser. -export class TarMetadataMeter extends Transform { - private readonly block = Buffer.alloc(512); - private blockLength = 0; - private state: MeterState = { kind: "header" }; - private pendingPax: LocalPax | undefined; - private readonly pendingGnu = new Set<"L" | "K">(); - private pendingGnuPath: string | undefined; - private zeroBlocks = 0; - private entries = 0; - private remainingDecodedBytes: number; - private manifestBytes = 0; - - constructor(private readonly limits: TarMeterLimits, private readonly onMember?: (entry: TarEntryInfo) => void) { - super(); - if (!Number.isSafeInteger(limits.maxDecodedBytes) || limits.maxDecodedBytes < 0) { - throw new RangeError("maxDecodedBytes must be a non-negative safe integer"); - } - this.remainingDecodedBytes = limits.maxDecodedBytes; - } - - private countMember(): void { - if (this.entries >= this.limits.maxEntries) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT); - } - this.entries += 1; - } - - private invalid(message: string): ArchiveFormatError { - return new ArchiveFormatError(`invalid TAR header: ${message}`); - } - - private paddedSize(size: number): number { - const padded = Math.ceil(size / 512) * 512; - if (!Number.isSafeInteger(padded)) throw this.invalid("entry padding exceeds the safe integer range"); - return padded; - } - - private parseSize(): number { - const field = this.block.subarray(124, 136); - if ((field[0] ?? 0) & 0x80) { - if (field[0] !== 0x80) { - throw this.invalid("base-256 size is negative or malformed"); - } - let value = 0n; - for (const byte of field.subarray(1)) { - value = (value << 8n) | BigInt(byte); - } - if (value > BigInt(Number.MAX_SAFE_INTEGER)) { - throw this.invalid("base-256 size exceeds the safe integer range"); - } - return Number(value); - } - const zero = field.indexOf(0); - if (zero >= 0 && field.subarray(zero).some((byte) => byte !== 0 && byte !== 0x20)) { - throw this.invalid("size has non-padding bytes after NUL"); - } - const bytes = field.subarray(0, zero < 0 ? field.length : zero); - if (bytes.some((byte) => byte > 0x7f)) throw this.invalid("size is not ASCII octal"); - const text = bytes.toString("ascii").replace(/^ +| +$/g, ""); - if (!text || !/^[0-7]+$/.test(text)) throw this.invalid("size is not valid octal"); - const value = Number.parseInt(text, 8); - if (!Number.isSafeInteger(value)) throw this.invalid("octal size exceeds the safe integer range"); - return value; - } - - private finishHeader(): void { - if (this.block.every((byte) => byte === 0)) { - if (this.pendingPax) throw this.invalid("dangling PAX metadata"); - if (this.pendingGnu.size) throw this.invalid("dangling GNU metadata"); - this.blockLength = 0; - this.zeroBlocks += 1; - this.state = { kind: this.zeroBlocks === 2 ? "eof" : "header" }; - return; - } - if (this.zeroBlocks !== 0) throw this.invalid("nonzero header after one TAR zero block"); - validateTarHeader(this.block); - const nameEnd = this.block.subarray(0, 100).indexOf(0); - const name = this.block.subarray(0, nameEnd < 0 ? 100 : nameEnd); - if (name.length === 0) { - throw this.invalid("entry path is empty"); - } - if (![0x35, 0x44].includes(this.block[156]!) && name.at(-1) === 0x2f) { - throw this.invalid("non-directory entry path ends with a separator"); - } - let size = this.parseSize(); - let padded = this.paddedSize(size); - const type = this.block[156]!; - // Directory/link headers cannot carry bodies; PAX/GNU metadata can. - if ([0x31, 0x32, 0x35].includes(type) && size !== 0) { - throw this.invalid("directory or link has a nonzero body size"); - } - if ([0x78, 0x67, 0x4c, 0x4b, 0x58, 0x4e].includes(type) && size > this.limits.maxMetaEntryBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT); - } - if (type === 0x67 || type === 0x58 || type === 0x4e) { - throw this.invalid("global/old PAX and old GNU metadata are not supported"); - } - if (type === 0x78 || type === 0x4c || type === 0x4b) { - const metadataType = type === 0x78 ? "pax" : type === 0x4c ? "L" : "K"; - if (this.pendingPax || size === 0 || (metadataType === "pax" ? this.pendingGnu.size : this.pendingGnu.has(metadataType))) { - throw this.invalid("empty, repeated or mixed PAX/GNU metadata"); - } - const magic = this.block.subarray(257, 265).toString("latin1"); - if (magic !== "ustar\0" + "00" && magic !== "ustar \0") throw this.invalid("unrecognized PAX/GNU header format"); - if (metadataType !== "pax") this.pendingGnu.add(metadataType); - this.state = { kind: "metadata", type: metadataType, body: Buffer.alloc(size), used: 0, padding: padded - size }; - this.blockLength = 0; - return; - } - // Sparse headers retain their metadata-limit-before-format-error ordering. - if (this.pendingPax && type !== 0x53) { - size = paxMemberSize(this.pendingPax, type, size, this.block); - } - padded = this.paddedSize(size); - if (type === 0x53) { - if (this.block[482] !== 0 && this.block[482] !== 1) { - throw this.invalid("GNU sparse extension flag is not 0 or 1"); - } - if (this.block[482] === 0) { - throw this.invalid("GNU sparse entries are not supported"); - } - this.state = { kind: "sparse", dataRemaining: padded, metaBytes: 0 }; - } else { - this.countMember(); - if (this.pendingGnuPath && /[\\/]$/.test(this.pendingGnuPath) && ![0x35, 0x44].includes(type)) { - throw this.invalid("GNU effective non-directory path ends with a separator"); - } - const entry = rawTarMember(this.block, size, this.pendingPax?.path ?? this.pendingGnuPath); - const cost = tarManifestEntryCost(entry.path); - if (cost > this.limits.maxManifestBytes - this.manifestBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.MANIFEST_SIZE_EXCEEDS_LIMIT); - } - this.manifestBytes += cost; - this.onMember?.(entry); - this.pendingPax = undefined; - this.pendingGnu.clear(); - this.pendingGnuPath = undefined; - this.state = padded === 0 ? { kind: "header" } : { kind: "data", remaining: padded }; - } - this.blockLength = 0; - } - - private finishSparseHeader(state: Extract): void { - const metaBytes = state.metaBytes + 512; - if (metaBytes > this.limits.maxMetaEntryBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT); - } - if (this.block[504] !== 0 && this.block[504] !== 1) { - throw this.invalid("GNU sparse extension flag is not 0 or 1"); - } - if (this.block[504] === 1) { - this.state = { ...state, metaBytes }; - } else { - throw this.invalid("GNU sparse entries are not supported"); - } - this.blockLength = 0; - } - - private meter(chunk: Buffer): void { - let offset = 0; - while (offset < chunk.length) { - if (this.state.kind === "eof") { - if (chunk.subarray(offset).some((byte) => byte !== 0)) throw this.invalid("nonzero data after TAR EOF"); - return; - } - if (this.state.kind === "metadata") { - const state = this.state; - const take = Math.min(state.body.length - state.used, chunk.length - offset); - chunk.copy(state.body, state.used, offset, offset + take); - state.used += take; - offset += take; - if (state.used === state.body.length) { - if (state.type === "pax") this.pendingPax = parseLocalPax(state.body); - else { - const name = validateGnuMetadata(state.body, state.type); - if (state.type === "L") this.pendingGnuPath = name; - } - this.state = state.padding === 0 ? { kind: "header" } : { kind: "data", remaining: state.padding }; - } - continue; - } - if (this.state.kind === "data") { - const take = Math.min(this.state.remaining, chunk.length - offset); - offset += take; - const remaining = this.state.remaining - take; - this.state = remaining === 0 ? { kind: "header" } : { kind: "data", remaining }; - continue; - } - const take = Math.min(512 - this.blockLength, chunk.length - offset); - chunk.copy(this.block, this.blockLength, offset, offset + take); - this.blockLength += take; - offset += take; - if (this.blockLength === 512) { - if (this.state.kind === "sparse") this.finishSparseHeader(this.state); - else this.finishHeader(); - } - } - } - - override _transform(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null, data?: Buffer) => void): void { - try { - const admitted = Math.min(chunk.length, this.remainingDecodedBytes); - this.meter(chunk.subarray(0, admitted)); - this.remainingDecodedBytes -= admitted; - if (admitted < chunk.length) { - // Inspect only the overflow probe, never scan an unbounded tail. - if (this.state.kind === "eof" && chunk[admitted] !== 0) throw this.invalid("nonzero data after TAR EOF"); - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.DECODED_SIZE_EXCEEDS_LIMIT); - } - callback(null, chunk); - } catch (error) { - callback(error instanceof Error ? error : new Error(String(error))); - } - } - - override _flush(callback: (error?: Error | null) => void): void { - if (this.pendingPax) callback(this.invalid("dangling PAX metadata")); - else if (this.pendingGnu.size) callback(this.invalid("dangling GNU metadata")); - else if (this.state.kind === "eof") callback(); - else if (this.state.kind === "header" && this.blockLength === 0) callback(this.invalid("missing two-block TAR EOF")); - else callback(this.invalid(this.state.kind === "header" ? "truncated TAR header" : "truncated TAR entry")); - } -} - -async function isGzip(filePath: string): Promise { - const handle = await fs.promises.open(filePath, "r"); - try { - const magic = Buffer.alloc(2); - const { bytesRead } = await handle.read(magic, 0, 2, 0); - return bytesRead === 2 && magic[0] === 0x1f && magic[1] === 0x8b; - } finally { - await handle.close(); - } -} - -export async function preflightTarMetadata(params: { - archivePath: string; - limits: TarMeterLimits; - signal?: AbortSignal; - onMember?: (entry: TarEntryInfo) => void; -}): Promise { - const meter = new TarMetadataMeter(params.limits, params.onMember); - const sink = new Writable({ write(_chunk, _encoding, callback) { callback(); } }); - const gzip = await isGzip(params.archivePath); - const decoder = gzip ? createGunzip() : undefined; - const input = fs.createReadStream(params.archivePath); - try { - await (decoder - ? pipeline(input, decoder, meter, sink, { signal: params.signal }) - : pipeline(input, meter, sink, { signal: params.signal })); - } finally { - input.destroy(); - } -} diff --git a/src/archive-tar-pax.ts b/src/archive-tar-pax.ts deleted file mode 100644 index 93965fea..00000000 --- a/src/archive-tar-pax.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { ArchiveFormatError } from "./archive-errors.js"; - -export type LocalPax = { path?: string; linkpath?: string; size?: number }; - -function invalid(): never { - throw new ArchiveFormatError("invalid TAR header: unsupported or malformed PAX metadata"); -} - -function decimal(bytes: Buffer): number { - const text = bytes.toString("latin1"); - if (!/^(0|[1-9][0-9]*)$/.test(text)) invalid(); - const value = Number(text); - if (!Number.isSafeInteger(value)) invalid(); - return value; -} - -function ascii(bytes: Buffer): string { - if (bytes.length === 0 || bytes.some((byte) => byte < 0x20 || byte > 0x7e)) invalid(); - return bytes.toString("ascii"); -} - -// Keep this grammar aligned with native/src/tar_pax.rs. Downstream parsers -// disagree on duplicates, coercions and UTF-8 chunk boundaries. -export function parseLocalPax(body: Buffer): LocalPax { - if (body.length === 0) invalid(); - const result: LocalPax = {}; - const keys = new Set(); - let offset = 0; - while (offset < body.length) { - const space = body.indexOf(0x20, offset); - if (space < 0) invalid(); - const length = decimal(body.subarray(offset, space)); - if (length > body.length - offset || length <= space - offset + 3) invalid(); - const end = offset + length; - if (body[end - 1] !== 0x0a) invalid(); - const record = body.subarray(space + 1, end - 1); - if (record.includes(0x0a)) invalid(); - const equals = record.indexOf(0x3d); - if (equals <= 0) invalid(); - const key = ascii(record.subarray(0, equals)); - if (!/^[A-Za-z0-9_.-]+$/.test(key) || keys.has(key)) invalid(); - keys.add(key); - const value = record.subarray(equals + 1); - if (/^(LIBARCHIVE|SCHILY)\.xattr\..+$/.test(key)) { - // Inert bytes, never restored. Newlines would corrupt Rust's subsequent - // record lookup; NUL and non-UTF8 bytes in these values are harmless. - } else if (key === "path" || key === "linkpath") { - result[key] = ascii(value); - } else if (key === "size") { - result.size = decimal(value); - } else if (key === "uid" || key === "gid") { - decimal(value); - } else if (key === "uname" || key === "gname") { - ascii(value); - } else if (key === "mtime" || key === "atime" || key === "ctime") { - const text = ascii(value); - if (!/^-?(0|[1-9][0-9]*)(\.[0-9]+)?$/.test(text) || Math.abs(Number(text)) > 8_640_000_000_000) invalid(); - } else { - invalid(); - } - offset = end; - } - return result; -} - -function rawText(field: Buffer): string { - const zero = field.indexOf(0); - const value = field.subarray(0, zero < 0 ? field.length : zero); - return value.length === 0 ? "" : ascii(value); -} - -export function paxMemberSize(pax: LocalPax, type: number, rawSize: number, header: Buffer): number { - if (![0, 0x30, 0x31, 0x32, 0x35, 0x37].includes(type)) invalid(); - const rawName = rawText(header.subarray(0, 100)); - const rawLink = rawText(header.subarray(157, 257)); - if (header.subarray(257, 265).equals(Buffer.from("ustar\0" + "00"))) { - rawText(header.subarray(345, 500)); - } - const isLink = type === 0x31 || type === 0x32; - if (isLink !== (rawLink.length > 0)) invalid(); - const size = pax.size ?? rawSize; - if ([0x31, 0x32, 0x35].includes(type) && (rawSize !== 0 || size !== 0)) invalid(); - if (type !== 0x35 && (pax.path?.endsWith("/") || pax.path?.endsWith("\\") || rawName.endsWith("\\"))) invalid(); - if (pax.linkpath !== undefined && !isLink) invalid(); - return size; -} diff --git a/src/archive-tar-runtime.ts b/src/archive-tar-runtime.ts deleted file mode 100644 index 4c326db4..00000000 --- a/src/archive-tar-runtime.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { ArchiveFormatError } from "./archive-errors.js"; - -export type TarParserEntry = { - meta?: boolean; - size: number; - type?: string; - resume(): void; -}; - -export type TarParser = NodeJS.WritableStream & { - abort(error: Error): void; - on(event: "ignoredEntry", listener: (entry: TarParserEntry) => void): TarParser; - on(event: "entry", listener: (entry: TarParserEntry) => void): TarParser; - on(event: "meta", listener: (metadata: string) => void): TarParser; - on(event: "error", listener: (error: Error) => void): TarParser; - on(event: "end", listener: () => void): TarParser; -}; - -// Both callers parse an immutable staged stream only after fs-safe's complete -// decoded-byte admission, so node-tar must not add backend-specific policy. -export const NODE_TAR_RATIO_DISABLED = Number.POSITIVE_INFINITY; - -export type TarModule = { - Parser: new (options: { strict: true; maxMetaEntrySize: number }) => TarParser; - x(options: { - cwd: string; - strip: number; - gzip?: boolean; - signal?: AbortSignal; - preservePaths: false; - noChmod: true; - preserveOwner: false; - noMtime: true; - dmode: number; - strict: true; - maxMetaEntrySize: number; - maxDecompressionRatio: number; - filter?(this: TarParser, entryPath: string, entry: unknown): boolean; - onReadEntry(this: unknown, entry: unknown): void; - }): TarParser; - t(options: { - file: string; - strict: true; - maxMetaEntrySize: number; - maxDecompressionRatio: number; - onReadEntry(entry: AsyncIterable & { resume(): void }): void; - }): Promise; -}; - -export async function importOptionalTar(): Promise { - try { - return await import("tar"); - } catch (cause) { - throw new Error( - 'Optional archive dependency "tar" is not installed. Install it to use TAR archive helpers from @openclaw/fs-safe/archive.', - { cause }, - ); - } -} - -export function normalizeTarParserError(error: unknown): unknown { - const code = (error as { code?: unknown } | null)?.code; - if (typeof code !== "string" || !code.startsWith("TAR_")) { - return error; - } - const message = error instanceof Error ? error.message : String(error); - return new ArchiveFormatError(`invalid TAR archive: ${message}`, { - cause: error instanceof Error ? error : undefined, - }); -} diff --git a/src/archive-tar-stream.ts b/src/archive-tar-stream.ts new file mode 100644 index 00000000..87303980 --- /dev/null +++ b/src/archive-tar-stream.ts @@ -0,0 +1,97 @@ +import fs from "node:fs"; +import { Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; +import { GzipInput, validateGzipContainerTail } from "./archive-gzip-tail.js"; +import { ArchiveFormatError } from "./archive-errors.js"; +import type { TarMeterLimits } from "./archive-limits.js"; +import { TarParserStream, type AdmittedTarMember } from "./archive-tar-wasm.js"; + +async function gzipFile(filePath: string): Promise { + const handle = await fs.promises.open(filePath, "r"); + try { + const magic = Buffer.alloc(2); + const { bytesRead } = await handle.read(magic, 0, 2, 0); + return bytesRead === 2 && magic[0] === 31 && magic[1] === 139; + } finally { await handle.close(); } +} + +async function withTarStream(params: { + archivePath: string; limits: TarMeterLimits; signal?: AbortSignal; + onMember?: (entry: AdmittedTarMember) => void; +}, consume: (parser: TarParserStream) => Promise): Promise { + const gzip = await gzipFile(params.archivePath); + const parser = new TarParserStream(params.limits, params.onMember); + const input = fs.createReadStream(params.archivePath, { highWaterMark: 65536 }); + const decoder = gzip ? createGunzip() : undefined; + const gzipInput = decoder ? new GzipInput(decoder) : undefined; + const destroy = (error?: Error) => { + input.destroy(error); gzipInput?.destroy(error); decoder?.destroy(error); parser.destroy(error); + }; + const pumps = decoder && gzipInput + ? [pipeline(decoder, parser, { signal: params.signal }), pipeline(input, gzipInput, { signal: params.signal })] + : [pipeline(input, parser, { signal: params.signal })]; + // Either pump tears down both routes; join every pump even after the first failure. + const settled = Promise.all(pumps.map((pump) => pump.then(() => undefined, (cause: unknown) => { + const error = cause instanceof Error ? cause : new Error(String(cause)); + destroy(error); + return error; + }))).then((errors) => errors.find((error) => error !== undefined)); + try { + const result = await consume(parser); + const error = await settled; + if (error) throw error; + if (gzipInput) await validateGzipContainerTail(params.archivePath, gzipInput.tailOffset, params.signal); + return result; + } finally { + destroy(); + await settled; + } +} + +export async function inspectTar(params: { + archivePath: string; limits: TarMeterLimits; signal?: AbortSignal; + onMember?: (entry: AdmittedTarMember) => void; +}): Promise { + await withTarStream(params, async (parser) => { + await pipeline(parser, new Writable({ write(_chunk, _encoding, callback) { callback(); } })); + }); +} + +/** Replay in physical order, retaining at most one decoded chunk. Every range + * comes from complete admission of the immutable staged input. */ +export async function replayTar(params: { + archivePath: string; limits: TarMeterLimits; signal?: AbortSignal; + members: readonly T[]; + consume(member: T, payload: AsyncIterable): Promise; +}): Promise { + await withTarStream(params, async (parser) => { + const iterator = parser[Symbol.asyncIterator](); + let chunk: Buffer = Buffer.alloc(0); + let position = 0; + async function* take(length: number): AsyncGenerator { + while (length > 0) { + if (!chunk.length) { + const next = await iterator.next(); + if (next.done) throw new ArchiveFormatError("truncated admitted TAR range"); + chunk = next.value as Buffer; + } + const count = Math.min(length, chunk.length); + const bytes = chunk.subarray(0, count); + chunk = chunk.subarray(count); + position += count; length -= count; + yield bytes; + } + } + for (const member of params.members) { + if (member.offset < position) throw new ArchiveFormatError("invalid admitted TAR range order"); + for await (const _ of take(member.offset - position)) { /* Skip admitted gaps. */ } + const payload = take(member.size); + await params.consume(member, payload); + for await (const _ of payload) { /* Directories may carry ignored dump data. */ } + if (position !== member.offset + member.size) throw new ArchiveFormatError("incomplete TAR range consumption"); + } + // Includes unrequested/skipped members, trailer checks, and physical EOF. + while (!(await iterator.next()).done) { /* Drain the bounded parser stream. */ } + }); +} diff --git a/src/archive-tar-wasm.ts b/src/archive-tar-wasm.ts new file mode 100644 index 00000000..d63d9ea7 --- /dev/null +++ b/src/archive-tar-wasm.ts @@ -0,0 +1,103 @@ +import { readFileSync } from "node:fs"; +import { Transform, type TransformCallback } from "node:stream"; +import { ArchiveFormatError, ArchiveSecurityError, isArchiveTarPathErrorMessage } from "./archive-errors.js"; +import { ARCHIVE_LIMIT_ERROR_CODE, ArchiveLimitError, type TarMeterLimits } from "./archive-limits.js"; +import type { TarEntryInfo } from "./archive-tar.js"; + +export type AdmittedTarMember = TarEntryInfo & { offset: number }; +type Abi = { + memory: { buffer: ArrayBuffer }; + input_ptr(): number; + init(entries: number, metadata: number, decoded: number, manifest: number, windows: number): number; + push(length: number): number; + finish(): number; + dispose(): void; + text_ptr(): number; + text_len(): number; + member_type(): number; + member_size(): number; + member_offset(): number; + member_mode(): number; +}; +// Node exposes WebAssembly without DOM globals; keep the private ABI types local. +const wasm = (globalThis as unknown as { WebAssembly: { + Module: { new(bytes: Uint8Array): object; imports(module: object): unknown[] }; + Instance: new(module: object) => { exports: object }; +} }).WebAssembly; +let compiled: object | undefined; +function instance(): Abi { + // src tests and dist consumers resolve the same generated package artifact. + compiled ??= new wasm.Module(readFileSync(new URL("../dist/archive-parser.wasm", import.meta.url))); + if (wasm.Module.imports(compiled).length) throw new Error("TAR WASM unexpectedly requires host imports"); + return new wasm.Instance(compiled).exports as unknown as Abi; +} +const types = new Map([ + [0, "File"], [48, "File"], [49, "Link"], [50, "SymbolicLink"], + [51, "CharacterDevice"], [52, "BlockDevice"], [53, "Directory"], + [54, "FIFO"], [55, "ContiguousFile"], [68, "GNUDumpDir"], +]); + +function parserError(message: string): Error { + if (isArchiveTarPathErrorMessage(message)) return new ArchiveSecurityError("entry-path", message); + for (const code of Object.values(ARCHIVE_LIMIT_ERROR_CODE)) { + if (message.includes(code)) return new ArchiveLimitError(code); + } + return new ArchiveFormatError(message.replace(/^archive-header-invalid:/, "invalid TAR header:")); +} + +/** Backpressure-aware transport only; all TAR semantics live in the Rust core. */ +export class TarParserStream extends Transform { + private abi: Abi | undefined; + constructor(limits: TarMeterLimits, private readonly onMember?: (entry: AdmittedTarMember) => void) { + super(); + this.abi = instance(); + if (this.abi.init(limits.maxEntries, limits.maxMetaEntryBytes, limits.maxDecodedBytes, limits.maxManifestBytes, Number(process.platform === "win32")) !== 0) { + this.abi.dispose(); + this.abi = undefined; + throw new RangeError("invalid TAR parser limits"); + } + } + private bytes(pointer: number, length: number): Uint8Array { + const memory = this.abi!.memory.buffer; + if (!Number.isInteger(pointer) || !Number.isInteger(length) || pointer < 0 || length < 0 || + pointer > memory.byteLength || length > memory.byteLength - pointer) { + throw new ArchiveFormatError("invalid TAR WASM memory range"); + } + return new Uint8Array(memory, pointer, length); + } + private text(): string { + const abi = this.abi!; + return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(this.bytes(abi.text_ptr(), abi.text_len())); + } + override _transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback): void { + try { + const abi = this.abi!; + for (let offset = 0; offset < chunk.length;) { + const length = Math.min(65536, chunk.length - offset); + this.bytes(abi.input_ptr(), length).set(chunk.subarray(offset, offset + length)); + const used = abi.push(length); + if (used < 0) throw parserError(this.text()); + if (used === 0 || used > length) throw new ArchiveFormatError("TAR WASM made no progress"); + offset += used; + const type = abi.member_type(); + if (type >= 0) this.onMember?.({ + path: this.text(), type: types.get(type) ?? "Unsupported", size: abi.member_size(), + mode: abi.member_mode(), offset: abi.member_offset(), + }); + } + callback(null, chunk); + } catch (error) { callback(error instanceof Error ? error : new Error(String(error))); } + } + override _flush(callback: TransformCallback): void { + try { + if (this.abi!.finish() !== 0) throw parserError(this.text()); + callback(); + } catch (error) { callback(error instanceof Error ? error : new Error(String(error))); } + } + override _destroy(error: Error | null, callback: (error: Error | null) => void): void { + try { this.abi?.dispose(); } + catch (cause) { error ??= cause instanceof Error ? cause : new Error(String(cause)); } + finally { this.abi = undefined; } + callback(error); + } +} diff --git a/src/archive-tar.ts b/src/archive-tar.ts index fa0dbe53..7c925c65 100644 --- a/src/archive-tar.ts +++ b/src/archive-tar.ts @@ -31,33 +31,6 @@ const BLOCKED_TAR_ENTRY_TYPES = new Set([ "Socket", ]); -export function readTarEntryInfo(entry: unknown): TarEntryInfo { - const p = - typeof entry === "object" && entry !== null && "path" in entry - ? String((entry as { path: unknown }).path) - : ""; - const t = - typeof entry === "object" && entry !== null && "type" in entry - ? String((entry as { type: unknown }).type) - : ""; - const s = - typeof entry === "object" && - entry !== null && - "size" in entry && - typeof (entry as { size?: unknown }).size === "number" && - Number.isFinite((entry as { size: number }).size) - ? Math.max(0, Math.floor((entry as { size: number }).size)) - : 0; - const mode = - typeof entry === "object" && - entry !== null && - "mode" in entry && - typeof (entry as { mode?: unknown }).mode === "number" - ? (entry as { mode: number }).mode - : undefined; - return { path: p, type: t, size: s, mode }; -} - export function createTarEntryPreflightChecker(params: { rootDir: string; stripComponents?: number; diff --git a/src/archive.ts b/src/archive.ts index 411ef946..7a26124f 100644 --- a/src/archive.ts +++ b/src/archive.ts @@ -32,11 +32,6 @@ import { withStagedArchiveDestination, } from "./archive-staging.js"; import { mergePlannedArchiveIntoDestination, type ArchivePublicationEntry } from "./archive-merge.js"; -import { - createTarEntryPreflightChecker, - readTarEntryInfo, - type TarEntryInfo, -} from "./archive-tar.js"; import { loadZipArchiveWithPreflight } from "./archive-zip-preflight.js"; import { isZipSymlinkEntry, @@ -58,9 +53,7 @@ import { resolveArchiveFilteredEntryPolicy, shouldExtractArchiveEntry, } from "./archive-policy.js"; -import { importOptionalTar, NODE_TAR_RATIO_DISABLED, normalizeTarParserError } from "./archive-tar-runtime.js"; -import { preflightTarMetadata } from "./archive-tar-meta.js"; -import { createTarAdmissionPlan } from "./archive-tar-admission.js"; +import { extractWasmTar } from "./archive-tar-extract.js"; import type { ExtractArchiveOptions } from "./archive-options.js"; import { writeSiblingTempFile } from "./sibling-temp.js"; export type { ArchiveLogger, ExtractArchiveOptions } from "./archive-options.js"; @@ -348,114 +341,13 @@ export async function extractArchive(params: ExtractArchiveOptions): Promise { - const tar = await importOptionalTar(); const stagedArchive = await stageArchiveFileForExtraction({ archivePath: params.archivePath, limits, deadline, }); try { - const manifest: TarEntryInfo[] = []; - await preflightTarMetadata({ - archivePath: stagedArchive.path, - limits: tarLimits, - signal: deadline.signal, - onMember: (entry) => { manifest.push(entry); }, - }); - deadline.check(); - const destinationRealDir = await prepareArchiveDestinationDir(params.destDir); - await withStagedArchiveDestination({ - destinationRealDir, - run: async (stagingDir) => { - deadline.check(); - const strip = Math.max(0, Math.floor(params.stripComponents ?? 0)); - const checkTarEntrySafety = createTarEntryPreflightChecker({ - rootDir: destinationRealDir, - stripComponents: params.stripComponents, - limits, - entryFilter: params.entryFilter, - onFiltered, - }); - const admission = createTarAdmissionPlan(manifest, (entry) => { - deadline.check(); - return checkTarEntrySafety(entry); - }, strip); - const acceptedEntries: ArchivePublicationEntry[] = []; - // Extract privately, then merge through the safe-open boundary. - const extractor = tar.x({ - cwd: stagingDir, - // fs-safe owns stripping: node-tar counts the `.` and empty path - // components that stripArchivePath() drops, so the two disagree. - strip: 0, - gzip: params.tarGzip, - signal: deadline.signal, - preservePaths: false, - noChmod: true, - preserveOwner: false, - noMtime: true, - dmode: 0o700, - strict: true, - maxMetaEntrySize: tarLimits.maxMetaEntryBytes, - maxDecompressionRatio: NODE_TAR_RATIO_DISABLED, - filter(this: { abort(error: Error): void }, _entryPath, entry) { - try { - const info = readTarEntryInfo(entry); - const relPath = admission.consume(info); - if (!relPath) { - return false; - } - const kind = info.type === "Directory" || info.type === "GNUDumpDir" ? "directory" : "file"; - // Keep archived modes before changing node-tar's creation mode. - (entry as { path: string }).path = relPath; - acceptedEntries.push({ - path: relPath, - kind, - mode: resolveArchiveEntryMode({ - kind, - archivedMode: info.mode, - policy: params.entryModes, - }), - }); - (entry as { mode: number }).mode = kind === "directory" ? 0o700 : 0o600; - return true; - } catch (error) { - // Abort through the parser so pipeline tears down both the - // archive reader and unpacker instead of leaving a paused - // stream behind after a policy rejection. - this.abort(error instanceof Error ? error : new Error(String(error))); - return false; - } - }, - onReadEntry(entry) { - try { - deadline.check(); - } catch (err) { - const error = err instanceof Error ? err : new Error(String(err)); - // EventEmitter binds `this` to tar.Unpack, exposing abort(). - const emitter = this as unknown as { abort?: (error: Error) => void }; - emitter.abort?.(error); - } - }, - }); - try { - await pipeline(fsSync.createReadStream(stagedArchive.path), extractor, { - signal: deadline.signal, - }); - } catch (error) { - throw normalizeTarParserError(createPipelineTimeoutError(error, deadline)); - } - admission.finish(); - deadline.check(); - await mergePlannedArchiveIntoDestination({ - entries: acceptedEntries, - sourceDir: stagingDir, - destinationDir: params.destDir, - destinationRealDir, - deadline, - }); - deadline.check(); - }, - }); + await extractWasmTar({ archivePath: stagedArchive.path, options: { ...params, onFiltered }, limits, tarLimits, deadline }); } finally { await stagedArchive.cleanup(); } diff --git a/test/archive-gzip-container.test.ts b/test/archive-gzip-container.test.ts new file mode 100644 index 00000000..93c8c24a --- /dev/null +++ b/test/archive-gzip-container.test.ts @@ -0,0 +1,95 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { extractArchive, readArchiveEntry } from "../src/archive.js"; +import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; +import { __resetNativeLoaderForTest, __setNativeLoaderForTest } from "../src/native.js"; +import { paxNative } from "./helpers/archive-pax-native.js"; +import { useTempDirs } from "./helpers/vitest.js"; +import { completeGzip, gzipMember, gzipTar, invalidGzipContainers, sizedGzipMember } from "./helpers/archive-gzip-container.js"; + +const { tempRoot } = useTempDirs(); +afterEach(() => { __resetFsSafeNativeConfigForTest(); __resetNativeLoaderForTest(); }); + +describe.each(["off", "require"] as const)("compressed gzip container %s", (mode) => { + beforeEach(() => { + configureFsSafeNative({ mode }); + if (mode === "require" && paxNative) __setNativeLoaderForTest(() => paxNative); + }); + async function setup(bytes: Buffer) { + const base = await tempRoot("fs-safe-gzip-container-"); + const archivePath = path.join(base, "input.tgz"), destDir = path.join(base, "out"); + await fs.writeFile(archivePath, bytes); await fs.mkdir(destDir); + return { archivePath, destDir, timeoutMs: 10000 }; + } + const suite = mode === "require" && !paxNative ? it.skip : it; + suite.each([0, 1, 511, 512, 513, 10240 - completeGzip.length, 131072])("accepts %i physical zero tail bytes", async (size) => { + expect(gunzipSync(completeGzip)).toEqual(gzipTar); + const fixture = await setup(Buffer.concat([completeGzip, Buffer.alloc(size)])); + await extractArchive(fixture); + expect((await fs.readdir(fixture.destDir)).sort()).toEqual(["sentinel", "value"]); + expect(await fs.readFile(path.join(fixture.destDir, "value"), "utf8")).toBe("payload"); + expect(await readArchiveEntry(fixture.archivePath, "value", { maxBytes: 7 })).toEqual(Buffer.from("payload")); + expect(await readArchiveEntry(fixture.archivePath, "sentinel", { maxBytes: 3 })).toEqual(Buffer.from("end")); + }); + suite.each(invalidGzipContainers)("rejects %s without publication or selected bytes", async (_name, bytes) => { + const fixture = await setup(bytes); + await expect(extractArchive(fixture)).rejects.toThrow(); + expect(await fs.readdir(fixture.destDir)).toEqual([]); + await expect(readArchiveEntry(fixture.archivePath, "sentinel", { maxBytes: 3 })).rejects.toThrow(); + }); + suite.each([1, 511, 512, 513, gzipTar.length])("validates concatenated members split at decoded offset %i", async (split) => { + const bytes = Buffer.concat([gzipMember(gzipTar.subarray(0, split), true), gzipMember(gzipTar.subarray(split)), Buffer.alloc(513)]); + const fixture = await setup(bytes); + await extractArchive(fixture); + expect(await fs.readFile(path.join(fixture.destDir, "sentinel"), "utf8")).toBe("end"); + expect(await readArchiveEntry(fixture.archivePath, "value", { maxBytes: 7 })).toEqual(Buffer.from("payload")); + }); + suite.each([65534, 65535, 65536])("accepts an empty following member across physical offset %i without a padding gap", async (size) => { + const member = sizedGzipMember(gzipTar, size); + expect(member.length).toBe(size); + expect(gunzipSync(member)).toEqual(gzipTar); + const fixture = await setup(Buffer.concat([member, gzipMember(Buffer.alloc(0)), Buffer.alloc(513)])); + await extractArchive(fixture); + expect(await fs.readFile(path.join(fixture.destDir, "value"), "utf8")).toBe("payload"); + expect(await readArchiveEntry(fixture.archivePath, "sentinel", { maxBytes: 3 })).toEqual(Buffer.from("end")); + }); + suite("charges physical container padding against the original archive budget", async () => { + const bytes = Buffer.concat([completeGzip, Buffer.alloc(10240 - completeGzip.length)]); + const fixture = await setup(bytes); + await expect(extractArchive({ ...fixture, limits: { maxArchiveBytes: bytes.length - 1 } })) + .rejects.toMatchObject({ code: "archive-size-exceeds-limit" }); + expect(await fs.readdir(fixture.destDir)).toEqual([]); + await extractArchive({ ...fixture, limits: { maxArchiveBytes: bytes.length } }); + expect(await fs.readFile(path.join(fixture.destDir, "value"), "utf8")).toBe("payload"); + }); +}); + +describe.skipIf(process.platform === "win32").each(["off", "require"] as const)("bound system tar stdout %s", (mode) => { + it.skipIf(mode === "require" && !paxNative)("keeps exact stdout bytes and refuses a wrong directory identity", async () => { + const { execFileSync, spawnSync } = await import("node:child_process"); + const { fileURLToPath } = await import("node:url"); + configureFsSafeNative({ mode }); + if (mode === "require") __setNativeLoaderForTest(() => paxNative!); + const source = await tempRoot("fs-safe-tar-stdout-"); + const names = ["ordinary", "雪.txt", "line\n.txt", "long-" + "a".repeat(130)]; + for (const name of names) await fs.writeFile(path.join(source, name), `exact:${name}`); + const canonical = await fs.realpath(source), stat = await fs.stat(source, { bigint: true }); + const worker = fileURLToPath(new URL("../scripts/archive-system-tar-worker.cjs", import.meta.url)); + const args = [worker, source, canonical, String(stat.dev), String(stat.ino)]; + const refused = spawnSync(process.execPath, [...args.slice(0, -1), String(stat.ino + 1n)], { timeout: 10000 }); + expect(refused.status).toBe(78); expect(refused.stdout.length).toBe(0); + const bytes = execFileSync(process.execPath, args, { timeout: 30000, maxBuffer: 4 * 1024 * 1024 }); + const base = await tempRoot("fs-safe-tar-stdout-output-"); + const archivePath = path.join(base, "stdout.tgz"), destDir = path.join(base, "out"); + await fs.writeFile(archivePath, bytes); await fs.mkdir(destDir); + await extractArchive({ archivePath, destDir, timeoutMs: 10000 }); + expect(await fs.readFile(archivePath)).toEqual(bytes); + for (const name of names) { + const expected = Buffer.from(`exact:${name}`); + expect(await fs.readFile(path.join(destDir, name))).toEqual(expected); + expect(await readArchiveEntry(archivePath, name, { maxBytes: expected.length })).toEqual(expected); + } + }); +}); diff --git a/test/archive-gzip-tail.test.ts b/test/archive-gzip-tail.test.ts new file mode 100644 index 00000000..4455a69c --- /dev/null +++ b/test/archive-gzip-tail.test.ts @@ -0,0 +1,28 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { validateGzipContainerTail } from "../src/archive-gzip-tail.js"; +import { completeGzip } from "./helpers/archive-gzip-container.js"; +import { useTempDirs } from "./helpers/vitest.js"; + +const { tempRoot } = useTempDirs(); +afterEach(() => vi.restoreAllMocks()); +it("stops a cancelled suffix scan at its bounded read and closes the borrowed file", async () => { + const base = await tempRoot("fs-safe-gzip-tail-cancel-"); + const file = path.join(base, "input.tgz"); + await fs.writeFile(file, Buffer.concat([completeGzip, Buffer.alloc(131072)])); + const handle = await fs.open(file, "r"); + const originalRead = handle.read.bind(handle); + const controller = new AbortController(); + const reason = new Error("cancel gzip padding scan"); + const read = vi.spyOn(handle, "read").mockImplementation(async (...args: Parameters) => { + const result = await originalRead(...args); + controller.abort(reason); + return result; + }); + vi.spyOn(fs, "open").mockResolvedValue(handle); + await expect(validateGzipContainerTail(file, completeGzip.length, controller.signal)).rejects.toBe(reason); + expect(read).toHaveBeenCalledTimes(1); + expect(read.mock.calls[0]![2]).toBe(65536); + await expect(handle.stat()).rejects.toMatchObject({ code: "EBADF" }); +}); diff --git a/test/archive-pax-security.test.ts b/test/archive-pax-security.test.ts index aedd28b6..37953403 100644 --- a/test/archive-pax-security.test.ts +++ b/test/archive-pax-security.test.ts @@ -47,9 +47,9 @@ for (const mode of ["off", "require"] as const) { ["exponent", "size", "1e3"], ["hex", "size", "0x10"], ["leading zero", "size", "01"], ["unsafe integer", "size", "9007199254740992"], ["unsafe padding", "size", "9007199254740990991"], ["padding overflow", "size", "9007199254740991"], ["space", "size", " 1"], - ["NUL path", "path", "ok\0evil"], ["Unicode path", "path", "café"], - ["Unicode linkpath", "linkpath", "café"], ["Unicode owner", "uname", "café"], - ["non-ASCII key", "päth", "ok"], ["newline xattr", "SCHILY.xattr.user.binary", "ok\nbad"], + ["NUL path", "path", "ok\0evil"], + ["Unicode owner", "uname", "café"], + ["non-ASCII key", "päth", "ok"], ["charset", "hdrcharset", "BINARY"], ["charset alias", "charset", "UTF-8"], ["sparse map", "GNU.sparse.map", "0,1"], ["sparse name", "GNU.sparse.name", "raw"], ["sparse size", "GNU.sparse.size", "1"], ["sparse 1.0", "GNU.sparse.major", "1"], @@ -68,7 +68,7 @@ for (const mode of ["off", "require"] as const) { Buffer.from("09 path=a\n"), Buffer.from("+9 path=a\n"), Buffer.from("0 path=a\n"), Buffer.from("999999999999999999999999 path=a\n"), Buffer.from("9 path=a\0"), Buffer.from("9 path=a\ntrailing"), Buffer.from("9 path=a\n\n"), - paxRecord("path", "a\nb"), paxRecord("bad key", "value"), + paxRecord("bad key", "value"), Buffer.concat([paxRecord("path", "a"), paxRecord("path", "b")]), Buffer.concat([paxRecord("size", "1"), paxRecord("size", "7")]), Buffer.concat([paxRecord("SCHILY.xattr.user.binary", "a"), paxRecord("SCHILY.xattr.user.binary", "b")]), @@ -120,11 +120,9 @@ for (const mode of ["off", "require"] as const) { it("rejects ambiguous raw text, raw numbers and link fields even when overridden", async () => { for (const entry of [ - { ...member, path: "café" }, { ...member, linkPath: "not-a-link" }, { path: "link", type: "2", linkPath: "" }, { ...member, mutateHeader: (header: Buffer) => { header[124] = 0xb0; } }, - { ...member, mutateHeader: (header: Buffer) => { header.write("café", 345, "utf8"); } }, ]) await reject(tarFixture([paxHeader([["path", "safe"], ["size", "0"]]), entry])); }); diff --git a/test/archive-read-boundaries.test.ts b/test/archive-read-boundaries.test.ts index 37614f5c..f2f6b0d2 100644 --- a/test/archive-read-boundaries.test.ts +++ b/test/archive-read-boundaries.test.ts @@ -13,7 +13,7 @@ import { readArchiveEntry, } from "../src/archive.js"; import { resolveTarMeterLimits } from "../src/archive-limits.js"; -import { preflightTarMetadata } from "../src/archive-tar-meta.js"; +import { inspectTar } from "../src/archive-tar-stream.js"; import { __resetFsSafeNativeConfigForTest, configureFsSafeNative, @@ -60,9 +60,9 @@ describe("TAR metadata preflight boundaries", () => { await fs.writeFile(atLimit, tarFixture([{ path: "long-name", type: "L", body: "x".repeat(16) }, { path: "raw" }])); await fs.writeFile(pastLimit, tarFixture([{ path: "long-name", type: "L", body: "x".repeat(17) }, { path: "raw" }])); - await expect(preflightTarMetadata({ archivePath: atLimit, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 16 }) })) + await expect(inspectTar({ archivePath: atLimit, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 16 }) })) .resolves.toBeUndefined(); - await expect(preflightTarMetadata({ archivePath: pastLimit, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 16 }) })) + await expect(inspectTar({ archivePath: pastLimit, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 16 }) })) .rejects.toMatchObject({ code: ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT, }); @@ -75,9 +75,9 @@ describe("TAR metadata preflight boundaries", () => { await fs.writeFile(headerPath, Buffer.alloc(511)); await fs.writeFile(entryPath, tarFixture([{ path: "value", body: "payload" }], false).subarray(0, 513)); - await expect(preflightTarMetadata({ archivePath: headerPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath: headerPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .rejects.toThrow("truncated TAR header"); - await expect(preflightTarMetadata({ archivePath: entryPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath: entryPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .rejects.toThrow("truncated TAR entry"); }); @@ -104,15 +104,15 @@ describe("TAR metadata preflight boundaries", () => { await fs.writeFile(highBitsPath, highBits); await fs.writeFile(paddingPath, padding); - await expect(preflightTarMetadata({ archivePath: malformedPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath: malformedPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .rejects.toThrow("size is not valid octal"); - await expect(preflightTarMetadata({ archivePath: highBitsPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) - .rejects.toThrow("base-256 size exceeds the safe integer range"); - await expect(preflightTarMetadata({ archivePath: paddingPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath: highBitsPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + .rejects.toThrow("base-256 size is negative or overflows u64"); + await expect(inspectTar({ archivePath: paddingPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .rejects.toThrow("entry padding exceeds the safe integer range"); await fs.writeFile(paddingPath, negative); - await expect(preflightTarMetadata({ archivePath: paddingPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) - .rejects.toThrow("base-256 size is negative or malformed"); + await expect(inspectTar({ archivePath: paddingPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + .rejects.toThrow("base-256 size is negative or overflows u64"); }); it("rejects dangling PAX and malformed GNU sparse metadata", async () => { @@ -142,17 +142,17 @@ describe("TAR metadata preflight boundaries", () => { Buffer.concat([sparseExtension, repeatedExtension, finalExtension]), ); - await expect(preflightTarMetadata({ archivePath: paxPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath: paxPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .rejects.toThrow("dangling PAX metadata"); - await expect(preflightTarMetadata({ archivePath: sparseFlagPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath: sparseFlagPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .rejects.toThrow("GNU sparse extension flag is not 0 or 1"); - await expect(preflightTarMetadata({ archivePath: sparseExtensionPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath: sparseExtensionPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .rejects.toThrow("GNU sparse extension flag is not 0 or 1"); - await expect(preflightTarMetadata({ archivePath: sparseLimitPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 511 }) })) + await expect(inspectTar({ archivePath: sparseLimitPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 511 }) })) .rejects.toMatchObject({ code: ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT, }); - await expect(preflightTarMetadata({ archivePath: sparseUnsupportedPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath: sparseUnsupportedPath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .rejects.toThrow("GNU sparse entries are not supported"); }); @@ -164,7 +164,7 @@ describe("TAR metadata preflight boundaries", () => { updateTarChecksum(header); await fs.writeFile(archivePath, Buffer.concat([header, Buffer.alloc(512 + 1024)])); - await expect(preflightTarMetadata({ archivePath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .resolves.toBeUndefined(); }); @@ -172,11 +172,11 @@ describe("TAR metadata preflight boundaries", () => { const root = await tempRoot("fs-safe-tar-gzip-"); const archivePath = path.join(root, "fixture.tar.gz"); await fs.writeFile(archivePath, gzipSync(tarFixture([{ path: "value", body: "ok" }]))); - await expect(preflightTarMetadata({ archivePath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) + await expect(inspectTar({ archivePath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }) })) .resolves.toBeUndefined(); const controller = new AbortController(); controller.abort(new Error("deadline elapsed")); - await expect(preflightTarMetadata({ + await expect(inspectTar({ archivePath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }), signal: controller.signal, @@ -189,11 +189,11 @@ describe("TAR metadata preflight boundaries", () => { await fs.writeFile(archivePath, tarFixture([{ path: "value", body: "ok" }])); const createReadStream = vi.spyOn(fsSync, "createReadStream"); - await expect(preflightTarMetadata({ + await expect(inspectTar({ archivePath, limits: { ...resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }), maxDecodedBytes: Number.NaN }, - })).rejects.toThrow("maxDecodedBytes must be a non-negative safe integer"); - await expect(preflightTarMetadata({ + })).rejects.toThrow("invalid TAR parser limits"); + await expect(inspectTar({ archivePath: path.join(root, "missing.tar"), limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }), })).rejects.toMatchObject({ code: "ENOENT" }); @@ -212,7 +212,7 @@ describe("TAR metadata preflight boundaries", () => { return stream; }) as typeof fsSync.createReadStream); - await expect(preflightTarMetadata({ + await expect(inspectTar({ archivePath, limits: resolveTarMeterLimits({ maxMetaEntryBytes: 1024 }), })).rejects.toThrow("truncated TAR header"); diff --git a/test/archive-tar-decoded.test.ts b/test/archive-tar-decoded.test.ts index 62ae009e..3b9afbab 100644 --- a/test/archive-tar-decoded.test.ts +++ b/test/archive-tar-decoded.test.ts @@ -2,7 +2,7 @@ import { Readable, Writable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { describe, expect, it } from "vitest"; import { resolveExtractLimits, resolveTarMeterLimits } from "../src/archive-limits.js"; -import { TarMetadataMeter } from "../src/archive-tar-meta.js"; +import { TarParserStream } from "../src/archive-tar-wasm.js"; import { tarFixture } from "./helpers/archive-fuzz.js"; import { paxHeader } from "./helpers/archive-pax.js"; @@ -13,7 +13,7 @@ async function admit(bytes: Buffer, maxDecodedBytes: number, chunkSize = 511): P for (let offset = 0; offset < bytes.length; offset += chunkSize) yield bytes.subarray(offset, offset + chunkSize); } const output: Buffer[] = []; - await pipeline(Readable.from(chunks()), new TarMetadataMeter({ ...resolveTarMeterLimits(), maxDecodedBytes }), new Writable({ + await pipeline(Readable.from(chunks()), new TarParserStream({ ...resolveTarMeterLimits(), maxDecodedBytes }), new Writable({ write(chunk: Buffer, _encoding, callback) { output.push(chunk); callback(); }, })); return Buffer.concat(output); @@ -61,7 +61,7 @@ describe("absolute decoded TAR admission", () => { ["PAX metadata", tarFixture([paxHeader([["size", "0"]]), { path: "empty" }], false)], ] as const)("stops an unbounded %s tail at the ceiling plus one probe", async (_label, pattern) => { const ceiling = pattern.length * 4; - const meter = new TarMetadataMeter({ ...resolveTarMeterLimits(), maxDecodedBytes: ceiling }); + const meter = new TarParserStream({ ...resolveTarMeterLimits(), maxDecodedBytes: ceiling }); const error = new Promise((resolve) => meter.once("error", resolve)); let forwarded = 0; meter.on("data", (chunk: Buffer) => { forwarded += chunk.length; }); diff --git a/test/archive-tar-framing.test.ts b/test/archive-tar-framing.test.ts index d191a000..88db45aa 100644 --- a/test/archive-tar-framing.test.ts +++ b/test/archive-tar-framing.test.ts @@ -220,12 +220,12 @@ for (const backend of ["off", "auto-missing", "auto", "require"] as const) { await expect(nativeRead("directory", 7)).rejects.toThrow("archive entry is not a file: directory"); await expect(paxNative!.readArchiveEntryNative(fixture.archivePath, "tar", "value", 7, { ...limits, maxEntries: 1 }, signal)).rejects.toThrow("archive-entry-count-exceeds-limit"); await fs.writeFile(fixture.archivePath, gzip ? gzipSync(Buffer.concat([bytes, Buffer.from([1])])) : Buffer.concat([bytes, Buffer.from([1])])); - await expect(nativeRead("value", 6)).rejects.toThrow("archive-entry-extracted-size-exceeds-limit"); - await expect(nativeRead("directory", 7)).rejects.toThrow("archive entry is not a file: directory"); + await expect(nativeRead("value", 6)).rejects.toThrow("archive-header-invalid"); + await expect(nativeRead("directory", 7)).rejects.toThrow("archive-header-invalid"); await expect(nativeRead("absent", 7)).rejects.toThrow("archive-header-invalid"); }); - it.skipIf(backend === "off" || backend === "auto-missing")("keeps native directory modes private until the physical tail passes", async () => { + it.skipIf(backend === "off" || backend === "auto-missing")("rejects the physical tail before creating native staging entries", async () => { const prefix = tarFixture([{ path: "directory", type: "5", mode: 0o500 }, member]); const fixture = await setup(Buffer.concat([prefix, Buffer.from([1])]), gzip); const privateDir = path.join(path.dirname(fixture.destDir), "private-stage"); @@ -237,8 +237,7 @@ for (const backend of ["off", "auto-missing", "auto", "require"] as const) { { index: 1, path: "value", kind: "file", size: 7, mode: 0o600 }, ]; await expect(paxNative!.extractArchiveNative(fixture.archivePath, "tar", directory.fd, plan, resolveTarMeterLimits(), new AbortController().signal)).rejects.toThrow("archive-header-invalid"); - expect(await fs.readFile(path.join(privateDir, "value"), "utf8")).toBe("payload"); - if (process.platform !== "win32") expect((await fs.stat(path.join(privateDir, "directory"))).mode & 0o777).toBe(0o700); + expect(await fs.readdir(privateDir)).toEqual([]); expect(await fs.readdir(fixture.destDir)).toEqual(["sentinel"]); } finally { await directory.close(); diff --git a/test/archive-tar-gnu-meter.test.ts b/test/archive-tar-gnu-meter.test.ts index 285503af..caa82658 100644 --- a/test/archive-tar-gnu-meter.test.ts +++ b/test/archive-tar-gnu-meter.test.ts @@ -2,7 +2,7 @@ import { Readable, Writable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { describe, expect, it } from "vitest"; import { resolveTarMeterLimits } from "../src/archive-limits.js"; -import { TarMetadataMeter } from "../src/archive-tar-meta.js"; +import { TarParserStream } from "../src/archive-tar-wasm.js"; import { gnu, gnuFixture, invalidGnu, validGnu } from "./helpers/archive-gnu.js"; import { tarFixture } from "./helpers/archive-fuzz.js"; @@ -11,7 +11,7 @@ async function meter(bytes: Buffer, chunkSize: number, maxMetaEntryBytes = 1024) for (let offset = 0; offset < bytes.length; offset += chunkSize) yield bytes.subarray(offset, offset + chunkSize); } const output: Buffer[] = []; - await pipeline(Readable.from(chunks()), new TarMetadataMeter(resolveTarMeterLimits({ maxMetaEntryBytes })), new Writable({ + await pipeline(Readable.from(chunks()), new TarParserStream(resolveTarMeterLimits({ maxMetaEntryBytes })), new Writable({ write(chunk: Buffer, _encoding, callback) { output.push(chunk); callback(); }, })); return Buffer.concat(output); diff --git a/test/archive-tar-ignored-meter.test.ts b/test/archive-tar-ignored-meter.test.ts index 35e53fcb..303fad7a 100644 --- a/test/archive-tar-ignored-meter.test.ts +++ b/test/archive-tar-ignored-meter.test.ts @@ -1,12 +1,10 @@ import { Readable, Writable } from "node:stream"; import { pipeline } from "node:stream/promises"; -import { Parser } from "tar"; import fc from "fast-check"; import { describe, expect, it } from "vitest"; -import { TarMetadataMeter } from "../src/archive-tar-meta.js"; +import { TarParserStream } from "../src/archive-tar-wasm.js"; import { resolveTarMeterLimits, tarManifestEntryCost, type TarMeterLimits } from "../src/archive-limits.js"; import { createTarEntryPreflightChecker, type TarEntryInfo } from "../src/archive-tar.js"; -import { createTarAdmissionPlan } from "../src/archive-tar-admission.js"; import { tarFixture } from "./helpers/archive-fuzz.js"; import { ignored, ignoredTypes, unsafeIgnoredPaths } from "./helpers/archive-ignored.js"; @@ -22,7 +20,7 @@ it("rejects raw linkname/type contradictions for all 256 flags before metadata o mutateHeader(header) { header[156] = type; }, }], false); const entries: TarEntryInfo[] = []; - const meter = new TarMetadataMeter(resolveTarMeterLimits({ maxEntries: 0, maxMetaEntryBytes: 0 }), (entry) => entries.push(entry)); + const meter = new TarParserStream(resolveTarMeterLimits({ maxEntries: 0, maxMetaEntryBytes: 0 }), (entry) => entries.push({ path: entry.path, type: entry.type, size: entry.size })); await expect(pipeline(Readable.from([header]), meter, new Writable({ write(_chunk, _encoding, callback) { callback(); } }))) .rejects.toMatchObject({ name: "ArchiveFormatError", code: "archive-header-invalid", message: `invalid TAR header: linkname ${isLink ? "required on a link" : "forbidden on a non-link"} header` }); @@ -30,35 +28,13 @@ it("rejects raw linkname/type contradictions for all 256 flags before metadata o } }); -it("confirms node-tar's classification for every raw typeflag", async () => { - const observed: number[] = []; - const named: string[] = []; - for (let byte = 0; byte < 256; byte++) { - const bytes = tarFixture([{ - path: "safe", linkPath: byte === 49 || byte === 50 ? "target" : undefined, - mutateHeader: (header) => { header[156] = byte; }, - }]); - const callbacks: number[] = []; - const parser = new Parser({ strict: true, onReadEntry(entry) { callbacks.push(byte); entry.resume(); } }); - parser.on("ignoredEntry", (entry) => { observed.push(byte); named.push(entry.type); }); - await pipeline(Readable.from([bytes]), parser); - expect(callbacks).toEqual(visible.includes(byte) ? [byte] : []); - } - expect(observed).toEqual(ignoredFlags); - expect(observed).toHaveLength(240); - expect(named.filter((type) => type === "Unsupported")).toHaveLength(235); - expect(named.filter((type) => type !== "Unsupported")).toEqual([ - "SolarisACL", "Inode", "ContinuationFile", "SparseFile", "TapeVolumeHeader", - ]); -}); - async function admit(bytes: Buffer, chunkSize: number, maxEntries = 50_000) { const entries: TarEntryInfo[] = []; const chunks = function* () { for (let offset = 0; offset < bytes.length; offset += chunkSize) yield bytes.subarray(offset, offset + chunkSize); }; const output: Buffer[] = []; - await pipeline(Readable.from(chunks()), new TarMetadataMeter(resolveTarMeterLimits({ maxEntries }), (entry) => entries.push(entry)), + await pipeline(Readable.from(chunks()), new TarParserStream(resolveTarMeterLimits({ maxEntries }), (entry) => entries.push({ path: entry.path, type: entry.type, size: entry.size })), new Writable({ write(chunk, _encoding, callback) { output.push(chunk); callback(); } })); expect(Buffer.concat(output)).toEqual(bytes); return entries; @@ -66,7 +42,7 @@ async function admit(bytes: Buffer, chunkSize: number, maxEntries = 50_000) { async function rejectsBeforeEmission(bytes: Buffer, code: string, limits: Partial = {}, expected: TarEntryInfo[] = []) { const entries: TarEntryInfo[] = []; - const meter = new TarMetadataMeter({ ...resolveTarMeterLimits(), ...limits }, (entry) => entries.push(entry)); + const meter = new TarParserStream({ ...resolveTarMeterLimits(), ...limits }, (entry) => entries.push({ path: entry.path, type: entry.type, size: entry.size })); await expect(pipeline(Readable.from([bytes]), meter, new Writable({ write(_chunk, _encoding, callback) { callback(); } }))) .rejects.toMatchObject({ code }); expect(entries).toEqual(expected); @@ -134,7 +110,7 @@ describe.each([1, 7, 511, 512, 513, 64 * 1024])("ignored raw admission chunk=%i" ]); }); it.each(ignoredTypes)("counts ignored %s headers before requesting their bodies", async (type) => { - const meter = new TarMetadataMeter(resolveTarMeterLimits({ maxEntries: 1 })); + const meter = new TarParserStream(resolveTarMeterLimits({ maxEntries: 1 })); meter.resume(); const failure = new Promise((resolve) => meter.once("error", resolve)); const prefix = Buffer.concat([tarFixture([ignored(type)], false), tarFixture([ignored(type, "second")]).subarray(0, 512)]); @@ -154,7 +130,7 @@ it("keeps unsupported raw flags under the same alias/collision policy for arbitr const entries = await admit(tarFixture(first ? [hidden, file] : [file, hidden]), chunk, 2); const calls: string[] = []; const check = createTarEntryPreflightChecker({ rootDir: process.cwd(), onFiltered: "skip-entry", entryFilter: ({ path }) => { calls.push(path); return "skip"; } }); - expect(() => createTarAdmissionPlan(entries, check, 0)).toThrow(expect.objectContaining({ code: "entry-path" })); + expect(() => entries.forEach(check)).toThrow(expect.objectContaining({ code: "entry-path" })); expect(calls).toEqual(["pkg/item"]); }, ), { numRuns: 100, seed: 47 }); diff --git a/test/archive-tar-manifest.test.ts b/test/archive-tar-manifest.test.ts index 284c3c64..8b7fbb54 100644 --- a/test/archive-tar-manifest.test.ts +++ b/test/archive-tar-manifest.test.ts @@ -8,7 +8,7 @@ import { pipeline } from "node:stream/promises"; import { createGzip } from "node:zlib"; import { describe, expect, it } from "vitest"; import { MAX_TAR_MANIFEST_BYTES, resolveTarMeterLimits, tarManifestEntryCost } from "../src/archive-limits.js"; -import { TarMetadataMeter } from "../src/archive-tar-meta.js"; +import { TarParserStream } from "../src/archive-tar-wasm.js"; import { manifestMember, nearMaxPath } from "./helpers/archive-admission.js"; import { tarFixture } from "./helpers/archive-fuzz.js"; import { paxNative } from "./helpers/archive-pax-native.js"; @@ -35,7 +35,7 @@ it("charges object/string overhead and UTF-8 bytes exactly at the boundary", asy expect(tarManifestEntryCost(name)).toBe(cost); for (const maximum of [cost - 1, cost]) { let emitted = 0; - const meter = new TarMetadataMeter({ ...resolveTarMeterLimits(), maxManifestBytes: maximum }, () => { emitted++; }); + const meter = new TarParserStream({ ...resolveTarMeterLimits(), maxManifestBytes: maximum }, () => { emitted++; }); meter.resume(); meter.on("error", () => {}); const error = await new Promise((resolve) => meter.end(tarFixture([{ path: name }]), resolve)); @@ -62,7 +62,7 @@ describe.each(["GNU", "PAX"] as const)("streamed %s manifest retention", (extens it("stops before emitting the overflowing member or requesting another body", async () => { let emitted = 0; const limits = resolveTarMeterLimits(); - const meter = new TarMetadataMeter(limits, () => { emitted++; }); + const meter = new TarParserStream(limits, () => { emitted++; }); meter.resume(); meter.on("error", () => {}); const member = manifestMember(extension); diff --git a/test/archive-tar-meter.test.ts b/test/archive-tar-meter.test.ts index eedc90e5..1ed75223 100644 --- a/test/archive-tar-meter.test.ts +++ b/test/archive-tar-meter.test.ts @@ -2,7 +2,7 @@ import { Readable, Writable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { describe, expect, it } from "vitest"; import { resolveTarMeterLimits, type ArchiveExtractLimits } from "../src/archive-limits.js"; -import { TarMetadataMeter } from "../src/archive-tar-meta.js"; +import { TarParserStream } from "../src/archive-tar-wasm.js"; import { tarFixture } from "./helpers/archive-fuzz.js"; import { paxArchive, paxHeader } from "./helpers/archive-pax.js"; import { malformedTarFraming } from "./helpers/archive-tar-framing.js"; @@ -12,7 +12,7 @@ async function meter(bytes: Buffer, chunkSize: number): Promise { function* chunks() { for (let offset = 0; offset < bytes.length; offset += chunkSize) yield bytes.subarray(offset, offset + chunkSize); } - await pipeline(Readable.from(chunks()), new TarMetadataMeter(resolveTarMeterLimits({ maxMetaEntryBytes: 1024 })), new Writable({ + await pipeline(Readable.from(chunks()), new TarParserStream(resolveTarMeterLimits({ maxMetaEntryBytes: 1024 })), new Writable({ write(chunk: Buffer, _encoding, callback) { output.push(chunk); callback(); }, })); return Buffer.concat(output); @@ -71,7 +71,7 @@ describe("raw TAR logical entry counts", () => { ]; it.each(cases)("rejects %s before asking its producer for body bytes", async (_label, prefix, options, code) => { - const meter = new TarMetadataMeter(resolveTarMeterLimits(options)); + const meter = new TarParserStream(resolveTarMeterLimits(options)); meter.resume(); const error = new Promise((resolve) => meter.once("error", resolve)); let bodyRequested = false; @@ -90,7 +90,7 @@ describe("raw TAR logical entry counts", () => { tarFixture([paxHeader([["size", "0"]])], false), header(700), tarFixture([{ path: "LongName", type: "L", body: "name\0" }, { path: "LongLink", type: "K", body: "link\0" }, member]), ]); - const meter = new TarMetadataMeter(resolveTarMeterLimits({ ...limits, maxEntries: 2 })); + const meter = new TarParserStream(resolveTarMeterLimits({ ...limits, maxEntries: 2 })); const output: Buffer[] = []; await pipeline(Readable.from([bytes]), meter, new Writable({ write(chunk, _encoding, callback) { output.push(chunk); callback(); } })); expect(Buffer.concat(output)).toEqual(bytes); diff --git a/test/archive-tar-mode-fields.test.ts b/test/archive-tar-mode-fields.test.ts index 2664d647..a9752782 100644 --- a/test/archive-tar-mode-fields.test.ts +++ b/test/archive-tar-mode-fields.test.ts @@ -80,3 +80,21 @@ describe.skipIf(process.platform === "win32" || process.getuid?.() === 0).each(r }); }, ); + +describe.each(["off", ...(paxNative ? ["require"] : [])] as const)("common malformed TAR modes %s", (backend) => { + it.each(["invalid!", "0000755x", "-0000400"])("uses explicit zero for malformed field %j", async (field) => { + configureFsSafeNative({ mode: backend as "off" | "require" }); + if (backend === "require") __setNativeLoaderForTest(() => paxNative!); + const base = await tempRoot("fs-safe-tar-mode-invalid-"); + const archivePath = path.join(base, "input.tar"); + const destDir = path.join(base, "out"); + await fs.mkdir(destDir); + await fs.writeFile(archivePath, tarFixture([{ path: "file", body: "DATA", mutateHeader(header) { header.write(field, 100); } }])); + try { + await extractArchive({ archivePath, destDir, entryModes: "preserve", timeoutMs: 10000 }); + const stat = await fs.stat(path.join(destDir, "file")); + if (process.platform !== "win32") expect(stat.mode & 0o777).toBe(0); + else expect(stat.isFile()).toBe(true); + } finally { await fs.chmod(path.join(destDir, "file"), 0o600).catch(() => undefined); } + }); +}); diff --git a/test/archive-tar-old-file.test.ts b/test/archive-tar-old-file.test.ts index 53acc19c..46948f7c 100644 --- a/test/archive-tar-old-file.test.ts +++ b/test/archive-tar-old-file.test.ts @@ -3,11 +3,11 @@ import path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { gzipSync } from "node:zlib"; -import { Parser } from "tar"; +import { TarParserStream } from "../src/archive-tar-wasm.js"; +import { resolveTarMeterLimits } from "../src/archive-limits.js"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { extractArchive, readArchiveEntry } from "../src/archive.js"; -import { createTarAdmissionPlan, rawTarMember } from "../src/archive-tar-admission.js"; -import { readTarEntryInfo, type TarEntryInfo } from "../src/archive-tar.js"; +import { type TarEntryInfo } from "../src/archive-tar.js"; import { __resetFsSafeNativeConfigForTest, configureFsSafeNative } from "../src/native-config.js"; import { tarFixture, type TarFixtureEntry } from "./helpers/archive-fuzz.js"; import { useTempDirs } from "./helpers/vitest.js"; @@ -20,41 +20,15 @@ function oldFile(entryPath: string): TarFixtureEntry { return { path: entryPath, body: payload, mutateHeader(header) { header[156] = 0; } }; } -describe("TAR regular-file parser agreement", () => { - it.each([0x00, 0x30])("matches node-tar's emitted type for raw byte %i", async (byte) => { - const bytes = tarFixture([{ - path: "./pkg//legacy.bin", body: payload, - mutateHeader(header) { header[156] = byte; }, - }]); - const member = rawTarMember(bytes.subarray(0, 512), payload.length); - const check = vi.fn(() => true); - const plan = createTarAdmissionPlan([member], check, 1); +describe("TAR regular-file admission", () => { + it.each([0x00, 0x30])("admits the exact file identity and range for raw byte %i", async (byte) => { + const bytes = tarFixture([{ path: "./pkg//legacy.bin", body: payload, + mutateHeader(header) { header[156] = byte; } }]); const parsed: TarEntryInfo[] = []; - await pipeline(Readable.from([bytes]), new Parser({ - strict: true, - onReadEntry(entry) { parsed.push(readTarEntryInfo(entry)); entry.resume(); }, - })); - - expect(parsed).toHaveLength(1); - expect(member.type).toBe(parsed[0]!.type); - if (byte === 0x30) expect(member.type).toBe("File"); - expect(plan.consume(parsed[0]!)).toBe("legacy.bin"); - expect(() => plan.finish()).not.toThrow(); - expect(check.mock.calls).toEqual([[member]]); - }); - - it.each(["OldFile", "File"])("accepts a matching %s plan but rejects a different parser type", (type) => { - const member = { path: "./pkg//legacy.bin", type, size: payload.length }; - const actual = { ...member, path: "pkg/legacy.bin" }; - const plan = createTarAdmissionPlan([member], () => true, 1); - expect(plan.consume(actual)).toBe("legacy.bin"); - expect(() => plan.finish()).not.toThrow(); - - const mismatch = createTarAdmissionPlan([member], () => true, 1); - expect(() => mismatch.consume({ ...actual, type: "Directory" })).toThrowError(expect.objectContaining({ - name: "ArchiveFormatError", code: "archive-header-invalid", - message: "invalid TAR header: parser disagrees with raw admission", - })); + const parser = new TarParserStream(resolveTarMeterLimits(), (entry) => parsed.push(entry)); + parser.resume(); + await pipeline(Readable.from([bytes]), parser); + expect(parsed).toEqual([{ path: "./pkg//legacy.bin", type: "File", size: payload.length, mode: 0o644, offset: 512 }]); }); }); diff --git a/test/archive-unified.test.ts b/test/archive-unified.test.ts new file mode 100644 index 00000000..70e75bcc --- /dev/null +++ b/test/archive-unified.test.ts @@ -0,0 +1,126 @@ +import fs from "node:fs/promises"; +import { createHash } from "node:crypto"; +import path from "node:path"; +import { Readable, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import * as zlib from "node:zlib"; +import { execFileSync } from "node:child_process"; +import { afterEach, describe, expect, it } from "vitest"; +import { extractArchive, readArchiveEntry } from "../src/archive.js"; +import { resolveTarMeterLimits } from "../src/archive-limits.js"; +import { TarParserStream, type AdmittedTarMember } from "../src/archive-tar-wasm.js"; +import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; +import { __setNativeLoaderForTest, __resetNativeLoaderForTest } from "../src/native.js"; +import { paxNative } from "./helpers/archive-pax-native.js"; +import { unicodeNames, unifiedFixture, rawUnicodeFixture } from "./helpers/archive-unified.js"; +import { paxHeader } from "./helpers/archive-pax.js"; +import { tarFixture } from "./helpers/archive-fuzz.js"; +import { useTempDirs } from "./helpers/vitest.js"; + +const { tempRoot } = useTempDirs(); +const hash = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +afterEach(() => { __resetFsSafeNativeConfigForTest(); __resetNativeLoaderForTest(); }); + +async function parse(bytes: Buffer, chunkSize: number) { + const members: AdmittedTarMember[] = []; + function* chunks() { for (let offset = 0; offset < bytes.length; offset += chunkSize) yield bytes.subarray(offset, offset + chunkSize); } + await pipeline(Readable.from(chunks()), new TarParserStream(resolveTarMeterLimits(), (entry) => members.push(entry)), + new Writable({ write(_chunk, _encoding, callback) { callback(); } })); + return members; +} + +it.each([511, 512, 513])("preserves strict Unicode at metadata offset %i under arbitrary WASM chunks", async (alignment) => { + const { bytes, payload } = unifiedFixture("雪.txt", 1, 700, alignment); + expect(bytes.indexOf(Buffer.from("雪"))).toBe(512 + alignment); + for (const chunk of [1, 2, 3, 7, 511, 512, 513, 1023, 65536]) { + const entries = await parse(bytes, chunk); + expect(entries.map(({ path, type, size }) => ({ path, type, size }))).toEqual([ + { path: "雪.txt", type: "File", size: 700 }, { path: "sentinel", type: "File", size: 3 }, + ]); + expect(hash(bytes.subarray(entries[0]!.offset, entries[0]!.offset + entries[0]!.size))).toBe(hash(payload)); + expect(bytes.subarray(entries[1]!.offset, entries[1]!.offset + 3).toString()).toBe("end"); + } +}); + +let bzip = false; +try { execFileSync("bzip2", ["--help"], { stdio: "ignore" }); bzip = true; } catch { /* Not installed on every host. */ } +const routes = [ + { mode: "off", codec: "tar" }, { mode: "off", codec: "gzip" }, + ...(paxNative ? [{ mode: "require", codec: "tar" }, { mode: "require", codec: "gzip" }, + ...(typeof zlib.zstdCompressSync === "function" ? [{ mode: "require", codec: "zstd" }] : []), + ...(bzip ? [{ mode: "require", codec: "bzip2" }] : [])] : []), +] as const; + +describe.each(routes)("unified public TAR $mode/$codec", ({ mode, codec }) => { + async function setup(bytes: Buffer) { + configureFsSafeNative({ mode: mode as "off" | "require" }); + if (mode === "require") __setNativeLoaderForTest(() => paxNative!); + const base = await tempRoot("fs-safe-unified-"); + const archivePath = path.join(base, "input.tar"); + const destDir = path.join(base, "out"); + await fs.mkdir(destDir); + const encoded = codec === "gzip" ? zlib.gzipSync(bytes) : codec === "zstd" ? zlib.zstdCompressSync(bytes) + : codec === "bzip2" ? execFileSync("bzip2", ["-c"], { input: bytes }) : bytes; + await fs.writeFile(archivePath, encoded); + const kind = codec === "zstd" ? "tar-zstd" as const : codec === "bzip2" ? "tar-bzip2" as const : "tar" as const; + return { archivePath, destDir, kind, timeoutMs: 10000 }; + } + it.each(unicodeNames)("retains exact PAX name %j, payload hash and sentinel", async (name) => { + const { bytes, payload } = unifiedFixture(name); + const fixture = await setup(bytes); + const entries = await parse(bytes, 7); + if (mode === "require") { + const native = await paxNative!.inspectArchiveNative(fixture.archivePath, fixture.kind, resolveTarMeterLimits(), new AbortController().signal); + expect(native.map(({ path, size, mode }) => ({ path, size, mode }))) + .toEqual(entries.map(({ path, size, mode }) => ({ path, size, mode }))); + } + expect(hash(await readArchiveEntry(fixture.archivePath, name, { maxBytes: payload.length, kind: fixture.kind }))).toBe(hash(payload)); + await expect(readArchiveEntry(fixture.archivePath, name, { maxBytes: payload.length - 1, kind: fixture.kind })) + .rejects.toMatchObject({ code: "archive-entry-extracted-size-exceeds-limit" }); + // TAR interpretation accepts LF on Windows; its filesystem cannot create that name. + if (process.platform !== "win32" || !name.includes("\n")) { + await extractArchive(fixture); + expect((await fs.readdir(fixture.destDir)).sort()).toEqual([name, "sentinel"].sort()); + expect(hash(await fs.readFile(path.join(fixture.destDir, name)))).toBe(hash(payload)); + } + expect((await readArchiveEntry(fixture.archivePath, "sentinel", { maxBytes: 3, kind: fixture.kind })).toString()).toBe("end"); + }); + it.each([[1, 700], [700, 1], [700, 0]])("keeps newline xattrs inert before/after effective size %i -> %i", async (raw, size) => { + for (const first of [false, true]) { + const { bytes, payload } = unifiedFixture("雪.txt", raw, size, 511, first); + const fixture = await setup(bytes); + await extractArchive(fixture); + expect(await fs.readFile(path.join(fixture.destDir, "雪.txt"))).toEqual(payload); + expect((await readArchiveEntry(fixture.archivePath, "sentinel", { maxBytes: 3, kind: fixture.kind })).toString()).toBe("end"); + } + }); + it.each(["café", "雪.txt", "raw\nname"])("accepts PAX-attached raw UTF-8 %j", async (name) => { + const fixture = await setup(rawUnicodeFixture(name)); + expect((await readArchiveEntry(fixture.archivePath, name, { maxBytes: 11, kind: fixture.kind })).toString()).toBe("raw payload"); + if (process.platform !== "win32" || !name.includes("\n")) { + await extractArchive(fixture); + expect(await fs.readFile(path.join(fixture.destDir, name), "utf8")).toBe("raw payload"); + } + }); + it("admits strict Unicode/newline linkpath without authorizing link creation", async () => { + const bytes = tarFixture([paxHeader([["linkpath", "雪\ntarget"]]), { path: "link", type: "2", linkPath: "raw-target" }, + { path: "sentinel", body: "end" }]); + const fixture = await setup(bytes); + await expect(extractArchive(fixture)).rejects.toMatchObject({ code: "entry-link" }); + expect(await fs.readdir(fixture.destDir)).toEqual([]); + await extractArchive({ ...fixture, onFiltered: "skip-entry", entryFilter: ({ kind }) => kind === "symlink" ? "skip" : "extract" }); + expect(await fs.readdir(fixture.destDir)).toEqual(["sentinel"]); + }); + it.each([0, 345, 157])("rejects invalid raw UTF-8 at field %i despite a valid PAX override", async (offset) => { + const bytes = tarFixture([paxHeader([["path", "valid"]]), { path: "raw", mutateHeader(header) { header[offset] = 0xff; } }]); + const fixture = await setup(bytes); + await expect(extractArchive(fixture)).rejects.toMatchObject({ code: "entry-path" }); + expect(await fs.readdir(fixture.destDir)).toEqual([]); + await expect(readArchiveEntry(fixture.archivePath, "valid", { maxBytes: 10, kind: fixture.kind })).rejects.toMatchObject({ code: "entry-path" }); + }); + it.each([Buffer.from([0xff]), Buffer.from("before\0after")])("rejects malformed structural PAX bytes %#", async (value) => { + const fixture = await setup(tarFixture([paxHeader([["path", value]]), { path: "raw" }])); + await expect(extractArchive(fixture)).rejects.toMatchObject({ code: "archive-header-invalid" }); + expect(await fs.readdir(fixture.destDir)).toEqual([]); + }); +}); diff --git a/test/archive-wasm-abi.test.ts b/test/archive-wasm-abi.test.ts new file mode 100644 index 00000000..3f5ed5be --- /dev/null +++ b/test/archive-wasm-abi.test.ts @@ -0,0 +1,92 @@ +import { readFileSync } from "node:fs"; +import { expect, it } from "vitest"; +import { tarFixture } from "./helpers/archive-fuzz.js"; +import { unifiedFixture } from "./helpers/archive-unified.js"; + +const module = new WebAssembly.Module(readFileSync(new URL("../dist/archive-parser.wasm", import.meta.url))); +function parser() { + return new WebAssembly.Instance(module).exports as unknown as { + memory: WebAssembly.Memory; input_ptr(): number; init(a: number, b: number, c: number, d: number, windows: number): number; + push(length: number): number; finish(): number; dispose(): void; text_ptr(): number; text_len(): number; + member_type(): number; + }; +} +function text(p: ReturnType) { + return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }) + .decode(new Uint8Array(p.memory.buffer, p.text_ptr(), p.text_len())); +} +it("has no imports, rejects invalid limits and inbox lengths, and bounds linear memory", () => { + expect(WebAssembly.Module.imports(module)).toEqual([]); + const p = parser(); + for (const value of [NaN, Infinity, -1]) expect(p.init(value, 1024, 10000, 4096, 0)).toBe(-1); + expect(p.push(1)).toBe(-1); + expect(p.init(10, 1024, 10000, 4096, 0)).toBe(0); + for (const length of [0, 65537, -1, 0x7fffffff]) expect(p.push(length)).toBe(-1); + expect(p.input_ptr() + 65536).toBeLessThanOrEqual(p.memory.buffer.byteLength); + expect(() => p.memory.grow(4096)).toThrow(); + p.dispose(); + expect(p.push(1)).toBe(-1); + expect(p.finish()).toBe(-1); +}); +it("keeps concurrent parser states isolated and consumes one bounded event at a time", () => { + const parsers = [parser(), parser()]; + const names = ["雪.txt", "line\n.txt"]; + const bytes = names.map((name) => unifiedFixture(name).bytes); + const offsets = [0, 0]; + const observed: string[][] = [[], []]; + for (const p of parsers) expect(p.init(10, 1024, 20000, 4096, 0)).toBe(0); + while (offsets.some((offset, i) => offset < bytes[i]!.length)) { + parsers.forEach((p, i) => { + const offset = offsets[i]!; + if (offset === bytes[i]!.length) return; + const chunk = bytes[i]!.subarray(offset, offset + 7); + new Uint8Array(p.memory.buffer, p.input_ptr(), chunk.length).set(chunk); + const used = p.push(chunk.length); + expect(used).toBeGreaterThan(0); + expect(used).toBeLessThanOrEqual(chunk.length); + offsets[i]! += used; + if (p.member_type() >= 0) observed[i]!.push(text(p)); + }); + } + expect(observed).toEqual(names.map((name) => [name, "sentinel"])); + for (const p of parsers) { expect(p.finish()).toBe(0); p.dispose(); } +}); +it("fails metadata allocation within the memory ceiling without accepting its body", () => { + const p = parser(); + const size = 300 * 1024 * 1024; + expect(p.init(10, size, size + 1024, 4096, 0)).toBe(0); + const header = tarFixture([{ path: "PaxHeader", type: "x", mutateHeader(block) { + block.write(`${size.toString(8).padStart(11, "0")}\0`, 124); + } }], false).subarray(0, 512); + new Uint8Array(p.memory.buffer, p.input_ptr(), 512).set(header); + expect(p.push(512)).toBe(-1); + expect(text(p)).toContain("archive-meta-entry-size-exceeds-limit"); + expect(p.push(1)).toBe(-1); + p.dispose(); +}); + +it("applies the host Windows path policy to raw names even when overridden", () => { + const fixture = unifiedFixture("safe").bytes; + // The effective name stays safe; the member's raw header carries an ADS spelling. + const rawOffset = 1024; + fixture.fill(0, rawOffset, rawOffset + 100); + fixture.write("file:stream", rawOffset); + fixture.fill(32, rawOffset + 148, rawOffset + 156); + const sum = fixture.subarray(rawOffset, rawOffset + 512).reduce((a, b) => a + b, 0); + fixture.write(`${sum.toString(8).padStart(6, "0")}\0 `, rawOffset + 148); + for (const windows of [0, 1]) { + const p = parser(); + expect(p.init(10, 1024, 10000, 4096, windows)).toBe(0); + let offset = 0; + while (offset < fixture.length) { + const bytes = fixture.subarray(offset, offset + 512); + new Uint8Array(p.memory.buffer, p.input_ptr(), bytes.length).set(bytes); + const used = p.push(bytes.length); + if (used < 0) break; + offset += used; + } + if (windows) { expect(offset).toBe(rawOffset); expect(text(p)).toContain("archive-entry-path-invalid"); } + else { expect(offset).toBe(fixture.length); expect(p.finish()).toBe(0); } + p.dispose(); + } +}); diff --git a/test/coverage-gaps.test.ts b/test/coverage-gaps.test.ts index 6867f067..103f1ba7 100644 --- a/test/coverage-gaps.test.ts +++ b/test/coverage-gaps.test.ts @@ -13,7 +13,6 @@ import { } from "../src/absolute-path.js"; import { createTarEntryPreflightChecker, - readTarEntryInfo, } from "../src/archive-tar.js"; import { resolveArchiveKind, resolvePackedRootDir } from "../src/archive-kind.js"; import { pathExists, pathExistsSync } from "../src/fs.js"; @@ -147,15 +146,7 @@ describe("archive kind and tar preflight helpers", () => { await expect(resolvePackedRootDir(root)).rejects.toThrow("unexpected archive layout"); }); - it("normalizes tar entries and rejects unsafe entries", () => { - expect(readTarEntryInfo({ path: "a.txt", type: "File", size: 4.9 })).toEqual({ - path: "a.txt", - type: "File", - size: 4, - }); - expect(readTarEntryInfo({ path: "a.txt", type: "File", size: -1 })).toMatchObject({ size: 0 }); - expect(readTarEntryInfo(null)).toEqual({ path: "", type: "", size: 0 }); - + it("rejects unsafe entries through the public TAR checker", () => { const check = createTarEntryPreflightChecker({ rootDir: "/tmp/extract", stripComponents: 1, diff --git a/test/helpers/archive-gzip-container.ts b/test/helpers/archive-gzip-container.ts new file mode 100644 index 00000000..9289c94b --- /dev/null +++ b/test/helpers/archive-gzip-container.ts @@ -0,0 +1,69 @@ +import { deflateRawSync } from "node:zlib"; +import { tarFixture } from "./archive-fuzz.js"; + +// Independent RFC 1952 construction: raw DEFLATE plus explicit CRC32/ISIZE. +function crc32(bytes: Buffer): number { + let crc = 0xffffffff; + for (const byte of bytes) { + crc ^= byte; + for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + } + return (crc ^ 0xffffffff) >>> 0; +} +export function gzipMember(bytes: Buffer, optionalHeader = false): Buffer { + let header: Buffer = Buffer.from([31, 139, 8, optionalHeader ? 30 : 0, 0, 0, 0, 0, 0, 255]); + if (optionalHeader) { + header = Buffer.concat([header, Buffer.from([3, 0, 0, 255, 10]), Buffer.from("name\0comment\0")]); + const checksum = Buffer.alloc(2); + checksum.writeUInt16LE(crc32(header) & 0xffff); + header = Buffer.concat([header, checksum]); + } + const trailer = Buffer.alloc(8); + trailer.writeUInt32LE(crc32(bytes)); + trailer.writeUInt32LE(bytes.length >>> 0, 4); + return Buffer.concat([header, deflateRawSync(bytes), trailer]); +} +export function sizedGzipMember(bytes: Buffer, size: number): Buffer { + const member = gzipMember(bytes); + const extraLength = size - member.length - 2; + if (extraLength < 0 || extraLength > 0xffff) throw new RangeError("invalid gzip extra length"); + const header = Buffer.from(member.subarray(0, 10)); + header[3] |= 4; + const length = Buffer.alloc(2); + length.writeUInt16LE(extraLength); + return Buffer.concat([header, length, Buffer.alloc(extraLength), member.subarray(10)]); +} +export const gzipTar = tarFixture([{ path: "value", body: "payload" }, { path: "sentinel", body: "end" }]); +export const completeGzip = gzipMember(gzipTar); +const corrupt = (bytes: Buffer, offset: number) => { + const result = Buffer.from(bytes); result[offset] ^= 1; return result; +}; +const empty = gzipMember(Buffer.alloc(0)); +export const invalidGzipContainers: Array<[string, Buffer]> = [ + ["nonzero immediately after member", Buffer.concat([completeGzip, Buffer.from([1])])], + ...[1, 511, 512, 513, 65536, 131072].map((size): [string, Buffer] => + [`nonzero after ${size} zeros`, Buffer.concat([completeGzip, Buffer.alloc(size), Buffer.from([1])])]), + ["member after padding", Buffer.concat([completeGzip, Buffer.alloc(1), empty])], + ...[65534, 65535].map((size): [string, Buffer] => + [`empty member in a later chunk after padding at ${size}`, Buffer.concat([ + sizedGzipMember(gzipTar, size), Buffer.alloc(65536 - size), empty, + ])]), + ["incomplete following magic", Buffer.concat([completeGzip, Buffer.from([31])])], + ["incomplete following header", Buffer.concat([completeGzip, Buffer.from([31, 139, 8])])], + ["bad following header", Buffer.concat([completeGzip, Buffer.from([31, 139, 0, 0, 0, 0, 0, 0, 0, 0])])], + ["bad second CRC", Buffer.concat([completeGzip, corrupt(empty, empty.length - 8)])], + ["truncated second trailer", Buffer.concat([completeGzip, empty.subarray(0, -3)])], + ...[false, true].flatMap((padding): Array<[string, Buffer]> => { + const tail = padding ? Buffer.alloc(10240) : Buffer.alloc(0); + return [ + ["truncated body", completeGzip.subarray(0, Math.floor(completeGzip.length / 2))], + ["missing trailer", completeGzip.subarray(0, -8)], + ["truncated CRC", completeGzip.subarray(0, -6)], + ["missing ISIZE", completeGzip.subarray(0, -4)], + ["bad CRC", corrupt(completeGzip, completeGzip.length - 8)], + ["bad ISIZE", corrupt(completeGzip, completeGzip.length - 4)], + ["bad method", corrupt(completeGzip, 2)], + ["bad header CRC", corrupt(gzipMember(gzipTar, true), 10)], + ].map(([name, bytes]) => [`${name}, padding=${padding}`, Buffer.concat([bytes as Buffer, tail])]); + }), +]; diff --git a/test/helpers/archive-unified.ts b/test/helpers/archive-unified.ts new file mode 100644 index 00000000..bcdbf7fd --- /dev/null +++ b/test/helpers/archive-unified.ts @@ -0,0 +1,31 @@ +import { Pax } from "tar"; +import { paxHeader, paxRecord } from "./archive-pax.js"; +import { tarFixture } from "./archive-fuzz.js"; + +export const unicodeNames = ["雪.txt", "café", "\ufeffBOM", "0", "123", "1e3", "01", "0x10", "-1", "1.0", "line\n.txt"]; +export function unifiedFixture(name: string, rawSize = 1, size = 700, alignment?: number, sizeFirst = false) { + const payload = Buffer.alloc(size, 0xa7); + // Exercise the real npm producer's record encoder, not only our fixture encoder. + const pathRecord = Buffer.from(new Pax({ path: name }).encodeBody()); + let prefix = Buffer.alloc(0); + if (alignment !== undefined) { + const valueOffset = pathRecord.indexOf(Buffer.from(name)); + for (let count = 0; count < 1024; count++) { + const candidate = paxRecord("SCHILY.xattr.pad", "a".repeat(count)); + if (candidate.length + valueOffset === alignment) { prefix = candidate; break; } + } + if (!prefix.length) throw new Error("could not align fixture"); + } + const binary = paxRecord("SCHILY.xattr.binary", Buffer.from([0xff, 0, 10, 0xfe])); + const effectiveSize = paxRecord("size", String(size)); + const body = Buffer.concat([prefix, pathRecord, ...(sizeFirst ? [effectiveSize, binary] : [binary, effectiveSize])]); + const bytes = tarFixture([ + { path: "PaxHeader", type: "x", body }, + { path: "raw", body: payload, mutateHeader(header) { header.write(`${rawSize.toString(8).padStart(11, "0")}\0`, 124); } }, + { path: "sentinel", body: "end" }, + ]); + return { name, payload, bytes }; +} +export function rawUnicodeFixture(name: string) { + return tarFixture([paxHeader([["mtime", "1.25"]]), { path: name, body: "raw payload" }, { path: "sentinel", body: "end" }]); +}