diff --git a/Cargo.lock b/Cargo.lock index 931b23ead8..61a42da9eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5334,6 +5334,7 @@ dependencies = [ "url", "walkdir", "zip", + "zstd", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5e90901ab8..9f66dfc6ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -262,6 +262,9 @@ flate2 = "1" tar = "0.4" base64 = "0.22" zip = "2" +# zstd for transparent decompression of the npm-shipped `*.a.zst` archives. +# Statically vendored (no system libzstd dependency on the user's machine). +zstd = "0.13" # Internal crates perry-parser = { path = "crates/perry-parser" } diff --git a/crates/perry/Cargo.toml b/crates/perry/Cargo.toml index 897b3f4fb2..9d95b5cce7 100644 --- a/crates/perry/Cargo.toml +++ b/crates/perry/Cargo.toml @@ -48,6 +48,7 @@ flate2.workspace = true tar.workspace = true base64.workspace = true zip.workspace = true +zstd.workspace = true jsonwebtoken.workspace = true semver = "1.0" dotenvy = "0.15" diff --git a/crates/perry/src/commands/compile.rs b/crates/perry/src/commands/compile.rs index f503319387..67aa5180b1 100644 --- a/crates/perry/src/commands/compile.rs +++ b/crates/perry/src/commands/compile.rs @@ -26,6 +26,7 @@ mod bundle_ios; mod cjs_wrap; mod codegen_steps; mod collect_modules; +mod compressed_libs; mod env_fold; mod harmonyos_shim; mod host_config; diff --git a/crates/perry/src/commands/compile/compressed_libs.rs b/crates/perry/src/commands/compile/compressed_libs.rs new file mode 100644 index 0000000000..6ae48f10ce --- /dev/null +++ b/crates/perry/src/commands/compile/compressed_libs.rs @@ -0,0 +1,120 @@ +//! Transparent decompression of bundled, compressed static archives. +//! +//! The per-platform npm packages (`@perryts/perry-linux-arm64`, …) ship their +//! prebuilt static archives zstd-compressed (`libperry_runtime.a.zst`, +//! `libperry_stdlib.a.zst`, …) rather than raw `.a`, so the published tarball +//! stays under npm's registry upload limit. The uncompressed archives total +//! ~750 MB per platform; npm rejects the raw upload with HTTP 413 (Payload Too +//! Large). zstd brings the published package comfortably under the limit while +//! keeping every feature — the archives still contain the full stdlib so +//! out-of-tree `perry compile` can link any program. +//! +//! [`find_library_with_candidates`](super::library_search::find_library_with_candidates) +//! calls [`decompressed_archive`] when a candidate `.a` is absent but a sibling +//! `.a.zst` exists: the archive is decompressed once into a per-user cache and +//! the cached `.a` is linked. The cache slot is keyed on the compressed file's +//! size + mtime, so a new release lands in a fresh slot and a stale archive is +//! never served. Decompression is a one-time cost per machine per release; +//! later compiles reuse the cache. zstd is statically vendored into the perry +//! binary, so this adds no system-library dependency on the user's machine. +//! +//! This path is purely additive: installs that ship raw `.a` (Homebrew, apt, +//! in-tree dev builds) match a `.a` candidate first and never reach it. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context, Result}; + +/// Extension appended to a bundled archive to mark it zstd-compressed. +const COMPRESSED_EXT: &str = "zst"; + +/// `libperry_runtime.a` → `libperry_runtime.a.zst`. +pub(super) fn compressed_sibling(path: &Path) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push("."); + name.push(COMPRESSED_EXT); + PathBuf::from(name) +} + +/// Decompress `compressed` (a `*.a.zst`) into the per-user cache and return the +/// path to the decompressed archive, reusing an existing cache entry when one +/// matches. `lib_name` is the canonical archive filename (e.g. +/// `libperry_runtime.a`); it is preserved in the cache so both full-path and +/// `-L -l` link styles resolve the result. +pub(super) fn decompressed_archive(compressed: &Path, lib_name: &str) -> Result { + let meta = + fs::metadata(compressed).with_context(|| format!("stat {}", compressed.display()))?; + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); + // One slot per (size, mtime): a different release (different bytes) gets a + // distinct slot and can never shadow or be confused with a previous one. + let slot = cache_root()?.join(format!("{:x}-{:x}", meta.len(), mtime)); + let out = slot.join(lib_name); + if fs::metadata(&out).map(|m| m.len() > 0).unwrap_or(false) { + return Ok(out); + } + + fs::create_dir_all(&slot).with_context(|| format!("create cache dir {}", slot.display()))?; + eprintln!( + " decompressing bundled {} (one-time; cached under {})", + lib_name, + slot.display() + ); + + // Decompress to a process-unique temp file, then atomically rename, so a + // concurrent compile never links a half-written archive. + let tmp = slot.join(format!(".{}.{}.tmp", lib_name, std::process::id())); + let result = (|| -> Result<()> { + let input = + fs::File::open(compressed).with_context(|| format!("open {}", compressed.display()))?; + // `Decoder::new` wraps the reader in its own `BufReader`. + let mut decoder = zstd::Decoder::new(input) + .with_context(|| format!("init zstd decoder for {}", compressed.display()))?; + let mut out_file = + fs::File::create(&tmp).with_context(|| format!("create {}", tmp.display()))?; + io::copy(&mut decoder, &mut out_file) + .with_context(|| format!("zstd-decompress {}", compressed.display()))?; + // Propagate fsync failures (disk full, I/O error) so a truncated temp + // file is cleaned up below rather than renamed into the cache. + out_file + .sync_all() + .with_context(|| format!("flush {}", tmp.display()))?; + Ok(()) + })(); + if let Err(e) = result { + let _ = fs::remove_file(&tmp); + return Err(e); + } + + match fs::rename(&tmp, &out) { + Ok(()) => Ok(out), + Err(_) => { + // Lost a race with a sibling process (or a cross-device rename): + // use the finished archive if it now exists, else fail loudly. + let _ = fs::remove_file(&tmp); + if fs::metadata(&out).map(|m| m.len() > 0).unwrap_or(false) { + Ok(out) + } else { + Err(anyhow!("failed to finalize decompressed {}", out.display())) + } + } + } +} + +/// Root directory for decompressed archives. Honors `PERRY_LIB_CACHE_DIR`, +/// otherwise the platform cache dir, otherwise the system temp dir. +fn cache_root() -> Result { + if let Ok(dir) = std::env::var("PERRY_LIB_CACHE_DIR") { + if !dir.is_empty() { + return Ok(PathBuf::from(dir)); + } + } + let base = dirs::cache_dir().unwrap_or_else(std::env::temp_dir); + Ok(base.join("perry").join("libs")) +} diff --git a/crates/perry/src/commands/compile/library_search.rs b/crates/perry/src/commands/compile/library_search.rs index 4ae90fd4cb..4dfb952bd5 100644 --- a/crates/perry/src/commands/compile/library_search.rs +++ b/crates/perry/src/commands/compile/library_search.rs @@ -646,6 +646,25 @@ pub(super) fn find_library_with_candidates( if path.exists() { return Ok(path.clone()); } + // npm per-platform packages ship `*.a.zst` (the raw archives exceed + // npm's tarball upload limit). When only the compressed sibling is + // present, decompress it once into a per-user cache and link that. + let compressed = super::compressed_libs::compressed_sibling(path); + if compressed.exists() { + let lib_name = path.file_name().and_then(|s| s.to_str()).unwrap_or(name); + match super::compressed_libs::decompressed_archive(&compressed, lib_name) { + Ok(decompressed) => return Ok(decompressed), + // A compressed archive is present but couldn't be expanded + // (corrupt download, out of disk, …). Surface the real cause + // loudly here — otherwise it's masked by the generic "library + // not found" error the caller raises after exhausting candidates. + Err(e) => eprintln!( + " error: failed to decompress {}: {:#}", + compressed.display(), + e + ), + } + } } Err(candidates) } diff --git a/scripts/stage-npm.sh b/scripts/stage-npm.sh index f7bd3a5dc7..76dc79c603 100755 --- a/scripts/stage-npm.sh +++ b/scripts/stage-npm.sh @@ -174,6 +174,28 @@ for entry in "${PLATFORMS[@]}"; do done fi + # Compress the static archives so the published npm tarball stays under + # npm's registry upload limit (the raw archives total ~750 MB per platform; + # npm rejects the upload with HTTP 413). The perry binary decompresses them + # transparently on first use into a per-user cache (see compressed_libs.rs). + # The binary in bin/ is left raw — it is exec'd directly. Set + # PERRY_NPM_NO_COMPRESS=1 for local staging where uncompressed libs are handy. + if [ "${PERRY_NPM_NO_COMPRESS:-0}" != "1" ] && [ -d "$pkg_dir/lib" ]; then + if ! command -v zstd >/dev/null 2>&1; then + echo " error: zstd not found but archive compression is required" >&2 + echo " install zstd or set PERRY_NPM_NO_COMPRESS=1" >&2 + exit 1 + fi + for f in "$pkg_dir"/lib/*; do + [ -f "$f" ] || continue + case "$f" in + *.zst) continue ;; + esac + zstd -19 -T0 -q -f --rm "$f" + echo " compressed $(basename "$f") -> $(basename "$f").zst" + done + fi + render_template "$pkg_dir/package.json.tmpl" "$pkg_dir/package.json" if [ -f "$LICENSE_SRC" ]; then cp "$LICENSE_SRC" "$pkg_dir/LICENSE"