From 9ee4a35369d310e78517b43cf23ab3f1598198c4 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Thu, 7 May 2026 11:23:44 +0800 Subject: [PATCH 1/8] perf(pm): zero-copy tar entries in extract_tarball_sync `ExtractedEntry` previously held an owned `Vec` copied via `read_to_end` for every file. Each tarball with N files paid N allocs + N memcpys before the parallel-write phase even started. Switched to `(data_offset, data_len)` indices into the decompressed buffer; the rayon write phase slices into the shared `&[u8]` directly. Tar Slip + path-traversal guards untouched, plus a buffer-overrun check on the recorded offsets. This is 3/3 of the triplet from #2902, split out for independent A/B benchmarking against the recently-merged baseline #2887. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/pm/src/util/extractor.rs | 131 ++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 47 deletions(-) diff --git a/crates/pm/src/util/extractor.rs b/crates/pm/src/util/extractor.rs index 29b71149b0..a77bf5afc1 100644 --- a/crates/pm/src/util/extractor.rs +++ b/crates/pm/src/util/extractor.rs @@ -41,9 +41,15 @@ fn estimate_uncompressed_size(gzip_data: &[u8]) -> usize { } } +/// Tarball entry recorded during the parse pass, deferred until parallel +/// write. `data_offset`/`data_len` index into the decompressed buffer +/// (lifetime-erased from the buffer for reuse during `par_iter`); avoids +/// the per-file `Vec` allocation+copy that `read_to_end` previously +/// did for every file in the archive. struct ExtractedEntry { path: PathBuf, - content: Vec, + data_offset: usize, + data_len: usize, mode: u32, } @@ -70,7 +76,7 @@ fn extract_tarball_sync(gzip_bytes: Bytes, estimated_size: usize, dest: &Path) - use rayon::prelude::*; use std::collections::HashSet; use std::fs; - use std::io::{Cursor, Read, Write}; + use std::io::Write; // Decompress gzip using libdeflate let mut output = vec![0u8; estimated_size]; @@ -90,49 +96,11 @@ fn extract_tarball_sync(gzip_bytes: Bytes, estimated_size: usize, dest: &Path) - }; output.truncate(actual_size); - // Parse tar entries - let cursor = Cursor::new(&output[..]); - let mut archive = tar::Archive::new(cursor); - let mut entries = Vec::new(); - - for entry_result in archive.entries()? { - let mut entry = entry_result.with_context(|| "Failed to read tar entry")?; - let path = entry - .path() - .with_context(|| "Failed to get entry path")? - .into_owned(); - - // Guard against path traversal (Tar Slip): reject absolute paths - // and entries containing ".." components. - if path.is_absolute() - || path - .components() - .any(|c| matches!(c, std::path::Component::ParentDir)) - { - tracing::warn!("Skipping tar entry with unsafe path: {}", path.display()); - continue; - } - - let full_path = dest.join(&path); - - if !entry.header().entry_type().is_dir() { - let mut content = Vec::new(); - entry - .read_to_end(&mut content) - .with_context(|| format!("Failed to read tar entry: {}", path.display()))?; - - // Normalize to npm/pnpm convention: 0o755 if any exec bit is set, - // else 0o644. Preserving raw tar modes (e.g. 0o640 in google-protobuf) - // breaks world-readability in containers and multi-user setups. - let raw_mode = entry.header().mode().unwrap_or(0o644); - let mode = if raw_mode & 0o111 != 0 { 0o755 } else { 0o644 }; - entries.push(ExtractedEntry { - path: full_path, - content, - mode, - }); - } - } + // Parse tar entries — record offsets into `output` instead of copying + // each file's bytes. The buffer outlives the parallel write phase and + // is read-only after the parse loop, so concurrent slice reads are + // safe. + let entries = parse_tar_entries(&output, dest)?; // Collect every ancestor directory (up to `dest`), then create them // shallowest-first so a single mkdir() per dir is sufficient. @@ -152,11 +120,14 @@ fn extract_tarball_sync(gzip_bytes: Bytes, estimated_size: usize, dest: &Path) - fs::create_dir(dir).ok(); } - // Write files in parallel using rayon + // Write files in parallel using rayon. Each task slices into the + // shared `output` buffer (immutable borrow) — no per-file copy. + let buf: &[u8] = &output; entries.par_iter().try_for_each(|entry| -> Result<()> { let mut file = fs::File::create(&entry.path) .with_context(|| format!("Failed to create: {}", entry.path.display()))?; - file.write_all(&entry.content) + let data = &buf[entry.data_offset..entry.data_offset + entry.data_len]; + file.write_all(data) .with_context(|| format!("Failed to write: {}", entry.path.display()))?; // Skip chmod for 0o644 (most files) — File::create() already produces @@ -181,3 +152,69 @@ fn extract_tarball_sync(gzip_bytes: Bytes, estimated_size: usize, dest: &Path) - Ok(()) } + +/// Walk the tar stream and collect file entries as `(path, offset, len, mode)` +/// tuples into the decompressed `buf`. Skips directories, ignores any +/// path-traversal entries. +fn parse_tar_entries(buf: &[u8], dest: &Path) -> Result> { + use std::io::Cursor; + + let cursor = Cursor::new(buf); + let mut archive = tar::Archive::new(cursor); + let mut entries = Vec::new(); + + for entry_result in archive.entries()? { + let entry = entry_result.with_context(|| "Failed to read tar entry")?; + let path = entry + .path() + .with_context(|| "Failed to get entry path")? + .into_owned(); + + // Guard against path traversal (Tar Slip): reject absolute paths + // and entries containing ".." components. + if path.is_absolute() + || path + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + tracing::warn!("Skipping tar entry with unsafe path: {}", path.display()); + continue; + } + + if entry.header().entry_type().is_dir() { + continue; + } + + // tar::Entry exposes the byte offset and size inside the parsed + // stream — that's the same `buf` we'll later slice in `par_iter`. + let data_offset = entry.raw_file_position() as usize; + let data_len = entry.size() as usize; + if data_offset + .checked_add(data_len) + .is_none_or(|end| end > buf.len()) + { + anyhow::bail!( + "Tar entry {} extends past decompressed buffer ({}..+{} > {})", + path.display(), + data_offset, + data_len, + buf.len() + ); + } + + let full_path = dest.join(&path); + // Normalize to npm/pnpm convention: 0o755 if any exec bit is set, + // else 0o644. Preserving raw tar modes (e.g. 0o640 in google-protobuf) + // breaks world-readability in containers and multi-user setups. + let raw_mode = entry.header().mode().unwrap_or(0o644); + let mode = if raw_mode & 0o111 != 0 { 0o755 } else { 0o644 }; + entries.push(ExtractedEntry { + path: full_path, + data_offset, + data_len, + mode, + }); + } + + Ok(entries) +} From 47d62c01a2d326f1f21e3c76f7e0429e151db315 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Thu, 7 May 2026 13:32:57 +0800 Subject: [PATCH 2/8] chore(bench): extend pcap to install phase + utoo-next baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pcap-only bench was previously a one-off that captured `p1_resolve` across `utoo` and `bun`, and assumed the project tree was already cloned by `pm-bench-phases.sh` running in the same job. That gave us metadata fan-out, but install-phase regressions (#2902 / #2903 / #2904 / #2905 σ widening on `p0_full_cold`) live in the tarball download path, not in resolve. This commit makes the pcap bench self-contained and covers both phases for three PMs: - Self-clone the project if `$PROJECT_DIR` is missing (mirrors `pm-bench-phases.sh`), so this script runs as a standalone CI job. - Add a `-install` capture per PM: lock pre-existing, `cache + node_modules` wiped, then ` install`. This is the cold-tarball-download phase where the σ-widening lives. - Add `utoo-next` as a third PM: built upstream by `build-linux`'s bench-baseline step (now also gated on `pm-bench-pcap`), downloaded via the same artifact path as `bench-phases-linux`. Skipped in local runs where `$UTOO_NEXT_BIN` is unset. Workflow change: - `pm-bench-pcap-linux` now downloads the `utoo-next-linux-x64` artifact and exports `UTOO_NEXT_BIN` exactly like `bench-phases-linux` does. - `Build next branch utoo` and `Upload utoo-next binary` steps in `build-linux` now also fire for `inputs.target == 'pm-bench-pcap'`, not only `pm-bench-phases`. Outputs in `/tmp/pm-bench-pcap`: dns.txt utoo-{resolve,install}.{pcap,log} utoo-next-{resolve,install}.{pcap,log} (when UTOO_NEXT_BIN set) bun-{resolve,install}.{pcap,log} Drives the analysis of whether the install hot-path's increased concurrency (FuturesUnordered streaming, zero-copy tar, TTY-gate) saturates outbound TCP and starves the download path. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/pm-e2e-bench.yml | 17 +++++- bench/pm-bench-pcap.sh | 91 ++++++++++++++++++++++-------- 2 files changed, 82 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pm-e2e-bench.yml b/.github/workflows/pm-e2e-bench.yml index ad0e1652c6..ee02104c5f 100644 --- a/.github/workflows/pm-e2e-bench.yml +++ b/.github/workflows/pm-e2e-bench.yml @@ -154,7 +154,7 @@ jobs: - name: Build next branch utoo (bench baseline) if: > (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) || - (github.event_name == 'workflow_dispatch' && inputs.target == 'pm-bench-phases') + (github.event_name == 'workflow_dispatch' && (inputs.target == 'pm-bench-phases' || inputs.target == 'pm-bench-pcap')) run: | # The previous Upload step has already shipped the PR binary; we # can now overwrite target/.../release/utoo safely. @@ -168,7 +168,7 @@ jobs: - name: Upload utoo-next binary if: > (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) || - (github.event_name == 'workflow_dispatch' && inputs.target == 'pm-bench-phases') + (github.event_name == 'workflow_dispatch' && (inputs.target == 'pm-bench-phases' || inputs.target == 'pm-bench-pcap')) uses: actions/upload-artifact@v4 with: name: utoo-next-linux-x64 @@ -832,6 +832,19 @@ jobs: - name: Install pcap tools # tcpdump + dnsutils on top of the standard bench tooling. run: sudo apt-get update && sudo apt-get install -y tcpdump dnsutils hyperfine time + # The utoo-next binary is built upstream by build-linux's + # "Build next branch utoo (bench baseline)" step. + - name: Download utoo-next binary + uses: actions/download-artifact@v4 + with: + name: utoo-next-linux-x64 + path: /tmp/utoo-next-dist + - name: Install utoo-next + run: | + chmod +x /tmp/utoo-next-dist/utoo + mv /tmp/utoo-next-dist/utoo /tmp/utoo-next + echo "Baseline utoo (next) version: $(/tmp/utoo-next --version)" + echo "UTOO_NEXT_BIN=/tmp/utoo-next" >> $GITHUB_ENV - name: Capture pcap env: PROJECT: ${{ github.event.inputs.project || 'ant-design' }} diff --git a/bench/pm-bench-pcap.sh b/bench/pm-bench-pcap.sh index c8e869d78e..de5cbaa15f 100755 --- a/bench/pm-bench-pcap.sh +++ b/bench/pm-bench-pcap.sh @@ -1,30 +1,44 @@ #!/bin/bash -# One-off network trace capture: runs a single cold p1_resolve per PM with -# tcpdump active, so we can inspect connection topology (IPs, concurrency, -# timing) on CI after local evidence was inconclusive. +# Network trace capture for the install hot path: runs a single cold +# p1_resolve and a single cold p3_install per PM with tcpdump active, +# so we can inspect connection topology (concurrency, timing, request +# bursts, RST/dup-acks) when phase-bench σ widens unexplainedly. # -# Outputs: -# $PCAP_DIR/dns.txt — A records for the registry host -# $PCAP_DIR/{bun,utoo}.pcap — tcpdump capture of the resolve phase -# $PCAP_DIR/{bun,utoo}.log — stdout/stderr of the run +# PMs: +# utoo — the binary on $PATH (built by build-linux from the +# dispatched ref) +# utoo-next — UTOO_NEXT_BIN, the bench baseline (skipped if unset) +# bun — latest bun +# +# Phases per PM: +# resolve → lockfile generation only (metadata fan-out) +# install → tarball download + extract with lock present and cache +# cold (this is where p3_cold_install regressions live) +# +# Outputs in $PCAP_DIR: +# dns.txt — A records for the registry host +# -{resolve,install}.pcap — tcpdump capture +# -{resolve,install}.log — /usr/bin/time -v output set -eo pipefail PROJECT=${PROJECT:-ant-design} REGISTRY=${REGISTRY:-https://registry.npmjs.org} BENCH_DIR=${BENCH_DIR:-/tmp/pm-bench} PCAP_DIR=${PCAP_DIR:-/tmp/pm-bench-pcap} -UTOO_CACHE=/tmp/utoo-pcap-cache -BUN_CACHE=/tmp/bun-pcap-cache -export BUN_INSTALL_CACHE_DIR="$BUN_CACHE" -mkdir -p "$PCAP_DIR" +# Per-PM cache dirs so a wipe between phases is unambiguous. +UTOO_CACHE=${UTOO_CACHE:-/tmp/utoo-pcap-cache} +UTOO_NEXT_CACHE=${UTOO_NEXT_CACHE:-/tmp/utoo-next-pcap-cache} +BUN_CACHE=${BUN_CACHE:-/tmp/bun-pcap-cache} -# Project must already be cloned by pm-bench-phases.sh (this script is meant -# to run after it, reusing the clone). +mkdir -p "$PCAP_DIR" "$BENCH_DIR" + +# Self-clone the project if missing. Mirrors pm-bench-phases.sh so this +# script is runnable as a standalone CI job. PROJECT_DIR="$BENCH_DIR/$PROJECT" if [ ! -d "$PROJECT_DIR" ]; then - echo "missing $PROJECT_DIR — run pm-bench-phases.sh first" >&2 - exit 1 + echo "=== cloning $PROJECT ===" + git clone --depth=1 "https://github.com/ant-design/${PROJECT}.git" "$PROJECT_DIR" fi # Extract hostname (strip scheme + any path) for DNS lookup. @@ -44,17 +58,13 @@ echo "=== DNS records for $HOST ===" IFACE=$(ip route | awk '/default/ {print $5; exit}') echo "capturing on interface: $IFACE" +# Run a single command under tcpdump, write pcap+log to $PCAP_DIR/.*. capture_one() { local name=$1 shift local pcap="$PCAP_DIR/$name.pcap" local log="$PCAP_DIR/$name.log" - cd "$PROJECT_DIR" - # Cold: nothing reused. - rm -f package-lock.json bun.lock - rm -rf "$UTOO_CACHE" "$BUN_CACHE" node_modules - echo "=== capturing $name ===" # tcpdump as root; ubuntu-latest runners allow sudo without password. sudo tcpdump -s 0 -i "$IFACE" -w "$pcap" 'tcp port 443' >/dev/null 2>&1 & @@ -73,11 +83,44 @@ capture_one() { echo " → $pcap ($(wc -c <"$pcap") bytes), log: $log" } -capture_one utoo \ - utoo deps --registry="$REGISTRY" --cache-dir="$UTOO_CACHE" +# Capture both phases (resolve, install) for a single PM. Wipes lock + +# cache + node_modules before resolve; keeps the lock between resolve +# and install but wipes cache + node_modules so install is truly cold. +run_pm_phases() { + local pm_name=$1 # display name (utoo, utoo-next, bun) + local pm_bin=$2 # absolute path / `bun` + local cache_dir=$3 + + cd "$PROJECT_DIR" + rm -f package-lock.json bun.lock + rm -rf "$cache_dir" node_modules + + if [ "$pm_name" = "bun" ]; then + BUN_INSTALL_CACHE_DIR="$cache_dir" \ + capture_one "${pm_name}-resolve" \ + "$pm_bin" install --lockfile-only --registry="$REGISTRY" + rm -rf "$cache_dir" node_modules + BUN_INSTALL_CACHE_DIR="$cache_dir" \ + capture_one "${pm_name}-install" \ + "$pm_bin" install --registry="$REGISTRY" + else + capture_one "${pm_name}-resolve" \ + "$pm_bin" deps --registry="$REGISTRY" --cache-dir="$cache_dir" + rm -rf "$cache_dir" node_modules + capture_one "${pm_name}-install" \ + "$pm_bin" install --registry="$REGISTRY" --cache-dir="$cache_dir" + fi +} + +run_pm_phases utoo "$(command -v utoo)" "$UTOO_CACHE" + +if [ -n "${UTOO_NEXT_BIN:-}" ] && [ -x "$UTOO_NEXT_BIN" ]; then + run_pm_phases utoo-next "$UTOO_NEXT_BIN" "$UTOO_NEXT_CACHE" +else + echo "skip utoo-next: UTOO_NEXT_BIN not set or not executable" +fi -capture_one bun \ - bun install --lockfile-only --registry="$REGISTRY" +run_pm_phases bun "$(command -v bun)" "$BUN_CACHE" echo "done. files:" ls -lh "$PCAP_DIR" From 5d62ce672f6ce2b63dcfc9f56e7684235416825d Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Thu, 7 May 2026 16:11:40 +0800 Subject: [PATCH 3/8] chore(bench): add tshark TCP-stress metrics to pcap bench MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building on the install-phase pcap capture from the previous commit, post-process each .pcap with tshark to extract pre-TLS metrics that directly probe the "install greediness starves download" hypothesis without needing TLS session-key dumping: zero_windows — receive buffer full → server paused. Direct evidence that the app's tokio runtime is not draining the socket fast enough between extracts. retransmits — server resent because ACK was late. Indirect evidence of receive-side stall. duplicate_acks — receiver re-sent ACK because it perceived a gap. stream_gap_* — inter-packet gap distribution per TCP stream (p50 / p99 / max in microseconds). p99 / max measure the longest pause an active connection experienced — if utoo shows multi-hundred-ms gaps where utoo-next shows tens of ms, install is freezing the runtime mid-download. Per-capture summaries land at $PCAP_DIR/.summary.json. They are aggregated into a top-level summary.json via jq -s, so artifact consumers can compare metrics across PMs without re-parsing the 100s of MB of raw pcaps. Single-pass tshark over the pcap with -T fields keeps cost bounded to ~1 minute per 1 GB of capture; the full analysis pass runs after all captures so it does not bleed into wall-clock measurement. Workflow change: Install pcap tools step now also installs tshark + jq, with wireshark-common pre-seeded so tshark installs non-interactively (we only read existing pcaps, no setuid dumpcap needed). Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/pm-e2e-bench.yml | 9 +- bench/pm-bench-pcap.sh | 145 ++++++++++++++++++++++++++++- 2 files changed, 149 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pm-e2e-bench.yml b/.github/workflows/pm-e2e-bench.yml index ee02104c5f..c091713b73 100644 --- a/.github/workflows/pm-e2e-bench.yml +++ b/.github/workflows/pm-e2e-bench.yml @@ -830,8 +830,13 @@ jobs: with: bun-version: latest - name: Install pcap tools - # tcpdump + dnsutils on top of the standard bench tooling. - run: sudo apt-get update && sudo apt-get install -y tcpdump dnsutils hyperfine time + # tcpdump captures, tshark + jq for post-capture metrics analysis. + # Pre-seed wireshark-common so tshark installs non-interactively + # (no need for setuid dumpcap; we only read existing pcaps). + run: | + echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections + sudo DEBIAN_FRONTEND=noninteractive apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y tcpdump tshark dnsutils hyperfine time jq # The utoo-next binary is built upstream by build-linux's # "Build next branch utoo (bench baseline)" step. - name: Download utoo-next binary diff --git a/bench/pm-bench-pcap.sh b/bench/pm-bench-pcap.sh index de5cbaa15f..f5aa9a1b76 100755 --- a/bench/pm-bench-pcap.sh +++ b/bench/pm-bench-pcap.sh @@ -1,8 +1,10 @@ #!/bin/bash # Network trace capture for the install hot path: runs a single cold # p1_resolve and a single cold p3_install per PM with tcpdump active, -# so we can inspect connection topology (concurrency, timing, request -# bursts, RST/dup-acks) when phase-bench σ widens unexplainedly. +# then post-processes each pcap with tshark to extract TCP stress +# metrics — directly comparing utoo (greedy install) vs utoo-next +# (baseline) on signals that prove or refute "install starves +# download" without needing TLS decryption. # # PMs: # utoo — the binary on $PATH (built by build-linux from the @@ -17,8 +19,10 @@ # # Outputs in $PCAP_DIR: # dns.txt — A records for the registry host -# -{resolve,install}.pcap — tcpdump capture +# -{resolve,install}.pcap — raw tcpdump capture # -{resolve,install}.log — /usr/bin/time -v output +# -{resolve,install}.summary.json — per-capture tshark metrics +# summary.json — aggregated metrics for all captures set -eo pipefail PROJECT=${PROJECT:-ant-design} @@ -122,5 +126,140 @@ fi run_pm_phases bun "$(command -v bun)" "$BUN_CACHE" +# --- post-capture analysis: tshark metrics per pcap --------------------- +# Extract TCP-level stress signals to validate the "install greediness +# starves download" hypothesis. All of these are pre-TLS so we don't need +# session-key dumping: +# +# zero_windows — receive buffer full → server paused. Direct evidence +# that the app (utoo's tokio runtime) is not draining +# the socket fast enough. +# retransmits — server resent because ACK was late. Indirect evidence +# of receive-side stall. +# duplicate_acks — receiver re-sent ACK because it perceived a gap. +# stream_gap_* — inter-packet gap distribution per TCP stream. p99 / +# max measures the longest pause an active connection +# experienced — if utoo shows multi-hundred-ms gaps +# while utoo-next shows tens of ms, install is freezing +# the runtime mid-download. +analyze_pcap() { + local name=$1 + local pcap="$PCAP_DIR/$name.pcap" + local log="$PCAP_DIR/$name.log" + local summary="$PCAP_DIR/$name.summary.json" + + if [ ! -f "$pcap" ]; then + echo " skip analyze: $pcap missing" >&2 + return + fi + + echo "=== analyzing $name ===" + + local pcap_bytes + pcap_bytes=$(wc -c < "$pcap") + + # /usr/bin/time -v writes "Elapsed (wall clock) time (h:mm:ss or m:ss): 0:19.05" + local wall_str + wall_str=$(grep -oE 'Elapsed \(wall clock\) time[^:]*: [0-9:.]+' "$log" 2>/dev/null \ + | awk -F': ' '{print $NF}') + local wall_s + wall_s=$(awk -v t="${wall_str:-0}" 'BEGIN{ + n = split(t, p, ":") + if (n == 3) printf "%.3f", p[1]*3600 + p[2]*60 + p[3] + else if (n == 2) printf "%.3f", p[1]*60 + p[2] + else printf "%.3f", t+0 + }') + + # Single-pass extraction of analysis flags + per-packet stream/time. + # Each emitted line is `tcp.stream,frame.time_relative,zwin,retx,fast_retx,dupack,ooo`. + # Empty fields where the analysis flag does not apply. + local stats_tmp + stats_tmp=$(mktemp) + sudo tshark -r "$pcap" -T fields \ + -e tcp.stream \ + -e frame.time_relative \ + -e tcp.analysis.zero_window \ + -e tcp.analysis.retransmission \ + -e tcp.analysis.fast_retransmission \ + -e tcp.analysis.duplicate_ack \ + -e tcp.analysis.out_of_order \ + -E separator=, -E quote=n -E header=n 2>/dev/null \ + > "$stats_tmp" + + local total_packets zero_windows retransmits fast_retx dup_acks out_of_order distinct_streams + total_packets=$(wc -l < "$stats_tmp") + zero_windows=$(awk -F, '$3 != "" {c++} END {print c+0}' "$stats_tmp") + retransmits=$(awk -F, '$4 != "" {c++} END {print c+0}' "$stats_tmp") + fast_retx=$(awk -F, '$5 != "" {c++} END {print c+0}' "$stats_tmp") + dup_acks=$(awk -F, '$6 != "" {c++} END {print c+0}' "$stats_tmp") + out_of_order=$(awk -F, '$7 != "" {c++} END {print c+0}' "$stats_tmp") + distinct_streams=$(awk -F, 'NF>0 && $1!="" {print $1}' "$stats_tmp" | sort -nu | wc -l) + + # Per-stream inter-packet gaps in microseconds. Awk keeps prev_time + # per stream id so the input doesn't need to be sorted. + local gaps_tmp + gaps_tmp=$(mktemp) + awk -F, ' + $1 != "" && $2 != "" { + if ($1 in prev_time) { + delta_us = ($2 - prev_time[$1]) * 1e6 + if (delta_us > 0) print int(delta_us) + } + prev_time[$1] = $2 + }' "$stats_tmp" | sort -n > "$gaps_tmp" + + local gap_count gap_p50 gap_p99 gap_max + gap_count=$(wc -l < "$gaps_tmp") + gap_p50=0; gap_p99=0; gap_max=0 + if [ "$gap_count" -gt 0 ]; then + local p50_idx=$(( gap_count / 2 )) + local p99_idx=$(( gap_count * 99 / 100 )) + [ "$p50_idx" -lt 1 ] && p50_idx=1 + [ "$p99_idx" -lt 1 ] && p99_idx=1 + gap_p50=$(sed -n "${p50_idx}p" "$gaps_tmp") + gap_p99=$(sed -n "${p99_idx}p" "$gaps_tmp") + gap_max=$(tail -1 "$gaps_tmp") + fi + + cat > "$summary" </dev/null && [ "${#SUMMARIES[@]}" -gt 0 ]; then + jq -s '{captures: .}' "${SUMMARIES[@]}" > "$PCAP_DIR/summary.json" +else + echo "skip summary aggregation: jq not available or no summaries" >&2 +fi + echo "done. files:" ls -lh "$PCAP_DIR" From ade7c6f6ae18a2f572444c64b6bd1db80c92a822 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Thu, 7 May 2026 16:28:37 +0800 Subject: [PATCH 4/8] fix(bench): make tshark analysis best-effort, awk-only wall-time parse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The analysis pass aborted the whole job on the first .pcap because the wall-time grep returned no match, and `set -eo pipefail` propagated that exit-1 through `local x=$(grep | awk)` (the multi-line `local x; x=$(...)` form does NOT mask the exit code, unlike `local x=$(...)` on one line — bash gotcha). Two-part fix: 1. Drop into `set +e` / `set +o pipefail` for the analysis function body. The metrics are diagnostic — one tshark hiccup or an empty log line should not nuke a 25-minute capture run. Strict mode is restored at the end of the function so the rest of the script keeps its safety net. 2. Replace `grep -oE | awk` with awk-only. awk returns 0 even when no record matches, so empty-result log files no longer trip pipefail. Same parse, fewer pipes. Co-Authored-By: Claude Opus 4.7 (1M context) --- bench/pm-bench-pcap.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bench/pm-bench-pcap.sh b/bench/pm-bench-pcap.sh index f5aa9a1b76..8b309906bc 100755 --- a/bench/pm-bench-pcap.sh +++ b/bench/pm-bench-pcap.sh @@ -150,18 +150,23 @@ analyze_pcap() { if [ ! -f "$pcap" ]; then echo " skip analyze: $pcap missing" >&2 - return + return 0 fi echo "=== analyzing $name ===" + # Best-effort analysis: a single tshark hiccup or empty log shouldn't + # take down the whole job. The metrics are diagnostic, not load-bearing. + set +e + set +o pipefail + local pcap_bytes pcap_bytes=$(wc -c < "$pcap") - # /usr/bin/time -v writes "Elapsed (wall clock) time (h:mm:ss or m:ss): 0:19.05" + # /usr/bin/time -v writes "Elapsed (wall clock) time (h:mm:ss or m:ss): 0:19.05". + # Use awk-only so a no-match never returns non-zero. local wall_str - wall_str=$(grep -oE 'Elapsed \(wall clock\) time[^:]*: [0-9:.]+' "$log" 2>/dev/null \ - | awk -F': ' '{print $NF}') + wall_str=$(awk -F': ' '/Elapsed \(wall clock\)/ {print $NF}' "$log" 2>/dev/null) local wall_s wall_s=$(awk -v t="${wall_str:-0}" 'BEGIN{ n = split(t, p, ":") @@ -244,6 +249,10 @@ EOF rm -f "$stats_tmp" "$gaps_tmp" echo " packets=$total_packets streams=$distinct_streams retx=$retransmits zwin=$zero_windows dup_ack=$dup_acks gap_p99=${gap_p99}us gap_max=${gap_max}us" + + # Restore the file-level strict mode now that analysis is done. + set -e + set -o pipefail } echo "=== analysis pass ===" From 22d0fc0e52fe5c82914999d5c6af9499b5f52fd5 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Thu, 7 May 2026 17:09:49 +0800 Subject: [PATCH 5/8] chore(bench): add iostat sampling to pcap bench for disk-saturation diagnosis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TCP-level analysis (zero-copy retx=123 vs baseline 4-18) gave strong evidence that utoo's receiver runtime is under back-pressure during install, but it doesn't tell us *why*. The leading hypothesis is disk IO saturation: rayon's parallel `fs::create + write_all` over 80k+ files in the ant-design tarball burst can outrun GitHub Actions runners' Azure-disk IOPS budget, blocking write threads → tokio threads back up → socket buffers fill. This commit adds an iostat-x sampler to each capture: capture_one() now spawns `iostat -x -y 1` in parallel with tcpdump, writing per-second device samples to $PCAP_DIR/.iostat.txt. Both samplers are torn down with the workload command. analyze_pcap() parses the iostat log via column-position lookup (sysstat header row → column index map) and extracts: io_util_max_pct — peak disk-busy percentage io_util_avg_pct — average disk-busy percentage io_w_iops_max — peak write IOPS io_w_kbs_max — peak write throughput (kB/s) io_w_await_max_ms — peak write queue wait (ms) io_samples — sample count for sanity check These six fields land in summary.json alongside the TCP metrics, so artifact consumers can directly cross-correlate disk pressure with TCP back-pressure within the same capture window. The decision rule: * If io_util_max_pct stays high (>80%) on the experiment branch while baseline same-PM utoo-next stays low → install path is saturating disk and that's the mechanism. * If both branches show similar low %util, disk is not the bottleneck and we keep looking (e.g. CPU contention). Workflow: apt install adds `sysstat` (iostat lives there). It is preinstalled on ubuntu-latest images today, but pinning the dependency makes future image rebuilds resilient. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/pm-e2e-bench.yml | 5 ++- bench/pm-bench-pcap.sh | 69 ++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pm-e2e-bench.yml b/.github/workflows/pm-e2e-bench.yml index c091713b73..74c90ece5d 100644 --- a/.github/workflows/pm-e2e-bench.yml +++ b/.github/workflows/pm-e2e-bench.yml @@ -830,13 +830,14 @@ jobs: with: bun-version: latest - name: Install pcap tools - # tcpdump captures, tshark + jq for post-capture metrics analysis. + # tcpdump captures, tshark + jq for post-capture metrics analysis, + # sysstat (iostat) for per-capture disk-saturation sampling. # Pre-seed wireshark-common so tshark installs non-interactively # (no need for setuid dumpcap; we only read existing pcaps). run: | echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections sudo DEBIAN_FRONTEND=noninteractive apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y tcpdump tshark dnsutils hyperfine time jq + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y tcpdump tshark dnsutils hyperfine time jq sysstat # The utoo-next binary is built upstream by build-linux's # "Build next branch utoo (bench baseline)" step. - name: Download utoo-next binary diff --git a/bench/pm-bench-pcap.sh b/bench/pm-bench-pcap.sh index 8b309906bc..7a0f7c8192 100755 --- a/bench/pm-bench-pcap.sh +++ b/bench/pm-bench-pcap.sh @@ -62,29 +62,42 @@ echo "=== DNS records for $HOST ===" IFACE=$(ip route | awk '/default/ {print $5; exit}') echo "capturing on interface: $IFACE" -# Run a single command under tcpdump, write pcap+log to $PCAP_DIR/.*. +# Run a single command under tcpdump + iostat, write pcap+log+iostat +# samples to $PCAP_DIR/.*. iostat is sampled at 1Hz with the `-y` +# flag so we drop the since-boot summary and only record live samples +# during the capture window. Disk-saturation diagnosis (is install +# starving download via blocking on writes?) needs %util / w/s / w_await +# alongside the TCP signals. capture_one() { local name=$1 shift local pcap="$PCAP_DIR/$name.pcap" local log="$PCAP_DIR/$name.log" + local iostat_log="$PCAP_DIR/$name.iostat.txt" echo "=== capturing $name ===" # tcpdump as root; ubuntu-latest runners allow sudo without password. sudo tcpdump -s 0 -i "$IFACE" -w "$pcap" 'tcp port 443' >/dev/null 2>&1 & local tcpdump_pid=$! - # Let tcpdump bind before the workload starts. + # iostat -x = extended stats, -y = skip the since-boot first sample, + # `1` = 1-second interval. Runs unprivileged; sysstat is preinstalled + # on ubuntu-latest. + iostat -x -y 1 > "$iostat_log" 2>/dev/null & + local iostat_pid=$! + # Let tcpdump and iostat bind before the workload starts. sleep 1 /usr/bin/time -v "$@" >"$log" 2>&1 || echo " run exited with $?" sudo kill "$tcpdump_pid" 2>/dev/null || true + kill "$iostat_pid" 2>/dev/null || true wait "$tcpdump_pid" 2>/dev/null || true + wait "$iostat_pid" 2>/dev/null || true # Make pcap readable by the later upload-artifact step (tcpdump writes # as root). sudo chmod 644 "$pcap" - echo " → $pcap ($(wc -c <"$pcap") bytes), log: $log" + echo " → $pcap ($(wc -c <"$pcap") bytes), log: $log, iostat: $iostat_log" } # Capture both phases (resolve, install) for a single PM. Wipes lock + @@ -226,6 +239,47 @@ analyze_pcap() { gap_max=$(tail -1 "$gaps_tmp") fi + # iostat parsing — extract %util, w/s, wkB/s, w_await peaks across + # all per-second samples. Sysstat 12+ output has a `Device` header row + # before each sampling block; the column order is stable: + # Device r/s rkB/s rrqm/s %rrqm r_await rareq-sz w/s wkB/s wrqm/s + # %wrqm w_await wareq-sz d/s dkB/s drqm/s %drqm d_await dareq-sz + # f/s f_await aqu-sz %util + # We pin to header positions so a sysstat ABI shift is a clear failure + # rather than a silently-wrong number. + local iostat_log="$PCAP_DIR/$name.iostat.txt" + local io_util_max=0 io_util_avg=0 io_w_iops_max=0 io_w_kbs_max=0 io_w_await_max=0 io_samples=0 + if [ -s "$iostat_log" ]; then + # The awk script below tracks header column positions (in case sysstat + # changes ordering between versions) and then walks every device row, + # accumulating max/avg of %util plus max of write-side metrics. + read -r io_util_max io_util_avg io_w_iops_max io_w_kbs_max io_w_await_max io_samples <<< "$(awk ' + /^Device/ { + # Reset column map per header block (in case columns drift). + for (i = 1; i <= NF; i++) col[$i] = i + in_dev = 1 + next + } + /^$/ { in_dev = 0; next } + in_dev && NF >= col["%util"] && col["%util"] > 0 { + u = $col["%util"] + 0 + ws = $col["w/s"] + 0 + wkbs = $col["wkB/s"] + 0 + wa = $col["w_await"] + 0 + if (u > util_max) util_max = u + util_sum += u + util_n++ + if (ws > w_iops_max) w_iops_max = ws + if (wkbs > w_kbs_max) w_kbs_max = wkbs + if (wa > w_await_max) w_await_max = wa + } + END { + util_avg = util_n > 0 ? util_sum / util_n : 0 + printf "%.2f %.2f %.0f %.0f %.2f %d", util_max+0, util_avg+0, w_iops_max+0, w_kbs_max+0, w_await_max+0, util_n+0 + } + ' "$iostat_log")" + fi + cat > "$summary" < Date: Thu, 7 May 2026 20:01:10 +0800 Subject: [PATCH 6/8] perf(pm): chunked write in extract_tarball_sync to bound disk burst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Direct A/B follow-up: pcap+iostat instrumentation on this branch (par_iter zero-copy) showed install-time disk burst saturating GHA's disk queue — util_max=92%, w_await peaks of 490ms, paired with TCP-level retx=123 vs utoo-next's retx=4 in the same run. The mechanism: rayon spreads every individual file write across all workers, so each tarball's extract submits N concurrent IO syscalls and the in-kernel queue depth blows up. Write threads block on the saturated queue, the tokio runtime can't drain socket buffers between extracts, ACKs delay, server retransmits. Switch from `par_iter()` to `par_chunks(64)` with a sequential body inside each chunk. Cross-thread parallelism is preserved: each rayon worker still picks up a chunk and runs in parallel with sibling chunks. What changes is the in-flight write count *per worker* — it goes from "as many as rayon will divide into me" down to one. The disk queue stays drained. Same pcap+iostat A/B with this commit: retx 123 → 10 (Δ utoo-vs-utoo-next +119 → +4) zwin 9 → 1 util_max 92% → 81% (no longer saturating) util_avg 19% → 19% (work done, just smoother) w_await 490ms → 160ms (disk queue not backed up) wall 18s → 20s (run-to-run noise, no real loss) Companion to the iter() / rayon-cap experiments on experiment/zero-copy-{iter-write,rayon-cap}: par_chunks is the only option that brought retx back to baseline *without* increasing zwin or wall, so it's the recommended ship form. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/pm/src/util/extractor.rs | 48 +++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/crates/pm/src/util/extractor.rs b/crates/pm/src/util/extractor.rs index a77bf5afc1..bdcd9e8d98 100644 --- a/crates/pm/src/util/extractor.rs +++ b/crates/pm/src/util/extractor.rs @@ -120,24 +120,38 @@ fn extract_tarball_sync(gzip_bytes: Bytes, estimated_size: usize, dest: &Path) - fs::create_dir(dir).ok(); } - // Write files in parallel using rayon. Each task slices into the - // shared `output` buffer (immutable borrow) — no per-file copy. + // Write files using par_chunks for batched parallelism. par_iter + // spread every individual write across all rayon workers, queuing + // N concurrent IO syscalls into the kernel scheduler at burst — + // pcap+iostat A/B on GHA 2-core showed util_max=92% + w_await peaks + // of 490ms paired with TCP-level retx=123 on the install hot path. + // par_chunks(64) keeps cross-thread parallelism (each rayon worker + // takes a 64-file chunk in parallel with sibling chunks) but bounds + // in-flight write count per worker to 1. The same A/B with this + // change collapsed retx 123 → 10, w_await 490ms → 160ms, util_max + // 92% → 81%, with no observable wall-time loss vs par_iter (20s vs + // 18s, within run-to-run noise). + const WRITE_CHUNK_SIZE: usize = 64; let buf: &[u8] = &output; - entries.par_iter().try_for_each(|entry| -> Result<()> { - let mut file = fs::File::create(&entry.path) - .with_context(|| format!("Failed to create: {}", entry.path.display()))?; - let data = &buf[entry.data_offset..entry.data_offset + entry.data_len]; - file.write_all(data) - .with_context(|| format!("Failed to write: {}", entry.path.display()))?; - - // Skip chmod for 0o644 (most files) — File::create() already produces - // this via umask (0o666 & ~0o022 = 0o644). - #[cfg(unix)] - if entry.mode != 0o644 { - fs::set_permissions(&entry.path, fs::Permissions::from_mode(entry.mode)).ok(); - } - Ok(()) - })?; + entries + .par_chunks(WRITE_CHUNK_SIZE) + .try_for_each(|chunk| -> Result<()> { + for entry in chunk { + let mut file = fs::File::create(&entry.path) + .with_context(|| format!("Failed to create: {}", entry.path.display()))?; + let data = &buf[entry.data_offset..entry.data_offset + entry.data_len]; + file.write_all(data) + .with_context(|| format!("Failed to write: {}", entry.path.display()))?; + + // Skip chmod for 0o644 (most files) — File::create() already produces + // this via umask (0o666 & ~0o022 = 0o644). + #[cfg(unix)] + if entry.mode != 0o644 { + fs::set_permissions(&entry.path, fs::Permissions::from_mode(entry.mode)).ok(); + } + } + Ok(()) + })?; // Set directory permissions #[cfg(unix)] From 1040690df06a2af352bca20ade19e73e23b9c6e0 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Thu, 7 May 2026 22:54:59 +0800 Subject: [PATCH 7/8] fix(pm): use try_from for tar offset/size to avoid 32-bit truncation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Gemini review feedback on #2905 (lines 204-205): the previous \`as usize\` cast on \`entry.raw_file_position()\` and \`entry.size()\` silently truncates on 32-bit targets when a tar header claims a >4GB value. utoo only ships 64-bit binaries, so this is defensive — but the \`try_from\` path also turns a malformed tar header into a clean error instead of a buffer-overrun bug. The buffer-overrun guard a few lines below catches the same class of issue, so this is belt-and-suspenders. Gemini's second comment on \`is_none_or\` does not apply: the project pins to \`nightly-2026-04-02\` (rust-toolchain.toml), which is far past \`is_none_or\`'s 1.82 stabilization. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/pm/src/util/extractor.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/pm/src/util/extractor.rs b/crates/pm/src/util/extractor.rs index bdcd9e8d98..50c6f36ed6 100644 --- a/crates/pm/src/util/extractor.rs +++ b/crates/pm/src/util/extractor.rs @@ -200,9 +200,14 @@ fn parse_tar_entries(buf: &[u8], dest: &Path) -> Result> { } // tar::Entry exposes the byte offset and size inside the parsed - // stream — that's the same `buf` we'll later slice in `par_iter`. - let data_offset = entry.raw_file_position() as usize; - let data_len = entry.size() as usize; + // stream — that's the same `buf` we'll later slice in par_chunks. + // Use try_from rather than `as usize` so a malformed tar header + // claiming a >usize::MAX size fails loudly instead of silently + // truncating into a buffer-overrun bug on 32-bit targets. + let data_offset = usize::try_from(entry.raw_file_position()) + .with_context(|| format!("Tar entry {} offset exceeds usize", path.display()))?; + let data_len = usize::try_from(entry.size()) + .with_context(|| format!("Tar entry {} size exceeds usize", path.display()))?; if data_offset .checked_add(data_len) .is_none_or(|end| end > buf.len()) From 873738248108f440696cef3cdbf423e953107971 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Thu, 7 May 2026 23:31:20 +0800 Subject: [PATCH 8/8] refactor(pm): use bytes::Bytes for zero-copy tar entry slicing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous form stored \`(data_offset, data_len)\` and reconstructed the slice at every write call site: let data = &buf[entry.data_offset..entry.data_offset + entry.data_len]; file.write_all(data)?; That worked but leaked offset/len arithmetic into the hot loop and forced the writer to know the parent buffer existed. \`bytes::Bytes\` is already a workspace dep — converting the decompressed buffer into \`Bytes\` once after \`output.truncate\` lets each entry hold a reference-counted slice that shares the underlying Arc allocation (no data copy, one atomic increment per slice). Diff at the call site: -let data = &buf[entry.data_offset..entry.data_offset + entry.data_len]; -file.write_all(data)?; +file.write_all(&entry.data)?; The Tar Slip / path-traversal guards and buffer-overrun check are preserved — \`Bytes::slice\` panics on out-of-bounds, so the explicit length validation is still load-bearing. The atomic ref-count cost (~ns per slice) is negligible against the write-syscall path; pcap+iostat metrics from the prior par_chunks(64) work are unaffected. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/pm/src/util/extractor.rs | 40 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/crates/pm/src/util/extractor.rs b/crates/pm/src/util/extractor.rs index 50c6f36ed6..ffe6170a61 100644 --- a/crates/pm/src/util/extractor.rs +++ b/crates/pm/src/util/extractor.rs @@ -42,14 +42,14 @@ fn estimate_uncompressed_size(gzip_data: &[u8]) -> usize { } /// Tarball entry recorded during the parse pass, deferred until parallel -/// write. `data_offset`/`data_len` index into the decompressed buffer -/// (lifetime-erased from the buffer for reuse during `par_iter`); avoids -/// the per-file `Vec` allocation+copy that `read_to_end` previously -/// did for every file in the archive. +/// write. `data` is a `bytes::Bytes` slice that shares the underlying +/// decompressed buffer via Arc, avoiding both the per-file `Vec` +/// allocation+copy that `read_to_end` previously did and the +/// offset/len arithmetic the previous zero-copy form needed at every +/// write call site. struct ExtractedEntry { path: PathBuf, - data_offset: usize, - data_len: usize, + data: Bytes, mode: u32, } @@ -96,11 +96,16 @@ fn extract_tarball_sync(gzip_bytes: Bytes, estimated_size: usize, dest: &Path) - }; output.truncate(actual_size); - // Parse tar entries — record offsets into `output` instead of copying - // each file's bytes. The buffer outlives the parallel write phase and - // is read-only after the parse loop, so concurrent slice reads are - // safe. - let entries = parse_tar_entries(&output, dest)?; + // Move the decompressed buffer into `Bytes` so per-entry slices below + // are reference-counted views into the same Arc-backed allocation — + // no data copy, no offset arithmetic at the write call site. + let buf = Bytes::from(output); + + // Parse tar entries — record `Bytes` slices into `buf` instead of + // copying each file's bytes. The buffer outlives the parallel write + // phase and is read-only after the parse loop, so concurrent slice + // reads are safe. + let entries = parse_tar_entries(&buf, dest)?; // Collect every ancestor directory (up to `dest`), then create them // shallowest-first so a single mkdir() per dir is sufficient. @@ -132,15 +137,13 @@ fn extract_tarball_sync(gzip_bytes: Bytes, estimated_size: usize, dest: &Path) - // 92% → 81%, with no observable wall-time loss vs par_iter (20s vs // 18s, within run-to-run noise). const WRITE_CHUNK_SIZE: usize = 64; - let buf: &[u8] = &output; entries .par_chunks(WRITE_CHUNK_SIZE) .try_for_each(|chunk| -> Result<()> { for entry in chunk { let mut file = fs::File::create(&entry.path) .with_context(|| format!("Failed to create: {}", entry.path.display()))?; - let data = &buf[entry.data_offset..entry.data_offset + entry.data_len]; - file.write_all(data) + file.write_all(&entry.data) .with_context(|| format!("Failed to write: {}", entry.path.display()))?; // Skip chmod for 0o644 (most files) — File::create() already produces @@ -170,10 +173,10 @@ fn extract_tarball_sync(gzip_bytes: Bytes, estimated_size: usize, dest: &Path) - /// Walk the tar stream and collect file entries as `(path, offset, len, mode)` /// tuples into the decompressed `buf`. Skips directories, ignores any /// path-traversal entries. -fn parse_tar_entries(buf: &[u8], dest: &Path) -> Result> { +fn parse_tar_entries(buf: &Bytes, dest: &Path) -> Result> { use std::io::Cursor; - let cursor = Cursor::new(buf); + let cursor = Cursor::new(buf.as_ref()); let mut archive = tar::Archive::new(cursor); let mut entries = Vec::new(); @@ -200,7 +203,7 @@ fn parse_tar_entries(buf: &[u8], dest: &Path) -> Result> { } // tar::Entry exposes the byte offset and size inside the parsed - // stream — that's the same `buf` we'll later slice in par_chunks. + // stream — the same `buf` we now reference-share via `Bytes`. // Use try_from rather than `as usize` so a malformed tar header // claiming a >usize::MAX size fails loudly instead of silently // truncating into a buffer-overrun bug on 32-bit targets. @@ -229,8 +232,7 @@ fn parse_tar_entries(buf: &[u8], dest: &Path) -> Result> { let mode = if raw_mode & 0o111 != 0 { 0o755 } else { 0o644 }; entries.push(ExtractedEntry { path: full_path, - data_offset, - data_len, + data: buf.slice(data_offset..data_offset + data_len), mode, }); }