From c8b2ac955beb0582e62c5aa0ffeaae4364ed6268 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Tue, 21 Jul 2026 22:16:33 +0800 Subject: [PATCH 1/2] test(pm): add downloader race repro workflow --- .github/workflows/downloader-race-repro.yml | 52 ++++ crates/pm/src/util/downloader.rs | 306 ++++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 .github/workflows/downloader-race-repro.yml diff --git a/.github/workflows/downloader-race-repro.yml b/.github/workflows/downloader-race-repro.yml new file mode 100644 index 0000000000..339851a2ef --- /dev/null +++ b/.github/workflows/downloader-race-repro.yml @@ -0,0 +1,52 @@ +name: downloader-race-repro + +on: + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/downloader-race-repro.yml" + - "crates/pm/src/util/downloader.rs" + +permissions: + contents: read + +jobs: + linux-repro: + name: linux downloader race repro + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly-2026-01-04 + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + with: + shared-key: downloader-race-repro-linux + + - name: Try to reproduce zero-length resolved cache + shell: bash + run: | + set -uo pipefail + + test_name='downloader::tests::repro_download_can_leave_zero_file_if_racing_process_dies_after_resolved' + for attempt in $(seq 1 30); do + echo "::group::attempt ${attempt}" + cargo test -p utoo-pm "${test_name}" -- --ignored --nocapture + status=$? + echo "::endgroup::" + + if [ "${status}" -eq 0 ]; then + echo "::notice::Reproduced zero-length package.json in resolved downloader cache on attempt ${attempt}." + exit 0 + fi + + echo "attempt ${attempt} did not hit the zero-length window" + done + + echo "::error::Did not reproduce zero-length package.json after 30 attempts." + exit 1 diff --git a/crates/pm/src/util/downloader.rs b/crates/pm/src/util/downloader.rs index 8202833c24..6ce496c07c 100644 --- a/crates/pm/src/util/downloader.rs +++ b/crates/pm/src/util/downloader.rs @@ -307,10 +307,17 @@ mod tests { use flate2::Compression; use flate2::write::GzEncoder; use mockito::Server; + use std::env; use std::io::Write; + use std::path::PathBuf; + use std::process::{Child, Command}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::{Duration, Instant}; use tar::Builder; use tempfile::TempDir; use tokio::task; + use tokio::time::sleep; // Helper to create a simple tar.gz archive in memory fn create_tar_gz() -> Vec { @@ -330,6 +337,80 @@ mod tests { encoder.finish().unwrap() } + fn create_package_tar_gz() -> (Vec, String) { + let package_json = r#"{"name":"race-pkg","version":"1.0.0"}"#.to_string(); + let mut tar_data = Vec::new(); + { + let mut tar = Builder::new(&mut tar_data); + append_tar_file(&mut tar, "package/package.json", package_json.as_bytes()); + + for index in 0..128 { + let content = format!( + "file-{index}\n{}", + "0123456789abcdefghijklmnopqrstuvwxyz".repeat(256) + ); + append_tar_file( + &mut tar, + &format!("package/lib/file-{index}.txt"), + content.as_bytes(), + ); + } + + tar.finish().unwrap(); + } + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&tar_data).unwrap(); + (encoder.finish().unwrap(), package_json) + } + + fn create_large_package_json_tar_gz() -> (Vec, usize) { + let package_json = format!( + r#"{{"name":"race-pkg","version":"1.0.0","padding":"{}"}}"#, + "x".repeat(64 * 1024 * 1024) + ); + let package_json_len = package_json.len(); + let mut tar_data = Vec::new(); + { + let mut tar = Builder::new(&mut tar_data); + append_tar_file(&mut tar, "package/package.json", package_json.as_bytes()); + tar.finish().unwrap(); + } + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&tar_data).unwrap(); + (encoder.finish().unwrap(), package_json_len) + } + + fn append_tar_file(tar: &mut Builder<&mut Vec>, path: &str, content: &[u8]) { + let mut header = tar::Header::new_gnu(); + header.set_path(path).unwrap(); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append(&header, content).unwrap(); + } + + fn spawn_gated_download_helper( + test_binary: &Path, + url: &str, + dest: &Path, + ready_dir: &Path, + start_file: &Path, + ) -> Child { + Command::new(test_binary) + .arg("util::downloader::tests::download_process_helper") + .arg("--ignored") + .arg("--nocapture") + .env("UTOO_DOWNLOAD_HELPER", "1") + .env("UTOO_DOWNLOAD_HELPER_URL", url) + .env("UTOO_DOWNLOAD_HELPER_DEST", dest) + .env("UTOO_DOWNLOAD_HELPER_READY_DIR", ready_dir) + .env("UTOO_DOWNLOAD_HELPER_START_FILE", start_file) + .spawn() + .unwrap() + } + #[tokio::test] async fn test_download_idempotent() { let tar_gz = create_tar_gz(); @@ -369,4 +450,229 @@ mod tests { assert_eq!(content, "hello world"); _m.assert(); } + + #[tokio::test] + #[ignore] + async fn test_download_shared_dest_across_processes() { + let (tar_gz, package_json) = create_package_tar_gz(); + let body = Arc::new(tar_gz); + let mut server = Server::new_async().await; + let child_count = 8; + let _m = server + .mock("GET", "/pkg.tgz") + .with_status(200) + .with_header("content-type", "application/gzip") + .with_chunked_body(move |writer| { + std::thread::sleep(Duration::from_millis(200)); + for chunk in body.chunks(1024) { + writer.write_all(chunk)?; + std::thread::sleep(Duration::from_millis(2)); + } + Ok(()) + }) + .expect_at_least(2) + .expect_at_most(child_count) + .create_async() + .await; + + let url = format!("{}/pkg.tgz", server.url()); + let temp_dir = TempDir::new().unwrap(); + let dest = temp_dir.path().join("pkg"); + let ready_dir = temp_dir.path().join("ready"); + let start_file = temp_dir.path().join("start"); + crate::fs::create_dir_all(&ready_dir).await.unwrap(); + let test_binary = env::current_exe().unwrap(); + let mut children = Vec::new(); + + for _ in 0..child_count { + children.push(spawn_gated_download_helper( + &test_binary, + &url, + &dest, + &ready_dir, + &start_file, + )); + } + + wait_for_ready_files(&ready_dir, child_count, Duration::from_secs(5)).await; + crate::fs::write(&start_file, "").await.unwrap(); + + let deadline = Instant::now() + Duration::from_secs(20); + let mut failures = Vec::new(); + for mut child in children { + loop { + match child.try_wait().unwrap() { + Some(status) => { + if !status.success() { + failures.push(status.to_string()); + } + break; + } + None if Instant::now() >= deadline => { + let _ = child.kill(); + failures.push("timed out".to_string()); + break; + } + None => sleep(Duration::from_millis(25)).await, + } + } + } + + assert!(failures.is_empty(), "download helpers failed: {failures:?}"); + assert!(dest.join("_resolved").exists()); + + let content = crate::fs::read_to_string(dest.join("package/package.json")) + .await + .unwrap(); + assert_eq!(content, package_json); + + for index in [0, 31, 63, 127] { + let path = dest.join(format!("package/lib/file-{index}.txt")); + let metadata = crate::fs::metadata(&path).await.unwrap(); + assert!(metadata.len() > 0, "{} was empty", path.display()); + } + + _m.assert(); + } + + #[tokio::test] + #[ignore] + async fn repro_download_can_leave_zero_file_if_racing_process_dies_after_resolved() { + let (tar_gz, expected_package_json_len) = create_large_package_json_tar_gz(); + let body = Arc::new(tar_gz); + let request_index = Arc::new(AtomicUsize::new(0)); + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/pkg.tgz") + .with_status(200) + .with_header("content-type", "application/gzip") + .with_chunked_body(move |writer| { + let index = request_index.fetch_add(1, Ordering::SeqCst); + if index == 1 { + std::thread::sleep(Duration::from_millis(500)); + } + + for chunk in body.chunks(64 * 1024) { + writer.write_all(chunk)?; + } + Ok(()) + }) + .expect(2) + .create_async() + .await; + + let url = format!("{}/pkg.tgz", server.url()); + let temp_dir = TempDir::new().unwrap(); + let dest = temp_dir.path().join("pkg"); + let package_json_path = dest.join("package/package.json"); + let ready_dir = temp_dir.path().join("ready"); + let start_file = temp_dir.path().join("start"); + crate::fs::create_dir_all(&ready_dir).await.unwrap(); + let test_binary = env::current_exe().unwrap(); + let children = vec![ + spawn_gated_download_helper(&test_binary, &url, &dest, &ready_dir, &start_file), + spawn_gated_download_helper(&test_binary, &url, &dest, &ready_dir, &start_file), + ]; + + wait_for_ready_files(&ready_dir, 2, Duration::from_secs(5)).await; + crate::fs::write(&start_file, "").await.unwrap(); + + wait_for_path(&dest.join("_resolved"), Duration::from_secs(5)).await; + wait_for_zero_len(&package_json_path, Duration::from_secs(15)).await; + + for child in &children { + if child.id() != 0 { + let _ = Command::new("kill") + .arg("-KILL") + .arg(child.id().to_string()) + .status(); + } + } + + for mut child in children { + let _ = child.wait(); + } + + assert!(dest.join("_resolved").exists()); + assert_eq!( + crate::fs::metadata(&package_json_path).await.unwrap().len(), + 0, + "racing writer finished instead of being killed after truncating package.json; expected full size was {expected_package_json_len}" + ); + + _m.assert(); + } + + async fn wait_for_path(path: &Path, timeout: Duration) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if path.exists() { + return; + } + sleep(Duration::from_millis(10)).await; + } + panic!("timed out waiting for {}", path.display()); + } + + async fn wait_for_ready_files(path: &Path, count: usize, timeout: Duration) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + let ready_count = std::fs::read_dir(path) + .map(|entries| entries.count()) + .unwrap_or(0); + if ready_count >= count { + return; + } + sleep(Duration::from_millis(10)).await; + } + panic!( + "timed out waiting for {count} ready files in {}", + path.display() + ); + } + + async fn wait_for_zero_len(path: &Path, timeout: Duration) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if let Ok(metadata) = crate::fs::metadata(path).await + && metadata.len() == 0 + { + return; + } + sleep(Duration::from_millis(10)).await; + } + panic!("timed out waiting for zero-length {}", path.display()); + } + + #[test] + #[ignore] + fn download_process_helper() { + if env::var("UTOO_DOWNLOAD_HELPER").ok().as_deref() != Some("1") { + return; + } + + let url = env::var("UTOO_DOWNLOAD_HELPER_URL").unwrap(); + let dest = PathBuf::from(env::var("UTOO_DOWNLOAD_HELPER_DEST").unwrap()); + if let Ok(ready_dir) = env::var("UTOO_DOWNLOAD_HELPER_READY_DIR") { + std::fs::create_dir_all(&ready_dir).unwrap(); + std::fs::write( + PathBuf::from(ready_dir).join(std::process::id().to_string()), + b"ready", + ) + .unwrap(); + + let start_file = PathBuf::from(env::var("UTOO_DOWNLOAD_HELPER_START_FILE").unwrap()); + while !start_file.exists() { + std::thread::sleep(Duration::from_millis(10)); + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + download(&url, &dest).await.unwrap(); + }); + } } From 1ff70d01ab2956294095e3997dd392abcb9e82e8 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Tue, 21 Jul 2026 22:28:41 +0800 Subject: [PATCH 2/2] test(pm): stress release cache race in gha --- .github/workflows/downloader-race-repro.yml | 224 ++++++++++++++++++-- 1 file changed, 201 insertions(+), 23 deletions(-) diff --git a/.github/workflows/downloader-race-repro.yml b/.github/workflows/downloader-race-repro.yml index 339851a2ef..f94575ca9b 100644 --- a/.github/workflows/downloader-race-repro.yml +++ b/.github/workflows/downloader-race-repro.yml @@ -2,51 +2,229 @@ name: downloader-race-repro on: workflow_dispatch: + inputs: + release_tag: + description: "utoo release tag to test" + required: false + default: "utoo-v1.1.4" + attempts: + description: "stress attempts" + required: false + default: "15" + parallelism: + description: "parallel utoo install processes" + required: false + default: "24" pull_request: paths: - ".github/workflows/downloader-race-repro.yml" - - "crates/pm/src/util/downloader.rs" permissions: contents: read jobs: - linux-repro: - name: linux downloader race repro + linux-release-repro: + name: linux release cache race repro runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 + env: + UTOO_RELEASE_TAG: "utoo-v1.1.4" + ATTEMPTS: "15" + PARALLELISM: "24" + INSTALL_TIMEOUT: "75s" + RESOLVED_WAIT_TIMEOUT: "45" + REGISTRY: "https://registry.npmjs.org" steps: - - uses: actions/checkout@v4 + - name: Initialize runtime paths + shell: bash + run: | + set -euo pipefail - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - toolchain: nightly-2026-01-04 + echo "CACHE_DIR=$RUNNER_TEMP/utoo-npm-cache" >> "$GITHUB_ENV" + echo "LOG_DIR=$RUNNER_TEMP/utoo-repro-logs" >> "$GITHUB_ENV" + mkdir -p "$RUNNER_TEMP/utoo-repro-logs" - - name: Cache cargo - uses: Swatinem/rust-cache@v2 - with: - shared-key: downloader-race-repro-linux + - name: Apply manual inputs + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + run: | + set -euo pipefail + + echo "UTOO_RELEASE_TAG=${{ github.event.inputs.release_tag }}" >> "$GITHUB_ENV" + echo "ATTEMPTS=${{ github.event.inputs.attempts }}" >> "$GITHUB_ENV" + echo "PARALLELISM=${{ github.event.inputs.parallelism }}" >> "$GITHUB_ENV" + + - name: Download utoo release + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + mkdir -p "$RUNNER_TEMP/utoo-release" "$LOG_DIR" + gh release download "$UTOO_RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --pattern "utoo-linux-x64.tar.gz" \ + --dir "$RUNNER_TEMP/utoo-release" + + tar -xzf "$RUNNER_TEMP/utoo-release/utoo-linux-x64.tar.gz" -C "$RUNNER_TEMP/utoo-release" + chmod +x "$RUNNER_TEMP/utoo-release/utoo" + echo "$RUNNER_TEMP/utoo-release" >> "$GITHUB_PATH" + "$RUNNER_TEMP/utoo-release/utoo" --version - - name: Try to reproduce zero-length resolved cache + - name: Create repro project template + shell: bash + run: | + set -euo pipefail + + mkdir -p "$RUNNER_TEMP/utoo-repro-template" + cat > "$RUNNER_TEMP/utoo-repro-template/package.json" <<'JSON' + { + "name": "utoo-cache-race-repro", + "version": "0.0.0", + "private": true, + "dependencies": { + "@babel/plugin-transform-react-jsx-source": "7.29.7", + "cookie-es": "3.1.1", + "d3-time-format": "4.1.0", + "date-fns": "4.4.0", + "recharts-scale": "0.4.5", + "tapable": "2.3.3" + } + } + JSON + + - name: Stress shared cache with release utoo shell: bash run: | set -uo pipefail - test_name='downloader::tests::repro_download_can_leave_zero_file_if_racing_process_dies_after_resolved' - for attempt in $(seq 1 30); do - echo "::group::attempt ${attempt}" - cargo test -p utoo-pm "${test_name}" -- --ignored --nocapture - status=$? + UTOO_BIN="$(command -v utoo)" + ZERO_LIST="$RUNNER_TEMP/zero-package-json.txt" + : > "$ZERO_LIST" + + scan_zero_package_json() { + find "$CACHE_DIR" -type f -path '*/package/package.json' -size 0c -print 2>/dev/null | sort + } + + dump_cache_summary() { + local attempt="$1" + local mode="$2" + { + echo "attempt=$attempt mode=$mode" + echo "cache_dir=$CACHE_DIR" + find "$CACHE_DIR" -maxdepth 4 -type f \( -name package.json -o -name _resolved \) \ + -printf '%s %p\n' 2>/dev/null | sort || true + } > "$LOG_DIR/cache-summary-${mode}-${attempt}.txt" + } + + start_installers() { + local attempt="$1" + local mode="$2" + local workdir="$RUNNER_TEMP/repro-${mode}-${attempt}" + mkdir -p "$workdir" + + PIDS=() + for idx in $(seq 1 "$PARALLELISM"); do + local project="$workdir/project-$idx" + mkdir -p "$project" + cp "$RUNNER_TEMP/utoo-repro-template/package.json" "$project/package.json" + ( + cd "$project" + UTOO_CACHE_DIR="$CACHE_DIR" UTOO_REGISTRY="$REGISTRY" \ + timeout --kill-after=3s "$INSTALL_TIMEOUT" "$UTOO_BIN" \ + --cache-dir "$CACHE_DIR" \ + --registry "$REGISTRY" \ + install --ignore-scripts \ + > "$LOG_DIR/install-${mode}-${attempt}-${idx}.log" 2>&1 + ) & + PIDS+=("$!") + done + } + + wait_installers() { + local status=0 + for pid in "${PIDS[@]}"; do + if ! wait "$pid"; then + status=1 + fi + done + return "$status" + } + + kill_after_resolved() { + local deadline=$((SECONDS + RESOLVED_WAIT_TIMEOUT)) + while [ "$SECONDS" -lt "$deadline" ]; do + if find "$CACHE_DIR" -type f -name _resolved -print -quit 2>/dev/null | grep -q .; then + sleep 0.02 + local killed=0 + for pid in $(printf '%s\n' "${PIDS[@]}" | shuf); do + local state + state="$(ps -o stat= -p "$pid" 2>/dev/null | tr -d ' ' || true)" + if [ -n "$state" ] && [ "${state#Z}" = "$state" ]; then + kill -9 "$pid" 2>/dev/null || true + killed=$((killed + 1)) + fi + done + echo "killed $killed installers after _resolved appeared" + return 0 + fi + sleep 0.01 + done + echo "no _resolved observed before kill deadline" + } + + run_attempt() { + local attempt="$1" + local mode="$2" + + rm -rf "$CACHE_DIR" + mkdir -p "$CACHE_DIR" + + echo "::group::attempt $attempt ($mode)" + start_installers "$attempt" "$mode" + if [ "$mode" = "kill-after-resolved" ]; then + kill_after_resolved + fi + + wait_installers || true + scan_zero_package_json | tee "$ZERO_LIST" + dump_cache_summary "$attempt" "$mode" echo "::endgroup::" - if [ "${status}" -eq 0 ]; then - echo "::notice::Reproduced zero-length package.json in resolved downloader cache on attempt ${attempt}." + if [ -s "$ZERO_LIST" ]; then + echo "::error::Reproduced zero-byte cache package.json with release $UTOO_RELEASE_TAG on attempt $attempt ($mode)." + cat "$ZERO_LIST" + return 0 + fi + + return 1 + } + + echo "utoo=$UTOO_BIN" + echo "release=$UTOO_RELEASE_TAG attempts=$ATTEMPTS parallelism=$PARALLELISM install_timeout=$INSTALL_TIMEOUT registry=$REGISTRY" + + for attempt in $(seq 1 "$ATTEMPTS"); do + if [ "$attempt" -eq 1 ] && run_attempt "$attempt" "passive"; then exit 0 fi - echo "attempt ${attempt} did not hit the zero-length window" + if run_attempt "$attempt" "kill-after-resolved"; then + exit 0 + fi + + echo "attempt $attempt did not leave a zero-byte package.json" done - echo "::error::Did not reproduce zero-length package.json after 30 attempts." + echo "::error::Did not reproduce a zero-byte cache package.json after $ATTEMPTS release-based attempts." exit 1 + + - name: Upload repro logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: utoo-release-cache-race-logs + path: | + ${{ runner.temp }}/utoo-repro-logs + ${{ runner.temp }}/zero-package-json.txt + if-no-files-found: ignore