perf(pm): iter1 — HTTP layer (gzip+brotli, DNS, conn pool, fire-and-forget)

bench/pm-bench-pcap.sh

@@ -19,12 +19,11 @@ export BUN_INSTALL_CACHE_DIR="$BUN_CACHE"
 
 mkdir -p "$PCAP_DIR"
 
-# Project must already be cloned by pm-bench-phases.sh (this script is meant
-# to run after it, reusing the clone).
 PROJECT_DIR="$BENCH_DIR/$PROJECT"
+mkdir -p "$BENCH_DIR"
 if [ ! -d "$PROJECT_DIR" ]; then
-  echo "missing $PROJECT_DIR — run pm-bench-phases.sh first" >&2
-  exit 1
+  echo "cloning $PROJECT into $PROJECT_DIR"
+  git clone --depth=1 "https://github.com/ant-design/${PROJECT}.git" "$PROJECT_DIR"
 fi
 
 # Extract hostname (strip scheme + any path) for DNS lookup.

crates/ruborist/Cargo.toml

@@ -7,57 +7,74 @@ license     = "MIT"
 
 # tombi: format.rules.table-keys-order.disabled = true
 [dependencies]
-anyhow       = { workspace = true }
-bytes        = "1"
-pathdiff     = { workspace = true }
-deno_semver  = "0.7"
-dirs         = { workspace = true }
-futures      = "0.3"
-parking_lot  = { workspace = true }
-petgraph     = "0.6"
-serde        = { version = "1.0", features = ["derive"] }
-serde_json   = { workspace = true }
-simd-json    = "0.17"
-thiserror    = { workspace = true }
-tokio-fs-ext = { workspace = true }
-tokio-retry  = "0.3"
-tracing      = { workspace = true }
+anyhow          = { workspace = true }
+bytes           = "1"
+pathdiff        = { workspace = true }
+crossbeam-queue = "0.3"
+dashmap         = "6"
+deno_semver     = "0.7"
+dirs            = { workspace = true }
+futures         = "0.3"
+parking_lot     = { workspace = true }
+petgraph        = "0.6"
+serde           = { version = "1.0", features = ["derive"] }
+serde_json      = { workspace = true }
+simd-json       = "0.17"
+thiserror       = { workspace = true }
+tokio-fs-ext    = { workspace = true }
+tokio-retry     = "0.3"
+tracing         = { workspace = true }
 # HTTP client - works on both native and WASM
-reqwest      = { version = "0.12", default-features = false, features = [
+#
+# We use `rustls-tls-native-roots-no-provider` (instead of the default
+# `rustls-tls-native-roots`) so reqwest doesn't pin the crypto provider
+# to `ring`. We then build our own `rustls::ClientConfig` with
+# `aws-lc-rs` (BoringSSL's crypto primitives) in `service/http.rs` and
+# pass it via `Client::use_preconfigured_tls`. Measured on CI against
+# npmjs.org: `ring` took 78 ms p50 / 154 ms max from CCS → first
+# AppData per handshake (CPU-bound; 128 parallel handshakes serialise
+# across 4 cores). aws-lc-rs's per-handshake CPU is roughly 3× faster.
+reqwest         = { version = "0.12", default-features = false, features = [
+  "brotli",
+  "gzip",
   "http2",
   "json",
-  "rustls-tls",
-  "rustls-tls-native-roots",
+  "rustls-tls-native-roots-no-provider",
   "socks"
 ] }
 # Shared URL-hash + staging helpers for native-only resolvers (git / http)
-sha2         = { workspace = true, optional = true }
-tempfile     = { workspace = true, optional = true }
+sha2            = { workspace = true, optional = true }
+tempfile        = { workspace = true, optional = true }
 # Git clone (gated on `native-git`)
-gix          = { workspace = true, features = [
+gix             = { workspace = true, features = [
   "blocking-http-transport-reqwest-rust-tls",
   "blocking-network-client",
 ], optional = true }
 # HTTP tarball extraction (gated on `http-tarball`)
 # Uses libdeflater to match pm's extractor and avoid pulling a second gzip
 # implementation (flate2) into the workspace.
-libdeflater  = { version = "1.25", optional = true }
-tar          = { version = "0.4", optional = true }
+libdeflater     = { version = "1.25", optional = true }
+tar             = { version = "0.4", optional = true }
 
 # Async runtime (works on both native and WASM with forked tokio)
 [dependencies.tokio]
 workspace = true
 
 [dev-dependencies]
-tokio = { workspace = true, features = ["macros", "rt"] }
+tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread"] }
 
 # Native-only dependencies (not compiled for WASM)
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-libc = "0.2"
+libc                = "0.2"
+# Custom rustls config with aws-lc-rs crypto provider — see service/http.rs.
+# wasm32 falls back to reqwest's default rustls + ring (transitive via reqwest).
+rustls              = { version = "0.23", default-features = false, features = ["aws-lc-rs", "logging", "std", "tls12"] }
+rustls-native-certs = "0.8"
 
 # WASM-only dependencies
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-js-sys = "0.3"  # For time in WASM
+js-sys               = "0.3"  # For time in WASM
+wasm-bindgen-futures = "0.4"  # spawn_local replaces tokio::spawn on wasm
 
 [features]
 default      = []

crates/ruborist/src/lib.rs

@@ -22,11 +22,13 @@
 //! }).await?;
 //! ```
 
+pub mod maybe_send;
 pub mod model;
 pub mod resolver;
 pub mod service;
 pub mod spec;
 pub mod traits;
+pub mod util;
 
 // ============================================================================
 // Re-exports: only items actually used by consumers
@@ -99,8 +101,3 @@ pub mod http {
     #[cfg(feature = "http-tarball")]
     pub use crate::resolver::http::{file_cache_slot, http_cache_slot};
 }
-
-/// Utility functions.
-pub mod util {
-    pub use crate::model::util::{PackageNameStr, parse_package_spec, read_package_json};
-}

crates/ruborist/src/maybe_send.rs

@@ -0,0 +1,27 @@
+//! `Send`/`Sync` shim that is `Send`/`Sync` on native targets and
+//! no-op on `wasm32`. Lets the resolver share a single trait surface
+//! across multi-thread tokio (where futures must be `Send` to be
+//! `tokio::spawn`-able) and wasm-bindgen (where `JsFuture` is `!Send`
+//! and tasks run via `wasm_bindgen_futures::spawn_local`).
+
+/// `Send` on native, no-op on wasm32.
+#[cfg(not(target_arch = "wasm32"))]
+pub trait MaybeSend: Send {}
+#[cfg(not(target_arch = "wasm32"))]
+impl<T: Send> MaybeSend for T {}
+
+#[cfg(target_arch = "wasm32")]
+pub trait MaybeSend {}
+#[cfg(target_arch = "wasm32")]
+impl<T> MaybeSend for T {}
+
+/// `Sync` on native, no-op on wasm32.
+#[cfg(not(target_arch = "wasm32"))]
+pub trait MaybeSync: Sync {}
+#[cfg(not(target_arch = "wasm32"))]
+impl<T: Sync> MaybeSync for T {}
+
+#[cfg(target_arch = "wasm32")]
+pub trait MaybeSync {}
+#[cfg(target_arch = "wasm32")]
+impl<T> MaybeSync for T {}

crates/ruborist/src/resolver/builder.rs

@@ -34,7 +34,7 @@ use crate::resolver::preload::{PreloadConfig, preload_manifests};
 use crate::resolver::registry::{ResolveError, resolve_registry_dep};
 use crate::spec::{Catalogs, PackageSpec, Protocol};
 use crate::traits::progress::{BuildEvent, EventReceiver, NoopReceiver};
-use crate::traits::registry::{RegistryClient, ResolvedPackage};
+use crate::traits::registry::{PreloadRegistry, RegistryClient, ResolvedPackage};
 
 /// Dispatch a git/github spec to the real `gix`-backed resolver when the
 /// `native-git` feature is enabled, otherwise error with a hint.
@@ -458,13 +458,16 @@ async fn process_file_dep<E>(
 ///
 /// # Returns
 /// The result of processing (reused, created, or skipped)
-pub async fn process_dependency<R: RegistryClient>(
+pub async fn process_dependency<R: RegistryClient + crate::maybe_send::MaybeSync>(
     graph: &mut DependencyGraph,
     registry: &R,
     node_index: NodeIndex,
     edge_info: &DependencyEdgeInfo,
     config: &BuildDepsConfig,
-) -> Result<ProcessResult, ResolveError<R::Error>> {
+) -> Result<ProcessResult, ResolveError<R::Error>>
+where
+    R::Error: crate::maybe_send::MaybeSend,
+{
     tracing::debug!(
         "Processing dependency {}@{} from [{:?}]",
         edge_info.name,
@@ -707,11 +710,14 @@ pub async fn process_dependency<R: RegistryClient>(
 /// // Add initial dependency edges to root...
 /// build_deps(&mut graph, &registry, PeerDeps::Include).await?;
 /// ```
-pub async fn build_deps<R: RegistryClient>(
+pub async fn build_deps<R: PreloadRegistry>(
     graph: &mut DependencyGraph,
     registry: &R,
     peer_deps: PeerDeps,
-) -> Result<(), ResolveError<R::Error>> {
+) -> Result<(), ResolveError<R::Error>>
+where
+    R::Error: crate::maybe_send::MaybeSend,
+{
     let config = BuildDepsConfig::default().with_peer_deps(peer_deps);
     build_deps_with_config(graph, registry, config, &NoopReceiver).await
 }
@@ -729,12 +735,15 @@ pub async fn build_deps<R: RegistryClient>(
 /// * `registry` - Registry client for fetching packages
 /// * `peer_deps` - How to handle peer dependencies
 /// * `receiver` - Event receiver for handling build events
-pub async fn build_deps_with_receiver<R: RegistryClient, E: EventReceiver>(
+pub async fn build_deps_with_receiver<R: PreloadRegistry, E: EventReceiver>(
     graph: &mut DependencyGraph,
     registry: &R,
     peer_deps: PeerDeps,
     receiver: &E,
-) -> Result<(), ResolveError<R::Error>> {
+) -> Result<(), ResolveError<R::Error>>
+where
+    R::Error: crate::maybe_send::MaybeSend,
+{
     let config = BuildDepsConfig::default().with_peer_deps(peer_deps);
     build_deps_with_config(graph, registry, config, receiver).await
 }
@@ -759,12 +768,15 @@ pub async fn build_deps_with_receiver<R: RegistryClient, E: EventReceiver>(
 ///
 /// build_deps_with_config(&mut graph, &registry, config, &receiver).await?;
 /// ```
-pub async fn build_deps_with_config<R: RegistryClient, E: EventReceiver>(
+pub async fn build_deps_with_config<R: PreloadRegistry, E: EventReceiver>(
     graph: &mut DependencyGraph,
     registry: &R,
     config: BuildDepsConfig,
     receiver: &E,
-) -> Result<(), ResolveError<R::Error>> {
+) -> Result<(), ResolveError<R::Error>>
+where
+    R::Error: crate::maybe_send::MaybeSend,
+{
     tracing::debug!(
         "Starting dependency tree build, peer_deps: {:?}, concurrency: {}, skip_preload: {}",
         config.peer_deps,
@@ -786,12 +798,14 @@ pub async fn build_deps_with_config<R: RegistryClient, E: EventReceiver>(
 }
 
 /// Run the preload phase to warm up the cache with manifests.
-async fn run_preload_phase<R: RegistryClient, E: EventReceiver>(
+async fn run_preload_phase<R: PreloadRegistry, E: EventReceiver>(
     graph: &DependencyGraph,
     registry: &R,
     config: &BuildDepsConfig,
     receiver: &E,
-) {
+) where
+    R::Error: crate::maybe_send::MaybeSend,
+{
     if config.skip_preload {
         return;
     }
@@ -804,18 +818,22 @@ async fn run_preload_phase<R: RegistryClient, E: EventReceiver>(
     }
 
     tracing::debug!("Preload phase: {} initial dependencies", initial_deps.len());
-    receiver.on_event(BuildEvent::PreloadStart {
-        count: initial_deps.len(),
-    });
+    // PreloadStart / PreloadComplete are emitted inside `preload_manifests`.
 
     let preload_config = PreloadConfig {
         peer_deps: config.peer_deps,
         concurrency: config.concurrency,
     };
 
+    // Wrap registry in Arc for the worker pool. Each worker holds an
+    // Arc<R> so resolution runs as an independent `tokio::spawn` task on
+    // the multi-thread runtime. UnifiedRegistry's Clone is shallow
+    // (Arc-based), so this clone is cheap.
+    let registry_arc = std::sync::Arc::new(registry.clone());
+
     let stats = preload_manifests(
         initial_deps,
-        registry,
+        registry_arc,
         preload_config,
         receiver,
         |_name, _manifest| {
@@ -829,21 +847,20 @@ async fn run_preload_phase<R: RegistryClient, E: EventReceiver>(
         stats.success_count,
         stats.failed_count
     );
-    receiver.on_event(BuildEvent::PreloadComplete {
-        success: stats.success_count,
-        failed: stats.failed_count,
-    });
 
     tracing::debug!("Preload phase: {:?}", start.elapsed());
 }
 
 /// Run the BFS traversal phase to build the dependency tree.
-async fn run_bfs_phase<R: RegistryClient, E: EventReceiver>(
+async fn run_bfs_phase<R: RegistryClient + crate::maybe_send::MaybeSync, E: EventReceiver>(
     graph: &mut DependencyGraph,
     registry: &R,
     config: &BuildDepsConfig,
     receiver: &E,
-) -> Result<(), ResolveError<R::Error>> {
+) -> Result<(), ResolveError<R::Error>>
+where
+    R::Error: crate::maybe_send::MaybeSend,
+{
     let start = tokio::time::Instant::now();
 
     let mut current_level = vec![graph.root_index];
@@ -961,10 +978,13 @@ use std::path::Path;
 /// let pkg: PackageJson = serde_json::from_str(&pkg_content)?;
 /// let lock = resolve(&pkg, &registry).await?;
 /// ```
-pub async fn resolve<R: RegistryClient>(
+pub async fn resolve<R: PreloadRegistry>(
     pkg: &PackageJson,
     registry: &R,
-) -> Result<PackageLock, ResolveError<R::Error>> {
+) -> Result<PackageLock, ResolveError<R::Error>>
+where
+    R::Error: crate::maybe_send::MaybeSend,
+{
     resolve_with_options(pkg, registry, PeerDeps::Include, &NoopReceiver).await
 }
 
@@ -975,12 +995,15 @@ pub async fn resolve<R: RegistryClient>(
 /// * `registry` - Registry client for fetching packages
 /// * `peer_deps` - How to handle peer dependencies
 /// * `receiver` - Event receiver for progress tracking
-pub async fn resolve_with_options<R: RegistryClient, E: EventReceiver>(
+pub async fn resolve_with_options<R: PreloadRegistry, E: EventReceiver>(
     pkg: &PackageJson,
     registry: &R,
     peer_deps: PeerDeps,
     receiver: &E,
-) -> Result<PackageLock, ResolveError<R::Error>> {
+) -> Result<PackageLock, ResolveError<R::Error>>
+where
+    R::Error: crate::maybe_send::MaybeSend,
+{
     // Create graph with root node
     let mut graph = DependencyGraph::from_package_json(PathBuf::from("."), pkg.clone());