inheritCargoArtifactsHook: symlink dependency rlib and rmeta files

CHANGELOG.md

@@ -16,6 +16,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   any installation hooks
 * **Breaking** (technically): `buildPackage` no longer installs cargo binary
   dependencies (i.e. when the `bindeps` feature is used) by default
+* `inheritCargoArtifactsHook` will now symlink dependency `.rlib` and `.rmeta`
+  files. This means that derivations which reuse existing cargo artifacts will
+  run faster as fewer files (and bytes!) need to be copied around
+* `cargoTarpaulin`'s default `cargoTarpaulinExtraArgs` no longer include
+  `--skip-clean`
 
 ### Changed
 * **Breaking**: dropped compatibility for Nix versions below 2.13.3

docs/API.md

@@ -622,7 +622,7 @@ Except where noted below, all derivation attributes are delegated to
   - Default value: `""`
 * `cargoTarpaulinExtraArgs`: additional flags to be passed in the cargo
   tarpaulin invocation
-  - Default value: `"--skip-clean --out Xml --output-dir $out"`
+  - Default value: `"--out Xml --output-dir $out"`
 
 #### Native build dependencies
 The `cargo-tarpaulin` package is automatically appended as a native build input to any
@@ -935,6 +935,7 @@ hooks:
 * `configureCargoVendoredDepsHook`
 * `inheritCargoArtifactsHook`
 * `installCargoArtifactsHook`
+* `rsync`
 * `zstd`
 
 ### `craneLib.mkDummySrc`

lib/cargoTarpaulin.nix

@@ -3,7 +3,7 @@
 }:
 
 { cargoExtraArgs ? ""
-, cargoTarpaulinExtraArgs ? "--skip-clean --out Xml --output-dir $out"
+, cargoTarpaulinExtraArgs ? "--out Xml --output-dir $out"
 , ...
 }@origArgs:
 let

lib/mkCargoDerivation.nix

@@ -5,6 +5,7 @@
 , crateNameFromCargoToml
 , inheritCargoArtifactsHook
 , installCargoArtifactsHook
+, rsync
 , stdenv
 , vendorCargoDeps
 , zstd
@@ -59,6 +60,7 @@ chosenStdenv.mkDerivation (cleanedArgs // {
     configureCargoVendoredDepsHook
     inheritCargoArtifactsHook
     installCargoArtifactsHook
+    rsync
     zstd
   ];
 

lib/setupHooks/inheritCargoArtifacts.nix

@@ -1,4 +1,5 @@
 { makeSetupHook
+, rsync
 }:
 
 makeSetupHook

lib/setupHooks/inheritCargoArtifactsHook.sh

@@ -24,26 +24,47 @@ inheritCargoArtifacts() {
   elif [ -d "${preparedArtifacts}" ]; then
     echo "copying cargo artifacts from ${preparedArtifacts} to ${cargoTargetDir}"
 
-    # NB: rustc doesn't like it when artifacts are either symlinks or hardlinks to the store
-    # (it tries to truncate files instead of unlinking and recreating them)
-    # so we're forced to do a full copy here :(
+    # Dependency .rlib and .rmeta files are content addressed and thus are not written to after
+    # being built (since changing `Cargo.lock` would rebuild everything anyway), which makes them
+    # good candidates for symlinking (esp. since they can make up 60-70% of the artifact directory
+    # on most projects). Thus we ignore them when copying all other artifacts below as we will
+    # symlink them afterwards. Note that we scope these checks to the `/deps` subdirectory; the
+    # workspace's own .rlib and .rmeta files appear one directory up (and these may require being
+    # writable depending on how the actual workspace build is being invoked, so we'll leave them
+    # alone).
+    #
+    # NB: keep the executable bit only if set on the original file
+    # but make all files writable as sometimes read-only files will make the build choke
     #
-    # Notes:
-    # - --no-target-directory to avoid nesting (i.e. `./target/target`)
-    # - preserve timestamps to avoid rebuilding
-    # - no-preserve ownership (root) so we can make the files writable
-    cp -r "${preparedArtifacts}" \
-      --no-target-directory "${cargoTargetDir}" \
-      --preserve=timestamps \
-      --no-preserve=ownership
-
-    # Keep existing permissions (e.g. exectuable), but also make things writable
-    # since the store is read-only and cargo would otherwise choke
-    chmod -R u+w "${cargoTargetDir}"
-
     # NB: cargo also doesn't like it if `.cargo-lock` files remain with a
-    # timestamp in the distant past so we need to delete them here
-    find "${cargoTargetDir}" -name '.cargo-lock' -delete
+    # timestamp in the distant past so we avoid copying them here
+    rsync \
+      --recursive \
+      --links \
+      --times \
+      --chmod=u+w \
+      --executability \
+      --exclude 'deps/*.rlib' \
+      --exclude 'deps/*.rmeta' \
+      --exclude '.cargo-lock' \
+      "${preparedArtifacts}/" \
+      "${cargoTargetDir}/"
+
+    local linkCandidates=$(mktemp linkCandidatesXXXX.txt)
+    find "${preparedArtifacts}" \
+      '(' -path '*/deps/*.rlib' -or -path '*/deps/*.rmeta' ')' \
+      -printf "%P\n" \
+      >"${linkCandidates}"
+
+    # Next create any missing directories up front so we can avoid redundant checks later
+    cat "${linkCandidates}" \
+      | xargs --no-run-if-empty -n1 dirname \
+      | sort -u \
+      | (cd "${cargoTargetDir}"; xargs --no-run-if-empty mkdir -p)
+
+    # Lastly do the actual symlinking
+    cat "${linkCandidates}" \
+      | xargs -P ${NIX_BUILD_CORES} -I '##{}##' ln -s "${preparedArtifacts}/##{}##" "${cargoTargetDir}/##{}##"
   else
     echo unable to copy cargo artifacts, \"${preparedArtifacts}\" looks invalid
     false