diff --git a/CHANGELOG.md b/CHANGELOG.md index 67af7431..d3857867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. any installation hooks * **Breaking** (technically): `buildPackage` no longer installs cargo binary dependencies (i.e. when the `bindeps` feature is used) by default +* `inheritCargoArtifactsHook` will now symlink dependency `.rlib` and `.rmeta` + files. This means that derivations which reuse existing cargo artifacts will + run faster as fewer files (and bytes!) need to be copied around +* `cargoTarpaulin`'s default `cargoTarpaulinExtraArgs` no longer include + `--skip-clean` ### Changed * **Breaking**: dropped compatibility for Nix versions below 2.13.3 diff --git a/docs/API.md b/docs/API.md index d2326346..c5877d05 100644 --- a/docs/API.md +++ b/docs/API.md @@ -622,7 +622,7 @@ Except where noted below, all derivation attributes are delegated to - Default value: `""` * `cargoTarpaulinExtraArgs`: additional flags to be passed in the cargo tarpaulin invocation - - Default value: `"--skip-clean --out Xml --output-dir $out"` + - Default value: `"--out Xml --output-dir $out"` #### Native build dependencies The `cargo-tarpaulin` package is automatically appended as a native build input to any @@ -935,6 +935,7 @@ hooks: * `configureCargoVendoredDepsHook` * `inheritCargoArtifactsHook` * `installCargoArtifactsHook` +* `rsync` * `zstd` ### `craneLib.mkDummySrc` diff --git a/lib/cargoTarpaulin.nix b/lib/cargoTarpaulin.nix index 5ac007d9..1b094f82 100644 --- a/lib/cargoTarpaulin.nix +++ b/lib/cargoTarpaulin.nix @@ -3,7 +3,7 @@ }: { cargoExtraArgs ? "" -, cargoTarpaulinExtraArgs ? "--skip-clean --out Xml --output-dir $out" +, cargoTarpaulinExtraArgs ? "--out Xml --output-dir $out" , ... }@origArgs: let diff --git a/lib/mkCargoDerivation.nix b/lib/mkCargoDerivation.nix index 76e35cb6..929acb45 100644 --- a/lib/mkCargoDerivation.nix +++ b/lib/mkCargoDerivation.nix @@ -5,6 +5,7 @@ , crateNameFromCargoToml , inheritCargoArtifactsHook , installCargoArtifactsHook +, rsync , stdenv , vendorCargoDeps , zstd @@ -59,6 +60,7 @@ chosenStdenv.mkDerivation (cleanedArgs // { configureCargoVendoredDepsHook inheritCargoArtifactsHook installCargoArtifactsHook + rsync zstd ]; diff --git a/lib/setupHooks/inheritCargoArtifacts.nix b/lib/setupHooks/inheritCargoArtifacts.nix index afb3d67e..d4a8a8fe 100644 --- a/lib/setupHooks/inheritCargoArtifacts.nix +++ b/lib/setupHooks/inheritCargoArtifacts.nix @@ -1,4 +1,5 @@ { makeSetupHook +, rsync }: makeSetupHook diff --git a/lib/setupHooks/inheritCargoArtifactsHook.sh b/lib/setupHooks/inheritCargoArtifactsHook.sh index 2456ef1c..bc9e57a8 100644 --- a/lib/setupHooks/inheritCargoArtifactsHook.sh +++ b/lib/setupHooks/inheritCargoArtifactsHook.sh @@ -24,26 +24,47 @@ inheritCargoArtifacts() { elif [ -d "${preparedArtifacts}" ]; then echo "copying cargo artifacts from ${preparedArtifacts} to ${cargoTargetDir}" - # NB: rustc doesn't like it when artifacts are either symlinks or hardlinks to the store - # (it tries to truncate files instead of unlinking and recreating them) - # so we're forced to do a full copy here :( + # Dependency .rlib and .rmeta files are content addressed and thus are not written to after + # being built (since changing `Cargo.lock` would rebuild everything anyway), which makes them + # good candidates for symlinking (esp. since they can make up 60-70% of the artifact directory + # on most projects). Thus we ignore them when copying all other artifacts below as we will + # symlink them afterwards. Note that we scope these checks to the `/deps` subdirectory; the + # workspace's own .rlib and .rmeta files appear one directory up (and these may require being + # writable depending on how the actual workspace build is being invoked, so we'll leave them + # alone). + # + # NB: keep the executable bit only if set on the original file + # but make all files writable as sometimes read-only files will make the build choke # - # Notes: - # - --no-target-directory to avoid nesting (i.e. `./target/target`) - # - preserve timestamps to avoid rebuilding - # - no-preserve ownership (root) so we can make the files writable - cp -r "${preparedArtifacts}" \ - --no-target-directory "${cargoTargetDir}" \ - --preserve=timestamps \ - --no-preserve=ownership - - # Keep existing permissions (e.g. exectuable), but also make things writable - # since the store is read-only and cargo would otherwise choke - chmod -R u+w "${cargoTargetDir}" - # NB: cargo also doesn't like it if `.cargo-lock` files remain with a - # timestamp in the distant past so we need to delete them here - find "${cargoTargetDir}" -name '.cargo-lock' -delete + # timestamp in the distant past so we avoid copying them here + rsync \ + --recursive \ + --links \ + --times \ + --chmod=u+w \ + --executability \ + --exclude 'deps/*.rlib' \ + --exclude 'deps/*.rmeta' \ + --exclude '.cargo-lock' \ + "${preparedArtifacts}/" \ + "${cargoTargetDir}/" + + local linkCandidates=$(mktemp linkCandidatesXXXX.txt) + find "${preparedArtifacts}" \ + '(' -path '*/deps/*.rlib' -or -path '*/deps/*.rmeta' ')' \ + -printf "%P\n" \ + >"${linkCandidates}" + + # Next create any missing directories up front so we can avoid redundant checks later + cat "${linkCandidates}" \ + | xargs --no-run-if-empty -n1 dirname \ + | sort -u \ + | (cd "${cargoTargetDir}"; xargs --no-run-if-empty mkdir -p) + + # Lastly do the actual symlinking + cat "${linkCandidates}" \ + | xargs -P ${NIX_BUILD_CORES} -I '##{}##' ln -s "${preparedArtifacts}/##{}##" "${cargoTargetDir}/##{}##" else echo unable to copy cargo artifacts, \"${preparedArtifacts}\" looks invalid false