[8.8.0] Cherry-pick remote repo contents cache feature

src/main/cpp/blaze.cc

@@ -548,6 +548,11 @@ static vector<string> GetServerExeArgs(const blaze_util::Path &jvm_path,
   } else {
     result.push_back("--nowindows_enable_symlinks");
   }
+  if (startup_options.remote_repo_contents_cache) {
+    result.push_back("--experimental_remote_repo_contents_cache");
+    // Don't set the flag to false if it's not set - non-OSS Blaze does not know
+    // about this flag.
+  }
   // We use this syntax so that the logic in AreStartupOptionsDifferent() that
   // decides whether the server needs killing is simpler. This is parsed by
   // the Java code where --noclient_debug and --client_debug=false are

src/main/cpp/startup_options.cc

@@ -64,7 +64,7 @@ void StartupOptions::OverrideOptionSourcesKey(const std::string &flag_name,
   option_sources_key_override_[flag_name] = new_name;
 }
 
-StartupOptions::StartupOptions(const string &product_name,
+StartupOptions::StartupOptions(const string& product_name,
                                bool lock_install_base)
     : product_name(product_name),
       lock_install_base(lock_install_base),
@@ -101,7 +101,8 @@ StartupOptions::StartupOptions(const string &product_name,
       cgroup_parent(),
       run_in_user_cgroup(false),
 #endif
-      windows_enable_symlinks(false) {
+      windows_enable_symlinks(false),
+      remote_repo_contents_cache(false) {
 #if defined(_WIN32) || defined(__CYGWIN__)
   string windows_unix_root = DetectBashAndExportBazelSh();
   if (!windows_unix_root.empty()) {
@@ -136,6 +137,8 @@ StartupOptions::StartupOptions(const string &product_name,
   RegisterNullaryStartupFlag("write_command_log", &write_command_log);
   RegisterNullaryStartupFlag("windows_enable_symlinks",
                              &windows_enable_symlinks);
+  RegisterNullaryStartupFlag("experimental_remote_repo_contents_cache",
+                             &remote_repo_contents_cache);
 #ifdef __linux__
   RegisterNullaryStartupFlag("experimental_run_in_user_cgroup",
                              &run_in_user_cgroup);

src/main/cpp/startup_options.h

@@ -294,6 +294,10 @@ class StartupOptions {
   // developer mode to be enabled.
   bool windows_enable_symlinks;
 
+  // Whether to use a remote cache to store the contents of reproducible
+  // external repositories.
+  bool remote_repo_contents_cache;
+
  protected:
   // Constructor for subclasses only so that site-specific extensions of this
   // class can override the product name. The product_name must be capitalized,

src/main/java/com/google/devtools/build/lib/BUILD

@@ -281,6 +281,16 @@ java_library(
     ],
 )
 
+java_library(
+    name = "runtime/remote_repo_contents_cache",
+    srcs = ["runtime/RemoteRepoContentsCache.java"],
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib/cmdline",
+        "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+    ],
+)
+
 java_library(
     name = "runtime",
     srcs = glob(
@@ -301,6 +311,7 @@ java_library(
             "runtime/MemoryPressureEvent.java",
             "runtime/MemoryPressureOptions.java",
             "runtime/MemoryPressureStatCollector.java",
+            "runtime/RemoteRepoContentsCache.java",
             "runtime/StarlarkOptionsParser.java",
             "runtime/TestSummaryOptions.java",
         ],
@@ -314,6 +325,7 @@ java_library(
         ":runtime/command_dispatcher",
         ":runtime/command_line_path_factory",
         ":runtime/memory_pressure",
+        ":runtime/remote_repo_contents_cache",
         ":runtime/test_summary_options",
         ":starlark_options_parser",
         "//src/main/java/com/google/devtools/build/lib/actions",

src/main/java/com/google/devtools/build/lib/actions/FileStateValue.java

@@ -153,6 +153,9 @@ private static FileStateValue createRegularFileStateValueFromPath(
       throws InconsistentFilesystemException {
     checkState(stat.isFile(), path);
 
+    if (stat instanceof FileStatusWithMetadata fileStatusWithMetadata) {
+      return new RegularFileStateValueWithMetadata(fileStatusWithMetadata.getMetadata());
+    }
     try {
       // If the digest will be injected, we can skip calling getFastDigest, but we need to store a
       // contents proxy because if the digest is injected but is not available from the filesystem,
@@ -386,6 +389,77 @@ public String prettyPrint() {
     }
   }
 
+  /**
+   * Implementation of {@link FileStateValue} for regular files when its metadata is backed by a
+   * {@link FileArtifactValue}.
+   */
+  public static final class RegularFileStateValueWithMetadata extends FileStateValue {
+    private final FileArtifactValue metadata;
+
+    @VisibleForTesting
+    public RegularFileStateValueWithMetadata(FileArtifactValue metadata) {
+      this.metadata = checkNotNull(metadata);
+    }
+
+    @Override
+    public FileStateType getType() {
+      return FileStateType.REGULAR_FILE;
+    }
+
+    @Override
+    public long getSize() {
+      return metadata.getSize();
+    }
+
+    @Override
+    @Nullable
+    public byte[] getDigest() {
+      return metadata.getDigest();
+    }
+
+    @Override
+    public FileContentsProxy getContentsProxy() {
+      return metadata.getContentsProxy();
+    }
+
+    public FileArtifactValue getMetadata() {
+      return metadata;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof RegularFileStateValueWithMetadata other)) {
+        return false;
+      }
+      return other.metadata.equals(this.metadata);
+    }
+
+    @Override
+    public int hashCode() {
+      return metadata.hashCode();
+    }
+
+    @Override
+    public byte[] getValueFingerprint() {
+      Fingerprint fp = new Fingerprint().addLong(getSize());
+      fp.addBytes(getDigest());
+      return fp.digestAndReset();
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this).add("metadata", metadata).toString();
+    }
+
+    @Override
+    public String prettyPrint() {
+      return String.format("regular file with size of %d and %s", getSize(), metadata);
+    }
+  }
+
   /** Implementation of {@link FileStateValue} for special files that exist. */
   @VisibleForTesting
   public static final class SpecialFileStateValue extends FileStateValue {

src/main/java/com/google/devtools/build/lib/analysis/BlazeDirectories.java

@@ -141,6 +141,7 @@ public Path getWorkspace() {
   }
 
   /** Returns working directory of the server. */
+  @Nullable
   public Path getWorkingDirectory() {
     return workspace;
   }

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -51,6 +51,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/rules:repository/local_repository_rule",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/new_local_repository_function",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/new_local_repository_rule",
+        "//src/main/java/com/google/devtools/build/lib:runtime/remote_repo_contents_cache",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:mutable_supplier",
         "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -99,8 +99,9 @@
 import com.google.devtools.build.lib.runtime.CommonCommandOptions;
 import com.google.devtools.build.lib.runtime.InfoItem;
 import com.google.devtools.build.lib.runtime.ProcessWrapper;
+import com.google.devtools.build.lib.runtime.RemoteRepoContentsCache;
 import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
-import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutorFactory;
+import com.google.devtools.build.lib.runtime.RepositoryRemoteHelpersFactory;
 import com.google.devtools.build.lib.runtime.ServerBuilder;
 import com.google.devtools.build.lib.runtime.WorkspaceBuilder;
 import com.google.devtools.build.lib.server.FailureDetails.ExternalRepository;
@@ -151,6 +152,7 @@ public class BazelRepositoryModule extends BlazeModule {
   private final ImmutableMap<String, RepositoryFunction> repositoryHandlers;
   private final AtomicBoolean isFetch = new AtomicBoolean(false);
   private final StarlarkRepositoryFunction starlarkRepositoryFunction;
+  private RepositoryDelegatorFunction repositoryDelegatorFunction;
   private final RepositoryCache repositoryCache = new RepositoryCache();
   private final MutableSupplier<Map<String, String>> repoEnvironmentSupplier =
       new MutableSupplier<>();
@@ -247,9 +249,10 @@ public void workspaceInit(
           SkyframeExecutorRepositoryHelpersHolder.create(
               new RepositoryDirectoryDirtinessChecker()));
     }
+    builder.setRepoContentsCachePathSupplier(repositoryCache.getRepoContentsCache()::getPath);
 
     // Create the repository function everything flows through.
-    RepositoryDelegatorFunction repositoryDelegatorFunction =
+    repositoryDelegatorFunction =
         new RepositoryDelegatorFunction(
             repositoryHandlers,
             starlarkRepositoryFunction,
@@ -397,19 +400,19 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
       if (repoContentsCachePath != null
           && env.getWorkspace() != null
           && repoContentsCachePath.startsWith(env.getWorkspace())) {
-        // Having the repo contents cache inside the workspace is very dangerous. During the
-        // lifetime of a Bazel invocation, we treat files inside the workspace as immutable. This
-        // can cause mysterious failures if we write files inside the workspace during the
+        // Having the repo contents cache inside the main repo is very dangerous. During the
+        // lifetime of a Bazel invocation, we treat files inside the main repo as immutable. This
+        // can cause mysterious failures if we write files inside the main repo during the
         // invocation, as is often the case with the repo contents cache.
         // TODO: wyv@ - This is a crude check that disables some use cases (such as when the output
         //   base itself is inside the main repo). Investigate a better check.
         repositoryCache.getRepoContentsCache().setPath(null);
         throw new AbruptExitException(
             detailedExitCode(
                 """
-                The repo contents cache [%s] is inside the workspace [%s]. This can cause spurious \
+                The repo contents cache [%s] is inside the main repo [%s]. This can cause spurious \
                 failures. Disable the repo contents cache with `--repo_contents_cache=`, or \
-                specify `--repo_contents_cache=<path outside the workspace>`.
+                specify `--repo_contents_cache=<path outside the main repo>`.
                 """
                     .formatted(repoContentsCachePath, env.getWorkspace()),
                 Code.BAD_REPO_CONTENTS_CACHE));
@@ -425,14 +428,12 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
                   "could not acquire lock on repo contents cache", Code.BAD_REPO_CONTENTS_CACHE),
               e);
         }
-        if (!repoOptions.repoContentsCacheGcMaxAge.isZero()) {
-          env.addIdleTask(
-              repositoryCache
-                  .getRepoContentsCache()
-                  .createGcIdleTask(
-                      repoOptions.repoContentsCacheGcMaxAge,
-                      repoOptions.repoContentsCacheGcIdleDelay));
-        }
+        env.addIdleTask(
+            repositoryCache
+                .getRepoContentsCache()
+                .createGcIdleTask(
+                    repoOptions.repoContentsCacheGcMaxAge,
+                    repoOptions.repoContentsCacheGcIdleDelay));
       }
 
       try {
@@ -657,13 +658,16 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
             Optional.of(RootedPath.toRootedPath(Root.absoluteRoot(filesystem), resolvedFile));
       }
 
-      RepositoryRemoteExecutorFactory remoteExecutorFactory =
-          env.getRuntime().getRepositoryRemoteExecutorFactory();
+      RepositoryRemoteHelpersFactory repositoryRemoteHelpersFactory =
+          env.getRuntime().getRepositoryHelpersFactory();
       RepositoryRemoteExecutor remoteExecutor = null;
-      if (remoteExecutorFactory != null) {
-        remoteExecutor = remoteExecutorFactory.create();
+      RemoteRepoContentsCache remoteRepoContentsCache = null;
+      if (repositoryRemoteHelpersFactory != null) {
+        remoteExecutor = repositoryRemoteHelpersFactory.createExecutor();
+        remoteRepoContentsCache = repositoryRemoteHelpersFactory.createRepoContentsCache();
       }
       starlarkRepositoryFunction.setRepositoryRemoteExecutor(remoteExecutor);
+      repositoryDelegatorFunction.setRemoteRepoContentsCache(remoteRepoContentsCache);
       singleExtensionEvalFunction.setRepositoryRemoteExecutor(remoteExecutor);
 
       clock = env.getClock();
@@ -716,10 +720,18 @@ private String getAbsolutePath(String path, CommandEnvironment env) {
    */
   @Nullable
   private Path toPath(PathFragment path, CommandEnvironment env) {
-    if (path.isEmpty() || env.getBlazeWorkspace().getWorkspace() == null) {
+    if (path.isEmpty() || env.getDirectories().getWorkspace() == null) {
       return null;
     }
-    return env.getBlazeWorkspace().getWorkspace().getRelative(path);
+    // It is important to use getWorkspace() here, not getWorkingDirectory(). Both Paths have the
+    // same underlying PathFragment, but may differ in their FileSystem if the remote repo contents
+    // cache is in use. getWorkspace() uses the same FileSystem as everything other than the
+    // workspace directory, while getWorkingDirectory() uses the workspace directory's FileSystem.
+    // Even though the users of the returned Path may end up writing to it, they are not expected to
+    // update source files within the workspace. Thus, the correct FileSystem is the one from
+    // getWorkspace(), which e.g. allows moves from the external directory under the output base to
+    // the local repo contents cache without crossing FileSystems.
+    return env.getDirectories().getWorkspace().getRelative(path);
   }
 
   @Override

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java

@@ -47,7 +47,7 @@ public abstract class BazelLockFileValue implements SkyValue {
   // https://cs.opensource.google/bazel/bazel/+/release-7.3.0:src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java;l=120-127;drc=5f5355b75c7c93fba1e15f6658f308953f4baf51
   // While this hack exists on 7.x, lockfile version increments should be done 2 at a time (i.e.
   // keep this number even).
-  public static final int LOCK_FILE_VERSION = 24;
+  public static final int LOCK_FILE_VERSION = 26;
 
   /** A valid empty lockfile. */
   public static final BazelLockFileValue EMPTY_LOCKFILE = builder().build();

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/DelegateTypeAdapterFactory.java

@@ -19,6 +19,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import com.google.gson.Gson;
 import com.google.gson.TypeAdapter;
 import com.google.gson.TypeAdapterFactory;
@@ -34,6 +35,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.function.Function;
 import javax.annotation.Nullable;
 import net.starlark.java.eval.Dict;
@@ -71,6 +74,14 @@ private DelegateTypeAdapterFactory(
           raw -> new LinkedHashMap<>((Map<?, ?>) raw),
           delegate -> ImmutableMap.copyOf((Map<?, ?>) delegate));
 
+  public static final TypeAdapterFactory IMMUTABLE_SORTED_MAP =
+      new DelegateTypeAdapterFactory<>(
+          ImmutableSortedMap.class,
+          SortedMap.class,
+          TreeMap.class,
+          raw -> new TreeMap<>((SortedMap<?, ?>) raw),
+          delegate -> ImmutableSortedMap.copyOf((SortedMap<?, ?>) delegate));
+
   public static final TypeAdapterFactory IMMUTABLE_BIMAP =
       new DelegateTypeAdapterFactory<>(
           ImmutableBiMap.class,