[8.8.0] Add a remote repo contents cache

src/main/cpp/blaze.cc

@@ -548,6 +548,11 @@ static vector<string> GetServerExeArgs(const blaze_util::Path &jvm_path,
   } else {
     result.push_back("--nowindows_enable_symlinks");
   }
+  if (startup_options.remote_repo_contents_cache) {
+    result.push_back("--experimental_remote_repo_contents_cache");
+    // Don't set the flag to false if it's not set - non-OSS Blaze does not know
+    // about this flag.
+  }
   // We use this syntax so that the logic in AreStartupOptionsDifferent() that
   // decides whether the server needs killing is simpler. This is parsed by
   // the Java code where --noclient_debug and --client_debug=false are

src/main/cpp/startup_options.cc

@@ -64,7 +64,7 @@ void StartupOptions::OverrideOptionSourcesKey(const std::string &flag_name,
   option_sources_key_override_[flag_name] = new_name;
 }
 
-StartupOptions::StartupOptions(const string &product_name,
+StartupOptions::StartupOptions(const string& product_name,
                                bool lock_install_base)
     : product_name(product_name),
       lock_install_base(lock_install_base),
@@ -101,7 +101,8 @@ StartupOptions::StartupOptions(const string &product_name,
       cgroup_parent(),
       run_in_user_cgroup(false),
 #endif
-      windows_enable_symlinks(false) {
+      windows_enable_symlinks(false),
+      remote_repo_contents_cache(false) {
 #if defined(_WIN32) || defined(__CYGWIN__)
   string windows_unix_root = DetectBashAndExportBazelSh();
   if (!windows_unix_root.empty()) {
@@ -136,6 +137,8 @@ StartupOptions::StartupOptions(const string &product_name,
   RegisterNullaryStartupFlag("write_command_log", &write_command_log);
   RegisterNullaryStartupFlag("windows_enable_symlinks",
                              &windows_enable_symlinks);
+  RegisterNullaryStartupFlag("experimental_remote_repo_contents_cache",
+                             &remote_repo_contents_cache);
 #ifdef __linux__
   RegisterNullaryStartupFlag("experimental_run_in_user_cgroup",
                              &run_in_user_cgroup);

src/main/cpp/startup_options.h

@@ -294,6 +294,10 @@ class StartupOptions {
   // developer mode to be enabled.
   bool windows_enable_symlinks;
 
+  // Whether to use a remote cache to store the contents of reproducible
+  // external repositories.
+  bool remote_repo_contents_cache;
+
  protected:
   // Constructor for subclasses only so that site-specific extensions of this
   // class can override the product name. The product_name must be capitalized,

src/main/java/com/google/devtools/build/lib/BUILD

@@ -281,6 +281,16 @@ java_library(
     ],
 )
 
+java_library(
+    name = "runtime/remote_repo_contents_cache",
+    srcs = ["runtime/RemoteRepoContentsCache.java"],
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib/cmdline",
+        "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+    ],
+)
+
 java_library(
     name = "runtime",
     srcs = glob(
@@ -301,6 +311,7 @@ java_library(
             "runtime/MemoryPressureEvent.java",
             "runtime/MemoryPressureOptions.java",
             "runtime/MemoryPressureStatCollector.java",
+            "runtime/RemoteRepoContentsCache.java",
             "runtime/StarlarkOptionsParser.java",
             "runtime/TestSummaryOptions.java",
         ],
@@ -314,6 +325,7 @@ java_library(
         ":runtime/command_dispatcher",
         ":runtime/command_line_path_factory",
         ":runtime/memory_pressure",
+        ":runtime/remote_repo_contents_cache",
         ":runtime/test_summary_options",
         ":starlark_options_parser",
         "//src/main/java/com/google/devtools/build/lib/actions",

src/main/java/com/google/devtools/build/lib/actions/FileStateValue.java

@@ -153,6 +153,9 @@ private static FileStateValue createRegularFileStateValueFromPath(
       throws InconsistentFilesystemException {
     checkState(stat.isFile(), path);
 
+    if (stat instanceof FileStatusWithMetadata fileStatusWithMetadata) {
+      return new RegularFileStateValueWithMetadata(fileStatusWithMetadata.getMetadata());
+    }
     try {
       // If the digest will be injected, we can skip calling getFastDigest, but we need to store a
       // contents proxy because if the digest is injected but is not available from the filesystem,
@@ -386,6 +389,77 @@ public String prettyPrint() {
     }
   }
 
+  /**
+   * Implementation of {@link FileStateValue} for regular files when its metadata is backed by a
+   * {@link FileArtifactValue}.
+   */
+  public static final class RegularFileStateValueWithMetadata extends FileStateValue {
+    private final FileArtifactValue metadata;
+
+    @VisibleForTesting
+    public RegularFileStateValueWithMetadata(FileArtifactValue metadata) {
+      this.metadata = checkNotNull(metadata);
+    }
+
+    @Override
+    public FileStateType getType() {
+      return FileStateType.REGULAR_FILE;
+    }
+
+    @Override
+    public long getSize() {
+      return metadata.getSize();
+    }
+
+    @Override
+    @Nullable
+    public byte[] getDigest() {
+      return metadata.getDigest();
+    }
+
+    @Override
+    public FileContentsProxy getContentsProxy() {
+      return metadata.getContentsProxy();
+    }
+
+    public FileArtifactValue getMetadata() {
+      return metadata;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof RegularFileStateValueWithMetadata other)) {
+        return false;
+      }
+      return other.metadata.equals(this.metadata);
+    }
+
+    @Override
+    public int hashCode() {
+      return metadata.hashCode();
+    }
+
+    @Override
+    public byte[] getValueFingerprint() {
+      Fingerprint fp = new Fingerprint().addLong(getSize());
+      fp.addBytes(getDigest());
+      return fp.digestAndReset();
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this).add("metadata", metadata).toString();
+    }
+
+    @Override
+    public String prettyPrint() {
+      return String.format("regular file with size of %d and %s", getSize(), metadata);
+    }
+  }
+
   /** Implementation of {@link FileStateValue} for special files that exist. */
   @VisibleForTesting
   public static final class SpecialFileStateValue extends FileStateValue {

src/main/java/com/google/devtools/build/lib/analysis/ConfiguredRuleClassProvider.java

@@ -126,12 +126,12 @@ public BundledFileSystem() {
     // See b/226379109 for details.
 
     @Override
-    protected synchronized byte[] getFastDigest(PathFragment path) {
+    public synchronized byte[] getFastDigest(PathFragment path) {
       return getDigest(path);
     }
 
     @Override
-    protected synchronized byte[] getDigest(PathFragment path) {
+    public synchronized byte[] getDigest(PathFragment path) {
       return getDigestFunction()
           .getHashFunction()
           .hashBytes(StringUnsafe.getInternalStringBytes(path.getPathString()))

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -51,6 +51,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/rules:repository/local_repository_rule",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/new_local_repository_function",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/new_local_repository_rule",
+        "//src/main/java/com/google/devtools/build/lib:runtime/remote_repo_contents_cache",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:mutable_supplier",
         "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -99,6 +99,7 @@
 import com.google.devtools.build.lib.runtime.CommonCommandOptions;
 import com.google.devtools.build.lib.runtime.InfoItem;
 import com.google.devtools.build.lib.runtime.ProcessWrapper;
+import com.google.devtools.build.lib.runtime.RemoteRepoContentsCache;
 import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
 import com.google.devtools.build.lib.runtime.RepositoryRemoteHelpersFactory;
 import com.google.devtools.build.lib.runtime.ServerBuilder;
@@ -151,6 +152,7 @@ public class BazelRepositoryModule extends BlazeModule {
   private final ImmutableMap<String, RepositoryFunction> repositoryHandlers;
   private final AtomicBoolean isFetch = new AtomicBoolean(false);
   private final StarlarkRepositoryFunction starlarkRepositoryFunction;
+  private RepositoryDelegatorFunction repositoryDelegatorFunction;
   private final RepositoryCache repositoryCache = new RepositoryCache();
   private final MutableSupplier<Map<String, String>> repoEnvironmentSupplier =
       new MutableSupplier<>();
@@ -249,7 +251,7 @@ public void workspaceInit(
     }
 
     // Create the repository function everything flows through.
-    RepositoryDelegatorFunction repositoryDelegatorFunction =
+    repositoryDelegatorFunction =
         new RepositoryDelegatorFunction(
             repositoryHandlers,
             starlarkRepositoryFunction,
@@ -658,10 +660,13 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
       RepositoryRemoteHelpersFactory repositoryRemoteHelpersFactory =
           env.getRuntime().getRepositoryHelpersFactory();
       RepositoryRemoteExecutor remoteExecutor = null;
+      RemoteRepoContentsCache remoteRepoContentsCache = null;
       if (repositoryRemoteHelpersFactory != null) {
         remoteExecutor = repositoryRemoteHelpersFactory.createExecutor();
+        remoteRepoContentsCache = repositoryRemoteHelpersFactory.createRepoContentsCache();
       }
       starlarkRepositoryFunction.setRepositoryRemoteExecutor(remoteExecutor);
+      repositoryDelegatorFunction.setRemoteRepoContentsCache(remoteRepoContentsCache);
       singleExtensionEvalFunction.setRepositoryRemoteExecutor(remoteExecutor);
 
       clock = env.getClock();

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD

@@ -34,6 +34,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/packages/semantics",
         "//src/main/java/com/google/devtools/build/lib/pkgcache",
         "//src/main/java/com/google/devtools/build/lib/profiler",
+        "//src/main/java/com/google/devtools/build/lib/remote",
         "//src/main/java/com/google/devtools/build/lib/repository:repository_events",
         "//src/main/java/com/google/devtools/build/lib/repository:request_repository_information_event",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repo_recorded_input",

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java

@@ -49,6 +49,7 @@
 import com.google.devtools.build.lib.profiler.Profiler;
 import com.google.devtools.build.lib.profiler.ProfilerTask;
 import com.google.devtools.build.lib.profiler.SilentCloseable;
+import com.google.devtools.build.lib.remote.RemoteExternalOverlayFileSystem;
 import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException;
 import com.google.devtools.build.lib.rules.repository.RepoRecordedInput;
 import com.google.devtools.build.lib.rules.repository.RepoRecordedInput.Dirents;
@@ -2315,6 +2316,19 @@ private StarlarkPath findCommandOnPath(String program) throws IOException {
   // Resolve the label given by value into a file path.
   protected StarlarkPath getPathFromLabel(Label label) throws EvalException, InterruptedException {
     RootedPath rootedPath = RepositoryFunction.getRootedPathFromLabel(label, env);
+    if (rootedPath == null) {
+      throw new NeedsSkyframeRestartException();
+    }
+    if (!label.getRepository().isMain()
+        && directories.getOutputBase().getFileSystem()
+            instanceof RemoteExternalOverlayFileSystem remoteFs) {
+      try {
+        remoteFs.ensureMaterialized(label.getRepository(), env.getListener());
+      } catch (IOException e) {
+        throw Starlark.errorf(
+            "Failed to materialize remote repo %s: %s", label.getRepository(), e.getMessage());
+      }
+    }
     StarlarkPath starlarkPath = new StarlarkPath(this, rootedPath.asPath());
     try {
       maybeWatch(