Skip to content

[8.8.0] Preserve order of recorded inputs #29068

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Wyverald merged 7 commits into from
Jun 22, 2026
Merged

[8.8.0] Preserve order of recorded inputs #29068

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4674c7d
[8.8.0] Fix NPE with remote repo contents cache
fmeum Nov 18, 2025
4e4919b
[8.8.0] Make remote repo contents cache less spammy
fmeum Nov 19, 2025
7545af3
Fix materialization edge cases in the remote repo contents cache
fmeum Nov 24, 2025
96f4080
[8.7.0] Fix `RemoteExternalOverlayFileSystem#resolveSymbolicLinks`
fmeum Nov 28, 2025
d9331e6
[8.7.0] Allow more general exceptions in `getConfiguration`
fmeum Dec 17, 2025
ac8b751
Preserve order of recorded inputs
fmeum Dec 15, 2025
a2591d5
Apply suggestion from @fmeum
fmeum Jun 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public abstract class BazelLockFileValue implements SkyValue {
// https://cs.opensource.google/bazel/bazel/+/release-7.3.0:src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java;l=120-127;drc=5f5355b75c7c93fba1e15f6658f308953f4baf51
// While this hack exists on 7.x, lockfile version increments should be done 2 at a time (i.e.
// keep this number even).
public static final int LOCK_FILE_VERSION = 24;
public static final int LOCK_FILE_VERSION = 26;

/** A valid empty lockfile. */
public static final BazelLockFileValue EMPTY_LOCKFILE = builder().build();
Expand Down
119 changes: 19 additions & 100 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@
import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_MAP;
import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_SET;
import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_SORTED_MAP;
import static com.google.devtools.build.lib.rules.repository.RepoRecordedInput.NeverUpToDateRepoRecordedInput.PARSE_FAILURE;
import static com.google.devtools.build.lib.util.StringEncoding.internalToUnicode;
import static com.google.devtools.build.lib.util.StringEncoding.unicodeToInternal;

import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableTable;
import com.google.common.collect.Table;
import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException;
import com.google.devtools.build.lib.bazel.repository.cache.DownloadCache;
import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
Expand Down Expand Up @@ -293,113 +294,34 @@ public Optional<T> read(JsonReader jsonReader) throws IOException {
}
}

/**
* Converts Guava tables into a JSON array of 3-tuples (one per cell). Each 3-tuple is a JSON
* array itself (rowKey, columnKey, value). For example, a JSON snippet could be: {@code [
* ["row1", "col1", "value1"], ["row2", "col2", "value2"], ... ]}
*/
public static final TypeAdapterFactory IMMUTABLE_TABLE =
new TypeAdapterFactory() {
@Nullable
@Override
@SuppressWarnings("unchecked")
public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {
if (typeToken.getRawType() != ImmutableTable.class) {
return null;
}
Type type = typeToken.getType();
if (!(type instanceof ParameterizedType)) {
return null;
}
Type[] typeArgs = ((ParameterizedType) typeToken.getType()).getActualTypeArguments();
if (typeArgs.length != 3) {
return null;
}
var rowTypeAdapter = (TypeAdapter<Object>) gson.getAdapter(TypeToken.get(typeArgs[0]));
var colTypeAdapter = (TypeAdapter<Object>) gson.getAdapter(TypeToken.get(typeArgs[1]));
var valTypeAdapter = (TypeAdapter<Object>) gson.getAdapter(TypeToken.get(typeArgs[2]));
if (rowTypeAdapter == null || colTypeAdapter == null || valTypeAdapter == null) {
return null;
}
return (TypeAdapter<T>)
new TypeAdapter<ImmutableTable<Object, Object, Object>>() {
@Override
public void write(JsonWriter jsonWriter, ImmutableTable<Object, Object, Object> t)
throws IOException {
jsonWriter.beginArray();
for (Table.Cell<Object, Object, Object> cell : t.cellSet()) {
jsonWriter.beginArray();
rowTypeAdapter.write(jsonWriter, cell.getRowKey());
colTypeAdapter.write(jsonWriter, cell.getColumnKey());
valTypeAdapter.write(jsonWriter, cell.getValue());
jsonWriter.endArray();
}
jsonWriter.endArray();
}

@Override
public ImmutableTable<Object, Object, Object> read(JsonReader jsonReader)
throws IOException {
var builder = ImmutableTable.builder();
jsonReader.beginArray();
while (jsonReader.peek() != JsonToken.END_ARRAY) {
jsonReader.beginArray();
builder.put(
rowTypeAdapter.read(jsonReader),
colTypeAdapter.read(jsonReader),
valTypeAdapter.read(jsonReader));
jsonReader.endArray();
}
jsonReader.endArray();
return builder.buildOrThrow();
}
};
}
};

private static final TypeAdapter<RepoRecordedInput.File> REPO_RECORDED_INPUT_FILE_TYPE_ADAPTER =
// This is needed because Bazel uses a custom String encoding internally, see StringEncoding for
// details.
public static final TypeAdapter<String> STRING_ADAPTER =
new TypeAdapter<>() {
@Override
public void write(JsonWriter jsonWriter, RepoRecordedInput.File value) throws IOException {
jsonWriter.value(value.toStringInternal());
public void write(JsonWriter jsonWriter, String s) throws IOException {
jsonWriter.value(internalToUnicode(s));
}

@Override
public RepoRecordedInput.File read(JsonReader jsonReader) throws IOException {
return (RepoRecordedInput.File)
RepoRecordedInput.File.PARSER.parse(jsonReader.nextString());
public String read(JsonReader jsonReader) throws IOException {
return unicodeToInternal(jsonReader.nextString());
}
};

private static final TypeAdapter<RepoRecordedInput.Dirents>
REPO_RECORDED_INPUT_DIRENTS_TYPE_ADAPTER =
new TypeAdapter<>() {
@Override
public void write(JsonWriter jsonWriter, RepoRecordedInput.Dirents value)
throws IOException {
jsonWriter.value(value.toStringInternal());
}

@Override
public RepoRecordedInput.Dirents read(JsonReader jsonReader) throws IOException {
return (RepoRecordedInput.Dirents)
RepoRecordedInput.Dirents.PARSER.parse(jsonReader.nextString());
}
};

private static final TypeAdapter<RepoRecordedInput.EnvVar>
REPO_RECORDED_INPUT_ENV_VAR_TYPE_ADAPTER =
private static final TypeAdapter<RepoRecordedInput.WithValue>
REPO_RECORDED_INPUT_WITH_VALUE_TYPE_ADAPTER =
new TypeAdapter<>() {
@Override
public void write(JsonWriter jsonWriter, RepoRecordedInput.EnvVar value)
public void write(JsonWriter jsonWriter, RepoRecordedInput.WithValue value)
throws IOException {
jsonWriter.value(value.toStringInternal());
jsonWriter.value(internalToUnicode(value.toString()));
}

@Override
public RepoRecordedInput.EnvVar read(JsonReader jsonReader) throws IOException {
return (RepoRecordedInput.EnvVar)
RepoRecordedInput.EnvVar.PARSER.parse(jsonReader.nextString());
public RepoRecordedInput.WithValue read(JsonReader jsonReader) throws IOException {
return RepoRecordedInput.WithValue.parse(unicodeToInternal(jsonReader.nextString()))
.orElseGet(() -> new RepoRecordedInput.WithValue(PARSE_FAILURE, ""));
}
};

Expand Down Expand Up @@ -475,7 +397,7 @@ private static GsonBuilder newGsonBuilder() {
.registerTypeAdapterFactory(IMMUTABLE_BIMAP)
.registerTypeAdapterFactory(IMMUTABLE_SET)
.registerTypeAdapterFactory(OPTIONAL)
.registerTypeAdapterFactory(IMMUTABLE_TABLE)
.registerTypeAdapter(String.class, STRING_ADAPTER)
.registerTypeAdapter(Label.class, LABEL_TYPE_ADAPTER)
.registerTypeAdapter(RepoRuleId.class, REPO_RULE_ID_TYPE_ADAPTER)
.registerTypeAdapter(RepositoryName.class, REPOSITORY_NAME_TYPE_ADAPTER)
Expand All @@ -487,11 +409,8 @@ private static GsonBuilder newGsonBuilder() {
.registerTypeAdapter(ModuleExtensionId.IsolationKey.class, ISOLATION_KEY_TYPE_ADAPTER)
.registerTypeAdapter(AttributeValues.class, new AttributeValuesAdapter())
.registerTypeAdapter(byte[].class, BYTE_ARRAY_TYPE_ADAPTER)
.registerTypeAdapter(RepoRecordedInput.File.class, REPO_RECORDED_INPUT_FILE_TYPE_ADAPTER)
.registerTypeAdapter(
RepoRecordedInput.Dirents.class, REPO_RECORDED_INPUT_DIRENTS_TYPE_ADAPTER)
.registerTypeAdapter(
RepoRecordedInput.EnvVar.class, REPO_RECORDED_INPUT_ENV_VAR_TYPE_ADAPTER);
RepoRecordedInput.WithValue.class, REPO_RECORDED_INPUT_WITH_VALUE_TYPE_ADAPTER);
}

private GsonTypeAdapterUtil() {}
Expand Down
15 changes: 2 additions & 13 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/InnateRunnableExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableTable;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
Expand All @@ -39,7 +37,6 @@
import com.google.devtools.build.lib.skyframe.BzlLoadValue;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import java.util.Map.Entry;
import java.util.Optional;
import javax.annotation.Nullable;
import net.starlark.java.eval.Dict;
import net.starlark.java.eval.EvalException;
Expand Down Expand Up @@ -150,11 +147,6 @@ public byte[] getBzlTransitiveDigest() {
return loadedBzl.getTransitiveDigest();
}

@Override
public ImmutableMap<String, Optional<String>> getStaticEnvVars() {
return ImmutableMap.of();
}

@Override
public RunModuleExtensionResult run(
Environment env,
Expand Down Expand Up @@ -237,11 +229,8 @@ public RunModuleExtensionResult run(
attributesValue));
}
return new RunModuleExtensionResult(
ImmutableSortedMap.of(),
ImmutableSortedMap.of(),
ImmutableSortedMap.of(),
ImmutableList.of(),
generatedRepoSpecs.buildOrThrow(),
ModuleExtensionMetadata.REPRODUCIBLE,
ImmutableTable.of());
ModuleExtensionMetadata.REPRODUCIBLE);
}
}
28 changes: 4 additions & 24 deletions src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@
package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableTable;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.rules.repository.RepoRecordedInput;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
Expand All @@ -35,8 +33,7 @@ public abstract class LockFileModuleExtension {

public static Builder builder() {
return new AutoValue_LockFileModuleExtension.Builder()
.setModuleExtensionMetadata(Optional.empty())
.setRecordedRepoMappingEntries(ImmutableTable.of());
.setModuleExtensionMetadata(Optional.empty());
}

@SuppressWarnings("mutable")
Expand All @@ -45,19 +42,12 @@ public static Builder builder() {
@SuppressWarnings("mutable")
public abstract byte[] getUsagesDigest();

public abstract ImmutableSortedMap<RepoRecordedInput.File, String> getRecordedFileInputs();

public abstract ImmutableSortedMap<RepoRecordedInput.Dirents, String> getRecordedDirentsInputs();

public abstract ImmutableSortedMap<RepoRecordedInput.EnvVar, Optional<String>> getEnvVariables();
public abstract ImmutableList<RepoRecordedInput.WithValue> getRecordedInputs();

public abstract ImmutableMap<String, RepoSpec> getGeneratedRepoSpecs();

public abstract Optional<LockfileModuleExtensionMetadata> getModuleExtensionMetadata();

public abstract ImmutableTable<RepositoryName, String, RepositoryName>
getRecordedRepoMappingEntries();

public boolean isReproducible() {
return getModuleExtensionMetadata()
.map(LockfileModuleExtensionMetadata::getReproducible)
Expand All @@ -72,23 +62,13 @@ public abstract static class Builder {

public abstract Builder setUsagesDigest(byte[] digest);

public abstract Builder setRecordedFileInputs(
ImmutableSortedMap<RepoRecordedInput.File, String> value);

public abstract Builder setRecordedDirentsInputs(
ImmutableSortedMap<RepoRecordedInput.Dirents, String> value);

public abstract Builder setEnvVariables(
ImmutableSortedMap<RepoRecordedInput.EnvVar, Optional<String>> value);
public abstract Builder setRecordedInputs(ImmutableList<RepoRecordedInput.WithValue> value);

public abstract Builder setGeneratedRepoSpecs(ImmutableMap<String, RepoSpec> value);

public abstract Builder setModuleExtensionMetadata(
Optional<LockfileModuleExtensionMetadata> value);

public abstract Builder setRecordedRepoMappingEntries(
ImmutableTable<RepositoryName, String, RepositoryName> value);

public abstract LockFileModuleExtension build();
}

Expand Down
12 changes: 11 additions & 1 deletion src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,19 @@
package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableTable;
import com.google.devtools.build.docgen.annot.DocCategory;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkBaseExternalContext;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.rules.repository.RepoRecordedInput;
import com.google.devtools.build.lib.runtime.ProcessWrapper;
import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;
import net.starlark.java.annot.Param;
import net.starlark.java.annot.ParamType;
Expand Down Expand Up @@ -65,7 +69,9 @@ protected ModuleExtensionContext(
ModuleExtensionId extensionId,
StarlarkList<StarlarkBazelModule> modules,
Facts facts,
boolean rootModuleHasNonDevDependency) {
boolean rootModuleHasNonDevDependency,
ImmutableMap<String, Optional<String>> staticEnvVars,
ImmutableTable<RepositoryName, String, RepositoryName> staticRepoMappingEntries) {
super(
workingDirectory,
directories,
Expand All @@ -83,6 +89,10 @@ protected ModuleExtensionContext(
this.modules = modules;
this.facts = facts;
this.rootModuleHasNonDevDependency = rootModuleHasNonDevDependency;
// Record inputs to the extension that are known prior to evaluation.
RepoRecordedInput.EnvVar.wrap(staticEnvVars)
.forEach((input, value) -> recordInput(input, value.orElse(null)));
repoMappingRecorder.record(staticRepoMappingEntries);
}

@Override
Expand Down
Loading
Loading