Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -169,10 +169,20 @@ public static String deltaChange(
switch (type) {
case ADD -> new DeltaSingleAction(action, null, null);
case REMOVE -> new DeltaSingleAction(null, action, null);
case CDC -> new DeltaSingleAction(null, null, action);
case CDC -> new DeltaSingleAction(null, null, withDataChangeFalse(action));
});
}

/**
* Fixes a cdc action's dataChange at false, which the protocol requires of cdc actions and Delta
* readers need on the wire, whatever the commit wrote. The action's schema must carry the slot,
* which the reader's cdc schema adds for this override.
*/
private static Row withDataChangeFalse(Row action) {
int ordinal = action.getSchema().indexOf("dataChange");
return ordinal < 0 ? action : new DelegateRow(action, Map.of(ordinal, false));
}

private static String deltaFile(
String id,
String deletionVectorFileId,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import io.delta.kernel.internal.util.FileNames;
import io.delta.kernel.internal.util.Utils;
import io.delta.kernel.internal.util.VectorUtils;
import io.delta.kernel.types.BooleanType;
import io.delta.kernel.types.LongType;
import io.delta.kernel.types.MapType;
import io.delta.kernel.types.StringType;
Expand Down Expand Up @@ -79,6 +80,11 @@ public class DeltaTableQueryReader {
private static final Duration URL_TTL = Duration.ofHours(1);
private static final ObjectMapper JSON = new ObjectMapper();

// Kernel's cdc schema omits dataChange; the slot exists only so TableActions can override it
// to false, which Delta readers require on a cdc action.
private static final StructType CDC_SCHEMA =
DeltaAction.CDC.schema.add("dataChange", BooleanType.BOOLEAN);

// Commit files are read directly because Kernel 4.0's getChanges requires commitInfo fields that
// older Delta writers omit; only the commitInfo fields used here are read.
private static final StructType CHANGE_SCHEMA =
Expand All @@ -87,7 +93,7 @@ public class DeltaTableQueryReader {
.add(DeltaAction.METADATA.colName, DeltaAction.METADATA.schema)
.add(DeltaAction.ADD.colName, DeltaAction.ADD.schema)
.add(DeltaAction.REMOVE.colName, DeltaAction.REMOVE.schema)
.add(DeltaAction.CDC.colName, DeltaAction.CDC.schema)
.add(DeltaAction.CDC.colName, CDC_SCHEMA)
.add(
DeltaAction.COMMITINFO.colName,
new StructType()
Expand Down Expand Up @@ -883,7 +889,8 @@ yield new ChangeFile(
null,
remove.getDeletionVector());
}
// cdc files are always data changes and carry no stats or deletion vector.
// A cdc file is always emitted, whatever the log's dataChange says: TableActions fixes the
// action's own dataChange at false. A cdc action carries no stats or deletion vector.
case CDC -> {
StructType schema = action.getSchema();
yield new ChangeFile(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,50 @@ void changeDataFeedPrefersCdcFilesAndSkipsNoOpMerges() throws IOException {

List<String> delta = changeDataFeed(1L, null, 2L, null, "responseformat=delta", false);
assertEquals(List.of("cdc c 1", "remove b 2"), describe(delta.subList(2, delta.size())));
// Delta readers require dataChange on a cdc action, which the protocol fixes at false.
JsonNode cdc =
parse(delta.get(2)).path("file").path("deltaSingleAction").path("cdc");
assertTrue(cdc.path("dataChange").isBoolean(), delta.get(2));
assertFalse(cdc.path("dataChange").asBoolean(), delta.get(2));
// The parquet cdc line carries no dataChange.
String parquet = changeDataFeed(1L, null, 2L, null, null, false).get(2);
assertTrue(parquet.startsWith("{\"cdf\":"), parquet);
assertFalse(parquet.contains("dataChange"), parquet);
// Add and remove actions keep the dataChange their commit wrote.
List<String> deltaChanges = changeDataFeed(0L, null, 2L, null, "responseformat=delta", false);
assertEquals(
List.of("add a 0", "add b 0", "cdc c 1", "remove b 2"),
describe(deltaChanges.subList(2, deltaChanges.size())));
assertTrue(
parse(deltaChanges.get(2)).path("file").path("deltaSingleAction").path("add")
.path("dataChange")
.asBoolean(),
deltaChanges.get(2));
assertTrue(
parse(deltaChanges.get(5)).path("file").path("deltaSingleAction").path("remove")
.path("dataChange")
.asBoolean(),
deltaChanges.get(5));
}

@Test
void changeDataFeedWritesCdcDataChangeFalseWhateverTheCommitWrote() throws IOException {
writeChangeDataFeed();
// The cdc action of commit 1 wrote dataChange false; write one with true instead.
Files.writeString(
dir.resolve("_delta_log/00000000000000000001.json"),
String.join(
"\n",
remove("a", "2021-01-01", true),
add("d", "2021-01-01", 1, 10),
"{\"cdc\":{\"path\":\"_change_data/date=2021-01-01/c.parquet\","
+ "\"partitionValues\":{\"date\":\"2021-01-01\"},\"size\":1,"
+ "\"dataChange\":true}}")
+ "\n");
List<String> delta = changeDataFeed(1L, null, 1L, null, "responseformat=delta", false);
assertEquals(List.of("cdc c 1"), describe(delta.subList(2, delta.size())));
// The override wins: a cdc action's dataChange is false whatever the commit wrote.
assertFalse(parse(delta.get(2)).path("cdc").path("dataChange").asBoolean(), delta.get(2));
}

@Test
Expand Down
Loading