diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/TableActions.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/TableActions.java index fa533587c..a66be2fe6 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/TableActions.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/TableActions.java @@ -169,10 +169,20 @@ public static String deltaChange( switch (type) { case ADD -> new DeltaSingleAction(action, null, null); case REMOVE -> new DeltaSingleAction(null, action, null); - case CDC -> new DeltaSingleAction(null, null, action); + case CDC -> new DeltaSingleAction(null, null, withDataChangeFalse(action)); }); } + /** + * Fixes a cdc action's dataChange at false, which the protocol requires of cdc actions and Delta + * readers need on the wire, whatever the commit wrote. The action's schema must carry the slot, + * which the reader's cdc schema adds for this override. + */ + private static Row withDataChangeFalse(Row action) { + int ordinal = action.getSchema().indexOf("dataChange"); + return ordinal < 0 ? action : new DelegateRow(action, Map.of(ordinal, false)); + } + private static String deltaFile( String id, String deletionVectorFileId, diff --git a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaTableQueryReader.java b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaTableQueryReader.java index 378e745f7..5fafdddf8 100644 --- a/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaTableQueryReader.java +++ b/server/opensharing-server-core/src/main/java/io/opensharing/asset/table/delta/DeltaTableQueryReader.java @@ -26,6 +26,7 @@ import io.delta.kernel.internal.util.FileNames; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.internal.util.VectorUtils; +import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.LongType; import io.delta.kernel.types.MapType; import io.delta.kernel.types.StringType; @@ -79,6 +80,11 @@ public class DeltaTableQueryReader { private static final Duration URL_TTL = Duration.ofHours(1); private static final ObjectMapper JSON = new ObjectMapper(); + // Kernel's cdc schema omits dataChange; the slot exists only so TableActions can override it + // to false, which Delta readers require on a cdc action. + private static final StructType CDC_SCHEMA = + DeltaAction.CDC.schema.add("dataChange", BooleanType.BOOLEAN); + // Commit files are read directly because Kernel 4.0's getChanges requires commitInfo fields that // older Delta writers omit; only the commitInfo fields used here are read. private static final StructType CHANGE_SCHEMA = @@ -87,7 +93,7 @@ public class DeltaTableQueryReader { .add(DeltaAction.METADATA.colName, DeltaAction.METADATA.schema) .add(DeltaAction.ADD.colName, DeltaAction.ADD.schema) .add(DeltaAction.REMOVE.colName, DeltaAction.REMOVE.schema) - .add(DeltaAction.CDC.colName, DeltaAction.CDC.schema) + .add(DeltaAction.CDC.colName, CDC_SCHEMA) .add( DeltaAction.COMMITINFO.colName, new StructType() @@ -883,7 +889,8 @@ yield new ChangeFile( null, remove.getDeletionVector()); } - // cdc files are always data changes and carry no stats or deletion vector. + // A cdc file is always emitted, whatever the log's dataChange says: TableActions fixes the + // action's own dataChange at false. A cdc action carries no stats or deletion vector. case CDC -> { StructType schema = action.getSchema(); yield new ChangeFile( diff --git a/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/delta/DeltaTableQueryReaderTest.java b/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/delta/DeltaTableQueryReaderTest.java index 245fa9b49..170915833 100644 --- a/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/delta/DeltaTableQueryReaderTest.java +++ b/server/opensharing-server-core/src/test/java/io/opensharing/asset/table/delta/DeltaTableQueryReaderTest.java @@ -233,6 +233,50 @@ void changeDataFeedPrefersCdcFilesAndSkipsNoOpMerges() throws IOException { List delta = changeDataFeed(1L, null, 2L, null, "responseformat=delta", false); assertEquals(List.of("cdc c 1", "remove b 2"), describe(delta.subList(2, delta.size()))); + // Delta readers require dataChange on a cdc action, which the protocol fixes at false. + JsonNode cdc = + parse(delta.get(2)).path("file").path("deltaSingleAction").path("cdc"); + assertTrue(cdc.path("dataChange").isBoolean(), delta.get(2)); + assertFalse(cdc.path("dataChange").asBoolean(), delta.get(2)); + // The parquet cdc line carries no dataChange. + String parquet = changeDataFeed(1L, null, 2L, null, null, false).get(2); + assertTrue(parquet.startsWith("{\"cdf\":"), parquet); + assertFalse(parquet.contains("dataChange"), parquet); + // Add and remove actions keep the dataChange their commit wrote. + List deltaChanges = changeDataFeed(0L, null, 2L, null, "responseformat=delta", false); + assertEquals( + List.of("add a 0", "add b 0", "cdc c 1", "remove b 2"), + describe(deltaChanges.subList(2, deltaChanges.size()))); + assertTrue( + parse(deltaChanges.get(2)).path("file").path("deltaSingleAction").path("add") + .path("dataChange") + .asBoolean(), + deltaChanges.get(2)); + assertTrue( + parse(deltaChanges.get(5)).path("file").path("deltaSingleAction").path("remove") + .path("dataChange") + .asBoolean(), + deltaChanges.get(5)); + } + + @Test + void changeDataFeedWritesCdcDataChangeFalseWhateverTheCommitWrote() throws IOException { + writeChangeDataFeed(); + // The cdc action of commit 1 wrote dataChange false; write one with true instead. + Files.writeString( + dir.resolve("_delta_log/00000000000000000001.json"), + String.join( + "\n", + remove("a", "2021-01-01", true), + add("d", "2021-01-01", 1, 10), + "{\"cdc\":{\"path\":\"_change_data/date=2021-01-01/c.parquet\"," + + "\"partitionValues\":{\"date\":\"2021-01-01\"},\"size\":1," + + "\"dataChange\":true}}") + + "\n"); + List delta = changeDataFeed(1L, null, 1L, null, "responseformat=delta", false); + assertEquals(List.of("cdc c 1"), describe(delta.subList(2, delta.size()))); + // The override wins: a cdc action's dataChange is false whatever the commit wrote. + assertFalse(parse(delta.get(2)).path("cdc").path("dataChange").asBoolean(), delta.get(2)); } @Test