From 2aa7259c9fe24b7d72562a4dab62918c17c7888a Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Tue, 10 Mar 2026 11:47:41 -0700 Subject: [PATCH 01/80] [kernel-spark] Block non-additive schema change in dsv2 (#6206) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Before we support schema tracking log, we should disallow all non-additive schema change in dsv2. Column Mapping related code change is pure refactoring that for v2 to reuse the utilities. ## How was this patch tested? Unit test: - New test case for drop column in a nested struct - Modify existed tests for non-additive schema change to throws the same error as v1 Integration test: - drop column: should fail with non-additive schema change error - drop column: should succeed with unsafe column mapping schema change flag enabled - rename column: should fail with non-additive schema change error - rename column: should throw schema change error with unsafe flag enabled - type widening: should fail with non-additive schema change error when enable schema tracking ## Does this PR introduce _any_ user-facing changes? No --- .../sql/delta/test/DeltaV2SourceSuite.scala | 5 + .../resources/error/delta-error-classes.json | 9 + .../spark/sql/delta/DeltaColumnMapping.scala | 63 +++-- .../apache/spark/sql/delta/DeltaErrors.scala | 10 +- .../sql/delta/OptimisticTransaction.scala | 8 +- .../sql/delta/DeltaColumnMappingSuite.scala | 45 ++-- .../spark/sql/delta/DeltaSourceSuite.scala | 230 ++++++++++++++++++ .../v2/read/SparkMicroBatchStream.java | 113 +++++++-- .../spark/internal/v2/DeltaV2TestBase.java | 10 +- .../v2/read/SparkMicroBatchStreamTest.java | 117 ++++++--- 10 files changed, 511 insertions(+), 99 deletions(-) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index 94fd9065a33..bfe4e5462d6 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -59,6 +59,11 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { "add column: restarting with stale DataFrame should fail", "disallow to change schema after starting a streaming query", "allow to change schema before starting a streaming query", + "drop column: should fail with non-additive schema change error", + "drop column: should succeed with unsafe column mapping schema change flag enabled", + "rename column: should fail with non-additive schema change error", + "rename column: should throw schema change error with unsafe flag enabled", + "type widening: should fail with non-additive schema change error when enable schema tracking", // ========== startingVersion option tests ========== "startingVersion", diff --git a/spark/src/main/resources/error/delta-error-classes.json b/spark/src/main/resources/error/delta-error-classes.json index d4411a75144..1ba811544c2 100644 --- a/spark/src/main/resources/error/delta-error-classes.json +++ b/spark/src/main/resources/error/delta-error-classes.json @@ -2563,6 +2563,15 @@ ], "sqlState" : "42KD4" }, + "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_V2" : { + "message" : [ + "Streaming read is not supported on tables with read-incompatible schema changes (e.g. rename or drop or datatype changes).", + "Non-additive schema change handling is not supported in Delta source v2 yet.", + "For further information and possible next steps to resolve this issue, please review the documentation at ", + "Read schema: . Incompatible data schema: ." + ], + "sqlState" : "42KD4" + }, "DELTA_STREAMING_INITIAL_SNAPSHOT_TOO_LARGE" : { "message" : [ "Initial snapshot for Delta streaming at table '' (version ) contains files, which exceeds the maximum allowed files.", diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala index 672731e77d3..e4d373b524b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala @@ -657,17 +657,19 @@ trait DeltaColumnMappingBase extends DeltaLogging { * We detect DROP COLUMNS by checking if any physical name in `currentSchema` is missing in * `newSchema`. */ - def isDropColumnOperation(newMetadata: Metadata, currentMetadata: Metadata): Boolean = { + def isDropColumnOperation( + newSchema: StructType, + currentSchema: StructType, + isBothColumnMappingEnabled: Boolean): Boolean = { // We will need to compare the new schema's physical columns to the current schema's physical // columns. So, they both must have column mapping enabled. - if (newMetadata.columnMappingMode == NoMapping || - currentMetadata.columnMappingMode == NoMapping) { + if (!isBothColumnMappingEnabled) { return false } - val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newMetadata.schema) - val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentMetadata.schema) + val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newSchema) + val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentSchema) // are any of the current physical names missing in the new schema? currentPhysicalToLogicalMap @@ -704,17 +706,19 @@ trait DeltaColumnMappingBase extends DeltaLogging { * We detect RENAME COLUMNS by checking if any two columns with the same physical name have * different logical names */ - def isRenameColumnOperation(newMetadata: Metadata, currentMetadata: Metadata): Boolean = { + def isRenameColumnOperation( + newSchema: StructType, + currentSchema: StructType, + isBothColumnMappingEnabled: Boolean): Boolean = { // We will need to compare the new schema's physical columns to the current schema's physical // columns. So, they both must have column mapping enabled. - if (newMetadata.columnMappingMode == NoMapping || - currentMetadata.columnMappingMode == NoMapping) { + if (!isBothColumnMappingEnabled) { return false } - val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newMetadata.schema) - val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentMetadata.schema) + val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newSchema) + val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentSchema) // do any two columns with the same physical name have different logical names? currentPhysicalToLogicalMap @@ -723,6 +727,21 @@ trait DeltaColumnMappingBase extends DeltaLogging { } } + /** + * Returns true if there is a column mapping schema change (drop/rename) or an incompatible + * partition column change between the new and current schemas. + */ + def hasColMappingOrPartitionSchemaChange( + newSchema: StructType, + currentSchema: StructType, + newPartitionColumns: Seq[String], + oldPartitionColumns: Seq[String], + isBothColumnMappingEnabled: Boolean): Boolean = { + isDropColumnOperation(newSchema, currentSchema, isBothColumnMappingEnabled) || + isRenameColumnOperation(newSchema, currentSchema, isBothColumnMappingEnabled) || + !SchemaUtils.isPartitionCompatible(newPartitionColumns, oldPartitionColumns) + } + /** * Collects the column rename operations between the new schema and the current schema. * @param newSchema The new schema after a potential rename. @@ -758,22 +777,24 @@ trait DeltaColumnMappingBase extends DeltaLogging { */ def hasNoColumnMappingSchemaChanges(newMetadata: Metadata, oldMetadata: Metadata, allowUnsafeReadOnPartitionChanges: Boolean = false): Boolean = { - // Helper function to check no column mapping schema change and no repartition - def hasNoColMappingAndRepartitionSchemaChange( - newMetadata: Metadata, oldMetadata: Metadata): Boolean = { - isRenameColumnOperation(newMetadata, oldMetadata) || - isDropColumnOperation(newMetadata, oldMetadata) || - !SchemaUtils.isPartitionCompatible( - // if allow unsafe row read for partition change, ignore the check - if (allowUnsafeReadOnPartitionChanges) Seq.empty else newMetadata.partitionColumns, - if (allowUnsafeReadOnPartitionChanges) Seq.empty else oldMetadata.partitionColumns) + def hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata: Metadata, + oldMetadata: Metadata): Boolean = { + val isBothColumnMappingEnabled = + newMetadata.columnMappingMode != NoMapping && oldMetadata.columnMappingMode != NoMapping + hasColMappingOrPartitionSchemaChange( + newMetadata.schema, + oldMetadata.schema, + // if allow unsafe row read for partition change, ignore the check + if (allowUnsafeReadOnPartitionChanges) Seq.empty else newMetadata.partitionColumns, + if (allowUnsafeReadOnPartitionChanges) Seq.empty else oldMetadata.partitionColumns, + isBothColumnMappingEnabled) } val (oldMode, newMode) = (oldMetadata.columnMappingMode, newMetadata.columnMappingMode) if (oldMode != NoMapping && newMode != NoMapping) { require(oldMode == newMode, "changing mode is not supported") // Both changes are post column mapping enabled - !hasNoColMappingAndRepartitionSchemaChange(newMetadata, oldMetadata) + !hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata, oldMetadata) } else if (oldMode == NoMapping && newMode != NoMapping) { // The old metadata does not have column mapping while the new metadata does, in this case // we assume an upgrade has happened in between. @@ -790,7 +811,7 @@ trait DeltaColumnMappingBase extends DeltaLogging { Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> newMetadata.columnMappingMode.name) ) // use the same check - !hasNoColMappingAndRepartitionSchemaChange(newMetadata, upgradedMetadata) + !hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata, upgradedMetadata) } else { // Prohibit reading across a downgrade. val isDowngrade = oldMode != NoMapping && newMode == NoMapping diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala index ca622d0b40d..2c9a81a73f3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala @@ -3221,7 +3221,8 @@ trait DeltaErrorsBase spark: SparkSession, readSchema: StructType, incompatibleSchema: StructType, - detectedDuringStreaming: Boolean): Throwable = { + detectedDuringStreaming: Boolean, + isV2DataSource: Boolean = false): Throwable = { val docLink = "/versioning.html#column-mapping" val enableNonAdditiveSchemaEvolution = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING) @@ -3231,7 +3232,8 @@ trait DeltaErrorsBase generateDocsLinkOption(spark, docLink).getOrElse("-"), enableNonAdditiveSchemaEvolution, additionalProperties = Map( - "detectedDuringStreaming" -> detectedDuringStreaming.toString + "detectedDuringStreaming" -> detectedDuringStreaming.toString, + "isV2DataSource" -> isV2DataSource.toString )) } @@ -4358,7 +4360,9 @@ class DeltaStreamingNonAdditiveSchemaIncompatibleException( val enableNonAdditiveSchemaEvolution: Boolean = false, val additionalProperties: Map[String, String] = Map.empty) extends DeltaUnsupportedOperationException( - errorClass = if (enableNonAdditiveSchemaEvolution) { + errorClass = if (additionalProperties.getOrElse("isV2DataSource", "false") == "true") { + "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_V2" + } else if (enableNonAdditiveSchemaEvolution) { "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_USE_SCHEMA_LOG" } else { "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE" diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index b83d921d4db..031f0779ec9 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -1432,11 +1432,15 @@ trait OptimisticTransactionImpl extends TransactionHelper newMode = _newMetadata.columnMappingMode ) + val isBothColumnMappingEnabled = + _newMetadata.columnMappingMode != NoMapping && + _currentMetadata.columnMappingMode != NoMapping + def dropColumnOp: Boolean = DeltaColumnMapping.isDropColumnOperation( - _newMetadata, _currentMetadata) + _newMetadata.schema, _currentMetadata.schema, isBothColumnMappingEnabled) def renameColumnOp: Boolean = DeltaColumnMapping.isRenameColumnOperation( - _newMetadata, _currentMetadata) + _newMetadata.schema, _currentMetadata.schema, isBothColumnMappingEnabled) def columnMappingChange: Boolean = isColumnMappingUpgrade || dropColumnOp || renameColumnOp diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala index 52eb9c053dc..af9a9b346fd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala @@ -1413,9 +1413,14 @@ class DeltaColumnMappingSuite extends QueryTest // case 1: currentSchema compared with itself var currentMetadata = getMetadata() var newMetadata = getMetadata() + def isBothCMEnabled: Boolean = + newMetadata.columnMappingMode != NoMapping && + currentMetadata.columnMappingMode != NoMapping assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 2: add a top-level column @@ -1423,8 +1428,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 3: add a nested column @@ -1432,8 +1439,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 4: drop a top-level column @@ -1441,8 +1450,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 5: drop a nested column @@ -1450,8 +1461,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 6: rename a top-level column @@ -1459,8 +1472,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 7: rename a nested column @@ -1468,8 +1483,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala index 1abb516c5d6..1f95c6b1916 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala @@ -2275,6 +2275,236 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } + test("drop column: should fail with non-additive schema change error") { + withTempDir { inputDir => + withTempDir { checkpointDir => + // Create a table with column mapping enabled (required for drop/rename column) + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i.toString, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("noop") + .start() + } + + val q = startQuery() + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate dropping "value" column by committing new metadata with only "id". + // This is a non-additive schema change (column removal). + withMetadata(deltaLog, StructType.fromDDL("id STRING")) + + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains( + "Streaming read is not supported on tables with read-incompatible schema changes")) + } finally { + q.stop() + } + } + } + } + + test("drop column: should succeed with unsafe column mapping schema change flag enabled") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + // Enable the unsafe flag that allows streaming to continue past column mapping + // schema changes (drop/rename) instead of failing. + withSQLConf( + DeltaSQLConf + .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES + .key -> "true" + ) { + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i.toString, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + val q = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("delta") + .start(outputDir.getCanonicalPath) + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate dropping "value" column by committing new metadata with only "id". + withMetadata(deltaLog, StructType.fromDDL("id STRING")) + + // Write more data after the drop column schema change + (5 until 10).foreach { i => + Seq(i.toString).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + // With the unsafe flag, the stream should process all data without failing. + // Post-drop rows have null for the removed "value" column. + q.processAllAvailable() + + checkAnswer( + spark.read.format("delta").load(outputDir.getAbsolutePath), + (0 until 5).map(i => (i.toString, s"val$i")).toDF("id", "value") union + (5 until 10).map(i => (i.toString, null: String)).toDF("id", "value")) + } finally { + q.stop() + } + } + } + } + + test("rename column: should fail with non-additive schema change error") { + withTempDir { inputDir => + withTempDir { checkpointDir => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i.toString, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("noop") + .start() + } + + var q = startQuery() + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate renaming "value" -> "renamed_value" by committing new metadata. + // Column rename is a non-additive schema change under column mapping. + withMetadata(deltaLog, StructType.fromDDL("id STRING, renamed_value STRING")) + + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains( + "Streaming read is not supported on tables with read-incompatible schema changes")) + } finally { + q.stop() + } + } + } + } + + test("rename column: should throw schema change error with unsafe flag enabled") { + withTempDir { inputDir => + withTempDir { checkpointDir => + // The unsafe flag only helps with drop-column; rename is still blocked because + // it changes logical column identity, which can silently corrupt data. + withSQLConf( + DeltaSQLConf + .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES + .key -> "true" + ) { + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i.toString, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("noop") + .start() + } + + var q = startQuery() + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate renaming "value" -> "renamed_value" by committing new metadata. + withMetadata(deltaLog, StructType.fromDDL("id STRING, renamed_value STRING")) + + // Even with the unsafe flag, rename still fails - but with a different error + // ("Detected schema change") rather than the non-additive schema change error. + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change in version 6")) + } finally { + q.stop() + } + } + } + } + } + + test("type widening: should fail with non-additive schema change error " + + "when enable schema tracking") { + withTempDir { inputDir => + withTempDir { checkpointDir => + withSQLConf( + DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "true" + ) { + // Table with type widening enabled so that type changes are allowed but tracked. + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("noop") + .start() + } + + var q = startQuery() + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate widening "id" from INT -> BIGINT by committing new metadata. + // Type widening is a non-additive schema change for streaming reads. + withMetadata(deltaLog, StructType.fromDDL("id BIGINT, value STRING")) + + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains( + "Streaming read is not supported on tables with read-incompatible schema changes")) + } finally { + q.stop() + } + } + } + } + } + test("handling nullability schema changes") { withTable("srcTable") { withTempDirs { (srcTblDir, checkpointDir, checkpointDir2) => diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java index bec8e76f217..cead0c1c2ae 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java @@ -34,6 +34,7 @@ import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.RemoveFile; import io.delta.kernel.internal.util.ColumnMapping; +import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode; import io.delta.kernel.internal.util.Preconditions; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.internal.util.VectorUtils; @@ -57,12 +58,14 @@ import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.connector.read.streaming.*; +import org.apache.spark.sql.delta.DeltaColumnMapping; import org.apache.spark.sql.delta.DeltaErrors; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.delta.DeltaStartingVersion; import org.apache.spark.sql.delta.DeltaTimeTravelSpec; import org.apache.spark.sql.delta.StartingVersion; import org.apache.spark.sql.delta.StartingVersionLatest$; +import org.apache.spark.sql.delta.TypeWidening; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.delta.sources.DeltaSource; import org.apache.spark.sql.delta.sources.DeltaSourceOffset; @@ -217,7 +220,7 @@ public SparkMicroBatchStream( boolean isStreamingFromColumnMappingTable = ColumnMapping.getColumnMappingMode( this.snapshotAtSourceInit.getMetadata().getConfiguration()) - != ColumnMapping.ColumnMappingMode.NONE; + != ColumnMappingMode.NONE; boolean isTypeWideningSupportedInProtocol = this.snapshotAtSourceInit.getProtocol().supportsFeature(TYPE_WIDENING_RW_PREVIEW_FEATURE) || this.snapshotAtSourceInit.getProtocol().supportsFeature(TYPE_WIDENING_RW_FEATURE); @@ -967,7 +970,32 @@ private void checkReadIncompatibleSchemaChanges( newMetadata.getId(), oldMetadata.getId()); } - // TODO(#5319): schema tracking for non-additive schema changes + // Partition column change will be ignored if user enable the unsafe flag + Seq newPartitionColumns, oldPartitionColumns; + if (schemaReadOptions.allowUnsafeStreamingReadOnPartitionColumnChanges()) { + newPartitionColumns = (Seq) Seq$.MODULE$.empty(); + oldPartitionColumns = (Seq) Seq$.MODULE$.empty(); + } else { + newPartitionColumns = + CollectionConverters.asScala( + VectorUtils.toJavaList(newMetadata.getPartitionColumns()).stream() + .map(Object::toString) + .collect(Collectors.toList())) + .toSeq(); + oldPartitionColumns = + CollectionConverters.asScala( + VectorUtils.toJavaList(oldMetadata.getPartitionColumns()).stream() + .map(Object::toString) + .collect(Collectors.toList())) + .toSeq(); + } + + checkNonAdditiveSchemaChanges( + oldMetadata, + newMetadata, + oldPartitionColumns, + newPartitionColumns, + validatedDuringStreamStart); // Other standard read compatibility changes if (!validatedDuringStreamStart @@ -980,25 +1008,6 @@ private void checkReadIncompatibleSchemaChanges( // check whether we can use `schema` (the fixed source schema we use in the same run of the // query) to read these new files safely. boolean backfilling = version < snapshotAtSourceInit.getVersion(); - // Partition column change will be ignored if user enable the unsafe flag - Seq newPartitionColumns, oldPartitionColumns; - if (schemaReadOptions.allowUnsafeStreamingReadOnPartitionColumnChanges()) { - newPartitionColumns = (Seq) Seq$.MODULE$.empty(); - oldPartitionColumns = (Seq) Seq$.MODULE$.empty(); - } else { - newPartitionColumns = - CollectionConverters.asScala( - VectorUtils.toJavaList(newMetadata.getPartitionColumns()).stream() - .map(Object::toString) - .collect(Collectors.toList())) - .toSeq(); - oldPartitionColumns = - CollectionConverters.asScala( - VectorUtils.toJavaList(oldMetadata.getPartitionColumns()).stream() - .map(Object::toString) - .collect(Collectors.toList())) - .toSeq(); - } DeltaStreamUtils.SchemaCompatibilityResult checkResult = DeltaStreamUtils.checkSchemaChangesWhenNoSchemaTracking( @@ -1024,6 +1033,68 @@ private void checkReadIncompatibleSchemaChanges( } } + // TODO(#5319): schema tracking for non-additive schema changes + // TODO(#5319): Extract the entire non-additive schema check into a static utility and share it + // with v1 by refactoring DeltaColumnMapping.hasNoColumnMappingSchemaChanges so it can be reused + // by both v1 and v2. + // Non-additive schema changes include rename column, drop column and change column type + private void checkNonAdditiveSchemaChanges( + Metadata oldMetadata, + Metadata newMetadata, + Seq oldPartitionColumns, + Seq newPartitionColumns, + boolean validatedDuringStreamStart) { + StructType sparkNewSchema = + SchemaUtils.convertKernelSchemaToSparkSchema(newMetadata.getSchema()); + StructType sparkOldSchema = + SchemaUtils.convertKernelSchemaToSparkSchema(oldMetadata.getSchema()); + + boolean shouldTrackSchema; + if (schemaReadOptions.typeWideningEnabled() + && schemaReadOptions.enableSchemaTrackingForTypeWidening() + && TypeWidening.containsWideningTypeChanges(sparkOldSchema, sparkNewSchema)) { + // If schema tracking is enabled for type widening, we will detect widening type changes and + // block the stream until the user sets `allowSourceColumnTypeChange` - similar to handling + // DROP/RENAME for column mapping. + shouldTrackSchema = true; + } else if (schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges()) { + shouldTrackSchema = false; + } else { + ColumnMappingMode NONE = ColumnMappingMode.NONE; + ColumnMappingMode oldMode = + ColumnMapping.getColumnMappingMode(oldMetadata.getConfiguration()); + ColumnMappingMode newMode = + ColumnMapping.getColumnMappingMode(newMetadata.getConfiguration()); + if (oldMode != NONE && newMode != NONE) { + Preconditions.checkArgument(oldMode == newMode, "changing mode is not supported"); + shouldTrackSchema = + DeltaColumnMapping.hasColMappingOrPartitionSchemaChange( + sparkNewSchema, + sparkOldSchema, + newPartitionColumns, + oldPartitionColumns, + /* isBothColumnMappingEnabled */ true); + } else if (oldMode == NONE && newMode != NONE) { + // TODO(#5319): We should disallow user to upgrade column mapping mode for now since we + // don't support schema tracking + shouldTrackSchema = true; + } else { + // Prohibit reading across a downgrade. + shouldTrackSchema = oldMode != NONE && newMode == NONE; + } + } + + if (shouldTrackSchema) { + throw (RuntimeException) + DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges( + spark, + sparkOldSchema, + sparkNewSchema, + !validatedDuringStreamStart, + /* isV2DataSource= */ true); + } + } + /** * Check read-incompatible schema changes during stream (re)start so we could fail fast. * diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java index f07f57fca8f..f05c3223c71 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java @@ -68,13 +68,19 @@ protected void createEmptyTestTable(String path, String tableName) { protected void createSchemaEvolutionTestTable(String path, String tableName) { spark.sql( String.format( - "CREATE TABLE %s (id INT NOT NULL, name String, value FLOAT) USING delta LOCATION '%s'" + "CREATE TABLE %s (id INT NOT NULL, " + + "name String, value FLOAT, " + + "info STRUCT) USING delta LOCATION '%s'" + "TBLPROPERTIES (" + "'delta.columnMapping.mode' = 'name', " + "'delta.enableTypeWidening' = 'true')", tableName, path)); spark.sql( - String.format("INSERT INTO %s VALUES (1, 'Alice', 10.5), (2,'Bob', NULL)", tableName)); + String.format( + "INSERT INTO %s VALUES " + + "(1, 'Alice', 10.5, named_struct('col1', 27, 'col2', 'LA')), " + + "(2,'Bob', NULL, named_struct('col1', 30, 'col2', 'NYC'))", + tableName)); } protected static void createPartitionedTable(String tableName, String path) { diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java index 6e96bbb17d8..def6dcc3605 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java @@ -17,7 +17,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.assumeFalse; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.DeltaV2TestBase; @@ -2256,26 +2255,6 @@ public void testSchemaEvolution_onForwardNonAdditiveChanges_throwsError( + "_" + System.nanoTime(); - // TODO(#5318): Skip forward-fill drop column test until checkNonAdditiveSchemaChanges is - // implemented. - // - // `isReadCompatible` cannot distinguish between safe "backfill + add column" and breaking - // "forward-fill + drop column" scenarios. Both involve reading [A,B] data with [A,B,C] schema, - // which passes compatibility checks. However, the latter is a breaking change. - // - // Special handling is needed to detect "forward-fill + drop column" scenarios. DSv1 solves - // this via `checkNonAdditiveSchemaChanges`, which relies on - // DeltaColumnMapping.hasNoColumnMappingSchemaChanges. - // - // DSv1 has the same limitation when `checkNonAdditiveSchemaChanges` is removed. Since - // non-additive schema change detection is the next work item and provides the complete - // solution for all column mapping changes, implementing a partial fix now would create - // duplicate logic. DSv1 also behaved the same way before `checkNonAdditiveSchemaChanges` - // was implemented, so we skip this test as a short-term solution for now. - assumeFalse( - testDescription.contains("Drop"), - "Skipping drop column test - checkNonAdditiveSchemaChanges not yet implemented in v2 (TODO #5318)"); - createSchemaEvolutionTestTable(testTablePath, testTableName); // Try to read from version 0, which should include commits with METADATA actions @@ -2322,9 +2301,9 @@ public void testSchemaEvolution_onForwardNonAdditiveChanges_throwsError( }, String.format("DSv1 should throw on METADATA for scenario: %s", testDescription)); - DeltaIllegalStateException dsv2Exception = + DeltaUnsupportedOperationException dsv2Exception = assertThrows( - DeltaIllegalStateException.class, + DeltaUnsupportedOperationException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges( @@ -2349,7 +2328,23 @@ public void testSchemaEvolution_onForwardNonAdditiveChanges_throwsError( } }, String.format("DSv2 should throw on METADATA for scenario: %s", testDescription)); - // TODO(#5318): assert two exceptions are equal after schema tracking is supported in v2 + + // TODO(#5319): assertEqual after schema tracking log is supported + String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; + assertTrue( + dsv1Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v1 connector error class should start with %s, but got: %s", + expectedPrefix, dsv1Exception.getErrorClass())); + assertTrue( + dsv2Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v2 connector error class should start with %s, but got: %s", + expectedPrefix, dsv2Exception.getErrorClass())); + assertEquals( + dsv1Exception.getMessageParameters(), + dsv2Exception.getMessageParameters(), + "v1 connector and v2 connector should throw the same error messages on forward-fill non-additive schema changes"); } finally { // recover spark config to original state sparkConf.forEach((key, value) -> spark.conf().unset(key)); @@ -2420,9 +2415,9 @@ public void testSchemaEvolution_onBackfillNonAdditiveChanges_throwsError( }, String.format("DSv1 should throw on METADATA for scenario: %s", testDescription)); - DeltaIllegalStateException dsv2Exception = + DeltaUnsupportedOperationException dsv2Exception = assertThrows( - DeltaIllegalStateException.class, + DeltaUnsupportedOperationException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges( @@ -2447,7 +2442,23 @@ public void testSchemaEvolution_onBackfillNonAdditiveChanges_throwsError( } }, String.format("DSv2 should throw on METADATA for scenario: %s", testDescription)); - // TODO(#5318): assert two exceptions are equal after schema tracking is supported in v2 + + // TODO(#5319): assertEqual after schema tracking log is supported + String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; + assertTrue( + dsv1Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v1 connector error class should start with %s, but got: %s", + expectedPrefix, dsv1Exception.getErrorClass())); + assertTrue( + dsv2Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v2 connector error class should start with %s, but got: %s", + expectedPrefix, dsv2Exception.getErrorClass())); + assertEquals( + dsv1Exception.getMessageParameters(), + dsv2Exception.getMessageParameters(), + "v1 connector and v2 connector should throw the same error messages on backfill non-additive schema changes"); } finally { // recover spark config to original state sparkConf.forEach((key, value) -> spark.conf().unset(key)); @@ -2476,7 +2487,10 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws // Execute schema change before source initialization to ensure backfill change spark.sql(String.format("ALTER table %s RENAME COLUMN id TO userId", testTableName)); - spark.sql(String.format("INSERT INTO %s VALUES (3, 'Cathy', 5)", testTableName)); + spark.sql( + String.format( + "INSERT INTO %s VALUES (3, 'Cathy', 5, named_struct('col1', 18, 'col2', 'SF'))", + testTableName)); // Record the version prior to reverting schema change long incompatibleSchemaVersion = DeltaLog.forTable(spark, new Path(testTablePath)) @@ -2484,7 +2498,10 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws .version(); // Revert the schema change spark.sql(String.format("ALTER table %s RENAME COLUMN userId TO id", testTableName)); - spark.sql(String.format("INSERT INTO %s VALUES (4, 'David', 8)", testTableName)); + spark.sql( + String.format( + "INSERT INTO %s VALUES (4, 'David', 8, named_struct('col1', 47, 'col2', 'DC'))", + testTableName)); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); String tableId = deltaLog.tableId(); @@ -2517,11 +2534,9 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - // TODO(#5318): Change error type to - // DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges - DeltaIllegalStateException dsv2Exception = + DeltaUnsupportedOperationException dsv2Exception = assertThrows( - DeltaIllegalStateException.class, + DeltaUnsupportedOperationException.class, () -> stream.latestOffset(startOffset, readLimit), String.format( "DSv2 should throw error on stream start for scenario: %s", testDescription)); @@ -2530,6 +2545,23 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws .anyMatch( element -> element.toString().contains("checkReadIncompatibleSchemaChangeOnStreamStartOnce")); + + // TODO(#5319): assertEqual after schema tracking log is supported + String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; + assertTrue( + dsv1Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v1 connector error class should start with %s, but got: %s", + expectedPrefix, dsv1Exception.getErrorClass())); + assertTrue( + dsv2Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v2 connector error class should start with %s, but got: %s", + expectedPrefix, dsv2Exception.getErrorClass())); + assertEquals( + dsv1Exception.getMessageParameters(), + dsv2Exception.getMessageParameters(), + "v1 connector and v2 connector should throw the same error messages on stream start schema changes"); } /** Provides test scenarios that generate additive schema changes actions. */ @@ -2629,18 +2661,31 @@ private static Stream nonAdditiveSchemaEvolutionScenarios() { /* sparkConf */ Map.of(), "Rename column"), - // Drop both nullable and non-nullable columns + // Drop nullable, non-nullable and struct columns + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("ALTER TABLE %s DROP COLUMNS (id, value, info)", tableName); + }, + /* sparkConf */ Map.of( + DeltaSQLConf + .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES() + .key(), + "false"), + "Drop nullable, non-nullable and struct columns"), + + // Drop column in nested struct Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { - sql("ALTER TABLE %s DROP COLUMNS (id, value)", tableName); + sql("ALTER TABLE %s DROP COLUMNS info.col1", tableName); }, /* sparkConf */ Map.of( DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES() .key(), "false"), - "Drop both nullable and non-nullable columns"), + "Drop column in nested struct"), // Widen INT column to BIGINT Arguments.of( From 58309c897e41296ee84b4f0714867c4a229b130b Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:29:38 -0700 Subject: [PATCH 02/80] [kernel-spark] Add e2e tests for type widening and relaxing nullability (#6227) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🥞 Stacked PR Use this [link](https://github.com/delta-io/delta/pull/6227/files) to review incremental changes. - [**stack/type-nullability-e2e-test**](https://github.com/delta-io/delta/pull/6227) [[Files changed](https://github.com/delta-io/delta/pull/6227/files)] - [stack/forbid-schema-evolution-with-staleDF-v2](https://github.com/delta-io/delta/pull/6228) [[Files changed](https://github.com/delta-io/delta/pull/6228/files/cafe746303c3e32bdcf5499c8e99fd6a59c033c4..94b77dc62d21e896e9f4fdafd3ff17fafddd606f)] --------- #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Added schema evolution e2e tests for type widening and relax nullability for both refresh dataframe and stale dataframe cases. Refresh dataframe cases work for both v1 & v2 connector, while stale dataframe cases only work for v1 connector. ## How was this patch tested? test-only PR ## Does this PR introduce _any_ user-facing changes? No --- .../sql/delta/test/DeltaV2SourceSuite.scala | 8 + .../spark/sql/delta/DeltaSourceSuite.scala | 227 +++++++++++++++++- 2 files changed, 234 insertions(+), 1 deletion(-) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index bfe4e5462d6..be5ed48e6c3 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -57,6 +57,8 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { // === Schema Evolution === "add column: restarting with new DataFrame should recover", "add column: restarting with stale DataFrame should fail", + "relax nullability: restarting with new DataFrame should recover", + "type widening: restarting with new DataFrame should recover", "disallow to change schema after starting a streaming query", "allow to change schema before starting a streaming query", "drop column: should fail with non-additive schema change error", @@ -119,6 +121,12 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { "excludeRegex works and doesn't mess up offsets across restarts - parquet version", "SC-46515: deltaSourceIgnoreChangesError contains removeFile, version, tablePath", + // === Schema Evolution === + // TODO(#6232): enable the two tests after spark streaming engine supports leaf node projection + // for datasource v2 such that we can adopt the two schema changes without refresh the dataframe + "relax nullability: restarting with stale DataFrame should recover", + "type widening: restarting with stale DataFrame should recover", + // === Data Loss Detection === "fail on data loss - starting from missing files", "fail on data loss - gaps of files", diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala index 1f95c6b1916..276d96aa095 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala @@ -47,7 +47,7 @@ import org.apache.spark.sql.execution.streaming._ import org.apache.spark.sql.functions.when import org.apache.spark.sql.streaming.{OutputMode, StreamingQuery, StreamingQueryException, Trigger} import org.apache.spark.sql.streaming.util.StreamManualClock -import org.apache.spark.sql.types.{IntegerType, NullType, StringType, StructField, StructType} +import org.apache.spark.sql.types.{IntegerType, LongType, NullType, StringType, StructField, StructType} import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.{ManualClock, Utils} @@ -2188,6 +2188,7 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") + // use delta sink because we need to check the result .format("delta") .start(outputDir.getCanonicalPath) } @@ -2275,6 +2276,230 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } + test("relax nullability: restarting with new DataFrame should recover") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` " + + "(id STRING NOT NULL) USING DELTA") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 2).foreach { i => + Seq(i.toString).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + // use delta sink because we need to check the result + .format("delta") + .start(outputDir.getCanonicalPath) + } + + var q = startQuery() + try { + q.processAllAvailable() + + // Clear delta log cache + DeltaLog.clearCache() + // Relax nullability from "id STRING NOT NULL" to "id STRING" using the non-cached + // `DeltaLog` to mimic the case that the schema change happens on a different cluster + withMetadata(deltaLog, StructType.fromDDL("id STRING")) + // Insert a null row + Seq(Option.empty[String]).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + + // The streaming query should fail when detecting a schema change + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change")) + + // Restarting the query with a new DataFrame should recover from the schema change + q = startQuery() + q.processAllAvailable() + val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) + assert(outputDf.schema("id").nullable, + "Expected 'id' column to be nullable after relaxing nullability") + checkAnswer(outputDf.orderBy("id"), + Seq(Row(null), Row("0"), Row("1"))) + } finally { + q.stop() + } + } + } + + test("relax nullability: restarting with stale DataFrame should recover") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` " + + "(id STRING NOT NULL) USING DELTA") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 2).foreach { i => + Seq(i.toString).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + + var q = df.writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + // use delta sink because we need to check the result + .format("delta") + .start(outputDir.getCanonicalPath) + try { + q.processAllAvailable() + + // Clear delta log cache + DeltaLog.clearCache() + // Relax nullability from "id STRING NOT NULL" to "id STRING" using the non-cached + // `DeltaLog` to mimic the case that the schema change happens on a different cluster + withMetadata(deltaLog, StructType.fromDDL("id STRING")) + // Insert a null row + Seq(Option.empty[String]).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + + // The streaming query should fail when detecting a schema change + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change")) + + // Restarting the query with the stale DataFrame should still recover + q = df.writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + .format("delta") + .start(outputDir.getCanonicalPath) + q.processAllAvailable() + val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) + assert(outputDf.schema("id").nullable, + "Expected 'id' column to be nullable after relaxing nullability") + checkAnswer(outputDf.orderBy("id"), + Seq(Row(null), Row("0"), Row("1"))) + } finally { + q.stop() + } + } + } + + test("type widening: restarting with new DataFrame should recover") { + withSQLConf( + DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "false", + DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey -> "true", + DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "true") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT) " + + "USING DELTA TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 2).foreach { i => + Seq(i).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + // use delta sink because we need to check the result + .format("delta") + .start(outputDir.getCanonicalPath) + } + + var q = startQuery() + try { + q.processAllAvailable() + + // Clear delta log cache + DeltaLog.clearCache() + // Widen the column type from INT to BIGINT using the non-cached `DeltaLog` to mimic + // the case that the schema change happens on a different cluster + withMetadata(deltaLog, StructType.fromDDL("id BIGINT")) + // 2^31 cannot be represented as an int, so it will be inserted as a bigint + Seq(2147483648L).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + + // The streaming query should fail when detecting a schema change + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change")) + + // Restarting the query with a new DataFrame should recover from the schema change + q = startQuery() + q.processAllAvailable() + val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) + assert(outputDf.schema("id").dataType === LongType, + s"Expected 'id' column to be LongType after type widening but got " + + s"${outputDf.schema("id").dataType}") + checkAnswer(outputDf.orderBy("id"), + Seq(Row(0L), Row(1L), Row(2147483648L))) + } finally { + q.stop() + } + } + } + } + + test("type widening: restarting with stale DataFrame should recover") { + withSQLConf( + DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "false", + DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey -> "true", + DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "true") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT) " + + "USING DELTA TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 2).foreach { i => + Seq(i).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + + var q = df.writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + // use delta sink because we need to check the result + .format("delta") + .start(outputDir.getCanonicalPath) + try { + q.processAllAvailable() + + // Clear delta log cache + DeltaLog.clearCache() + // Widen the column type from INT to BIGINT using the non-cached `DeltaLog` to mimic + // the case that the schema change happens on a different cluster + withMetadata(deltaLog, StructType.fromDDL("id BIGINT")) + // 2^31 cannot be represented as an int, so it will be inserted as a bigint + Seq(2147483648L).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + + // The streaming query should fail when detecting a schema change + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change")) + + // Restarting the query with the stale DataFrame should still recover + q = df.writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + .format("delta") + .start(outputDir.getCanonicalPath) + q.processAllAvailable() + val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) + assert(outputDf.schema("id").dataType === LongType, + s"Expected 'id' column to be LongType after type widening but got " + + s"${outputDf.schema("id").dataType}") + checkAnswer(outputDf.orderBy("id"), + Seq(Row(0L), Row(1L), Row(2147483648L))) + } finally { + q.stop() + } + } + } + } + test("drop column: should fail with non-additive schema change error") { withTempDir { inputDir => withTempDir { checkpointDir => From 98c7a273769c0dea0f91cf850620eec27f55e15a Mon Sep 17 00:00:00 2001 From: Xin Huang <42597328+huan233usc@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:13:40 -0700 Subject: [PATCH 03/80] [Spark] Double DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES default from 50000 to 100000 (#6240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Double the default value of `DELTA_STREAMING_INITIAL_SNAPSHOT_MAX_FILES` from 50,000 to 100,000 to better support larger tables for V2 streaming initial snapshots out of the box. ## How was this patch tested? Existing tests continue to pass since this is a default-value-only change. Tests that explicitly set this config (e.g., `SparkMicroBatchStreamTest`) are unaffected. ## Does this PR introduce _any_ user-facing changes? Yes. The default maximum number of files allowed in the initial snapshot for V2 streaming is increased from 50,000 to 100,000. Users with tables between 50k–100k files will no longer hit the limit without explicitly raising the config. This change only affects the unreleased master branch and the future release. --- .../scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index b57dca07129..08c731a7459 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -3166,7 +3166,7 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .internal() .doc("Maximum number of files allowed in initial snapshot for V2 streaming.") .intConf - .createWithDefault(50000) + .createWithDefault(100000) } object DeltaSQLConf extends DeltaSQLConfBase From 351b5bc5f8a10f2e734ea4d9fcb382bb2879509d Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Tue, 10 Mar 2026 15:02:35 -0700 Subject: [PATCH 04/80] [kernel-spark] Schema validation during stream restart e2e test for type widening and relaxing nullability (#6228) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🥞 Stacked PR Use this [link](https://github.com/delta-io/delta/pull/6228/files/58309c897e41296ee84b4f0714867c4a229b130b..781fa7a841d9f7dc1e6551426f12bf5fe90f8055) to review incremental changes. - [stack/type-nullability-e2e-test](https://github.com/delta-io/delta/pull/6227) [[Files changed](https://github.com/delta-io/delta/pull/6227/files)] - [**stack/forbid-schema-evolution-with-staleDF-v2**](https://github.com/delta-io/delta/pull/6228) [[Files changed](https://github.com/delta-io/delta/pull/6228/files/58309c897e41296ee84b4f0714867c4a229b130b..781fa7a841d9f7dc1e6551426f12bf5fe90f8055)] --------- #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Add schema validation during stream restart e2e tests for type widening and relaxing nullability to block user from adopt these two schema change without refresh the dataframe in v2 connector ## How was this patch tested? Test-only PR ## Does this PR introduce _any_ user-facing changes? No --- .../sql/delta/test/DeltaV2SourceSuite.scala | 3 +- .../v2/read/SparkMicroBatchStream.java | 1 - .../internal/v2/V2StreamingReadTest.java | 156 +++++++++++++----- 3 files changed, 121 insertions(+), 39 deletions(-) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index be5ed48e6c3..d62e1cdc15f 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -123,7 +123,8 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { // === Schema Evolution === // TODO(#6232): enable the two tests after spark streaming engine supports leaf node projection - // for datasource v2 such that we can adopt the two schema changes without refresh the dataframe + // for datasource v2 such that we can adopt the two schema changes without refreshing the + // dataframe "relax nullability: restarting with stale DataFrame should recover", "type widening: restarting with stale DataFrame should recover", diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java index cead0c1c2ae..2ef4750f225 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java @@ -1199,7 +1199,6 @@ private void validateSchemaCompatibilityOnStartup( } // Compare the structural schema of the analysis-time schema and snapshot schema. - // TODO(#5319): add tests for type widening, partition column, & nullability changes. if (!DataType.equalsStructurally(querySchema, snapshotSchema, /* ignoreNullability */ false)) { throw DeltaErrors.streamingSchemaMismatchOnRestart(querySchema, snapshotSchema); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java index ba142f97118..a529b52e507 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java @@ -35,8 +35,6 @@ import org.apache.spark.sql.delta.stats.StatisticsCollection; import org.apache.spark.sql.streaming.StreamingQuery; import org.apache.spark.sql.streaming.StreamingQueryException; -import org.apache.spark.sql.types.DataTypes; -import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.*; import org.junit.jupiter.api.io.TempDir; import scala.Option; @@ -315,22 +313,62 @@ public void testNestedColumnAdditionDetectedOnRestart(@TempDir File deltaTablePa String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); // Create table with struct column: data STRUCT - StructType initialSchema = - DataTypes.createStructType( - Arrays.asList( - DataTypes.createStructField("id", DataTypes.StringType, true), - DataTypes.createStructField( - "data", - DataTypes.createStructType( - Arrays.asList( - DataTypes.createStructField("x", DataTypes.IntegerType, true))), - true))); + spark.sql( + str("CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", tablePath)); + spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); - spark - .createDataFrame(Arrays.asList(RowFactory.create("0", RowFactory.create(1))), initialSchema) - .write() - .format("delta") - .save(tablePath); + // Start streaming and process initial data + File checkpointDir = new File(deltaTablePath, "_checkpoint"); + Dataset streamingDF = spark.readStream().table(dsv2TableRef); + StreamingQuery query = + streamingDF + .writeStream() + .format("noop") + .option("checkpointLocation", checkpointDir.getAbsolutePath()) + .start(); + query.processAllAvailable(); + query.stop(); + + // Evolve struct via ALTER TABLE (metadata-only, no file deletion): + // add nested field y -> data STRUCT + spark.sql(str("ALTER TABLE delta.`%s` ADD COLUMNS (data.y INT)", tablePath)); + + // Restart with stale DataFrame — should fail with schema mismatch + StreamingQueryException ex = + assertThrows( + StreamingQueryException.class, + () -> { + StreamingQuery q = + streamingDF + .writeStream() + .format("noop") + .option("checkpointLocation", checkpointDir.getAbsolutePath()) + .start(); + try { + q.processAllAvailable(); + } finally { + q.stop(); + } + }); + assertInstanceOf(DeltaIllegalStateException.class, ex.cause()); + assertTrue( + ex.getMessage().contains("DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART"), + "Expected DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART but got: " + ex.getMessage()); + } + + // TODO(#6232): v2 source cannot adopt type widening schema change without refreshing the + // dataframe due to the lack of support in spark stream engine. Throw an error at stream + // start time to instruct user. + @Test + public void testNestedTypeWideningDetectedOnRestart(@TempDir File deltaTablePath) + throws Exception { + String tablePath = deltaTablePath.getAbsolutePath(); + String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); + + // Create table with struct column: data STRUCT + spark.sql( + str("CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", tablePath)); + spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); // Start streaming and process initial data File checkpointDir = new File(deltaTablePath, "_checkpoint"); @@ -344,26 +382,70 @@ public void testNestedColumnAdditionDetectedOnRestart(@TempDir File deltaTablePa query.processAllAvailable(); query.stop(); - // Evolve struct: add nested field y -> data STRUCT - StructType evolvedSchema = - DataTypes.createStructType( - Arrays.asList( - DataTypes.createStructField("id", DataTypes.StringType, true), - DataTypes.createStructField( - "data", - DataTypes.createStructType( - Arrays.asList( - DataTypes.createStructField("x", DataTypes.IntegerType, true), - DataTypes.createStructField("y", DataTypes.IntegerType, true))), - true))); - spark - .createDataFrame( - Arrays.asList(RowFactory.create("1", RowFactory.create(2, 3))), evolvedSchema) - .write() - .format("delta") - .mode("overwrite") - .option("overwriteSchema", "true") - .save(tablePath); + // Widen nested field type via ALTER TABLE (metadata-only, no file deletion): + // data.x INT -> data.x BIGINT + spark.sql( + str( + "ALTER TABLE delta.`%s` SET TBLPROPERTIES ('delta.enableTypeWidening' = 'true')", + tablePath)); + spark.sql(str("ALTER TABLE delta.`%s` ALTER COLUMN data.x TYPE BIGINT", tablePath)); + + // Restart with stale DataFrame — should fail with schema mismatch + StreamingQueryException ex = + assertThrows( + StreamingQueryException.class, + () -> { + StreamingQuery q = + streamingDF + .writeStream() + .format("noop") + .option("checkpointLocation", checkpointDir.getAbsolutePath()) + .start(); + try { + q.processAllAvailable(); + } finally { + q.stop(); + } + }); + assertInstanceOf(DeltaIllegalStateException.class, ex.cause()); + assertTrue( + ex.getMessage().contains("DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART"), + "Expected DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART but got: " + ex.getMessage()); + } + + // TODO(#6232): v2 source cannot adopt type widening schema change without refreshing the + // dataframe due to the lack of support in spark stream engine. Throw an error at stream + // start time to instruct user. + @Test + public void testNestedNullabilityRelaxDetectedOnRestart(@TempDir File deltaTablePath) + throws Exception { + String tablePath = deltaTablePath.getAbsolutePath(); + String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); + + // Create table via SQL DDL to preserve the NOT NULL constraint on the nested field. + // DataFrame writes go through ImplicitMetadataOperation which calls schema.asNullable, + // forcing all fields (including nested ones) to nullable — losing the NOT NULL. + spark.sql( + str( + "CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", + tablePath)); + spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); + + // Start streaming and process initial data + File checkpointDir = new File(deltaTablePath, "_checkpoint"); + Dataset streamingDF = spark.readStream().table(dsv2TableRef); + StreamingQuery query = + streamingDF + .writeStream() + .format("noop") + .option("checkpointLocation", checkpointDir.getAbsolutePath()) + .start(); + query.processAllAvailable(); + query.stop(); + + // Relax nullability via ALTER TABLE (metadata-only, no file deletion): + // data.x NOT NULL -> data.x nullable + spark.sql(str("ALTER TABLE delta.`%s` ALTER COLUMN data.x DROP NOT NULL", tablePath)); // Restart with stale DataFrame — should fail with schema mismatch StreamingQueryException ex = From 2568507f56d89b3897e0374a762e8c9da5525e27 Mon Sep 17 00:00:00 2001 From: openinx Date: Tue, 10 Mar 2026 15:43:53 -0700 Subject: [PATCH 05/80] [Spark] Add the DataFrame spark unitycatalog integration tests. (#5837) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Add the DataFrame basic test coverage for preventing future regression. ## How was this patch tested? Do the locally unit tests and remotely integration tests. ## Does this PR introduce _any_ user-facing changes? No. --------- Signed-off-by: openinx --- .../UCDeltaTableDataFrameReadTest.java | 186 ++++++++++++ .../UCDeltaTableDataFrameWriteTest.java | 272 ++++++++++++++++++ .../sparkuctest/UCDeltaTableDeltaAPITest.java | 180 ++++++++++++ .../UCDeltaTableIntegrationBaseTest.java | 19 +- .../io/sparkuctest/UCDeltaTableReadTest.java | 10 - 5 files changed, 655 insertions(+), 12 deletions(-) create mode 100644 spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameReadTest.java create mode 100644 spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java create mode 100644 spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameReadTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameReadTest.java new file mode 100644 index 00000000000..43a101c2ac2 --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameReadTest.java @@ -0,0 +1,186 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import static org.apache.spark.sql.functions.col; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.spark.sql.DataFrameReader; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * DataFrame read test suite for Delta Table operations through Unity Catalog. + * + *

Covers spark.table(), DataFrameReader, time travel, column pruning, and filter. Most tests run + * against both EXTERNAL and MANAGED table types. + */ +public class UCDeltaTableDataFrameReadTest extends UCDeltaTableIntegrationBaseTest { + + @TestAllTableTypes + public void testReadViaSparkTable(TableType tableType) throws Exception { + withNewTable( + "df_read_spark_table", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + assertThat(ids(spark().table(tableName).orderBy("id"))).containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testReadViaDataFrameReader(TableType tableType) throws Exception { + withNewTable( + "df_read_reader", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + assertThat(ids(deltaDFReader().table(tableName).orderBy("id"))).containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testTimeTravelByVersion(TableType tableType) throws Exception { + withNewTable( + "df_time_travel_version", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + long v1 = currentVersion(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + assertThat(ids(deltaDFReader().option("versionAsOf", v1).table(tableName).orderBy("id"))) + .containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testTimeTravelByTimestamp(TableType tableType) throws Exception { + withNewTable( + "df_time_travel_ts", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + String ts = currentTimestamp(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + assertThat( + ids(deltaDFReader().option("timestampAsOf", ts).table(tableName).orderBy("id"))) + .containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testColumnPruning(TableType tableType) throws Exception { + withNewTable( + "df_column_pruning", + "id INT, name STRING, value INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'Alice', 100), (2, 'Bob', 200)", tableName); + List rows = + spark().table(tableName).select("id", "name").orderBy("id").collectAsList(); + assertThat(rows.get(0).schema().fieldNames()).containsExactly("id", "name"); + assertThat(rows.stream().map(r -> r.getInt(0)).collect(Collectors.toList())) + .containsExactly(1, 2); + assertThat(rows.stream().map(r -> r.getString(1)).collect(Collectors.toList())) + .containsExactly("Alice", "Bob"); + }); + } + + @Test + public void testReadViaPath() throws Exception { + withNewTable( + "df_read_via_path", + "id INT", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + String tablePath = + sql("DESCRIBE EXTENDED %s", tableName).stream() + .filter(row -> row.size() >= 2 && "Location".equals(row.get(0))) + .map(row -> row.get(1)) + .findFirst() + .orElseThrow(() -> new AssertionError("Could not retrieve table location")); + Assertions.assertThrows( + Exception.class, + () -> spark().read().format("delta").load(tablePath).collect(), + "Path-based access should fail for managed tables"); + }); + } + + @TestAllTableTypes + public void testChangeDataFeedViaDataFrameAPI(TableType tableType) throws Exception { + withNewTable( + "df_cdf_reader", + "id INT", + null, + tableType, + "'delta.enableChangeDataFeed'='true'", + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + long currentVersion = currentVersion(tableName); + List rows = + deltaDFReader() + .option("readChangeFeed", "true") + .option("startingVersion", currentVersion) + .table(tableName) + .orderBy("id") + .collectAsList(); + assertThat(rows.stream().map(r -> r.getInt(0)).collect(Collectors.toList())) + .containsExactly(4, 5); + }); + } + + @TestAllTableTypes + public void testEmptyTableRead(TableType tableType) throws Exception { + withNewTable( + "df_empty_read", + "id INT", + tableType, + tableName -> assertThat(spark().table(tableName).collectAsList()).isEmpty()); + } + + @TestAllTableTypes + public void testFilter(TableType tableType) throws Exception { + withNewTable( + "df_filter", + "id INT, category STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A'), (4, 'B')", tableName); + assertThat( + ids(spark().table(tableName).filter(col("category").equalTo("A")).orderBy("id"))) + .containsExactly(1, 3); + }); + } + + private DataFrameReader deltaDFReader() { + return spark().read().format("delta"); + } + + private List ids(Dataset df) { + return df.collectAsList().stream().map(r -> r.getInt(0)).collect(Collectors.toList()); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java new file mode 100644 index 00000000000..e4a60b1ec6d --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java @@ -0,0 +1,272 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import static org.apache.spark.sql.functions.col; +import static org.apache.spark.sql.functions.lit; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +/** + * DataFrame write test suite for Delta Table operations through Unity Catalog. + * + *

Covers DataFrame Writer V1 (insertInto, save) and Writer V2 (writeTo) operations. Tests run + * against both EXTERNAL and MANAGED table types. + */ +public class UCDeltaTableDataFrameWriteTest extends UCDeltaTableIntegrationBaseTest { + + // Writer V1: insertInto + + @TestAllTableTypes + public void testInsertIntoAppend(TableType tableType) throws Exception { + withNewTable( + "insert_into_append_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(4, 5).write().mode("append").insertInto(tableName); + check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); + }); + } + + @TestAllTableTypes + public void testInsertIntoOverwrite(TableType tableType) throws Exception { + withNewTable( + "insert_into_overwrite_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(9).write().mode("overwrite").insertInto(tableName); + check(tableName, List.of(row("9"))); + }); + } + + @TestAllTableTypes + public void testInsertIntoReplaceWhere(TableType tableType) throws Exception { + withNewTable( + "insert_into_replace_where_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(9).write().mode("overwrite").option("replaceWhere", "id > 1").insertInto(tableName); + check(tableName, List.of(row("1"), row("9"))); + }); + } + + @TestAllTableTypes + public void testSaveAsTableAppend(TableType tableType) throws Exception { + withNewTable( + "save_as_table_append_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(4, 5).write().format("delta").mode("append").saveAsTable(tableName); + check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); + }); + } + + // TODO: Add saveAsTable overwrite/replaceWhere coverage once UCSingleCatalog supports REPLACE + // TABLE AS SELECT (RTAS). Currently, saveAsTable with mode("overwrite") routes through Spark's + // V2 catalog path as RTAS, which throws UnsupportedOperationException in UCSingleCatalog. + + @Test + public void testSaveByPathBlockedForManagedTable() throws Exception { + withNewTable( + "save_path_blocked_test", + "id INT", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + String tablePath = + sql("DESCRIBE FORMATTED %s", tableName).stream() + .filter(r -> r.size() >= 2 && "Location".equalsIgnoreCase(r.get(0).trim())) + .map(r -> r.get(1).trim()) + .findFirst() + .orElseThrow(); + assertThatThrownBy(() -> intDf(4).write().format("delta").mode("append").save(tablePath)) + .satisfies( + e -> + assertThat(e.getMessage()) + .containsAnyOf( + "Unable to load credentials", + "DELTA_PATH_BASED_ACCESS_TO_CATALOG_MANAGED_TABLE_BLOCKED", + "Path-based access is not allowed")); + check(tableName, List.of(row("1"), row("2"), row("3"))); + }); + } + + // Writer V2: writeTo + + @TestAllTableTypes + public void testWriteToAppend(TableType tableType) throws Exception { + withNewTable( + "write_to_append_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(4, 5).writeTo(tableName).append(); + check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); + }); + } + + @TestAllTableTypes + public void testWriteToOverwrite(TableType tableType) throws Exception { + withNewTable( + "write_to_overwrite_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(9).writeTo(tableName).overwrite(lit(true)); + check(tableName, List.of(row("9"))); + }); + } + + @TestAllTableTypes + public void testWriteToOverwriteWithCondition(TableType tableType) throws Exception { + withNewTable( + "write_to_overwrite_condition_test", + "id INT, category STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A')", tableName); + spark() + .createDataFrame( + List.of(RowFactory.create(9, "A")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("category", DataTypes.StringType)) + .writeTo(tableName) + .overwrite(col("category").equalTo("A")); + check(tableName, List.of(row("2", "B"), row("9", "A"))); + }); + } + + @TestAllTableTypes + public void testWriteToOverwritePartitions(TableType tableType) throws Exception { + withNewTable( + "write_to_overwrite_partitions_test", + "id INT, category STRING", + "category", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'A'), (3, 'B')", tableName); + spark() + .createDataFrame( + List.of(RowFactory.create(9, "A")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("category", DataTypes.StringType)) + .writeTo(tableName) + .overwritePartitions(); + // Only partition 'A' is replaced; partition 'B' remains untouched. + check(tableName, List.of(row("3", "B"), row("9", "A"))); + }); + } + + @Test + public void testWriteToCreateNewManagedTable() throws Exception { + String tableName = fullTableName("write_to_create_test"); + try { + intDf(1, 2) + .writeTo(tableName) + .using("delta") + .tableProperty("delta.feature.catalogManaged", "supported") + .create(); + check(tableName, List.of(row("1"), row("2"))); + } finally { + sql("DROP TABLE IF EXISTS %s", tableName); + } + } + + // TODO: Enable testMergeSchema for UC_REMOTE=true once UC managed tables support schema + // evolution (mergeSchema). Currently, saveAsTable with mergeSchema triggers a schema change + // that requires UC to update the table metadata, which is not yet supported for managed tables. + @TestAllTableTypes + public void testMergeSchema(TableType tableType) throws Exception { + Assumptions.assumeFalse( + isUCRemoteConfigured(), "mergeSchema not yet supported for UC managed tables remotely"); + withNewTable( + "merge_schema_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2)", tableName); + spark() + .createDataFrame( + List.of(RowFactory.create(3, "extra")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("name", DataTypes.StringType)) + .write() + .format("delta") + .mode("append") + .option("mergeSchema", "true") + .saveAsTable(tableName); + check(tableName, List.of(row("1", "null"), row("2", "null"), row("3", "extra"))); + }); + } + + @TestAllTableTypes + public void testWriteToPartitionedTable(TableType tableType) throws Exception { + withNewTable( + "df_partitioned_write_test", + "id INT, category STRING", + "category", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B')", tableName); + spark() + .createDataFrame( + List.of(RowFactory.create(3, "A"), RowFactory.create(4, "B")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("category", DataTypes.StringType)) + .write() + .mode("append") + .insertInto(tableName); + check(tableName, List.of(row("1", "A"), row("2", "B"), row("3", "A"), row("4", "B"))); + }); + } + + private Dataset intDf(Integer... ids) { + return spark() + .createDataFrame( + Arrays.stream(ids).map(RowFactory::create).collect(Collectors.toList()), + new StructType().add("id", DataTypes.IntegerType)); + } + + private static List row(String... values) { + return List.of(values); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java new file mode 100644 index 00000000000..94753132fca --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java @@ -0,0 +1,180 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import static org.apache.spark.sql.functions.col; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.delta.tables.DeltaTable; +import java.util.List; +import java.util.Map; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; + +/** + * Programmatic DeltaTable API test suite for Delta Table operations through Unity Catalog. + * + *

Covers DeltaTable.forName(), update(), delete(), merge(), history(), optimize(), and restore() + * via the io.delta.tables.DeltaTable API. These go through different code paths than SQL + * equivalents tested in UCDeltaTableDMLTest and UCDeltaUtilityTest. Tests run against both EXTERNAL + * and MANAGED table types. + */ +public class UCDeltaTableDeltaAPITest extends UCDeltaTableIntegrationBaseTest { + + @TestAllTableTypes + public void testForNameAndToDF(TableType tableType) throws Exception { + withNewTable( + "dt_api_read", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + assertThat(forName(tableName).toDF().orderBy("id").collectAsList()) + .extracting(r -> r.getInt(0)) + .containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testUpdate(TableType tableType) throws Exception { + withNewTable( + "dt_api_update", + "id INT, status STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'pending'), (2, 'pending'), (3, 'done')", tableName); + forName(tableName).updateExpr("id = 1", Map.of("status", "'processed'")); + check(tableName, List.of(row("1", "processed"), row("2", "pending"), row("3", "done"))); + }); + } + + @TestAllTableTypes + public void testDelete(TableType tableType) throws Exception { + withNewTable( + "dt_api_delete", + "id INT, active BOOLEAN", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, true), (2, false), (3, true)", tableName); + forName(tableName).delete(col("active").equalTo(false)); + check(tableName, List.of(row("1", "true"), row("3", "true"))); + }); + } + + @TestAllTableTypes + public void testMerge(TableType tableType) throws Exception { + withNewTable( + "dt_api_merge", + "id INT, value STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'old1'), (2, 'old2')", tableName); + Dataset source = + spark() + .createDataFrame( + List.of(RowFactory.create(2, "updated2"), RowFactory.create(3, "new3")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("value", DataTypes.StringType)); + forName(tableName) + .as("target") + .merge(source.as("source"), "target.id = source.id") + .whenMatched() + .updateExpr(Map.of("value", "source.value")) + .whenNotMatched() + .insertExpr(Map.of("id", "source.id", "value", "source.value")) + .execute(); + check(tableName, List.of(row("1", "old1"), row("2", "updated2"), row("3", "new3"))); + }); + } + + @TestAllTableTypes + public void testHistory(TableType tableType) throws Exception { + withNewTable( + "dt_api_history", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1)", tableName); + sql("INSERT INTO %s VALUES (2)", tableName); + // CREATE TABLE (v0) + 2 INSERTs (v1, v2) + assertThat(forName(tableName).history().collectAsList()).hasSize(3); + }); + } + + @TestAllTableTypes + public void testOptimize(TableType tableType) throws Exception { + withNewTable( + "dt_api_optimize", + "id INT, category STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A')", tableName); + if (tableType == TableType.MANAGED) { + assertThatThrownBy(() -> forName(tableName).optimize().executeCompaction()) + .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION"); + } else { + assertThat(forName(tableName).optimize().executeCompaction().collectAsList()) + .isNotEmpty(); + assertThat(forName(tableName).optimize().executeZOrderBy("category").collectAsList()) + .isNotEmpty(); + } + }); + } + + @TestAllTableTypes + public void testRestoreToVersion(TableType tableType) throws Exception { + withNewTable( + "dt_api_restore_version", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + long v1 = currentVersion(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + forName(tableName).restoreToVersion(v1); + check(tableName, List.of(row("1"), row("2"), row("3"))); + }); + } + + @TestAllTableTypes + public void testRestoreToTimestamp(TableType tableType) throws Exception { + withNewTable( + "dt_api_restore_ts", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + String ts = currentTimestamp(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + forName(tableName).restoreToTimestamp(ts); + check(tableName, List.of(row("1"), row("2"), row("3"))); + }); + } + + private DeltaTable forName(String tableName) { + return DeltaTable.forName(spark(), tableName); + } + + private static List row(String... values) { + return List.of(values); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java index 05533a0a2c2..0027003f083 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java @@ -209,8 +209,7 @@ protected void withNewTable( String tableProperties, TestCode testCode) throws Exception { - UnityCatalogInfo uc = unityCatalogInfo(); - String fullTableName = uc.catalogName() + "." + uc.schemaName() + "." + tableName; + String fullTableName = fullTableName(tableName); // Create th partition cause. StringBuilder partitionCause = new StringBuilder(); @@ -304,6 +303,22 @@ protected void withNewTable( withNewTable(tableName, tableSchema, null, tableType, testCode); } + /** Returns the fully qualified table name for a given simple table name. */ + protected String fullTableName(String simpleName) { + UnityCatalogInfo uc = unityCatalogInfo(); + return uc.catalogName() + "." + uc.schemaName() + "." + simpleName; + } + + /** Returns the current (latest) version of the table. */ + protected long currentVersion(String tableName) { + return Long.parseLong(sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(0)); + } + + /** Returns the timestamp of the current (latest) version. */ + protected String currentTimestamp(String tableName) { + return sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(1); + } + /** Functional interface for test code that takes a temporary directory. */ @FunctionalInterface protected interface TempDirCode { diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java index 29dad0c7fd5..66a9169ff35 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java @@ -141,14 +141,4 @@ private void check(List> actual, List> expected) { String.format("Query results do not match.\nExpected: %s\nActual: %s", expected, actual)); } } - - /** Returns the current (latest) version of the table. */ - private long currentVersion(String tableName) { - return Long.parseLong(sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(0)); - } - - /** Returns the timestamp of the current (latest) version. */ - private String currentTimestamp(String tableName) { - return sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(1); - } } From 647554123f9e8c9fbd722bb31d2a10c9db90fa63 Mon Sep 17 00:00:00 2001 From: Eames Trinh Date: Tue, 10 Mar 2026 23:58:28 +0100 Subject: [PATCH 06/80] Delta Error Followup for Logging all operations that try to illegally repartition the table's columns (#6238) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description - Updated partition column change tests in `OptimisticTransactionSuite` to use `checkError` instead of calling `ex.getErrorClass` directly - Each `checkError` call now verifies the error condition (`DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE`), SQLSTATE (`42P10`), and named parameter values (`operation`, `oldPartitionColumns`, `newPartitionColumns`) - Added expected operation name to the parameterized blocked-operations test cases to support per-case parameter verification - Collapsed the `DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE` error message from three lines into a single line in `delta-error-classes.json`, `databricks-error-classes.json`, and `delta-error-classes-data-classification.json` (the multi-line split was not justified as the column lists render as single-line comma-separated strings) ## How was this patch tested? No new tests added. Existing tests in `OptimisticTransactionSuite` covering partition column change validation were updated to use `checkError` as required by the error message review guidelines. ## Does this PR introduce _any_ user-facing changes? N/A --- .../resources/error/delta-error-classes.json | 4 +- .../delta/OptimisticTransactionSuite.scala | 70 ++++++++++++------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/spark/src/main/resources/error/delta-error-classes.json b/spark/src/main/resources/error/delta-error-classes.json index 1ba811544c2..1ca30d33066 100644 --- a/spark/src/main/resources/error/delta-error-classes.json +++ b/spark/src/main/resources/error/delta-error-classes.json @@ -3152,9 +3152,7 @@ }, "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE" : { "message" : [ - "Cannot change partition columns during operation.", - "- Old partition columns: .", - "- New partition columns: ." + "Cannot change partition columns during operation (old: , new: )." ], "sqlState" : "42P10" }, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala index 87fd5c2ae62..b723e9fc54d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala @@ -1198,16 +1198,22 @@ class OptimisticTransactionSuite val logRecords = Log4jUsageLogger.track { exceptionClassOpt match { - case Some(exceptionClass) => - val ex = intercept[Throwable] { - txn.commit( - Seq(newMetadata), - DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) + case Some(_) => + checkError( + intercept[DeltaAnalysisException] { + txn.commit( + Seq(newMetadata), + DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) + ) + }, + condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", + sqlState = "42P10", + parameters = Map( + "operation" -> "UPDATE", + "oldPartitionColumns" -> "col1", + "newPartitionColumns" -> "col2" ) - } - assert(ex.getClass == exceptionClass) - assert(ex.asInstanceOf[DeltaAnalysisException].getErrorClass == - "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE") + ) case None => // Should succeed without throwing txn.commit( @@ -1245,41 +1251,49 @@ class OptimisticTransactionSuite val txn = deltaLog.startTransaction() val newMetadata = txn.metadata.copy(partitionColumns = Seq("col2")) - val ex = intercept[DeltaAnalysisException] { - txn.commit( - Seq(newMetadata), - DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) + checkError( + intercept[DeltaAnalysisException] { + txn.commit( + Seq(newMetadata), + DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) + ) + }, + condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", + sqlState = "42P10", + parameters = Map( + "operation" -> "UPDATE", + "oldPartitionColumns" -> "col1", + "newPartitionColumns" -> "col2" ) - } - assert(ex.getErrorClass == "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE") + ) assertPartitionColumns(tablePath, Seq("col1")) } } Seq( // Recreation of running DFv1 .save() overwrite changing partition cols. - ("blocked for Write(Overwrite, partitionBy)", + ("blocked for Write(Overwrite, partitionBy)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")))), // Recreation of running DFv1 .save() replaceWhere changing partition cols. - ("blocked for Write(Overwrite, partitionBy, predicate)", + ("blocked for Write(Overwrite, partitionBy, predicate)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")), predicate = Some("col1=0"))), // Recreation of running DFv1 .save() DPO changing partition cols. - ("blocked for Write(Overwrite, partitionBy, DPO)", + ("blocked for Write(Overwrite, partitionBy, DPO)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")), isDynamicPartitionOverwrite = Some(true))), // Recreation of running DFv1 .saveAsTable() overwrite changing partition cols. - ("blocked for ReplaceTable(isV1SaveAsTableOverwrite=true)", + ("blocked for ReplaceTable(isV1SaveAsTableOverwrite=true)", "CREATE OR REPLACE TABLE AS SELECT", (newMeta: Metadata) => DeltaOperations.ReplaceTable( metadata = newMeta, isManaged = true, orCreate = true, asSelect = true, isV1SaveAsTableOverwrite = Some(true))) - ).foreach { case (testSuffix, mkOp) => + ).foreach { case (testSuffix, expectedOpName, mkOp) => test(s"partition column changes $testSuffix") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath @@ -1289,10 +1303,18 @@ class OptimisticTransactionSuite val txn = deltaLog.startTransaction() val newMetadata = txn.metadata.copy(partitionColumns = Seq("col2")) - val ex = intercept[DeltaAnalysisException] { - txn.commit(Seq(newMetadata), mkOp(newMetadata)) - } - assert(ex.getErrorClass == "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE") + checkError( + intercept[DeltaAnalysisException] { + txn.commit(Seq(newMetadata), mkOp(newMetadata)) + }, + condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", + sqlState = "42P10", + parameters = Map( + "operation" -> expectedOpName, + "oldPartitionColumns" -> "col1", + "newPartitionColumns" -> "col2" + ) + ) assertPartitionColumns(tablePath, Seq("col1")) } } From e6e525e5ada8ee0c5f20e007c0b48d84821ad2f5 Mon Sep 17 00:00:00 2001 From: Xin Huang <42597328+huan233usc@users.noreply.github.com> Date: Tue, 10 Mar 2026 20:34:51 -0700 Subject: [PATCH 07/80] [Kernel-Spark] Phase 2: Vectorized reader support for Deletion Vectors (#5775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🥞 Stacked PR Use this [link](https://github.com/delta-io/delta/pull/5775/files) to review incremental changes. - [**stack/dv_pr3_phase2_vectorized**](https://github.com/delta-io/delta/pull/5775) [[Files changed](https://github.com/delta-io/delta/pull/5775/files)] - [stack/dv_pr4_phase3_file_splitting](https://github.com/delta-io/delta/pull/5776) [[Files changed](https://github.com/delta-io/delta/pull/5776/files/463011af2fae489f7f3fe8c39fe86acbba06277b..0e534ed9c0257735523532c8c6f8e15ea33ae114)] --------- #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [x] Kernel - [ ] Other (fill in here) ## Description Enable vectorized (columnar batch) reading for DV tables in the Spark V2 connector. ### Changes: - `ColumnVectorWithFilter`: Wraps `ColumnVector` with row ID mapping for zero-copy filtering (follows Iceberg pattern) - `DeletionVectorReadFunction`: Extended to handle `ColumnarBatch` in addition to `InternalRow` - Enable vectorized reader for DV tables (remove `!isTableSupportDv` condition) ### How it works: 1. For `ColumnarBatch`: Build live row mapping, wrap columns with `ColumnVectorWithFilter` 2. For `InternalRow`: Filter deleted rows, project using `ProjectingInternalRow` ## How was this patch tested? - `ColumnVectorWithFilterTest`: Unit tests for various data types and filtering scenarios - `DeletionVectorVectorizedReaderTest`: Integration tests verifying vectorized path is used - All DV golden table tests pass with vectorized reading ## Does this PR introduce _any_ user-facing changes? No --------- Co-authored-by: Cursor --- .../ColumnVectorWithFilter.java | 152 +++++++++++++++ .../DeletionVectorReadFunction.java | 91 ++++++++- .../internal/v2/utils/PartitionUtils.java | 6 +- .../internal/v2/InternalRowTestUtils.java | 29 +++ .../ColumnVectorWithFilterTest.java | 184 ++++++++++++++++++ .../DeletionVectorReadFunctionTest.java | 111 ++++++++++- 6 files changed, 555 insertions(+), 18 deletions(-) create mode 100644 spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilter.java create mode 100644 spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilterTest.java diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilter.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilter.java new file mode 100644 index 00000000000..cc8c64a77ea --- /dev/null +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilter.java @@ -0,0 +1,152 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.read.deletionvector; + +import org.apache.spark.sql.types.Decimal; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.apache.spark.sql.vectorized.ColumnarArray; +import org.apache.spark.sql.vectorized.ColumnarMap; +import org.apache.spark.unsafe.types.UTF8String; + +/** + * A column vector that applies row-level filtering using a row ID mapping. + * + *

Wraps an existing column vector and remaps row indices during data access, effectively + * filtering the original data to only expose the live subset of rows without copying data. + * + *

Follows Apache Iceberg's ColumnVectorWithFilter pattern. + */ +public class ColumnVectorWithFilter extends ColumnVector { + private final ColumnVector delegate; + private final int[] rowIdMapping; + private volatile ColumnVectorWithFilter[] children = null; + + public ColumnVectorWithFilter(ColumnVector delegate, int[] rowIdMapping) { + super(delegate.dataType()); + this.delegate = delegate; + this.rowIdMapping = rowIdMapping; + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public boolean hasNull() { + return delegate.hasNull(); + } + + @Override + public int numNulls() { + // Computing the actual number of nulls with rowIdMapping is expensive. + // It is OK to overestimate and return the number of nulls in the original vector. + return delegate.numNulls(); + } + + @Override + public boolean isNullAt(int rowId) { + return delegate.isNullAt(rowIdMapping[rowId]); + } + + @Override + public boolean getBoolean(int rowId) { + return delegate.getBoolean(rowIdMapping[rowId]); + } + + @Override + public byte getByte(int rowId) { + return delegate.getByte(rowIdMapping[rowId]); + } + + @Override + public short getShort(int rowId) { + return delegate.getShort(rowIdMapping[rowId]); + } + + @Override + public int getInt(int rowId) { + return delegate.getInt(rowIdMapping[rowId]); + } + + @Override + public long getLong(int rowId) { + return delegate.getLong(rowIdMapping[rowId]); + } + + @Override + public float getFloat(int rowId) { + return delegate.getFloat(rowIdMapping[rowId]); + } + + @Override + public double getDouble(int rowId) { + return delegate.getDouble(rowIdMapping[rowId]); + } + + @Override + public ColumnarArray getArray(int rowId) { + return delegate.getArray(rowIdMapping[rowId]); + } + + @Override + public ColumnarMap getMap(int rowId) { + return delegate.getMap(rowIdMapping[rowId]); + } + + @Override + public Decimal getDecimal(int rowId, int precision, int scale) { + return delegate.getDecimal(rowIdMapping[rowId], precision, scale); + } + + @Override + public UTF8String getUTF8String(int rowId) { + return delegate.getUTF8String(rowIdMapping[rowId]); + } + + @Override + public byte[] getBinary(int rowId) { + return delegate.getBinary(rowIdMapping[rowId]); + } + + /** + * Returns the child vector at the given ordinal, wrapped with the same row ID mapping. + * + *

Uses double-checked locking for thread-safe lazy initialization: the volatile {@code + * children} field allows lock-free reads after initialization, while the synchronized block + * ensures at-most-once creation. All children are created eagerly within the lock to avoid + * per-element races. + */ + @Override + public ColumnVector getChild(int ordinal) { + if (children == null) { + synchronized (this) { + if (children == null) { + // Eagerly create all children to avoid race condition on children[ordinal] access + StructType structType = (StructType) dataType(); + ColumnVectorWithFilter[] newChildren = + new ColumnVectorWithFilter[structType.fields().length]; + for (int i = 0; i < newChildren.length; i++) { + newChildren[i] = new ColumnVectorWithFilter(delegate.getChild(i), rowIdMapping); + } + children = newChildren; + } + } + } + return children[ordinal]; + } +} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java index 3a0f3718400..77e93e80dc9 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java @@ -17,9 +17,12 @@ import io.delta.spark.internal.v2.utils.CloseableIterator; import java.io.Serializable; +import java.util.Arrays; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.ProjectingInternalRow; import org.apache.spark.sql.execution.datasources.PartitionedFile; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.apache.spark.sql.vectorized.ColumnarBatch; import scala.Function1; import scala.collection.Iterator; import scala.runtime.AbstractFunction1; @@ -37,6 +40,9 @@ * *

The returned iterator implements {@link java.io.Closeable} to ensure proper resource cleanup * of the underlying Parquet reader, even when the iterator is not fully consumed. + * + *

The reader mode (row-based vs vectorized) is determined once at scan planning time and does + * not change mid-stream. See {@link #wrap}. */ public class DeletionVectorReadFunction extends AbstractFunction1> implements Serializable { @@ -48,27 +54,34 @@ public class DeletionVectorReadFunction private final Function1> baseReadFunc; private final DeletionVectorSchemaContext dvSchemaContext; + private final boolean isVectorizedReader; private DeletionVectorReadFunction( Function1> baseReadFunc, - DeletionVectorSchemaContext dvSchemaContext) { + DeletionVectorSchemaContext dvSchemaContext, + boolean isVectorizedReader) { this.baseReadFunc = baseReadFunc; this.dvSchemaContext = dvSchemaContext; + this.isVectorizedReader = isVectorizedReader; } @Override public Iterator apply(PartitionedFile file) { + if (isVectorizedReader) { + return applyBatch(file); + } else { + return applyRow(file); + } + } + + /** Row-based: filter deleted rows and project out the DV column. */ + private Iterator applyRow(PartitionedFile file) { int dvColumnIndex = dvSchemaContext.getDvColumnIndex(); - // Use pre-computed ordinals from DeletionVectorSchemaContext. ProjectingInternalRow projection = ProjectingInternalRow.apply( dvSchemaContext.getOutputSchema(), dvSchemaContext.getOutputColumnOrdinals()); - // Wrap the base iterator as CloseableIterator to preserve close() through filter/map. - // This ensures proper resource cleanup even when the iterator is not fully consumed. - Iterator baseIterator = baseReadFunc.apply(file); - - return CloseableIterator.wrap(baseIterator) + return CloseableIterator.wrap(baseReadFunc.apply(file)) .filterCloseable(row -> row.getByte(dvColumnIndex) == ROW_NOT_DELETED) .mapCloseable( row -> { @@ -77,10 +90,70 @@ public Iterator apply(PartitionedFile file) { }); } + /** Vectorized: filter active rows and project out the DV column from each batch. */ + @SuppressWarnings("unchecked") + private Iterator applyBatch(PartitionedFile file) { + int dvColumnIndex = dvSchemaContext.getDvColumnIndex(); + + // In vectorized mode Spark passes ColumnarBatch via type erasure as InternalRow. + Iterator baseIterator = (Iterator) (Iterator) baseReadFunc.apply(file); + return (Iterator) + (Iterator) + CloseableIterator.wrap(baseIterator) + .mapCloseable( + item -> { + if (item instanceof ColumnarBatch) { + return filterAndProjectBatch((ColumnarBatch) item, dvColumnIndex); + } + throw new IllegalStateException( + "Expected ColumnarBatch when vectorized reader is enabled, but got: " + + item.getClass()); + }); + } + + /** Filter active rows and project out the DV column from a ColumnarBatch. */ + private static ColumnarBatch filterAndProjectBatch(ColumnarBatch batch, int dvColumnIndex) { + int[] activeRows = findActiveRows(batch, dvColumnIndex); + ColumnVector[] filteredColumns = buildFilteredColumns(batch, dvColumnIndex, activeRows); + return new ColumnarBatch(filteredColumns, activeRows.length); + } + + /** + * Build projected output columns by dropping the DV column and applying active row mapping. The + * excluded DV column's lifecycle is managed by the base Spark vectorized reader, not by us. + */ + private static ColumnVector[] buildFilteredColumns( + ColumnarBatch batch, int dvColumnIndex, int[] activeRows) { + ColumnVector[] filteredColumns = new ColumnVector[batch.numCols() - 1]; + int outputIndex = 0; + for (int inputIndex = 0; inputIndex < batch.numCols(); inputIndex++) { + if (inputIndex == dvColumnIndex) { + continue; + } + filteredColumns[outputIndex++] = + new ColumnVectorWithFilter(batch.column(inputIndex), activeRows); + } + return filteredColumns; + } + + /** Find indices of rows where DV column is 0 (not deleted). */ + private static int[] findActiveRows(ColumnarBatch batch, int dvColumnIndex) { + ColumnVector dvColumn = batch.column(dvColumnIndex); + int[] temp = new int[batch.numRows()]; + int count = 0; + for (int i = 0; i < batch.numRows(); i++) { + if (dvColumn.getByte(i) == ROW_NOT_DELETED) { + temp[count++] = i; + } + } + return Arrays.copyOf(temp, count); + } + /** Factory method to wrap a reader function with DV filtering. */ public static DeletionVectorReadFunction wrap( Function1> baseReadFunc, - DeletionVectorSchemaContext dvSchemaContext) { - return new DeletionVectorReadFunction(baseReadFunc, dvSchemaContext); + DeletionVectorSchemaContext dvSchemaContext, + boolean isVectorizedReader) { + return new DeletionVectorReadFunction(baseReadFunc, dvSchemaContext, isVectorizedReader); } } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java index 6dc0b7b3193..f88a3c57223 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java @@ -211,9 +211,8 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( .map(DeletionVectorSchemaContext::getSchemaWithDvColumn) .orElse(readDataSchema); - // TODO(https://github.com/delta-io/delta/issues/5859): Enable vectorized reader for DV tables boolean enableVectorizedReader = - !isTableSupportDv && ParquetUtils.isBatchReadSupportedForSchema(sqlConf, readDataSchema); + ParquetUtils.isBatchReadSupportedForSchema(sqlConf, finalReadDataSchema); scala.collection.immutable.Map optionsWithVectorizedReading = scalaOptions.$plus( new Tuple2<>( @@ -249,7 +248,8 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( // Wrap reader to filter deleted rows and remove internal columns if DV is enabled. if (dvSchemaContext.isPresent()) { - readFunc = DeletionVectorReadFunction.wrap(readFunc, dvSchemaContext.get()); + readFunc = + DeletionVectorReadFunction.wrap(readFunc, dvSchemaContext.get(), enableVectorizedReader); } return new SparkReaderFactory(readFunc, enableVectorizedReader); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java index 222908952b4..f24e7ba636d 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java @@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.execution.datasources.PartitionedFile; +import org.apache.spark.sql.vectorized.ColumnarBatch; import org.apache.spark.unsafe.types.UTF8String; import scala.Function1; import scala.collection.Iterator; @@ -48,6 +49,23 @@ public Iterator apply(PartitionedFile file) { }; } + /** + * Create a mock reader that returns ColumnarBatch objects through Iterator<InternalRow>. + * + *

Mimics Spark vectorized mode where ColumnarBatch is passed via type erasure. + */ + @SuppressWarnings("unchecked") + public static Function1> mockBatchReader( + List batches) { + return new AbstractFunction1>() { + @Override + public Iterator apply(PartitionedFile file) { + return (Iterator) + (Iterator) CollectionConverters.asScala(batches.iterator()); + } + }; + } + /** Collect all rows from iterator into a list, copying each row. */ public static List collectRows(Iterator iter) { List result = new ArrayList<>(); @@ -57,6 +75,17 @@ public static List collectRows(Iterator iter) { return result; } + /** Collect all ColumnarBatch objects from an iterator (for vectorized mode testing). */ + @SuppressWarnings("unchecked") + public static List collectBatches(Iterator iter) { + Iterator objectIter = (Iterator) (Iterator) iter; + List result = new ArrayList<>(); + while (objectIter.hasNext()) { + result.add((ColumnarBatch) objectIter.next()); + } + return result; + } + /** Create an InternalRow from values. Strings are converted to UTF8String. */ public static InternalRow row(Object... values) { Object[] converted = new Object[values.length]; diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilterTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilterTest.java new file mode 100644 index 00000000000..552e827296a --- /dev/null +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilterTest.java @@ -0,0 +1,184 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.read.deletionvector; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.function.IntFunction; +import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector; +import org.apache.spark.sql.execution.vectorized.WritableColumnVector; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.junit.jupiter.api.Test; + +public class ColumnVectorWithFilterTest { + + @Test + void testIntegerColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(5, DataTypes.IntegerType)) { + for (int i = 0; i < 5; i++) { + delegate.putInt(i, (i + 1) * 10); // [10, 20, 30, 40, 50] + } + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {1, 3}); + + assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {20, 40}); + } + } + + @Test + void testLongColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.LongType)) { + for (int i = 0; i < 4; i++) { + delegate.putLong(i, (i + 1) * 100L); + } + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); + + assertMappedValues(mappedVector, mappedVector::getLong, new Long[] {100L, 300L}); + } + } + + @Test + void testDoubleColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.DoubleType)) { + delegate.putDouble(0, 1.1); + delegate.putDouble(1, 2.2); + delegate.putDouble(2, 3.3); + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); + + assertMappedValues(mappedVector, mappedVector::getDouble, new Double[] {1.1, 3.3}); + } + } + + @Test + void testBooleanColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.BooleanType)) { + delegate.putBoolean(0, true); + delegate.putBoolean(1, false); + delegate.putBoolean(2, true); + delegate.putBoolean(3, false); + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {1, 2}); + + assertMappedValues(mappedVector, mappedVector::getBoolean, new Boolean[] {false, true}); + } + } + + @Test + void testStringColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.StringType)) { + delegate.putByteArray(0, "alice".getBytes()); + delegate.putByteArray(1, "bob".getBytes()); + delegate.putByteArray(2, "charlie".getBytes()); + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {2}); + + assertMappedValues( + mappedVector, i -> mappedVector.getUTF8String(i).toString(), new String[] {"charlie"}); + } + } + + @Test + void testStructColumnVector() { + StructType structType = + new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); + try (WritableColumnVector delegate = new OnHeapColumnVector(3, structType)) { + WritableColumnVector idChild = (WritableColumnVector) delegate.getChild(0); + for (int i = 0; i < 3; i++) { + idChild.putInt(i, i + 1); // [1, 2, 3] + } + WritableColumnVector nameChild = (WritableColumnVector) delegate.getChild(1); + nameChild.putByteArray(0, "a".getBytes()); + nameChild.putByteArray(1, "b".getBytes()); + nameChild.putByteArray(2, "c".getBytes()); + + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); + + ColumnVector mappedIdColumn = mappedVector.getChild(0); + ColumnVector mappedNameColumn = mappedVector.getChild(1); + + assertMappedValues(mappedIdColumn, mappedIdColumn::getInt, new Integer[] {1, 3}); + assertMappedValues( + mappedNameColumn, + i -> mappedNameColumn.getUTF8String(i).toString(), + new String[] {"a", "c"}); + } + } + + @Test + void testNullColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.IntegerType)) { + // original rows: [10, null, 30, null] + delegate.putInt(0, 10); + delegate.putNull(1); + delegate.putInt(2, 30); + delegate.putNull(3); + // selected rows (in order): [3, 0, 1] => [null, 10, null] + ColumnVectorWithFilter mappedVector = + new ColumnVectorWithFilter(delegate, new int[] {3, 0, 1}); + + assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {null, 10, null}); + } + } + + @Test + void testAllNullColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { + delegate.putNull(0); + delegate.putNull(1); + delegate.putNull(2); + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {2, 0}); + + assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {null, null}); + } + } + + @Test + void testNoneSelectColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { + delegate.putInt(0, 10); + delegate.putInt(1, 20); + delegate.putInt(2, 30); + + ColumnVectorWithFilter noRowsSelected = new ColumnVectorWithFilter(delegate, new int[] {}); + assertEquals(DataTypes.IntegerType, noRowsSelected.dataType()); + } + } + + @Test + void testAllSelectColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { + delegate.putInt(0, 10); + delegate.putInt(1, 20); + delegate.putInt(2, 30); + + // all select + ColumnVectorWithFilter allRowsSelected = + new ColumnVectorWithFilter(delegate, new int[] {0, 1, 2}); + assertMappedValues(allRowsSelected, allRowsSelected::getInt, new Integer[] {10, 20, 30}); + } + } + + private static void assertMappedValues( + ColumnVector vector, IntFunction getter, T[] expected) { + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(vector.isNullAt(i), "Expected null at mapped row " + i); + } else { + assertFalse(vector.isNullAt(i), "Expected non-null at mapped row " + i); + assertEquals(expected[i], getter.apply(i), "Mismatch at mapped row " + i); + } + } + } +} diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java index 1747dc96da0..5c914ac4d2f 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java @@ -16,11 +16,17 @@ package io.delta.spark.internal.v2.read.deletionvector; import static io.delta.spark.internal.v2.InternalRowTestUtils.*; +import static org.junit.jupiter.api.Assertions.*; +import java.nio.charset.StandardCharsets; import java.util.List; import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector; +import org.apache.spark.sql.execution.vectorized.WritableColumnVector; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.apache.spark.sql.vectorized.ColumnarBatch; import org.junit.jupiter.api.Test; public class DeletionVectorReadFunctionTest { @@ -29,6 +35,8 @@ public class DeletionVectorReadFunctionTest { new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); private static final StructType PARTITION_SCHEMA = new StructType(); + // ===== Row-based tests ===== + @Test public void testFilterDeletedRowsAndProjectRemovesDvColumn() { // Input: 3 rows, middle one is deleted. @@ -41,9 +49,9 @@ public void testFilterDeletedRowsAndProjectRemovesDvColumn() { DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = - DeletionVectorReadFunction.wrap(mockReader(inputRows), context); + DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); - List result = collectRows(readFunc.apply(null)); + List result = collectRows(readFunc.apply(/* file= */ null)); // Verify filtered and projected output (DV column removed, deleted row filtered). assertRowsEquals(result, List.of(row(1, "alice"), row(3, "charlie"))); @@ -57,9 +65,9 @@ public void testAllRowsDeleted() { DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = - DeletionVectorReadFunction.wrap(mockReader(inputRows), context); + DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); - List result = collectRows(readFunc.apply(null)); + List result = collectRows(readFunc.apply(/* file= */ null)); assertRowsEquals(result, List.of()); } @@ -72,10 +80,101 @@ public void testNoRowsDeleted() { DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = - DeletionVectorReadFunction.wrap(mockReader(inputRows), context); + DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); - List result = collectRows(readFunc.apply(null)); + List result = collectRows(readFunc.apply(/* file= */ null)); assertRowsEquals(result, List.of(row(1, "alice"), row(2, "bob"), row(3, "charlie"))); } + + // ===== ColumnarBatch (vectorized) tests ===== + + @Test + public void testBatchFilterDeletedRowsAndProjectRemovesDvColumn() { + // 3 rows: row 1 deleted, rows 0 and 2 kept. + ColumnarBatch inputBatch = createBatch(new int[] {1, 2, 3}, new byte[] {0, 1, 0}); + List result = runBatchRead(inputBatch); + + assertEquals(1, result.size()); + // Filtered: row 0 -> original 0 (id=1), row 1 -> original 2 (id=3) + assertBatchRows(result.get(0), new int[] {1, 3}, new String[] {"name_0", "name_2"}); + } + + @Test + public void testBatchAllRowsDeleted() { + ColumnarBatch inputBatch = createBatch(new int[] {1, 2}, new byte[] {1, 1}); + List result = runBatchRead(inputBatch); + + assertEquals(1, result.size()); + assertBatchRows(result.get(0), new int[] {}, new String[] {}); + } + + @Test + public void testBatchNoRowsDeleted() { + ColumnarBatch inputBatch = createBatch(new int[] {1, 2, 3}, new byte[] {0, 0, 0}); + List result = runBatchRead(inputBatch); + + assertEquals(1, result.size()); + assertBatchRows( + result.get(0), new int[] {1, 2, 3}, new String[] {"name_0", "name_1", "name_2"}); + } + + @Test + public void testBatchMultipleBatchesWithDifferentBatch() { + ColumnarBatch allDeleted = createBatch(new int[] {1, 2, 3}, new byte[] {1, 1, 1}); + ColumnarBatch mixed = createBatch(new int[] {4, 5, 6}, new byte[] {0, 1, 0}); + ColumnarBatch allLive = createBatch(new int[] {7}, new byte[] {0}); + + // Ordering 1: allDeleted, mixed, allLive + List result1 = runBatchRead(allDeleted, mixed, allLive); + assertEquals(3, result1.size()); + assertBatchRows(result1.get(0), new int[] {}, new String[] {}); + assertBatchRows(result1.get(1), new int[] {4, 6}, new String[] {"name_0", "name_2"}); + assertBatchRows(result1.get(2), new int[] {7}, new String[] {"name_0"}); + + // Ordering 2: allLive, allDeleted, mixed + List result2 = runBatchRead(allLive, allDeleted, mixed); + assertEquals(3, result2.size()); + assertBatchRows(result2.get(0), new int[] {7}, new String[] {"name_0"}); + assertBatchRows(result2.get(1), new int[] {}, new String[] {}); + assertBatchRows(result2.get(2), new int[] {4, 6}, new String[] {"name_0", "name_2"}); + } + + private List runBatchRead(ColumnarBatch... inputBatches) { + DeletionVectorSchemaContext context = + new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); + DeletionVectorReadFunction readFunc = + DeletionVectorReadFunction.wrap(mockBatchReader(List.of(inputBatches)), context, true); + return collectBatches(readFunc.apply(/* file= */ null)); + } + + private static void assertBatchRows( + ColumnarBatch batch, int[] expectedIds, String[] expectedNames) { + assertEquals(expectedIds.length, expectedNames.length, "Expected id/name lengths must match"); + assertEquals(expectedIds.length, batch.numRows(), "Unexpected number of filtered rows"); + for (int i = 0; i < expectedIds.length; i++) { + assertEquals(expectedIds[i], batch.column(0).getInt(i)); + assertEquals(expectedNames[i], batch.column(1).getUTF8String(i).toString()); + } + } + + /** + * Creates a ColumnarBatch with columns [id (int), name (string), is_row_deleted (byte)]. + * + *

Name values are auto-generated as "name_0", "name_1", etc. + */ + private static ColumnarBatch createBatch(int[] ids, byte[] deletionVector) { + int numRows = ids.length; + WritableColumnVector idCol = new OnHeapColumnVector(numRows, DataTypes.IntegerType); + WritableColumnVector nameCol = new OnHeapColumnVector(numRows, DataTypes.StringType); + WritableColumnVector dvCol = new OnHeapColumnVector(numRows, DataTypes.ByteType); + + for (int i = 0; i < numRows; i++) { + idCol.putInt(i, ids[i]); + nameCol.putByteArray(i, ("name_" + i).getBytes(StandardCharsets.UTF_8)); + dvCol.putByte(i, deletionVector[i]); + } + + return new ColumnarBatch(new ColumnVector[] {idCol, nameCol, dvCol}, numRows); + } } From 5aa13204a790e5c92dd3ca5e61b35c6ebf84e789 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 11 Mar 2026 11:39:40 +0800 Subject: [PATCH 08/80] [Spark] Add StringStartsWith filter pushdown to ExpressionUtils (#6104) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Spark's StringStartsWith(attribute, value) filter was not converted to a Kernel predicate, meaning WHERE col LIKE 'prefix%' queries had no filter pushdown in the V2 connector. The Kernel's STARTS_WITH predicate maps directly: STARTS_WITH(column, string_literal). ## How was this patch tested? ## Does this PR introduce _any_ user-facing changes? No --------- Co-authored-by: Claude Sonnet 4.6 --- .../internal/v2/utils/ExpressionUtils.java | 7 + .../v2/read/SparkGoldenTableTest.java | 60 ++++++-- .../v2/read/SparkScanBuilderTest.java | 141 ++++++++++++------ .../v2/utils/ExpressionUtilsTest.java | 28 ++++ 4 files changed, 184 insertions(+), 52 deletions(-) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java index 7343834ca27..4753c650f51 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java @@ -57,6 +57,7 @@ public final class ExpressionUtils { *

  • Null tests: IsNull, IsNotNull *
  • Null-safe comparison: EqualNullSafe *
  • Logical operators: And, Or, Not + *
  • String prefix: StringStartsWith * * * @param filter the Spark SQL filter to convert @@ -130,6 +131,12 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( return new ConvertedPredicate( Optional.of(new Predicate("IS_NOT_NULL", kernelColumn(f.attribute())))); } + if (filter instanceof StringStartsWith) { + StringStartsWith f = (StringStartsWith) filter; + return new ConvertedPredicate( + convertValueToKernelLiteral(f.value()) + .map(l -> new Predicate("STARTS_WITH", kernelColumn(f.attribute()), l))); + } if (filter instanceof org.apache.spark.sql.sources.And) { org.apache.spark.sql.sources.And f = (org.apache.spark.sql.sources.And) filter; ConvertedPredicate left = convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java index fda6c0f4003..48f4fc5886e 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java @@ -194,7 +194,7 @@ public void testDsv2Internal() throws Exception { // input filters new Filter[] { new GreaterThan("cnt", 10), // supported data filter - new StringStartsWith("name", "foo"), // unsupported data filter + new StringStartsWith("name", "foo"), // supported data filter new EqualTo("date", "2025-09-01"), // supported partition filter new StringEndsWith("city", "York"), // unsupported partition filter }, @@ -205,10 +205,15 @@ public void testDsv2Internal() throws Exception { new StringEndsWith("city", "York"), }, // expected pushed filters - new Filter[] {new GreaterThan("cnt", 10), new EqualTo("date", "2025-09-01")}, + new Filter[] { + new GreaterThan("cnt", 10), + new StringStartsWith("name", "foo"), + new EqualTo("date", "2025-09-01") + }, // expected pushed kernel predicates new Predicate[] { new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo")), new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")) }, // expected data filters @@ -217,7 +222,10 @@ public void testDsv2Internal() throws Exception { Optional.of( new Predicate( "AND", - new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate( + "AND", + new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), new Predicate("=", new Column("date"), Literal.ofString("2025-09-01"))))); // case 2: OR and NOT filters @@ -240,11 +248,17 @@ public void testDsv2Internal() throws Exception { }, // expected pushed filters new Filter[] { + new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), + new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, // expected pushed kernel predicates new Predicate[] { + new Predicate( + "OR", + new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), new Predicate( "OR", new Predicate("=", new Column("cnt"), Literal.ofInt(50)), @@ -254,7 +268,13 @@ public void testDsv2Internal() throws Exception { new Predicate( "AND", new Predicate(">", new Column("cnt"), Literal.ofInt(100)), - new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")))) + new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")))), + new Predicate( + "NOT", + new Predicate( + "OR", + new Predicate("=", new Column("name"), Literal.ofString("foo")), + new Predicate("STARTS_WITH", new Column("city"), Literal.ofString("New")))) }, // expected data filters new Filter[] { @@ -264,19 +284,39 @@ public void testDsv2Internal() throws Exception { new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, // expected kernel scan builder predicate + // reduce(And::new) over 4 predicates gives left-associative nesting: + // AND(AND(AND(pred1, pred2), pred3), pred4) Optional.of( new Predicate( "AND", new Predicate( - "OR", - new Predicate("=", new Column("cnt"), Literal.ofInt(50)), - new Predicate("=", new Column("date"), Literal.ofString("2025-10-01"))), + "AND", + new Predicate( + "AND", + new Predicate( + "OR", + new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate( + "STARTS_WITH", new Column("name"), Literal.ofString("foo"))), + new Predicate( + "OR", + new Predicate("=", new Column("cnt"), Literal.ofInt(50)), + new Predicate( + "=", new Column("date"), Literal.ofString("2025-10-01")))), + new Predicate( + "NOT", + new Predicate( + "AND", + new Predicate(">", new Column("cnt"), Literal.ofInt(100)), + new Predicate( + "=", new Column("date"), Literal.ofString("2025-09-01"))))), new Predicate( "NOT", new Predicate( - "AND", - new Predicate(">", new Column("cnt"), Literal.ofInt(100)), - new Predicate("=", new Column("date"), Literal.ofString("2025-09-01"))))))); + "OR", + new Predicate("=", new Column("name"), Literal.ofString("foo")), + new Predicate( + "STARTS_WITH", new Column("city"), Literal.ofString("New"))))))); // check SupportsRuntimeV2Filtering // city = 'hz' AND date = '20180520' diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java index f760f8d1775..482f9fa8416 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java @@ -150,15 +150,41 @@ public void testPushFilters_singleUnsupportedDataFilter(@TempDir File tempDir) t checkSupportsPushDownFilters( builder, - new Filter[] {new StringStartsWith("name", "test")}, // input filters - new Filter[] {new StringStartsWith("name", "test")}, // expected post-scan filters - new Filter[] {}, // expected pushed filters + new Filter[] {new StringEndsWith("name", "test")}, // input filters + new Filter[] { + new StringEndsWith("name", "test") + }, // expected post-scan filters (unsupported, stays for row-level eval) + new Filter[] {}, // expected pushed filters (nothing pushed) new Predicate[] {}, // expected pushed kernel predicates - new Filter[] {new StringStartsWith("name", "test")}, // expected data filters + new Filter[] {new StringEndsWith("name", "test")}, // expected data filters Optional.empty() // expected kernelScanBuilder.predicate ); } + @Test + public void testPushFilters_singleSupportedDataFilter_StringStartsWith(@TempDir File tempDir) + throws Exception { + SparkScanBuilder builder = createTestScanBuilder(tempDir); + + checkSupportsPushDownFilters( + builder, + new Filter[] {new StringStartsWith("name", "test")}, // input filters + new Filter[] { + new StringStartsWith("name", "test") + }, // expected post-scan filters (data filter still needs row-level eval) + new Filter[] {new StringStartsWith("name", "test")}, // expected pushed filters + new Predicate[] { + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("test")) + }, // expected pushed kernel predicates + new Filter[] {new StringStartsWith("name", "test")}, // expected data filters + Optional.of( + new Predicate( + "STARTS_WITH", + new Column("name"), + Literal.ofString("test"))) // expected kernelScanBuilder.predicate + ); + } + @Test public void testPushFilters_multiSupportedDataFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); @@ -197,17 +223,17 @@ public void testPushFilters_mixedSupportedAndUnsupportedDataFilters(@TempDir Fil // input filters new Filter[] { new EqualTo("id", 100), // supported - new StringStartsWith("name", "test") // unsupported + new StringEndsWith("name", "test") // unsupported }, // expected post-scan filters - new Filter[] {new EqualTo("id", 100), new StringStartsWith("name", "test")}, - // expected pushed filters + new Filter[] {new EqualTo("id", 100), new StringEndsWith("name", "test")}, + // expected pushed filters (only the supported EqualTo is pushed) new Filter[] {new EqualTo("id", 100)}, // expected pushed kernel predicates new Predicate[] {new Predicate("=", new Column("id"), Literal.ofInt(100))}, // expected data filters - new Filter[] {new EqualTo("id", 100), new StringStartsWith("name", "test")}, - // expected kernelScanBuilder.predicate + new Filter[] {new EqualTo("id", 100), new StringEndsWith("name", "test")}, + // expected kernelScanBuilder.predicate (only EqualTo was pushed to kernel) Optional.of(new Predicate("=", new Column("id"), Literal.ofInt(100)))); } @@ -240,11 +266,17 @@ public void testPushFilters_singleUnsupportedPartitionFilter(@TempDir File tempD checkSupportsPushDownFilters( builder, new Filter[] {new StringStartsWith("dep_id", "1")}, // input filters - new Filter[] {new StringStartsWith("dep_id", "1")}, // expected - new Filter[] {}, // expected pushed filters - new Predicate[] {}, // expected pushed kernel predicates + new Filter[] {}, // expected post-scan filters (partition filter, fully pushed) + new Filter[] {new StringStartsWith("dep_id", "1")}, // expected pushed filters + new Predicate[] { + new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1")) + }, // expected pushed kernel predicates new Filter[] {}, // expected data filters - Optional.empty() // expected kernelScanBuilder.predicate + Optional.of( + new Predicate( + "STARTS_WITH", + new Column("dep_id"), + Literal.ofString("1"))) // expected kernelScanBuilder.predicate ); } @@ -290,15 +322,22 @@ public void testPushFilters_mixedSupportedAndUnsupportedPartitionFilters(@TempDi new StringStartsWith("dep_id", "1") // unsupported }, // expected post-scan filters - new Filter[] {new StringStartsWith("dep_id", "1")}, + new Filter[] {}, // expected pushed filters - new Filter[] {new EqualTo("dep_id", 1)}, + new Filter[] {new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")}, // expected pushed kernel predicates - new Predicate[] {new Predicate("=", new Column("dep_id"), Literal.ofInt(1))}, + new Predicate[] { + new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1")) + }, // expected data filters new Filter[] {}, // expected kernelScanBuilder.predicate - Optional.of(new Predicate("=", new Column("dep_id"), Literal.ofInt(1)))); + Optional.of( + new Predicate( + "AND", + new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1"))))); } @Test @@ -323,25 +362,29 @@ public void testPushFilters_mixedFilters(@TempDir File tempDir) throws Exception // expected pushed filters new Filter[] { new EqualTo("id", 100), // data filter, supported + new StringStartsWith("name", "foo"), // data filter, supported new GreaterThan("dep_id", 1) // partition filter, supported }, // expected pushed kernel predicates new Predicate[] { new Predicate("=", new Column("id"), Literal.ofInt(100)), + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo")), new Predicate(">", new Column("dep_id"), Literal.ofInt(1)) }, // expected data filters new Filter[] { new EqualTo("id", 100), // data filter, supported - new StringStartsWith("name", "foo") // data filter, unsupported + new StringStartsWith("name", "foo") // data filter, supported }, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "AND", - Arrays.asList( + new Predicate( + "AND", new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1)))))); + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), + new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))); } @Test @@ -380,18 +423,20 @@ public void testPushFilters_ORSupportedAndUnsupportedDataFilters(@TempDir File t throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); + // OR(supported, unsupported) cannot be partially pushed: if one branch is unsupported, + // the whole OR must remain for post-scan evaluation and nothing is pushed to the kernel. checkSupportsPushDownFilters( builder, // input filters - new Filter[] {new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo"))}, - // expected post-scan filters - new Filter[] {new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo"))}, - // expected pushed filters + new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, + // expected post-scan filters (whole OR stays, since one branch is unsupported) + new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, + // expected pushed filters (nothing pushed) new Filter[] {}, // expected pushed kernel predicates new Predicate[] {}, // expected data filters - new Filter[] {new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo"))}, + new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, // expected kernelScanBuilder.predicate Optional.empty()); } @@ -431,21 +476,20 @@ public void testPushFilters_ORSupportedDataAndPartitionFilters(@TempDir File tem } /* - * (supportedPartitionFilterA AND unsupportedPartitionFilterB) OR supportedPartitionFilterC - * will be converted to A OR C, because of the partial pushdown of AND + * (partitionFilterA AND partitionFilterB) OR partitionFilterC + * where A = EqualTo("dep_id", 1), B = StringStartsWith("dep_id", "1"), C = GreaterThan("dep_id", 2) + * All three are fully supported partition filters, so the whole expression is pushed down. * - * Expected post-scan filters: (A AND B) OR C - * Expected pushed filters: none (TODO: should be sparkFilterA OR sparkFilterC) - * Expected pushed kernel predicates: predicateA OR predicateC + * Expected post-scan filters: none (all partition filters, fully pushed) + * Expected pushed filters: (A AND B) OR C + * Expected pushed kernel predicates: (predicateA AND predicateB) OR predicateC * Expected data filters: none - * Expected kernelScanBuilder.predicate: predicateA OR predicateC + * Expected kernelScanBuilder.predicate: (predicateA AND predicateB) OR predicateC */ @Test public void testPushFilters_mixedORandAND(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); - // (supportedPartitionFilterA AND unsupportedPartitionFilterB) OR supportedPartitionFilterC - checkSupportsPushDownFilters( builder, // input filters @@ -455,19 +499,24 @@ public void testPushFilters_mixedORandAND(@TempDir File tempDir) throws Exceptio new GreaterThan("dep_id", 2)) }, // expected post-scan filters + new Filter[] {}, + // expected pushed filters new Filter[] { new Or( new And(new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")), new GreaterThan("dep_id", 2)) }, - // expected pushed filters - new Filter[] {}, // expected pushed kernel predicates new Predicate[] { new Predicate( "OR", Arrays.asList( - new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate( + "AND", + Arrays.asList( + new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate( + "STARTS_WITH", new Column("dep_id"), Literal.ofString("1")))), new Predicate(">", new Column("dep_id"), Literal.ofInt(2)))) }, // expected data filters @@ -477,7 +526,12 @@ public void testPushFilters_mixedORandAND(@TempDir File tempDir) throws Exceptio new Predicate( "OR", Arrays.asList( - new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate( + "AND", + Arrays.asList( + new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate( + "STARTS_WITH", new Column("dep_id"), Literal.ofString("1")))), new Predicate(">", new Column("dep_id"), Literal.ofInt(2)))))); } @@ -612,20 +666,23 @@ public void testPushFilters_NOTSupportedDataORUnsupportedDataFilters(@TempDir Fi throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); + // NOT(OR(supported, unsupported)): the OR branch contains an unsupported filter + // (StringEndsWith), + // so the whole NOT(OR(...)) cannot be pushed to the kernel. checkSupportsPushDownFilters( builder, // input filters new Filter[] { - new Not(new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo"))), + new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))), }, - // expected post-scan filters - new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo")))}, - // expected pushed filters + // expected post-scan filters (whole NOT(OR) stays, unsupported branch blocks pushdown) + new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo")))}, + // expected pushed filters (nothing pushed) new Filter[] {}, // expected pushed kernel predicates new Predicate[] {}, // expected data filters - new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo")))}, + new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo")))}, // expected kernelScanBuilder.predicate Optional.empty()); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java index f80d236c3c1..8d69fb6f360 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java @@ -109,6 +109,34 @@ public void testEqualNullSafeFilter() { assertEquals(2, nonNullResult.get().getChildren().size()); } + static Stream stringStartsWithProvider() { + return Stream.of(Arguments.of("non-empty prefix", "Al"), Arguments.of("empty prefix", "")); + } + + @ParameterizedTest(name = "StringStartsWith with {0} should be converted") + @MethodSource("stringStartsWithProvider") + public void testStringStartsWithFilter(String desc, String prefix) { + StringStartsWith filter = new StringStartsWith("name", prefix); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "StringStartsWith filter should be converted"); + assertFalse(result.isPartial(), "StringStartsWith filter should be fully converted"); + assertEquals("STARTS_WITH", result.get().getName()); + // Children: column + string literal + assertEquals(2, result.get().getChildren().size()); + } + + @Test + public void testStringStartsWithFilter_NullValue() { + // A null prefix cannot be converted — treated as unsupported, falls back to post-scan + StringStartsWith filter = new StringStartsWith("name", null); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertFalse(result.isPresent(), "StringStartsWith with null value should not be converted"); + } + // Test data provider for parameterized literal conversion tests static Stream valueTypesProvider() { return Stream.of( From 6b9a07a6e3284c8eb49352f05154330a14674fef Mon Sep 17 00:00:00 2001 From: openinx Date: Tue, 10 Mar 2026 21:37:20 -0700 Subject: [PATCH 09/80] [Spark] Add the DataFrame spark unitycatalog streaming integration tests (#6210) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Add the DataFrame streaming integration tests to cover the spark + uc case. ## How was this patch tested? Run in both locally UC server and remote UC server. ## Does this PR introduce _any_ user-facing changes? No. --------- Signed-off-by: openinx --- .../UCDeltaTableDataFrameStreamingTest.java | 417 ++++++++++++++++++ .../UCDeltaTableDataFrameWriteTest.java | 4 - .../sparkuctest/UCDeltaTableDeltaAPITest.java | 4 - .../UCDeltaTableIntegrationBaseTest.java | 5 + 4 files changed, 422 insertions(+), 8 deletions(-) create mode 100644 spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java new file mode 100644 index 00000000000..920df9bfeb3 --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java @@ -0,0 +1,417 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.spark.api.java.function.VoidFunction2; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.streaming.DataStreamReader; +import org.apache.spark.sql.streaming.StreamingQuery; +import org.apache.spark.sql.streaming.Trigger; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.io.TempDir; + +/** + * DataFrame streaming test suite for Delta Table operations through Unity Catalog. + * + *

    Covers streaming read and write via Structured Streaming. Uses {@link Trigger#AvailableNow()} + * for deterministic, testable streaming without manual termination. Tests run against both EXTERNAL + * and MANAGED table types. + */ +public class UCDeltaTableDataFrameStreamingTest extends UCDeltaTableIntegrationBaseTest { + + /** No-op foreachBatch sink used by negative tests that only need the stream to start. */ + private static final VoidFunction2, Long> NOOP_BATCH = (df, id) -> {}; + + @TempDir private Path tempDir; + + private int checkpointCount; + + @TestAllTableTypes + public void testStreamingReadWrite(TableType tableType) throws Exception { + withNewTable( + "streaming_rw_src", + "id INT", + tableType, + srcName -> + withNewTable( + "streaming_rw_sink", + "id INT", + tableType, + sinkName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", srcName); + String ck = checkpoint(); + + // AvailableNow: process all existing data and terminate. + spark() + .readStream() + .format("delta") + .table(srcName) + .writeStream() + .format("delta") + .outputMode("append") + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", ck) + .toTable(sinkName) + .awaitTermination(); + check(sinkName, List.of(row("1"), row("2"), row("3"))); + + // Continuous: reuse same checkpoint so the query resumes from where + // AvailableNow left off and only picks up newly inserted rows. + StreamingQuery query = + spark() + .readStream() + .format("delta") + .table(srcName) + .writeStream() + .format("delta") + .outputMode("append") + .option("checkpointLocation", ck) + .toTable(sinkName); + try { + sql("INSERT INTO %s VALUES (4), (5)", srcName); + query.processAllAvailable(); + check(sinkName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); + } finally { + query.stop(); + } + })); + } + + @TestAllTableTypes + public void testStreamingReadFromVersion(TableType tableType) throws Exception { + withNewTable( + "streaming_version_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + long v1 = currentVersion(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + List result = new ArrayList<>(); + spark() + .readStream() + .format("delta") + .option("startingVersion", v1 + 1) + .table(tableName) + .writeStream() + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .foreachBatch((VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) + .start() + .awaitTermination(); + assertThat(result).containsExactlyInAnyOrder(4, 5); + }); + } + + /** + * Verifies incremental streaming via a foreachBatch sink (in-memory accumulator). Complements + * {@link #testStreamingReadWrite}, which covers the same data-arrival scenario but writes to a + * Delta table sink and validates via SQL. + */ + @TestAllTableTypes + public void testStreamingContinuous(TableType tableType) throws Exception { + withNewTable( + "streaming_continuous_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + List result = new ArrayList<>(); + StreamingQuery query = + spark() + .readStream() + .format("delta") + .table(tableName) + .writeStream() + .option("checkpointLocation", checkpoint()) + .foreachBatch( + (VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) + .start(); + try { + query.processAllAvailable(); + assertThat(result).containsExactlyInAnyOrder(1, 2, 3); + + sql("INSERT INTO %s VALUES (4), (5)", tableName); + query.processAllAvailable(); + assertThat(result).containsExactlyInAnyOrder(1, 2, 3, 4, 5); + } finally { + query.stop(); + } + }); + } + + /** + * Verifies that {@code maxFilesPerTrigger=1} causes the stream to process each of the 3 separate + * commits as its own micro-batch, producing exactly 3 batches total under {@link + * Trigger#AvailableNow()}. + */ + @TestAllTableTypes + public void testStreamingMaxFilesPerTrigger(TableType tableType) throws Exception { + withNewTable( + "streaming_max_files_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1)", tableName); + sql("INSERT INTO %s VALUES (2)", tableName); + sql("INSERT INTO %s VALUES (3)", tableName); + + List result = new ArrayList<>(); + List batchIds = new ArrayList<>(); + spark() + .readStream() + .format("delta") + .option("maxFilesPerTrigger", 1) + .table(tableName) + .writeStream() + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .foreachBatch( + (VoidFunction2, Long>) + (df, batchId) -> { + result.addAll(ids(df)); + batchIds.add(batchId); + }) + .start() + .awaitTermination(); + assertThat(result).containsExactlyInAnyOrder(1, 2, 3); + assertThat(batchIds).hasSize(3); + }); + } + + /** + * Verifies that streaming from a table that has rows deleted (RemoveFile with dataChange=true) + * fails with a clear error directing the user to the {@code ignoreDeletes} option. + */ + @TestAllTableTypes + public void testStreamingDeleteFailsWithHelpfulError(TableType tableType) throws Exception { + withNewTable( + "streaming_delete_error_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + List result = new ArrayList<>(); + StreamingQuery query = + spark() + .readStream() + .format("delta") + .table(tableName) + .writeStream() + .option("checkpointLocation", checkpoint()) + .foreachBatch( + (VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) + .start(); + try { + query.processAllAvailable(); + assertThat(result).containsExactlyInAnyOrder(1, 2, 3); + + sql("DELETE FROM %s WHERE id = 1", tableName); + assertStreamingThrowsContaining(query::processAllAvailable, "ignoreDeletes"); + } finally { + query.stop(); + } + }); + } + + /** + * CDF streaming reads work for EXTERNAL tables but fail for MANAGED tables. + * + *

    For EXTERNAL: verifies that inserts and a delete produce the expected typed change events. + * For MANAGED: verifies the stream fails with an error containing "not supported" and "CDC". + */ + @TestAllTableTypes + public void testStreamingCDFRead(TableType tableType) throws Exception { + withNewTable( + "streaming_cdf_read_test", + "id INT", + null, + tableType, + "'delta.enableChangeDataFeed'='true'", + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + long insertVersion = currentVersion(tableName); + sql("DELETE FROM %s WHERE id = 1", tableName); + + if (tableType == TableType.EXTERNAL) { + List changes = new ArrayList<>(); + spark() + .readStream() + .format("delta") + .option("readChangeFeed", "true") + .option("startingVersion", insertVersion) + .table(tableName) + .writeStream() + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .foreachBatch( + (VoidFunction2, Long>) + (df, id) -> changes.addAll(df.select("id", "_change_type").collectAsList())) + .start() + .awaitTermination(); + assertThat(changes) + .extracting(r -> r.getString(1)) + .containsExactlyInAnyOrder("insert", "insert", "insert", "delete"); + assertThat(changes) + .filteredOn(r -> "delete".equals(r.getString(1))) + .extracting(r -> r.getInt(0)) + .containsExactly(1); + } else { + assertInvalidStreamOption( + tableName, + r -> r.option("readChangeFeed", "true").option("startingVersion", insertVersion), + "not supported", + "CDC"); + } + }); + } + + /** + * Delta streaming sink does not support {@code complete} output mode. Verifies the stream fails + * with a clear error mentioning "complete". + */ + @TestAllTableTypes + public void testStreamingWriteCompleteModeNotSupported(TableType tableType) throws Exception { + withNewTable( + "streaming_complete_mode_src", + "id INT", + tableType, + srcName -> + withNewTable( + "streaming_complete_mode_sink", + "id INT", + tableType, + sinkName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", srcName); + assertStreamingThrowsContaining( + () -> + spark() + .readStream() + .format("delta") + .table(srcName) + .writeStream() + .format("delta") + .outputMode("complete") + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .toTable(sinkName) + .awaitTermination(), + "complete"); + })); + } + + /** + * Verifies that invalid or unsupported streaming read options are rejected with clear errors. + * + *

      + *
    • Negative {@code startingVersion} is rejected (both table types). + *
    • {@code startingVersion} beyond the table history is rejected (both table types). + *
    • {@code ignoreChanges} and {@code ignoreDeletes} fail with a "not supported" error for + * MANAGED tables; EXTERNAL tables accept these options silently. + *
    + */ + @TestAllTableTypes + public void testStreamingInvalidOptions(TableType tableType) throws Exception { + withNewTable( + "streaming_invalid_options_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1)", tableName); + + assertInvalidStreamOption( + tableName, r -> r.option("startingVersion", -1), "startingVersion"); + assertInvalidStreamOption(tableName, r -> r.option("startingVersion", 999999), "999999"); + + if (tableType == TableType.MANAGED) { + assertInvalidStreamOption( + tableName, + r -> r.option("ignoreChanges", "true"), + "not supported", + "ignoreChanges"); + assertInvalidStreamOption( + tableName, + r -> r.option("ignoreDeletes", "true"), + "not supported", + "ignoreDeletes"); + } + }); + } + + /** + * Starts a streaming read with options applied by {@code configure}, then asserts the stream + * fails with a message containing all {@code fragments} (case-insensitive). + */ + private void assertInvalidStreamOption( + String tableName, + Function configure, + String... fragments) { + assertStreamingThrowsContaining( + () -> + configure + .apply(spark().readStream().format("delta")) + .table(tableName) + .writeStream() + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .foreachBatch(NOOP_BATCH) + .start() + .awaitTermination(), + fragments); + } + + /** + * Asserts that {@code action} throws an exception whose cause chain contains all the given {@code + * fragments} (case-insensitive). + */ + private static void assertStreamingThrowsContaining( + ThrowingCallable action, String... fragments) { + assertThatThrownBy(action) + .satisfies( + e -> { + StringBuilder full = new StringBuilder(); + for (Throwable t = e; t != null; t = t.getCause()) { + if (t.getMessage() != null) full.append(t.getMessage()).append(' '); + } + String msg = full.toString(); + for (String fragment : fragments) { + assertThat(msg).containsIgnoringCase(fragment); + } + }); + } + + private String checkpoint() throws IOException { + Path ckDir = tempDir.resolve("ck-" + checkpointCount++); + Files.createDirectory(ckDir); + return ckDir.toString(); + } + + private List ids(Dataset df) { + return df.collectAsList().stream().map(r -> r.getInt(0)).collect(Collectors.toList()); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java index e4a60b1ec6d..aeaff4e29cd 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java @@ -265,8 +265,4 @@ private Dataset intDf(Integer... ids) { Arrays.stream(ids).map(RowFactory::create).collect(Collectors.toList()), new StructType().add("id", DataTypes.IntegerType)); } - - private static List row(String... values) { - return List.of(values); - } } diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java index 94753132fca..6904f883030 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java @@ -173,8 +173,4 @@ public void testRestoreToTimestamp(TableType tableType) throws Exception { private DeltaTable forName(String tableName) { return DeltaTable.forName(spark(), tableName); } - - private static List row(String... values) { - return List.of(values); - } } diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java index 0027003f083..3d4d4e7879a 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java @@ -319,6 +319,11 @@ protected String currentTimestamp(String tableName) { return sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(1); } + /** Helper to build an expected row as a list of string values. */ + protected static List row(String... values) { + return List.of(values); + } + /** Functional interface for test code that takes a temporary directory. */ @FunctionalInterface protected interface TempDirCode { From 1b69a8d14b99697236163775fc476cd90e6d7bf4 Mon Sep 17 00:00:00 2001 From: Eames Trinh Date: Wed, 11 Mar 2026 16:46:22 +0100 Subject: [PATCH 10/80] Allow SQL insert BY NAME queries to have schema evolution (#6186) --- .../spark/sql/delta/DeltaAnalysis.scala | 59 ++++++++++++++---- .../sql/delta/sources/DeltaSQLConf.scala | 12 ++++ .../DeltaInsertIntoSchemaEvolutionSuite.scala | 60 +++++++++++++++---- .../sql/delta/DeltaInsertIntoTableSuite.scala | 3 +- 4 files changed, 109 insertions(+), 25 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala index 19bfeae4345..1d82514fc20 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala @@ -101,7 +101,12 @@ class DeltaAnalysis(session: SparkSession) // since we only want to up-cast for SQL insert into by name case a @ AppendDelta(r, d) if a.isByName && a.origin.sqlText.nonEmpty && needsSchemaAdjustmentByName(a.query, r.output, d, a.writeOptions) => - val projection = resolveQueryColumnsByName(a.query, r.output, d, a.writeOptions) + val projection = resolveQueryColumnsByName( + query = a.query, + targetAttrs = r.output, + deltaTable = d, + writeOptions = a.writeOptions, + allowSchemaEvolution = true) if (projection != a.query) { a.copy(query = projection) } else { @@ -282,7 +287,12 @@ class DeltaAnalysis(session: SparkSession) // since we only want to up-cast for SQL insert into by name case o @ OverwriteDelta(r, d) if o.isByName && o.origin.sqlText.nonEmpty && needsSchemaAdjustmentByName(o.query, r.output, d, o.writeOptions) => - val projection = resolveQueryColumnsByName(o.query, r.output, d, o.writeOptions) + val projection = resolveQueryColumnsByName( + query = o.query, + targetAttrs = r.output, + deltaTable = d, + writeOptions = o.writeOptions, + allowSchemaEvolution = true) if (projection != o.query) { val aliases = AttributeMap(o.query.output.zip(projection.output).collect { case (l: AttributeReference, r: AttributeReference) if !l.sameRef(r) => (l, r) @@ -308,7 +318,12 @@ class DeltaAnalysis(session: SparkSession) // INSERT OVERWRITE by name // OverwriteDelta.byName is also used for DataFrame append so we check for the SQL origin // text since we only want to up-cast for SQL insert into by name - resolveQueryColumnsByName(o.query, r.output, d, o.writeOptions) + resolveQueryColumnsByName( + query = o.query, + targetAttrs = r.output, + deltaTable = d, + writeOptions = o.writeOptions, + allowSchemaEvolution = true) } else { o.query } @@ -973,12 +988,19 @@ class DeltaAnalysis(session: SparkSession) query: LogicalPlan, targetAttrs: Seq[Attribute], deltaTable: DeltaTableV2, - writeOptions: Map[String, String]): LogicalPlan = { - insertIntoByNameMissingColumn(query, targetAttrs, deltaTable) + writeOptions: Map[String, String], + allowSchemaEvolution: Boolean = false): LogicalPlan = { + // Schema evolution is only effective when mergeSchema is enabled in write options AND + // the feature is enabled via SQL conf. + val effectiveSchemaEvolution = allowSchemaEvolution && + new DeltaOptions(deltaTable.options ++ writeOptions, conf).canMergeSchema && + session.conf.get(DeltaSQLConf.DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED) + + insertIntoByNameMissingColumn(query, targetAttrs, deltaTable, effectiveSchemaEvolution) // This is called before resolveOutputColumns in postHocResolutionRules, so we need to duplicate // the schema validation here. - if (query.output.length > targetAttrs.length) { + if (!effectiveSchemaEvolution && query.output.length > targetAttrs.length) { throw QueryCompilationErrors.cannotWriteTooManyColumnsToTableError( tableName = deltaTable.name(), expected = targetAttrs.map(_.name), @@ -988,7 +1010,12 @@ class DeltaAnalysis(session: SparkSession) val project = query.output.map { attr => val targetAttr = targetAttrs.find(t => session.sessionState.conf.resolver(t.name, attr.name)) .getOrElse { - throw DeltaErrors.missingColumn(attr, targetAttrs) + if (effectiveSchemaEvolution) { + attr + } else { + // Extra columns in the source are not allowed when schema evolution is disabled. + throw DeltaErrors.missingColumn(attr, targetAttrs) + } } addCastToColumn(attr, targetAttr, deltaTable.name(), typeWideningMode = getTypeWideningMode(deltaTable, writeOptions) @@ -1081,10 +1108,20 @@ class DeltaAnalysis(session: SparkSession) private def insertIntoByNameMissingColumn( query: LogicalPlan, targetAttrs: Seq[Attribute], - deltaTable: DeltaTableV2): Unit = { - if (query.output.length < targetAttrs.length) { - // Some columns are not specified. We don't allow schema evolution in INSERT INTO BY NAME, so - // we need to ensure the missing columns must be generated columns. + deltaTable: DeltaTableV2, + allowSchemaEvolution: Boolean = false): Unit = { + // When allowing the source schema to contain extra columns, it can still + // be missing required columns from the target schema. + // + // The total column count alone is not sufficient for validation. + // A source may have more columns overall but still omit specific columns + // that are required by the target schema. + // + // Example: + // Target: [a, b] + // Source: [a, x, y] + // Since the source has 3 columns vs target's 2, but is missing column b, this should be caught. + if (allowSchemaEvolution || query.output.length < targetAttrs.length) { val userSpecifiedNames = if (session.sessionState.conf.caseSensitiveAnalysis) { query.output.map(a => (a.name, a)).toMap } else { diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index 08c731a7459..db5baf09064 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -767,6 +767,18 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(DeltaUtils.isTesting) + val DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED = + buildConf("insert.byName.schemaEvolution.enabled") + .internal() + .doc( + """When enabled, SQL INSERT INTO BY NAME operations allow schema evolution: extra columns in + |the source that are not in the target table schema are added to the target schema when + |schema evolution (mergeSchema) is also enabled. Disable this flag to revert to the old + |behavior where extra columns always cause an error, regardless of schema evolution + |settings.""".stripMargin) + .booleanConf + .createWithDefault(true) + val DELTA_SCHEMA_TYPE_CHECK = buildConf("schema.typeCheck.enabled") .doc( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala index aabdc600faf..abfd74cb3ac 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.SparkThrowable import org.apache.spark.sql.SaveMode import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -97,9 +98,8 @@ class DeltaInsertIntoSchemaEvolutionSuite extends DeltaInsertIntoTest { ) - // Adding new top-level columns with schema evolution is allowed for all inserts except SQL - // inserts by name, but dataframe inserts by name don't support implicit casting and will fail - // due to the type mismatch. + // Adding new top-level columns with schema evolution is allowed for all inserts, but dataframe + // inserts by name don't support implicit casting and will fail due to the type mismatch. testInserts(s"insert with extra top-level column and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int", Seq("""{ "a": 1, "b": 2 }""")), @@ -140,22 +140,32 @@ class DeltaInsertIntoSchemaEvolutionSuite extends DeltaInsertIntoTest { withSchemaEvolution = schemaEvolution ) + // SQL inserts by name fail with a different error in the analysis when there's an extra column + // and schema evolution is disabled. testInserts(s"insert with extra top-level column and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long, c int", Seq("""{ "a": 1, "b": 4, "c": 5 }""")), - expectedResult = ExpectedResult.Failure(ex => { - checkError( - ex, - "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS", - parameters = Map( - "tableName" -> s"`$catalogName`.`default`.`target`", - "tableColumns" -> "`a`, `b`", - "dataColumns" -> "`a`, `b`, `c`" - )) - }), + expectedResult = if (schemaEvolution) { + ExpectedResult.Success( + expected = new StructType() + .add("a", IntegerType) + .add("b", IntegerType) + .add("c", IntegerType)) + } else { + ExpectedResult.Failure(ex => { + checkError( + ex, + "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS", + parameters = Map( + "tableName" -> s"`$catalogName`.`default`.`target`", + "tableColumns" -> "`a`, `b`", + "dataColumns" -> "`a`, `b`, `c`" + )) + }) + }, includeInserts = insertsSQL.intersect(insertsByName) -- Set( // It's not possible to specify a column that doesn't exist in the target using SQL with an // explicit column list. @@ -273,4 +283,28 @@ class DeltaInsertIntoSchemaEvolutionSuite extends DeltaInsertIntoTest { withSchemaEvolution = schemaEvolution ) } + + // When DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED is disabled, SQL INSERT BY NAME with extra + // top-level columns should fail even when schema evolution is enabled. + test("insert by name with extra top-level column and implicit cast fails " + + "when byNameSchemaEvolution is disabled") { + withTable("target") { + sql("CREATE TABLE target (a INT, b INT) USING DELTA") + withSQLConf( + DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true", + DeltaSQLConf.DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED.key -> "false") { + val ex = intercept[SparkThrowable] { + sql("INSERT INTO target BY NAME SELECT 1 AS a, 2L AS b, 3 AS c") + } + checkError( + ex, + "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS", + parameters = Map( + "tableName" -> s"`$catalogName`.`default`.`target`", + "tableColumns" -> "`a`, `b`", + "dataColumns" -> "`a`, `b`, `c`" + )) + } + } + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala index 2ddd9397305..c9f468c2adf 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala @@ -168,7 +168,8 @@ class DeltaInsertIntoSQLSuite // true (due to case sensitivity) so that we call resolveQueryColumnsByName and hit the right // code path. - // when the number of columns does not match, throw an arity mismatch error. + // when the number of columns does not match and schema evolution is disabled, throw + // an arity mismatch error. testInsertByNameError( targetSchema = "(A int)", expectedErrorClass = "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS") From 7081621933222cb7965e1a6bbec3b5ec7a399796 Mon Sep 17 00:00:00 2001 From: littlegrasscao <45191106+littlegrasscao@users.noreply.github.com> Date: Wed, 11 Mar 2026 11:32:29 -0700 Subject: [PATCH 11/80] Upgrade delta sharing client to 1.3.10 (#6201) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [X] Other (sharing) ## Description Upgrade delta sharing client to 1.3.10 ## How was this patch tested? CI pipeline ## Does this PR introduce _any_ user-facing changes? No --------- Co-authored-by: Sun Cao --- build.sbt | 2 +- .../spark/DeltaFormatSharingSource.scala | 6 ++- .../sharing/spark/DeltaSharingCDFUtils.scala | 5 +- .../spark/DeltaSharingDataSource.scala | 36 +++++++++----- .../sharing/spark/DeltaSharingFileIndex.scala | 3 +- .../sharing/spark/DeltaSharingUtils.scala | 13 +++-- .../spark/DeltaSharingCDFUtilsSuite.scala | 13 +++-- .../DeltaSharingDataSourceDeltaSuite.scala | 47 +++++++++++++++++++ .../spark/DeltaSharingFileIndexSuite.scala | 13 +++-- .../spark/DeltaSharingUtilsSuite.scala | 19 +++++--- .../TestClientForDeltaFormatSharing.scala | 16 +++++-- 11 files changed, 134 insertions(+), 39 deletions(-) diff --git a/build.sbt b/build.sbt index 14c113d59b1..8f4e27951fe 100644 --- a/build.sbt +++ b/build.sbt @@ -822,7 +822,7 @@ lazy val sharing = (project in file("sharing")) libraryDependencies ++= Seq( "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", - "io.delta" %% "delta-sharing-client" % "1.3.9", + "io.delta" %% "delta-sharing-client" % "1.3.10", // Test deps "org.scalatest" %% "scalatest" % scalaTestVersion % "test", diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala index 4139dddd4ea..6404a36666f 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala @@ -520,7 +520,8 @@ case class DeltaFormatSharingSource( versionAsOf = Some(startingOffset.reservoirVersion), timestampAsOf = None, jsonPredicateHints = None, - refreshToken = None + refreshToken = None, + fileIdHash = None ) val refreshFunc = DeltaSharingUtils.getRefresherForGetFiles( client = client, @@ -543,7 +544,8 @@ case class DeltaFormatSharingSource( val tableFiles = client.getFiles( table = table, startingVersion = startingOffset.reservoirVersion, - endingVersion = Some(endingVersionForQuery) + endingVersion = Some(endingVersionForQuery), + fileIdHash = None ) val refreshFunc = DeltaSharingUtils.getRefresherForGetFilesWithStartingVersion( client = client, diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala index 7b8a8294c67..ae91cc4646d 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala @@ -52,8 +52,9 @@ object DeltaSharingCDFUtils extends Logging { // and also any metadata changes between [startingVersion, endingVersion], to put them in the // delta log. This is to allow delta library to check the metadata change and handle it // properly -- currently it throws error for column mapping changes. - val deltaTableFiles = - client.getCDFFiles(table, options.cdfOptions, includeHistoricalMetadata = true) + val deltaTableFiles = client.getCDFFiles( + table, options.cdfOptions, includeHistoricalMetadata = true, fileIdHash = None + ) logInfo( s"Fetched ${deltaTableFiles.lines.size} lines with cdf options ${options.cdfOptions} " + s"for table ${table} from delta sharing server, took ${getDuration(startTime)}s." diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala index 59c6856ac11..5804b03f358 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala @@ -78,7 +78,8 @@ private[sharing] class DeltaSharingDataSource path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, - responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET + responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, + callerOrg = options.callerOrg ) val schemaToUse = deltaLog.snapshot().schema if (schemaToUse.isEmpty) { @@ -105,7 +106,8 @@ private[sharing] class DeltaSharingDataSource responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. - readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(",") + readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(","), + callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, @@ -199,7 +201,8 @@ private[sharing] class DeltaSharingDataSource path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, - responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET + responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, + callerOrg = options.callerOrg ) DeltaSharingSource(SparkSession.active, deltaLog, options) } else if (responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { @@ -217,7 +220,8 @@ private[sharing] class DeltaSharingDataSource responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. - readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(",") + readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(","), + callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, @@ -273,7 +277,8 @@ private[sharing] class DeltaSharingDataSource shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, versionAsOf = None, - timestampAsOf = None + timestampAsOf = None, + callerOrg = options.callerOrg ) logInfo(s"Streaming format resolved via getMetadata: ${deltaTableMetadata.respondedFormat} " + s"for path:$path") @@ -293,7 +298,8 @@ private[sharing] class DeltaSharingDataSource shareCredentialsOptions: Map[String, String], forStreaming: Boolean, versionAsOf: Option[Long], - timestampAsOf: Option[String]): (DeltaSharingTable, DeltaTableMetadata) = { + timestampAsOf: Option[String], + callerOrg: Option[String] = None): (DeltaSharingTable, DeltaTableMetadata) = { val responseFormat = { if (sqlContext.sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_SHARING_FORCE_DELTA_FORMAT)) { @@ -320,7 +326,8 @@ private[sharing] class DeltaSharingDataSource forStreaming = forStreaming, // Indicating that the client is able to process response format in both parquet and delta. responseFormat = responseFormat, - readerFeatures = readerFeatures + readerFeatures = readerFeatures, + callerOrg = callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, @@ -356,7 +363,8 @@ private[sharing] class DeltaSharingDataSource path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = false, - responseFormat = options.responseFormat + responseFormat = options.responseFormat, + callerOrg = options.callerOrg ) deltaLog.createRelation( options.versionAsOf, @@ -377,7 +385,8 @@ private[sharing] class DeltaSharingDataSource responseFormat = options.responseFormat, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. - readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(",") + readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(","), + callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, @@ -439,7 +448,8 @@ private[sharing] class DeltaSharingDataSource shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = false, versionAsOf = options.versionAsOf, - timestampAsOf = options.timestampAsOf + timestampAsOf = options.timestampAsOf, + callerOrg = options.callerOrg ) if (deltaTableMetadata.respondedFormat == DeltaSharingOptions.RESPONSE_FORMAT_PARQUET) { @@ -450,7 +460,8 @@ private[sharing] class DeltaSharingDataSource options.shareCredentialsOptions, forStreaming = false, responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, - initDeltaTableMetadata = Some(deltaTableMetadata) + initDeltaTableMetadata = Some(deltaTableMetadata), + callerOrg = options.callerOrg ) deltaLog.createRelation(options.versionAsOf, options.timestampAsOf, options.cdfOptions) } else if (deltaTableMetadata.respondedFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { @@ -468,7 +479,8 @@ private[sharing] class DeltaSharingDataSource responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. - readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(",") + readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(","), + callerOrg = options.callerOrg ) getHadoopFsRelationForDeltaSnapshotQuery( path = path, diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala index c29dc2b805a..8d40daf58a6 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala @@ -149,7 +149,8 @@ case class DeltaSharingFileIndex( versionAsOf = params.options.versionAsOf, timestampAsOf = params.options.timestampAsOf, jsonPredicateHints = jsonPredicateHints, - refreshToken = None + refreshToken = None, + fileIdHash = None ) logInfo( s"Fetched ${deltaTableFiles.lines.size} lines for table $table with version " + diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala index b27ea008763..2f7e9b46e71 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala @@ -197,7 +197,8 @@ object DeltaSharingUtils extends Logging { versionAsOf = versionAsOf, timestampAsOf = timestampAsOf, jsonPredicateHints = jsonPredicateHints, - refreshToken = if (useRefreshToken) refreshTokenOpt else None + refreshToken = if (useRefreshToken) refreshTokenOpt else None, + fileIdHash = None ) getTableRefreshResult(tableFiles) } @@ -217,7 +218,8 @@ object DeltaSharingUtils extends Logging { val tableFiles = client.getCDFFiles( table = table, cdfOptions = cdfOptions, - includeHistoricalMetadata = true + includeHistoricalMetadata = true, + fileIdHash = None ) getTableRefreshResult(tableFiles) } @@ -236,7 +238,12 @@ object DeltaSharingUtils extends Logging { endingVersion: Option[Long]): RefresherFunction = { (_: Option[String]) => { val tableFiles = client - .getFiles(table = table, startingVersion = startingVersion, endingVersion = endingVersion) + .getFiles( + table = table, + startingVersion = startingVersion, + endingVersion = endingVersion, + fileIdHash = None + ) getTableRefreshResult(tableFiles) } } diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala index 8fe6319a33b..11f7752d55c 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala @@ -88,7 +88,9 @@ class TestDeltaSharingClientForCDFUtils( asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, - tokenRenewalThresholdInSeconds: Int = 600) + tokenRenewalThresholdInSeconds: Int = 600, + callerOrg: String = "", + skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { import CDFTesTUtils._ @@ -121,7 +123,8 @@ class TestDeltaSharingClientForCDFUtils( versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException("getFiles is not supported now.") } @@ -129,7 +132,8 @@ class TestDeltaSharingClientForCDFUtils( override def getFiles( table: Table, startingVersion: Long, - endingVersion: Option[Long] + endingVersion: Option[Long], + fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException(s"getFiles with startingVersion($startingVersion)") } @@ -137,7 +141,8 @@ class TestDeltaSharingClientForCDFUtils( override def getCDFFiles( table: Table, cdfOptions: Map[String, String], - includeHistoricalMetadata: Boolean + includeHistoricalMetadata: Boolean, + fileIdHash: Option[String] ): DeltaTableFiles = { numGetFileCalls += 1 DeltaTableFiles( diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala index f831130b361..314a9773686 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala @@ -1696,6 +1696,53 @@ trait DeltaSharingDataSourceDeltaSuiteBase } } } + test("callerOrg option is passed to DeltaSharingRestClient") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_caller_org" + withTable(deltaTableName) { + createSimpleTable(deltaTableName, enableCdf = false) + sql(s"""INSERT INTO $deltaTableName VALUES (1, "one")""") + + val sharedTableName = "shared_table_caller_org" + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName) + DeltaSharingUtils.overrideSingleBlock[Long]( + blockId = TestClientForDeltaFormatSharing.getBlockId( + sharedTableName, "getTableVersion"), + value = 1 + ) + + withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { + val profileFile = prepareProfileFile(inputDir) + val tablePath = + s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName" + + TestClientForDeltaFormatSharing.lastCallerOrg = "" + spark.read + .format("deltaSharing") + .option("responseFormat", "delta") + .option(DeltaSharingOptions.CALLER_ORG_OPTION, "test-org") + .load(tablePath) + .collect() + assert( + TestClientForDeltaFormatSharing.lastCallerOrg == "test-org", + "callerOrg should be passed through to the client" + ) + + TestClientForDeltaFormatSharing.lastCallerOrg = "" + spark.read + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .collect() + assert( + TestClientForDeltaFormatSharing.lastCallerOrg == "", + "callerOrg should be empty when not set" + ) + } + } + } + } } class DeltaSharingDataSourceDeltaSuite extends DeltaSharingDataSourceDeltaSuiteBase {} diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala index 4166444f2d0..080862487cb 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala @@ -100,7 +100,9 @@ class TestDeltaSharingClientForFileIndex( asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, - tokenRenewalThresholdInSeconds: Int = 600) + tokenRenewalThresholdInSeconds: Int = 600, + callerOrg: String = "", + skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { import TestUtils._ @@ -148,7 +150,8 @@ class TestDeltaSharingClientForFileIndex( versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { numGetFileCalls += 1 limit.foreach(lim => savedLimits = savedLimits :+ lim) @@ -175,7 +178,8 @@ class TestDeltaSharingClientForFileIndex( override def getFiles( table: Table, startingVersion: Long, - endingVersion: Option[Long] + endingVersion: Option[Long], + fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException(s"getFiles with startingVersion($startingVersion)") } @@ -183,7 +187,8 @@ class TestDeltaSharingClientForFileIndex( override def getCDFFiles( table: Table, cdfOptions: Map[String, String], - includeHistoricalMetadata: Boolean + includeHistoricalMetadata: Boolean, + fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException( s"getCDFFiles with cdfOptions:[$cdfOptions], " + diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala index ed9dcd2538e..3e682086969 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala @@ -133,7 +133,8 @@ class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() @@ -144,8 +145,12 @@ class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { ) } - override def getFiles(table: Table, startingVersion: Long, endingVersion: Option[Long]) - : DeltaTableFiles = { + override def getFiles( + table: Table, + startingVersion: Long, + endingVersion: Option[Long], + fileIdHash: Option[String] + ): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() DeltaTableFiles( @@ -158,7 +163,8 @@ class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { override def getCDFFiles( table: Table, cdfOptions: Map[String, String], - includeHistoricalMetadata: Boolean): DeltaTableFiles = { + includeHistoricalMetadata: Boolean, + fileIdHash: Option[String]): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() val cdc = getCdcStr() @@ -283,11 +289,12 @@ class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { lastRefreshToken = refreshToken super.getFiles(table, predicates, limit, versionAsOf, timestampAsOf, - jsonPredicateHints, refreshToken) + jsonPredicateHints, refreshToken, fileIdHash) } } diff --git a/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala b/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala index c8828f2c25b..1180ad2aa85 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala @@ -60,7 +60,9 @@ private[spark] class TestClientForDeltaFormatSharing( asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, - tokenRenewalThresholdInSeconds: Int = 600) + tokenRenewalThresholdInSeconds: Int = 600, + callerOrg: String = "", + skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { private val supportedReaderFeatures: Seq[String] = Seq( @@ -83,6 +85,8 @@ private[spark] class TestClientForDeltaFormatSharing( import TestClientForDeltaFormatSharing._ + TestClientForDeltaFormatSharing.lastCallerOrg = callerOrg + override def listAllTables(): Seq[Table] = throw new UnsupportedOperationException("not needed") override def getMetadata( @@ -144,7 +148,8 @@ private[spark] class TestClientForDeltaFormatSharing( versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { val tableFullName = s"${table.share}.${table.schema}.${table.name}" limit.foreach(lim => TestClientForDeltaFormatSharing.limits.put(tableFullName, lim)) @@ -206,7 +211,8 @@ private[spark] class TestClientForDeltaFormatSharing( override def getFiles( table: Table, startingVersion: Long, - endingVersion: Option[Long] + endingVersion: Option[Long], + fileIdHash: Option[String] ): DeltaTableFiles = { assert( endingVersion.isDefined, @@ -239,7 +245,8 @@ private[spark] class TestClientForDeltaFormatSharing( override def getCDFFiles( table: Table, cdfOptions: Map[String, String], - includeHistoricalMetadata: Boolean + includeHistoricalMetadata: Boolean, + fileIdHash: Option[String] ): DeltaTableFiles = { val suffix = cdfOptions .get(DeltaSharingOptions.CDF_START_VERSION) @@ -310,4 +317,5 @@ object TestClientForDeltaFormatSharing { val limits = scala.collection.mutable.Map[String, Long]() val requestedFormat = scala.collection.mutable.Map[String, String]() val jsonPredicateHints = scala.collection.mutable.Map[String, String]() + @volatile var lastCallerOrg: String = "" } From 8eff495d86c897553bc0bfba911a2aacfa86526e Mon Sep 17 00:00:00 2001 From: Ziya Mukhtarov Date: Wed, 11 Mar 2026 22:09:47 +0100 Subject: [PATCH 12/80] [Spark] Improve Error Handling for NullType in UDTs (#6254) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description This PR blocks writes with NullType in UDT columns, similar to how we block NullType in arrays/maps. NullType is typically used as a temporary placeholder, which is later evolved into some other type. However, UDTs cannot evolve over time, hence NullType inside UDT remains as NullType forever. Additionally, our old check missed arrays/maps inside UDT and didn't check whether they contain NullType or not. When trying to write such Parquet files, the Parquet writer failed. This PR fixes these problems. ## How was this patch tested? New unit tests. ## Does this PR introduce _any_ user-facing changes? No. --- .../resources/error/delta-error-classes.json | 6 ++ .../spark/sql/delta/schema/SchemaUtils.scala | 20 +++++-- .../apache/spark/sql/delta/DeltaSuite.scala | 56 ++++++++++++++++++- 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/spark/src/main/resources/error/delta-error-classes.json b/spark/src/main/resources/error/delta-error-classes.json index 1ca30d33066..c9ca79fb9f6 100644 --- a/spark/src/main/resources/error/delta-error-classes.json +++ b/spark/src/main/resources/error/delta-error-classes.json @@ -3260,6 +3260,12 @@ ], "sqlState" : "42846" }, + "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE" : { + "message" : [ + "Found NullType in column which is of user-defined type. Delta doesn't support writing NullType in user-defined types." + ], + "sqlState" : "22005" + }, "DELTA_VACUUM_RETENTION_PERIOD_NEGATIVE" : { "message" : [ "Retention period for Vacuum can't be less than 0 hours." diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala index d5f443c7a20..f1229f4db37 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala @@ -130,7 +130,14 @@ object SchemaUtils extends DeltaLogging { */ def dropNullTypeColumns(df: DataFrame): DataFrame = { val schema = df.schema - if (!typeExistsRecursively(schema)(_.isInstanceOf[NullType])) return df + def hasNullType(t: DataType): Boolean = + typeExistsRecursively(t) { + case _: NullType => true + case udt: UserDefinedType[_] => hasNullType(udt.sqlType) + case _ => false + } + + if (!hasNullType(schema)) return df def generateSelectExpr(sf: StructField, nameStack: Seq[String]): Column = sf.dataType match { case st: StructType => val nested = st.fields.flatMap { f => @@ -144,16 +151,21 @@ object SchemaUtils extends DeltaLogging { when(col(colName).isNull, null) .otherwise(struct(nested: _*)) .alias(sf.name) - case a: ArrayType if typeExistsRecursively(a)(_.isInstanceOf[NullType]) => + case a: ArrayType if hasNullType(a) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE", messageParameters = Array(colName, "ArrayType")) - case m: MapType if typeExistsRecursively(m)(_.isInstanceOf[NullType]) => + case m: MapType if hasNullType(m) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE", - messageParameters = Array(colName, "NullType")) + messageParameters = Array(colName, "MapType")) + case udt: UserDefinedType[_] if hasNullType(udt.sqlType) => + val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql + throw new DeltaAnalysisException( + errorClass = "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", + messageParameters = Array(colName, udt.userClass.getName)) case _ => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql col(colName).alias(sf.name) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala index 0b4070af4f5..7a75da71b0d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala @@ -46,7 +46,7 @@ import org.apache.spark.sql.functions.{asc, col, expr, lit, map_values, struct} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.StreamingQuery import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.{StringType, StructType} +import org.apache.spark.sql.types._ import org.apache.spark.util.Utils class DeltaSuite extends QueryTest @@ -241,6 +241,36 @@ class DeltaSuite extends QueryTest } } + test("Cannot create table with NullType UDT column") { + val table_name = "test_table" + withTable(table_name) { + checkError( + intercept[DeltaAnalysisException] { + Seq((1, new NullData())).toDF("id", "value") + .write.format("delta").saveAsTable(table_name) + }, + "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", + sqlState = Some("22005"), + parameters = Map("columnName" -> "value", "userClass" -> classOf[NullData].getName) + ) + } + } + + test("Cannot create table with NullType in a complex UDT column") { + val table_name = "test_table" + withTable(table_name) { + checkError( + intercept[DeltaAnalysisException] { + Seq((1, new ComplexData())).toDF("id", "value") + .write.format("delta").saveAsTable(table_name) + }, + "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", + sqlState = Some("22005"), + parameters = Map("columnName" -> "value", "userClass" -> classOf[ComplexData].getName) + ) + } + } + test("partitioned append - nulls") { val tempDir = Utils.createTempDir() Seq(Some(1), None).toDF() @@ -3286,3 +3316,27 @@ class DeltaWithCatalogOwnedBatch2Suite extends DeltaSuite { class DeltaWithCatalogOwnedBatch100Suite extends DeltaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } + +@SQLUserDefinedType(udt = classOf[NullUDT]) +class NullData extends Serializable + +class NullUDT extends UserDefinedType[NullData] { + override def sqlType: DataType = NullType + override def userClass: Class[NullData] = classOf[NullData] + override def serialize(obj: NullData): Any = null + override def deserialize(datum: Any): NullData = new NullData() +} + +@SQLUserDefinedType(udt = classOf[ComplexUDT]) +class ComplexData extends Serializable + +class ComplexUDT extends UserDefinedType[ComplexData] { + override def sqlType: DataType = new MapType( + StringType, + new ArrayType( + new StructType().add("a", IntegerType).add("b", new NullUDT), containsNull = true), + valueContainsNull = true) + override def userClass: Class[ComplexData] = classOf[ComplexData] + override def serialize(obj: ComplexData): Any = null + override def deserialize(datum: Any): ComplexData = new ComplexData() +} From ba06853cabe4500190ba680ccf90121bff01b530 Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:21:43 -0700 Subject: [PATCH 13/80] [kernel-spark] Support excludeRegex readOption in DSV2 (#6088) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description This PR adds support to excludeRegex read option in deltaDSv2 ## How was this patch tested? Unit tests: Create table partitioned by category with 4 versions of data. ``` Version 0: CREATE TABLE (empty) Version 1: ids {1,2} in category=alpha Version 2: ids {3,4} in category=beta Version 3: ids {5,6} in category=alpha Version 4: ids {7,8} in category=gamma ``` Test cases: "No excludeRegex - all versions", "Exclude alpha category", "Exclude beta category", "Exclude gamma category", "Exclude alpha and beta categories", "Regex matching no files", "Regex matching all files", "Exclude alpha versions 2 to 4", "Exclude beta version 2 only", "Initial snapshot - no excludeRegex", "Initial snapshot - exclude alpha". Enable integration test: "excludeRegex works and doesn't mess up offsets across restarts - parquet version" ## Does this PR introduce _any_ user-facing changes? No --- .../sql/delta/test/DeltaV2SourceSuite.scala | 4 +- .../v2/read/SparkMicroBatchStream.java | 8 + .../spark/internal/v2/read/SparkScan.java | 8 +- .../v2/read/SparkMicroBatchStreamTest.java | 205 ++++++++++++++++++ .../spark/internal/v2/read/SparkScanTest.java | 2 +- 5 files changed, 221 insertions(+), 6 deletions(-) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index d62e1cdc15f..d36873fe13e 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -67,6 +67,9 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { "rename column: should throw schema change error with unsafe flag enabled", "type widening: should fail with non-additive schema change error when enable schema tracking", + // === Read options === + "excludeRegex works and doesn't mess up offsets across restarts - parquet version", + // ========== startingVersion option tests ========== "startingVersion", "startingVersion latest", @@ -118,7 +121,6 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { // === read options === "skip change commits", - "excludeRegex works and doesn't mess up offsets across restarts - parquet version", "SC-46515: deltaSourceIgnoreChangesError contains removeFile, version, tablePath", // === Schema Evolution === diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java index 2ef4750f225..35d5b07d9bc 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java @@ -87,6 +87,7 @@ import scala.collection.immutable.Seq; import scala.collection.immutable.Seq$; import scala.jdk.javaapi.CollectionConverters; +import scala.util.matching.Regex; // TODO(#5318): Use DeltaErrors error framework for consistent error handling. public class SparkMicroBatchStream @@ -105,6 +106,7 @@ public class SparkMicroBatchStream private final String tableId; private final StructType readSchemaAtSourceInit; private final boolean shouldValidateOffsets; + private final Optional excludeRegex; private final SparkSession spark; private final String tablePath; private final StructType readDataSchema; @@ -210,6 +212,7 @@ public SparkMicroBatchStream( (Boolean) spark.sessionState().conf().getConf(DeltaSQLConf.STREAMING_OFFSET_VALIDATION()), "shouldValidateOffsets is null"); + this.excludeRegex = ScalaUtils.toJavaOptional(options.excludeRegex()); this.maxInitialSnapshotFiles = (Integer) spark @@ -427,6 +430,11 @@ public InputPartition[] planInputPartitions(Offset start, Offset end) { continue; } AddFile addFile = indexedFile.getAddFile(); + // TODO(#5319): Apply excludeRegex to RemoveFile/AddCDCFile when CDC is supported + if (excludeRegex.isPresent() + && excludeRegex.get().findFirstIn(addFile.getPath()).isDefined()) { + continue; + } PartitionedFile partitionedFile = PartitionUtils.buildPartitionedFile( addFile, partitionSchema, tablePath, ZoneId.of(sqlConf.sessionLocalTimeZone())); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java index 6c3fceacf62..95af5726540 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java @@ -60,18 +60,18 @@ public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntim DeltaOptions.STARTING_VERSION_OPTION(), DeltaOptions.STARTING_TIMESTAMP_OPTION(), DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION(), - DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION())); + DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION(), + DeltaOptions.EXCLUDE_REGEX_OPTION())); /** * Block list of DeltaOptions that are not supported for streaming in V2 connector. Only - * startingVersion, startingTimestamp, maxFilesPerTrigger, and maxBytesPerTrigger are supported. - * User-defined custom options (not in DeltaOptions) are allowed to pass through. + * startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, and excludeRegex + * are supported. User-defined custom options (not in DeltaOptions) are allowed to pass through. */ private static final Set UNSUPPORTED_STREAMING_OPTIONS = Collections.unmodifiableSet( new HashSet<>( Arrays.asList( - DeltaOptions.EXCLUDE_REGEX_OPTION().toLowerCase(), DeltaOptions.IGNORE_FILE_DELETION_OPTION().toLowerCase(), DeltaOptions.IGNORE_CHANGES_OPTION().toLowerCase(), DeltaOptions.IGNORE_DELETES_OPTION().toLowerCase(), diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java index def6dcc3605..b8424d1c764 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java @@ -27,10 +27,12 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -53,6 +55,7 @@ import org.apache.spark.sql.delta.sources.ReadMaxBytes; import org.apache.spark.sql.delta.storage.ClosableIterator; import org.apache.spark.sql.delta.util.JsonUtils; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -1647,6 +1650,208 @@ private static Stream planInputPartitionsParameters() { "Same version (3 to 3)")); } + // ================================================================================================ + // Tests for planInputPartitions with excludeRegex + // ================================================================================================ + + /** + * Parameterized test that verifies planInputPartitions correctly applies the excludeRegex read + * option. + * + *

    Uses a partitioned table so file paths contain partition directory names (e.g. + * "category=alpha/..."), making it straightforward to craft regex patterns that target specific + * subsets of files. + * + *

    Verifies correctness by extracting the {@code id} column (always at ordinal 0) from the + * reader output and comparing the resulting set of IDs against the expected set. + */ + @ParameterizedTest + @MethodSource("excludeRegexPlanInputPartitionsParameters") + public void testPlanInputPartitions_excludeRegex( + String excludeRegexPattern, + long fromVersion, + long toVersion, + int[] expectedIds, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = + "test_exclude_regex_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); + + // Create table partitioned by category with 4 versions of data. + // Version 0: CREATE TABLE (empty) + // Version 1: ids {1,2} in category=alpha + // Version 2: ids {3,4} in category=beta + // Version 3: ids {5,6} in category=alpha + // Version 4: ids {7,8} in category=gamma + sql( + "CREATE TABLE %s (id INT, name STRING, category STRING) " + + "USING delta LOCATION '%s' PARTITIONED BY (category)", + testTableName, testTablePath); + sql("INSERT INTO %s VALUES (1, 'Alice', 'alpha'), (2, 'Bob', 'alpha')", testTableName); + sql("INSERT INTO %s VALUES (3, 'Charlie', 'beta'), (4, 'David', 'beta')", testTableName); + sql("INSERT INTO %s VALUES (5, 'Eve', 'alpha'), (6, 'Frank', 'alpha')", testTableName); + sql("INSERT INTO %s VALUES (7, 'Grace', 'gamma'), (8, 'Harry', 'gamma')", testTableName); + + DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); + DeltaSourceOffset startOffset = + new DeltaSourceOffset( + deltaLog.tableId(), fromVersion, DeltaSourceOffset.BASE_INDEX(), isInitialSnapshot); + DeltaSourceOffset endOffset = + new DeltaSourceOffset( + deltaLog.tableId(), toVersion, DeltaSourceOffset.END_INDEX(), isInitialSnapshot); + + DeltaOptions options = createDeltaOptions("excludeRegex", excludeRegexPattern); + + // DSv2: planInputPartitions + createReaderFactory + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); + StructType fullSchema = deltaSnapshot.metadata().schema(); + StructType partitionSchema = deltaSnapshot.metadata().partitionSchema(); + Set partitionColNames = + Arrays.stream(partitionSchema.fieldNames()).collect(Collectors.toSet()); + StructType dataSchema = + new StructType( + Arrays.stream(fullSchema.fields()) + .filter(f -> !partitionColNames.contains(f.name())) + .toArray(StructField[]::new)); + + SparkMicroBatchStream stream = + new SparkMicroBatchStream( + snapshotManager, + snapshotManager.loadLatestSnapshot(), + spark.sessionState().newHadoopConf(), + spark, + options, + testTablePath, + dataSchema, + partitionSchema, + fullSchema, + new org.apache.spark.sql.sources.Filter[0], + Map$.MODULE$.empty()); + + InputPartition[] partitions = stream.planInputPartitions(startOffset, endOffset); + PartitionReaderFactory readerFactory = stream.createReaderFactory(); + + // Extract IDs (column ordinal 0) from reader output. + // We read only the id to avoid schema complications with partitioned tables where + // the vectorized reader may not include partition columns in the InternalRow. + List dsv2Ids = new ArrayList<>(); + for (InputPartition partition : partitions) { + if (readerFactory.supportColumnarReads(partition)) { + PartitionReader reader = + readerFactory.createColumnarReader(partition); + while (reader.next()) { + org.apache.spark.sql.vectorized.ColumnarBatch batch = reader.get(); + for (int rowId = 0; rowId < batch.numRows(); rowId++) { + dsv2Ids.add(batch.getRow(rowId).getInt(0)); + } + } + reader.close(); + } else { + PartitionReader reader = readerFactory.createReader(partition); + while (reader.next()) { + dsv2Ids.add(reader.get().getInt(0)); + } + reader.close(); + } + } + + List expected = + Arrays.stream(expectedIds).sorted().boxed().collect(Collectors.toList()); + Collections.sort(dsv2Ids); + + assertEquals( + expected, + dsv2Ids, + String.format("[%s] ID mismatch: expected=%s, got=%s", testDescription, expected, dsv2Ids)); + } + + /** Provides test parameters for the excludeRegex planInputPartitions test. */ + private static Stream excludeRegexPlanInputPartitionsParameters() { + return Stream.of( + Arguments.of( + /* excludeRegexPattern= */ (String) null, + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, + /* isInitialSnapshot= */ false, + "No excludeRegex - all versions"), + Arguments.of( + /* excludeRegexPattern= */ "category=alpha", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {3, 4, 7, 8}, + /* isInitialSnapshot= */ false, + "Exclude alpha category"), + Arguments.of( + /* excludeRegexPattern= */ "category=beta", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 5, 6, 7, 8}, + /* isInitialSnapshot= */ false, + "Exclude beta category"), + Arguments.of( + /* excludeRegexPattern= */ "category=gamma", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6}, + /* isInitialSnapshot= */ false, + "Exclude gamma category"), + Arguments.of( + /* excludeRegexPattern= */ "category=(alpha|beta)", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {7, 8}, + /* isInitialSnapshot= */ false, + "Exclude alpha and beta categories"), + Arguments.of( + /* excludeRegexPattern= */ "nonexistent_xyz_pattern", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, + /* isInitialSnapshot= */ false, + "Regex matching no files"), + Arguments.of( + /* excludeRegexPattern= */ "category=", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {}, + /* isInitialSnapshot= */ false, + "Regex matching all files"), + Arguments.of( + /* excludeRegexPattern= */ "category=alpha", + /* fromVersion= */ 2L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {3, 4, 7, 8}, + /* isInitialSnapshot= */ false, + "Exclude alpha versions 2 to 4"), + Arguments.of( + /* excludeRegexPattern= */ "category=beta", + /* fromVersion= */ 2L, + /* toVersion= */ 2L, + /* expectedIds= */ new int[] {}, + /* isInitialSnapshot= */ false, + "Exclude beta version 2 only"), + Arguments.of( + /* excludeRegexPattern= */ (String) null, + /* fromVersion= */ 0L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, + /* isInitialSnapshot= */ true, + "Initial snapshot - no excludeRegex"), + Arguments.of( + /* excludeRegexPattern= */ "category=alpha", + /* fromVersion= */ 0L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {3, 4, 7, 8}, + /* isInitialSnapshot= */ true, + "Initial snapshot - exclude alpha")); + } + /** * Helper method to convert InternalRow to Row for comparison. * diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java index e158a4a6539..961741010c5 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java @@ -445,7 +445,7 @@ public void testValidateStreamingOptions_UnsupportedOptions() { // Note: DeltaOptions uses CaseInsensitiveMap which lowercases keys during iteration assertEquals( "The following streaming options are not supported: [readchangefeed]. " - + "Supported options are: [startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger].", + + "Supported options are: [startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, excludeRegex].", exception.getMessage()); } From 7a426aea1262d41019ab32d3e92cc2e810f78312 Mon Sep 17 00:00:00 2001 From: Yi Li Date: Wed, 11 Mar 2026 17:22:04 -0700 Subject: [PATCH 14/80] [Spark] Send table schema to UC server when creating managed tables (#6204) #### Which Delta project/connector is this regarding? - [X] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Send table schema to UC server when creating managed tables ## How was this patch tested? Integration test ## Does this PR introduce _any_ user-facing changes? No. --------- Signed-off-by: Yi Li --- .../delta/commands/CreateDeltaTableLike.scala | 19 +++++-- .../sparkuctest/UCDeltaTableCreationTest.java | 50 +++++++++++++++++-- .../UCDeltaTableIntegrationBaseTest.java | 17 ++++++- .../io/sparkuctest/UnityCatalogSupport.java | 11 ++++ 4 files changed, 87 insertions(+), 10 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala index 04913683e94..2bed9506ac1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.types.StructType @@ -152,13 +153,23 @@ trait CreateDeltaTableLike extends SQLConfHelper { ++ additionalProperties, storage = storageProps, tracksPartitionsInCatalog = true) - } else { + } else if (allowCatalogManaged) { // Setting table properties is required for creating catalogManaged tables. - val properties: Map[String, String] = - if (allowCatalogManaged) UpdateCatalog.updatedProperties(snapshot) else Map.empty + table.copy( + // Here we use snapshot.schema instead of table.schema because it reflects the actual + // committed state of the table. + // Delta does not have a distinct storage type for Char/Varchar; in snapshots, they are + // represented in String type with extra type metadata. We convert them to back to the + // original Char/Varchar types when storing them in the catalog. + schema = CharVarcharUtils.getRawSchema(snapshot.schema), + partitionColumnNames = snapshot.metadata.partitionColumns, + properties = UpdateCatalog.updatedProperties(snapshot), + storage = storageProps, + tracksPartitionsInCatalog = true) + } else { table.copy( schema = new StructType(), - properties = properties, + properties = Map.empty, partitionColumnNames = Nil, // Remove write specific options when updating the catalog storage = storageProps, diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java index c0f66f4a163..737387d5798 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java @@ -27,6 +27,7 @@ import io.unitycatalog.client.model.DataSourceFormat; import io.unitycatalog.client.model.TableInfo; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -47,6 +48,7 @@ import org.apache.spark.sql.connector.catalog.TableCatalog; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; @@ -89,6 +91,16 @@ public class UCDeltaTableCreationTest extends UCDeltaTableIntegrationBaseTest { private static final String EXTERNAL_TBLPROPERTIES_CLAUSE = "TBLPROPERTIES ('Foo'='Bar')"; + /** + * Returns true if the Unity Catalog Spark version >0.4.0 so that it supports complex data types + * in columns and partition index. + */ + private static boolean isUcSparkNewerThan040() { + final int[] VER_0_4_0 = {0, 4, 0}; + int[] ucSparkVersion = getUnityCatalogSparkVersion(); + return Arrays.compare(ucSparkVersion, VER_0_4_0) > 0; + } + String tempDir; private Set tablesToCleanUp = new HashSet<>(); @@ -309,8 +321,11 @@ private void runTableCreationTest( if (replaceTable) { // First, create a different table to replace. sql( - "CREATE TABLE %s USING DELTA %s AS SELECT 0.1 AS col1", - fullTableName, MANAGED_TBLPROPERTIES_CLAUSE_OTHER); + "CREATE TABLE %s USING DELTA %s AS SELECT %s AS col1", + fullTableName, + MANAGED_TBLPROPERTIES_CLAUSE_OTHER, + // Older version UC Spark client can't support Decimal type + isUcSparkNewerThan040() ? "0.1" : "1"); tablesToCleanUp.add(fullTableName); } @@ -343,7 +358,8 @@ private void runTableCreationTest( comment, options.getExternalTableLocation(), withCluster, - options.getClusterColumn()); + options.getClusterColumn(), + options.getPartitionColumn()); } } @@ -437,6 +453,9 @@ public void testCreateOrReplaceTable(TableType tableType) throws Exception { @TestAllTableTypes public void testTableWithSupportedDataTypes(TableType tableType) throws Exception { + Assumptions.assumeTrue( + isUcSparkNewerThan040() || tableType != TableType.MANAGED, + "Older UC Spark package can't support uploading complex types to UC server for managed table"); String schema = // Numeric types "col_tinyint TINYINT, col_smallint SMALLINT, col_int INT, col_bigint BIGINT, " @@ -520,6 +539,9 @@ public void testTableWithSupportedDataTypes(TableType tableType) throws Exceptio @TestAllTableTypes public void testTableWithComplexTypes(TableType tableType) throws Exception { + Assumptions.assumeTrue( + isUcSparkNewerThan040() || tableType != TableType.MANAGED, + "Older UC Spark package can't support uploading complex types to UC server for managed table"); String schema = "id INT, arr ARRAY, " + "map_col MAP, " @@ -594,6 +616,7 @@ private void assertUCTableInfo( comment, externalTableLocation, false, + Optional.empty(), Optional.empty()); } @@ -605,7 +628,8 @@ private void assertUCTableInfo( String comment, String externalTableLocation, boolean withCluster, - Optional clusterColumn) + Optional clusterColumn, + Optional partitionColumn) throws ApiException { UnityCatalogInfo uc = unityCatalogInfo(); String catalogName = uc.catalogName(); @@ -628,9 +652,23 @@ private void assertUCTableInfo( // updated later and that would cause problem. List columns = tableInfo.getColumns(); assertThat(columns).isNotNull(); - assertThat(columns).isEmpty(); if (tableType == TableType.MANAGED) { + assertThat(columns).isNotEmpty(); + List columnNamesFromServer = + columns.stream().map(ColumnInfo::getName).collect(Collectors.toList()); + assertThat(columnNamesFromServer).containsExactlyInAnyOrderElementsOf(expectedColumns); + // Partition index is only set after UC-Spark 0.4.0 + if (isUcSparkNewerThan040() && partitionColumn.isPresent()) { + List matchingColumns = + columns.stream() + .filter(c -> c.getName().equals(partitionColumn.get())) + .collect(Collectors.toList()); + assertThat(matchingColumns).hasSize(1); + assertThat(matchingColumns.get(0).getPartitionIndex()).isEqualTo(0); + } else { + assertThat(columns.stream().anyMatch(c -> c.getPartitionIndex() != null)).isFalse(); + } // Delta sent properties of managed tables to server Map tablePropertiesFromServer = tableInfo.getProperties(); tablePropertiesFromServer.remove("table_type", "MANAGED"); // New property by Spark 4.1 @@ -688,6 +726,8 @@ private void assertUCTableInfo( && !expectedPropertiesWithVariableValue.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); assertThat(unexpectedTablePropertiesFromServer).isEmpty(); + } else { + assertThat(columns).isEmpty(); } // Also verify table using DESC EXTENDED diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java index 3d4d4e7879a..41ff9a9c723 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java @@ -75,7 +75,22 @@ Stream allTableTypesTestsFactory() { List tests = new ArrayList<>(); for (TableType tableType : ALL_TABLE_TYPES) { String testName = String.format("%s(%s)", method.getName(), tableType); - tests.add(DynamicTest.dynamicTest(testName, () -> method.invoke(this, tableType))); + tests.add( + DynamicTest.dynamicTest( + testName, + () -> { + try { + method.invoke(this, tableType); + } catch (java.lang.reflect.InvocationTargetException e) { + // Unwrap so JUnit sees the original exception. Without this, + // TestAbortedException (thrown by Assumptions) gets wrapped and + // JUnit treats the test as failed instead of skipped. + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + if (cause instanceof Error) throw (Error) cause; + throw e; + } + })); } containers.add(DynamicContainer.dynamicContainer(method.getName(), tests)); } diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java index 7a67cb7c346..46501f1d8a8 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import io.unitycatalog.client.ApiClient; import io.unitycatalog.client.ApiClientBuilder; +import io.unitycatalog.client.VersionUtils; import io.unitycatalog.client.api.CatalogsApi; import io.unitycatalog.client.api.SchemasApi; import io.unitycatalog.client.auth.TokenProvider; @@ -337,4 +338,14 @@ public void tearDownServer() { private void deleteRecursively(File file) { FileUtils.deleteQuietly(file); } + + /** Returns Unity Catalog Spark version, like [0, 4, 0]. */ + protected static int[] getUnityCatalogSparkVersion() { + String version = Preconditions.checkNotNull(VersionUtils.VERSION); + String[] parts = version.split("[.\\-]", 4); + int major = Integer.parseInt(parts[0]); + int minor = Integer.parseInt(parts[1]); + int patch = Integer.parseInt(parts[2]); + return new int[] {major, minor, patch}; + } } From e8440a67caf1b2901e168ee66c734ea4686b7d55 Mon Sep 17 00:00:00 2001 From: Leonid Lygin Date: Thu, 12 Mar 2026 03:26:43 +0100 Subject: [PATCH 15/80] [WIP] Add required java version into README.md (#6211) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [x] General repository ## Description Adds a note regarding the required Java version into the README.md. ## How was this patch tested? Inapplicable. ## Does this PR introduce _any_ user-facing changes? No. --------- Co-authored-by: Leonid Lygin Co-authored-by: Murali Ramanujam --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ebd2d24b76a..5a3c8ceaa50 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ We also adhere to the [Delta Lake Code of Conduct](https://github.com/delta-io/d ## Building Delta Lake is compiled using [SBT](https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html). +Ensure that your Java version is at least 17 (you can verify with `java -version`). To compile, run From 7a7e0f54a5eedceb2f74d2b28f72755bf6c43c7c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 12 Mar 2026 14:21:55 +0100 Subject: [PATCH 16/80] Fix potential NPE (#6253) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [x] Other (Iceberg) ## Description We have the following NPE: ``` java.lang.NullPointerException at java.base/java.util.Objects.requireNonNull(Objects.java:209) at org.apache.iceberg.SchemaParser.fromJson(SchemaParser.java:296) at org.apache.iceberg.ManifestReader.readPartitionSpec(ManifestReader.java:144) at org.apache.iceberg.ManifestReader.(ManifestReader.java:129) at org.apache.iceberg.ManifestFiles.read(ManifestFiles.java:138) at org.apache.iceberg.ManifestFiles.read(ManifestFiles.java:114) ``` Referring to this line: https://github.com/apache/iceberg/blob/apache-iceberg-1.9.0/core/src/main/java/org/apache/iceberg/SchemaParser.java#L296 This comes from the following line: https://github.com/apache/iceberg/blob/7dbafb438ee1e68d0047bebcb587265d7d87d8a1/core/src/main/java/org/apache/iceberg/ManifestReader.java#L144 The field `schema` is only set in V1 fields. I believe the reference implementation (Java OSS) also sets this field for V2/V3 to maximize compatibility, but this might be different for other writers. By passing in the spec, we don't execute the branch and we avoid the NPE: https://github.com/apache/iceberg/blob/7dbafb438ee1e68d0047bebcb587265d7d87d8a1/core/src/main/java/org/apache/iceberg/ManifestReader.java#L126-L130 ## How was this patch tested? ## Does this PR introduce _any_ user-facing changes? --- .../spark/sql/delta/IcebergFileManifest.scala | 6 +- .../delta/ConvertIcebergToDeltaSuite.scala | 81 ++++++++++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala index 9109c7d0d34..590348a622d 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala @@ -182,7 +182,11 @@ class IcebergFileManifest( val dataFiles = spark .createDataset(manifestFiles) - .flatMap(ManifestFiles.read(_, localTable.io()).asScala.map(new DataFileWrapper(_))) + .flatMap( + ManifestFiles.read(_, localTable.io(), localTable.specs()) + .asScala.map(new DataFileWrapper(_) + ) + ) dataFiles } diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala index b337076f91e..4338a819637 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine -import java.io.File +import java.io.{ByteArrayOutputStream, File} import java.text.SimpleDateFormat import java.util.TimeZone @@ -32,6 +32,8 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatsUtils import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.Path +import org.apache.avro.file.{DataFileReader, DataFileWriter, SeekableByteArrayInput} +import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord} import org.apache.iceberg.{Table, TableProperties} import org.apache.iceberg.hadoop.HadoopTables import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions @@ -591,6 +593,83 @@ trait ConvertIcebergToDeltaSuiteBase } } + /** + * Strips the "schema" metadata key from all manifest Avro files of the table's current snapshot. + * This simulates a V2 writer that omits the optional schema field. Without passing specsById + * to ManifestFiles.read, reading such manifests causes an NPE in SchemaParser.fromJson. + */ + private def stripSchemaFromManifests(table: Table): Unit = { + val manifests = table.currentSnapshot().dataManifests(table.io()).asScala + // scalastyle:off deltahadoopconfiguration + val conf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + manifests.foreach { manifest => + val path = new Path(manifest.path()) + val fs = path.getFileSystem(conf) + + // Read the entire manifest file into a byte array + val inputStream = fs.open(path) + val bytes = try { + org.apache.commons.io.IOUtils.toByteArray(inputStream) + } finally { + inputStream.close() + } + + val datumReader = new GenericDatumReader[GenericRecord]() + val reader = new DataFileReader[GenericRecord]( + new SeekableByteArrayInput(bytes), datumReader) + + // Collect records and metadata + val records = new java.util.ArrayList[GenericRecord]() + while (reader.hasNext) { + records.add(reader.next()) + } + val avroSchema = reader.getSchema + val metaKeys = reader.getMetaKeys.asScala.toSeq + + // Write back without the "schema" metadata key + val out = new ByteArrayOutputStream() + val datumWriter = new GenericDatumWriter[GenericRecord](avroSchema) + val writer = new DataFileWriter[GenericRecord](datumWriter) + val reservedKeys = Set("schema", "avro.schema", "avro.codec") + metaKeys.filterNot(reservedKeys.contains).foreach { key => + writer.setMeta(key, reader.getMeta(key)) + } + writer.create(avroSchema, out) + records.asScala.foreach(writer.append) + writer.close() + reader.close() + + // Overwrite the original file + val outputStream = fs.create(path, true) + try { + outputStream.write(out.toByteArray) + } finally { + outputStream.close() + } + } + } + + test("convert Iceberg table with manifest missing schema metadata") { + withTable(table) { + spark.sql( + s"""CREATE TABLE $table (id bigint, data string) + |USING iceberg PARTITIONED BY (data) + |TBLPROPERTIES ("format-version" = "2")""".stripMargin) + Seq((1L, "a"), (2L, "b")).toDF("id", "data") + .write.format("iceberg").mode("append").saveAsTable(table) + // Strip the "schema" metadata from manifest Avro files to simulate a V2 writer + // that omits the optional schema field. Without passing specsById to + // ManifestFiles.read, this causes an NPE in SchemaParser.fromJson. + val iceTable = readIcebergHadoopTable(tablePath) + stripSchemaFromManifests(iceTable) + convert(s"iceberg.`$tablePath`") + checkAnswer( + spark.read.format("delta").load(tablePath), + Row(1, "a") :: Row(2, "b") :: Nil) + } + } + test("convert Iceberg table with not null columns") { withTable(table) { spark.sql( From e003bdb9c3027bf258ca74b69f7bd005f247b164 Mon Sep 17 00:00:00 2001 From: yyanyy Date: Thu, 12 Mar 2026 10:27:44 -0700 Subject: [PATCH 17/80] [kernel-spark] Populate catalog stats in DSv2 scan (#5952) (#6203) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [x] Kernel - [ ] Other (fill in here) ## Description Convert CatalogStatistics to V2 connector Statistics and thread them through SparkTable → SparkScanBuilder → SparkScan.estimateStatistics(). Gate numRows/columnStats on cboEnabled || planStatsEnabled to match V1 LogicalRelation.computeStats() behavior. Use per-column avgLen from catalog stats in size estimation instead of defaultSize(), mirroring EstimationUtils.getSizePerRow() with UTF8String overhead for StringType columns. Resolves https://github.com/delta-io/delta/issues/5952 ## How was this patch tested? Unit test ## Does this PR introduce _any_ user-facing changes? No --- .../spark/internal/v2/catalog/SparkTable.java | 13 + .../spark/internal/v2/read/SparkScan.java | 103 ++++- .../internal/v2/read/SparkScanBuilder.java | 6 + .../spark/internal/v2/utils/StatsUtils.java | 170 +++++++ .../spark/internal/v2/DeltaV2TestBase.java | 24 + .../v2/read/SparkScanBuilderTest.java | 3 + .../spark/internal/v2/read/SparkScanTest.java | 414 ++++++++++++++++++ .../internal/v2/utils/StatsUtilsTest.java | 162 +++++++ 8 files changed, 889 insertions(+), 6 deletions(-) create mode 100644 spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StatsUtils.java create mode 100644 spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StatsUtilsTest.java diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java index 4ee7b22904b..4ad34c54e29 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java @@ -15,7 +15,9 @@ */ package io.delta.spark.internal.v2.catalog; +import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaOptional; import static io.delta.spark.internal.v2.utils.ScalaUtils.toScalaMap; +import static io.delta.spark.internal.v2.utils.StatsUtils.toV2Statistics; import static java.util.Objects.requireNonNull; import io.delta.kernel.Snapshot; @@ -35,6 +37,7 @@ import org.apache.spark.sql.connector.expressions.Expressions; import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.write.LogicalWriteInfo; import org.apache.spark.sql.connector.write.WriteBuilder; import org.apache.spark.sql.delta.DeltaTableUtils; @@ -236,12 +239,22 @@ public ScanBuilder newScanBuilder(CaseInsensitiveStringMap scanOptions) { Map combined = new HashMap<>(this.options); combined.putAll(scanOptions.asCaseSensitiveMap()); CaseInsensitiveStringMap merged = new CaseInsensitiveStringMap(combined); + Optional catalogStats = + catalogTable + .flatMap(ct -> toJavaOptional(ct.stats())) + .map( + stats -> + toV2Statistics( + stats, + schemaProvider.getDataSchema(), + schemaProvider.getPartitionSchema())); return new SparkScanBuilder( name(), initialSnapshot, snapshotManager, schemaProvider.getDataSchema(), schemaProvider.getPartitionSchema(), + catalogStats, merged); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java index 95af5726540..1fbd2aec235 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java @@ -41,11 +41,13 @@ import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.read.*; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.execution.datasources.*; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; +import org.apache.spark.sql.types.StringType; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; @@ -96,6 +98,7 @@ public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntim private final Predicate[] pushedToKernelFilters; private final Filter[] dataFilters; private final io.delta.kernel.Scan kernelScan; + private final Optional catalogStats; private final Configuration hadoopConf; private final CaseInsensitiveStringMap options; private final scala.collection.immutable.Map scalaOptions; @@ -123,6 +126,7 @@ public SparkScan( Predicate[] pushedToKernelFilters, Filter[] dataFilters, io.delta.kernel.Scan kernelScan, + Optional catalogStats, CaseInsensitiveStringMap options) { this.snapshotManager = Objects.requireNonNull(snapshotManager, "snapshotManager is null"); @@ -134,6 +138,7 @@ public SparkScan( pushedToKernelFilters == null ? new Predicate[0] : pushedToKernelFilters.clone(); this.dataFilters = dataFilters == null ? new Filter[0] : dataFilters.clone(); this.kernelScan = Objects.requireNonNull(kernelScan, "kernelScan is null"); + this.catalogStats = Objects.requireNonNull(catalogStats, "catalogStats is null"); this.options = Objects.requireNonNull(options, "options is null"); this.scalaOptions = ScalaUtils.toScalaMap(options); this.hadoopConf = SparkSession.active().sessionState().newHadoopConfWithOptions(scalaOptions); @@ -207,15 +212,49 @@ public String description() { @Override public Statistics estimateStatistics() { ensurePlanned(); + final long plannedBytes = estimatedSizeInBytes; + + // When catalog stats are available and CBO is enabled, combine table-level stats + // (for numRows/columnStats) with planned file stats (for sizeInBytes). + // This mirrors V1's LogicalRelation.computeStats() which gates column stats on + // conf.cboEnabled || conf.planStatsEnabled. + boolean useCatalogStats = sqlConf.cboEnabled() || sqlConf.planStatsEnabled(); + if (useCatalogStats && catalogStats.isPresent()) { + final Statistics stats = catalogStats.get(); + return new Statistics() { + @Override + public OptionalLong sizeInBytes() { + // Planned file size is authoritative (even if 0 for an empty table) + return OptionalLong.of(plannedBytes); + } + + @Override + public OptionalLong numRows() { + // TODO: Use accurate row count from planned files (sum of AddFile.numRecords) + // instead of catalog stats, which are stale (point-in-time from ANALYZE) and + // not adjusted for partition pruning. + return stats.numRows(); + } + + @Override + public Map columnStats() { + // TODO: After partition pruning, column stats (e.g. min, max, nullCount, + // distinctCount) could be tightened based on the pruned file-level stats. + return stats.columnStats(); + } + }; + } + + // No catalog stats available or CBO disabled — return stats from planned files only return new Statistics() { @Override public OptionalLong sizeInBytes() { - return OptionalLong.of(estimatedSizeInBytes); + return OptionalLong.of(plannedBytes); } @Override public OptionalLong numRows() { - // Row count is unknown at planning time. + // Row count is unknown without catalog stats return OptionalLong.empty(); } }; @@ -237,6 +276,10 @@ public OptionalLong numRows() { * (equivalent to Project output) * * + *

    When catalog column stats are available, uses per-column {@code avgLen} instead of {@code + * defaultSize()} for more accurate size estimation, mirroring {@code + * EstimationUtils.getSizePerRow()} behavior. + * *

    This provides consistent statistics with the v1 code path (LogicalRelation + visitUnaryNode * from Spark code directory). * @@ -251,16 +294,62 @@ private long computeEstimatedSizeWithColumnProjection(long totalBytes) { // Row overhead constant, matching EstimationUtils.getSizePerRow (from Spark) final int ROW_OVERHEAD = 8; - // TODO(#5952): update below with column stats from catalog when it becomes available + // Use avgLen from catalog column stats when available for more accurate estimation + Map avgLenByColumn = getAvgLenByColumn(); + final long fullSchemaRowSize = - ROW_OVERHEAD + dataSchema.defaultSize() + partitionSchema.defaultSize(); - final long outputRowSize = ROW_OVERHEAD + readSchema().defaultSize(); + ROW_OVERHEAD + + getSchemaSize(dataSchema, avgLenByColumn) + + getSchemaSize(partitionSchema, avgLenByColumn); + final long outputRowSize = ROW_OVERHEAD + getSchemaSize(readSchema(), avgLenByColumn); long estimatedBytes = (totalBytes * outputRowSize) / fullSchemaRowSize; return Math.max(1L, estimatedBytes); } + /** + * Returns a map of column name to avgLen from catalog column stats, used to improve size + * estimation accuracy when catalog stats are available. + */ + private Map getAvgLenByColumn() { + if (!catalogStats.isPresent()) { + return Collections.emptyMap(); + } + Map colStats = catalogStats.get().columnStats(); + if (colStats.isEmpty()) { + return Collections.emptyMap(); + } + Map result = new HashMap<>(); + for (Map.Entry entry : colStats.entrySet()) { + result.put(entry.getKey().fieldNames()[0], entry.getValue().avgLen()); + } + return result; + } + + /** + * Computes the estimated in-memory size for a schema, using avgLen from catalog stats when + * available, falling back to defaultSize(). Mirrors EstimationUtils.getSizePerRow(). For + * StringType columns with avgLen, adds UTF8String overhead (base + offset + numBytes = 12 bytes). + */ + private static long getSchemaSize(StructType schema, Map avgLenByColumn) { + long size = 0; + for (StructField field : schema.fields()) { + OptionalLong avgLen = avgLenByColumn.getOrDefault(field.name(), OptionalLong.empty()); + if (avgLen.isPresent()) { + if (field.dataType() instanceof StringType) { + // UTF8String: base + offset + numBytes (matching EstimationUtils.getSizePerRow) + size += avgLen.getAsLong() + 8 + 4; + } else { + size += avgLen.getAsLong(); + } + } else { + size += field.dataType().defaultSize(); + } + } + return size; + } + /** * Get the table path from the scan state. * @@ -458,13 +547,15 @@ public boolean equals(Object o) { // ignoring kernelScan because it is derived from Snapshot which is created from tablePath, // with pushed down filters that are also recorded in `pushedToKernelFilters` && Objects.equals(options, that.options) - && Objects.equals(appliedRuntimePredicates, that.appliedRuntimePredicates); + && Objects.equals(appliedRuntimePredicates, that.appliedRuntimePredicates) + && Objects.equals(catalogStats, that.catalogStats); } @Override public int hashCode() { int result = Objects.hash( + catalogStats, initialSnapshot.getPath(), initialSnapshot.getVersion(), dataSchema, diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java index d9a6e029047..e470c579457 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java @@ -25,6 +25,7 @@ import java.util.*; import java.util.stream.Collectors; import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.SupportsPushDownFilters; import org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns; import org.apache.spark.sql.sources.Filter; @@ -44,6 +45,7 @@ public class SparkScanBuilder private final DeltaSnapshotManager snapshotManager; private final StructType dataSchema; private final StructType partitionSchema; + private final Optional catalogStats; private final CaseInsensitiveStringMap options; private final Set partitionColumnSet; private StructType requiredDataSchema; @@ -63,6 +65,7 @@ public class SparkScanBuilder * @param snapshotManager the snapshot manager for this table * @param dataSchema the data schema (non-partition columns) * @param partitionSchema the partition schema + * @param catalogStats optional V2 Statistics converted from catalog stats * @param options scan options */ public SparkScanBuilder( @@ -71,12 +74,14 @@ public SparkScanBuilder( DeltaSnapshotManager snapshotManager, StructType dataSchema, StructType partitionSchema, + Optional catalogStats, CaseInsensitiveStringMap options) { this.initialSnapshot = requireNonNull(initialSnapshot, "initialSnapshot is null"); this.kernelScanBuilder = initialSnapshot.getScanBuilder(); this.snapshotManager = requireNonNull(snapshotManager, "snapshotManager is null"); this.dataSchema = requireNonNull(dataSchema, "dataSchema is null"); this.partitionSchema = requireNonNull(partitionSchema, "partitionSchema is null"); + this.catalogStats = requireNonNull(catalogStats, "catalogStats is null"); this.options = requireNonNull(options, "options is null"); this.requiredDataSchema = this.dataSchema; this.partitionColumnSet = @@ -167,6 +172,7 @@ public org.apache.spark.sql.connector.read.Scan build() { pushedKernelPredicates, dataFilters, kernelScanBuilder.build(), + catalogStats, options); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StatsUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StatsUtils.java new file mode 100644 index 00000000000..cc113b79b63 --- /dev/null +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StatsUtils.java @@ -0,0 +1,170 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.utils; + +import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaOptional; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; +import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat$; +import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; +import org.apache.spark.sql.connector.expressions.FieldReference; +import org.apache.spark.sql.connector.expressions.NamedReference; +import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +/** Utilities for converting catalog statistics to V2 connector statistics. */ +public final class StatsUtils { + + private StatsUtils() {} + + /** + * Convert {@link CatalogStatistics} to V2 connector {@link Statistics}. + * + * @param catalogStats the catalog statistics to convert + * @param dataSchema the data schema (non-partition columns) + * @param partitionSchema the partition schema + * @return V2 Statistics representation + */ + public static Statistics toV2Statistics( + CatalogStatistics catalogStats, StructType dataSchema, StructType partitionSchema) { + // Build a map of column name -> DataType from both schemas + Map columnTypes = new HashMap<>(); + for (StructField field : dataSchema.fields()) { + columnTypes.put(field.name(), field.dataType()); + } + for (StructField field : partitionSchema.fields()) { + columnTypes.put(field.name(), field.dataType()); + } + + Map colStatsMap = buildColumnStats(catalogStats, columnTypes); + + return new Statistics() { + @Override + public OptionalLong sizeInBytes() { + return OptionalLong.of(catalogStats.sizeInBytes().longValue()); + } + + @Override + public OptionalLong numRows() { + return toJavaOptional(catalogStats.rowCount()) + .map(r -> OptionalLong.of(r.longValue())) + .orElse(OptionalLong.empty()); + } + + @Override + public Map columnStats() { + return colStatsMap; + } + }; + } + + private static Map buildColumnStats( + CatalogStatistics catalogStats, Map columnTypes) { + Map colStats = + scala.collection.JavaConverters.mapAsJavaMapConverter(catalogStats.colStats()).asJava(); + + if (colStats.isEmpty()) { + return Collections.emptyMap(); + } + + Map result = new HashMap<>(); + for (Map.Entry entry : colStats.entrySet()) { + String colName = entry.getKey(); + CatalogColumnStat stat = entry.getValue(); + DataType dataType = columnTypes.get(colName); + + if (dataType == null) { + continue; + } + + NamedReference ref = FieldReference.apply(colName); + int version = stat.version(); + + // Eagerly parse min/max to avoid repeated fromExternalString calls + Optional minValue = + toJavaOptional(stat.min()) + .map( + minStr -> + CatalogColumnStat$.MODULE$.fromExternalString( + minStr, colName, dataType, version)); + Optional maxValue = + toJavaOptional(stat.max()) + .map( + maxStr -> + CatalogColumnStat$.MODULE$.fromExternalString( + maxStr, colName, dataType, version)); + OptionalLong distinctCount = + toJavaOptional(stat.distinctCount()) + .map(d -> OptionalLong.of(d.longValue())) + .orElse(OptionalLong.empty()); + OptionalLong nullCount = + toJavaOptional(stat.nullCount()) + .map(n -> OptionalLong.of(n.longValue())) + .orElse(OptionalLong.empty()); + OptionalLong avgLen = + toJavaOptional(stat.avgLen()) + .map(v -> OptionalLong.of(((Number) v).longValue())) + .orElse(OptionalLong.empty()); + OptionalLong maxLen = + toJavaOptional(stat.maxLen()) + .map(v -> OptionalLong.of(((Number) v).longValue())) + .orElse(OptionalLong.empty()); + + ColumnStatistics v2ColStats = + new ColumnStatistics() { + @Override + public OptionalLong distinctCount() { + return distinctCount; + } + + @Override + public Optional min() { + return minValue; + } + + @Override + public Optional max() { + return maxValue; + } + + @Override + public OptionalLong nullCount() { + return nullCount; + } + + @Override + public OptionalLong avgLen() { + return avgLen; + } + + @Override + public OptionalLong maxLen() { + return maxLen; + } + }; + result.put(ref, v2ColStats); + } + return Collections.unmodifiableMap(result); + } +} diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java index f05c3223c71..6d21bb0abe0 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java @@ -83,6 +83,30 @@ protected void createSchemaEvolutionTestTable(String path, String tableName) { tableName)); } + /** A runnable that can throw checked exceptions, for use with {@link #withSQLConf}. */ + @FunctionalInterface + protected interface ThrowingRunnable { + void run() throws Exception; + } + + /** + * Runs the given action with a Spark SQL configuration temporarily set, then restores the + * original value afterwards (similar to Scala's {@code withSQLConf}). + */ + protected void withSQLConf(String key, String value, ThrowingRunnable action) throws Exception { + scala.Option original = spark.conf().getOption(key); + spark.conf().set(key, value); + try { + action.run(); + } finally { + if (original.isDefined()) { + spark.conf().set(key, original.get()); + } else { + spark.conf().unset(key); + } + } + } + protected static void createPartitionedTable(String tableName, String path) { spark.sql( String.format( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java index 482f9fa8416..01f06aad9b0 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java @@ -69,6 +69,7 @@ public void testBuild_returnsScanWithExpectedSchema(@TempDir File tempDir) { snapshotManager, dataSchema, partitionSchema, + Optional.empty(), CaseInsensitiveStringMap.empty()); StructType expectedSparkSchema = @@ -112,6 +113,7 @@ public void testToMicroBatchStream_returnsSparkMicroBatchStream(@TempDir File te snapshotManager, dataSchema, partitionSchema, + Optional.empty(), CaseInsensitiveStringMap.empty()); Scan scan = builder.build(); @@ -745,6 +747,7 @@ private SparkScanBuilder createTestScanBuilder(File tempDir) { snapshotManager, dataSchema, partitionSchema, + Optional.empty(), CaseInsensitiveStringMap.empty()); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java index 961741010c5..39f3730ef20 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java @@ -9,18 +9,26 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.spark.sql.catalyst.TableIdentifier; +import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; +import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; +import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.expressions.Expression; import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.LiteralValue; +import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.expressions.filter.Predicate; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.types.DataTypes; @@ -810,4 +818,410 @@ public void testNotEqualsWithAndWithoutRuntimeFilter() { assertNotEquals(scan1, scan2); assertNotEquals(scan1.hashCode(), scan2.hashCode()); } + + // ================================================================================================ + // Tests for catalog statistics propagation + // ================================================================================================ + + /** + * Helper to inject CatalogStatistics into a catalog table via alterTableStats. This is needed + * because ANALYZE TABLE is not supported for V2 tables in the test environment. + */ + private CatalogTable injectCatalogStats(String tblName, CatalogStatistics stats) + throws Exception { + TableIdentifier tableId = new TableIdentifier(tblName); + spark.sessionState().catalog().alterTableStats(tableId, scala.Option.apply(stats)); + return spark.sessionState().catalog().getTableMetadata(tableId); + } + + @Test + public void testEstimateStatisticsWithCatalogStats_cboEnabled(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_cbo_enabled"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING, value DOUBLE) USING delta LOCATION '%s'", + tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a', 1.0), (2, 'b', 2.0)", tblName)); + + // Inject catalog stats with column stats for "id" + CatalogColumnStat idColStat = + new CatalogColumnStat( + scala.Option.apply(scala.math.BigInt.apply(2L)), // distinctCount + scala.Option.apply("1"), // min + scala.Option.apply("2"), // max + scala.Option.apply(scala.math.BigInt.apply(0L)), // nullCount + scala.Option.apply((Object) 4L), // avgLen + scala.Option.apply((Object) 4L), // maxLen + scala.Option.empty(), // histogram + CatalogColumnStat.VERSION()); + + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(1024L), + scala.Option.apply(scala.math.BigInt.apply(2L)), + buildColStatsMap(new String[] {"id"}, new CatalogColumnStat[] {idColStat})); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + withSQLConf( + "spark.sql.cbo.enabled", + "true", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // Should have numRows from catalog stats + assertTrue(stats.numRows().isPresent(), "numRows should be present with CBO enabled"); + assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); + + // sizeInBytes should still come from planned files (more accurate) + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + + // Should have column stats + Map colStats = stats.columnStats(); + assertNotNull(colStats, "columnStats should not be null"); + assertFalse(colStats.isEmpty(), "columnStats should not be empty"); + + // Check that column stats contain expected columns + ColumnStatistics idStats = colStats.get(FieldReference.apply("id")); + assertNotNull(idStats, "id column stats should be present"); + assertTrue(idStats.nullCount().isPresent(), "id nullCount should be present"); + assertTrue(idStats.distinctCount().isPresent(), "id distinctCount should be present"); + assertTrue(idStats.min().isPresent(), "id min should be present"); + assertTrue(idStats.max().isPresent(), "id max should be present"); + assertEquals(1, idStats.min().get(), "id min should be 1"); + assertEquals(2, idStats.max().get(), "id max should be 2"); + }); + } + + @Test + public void testEstimateStatisticsWithCatalogStats_cboDisabled(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_cbo_disabled"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tblName)); + + // Inject catalog stats + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(512L), + scala.Option.apply(scala.math.BigInt.apply(2L)), + scala.collection.immutable.Map$.MODULE$.empty()); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // With CBO disabled, numRows should be empty (matching V1 behavior) + assertFalse(stats.numRows().isPresent(), "numRows should be empty with CBO disabled"); + + // sizeInBytes should still come from planned files + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + }); + } + + @Test + public void testEstimateStatisticsWithCatalogStats_planStatsEnabled(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_plan_stats_enabled"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tblName)); + + // Inject catalog stats with numRows + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(512L), + scala.Option.apply(scala.math.BigInt.apply(2L)), + scala.collection.immutable.Map$.MODULE$.empty()); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + // CBO disabled but planStatsEnabled=true should still surface catalog stats + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> { + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "true", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // With planStatsEnabled, numRows should come from catalog stats + assertTrue( + stats.numRows().isPresent(), "numRows should be present with planStatsEnabled"); + assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); + + // sizeInBytes should still come from planned files + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + }); + }); + } + + @Test + public void testEstimateStatisticsWithoutCatalogStats(@TempDir File tempDir) throws Exception { + // Path-based table has no catalog stats + String path = tempDir.getAbsolutePath(); + String tblName = "stats_no_catalog"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); + + withSQLConf( + "spark.sql.cbo.enabled", + "true", + () -> { + // Path-based table — no catalog table, no stats + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, path); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // Without catalog stats, numRows should be empty + assertFalse(stats.numRows().isPresent(), "numRows should be empty for path-based table"); + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + }); + } + + @Test + public void testEstimateStatisticsWithPartitionedTableAndCatalogStats(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_partitioned"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING, part INT) USING delta " + + "PARTITIONED BY (part) LOCATION '%s'", + tblName, path)); + spark.sql( + String.format("INSERT INTO %s VALUES (1, 'a', 1), (2, 'b', 1), (3, 'c', 2)", tblName)); + + // Inject catalog stats with column stats for both data and partition columns + CatalogColumnStat idColStat = + new CatalogColumnStat( + scala.Option.apply(scala.math.BigInt.apply(3L)), + scala.Option.apply("1"), + scala.Option.apply("3"), + scala.Option.apply(scala.math.BigInt.apply(0L)), + scala.Option.empty(), + scala.Option.empty(), + scala.Option.empty(), + CatalogColumnStat.VERSION()); + + CatalogColumnStat partColStat = + new CatalogColumnStat( + scala.Option.apply(scala.math.BigInt.apply(2L)), + scala.Option.apply("1"), + scala.Option.apply("2"), + scala.Option.apply(scala.math.BigInt.apply(0L)), + scala.Option.empty(), + scala.Option.empty(), + scala.Option.empty(), + CatalogColumnStat.VERSION()); + + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(2048L), + scala.Option.apply(scala.math.BigInt.apply(3L)), + buildColStatsMap( + new String[] {"id", "part"}, new CatalogColumnStat[] {idColStat, partColStat})); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + withSQLConf( + "spark.sql.cbo.enabled", + "true", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + assertTrue(stats.numRows().isPresent(), "numRows should be present"); + assertEquals(3L, stats.numRows().getAsLong(), "numRows should be 3"); + + // Verify column stats include both data and partition columns + Map colStats = stats.columnStats(); + assertNotNull(colStats.get(FieldReference.apply("id")), "id stats should be present"); + assertNotNull(colStats.get(FieldReference.apply("part")), "part stats should be present"); + + // Check partition column stats + ColumnStatistics partStats = colStats.get(FieldReference.apply("part")); + assertTrue(partStats.min().isPresent(), "part min should be present"); + assertTrue(partStats.max().isPresent(), "part max should be present"); + assertEquals(1, partStats.min().get(), "part min should be 1"); + assertEquals(2, partStats.max().get(), "part max should be 2"); + }); + } + + @Test + public void testEstimatedSizeUsesAvgLenFromCatalogStats(@TempDir File tempDir) throws Exception { + // Verify that computeEstimatedSizeWithColumnProjection uses avgLen from catalog stats + // instead of defaultSize(), mirroring EstimationUtils.getSizePerRow() (#5952). + String path = tempDir.getAbsolutePath(); + String tblName = "stats_avglen"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'bb'), (3, 'ccc')", tblName)); + + // Create column stats with avgLen=5 for 'name' (STRING defaultSize is 20) + CatalogColumnStat nameColStat = + new CatalogColumnStat( + scala.Option.empty(), + scala.Option.empty(), + scala.Option.empty(), + scala.Option.empty(), + scala.Option.apply((Object) 5L), // avgLen = 5 (vs STRING defaultSize 20) + scala.Option.empty(), + scala.Option.empty(), + CatalogColumnStat.VERSION()); + + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(1024L), + scala.Option.apply(scala.math.BigInt.apply(3L)), + buildColStatsMap(new String[] {"name"}, new CatalogColumnStat[] {nameColStat})); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + // avgLen is used for sizeInBytes estimation regardless of CBO/planStats settings + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> { + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "false", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + + // Prune to only 'name' column to trigger column projection estimation + StructType prunedSchema = new StructType().add("name", DataTypes.StringType); + builder.pruneColumns(prunedSchema); + + SparkScan scan = (SparkScan) builder.build(); + + long totalBytes = getTotalBytes(scan); + long estimatedSize = getEstimatedSizeInBytes(scan); + assertTrue(totalBytes > 0, "totalBytes should be positive"); + + // Schema: dataSchema = (id INT, name STRING), partitionSchema = empty + // readSchema = (name STRING) after pruning + // + // With avgLen=5 for name (STRING: avgLen + 12 = 17, vs defaultSize 20): + // fullSchemaRowSize = 8 + (4 [INT default] + 17 [STRING avgLen]) = 29 + // outputRowSize = 8 + 17 = 25 + // estimated = (totalBytes * 25) / 29 + // + // Without avgLen (defaultSize only): + // fullSchemaRowSize = 8 + (4 + 20) = 32 + // outputRowSize = 8 + 20 = 28 + // estimated = (totalBytes * 28) / 32 + long expectedWithAvgLen = (totalBytes * 25) / 29; + long expectedWithoutAvgLen = (totalBytes * 28) / 32; + + assertEquals( + expectedWithAvgLen, + estimatedSize, + "estimatedSize should use avgLen from catalog stats"); + assertNotEquals( + expectedWithoutAvgLen, + estimatedSize, + "estimatedSize should differ from defaultSize-only calculation"); + }); + }); + } + + @Test + public void testEstimateStatisticsWithoutAnalyze(@TempDir File tempDir) throws Exception { + // Table exists in catalog but no stats were injected + String path = tempDir.getAbsolutePath(); + String tblName = "stats_no_analyze"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); + + withSQLConf( + "spark.sql.cbo.enabled", + "true", + () -> { + CatalogTable catalogTable = + spark.sessionState().catalog().getTableMetadata(new TableIdentifier(tblName)); + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // Without catalog stats, we fall back to file-only stats + assertFalse(stats.numRows().isPresent(), "numRows should be empty without catalog stats"); + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + }); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static scala.collection.immutable.Map buildColStatsMap( + String[] keys, CatalogColumnStat[] values) { + scala.collection.mutable.Builder b = scala.collection.immutable.Map$.MODULE$.newBuilder(); + for (int i = 0; i < keys.length; i++) { + b.$plus$eq(new scala.Tuple2<>(keys[i], values[i])); + } + return (scala.collection.immutable.Map) b.result(); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StatsUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StatsUtilsTest.java new file mode 100644 index 00000000000..8b35cd97fc5 --- /dev/null +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StatsUtilsTest.java @@ -0,0 +1,162 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; +import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; +import org.apache.spark.sql.connector.expressions.FieldReference; +import org.apache.spark.sql.connector.expressions.NamedReference; +import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Test; +import scala.Option; + +class StatsUtilsTest { + + @Test + void testToV2Statistics_sizeAndRowCount() { + StructType dataSchema = + new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); + StructType partitionSchema = new StructType(); + + CatalogStatistics catalogStats = + new CatalogStatistics( + BigInt(1024L), + Option.apply(BigInt(100L)), + scala.collection.immutable.Map$.MODULE$.empty()); + + Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); + + assertEquals(1024L, v2Stats.sizeInBytes().getAsLong(), "sizeInBytes should match"); + assertTrue(v2Stats.numRows().isPresent(), "numRows should be present"); + assertEquals(100L, v2Stats.numRows().getAsLong(), "numRows should match"); + assertTrue(v2Stats.columnStats().isEmpty(), "columnStats should be empty"); + } + + @Test + void testToV2Statistics_sizeOnlyNoRowCount() { + StructType dataSchema = new StructType().add("id", DataTypes.IntegerType); + StructType partitionSchema = new StructType(); + + CatalogStatistics catalogStats = + new CatalogStatistics( + BigInt(512L), Option.empty(), scala.collection.immutable.Map$.MODULE$.empty()); + + Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); + + assertEquals(512L, v2Stats.sizeInBytes().getAsLong(), "sizeInBytes should match"); + assertFalse(v2Stats.numRows().isPresent(), "numRows should be empty"); + } + + @Test + void testToV2Statistics_withColumnStats() { + StructType dataSchema = + new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); + StructType partitionSchema = new StructType().add("part", DataTypes.IntegerType); + + // Create column stats for "id" column + CatalogColumnStat idColStat = + new CatalogColumnStat( + Option.apply(BigInt(10L)), // distinctCount + Option.apply("1"), // min + Option.apply("100"), // max + Option.apply(BigInt(0L)), // nullCount + Option.apply((Object) 4L), // avgLen + Option.apply((Object) 4L), // maxLen + Option.empty(), // histogram + CatalogColumnStat.VERSION()); + + scala.collection.immutable.Map colStatsMap = + buildScalaMap(new String[] {"id"}, new CatalogColumnStat[] {idColStat}); + + CatalogStatistics catalogStats = + new CatalogStatistics(BigInt(2048L), Option.apply(BigInt(50L)), colStatsMap); + + Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); + + assertEquals(2048L, v2Stats.sizeInBytes().getAsLong()); + assertEquals(50L, v2Stats.numRows().getAsLong()); + + Map colStats = v2Stats.columnStats(); + assertEquals(1, colStats.size(), "Should have 1 column stat"); + + ColumnStatistics idStats = colStats.get(FieldReference.apply("id")); + assertNotNull(idStats, "id column stats should be present"); + assertEquals(10L, idStats.distinctCount().getAsLong(), "distinctCount should be 10"); + assertEquals(0L, idStats.nullCount().getAsLong(), "nullCount should be 0"); + assertEquals(4L, idStats.avgLen().getAsLong(), "avgLen should be 4"); + assertEquals(4L, idStats.maxLen().getAsLong(), "maxLen should be 4"); + assertTrue(idStats.min().isPresent(), "min should be present"); + assertTrue(idStats.max().isPresent(), "max should be present"); + assertEquals(1, idStats.min().get(), "min should be 1"); + assertEquals(100, idStats.max().get(), "max should be 100"); + } + + @Test + void testToV2Statistics_skipsColumnsNotInSchema() { + // Only "id" is in schema, "unknown" should be skipped + StructType dataSchema = new StructType().add("id", DataTypes.IntegerType); + StructType partitionSchema = new StructType(); + + CatalogColumnStat colStat = + new CatalogColumnStat( + Option.apply(BigInt(5L)), + Option.empty(), + Option.empty(), + Option.apply(BigInt(1L)), + Option.empty(), + Option.empty(), + Option.empty(), + CatalogColumnStat.VERSION()); + + scala.collection.immutable.Map colStatsMap = + buildScalaMap(new String[] {"id", "unknown"}, new CatalogColumnStat[] {colStat, colStat}); + + CatalogStatistics catalogStats = + new CatalogStatistics(BigInt(100L), Option.empty(), colStatsMap); + + Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); + + Map result = v2Stats.columnStats(); + assertEquals(1, result.size(), "Should only have 1 column stat (unknown skipped)"); + assertNotNull(result.get(FieldReference.apply("id")), "id stats should be present"); + } + + private static scala.math.BigInt BigInt(long value) { + return scala.math.BigInt.apply(value); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static scala.collection.immutable.Map buildScalaMap( + String[] keys, CatalogColumnStat[] values) { + scala.collection.mutable.Builder< + scala.Tuple2, + scala.collection.immutable.Map> + b = (scala.collection.mutable.Builder) scala.collection.immutable.Map$.MODULE$.newBuilder(); + for (int i = 0; i < keys.length; i++) { + b.$plus$eq(new scala.Tuple2<>(keys[i], values[i])); + } + return b.result(); + } +} From f1f8e5f3140742f87f956c7a7939f2338f39cd62 Mon Sep 17 00:00:00 2001 From: Min Yang Date: Thu, 12 Mar 2026 11:56:39 -0700 Subject: [PATCH 18/80] Add a flag to force stats collection during query optimizations (#5888) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description This PR adds a new Spark flag that will make all code paths in DataSkippingReader to collect stats. ## How was this patch tested? New tests. ## Does this PR introduce _any_ user-facing changes? No. --- .../sql/delta/sources/DeltaSQLConf.scala | 10 ++ .../sql/delta/stats/DataSkippingReader.scala | 53 ++++++- .../PreparedDeltaFileIndexRowCountSuite.scala | 150 ++++++++++++++++++ 3 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 spark/src/test/scala/org/apache/spark/sql/delta/stats/PreparedDeltaFileIndexRowCountSuite.scala diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index db5baf09064..e95f0f1007b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -470,6 +470,16 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELTA_ALWAYS_COLLECT_STATS = + buildConf("alwaysCollectStats.enabled") + .internal() + .doc("When true, row counts are collected from file statistics even when there are no " + + "data filters. This is useful for ensuring PreparedDeltaFileIndex always has row count " + + "information available. Note: this may have a small performance overhead as it requires " + + "summing numRecords from all files.") + .booleanConf + .createWithDefault(false) + val DELTA_LIMIT_PUSHDOWN_ENABLED = buildConf("stats.limitPushdown.enabled") .internal() diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala index a38739995bb..7ed7a76629a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala @@ -1263,7 +1263,10 @@ trait DataSkippingReaderBase partitionFilters: Seq[Expression], keepNumRecords: Boolean): (Seq[AddFile], DataSize) = recordFrameProfile( "Delta", "DataSkippingReader.filterOnPartitions") { - val df = if (keepNumRecords) { + val forceCollectRowCount = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS) + val shouldCollectStats = keepNumRecords || forceCollectRowCount + val df = if (shouldCollectStats) { // use withStats instead of allFiles so the `stats` column is already parsed val filteredFiles = DeltaLog.filterFileList(metadata.partitionSchema, withStats, partitionFilters) @@ -1278,7 +1281,36 @@ trait DataSkippingReaderBase } val files = convertDataFrameToAddFiles(df) val sizeInBytesByPartitionFilters = files.map(_.size).sum - files.toSeq -> DataSize(Some(sizeInBytesByPartitionFilters), None, Some(files.size)) + // Compute row count if we have stats available and forceCollectRowCount is enabled + val (rowCount, logicalRowCount) = if (forceCollectRowCount) { + sumRowCounts(files) + } else { + (None, None) + } + files.toSeq -> DataSize(Some(sizeInBytesByPartitionFilters), rowCount, Some(files.size), + logicalRowCount) + } + + /** + * Sums up the numPhysicalRecords and numLogicalRecords from the given AddFile objects. + * Returns (None, None) if any file is missing physical record stats. + * Returns (Some(physical), None) if any file is missing logical record stats. + */ + private def sumRowCounts(files: Seq[AddFile]): (Option[Long], Option[Long]) = { + var physicalRows = 0L + var logicalRows = 0L + var physicalMissing = false + var logicalMissing = false + files.foreach { file => + physicalMissing = physicalMissing || file.numPhysicalRecords.isEmpty + logicalMissing = logicalMissing || file.numLogicalRecords.isEmpty + physicalRows += file.numPhysicalRecords.getOrElse(0L) + logicalRows += file.numLogicalRecords.getOrElse(0L) + } + ( + if (physicalMissing) None else Some(physicalRows), + if (logicalMissing) None else Some(logicalRows) + ) } /** @@ -1335,13 +1367,24 @@ trait DataSkippingReaderBase if (filters == Seq(TrueLiteral) || filters.isEmpty || schema.isEmpty) { recordDeltaOperation(deltaLog, "delta.skipping.none") { // When there are no filters we can just return allFiles with no extra processing + val forceCollectRowCount = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS) + val shouldCollectStats = keepNumRecords || forceCollectRowCount + lazy val files = getAllFiles(shouldCollectStats) + // Compute row count if forceCollectRowCount is enabled + val (rowCount, logicalRowCount) = if (forceCollectRowCount) { + sumRowCounts(files) + } else { + (None, None) + } val dataSize = DataSize( bytesCompressed = sizeInBytesIfKnown, - rows = None, - files = numOfFilesIfKnown) + rows = rowCount, + files = numOfFilesIfKnown, + logicalRows = logicalRowCount) return DeltaScan( version = version, - files = getAllFiles(keepNumRecords), + files = files, total = dataSize, partition = dataSize, scanned = dataSize)( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/stats/PreparedDeltaFileIndexRowCountSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/stats/PreparedDeltaFileIndexRowCountSuite.scala new file mode 100644 index 00000000000..09cf3dff5f8 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/stats/PreparedDeltaFileIndexRowCountSuite.scala @@ -0,0 +1,150 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.stats + +import org.apache.spark.sql.delta.{DeltaLog, DeltaTable} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest + +import org.apache.spark.sql.{DataFrame, QueryTest} +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.test.SharedSparkSession + +/** + * Test suite to verify when preparedScan.scanned.rows is populated in PreparedDeltaFileIndex, + * and the behavior of the DELTA_ALWAYS_COLLECT_STATS flag. + */ +class PreparedDeltaFileIndexRowCountSuite + extends QueryTest + with DeltaSQLCommandTest { + + import testImplicits._ + + private def getDeltaScan(df: DataFrame): DeltaScan = { + val scans = df.queryExecution.optimizedPlan.collect { + case DeltaTable(prepared: PreparedDeltaFileIndex) => prepared.preparedScan + } + assert(scans.size == 1, s"Expected 1 DeltaScan, found ${scans.size}") + scans.head + } + + /** + * Test utility that creates a partitioned Delta table and verifies scanned.rows and + * scanned.logicalRows behavior. + * + * @param alwaysCollectStats value of the DELTA_ALWAYS_COLLECT_STATS flag + * @param queryTransform function to transform the base DataFrame (apply filters) + * @param expectedRowsDefined whether scanned.rows should be defined + * @param expectedRowCount expected row count if defined (None to skip validation) + * @param expectedLogicalRowsDefined whether scanned.logicalRows should be defined + * (defaults to same as expectedRowsDefined) + * @param expectedLogicalRowCount expected logical row count if defined (None to skip validation) + */ + private def testRowCountBehavior( + alwaysCollectStats: Boolean, + queryTransform: DataFrame => DataFrame, + expectedRowsDefined: Boolean, + expectedRowCount: Option[Long] = None, + expectedLogicalRowsDefined: Option[Boolean] = None, + expectedLogicalRowCount: Option[Long] = None): Unit = { + withTempDir { dir => + withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { + spark.range(100).toDF("id") + .withColumn("part", $"id" % 4) + .repartition(4) + .write.format("delta").partitionBy("part").save(dir.getAbsolutePath) + } + + DeltaLog.clearCache() + + withSQLConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS.key -> alwaysCollectStats.toString) { + val df = spark.read.format("delta").load(dir.getAbsolutePath) + val scan = getDeltaScan(queryTransform(df)) + + if (expectedRowsDefined) { + assert(scan.scanned.rows.isDefined, "scanned.rows should be defined") + expectedRowCount.foreach { expected => + assert(scan.scanned.rows.get == expected, + s"Expected $expected rows, got ${scan.scanned.rows.get}") + } + } else { + assert(scan.scanned.rows.isEmpty, "scanned.rows should be None") + } + + // logicalRows should follow the same defined/undefined pattern as rows by default + val logicalDefined = expectedLogicalRowsDefined.getOrElse(expectedRowsDefined) + if (logicalDefined) { + assert(scan.scanned.logicalRows.isDefined, "scanned.logicalRows should be defined") + expectedLogicalRowCount.foreach { expected => + assert(scan.scanned.logicalRows.get == expected, + s"Expected $expected logical rows, got ${scan.scanned.logicalRows.get}") + } + } else { + assert(scan.scanned.logicalRows.isEmpty, "scanned.logicalRows should be None") + } + } + } + } + + // Define query cases: (name, transform function, always collects rows) + private val queryCases: Seq[(String, DataFrame => DataFrame, Boolean)] = Seq( + ("no filter", identity[DataFrame], false), + ("TrueLiteral filter", _.where(lit(true)), false), + ("partition filter only", _.where($"part" === 1), false), + ("data filter", _.where($"id" === 50), true), + ("partition + data filter", _.where($"part" === 1).where($"id" === 49), true) + ) + + // Grid test: all query cases x flag values + for { + (caseName, queryTransform, alwaysCollectsRows) <- queryCases + alwaysCollectStats <- Seq(false, true) + } { + val flagDesc = s"alwaysCollectStats=$alwaysCollectStats" + // If the query type always collects rows, rows is always defined; otherwise depends on flag + val expectedRowsDefined = alwaysCollectsRows || alwaysCollectStats + + test(s"$caseName - $flagDesc") { + testRowCountBehavior( + alwaysCollectStats = alwaysCollectStats, + queryTransform = queryTransform, + expectedRowsDefined = expectedRowsDefined + ) + } + } + + test("alwaysCollectStats with missing stats returns None") { + withTempDir { dir => + // Create table without stats + withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { + spark.range(100).toDF("id") + .write.format("delta").save(dir.getAbsolutePath) + } + + DeltaLog.clearCache() + + withSQLConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS.key -> "true") { + val df = spark.read.format("delta").load(dir.getAbsolutePath) + val scan = getDeltaScan(df) + assert(scan.scanned.rows.isEmpty, "scanned.rows should be None when stats are missing") + assert(scan.scanned.logicalRows.isEmpty, + "scanned.logicalRows should be None when stats are missing") + } + } + } + +} From 2be5a1ffdc51913f46a367503e986596a4110f9b Mon Sep 17 00:00:00 2001 From: ChengJi Date: Thu, 12 Mar 2026 14:18:19 -0700 Subject: [PATCH 19/80] [UniForm] Use UnityCatalog as Iceberg committing catalog instead of HiveCatalog for Delta UniForm (#6189) ## Description Use UnityCatalog as Iceberg committing catalog instead of HiveCatalog for Delta UniForm ## How was this patch tested? Add UTs ``` build/sbt -DsparkVersion=4.0 "iceberg/testOnly org.apache.spark.sql.delta.uniform.UniFormConverterSuite" ``` --- .../IcebergConversionTransaction.scala | 70 +++++- .../icebergShaded/IcebergConverter.scala | 3 +- .../IcebergTransactionUtils.scala | 18 ++ .../delta/uniform/UniFormConverterSuite.scala | 117 ++++++++++ .../iceberg/unityCatalog/UnityCatalog.java | 140 ++++++++++++ .../UnityCatalogTableOperations.java | 203 ++++++++++++++++++ 6 files changed, 540 insertions(+), 11 deletions(-) create mode 100644 iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormConverterSuite.scala create mode 100644 icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalog.java create mode 100644 icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala index 71e69281d69..e18dd49faab 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala @@ -22,6 +22,7 @@ import java.util.function.Consumer import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer +import scala.jdk.OptionConverters._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeltaFileProviderUtils, DummySnapshot, IcebergConstants, NoMapping, Snapshot} @@ -33,11 +34,12 @@ import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.conf.Configuration -import shadedForDelta.org.apache.iceberg.{AppendFiles, DataFile, DeleteFiles, ExpireSnapshots, OverwriteFiles, PartitionSpec, PendingUpdate, RewriteFiles, Schema => IcebergSchema, Transaction => IcebergTransaction} +import shadedForDelta.org.apache.iceberg.{AppendFiles, BaseTransaction, DataFile, DeleteFiles, ExpireSnapshots, OverwriteFiles, PartitionSpec, PendingUpdate, RewriteFiles, Schema => IcebergSchema, TableMetadata, Transaction => IcebergTransaction} import shadedForDelta.org.apache.iceberg.MetadataUpdate import shadedForDelta.org.apache.iceberg.MetadataUpdate.{AddPartitionSpec, AddSchema} import shadedForDelta.org.apache.iceberg.mapping.MappingUtil import shadedForDelta.org.apache.iceberg.mapping.NameMappingParser +import shadedForDelta.org.apache.iceberg.unityCatalog.{UnityCatalog, UnityCatalogTableOperations} import shadedForDelta.org.apache.iceberg.util.LocationUtil import org.apache.spark.internal.MDC @@ -54,6 +56,9 @@ sealed trait IcebergConversionMode { // Used by Post-commit Delta UniForm (Iceberg conversion in Delta post commit hook) case object UNIFORM_POST_COMMIT_MODE extends IcebergConversionMode { } +// Used by atomic Delta UniForm +case object UNIFORM_CC_MODE extends IcebergConversionMode { +} /** * Used to prepare (convert) and then commit a set of Delta actions into the Iceberg table located * at the same path as [[postCommitSnapshot]] @@ -73,6 +78,7 @@ class IcebergConversionTransaction( protected val tableOp: IcebergTableOp = WRITE_TABLE, protected val lastConvertedIcebergSnapshotId: Option[Long] = None, protected val lastConvertedDeltaVersion: Option[Long] = None, + protected val lastConvertedIcebergMetadataPath: Option[String] = None, protected val metadataUpdates: java.util.ArrayList[MetadataUpdate] = new java.util.ArrayList[MetadataUpdate]() ) extends DeltaLogging { @@ -456,30 +462,75 @@ class IcebergConversionTransaction( committed = true } + /** + * Retrieves the converted Iceberg metadata location and its current snapshot. + * This method should only be called after a successful table conversion operation + * + * @return A tuple containing: + * - String: The path where the Iceberg metadata file was written + * - IcebergMetadata: The converted Iceberg metadata + * @throws IllegalStateException if the Iceberg metadata has not been converted + * @throws UnsupportedOperationException if called on non-UnityCatalogTableOperations + */ + def getConvertedIcebergMetadata: (String, TableMetadata) = + txn.asInstanceOf[BaseTransaction].underlyingOps() match { + case ops: UnityCatalogTableOperations => + ops.getLastWrittenTableMetadataWithLocation.toScala match { + case Some((metadataPath, tableMetadata)) => + (metadataPath, tableMetadata) + case _ => throw new IllegalStateException( + "Could not get converted Iceberg metadata: new written metadata not found") + } + case _ => + throw new IllegalStateException( + "Could not get converted Iceberg metadata:" + + " underlying UnityCatalogTableOperations not found" + ) + } + /////////////////////// // Protected Methods // /////////////////////// protected def createIcebergTxn(tableOpOpt: Option[IcebergTableOp] = None): IcebergTransaction = { - val hiveCatalog = IcebergTransactionUtils.createHiveCatalog(conf, metadataUpdates) - val icebergTableId = IcebergTransactionUtils - .convertSparkTableIdentifierToIcebergHive(catalogTable.identifier) + val baseMetadataPath = + (tableOpOpt.getOrElse(tableOp), lastConvertedIcebergMetadataPath) match { + case (CREATE_TABLE, None) => None + case (CREATE_TABLE, Some(_)) => + throw new IllegalStateException( + "Unexpected base metadata path for CREATE_TABLE operation") + case (op, None) => + throw new IllegalStateException(s"Missing base metadata path for $op operation") + case (_, Some(path)) => Some(path) + } + + val ucTable = new UnityCatalog( + metadataUpdates, + baseMetadataPath.toJava + ) + ucTable.initialize(null, new java.util.HashMap[String, String]()) + ucTable.setConf(conf) + + val icebergIdentifier = + IcebergTransactionUtils.convertSparkTableIdentifierToIceberg(catalogTable.identifier) - val tableExists = hiveCatalog.tableExists(icebergTableId) + val tableExists = ucTable.tableExists(icebergIdentifier) def tableBuilder = { - hiveCatalog - .buildTable(icebergTableId, icebergSchema) + val tableLocation = postCommitSnapshot.deltaLog.dataPath.toString + ucTable + .buildTable(icebergIdentifier, icebergSchema) .withPartitionSpec(partitionSpec) .withProperties(convert.properties.asJava) + .withLocation(tableLocation) } - tableOpOpt.getOrElse(tableOp) match { + val txn = tableOpOpt.getOrElse(tableOp) match { case WRITE_TABLE => if (tableExists) { recordFrameProfile("IcebergConversionTransaction", "loadTable") { - hiveCatalog.loadTable(icebergTableId).newTransaction() + ucTable.loadTable(icebergIdentifier).newTransaction() } } else { throw new IllegalStateException(s"Cannot write to table $tablePath. Table doesn't exist.") @@ -501,6 +552,7 @@ class IcebergConversionTransaction( throw new IllegalStateException(s"Cannot replace table $tablePath. Table doesn't exist.") } } + txn } //////////////////// diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala index bb76cf81c80..e8263ed8cf3 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala @@ -180,7 +180,7 @@ class IcebergConverter * Convert Iceberg Metadata for a complete snapshot. Used for conversion * after delta commits and in create table */ - private def convertSnapshotInternal( + protected def convertSnapshotInternal( snapshotToConvert: Snapshot, readSnapshotOpt: Option[Snapshot], lastConvertedInfo: LastConvertedIcebergInfo, @@ -321,7 +321,6 @@ class IcebergConverter icebergTxn.commit() logInfo(s"icebergTxn committed for table ${Option(catalogTable).map(_.identifier)} " + s"with converted delta version ${snapshotToConvert.version}") - validateIcebergCommit(snapshotToConvert, catalogTable) recordDeltaEvent( snapshotToConvert.deltaLog, diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala index b88c14aa3b0..563ef590753 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala @@ -240,6 +240,24 @@ object IcebergTransactionUtils catalog } + /** + * Encode Spark table identifier to Iceberg table identifier by putting "database" and "catalog" + * to the "namespace" in Iceberg table identifier + */ + def convertSparkTableIdentifierToIceberg( + identifier: SparkTableIdentifier): IcebergTableIdentifier = { + val namespace = (identifier.database, identifier.catalog) match { + case (Some(database), Some(catalog)) => Namespace.of(database, catalog) + case (Some(database), None) => Namespace.of(database) + case (None, Some(catalog)) => + throw new IllegalArgumentException( + "Spark does not allow the constructors to skip the `database` when `catalog` is used" + ) + case (None, None) => Namespace.empty() + } + IcebergTableIdentifier.of(namespace, identifier.table) + } + /** * Encode Spark table identifier to Iceberg table identifier by putting * only "database" to the "namespace" in Iceberg table identifier. diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormConverterSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormConverterSuite.scala new file mode 100644 index 00000000000..707be81a488 --- /dev/null +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormConverterSuite.scala @@ -0,0 +1,117 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.uniform + +import shadedForDelta.org.apache.iceberg.hadoop.HadoopTables + +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.delta.{DeltaLog, Snapshot} +import org.apache.spark.sql.delta.icebergShaded.{IcebergConverter, UNIFORM_CC_MODE} +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.test.SharedSparkSession + +class IcebergConverterForTest extends IcebergConverter { + def convertSnapshotAndReturnMetadataPath( + snapshotToConvert: Snapshot, + catalogTable: CatalogTable): String = { + val icebergTxn = convertSnapshotInternal( + snapshotToConvert, + readSnapshotOpt = None, + lastConvertedInfo = LastConvertedIcebergInfo( + icebergTable = None, + icebergSnapshotId = None, + deltaVersionConverted = None, + baseMetadataLocationOpt = None + ), + conversionContext = new ConversionContext( + conversionMode = UNIFORM_CC_MODE, + additionalDeltaActionsToCommit = None, + opType = "delta.iceberg.conversion.convertSnapshot" + ), + catalogTable + ) + icebergTxn.getConvertedIcebergMetadata._1 + } +} + +class UniFormConverterSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { + test("convertSnapshot writes Iceberg metadata and file count matches Delta snapshot") { + val tableName = "test_iceberg_converter" + withTable(tableName) { + spark.sql( + s"""CREATE TABLE $tableName (id INT, name STRING) USING DELTA + |TBLPROPERTIES ( + | 'delta.columnMapping.mode' = 'name', + | 'delta.enableIcebergCompatV2' = 'true', + | 'delta.universalFormat.enabledFormats' = 'iceberg' + |)""".stripMargin) + // Do write + spark.sql(s"INSERT INTO $tableName VALUES (1, 'alice'), (2, 'bob'), (3, 'carol')") + val tableId = TableIdentifier(tableName) + val deltaLog = DeltaLog.forTable(spark, tableId) + val snapshot = deltaLog.update() + val catalogTable = spark.sessionState.catalog.getTableMetadata(tableId) + // Trigger conversion + val converter = new IcebergConverterForTest() + val metadataPath = converter.convertSnapshotAndReturnMetadataPath(snapshot, catalogTable) + // Check match + val icebergTable = + new HadoopTables(deltaLog.newDeltaHadoopConf()).load(metadataPath) + val numFilesInIceberg = + icebergTable.currentSnapshot().summary().get("total-data-files").toInt + assert( + numFilesInIceberg == snapshot.numOfFiles, + s"Iceberg total-data-files ($numFilesInIceberg) must equal " + + s"Delta numOfFiles (${snapshot.numOfFiles})") + } + } + + test("convertSnapshot file count matches Delta snapshot after multiple inserts") { + val tableName = "test_iceberg_converter_multi" + withTable(tableName) { + spark.sql( + s"""CREATE TABLE $tableName (id INT) USING DELTA + |TBLPROPERTIES ( + | 'delta.columnMapping.mode' = 'name', + | 'delta.enableIcebergCompatV2' = 'true', + | 'delta.universalFormat.enabledFormats' = 'iceberg' + |)""".stripMargin) + // Do some writes + spark.sql(s"INSERT INTO $tableName VALUES (1)") + spark.sql(s"INSERT INTO $tableName VALUES (2)") + spark.sql(s"INSERT INTO $tableName VALUES (3)") + val tableId = TableIdentifier(tableName) + val deltaLog = DeltaLog.forTable(spark, tableId) + val snapshot = deltaLog.update() + val catalogTable = spark.sessionState.catalog.getTableMetadata(tableId) + assert(snapshot.numOfFiles == 3) + // Trigger conversion + val converter = new IcebergConverterForTest() + val metadataPath = converter.convertSnapshotAndReturnMetadataPath(snapshot, catalogTable) + // Check match + val icebergTable = new HadoopTables(deltaLog.newDeltaHadoopConf()).load(metadataPath) + val numFilesInIceberg = + icebergTable.currentSnapshot().summary().get("total-data-files").toInt + assert( + numFilesInIceberg == snapshot.numOfFiles, + s"Iceberg total-data-files ($numFilesInIceberg) must equal " + + s"Delta numOfFiles (${snapshot.numOfFiles})") + } + } +} diff --git a/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalog.java b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalog.java new file mode 100644 index 00000000000..39785230741 --- /dev/null +++ b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalog.java @@ -0,0 +1,140 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.unityCatalog; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.Configurable; +import org.apache.iceberg.io.CloseableGroup; +import org.apache.iceberg.io.FileIO; + +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotFoundException; + + +/** + * UnityCatalog manages delta table with iceberg metadata conversion in unity catalog + * Only newTableOps needs implementation as it is used to commit to Iceberg. All other methods are + * not required (eg, listTables, dropTable, renameTable) because the tables are managed by + * unity catalog outside of iceberg context. + */ +public class UnityCatalog extends BaseMetastoreCatalog implements Closeable, Configurable { + private static final String DEFAULT_FILE_IO_IMPL = "org.apache.iceberg.hadoop.HadoopFileIO"; + + + // Injectable factory for testing purposes. + static class FileIOFactory { + public FileIO newFileIO(String impl, Map properties, Object hadoopConf) { + return CatalogUtil.loadFileIO(impl, properties, hadoopConf); + } + } + + private Object conf; + + private Map catalogProperties; + + private FileIOFactory fileIOFactory; + + private CloseableGroup closeableGroup; + + private List metadataUpdates; + + // If set, all table option in this catalog will be built based on the snapshot baseMetadataLocation points to. + private final Optional baseMetadataLocation; + + public UnityCatalog( + List metadataUpdates + , Optional baseMetadataLocation + ) { + this.metadataUpdates = metadataUpdates; + this.baseMetadataLocation = baseMetadataLocation; + } + + @Override + protected TableOperations newTableOps(TableIdentifier tableIdentifier) { + FileIO fileIO = fileIOFactory.newFileIO(DEFAULT_FILE_IO_IMPL, catalogProperties, conf); + closeableGroup.addCloseable(fileIO); + return new UnityCatalogTableOperations( + fileIO + , tableIdentifier + , metadataUpdates + , baseMetadataLocation + ); + } + + @Override + public void initialize(String name, Map properties) { + this.catalogProperties = properties; + this.fileIOFactory = new FileIOFactory(); + this.closeableGroup = new CloseableGroup(); + } + + @Override + public void close() throws IOException { + if (closeableGroup != null) { + closeableGroup.close(); + } + } + + @Override + protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) { + throw new UnsupportedOperationException( + "UnityCatalog does not currently support defaultWarehouseLocation"); + } + + @Override + public void setConf(Object conf) { + this.conf = conf; + } + + @Override + public List listTables(Namespace namespace) { + throw new UnsupportedOperationException("UnityCatalog does not currently support listTables"); + } + + @Override + public boolean dropTable(TableIdentifier identifier, boolean purge) { + throw new UnsupportedOperationException("UnityCatalog does not currently support dropTable"); + } + + @Override + public void renameTable(TableIdentifier from, TableIdentifier to) { + throw new UnsupportedOperationException("UnityCatalog does not currently support renameTable"); + } + + // If the given metadataLocation is invalid, we also see it as the table doesn't exist + @Override + public boolean tableExists(TableIdentifier identifier) { + try { + loadTable(identifier); + return true; + } catch (NoSuchTableException | NotFoundException e) { + return false; + } + } +} diff --git a/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java new file mode 100644 index 00000000000..4bc3d534636 --- /dev/null +++ b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java @@ -0,0 +1,203 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.unityCatalog; + +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Instant; +import java.util.*; + +import org.apache.iceberg.BaseMetastoreTableOperations; +import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Option; +import scala.Tuple2; + +/** + * UnityCatalogTableOperations supports load and commit to Iceberg metadata through unity catalog + */ +public class UnityCatalogTableOperations extends BaseMetastoreTableOperations { + + private static final Logger LOG = LoggerFactory.getLogger(UnityCatalogTableOperations.class); + + /** + * Property to be set in translated Iceberg metadata files. + * Indicates the delta commit version # that it corresponds to. + */ + public static final String DELTA_VERSION_PROPERTY = "delta-version"; + public static final String DELTA_TIMESTAMP_PROPERTY = "delta-timestamp"; + public static final String BASE_DELTA_VERSION_PROPERTY = "base-delta-version"; + public static final String DELTA_HIGH_WATER_MARK_PROPERTY = "delta-high-water-mark"; + + public static final String DELTA_SQL_CONF_BYPASS_SEQUENCE_NUMBER_CHECK = + "spark.databricks.delta.uniform.bypassSnapshotSequenceNumberCheck"; + public static final String DELTA_SQL_CONF_BYPASS_FORMAT_VERSION_DOWNGRAGDE_CHECK = + "spark.databricks.delta.uniform.bypassFormatVersionDowngradeCheck"; + + private final FileIO fileIO; + private final TableIdentifier tableIdentifier; + private final Optional baseMetadataLocation; + private List metadataUpdates; + private Optional> lastWrittenTableMetadataWithLocation; + + public UnityCatalogTableOperations( + FileIO fileIO + , TableIdentifier tableIdentifier + , List metadataUpdates + , Optional baseMetadataLocation + ) { + this.fileIO = fileIO; + this.tableIdentifier = tableIdentifier; + this.metadataUpdates = metadataUpdates; + this.baseMetadataLocation = baseMetadataLocation; + this.lastWrittenTableMetadataWithLocation = Optional.empty(); + } + + @Override + public FileIO io() { + return fileIO; + } + + @Override + protected String tableName() { + return tableIdentifier.toString(); + } + + @Override + public void doCommit(TableMetadata base, TableMetadata metadata) { + TableMetadata.Builder builder = TableMetadata.buildFrom(metadata); + + // hasAddPartitionSpec indicates if the current commit attempt have added a new + // partition spec through metadata update. + boolean hasAddPartitionSpec = false; + boolean initialPartitionSpecExistsAndIsPartitioned = false; + for (PartitionSpec spec : metadata.specs()) { + if (spec.specId() == 0 && spec.isPartitioned()) { + initialPartitionSpecExistsAndIsPartitioned = true; + } + } + Long deltaVersion = Long.parseLong( + metadata.properties().get(DELTA_VERSION_PROPERTY)); + String baseDeltaVersionStr = + metadata.properties().get(BASE_DELTA_VERSION_PROPERTY); + + String deltaHighWaterMarkStr = + metadata.properties().get(DELTA_HIGH_WATER_MARK_PROPERTY); + + Schema lastAddedSchema = metadata.schema(); + for (MetadataUpdate update : metadataUpdates) { + // iceberg-core reassigns field id in its schema when firstly creates table metadata; + // we should always use the schema (with field ids assigned by delta) from MetadataUpdate + // because the parquet data files are already written with field ids assigned by Delta. + if (update instanceof MetadataUpdate.AddSchema) { + MetadataUpdate.AddSchema addSchema = (MetadataUpdate.AddSchema) update; + // lastColumnId must be monotonically increasing. + builder.setCurrentSchema( + addSchema.schema(), Math.max(metadata.lastColumnId(), addSchema.lastColumnId())); + lastAddedSchema = addSchema.schema(); + } else if (update instanceof MetadataUpdate.AddPartitionSpec) { + // Use the partition spec from MetadataUpdate because the partition spec contains + // the correct field ids assigned by Delta + PartitionSpec specToAdd = ((MetadataUpdate.AddPartitionSpec) update).spec().bind(lastAddedSchema); + if (!specToAdd.compatibleWith(metadata.spec())) { + builder.setDefaultPartitionSpec(specToAdd); + hasAddPartitionSpec = true; + } + } else { + update.applyTo(builder); + } + } + + // Remove the initial partitioned partition spec (id=0) if new partition spec is added + // because the initial partition spec has field ids assigned by iceberg-core + // which may mismatch with field id assigned by Delta + if (hasAddPartitionSpec && initialPartitionSpecExistsAndIsPartitioned) { + MetadataUpdate.RemovePartitionSpecs removeSpecs = new MetadataUpdate.RemovePartitionSpecs(Set.of(0)); + removeSpecs.applyTo(builder); + } + + metadata = builder.build(); + + if (base != current()) { + throw new CommitFailedException("Cannot commit changes based on stale table metadata"); + } + if (base == metadata) { + LOG.info("Nothing to commit."); + return; + } + + final int newVersion = currentVersion() + 1; + String newMetadataLocation = writeNewMetadata(metadata, newVersion); + lastWrittenTableMetadataWithLocation = Optional.of(Tuple2.apply(newMetadataLocation, metadata)); + } + + @Override + public TableMetadata refresh() { + // during Delta to iceberg conversion, the table should always exist in catalog and the only + // case with NoSuchTableException is the very first metadata conversion where iceberg + // metadata and metadata location table prop does not exist yet. + try { + return super.refresh(); + } catch (NoSuchTableException e) { + return null; + } + } + + @Override + public void doRefresh() { + LOG.debug("Getting metadata location for table {}", tableIdentifier); + String location = loadTableMetadataLocation(); + Preconditions.checkState( + location != null && !location.isEmpty(), + "Got null or empty location %s for table %s", + location, + tableIdentifier); + refreshFromMetadataLocation(location); + } + + /** + * Returns both the location of the Iceberg metadata file and its corresponding table metadata. + * + * @return An Optional containing a tuple of: + * - String: The path where the metadata file was written + * - TableMetadata: The corresponding Iceberg table metadata + */ + public Optional> getLastWrittenTableMetadataWithLocation() { + return lastWrittenTableMetadataWithLocation; + } + + private String loadTableMetadataLocation() { + if (baseMetadataLocation.isPresent()) { + return baseMetadataLocation.get(); + } + throw new NoSuchTableException("Cannot find iceberg table %s. Either the table does " + + "not exist or the corresponding Delta table has not been converted yet.", + tableIdentifier.toString()); + } + +} From f04a04df604df0b29b9b3eaab40357ce5e9bed06 Mon Sep 17 00:00:00 2001 From: Shlok Jhawar <34567765+shlokj@users.noreply.github.com> Date: Thu, 12 Mar 2026 14:59:36 -0700 Subject: [PATCH 20/80] Add overridable hook for winning transaction SQL in IdentityColumnConflictSuiteBase (#6277) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Add an overridable `sqlWithTotalRpcBound` method to `IdentityColumnConflictSuiteBase` that defaults to calling `sql()`. This is a no-op in OSS - it allows internal subclasses to override the method with custom assertions on the winning transaction's SQL execution path. ## How was this patch tested? No behavioral change - the new method defaults to `sql()`, identical to the existing behavior. All existing `IdentityColumnConflictSuite` tests pass unchanged. ## Does this PR introduce _any_ user-facing changes? No. Co-authored-by: Shlok Jhawar --- .../apache/spark/sql/delta/IdentityColumnConflictSuite.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala index 2bee10029b5..309689583fd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala @@ -134,6 +134,9 @@ trait IdentityColumnConflictSuiteBase None } + /** Executes the winning transaction SQL. Overridable for custom RPC assertions. */ + protected def sqlWithTotalRpcBound(sqlText: String): Unit = sql(sqlText) + /** * Helper function to test two concurrently running commands. Winning transaction commits before * current transaction commits. @@ -157,7 +160,7 @@ trait IdentityColumnConflictSuiteBase unblockUntilPreCommit(txnObserver) busyWaitFor(txnObserver.phases.preparePhase.hasEntered, timeout) - sql(winningTxn.sqlCommand.replace("{tblName}", tblName)) + sqlWithTotalRpcBound(winningTxn.sqlCommand.replace("{tblName}", tblName)) val expectedException = expectedExceptionClass(currentTxn, winningTxn) val events = Log4jUsageLogger.track { From 7b59b09f6c57de98a299e0b07eab8e0cc8622848 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Fri, 13 Mar 2026 05:21:21 +0530 Subject: [PATCH 21/80] Fix missing return in TransactionBuilderImpl early-exit path (#6271) ## Problem In `buildTransactionInternal()`, when `!needsMetadataOrProtocolUpdate`, a new `TransactionImpl` is constructed but never returned. Execution falls through to the full metadata/protocol update pipeline, doing unnecessary work and constructing a second `TransactionImpl`. Fixes #4713. ## Fix Add `return` before `new TransactionImpl(` so the early-exit path actually returns. The early return skips `TransactionMetadataFactory`, which also runs `validateRowTrackingConfigsNotMissing` on existing tables. To preserve this validation, it is moved into `validateWriteToExistingTable()` so it runs on every write regardless of whether metadata/protocol updates are needed. ## Validation - Confirmed by reading lines 230-247. - New unit test verifies the early-return path (uses a fake `delta.feature.*` property so the factory throws if the early return is bypassed). - Verified `LegacyRowTrackingSuite` passes (45/45 tests), including the row tracking config validation test. ## Regression Prevention - `TransactionBuilderImplSuite`: tests the no-update transaction path. - Existing `LegacyRowTrackingSuite`: covers the row tracking validation that was moved. --------- Co-authored-by: foo.bar --- .../internal/TransactionBuilderImpl.java | 13 ++- .../TransactionBuilderImplSuite.scala | 109 ++++++++++++++++++ 2 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionBuilderImplSuite.scala diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java index 26443fdfcdc..ceb72b90921 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java @@ -29,6 +29,7 @@ import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter; +import io.delta.kernel.internal.rowtracking.MaterializedRowTrackingColumn; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.types.StructType; import java.util.*; @@ -228,9 +229,8 @@ protected TransactionImpl buildTransactionInternal( || enablesDomainMetadataSupport; // domain metadata support added if (!needsMetadataOrProtocolUpdate) { - // TODO: fix this https://github.com/delta-io/delta/issues/4713 // Return early if there is no metadata or protocol updates and isCreateOrReplace=false - new TransactionImpl( + return new TransactionImpl( false, // isCreateOrReplace table.getDataPath(), latestSnapshot, @@ -314,7 +314,8 @@ protected TransactionImpl buildTransactionInternal( * Validates that Kernel can write to the existing table with the latest snapshot as provided. * This means (1) Kernel supports the reader and writer protocol of the table (2) if a transaction * identifier has been provided in this txn builder, a concurrent write has not already committed - * this transaction (3) Updating a partitioned table with clustering columns is not allowed. + * this transaction (3) Updating a partitioned table with clustering columns is not allowed (4) + * Row tracking configs are present when row tracking is enabled. */ protected void validateWriteToExistingTable( Engine engine, SnapshotImpl snapshot, boolean isCreateOrReplace) { @@ -338,6 +339,12 @@ protected void validateWriteToExistingTable( snapshot.getMetadata().getPartitionColNames(), inputLogicalClusteringColumns.get()); } + // Validate row tracking configs are present when row tracking is enabled. This must run + // on every write, including the early-return path that skips TransactionMetadataFactory. + if (!isCreateOrReplace) { + MaterializedRowTrackingColumn.validateRowTrackingConfigsNotMissing( + snapshot.getMetadata(), table.getPath(engine)); + } } /** diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionBuilderImplSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionBuilderImplSuite.scala new file mode 100644 index 00000000000..8533cceafb7 --- /dev/null +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionBuilderImplSuite.scala @@ -0,0 +1,109 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal + +import java.util.Optional + +import scala.collection.JavaConverters._ + +import io.delta.kernel.Operation +import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} +import io.delta.kernel.internal.checksum.CRCInfo +import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter +import io.delta.kernel.internal.fs.Path +import io.delta.kernel.internal.lang.Lazy +import io.delta.kernel.internal.metrics.SnapshotQueryContext +import io.delta.kernel.internal.snapshot.LogSegment +import io.delta.kernel.internal.util.FileNames +import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, stringStringMapValue} +import io.delta.kernel.test.MockEngineUtils +import io.delta.kernel.types.{IntegerType, StringType, StructType} +import io.delta.kernel.utils.FileStatus + +import org.scalatest.funsuite.AnyFunSuite + +class TransactionBuilderImplSuite extends AnyFunSuite with MockEngineUtils { + + /** + * Creates a mock snapshot whose metadata includes a `delta.feature.` property that + * TransactionMetadataFactory would attempt to process. This is used to verify the early-return + * path: if the code falls through to the factory, processing the unknown feature throws. + */ + private def createMockSnapshot( + dataPath: Path, + version: Long, + extraConfig: Map[String, String] = Map.empty): SnapshotImpl = { + val schema = new StructType().add("id", IntegerType.INTEGER) + val metadata = new Metadata( + "id", + Optional.empty(), + Optional.empty(), + new Format(), + schema.toJson, + schema, + buildArrayValue(java.util.Arrays.asList(), StringType.STRING), + Optional.of(123), + stringStringMapValue(extraConfig.asJava)) + val logPath = new Path(dataPath, "_delta_log") + val fs = FileStatus.of(FileNames.deltaFile(logPath, version), 1, 1) + val logSegment = new LogSegment( + logPath, + version, + Seq(fs).asJava, + Seq.empty.asJava, + Seq.empty.asJava, + fs, + Optional.empty(), + Optional.empty()) + val snapshotQueryContext = SnapshotQueryContext.forLatestSnapshot(dataPath.toString) + new SnapshotImpl( + dataPath, + logSegment.getVersion, + new Lazy(() => logSegment), + null, // logReplay - not needed; getCurrentCrcInfo is overridden below + new Protocol(1, 2), + metadata, + DefaultFileSystemManagedTableOnlyCommitter.INSTANCE, + snapshotQueryContext, + Optional.empty()) { + override def getCurrentCrcInfo: Optional[CRCInfo] = Optional.empty() + } + } + + test("early return when no metadata or protocol update is needed") { + // The snapshot metadata includes an unrecognized delta.feature.* property. If the code + // falls through to TransactionMetadataFactory (the bug), extractFeaturePropertyOverrides + // will try to resolve this feature and throw. The early-return path skips the factory + // entirely, so no exception is thrown. + val dataPath = new Path("/tmp/test-table") + val tableImpl = new TableImpl(dataPath.toString, () => System.currentTimeMillis()) + val snapshot = createMockSnapshot( + dataPath, + version = 0L, + extraConfig = Map("delta.feature.fakeFeatureForEarlyReturnTest" -> "supported")) + + val builder = new TransactionBuilderImpl(tableImpl, "test-engine", Operation.WRITE) + + // With the fix this returns immediately; without the fix this would throw + val txn = builder.buildTransactionInternal( + mockEngine(), + false, // isCreateOrReplace + Optional.of(snapshot)) + + // Verify the returned transaction does not mark protocol/metadata for update + assert(txn.getSchema(mockEngine()) === snapshot.getMetadata().getSchema()) + } +} From 66f39643b60696010e4406f7473465567980c84b Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Fri, 13 Mar 2026 05:21:51 +0530 Subject: [PATCH 22/80] Add previous-protocol DV check in Kernel IcebergCompat validation (#6278) ## Problem Spark's CheckDeletionVectorDisabled checks both newestProtocol and prevSnapshot to guard against concurrent writers enabling DVs. The Kernel's CHECK_HAS_NO_DELETION_VECTORS only checks newProtocol, missing the concurrent writer guard. ## Fix Add the previous protocol check to the Kernel's DV validation, matching the Spark implementation's defense-in-depth approach. ## Validation - Confirmed by comparing Spark IcebergCompat.scala with Kernel IcebergCompatMetadataValidatorAndUpdater.java. - New test verifies the check catches DV enablement from a previous protocol. ## Regression Prevention New test case explicitly validates the previous-protocol DV check. --------- Co-authored-by: foo.bar --- .../internal/TransactionMetadataFactory.java | 14 ++- ...bergCompatMetadataValidatorAndUpdater.java | 24 +++- ...rgCompatV2MetadataValidatorAndUpdater.java | 11 +- ...rgCompatV3MetadataValidatorAndUpdater.java | 11 +- ...erCompatV1MetadataValidatorAndUpdater.java | 14 ++- ...erCompatV3MetadataValidatorAndUpdater.java | 14 ++- ...atV2MetadataValidatorAndUpdaterSuite.scala | 39 ++++++- ...patV3MetadataValidatorAndUpdateSuite.scala | 8 +- ...atV1MetadataValidatorAndUpdaterSuite.scala | 107 ++++++++++++++---- ...atV3MetadataValidatorAndUpdaterSuite.scala | 78 ++++++++++--- 10 files changed, 261 insertions(+), 59 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java index 90ffb2a9d00..1f91cc18813 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java @@ -498,12 +498,18 @@ private void performIcebergCompatUpgradesAndValidation() { oldMetadata.getConfiguration(), metadata.getConfiguration())); }); + // Pass the previous protocol to IcebergCompat checks as defense-in-depth: if the read + // snapshot already had deletion vectors enabled, the check should reject it even when the + // new protocol in this transaction does not include DVs. The conflict checker may also + // catch this at commit time, but checking early gives a clearer error message. + Optional prevProtocol = latestSnapshotOpt.map(s -> s.getProtocol()); + // We must do our icebergWriterCompatV1 checks/updates FIRST since it has stricter column // mapping requirements (id mode) than icebergCompatV2. It also may enable icebergCompatV2. Optional icebergWriterCompatV1 = IcebergWriterCompatV1MetadataValidatorAndUpdater .validateAndUpdateIcebergWriterCompatV1Metadata( - isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol()); + isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergWriterCompatV1.isPresent()) { newMetadata = icebergWriterCompatV1; } @@ -511,7 +517,7 @@ private void performIcebergCompatUpgradesAndValidation() { Optional icebergWriterCompatV3 = IcebergWriterCompatV3MetadataValidatorAndUpdater .validateAndUpdateIcebergWriterCompatV3Metadata( - isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol()); + isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergWriterCompatV3.isPresent()) { newMetadata = icebergWriterCompatV3; } @@ -519,13 +525,13 @@ private void performIcebergCompatUpgradesAndValidation() { // TODO: refactor this method to use a single validator and updater. Optional icebergCompatV2Metadata = IcebergCompatV2MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV2Metadata( - isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol()); + isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergCompatV2Metadata.isPresent()) { newMetadata = icebergCompatV2Metadata; } Optional icebergCompatV3Metadata = IcebergCompatV3MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV3Metadata( - isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol()); + isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergCompatV3Metadata.isPresent()) { newMetadata = icebergCompatV3Metadata; } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java index 0125c48390e..cd64947fb51 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java @@ -78,21 +78,28 @@ public static class IcebergCompatInputContext { final boolean isCreatingNewTable; final Metadata newMetadata; final Protocol newProtocol; + /** + * The protocol from the previous snapshot, used to guard against concurrent writers that may + * have enabled incompatible features (e.g. deletion vectors). Empty for new tables. + */ + final Optional prevProtocol; public IcebergCompatInputContext( String compatFeatureName, boolean isCreatingNewTable, Metadata newMetadata, - Protocol newProtocol) { + Protocol newProtocol, + Optional prevProtocol) { this.compatFeatureName = compatFeatureName; this.isCreatingNewTable = isCreatingNewTable; this.newMetadata = newMetadata; this.newProtocol = newProtocol; + this.prevProtocol = prevProtocol; } public IcebergCompatInputContext withUpdatedMetadata(Metadata newMetadata) { return new IcebergCompatInputContext( - compatFeatureName, isCreatingNewTable, newMetadata, newProtocol); + compatFeatureName, isCreatingNewTable, newMetadata, newProtocol, prevProtocol); } } @@ -319,7 +326,18 @@ protected static IcebergCompatCheck hasOnlySupportedTypes( protected static final IcebergCompatCheck CHECK_HAS_NO_DELETION_VECTORS = (inputContext) -> { - if (inputContext.newProtocol.supportsFeature(DELETION_VECTORS_RW_FEATURE)) { + // Check both newProtocol and prevProtocol as defense-in-depth against cases where + // the previous snapshot already had deletion vectors enabled. This matches Spark's + // CheckDeletionVectorDisabled which checks both prevSnapshot and newestProtocol. + // Note: the conflict checker may also catch concurrent DV enablement at commit time. + boolean dvInNewProtocol = + inputContext.newProtocol.supportsFeature(DELETION_VECTORS_RW_FEATURE); + boolean dvInPrevProtocol = + inputContext + .prevProtocol + .map(prev -> prev.supportsFeature(DELETION_VECTORS_RW_FEATURE)) + .orElse(false); + if (dvInNewProtocol || dvInPrevProtocol) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(DELETION_VECTORS_RW_FEATURE)); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java index 7619792d1a0..12eda095b94 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java @@ -42,10 +42,17 @@ public class IcebergCompatV2MetadataValidatorAndUpdater * requirements */ public static Optional validateAndUpdateIcebergCompatV2Metadata( - boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol) { + boolean isCreatingNewTable, + Metadata newMetadata, + Protocol newProtocol, + Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( - INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol)); + INSTANCE.compatFeatureName(), + isCreatingNewTable, + newMetadata, + newProtocol, + prevProtocol)); } /** diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java index df7857a3672..9e5b3e32e94 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java @@ -64,10 +64,17 @@ public static void validateIcebergCompatV3Change( * requirements */ public static Optional validateAndUpdateIcebergCompatV3Metadata( - boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol) { + boolean isCreatingNewTable, + Metadata newMetadata, + Protocol newProtocol, + Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( - INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol)); + INSTANCE.compatFeatureName(), + isCreatingNewTable, + newMetadata, + newProtocol, + prevProtocol)); } /** diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java index 7f7b947bc58..408da7eec98 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java @@ -93,10 +93,17 @@ public static void validateIcebergWriterCompatV1Change( * requirements */ public static Optional validateAndUpdateIcebergWriterCompatV1Metadata( - boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol) { + boolean isCreatingNewTable, + Metadata newMetadata, + Protocol newProtocol, + Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( - INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol)); + INSTANCE.compatFeatureName(), + isCreatingNewTable, + newMetadata, + newProtocol, + prevProtocol)); } /// ////////////////////////////////////////////////////////////////////////////// @@ -119,7 +126,8 @@ public static Optional validateAndUpdateIcebergWriterCompatV1Metadata( .validateAndUpdateIcebergCompatV2Metadata( inputContext.isCreatingNewTable, inputContext.newMetadata, - inputContext.newProtocol)); + inputContext.newProtocol, + inputContext.prevProtocol)); /** * Current set of allowed table features for Iceberg writer compat V1. This combines the common diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java index aaef34cdb6a..4fb71571eff 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java @@ -64,10 +64,17 @@ public static void validateIcebergWriterCompatV3Change( * requirements */ public static Optional validateAndUpdateIcebergWriterCompatV3Metadata( - boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol) { + boolean isCreatingNewTable, + Metadata newMetadata, + Protocol newProtocol, + Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( - INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol)); + INSTANCE.compatFeatureName(), + isCreatingNewTable, + newMetadata, + newProtocol, + prevProtocol)); } /// ////////////////////////////////////////////////////////////////////////////// @@ -90,7 +97,8 @@ public static Optional validateAndUpdateIcebergWriterCompatV3Metadata( .validateAndUpdateIcebergCompatV3Metadata( inputContext.isCreatingNewTable, inputContext.newMetadata, - inputContext.newProtocol)); + inputContext.newProtocol, + inputContext.prevProtocol)); /** * Current set of allowed table features for Iceberg writer compat V3. This combines the common diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala index 482df6d5b77..8e2903fb95d 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala @@ -23,7 +23,7 @@ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.icebergcompat.IcebergCompatV2MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV2Metadata import io.delta.kernel.internal.tablefeatures.TableFeature -import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE} +import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, DELETION_VECTORS_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE} import io.delta.kernel.test.TestFixtures import io.delta.kernel.types._ @@ -70,7 +70,7 @@ class IcebergCompatV2MetadataValidatorAndUpdaterSuite isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { - validateAndUpdateIcebergCompatV2Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV2Metadata(isNewTable, metadata, protocol, Optional.empty()) } Seq(true, false).foreach { isNewTable => @@ -130,4 +130,39 @@ class IcebergCompatV2MetadataValidatorAndUpdaterSuite } } } + + /* --- prevProtocol DV check (concurrent writer protection) --- */ + + test("DV check catches deletion vectors enabled in previous protocol") { + val schema = new StructType().add("col", BooleanType.BOOLEAN) + val metadata = getCompatEnabledMetadata(schema) + // newProtocol does NOT support DVs + val newProtocol = getCompatEnabledProtocol() + // prevProtocol DOES support DVs (simulating a concurrent writer that enabled DVs) + val prevProtocol = Optional.of(getCompatEnabledProtocol(DELETION_VECTORS_RW_FEATURE)) + + val e = intercept[KernelException] { + validateAndUpdateIcebergCompatV2Metadata( + false, /* isCreatingNewTable */ + metadata, + newProtocol, + prevProtocol) + } + assert(e.getMessage.contains( + "Table features [deletionVectors] are incompatible with icebergCompatV2")) + } + + test("DV check passes when neither new nor previous protocol has DVs") { + val schema = new StructType().add("col", BooleanType.BOOLEAN) + val metadata = getCompatEnabledMetadata(schema) + val newProtocol = getCompatEnabledProtocol() + val prevProtocol = Optional.of(getCompatEnabledProtocol()) + + // Should not throw + validateAndUpdateIcebergCompatV2Metadata( + false, /* isCreatingNewTable */ + metadata, + newProtocol, + prevProtocol) + } } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala index 38677f50631..fd4a1c3c590 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala @@ -80,7 +80,7 @@ class IcebergCompatV3MetadataValidatorAndUpdaterSuite isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { - validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } Seq(true, false).foreach { isNewTable => @@ -90,7 +90,7 @@ class IcebergCompatV3MetadataValidatorAndUpdaterSuite val protocol = new Protocol(3, 7, Set.empty.asJava, Set("icebergCompatV3", "rowTracking").asJava) val e = intercept[KernelException] { - validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "icebergCompatV3: requires the feature 'columnMapping' to be enabled.")) @@ -126,13 +126,13 @@ class IcebergCompatV3MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "name") assert(TableConfig.ROW_TRACKING_ENABLED.fromMetadata(updatedMetadata.get())) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala index 490ac642fc4..75c52426e92 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala @@ -24,9 +24,9 @@ import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.icebergcompat.IcebergWriterCompatV1MetadataValidatorAndUpdater.validateAndUpdateIcebergWriterCompatV1Metadata import io.delta.kernel.internal.tablefeatures.TableFeature -import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_WRITER_COMPAT_V1, TYPE_WIDENING_RW_FEATURE} +import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, DELETION_VECTORS_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_WRITER_COMPAT_V1, TYPE_WIDENING_RW_FEATURE} import io.delta.kernel.internal.util.ColumnMapping -import io.delta.kernel.types.{ByteType, DataType, DecimalType, FieldMetadata, IntegerType, LongType, ShortType, StructField, StructType, TypeChange} +import io.delta.kernel.types.{BooleanType, ByteType, DataType, DecimalType, FieldMetadata, IntegerType, LongType, ShortType, StructField, StructType, TypeChange} class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite extends IcebergCompatV2MetadataValidatorAndUpdaterSuiteBase { @@ -35,7 +35,7 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol, Optional.empty()) } val icebergWriterCompatV1EnabledProps = Map( @@ -73,7 +73,8 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV1Metadata( true, /* isNewTable */ metadata, - getCompatEnabledProtocol()) + getCompatEnabledProtocol(), + Optional.empty()) } /* --- UNSUPPORTED_TYPES_CHECK tests --- */ @@ -85,7 +86,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( s"icebergWriterCompatV1 does not support the data types: ")) @@ -113,7 +118,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "id") // We correctly populate the column mapping metadata @@ -124,7 +133,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite enableIcebergWriterCompatV1 = true) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + @@ -145,7 +158,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( s"The value '$cmMode' for the property 'delta.columnMapping.mode' is" + @@ -169,7 +186,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "IcebergWriterCompatV1 requires column mapping field physical names be equal to " @@ -194,7 +215,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val updatedMetadata = - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) // No metadata update happens assert(!updatedMetadata.isPresent) } @@ -213,12 +238,20 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) assert(updatedMetadata.isPresent) assert(TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(updatedMetadata.get)) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV2' is" + @@ -237,7 +270,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV2' is" + @@ -275,8 +312,8 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite "catalogManaged") val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) - validateAndUpdateIcebergWriterCompatV1Metadata(true, metadata, protocol) - validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata(true, metadata, protocol, Optional.empty()) + validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } test("compatible type widening is allowed with icebergWriterCompatV1") { @@ -298,7 +335,7 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) // This should not throw an exception - validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } test("incompatible type widening throws exception with icebergWriterCompatV1") { @@ -319,7 +356,7 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains("icebergCompatV2 does not support type widening present in table")) @@ -340,7 +377,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val metadata = getCompatEnabledMetadata(cmTestSchema()) Seq(true, false).foreach { isNewTable => val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains(expectedErrorMessageContains)) } @@ -379,7 +420,8 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, activeFeatureMetadata, - getCompatEnabledProtocol()) + getCompatEnabledProtocol(), + Optional.empty()) } assert(e.getMessage.contains( s"Table features [$tableFeature] are incompatible with icebergWriterCompatV1")) @@ -457,7 +499,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val writerFeatures = Set("icebergWriterCompatV1", featureToIncludeStr) val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } // Since we run icebergCompatV2 validation as part of // ICEBERG_COMPAT_V2_ENABLED.postProcess we actually hit the missing feature error in the @@ -467,4 +513,25 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite } } } + + /* --- prevProtocol DV check flows through WriterCompatV1 -> CompatV2 delegation --- */ + + test("prevProtocol DV check flows through WriterCompatV1 delegation to CompatV2") { + val schema = new StructType().add("col", BooleanType.BOOLEAN) + val metadata = getCompatEnabledMetadata(schema) + val newProtocol = getCompatEnabledProtocol() + // prevProtocol has DVs enabled, simulating a table that had DVs before this transaction + val prevProtocol = Optional.of(getCompatEnabledProtocol(DELETION_VECTORS_RW_FEATURE)) + + val e = intercept[KernelException] { + validateAndUpdateIcebergWriterCompatV1Metadata( + false, + metadata, + newProtocol, + prevProtocol) + } + // The error comes from icebergCompatV2 since WriterCompatV1 delegates DV checking to V2 + assert(e.getMessage.contains( + "Table features [deletionVectors] are incompatible with icebergCompatV2")) + } } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala index 25965600827..330603f9509 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala @@ -34,7 +34,7 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } val icebergWriterCompatV3EnabledProps = Map( @@ -105,7 +105,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val metadata = getCompatEnabledMetadata(cmTestSchema()) Seq(true, false).foreach { isNewTable => val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains(expectedErrorMessageContains)) } @@ -120,7 +124,8 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, activeFeatureMetadata, - getCompatEnabledProtocol()) + getCompatEnabledProtocol(), + Optional.empty()) } assert(e.getMessage.contains( s"Table features [$tableFeature] are incompatible with icebergWriterCompatV3")) @@ -148,7 +153,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "id") // We correctly populate the column mapping metadata @@ -159,7 +168,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite true) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + @@ -177,7 +190,8 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV3Metadata( true, /* isNewTable */ metadata, - getCompatEnabledProtocol()) + getCompatEnabledProtocol(), + Optional.empty()) } Seq("name", "none").foreach { cmMode => @@ -191,7 +205,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( s"The value '$cmMode' for the property 'delta.columnMapping.mode' is" + @@ -215,7 +233,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "requires column mapping field physical names be equal to " @@ -240,7 +262,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val updatedMetadata = - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) // No metadata update happens assert(!updatedMetadata.isPresent) } @@ -255,7 +281,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( s"icebergWriterCompatV3 does not support the data types: ")) @@ -321,12 +351,20 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) assert(updatedMetadata.isPresent) assert(TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(updatedMetadata.get)) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV3' is" + @@ -346,7 +384,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV3' is" + @@ -389,8 +431,8 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite "catalogManaged") val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) - validateAndUpdateIcebergWriterCompatV3Metadata(true, metadata, protocol) - validateAndUpdateIcebergWriterCompatV3Metadata(false, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata(true, metadata, protocol, Optional.empty()) + validateAndUpdateIcebergWriterCompatV3Metadata(false, metadata, protocol, Optional.empty()) } Seq("collations", "defaultColumns").foreach { unsupportedIncompatibleFeature => @@ -475,7 +517,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val writerFeatures = Set("icebergWriterCompatV3", featureToIncludeStr) val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } // Since we run icebergCompatV3 validation as part of // ICEBERG_COMPAT_V3_ENABLED.postProcess we actually hit the missing feature error in the From 189ed1eae8ad76737eef0941ce7bd2cf375121bc Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Thu, 12 Mar 2026 17:21:33 -0700 Subject: [PATCH 23/80] [kernel-spark] Support skipChangeCommits and ignoreDeletes read option in dsv2 (#6246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🥞 Stacked PR Use this [link](https://github.com/delta-io/delta/pull/6246/files/6e9962d4a30a63ed14786830bae2b668004a76c9..2e111cf6ac9d1e5f84d83d94412b05486f543613) to review incremental changes. - [stack/ignoreDeletesV2](https://github.com/delta-io/delta/pull/6245) [[Files changed](https://github.com/delta-io/delta/pull/6245/files)] - [**stack/skipChangeCommitsV2**](https://github.com/delta-io/delta/pull/6246) [[Files changed](https://github.com/delta-io/delta/pull/6246/files/6e9962d4a30a63ed14786830bae2b668004a76c9..2e111cf6ac9d1e5f84d83d94412b05486f543613)] - [stack/ignoreChangesV2](https://github.com/delta-io/delta/pull/6249) [[Files changed](https://github.com/delta-io/delta/pull/6249/files/c1c942553a20800e318a5d2d384f869a3c0c5d3c..7209f24b489535e57a8396d712f0f1da577520d2)] - [stack/ignoreFileDeletionV2](https://github.com/delta-io/delta/pull/6250) [[Files changed](https://github.com/delta-io/delta/pull/6250/files/7209f24b489535e57a8396d712f0f1da577520d2..8690f78ca1bae137706f46ed1e05cdb05702ee6b)] --------- #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Support - skipChangeCommits read option is DSv2, which skip a commit that contains remove file actions regardless of if there is an add file action or not. - ignoreDeletes read option is DSv2, which skip a commit that contains only remove file actions but no add file action. ## How was this patch tested? Unit tests that test parity between v1 and v2 connector on both pure deletes commit (only remove) and change commit (add + remove). Integration tests: - streaming with skipChangeCommits = true skips both delete and change commits ## Does this PR introduce _any_ user-facing changes? No --- .../sql/delta/test/DeltaV2SourceSuite.scala | 10 +- .../spark/sql/delta/DeltaSourceSuite.scala | 168 +++++--- .../UCDeltaTableDataFrameStreamingTest.java | 5 - .../v2/read/SparkMicroBatchStream.java | 79 +++- .../spark/internal/v2/read/SparkScan.java | 9 +- .../spark/internal/v2/DeltaV2TestBase.java | 7 + .../v2/read/SparkMicroBatchStreamTest.java | 364 +++++++++++++++++- .../spark/internal/v2/read/SparkScanTest.java | 3 +- 8 files changed, 563 insertions(+), 82 deletions(-) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index d36873fe13e..986e4e951db 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -69,6 +69,9 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { // === Read options === "excludeRegex works and doesn't mess up offsets across restarts - parquet version", + "streaming with ignoreDeletes = true skips delete-only commits", + "streaming with ignoreDeletes = true still fails on change commits", + "streaming with skipChangeCommits = true skips both delete and change commits", // ========== startingVersion option tests ========== "startingVersion", @@ -103,7 +106,8 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { // ========== Error handling tests ========== "streaming query should fail when table is deleted and recreated with new id", - "SC-46515: deltaSourceIgnoreDeleteError contains removeFile, version, tablePath", + "deltaSourceIgnoreDeleteError contains removeFile, version, tablePath", + "deltaSourceIgnoreChangesError contains removeFile, version, tablePath", "excludeRegex throws good error on bad regex pattern", // ========== Misc tests ========== @@ -119,10 +123,6 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { "streaming delta source should not drop null columns", "streaming delta source should drop null columns without feature flag", - // === read options === - "skip change commits", - "SC-46515: deltaSourceIgnoreChangesError contains removeFile, version, tablePath", - // === Schema Evolution === // TODO(#6232): enable the two tests after spark streaming engine supports leaf node projection // for datasource v2 such that we can adopt the two schema changes without refreshing the diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala index 276d96aa095..24da9a2f6cc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala @@ -1820,7 +1820,7 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } - testQuietly("SC-46515: deltaSourceIgnoreChangesError contains removeFile, version, tablePath") { + testQuietly("deltaSourceIgnoreChangesError contains removeFile, version, tablePath") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) @@ -1862,7 +1862,7 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } - testQuietly("SC-46515: deltaSourceIgnoreDeleteError contains removeFile, version, tablePath") { + testQuietly("deltaSourceIgnoreDeleteError contains removeFile, version, tablePath") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) @@ -1899,6 +1899,126 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } + test("streaming with ignoreDeletes = true skips delete-only commits") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + // Write initial data + Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions( + inputDir.toString, Map(DeltaOptions.IGNORE_DELETES_OPTION -> "true", + "startingVersion" -> "0")) + + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3).map(Row(_))) + + // Delete all rows: produces only RemoveFile actions + io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() + + // Append new data after the delete + Seq(4, 5).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + + q.processAllAvailable() + + // The delete commit should be silently skipped; only inserts are processed + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3, 4, 5).map(Row(_))) + } finally { + q.stop() + } + } + } + + testQuietly("streaming with ignoreDeletes = true still fails on change commits") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions( + inputDir.toString, Map(DeltaOptions.IGNORE_DELETES_OPTION -> "true")) + df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + .processAllAvailable() + + // Overwrite produces both AddFile and RemoveFile actions (a change commit) + Seq(4, 5, 6).toDF("x") + .write + .mode("overwrite") + .format("delta") + .save(inputDir.toString) + + val e = intercept[StreamingQueryException] { + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + + try { + q.processAllAvailable() + } finally { + q.stop() + } + } + + assert(e.getCause.isInstanceOf[UnsupportedOperationException]) + assert(e.getCause.getMessage.contains( + "This is currently not supported. If this is going to happen regularly and you are okay" + + " to skip changes, set the option 'skipChangeCommits' to 'true'." + )) + } + } + + test("streaming with skipChangeCommits = true skips both delete and change commits") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions( + inputDir.toString, Map(DeltaOptions.SKIP_CHANGE_COMMITS_OPTION -> "true")) + + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3).map(Row(_))) + + // Delete all rows: produces only RemoveFile actions (delete-only commit) + io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() + + Seq(4, 5).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + + // Overwrite produces both AddFile and RemoveFile actions (change commit) + Seq(6, 7, 8).toDF("x") + .write + .mode("overwrite") + .format("delta") + .save(inputDir.toString) + + Seq(9, 10).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + + q.processAllAvailable() + + // Both the delete and overwrite commits are silently skipped; only inserts are processed + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3, 4, 5, 9, 10).map(Row(_))) + } finally { + q.stop() + } + } + } + test("fail on data loss - starting from missing files") { withTempDirs { (srcData, targetData, chkLocation) => def addData(): Unit = { @@ -2828,50 +2948,6 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } } - - test("skip change commits") { - withTempDir { inputDir => - val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) - withMetadata(deltaLog, StructType.fromDDL("value STRING")) - - val df = loadStreamWithOptions( - inputDir.getCanonicalPath, - Map(DeltaOptions.SKIP_CHANGE_COMMITS_OPTION -> "true")) - - testStream(df)( - // Add data to source table - AddToReservoir(inputDir, Seq("keep1", "update1", "drop1").toDF()), - AssertOnQuery { q => q.processAllAvailable(); true }, - CheckAnswer("keep1", "update1", "drop1"), - - // Update and delete rows - UpdateReservoir( - inputDir, - Map("value" -> when($"value" === "update1", "updated1").otherwise($"value")) - ), - DeleteFromReservoir(inputDir, $"value" === "drop1"), - CheckAnswer("keep1", "update1", "drop1"), - - // Merge data into source table - MergeIntoReservoir( - inputDir, - dfToMerge = Seq("keep1", "keep2", "keep3").toDF().as("merge1"), - mergeCondition = $"table.value" === $"merge1.value", - Map.empty - ), - MergeIntoReservoir( - inputDir, - dfToMerge = Seq("updated1", "keep4", "keep5").toDF().as("merge2"), - mergeCondition = $"table.value" === $"merge2.value", - Map("table.value" -> when($"table.value" === "updated1", "newlyUpdated1") - .otherwise($"table.value")) - ), - CheckAnswer( - "keep1", "update1", "drop1", "keep2", "keep3" - ) - ) - } - } } /** diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java index 920df9bfeb3..96078c94714 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java @@ -354,11 +354,6 @@ public void testStreamingInvalidOptions(TableType tableType) throws Exception { r -> r.option("ignoreChanges", "true"), "not supported", "ignoreChanges"); - assertInvalidStreamOption( - tableName, - r -> r.option("ignoreDeletes", "true"), - "not supported", - "ignoreDeletes"); } }); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java index 35d5b07d9bc..c37bd9f4c59 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java @@ -102,6 +102,7 @@ public class SparkMicroBatchStream private final Engine engine; private final DeltaSnapshotManager snapshotManager; private final DeltaOptions options; + private final boolean skipChangeCommits; private final SnapshotImpl snapshotAtSourceInit; private final String tableId; private final StructType readSchemaAtSourceInit; @@ -187,6 +188,7 @@ public SparkMicroBatchStream( this.spark = Objects.requireNonNull(spark, "spark is null"); this.engine = DefaultEngine.create(hadoopConf); this.options = Objects.requireNonNull(options, "options is null"); + this.skipChangeCommits = this.options.skipChangeCommits(); // Normalize tablePath to ensure it ends with "/" for consistent path construction String normalizedTablePath = Objects.requireNonNull(tablePath, "tablePath is null"); this.tablePath = @@ -809,7 +811,7 @@ private CloseableIterator processCommitToIndexedFiles( try { long version = commit.getVersion(); - // First pass: Validate the commit. + // First pass: Validate the commit and decide whether to skip it. // // We must validate the ENTIRE commit before emitting ANY files. This is a correctness // requirement: commits could contain both AddFiles and RemoveFiles. @@ -818,13 +820,15 @@ private CloseableIterator processCommitToIndexedFiles( // // TODO(#5318): consider caching the commit actions to avoid reading the same commit twice. // TODO(#5319): don't verify metadata action when schema tracking is enabled - validateCommit( - commit, - version, - startVersion, - snapshotAtSourceInit.getPath(), - endOffsetOpt, - /* verifyMetadataAction= */ true); + boolean shouldSkipCommit = + validateCommitAndDecideSkipping( + commit, + version, + startVersion, + snapshotAtSourceInit.getPath(), + endOffsetOpt, + /* verifyMetadataAction= */ true); + // Second pass: Build a lazy iterator of IndexedFiles. // // BEGIN (BASE_INDEX) + actual file actions + END (END_INDEX) @@ -837,9 +841,13 @@ private CloseableIterator processCommitToIndexedFiles( // buildOffsetFromIndexedFile to skip re-reading completed versions. // // See DeltaSource.addBeginAndEndIndexOffsetsForVersion for the Scala equivalent. + CloseableIterator fileActions = + shouldSkipCommit + ? Utils.toCloseableIterator(Collections.emptyIterator()) + : getFilesFromCommit(commit, version); return Utils.singletonCloseableIterator( new IndexedFile(version, DeltaSourceOffset.BASE_INDEX(), /* addFile= */ null)) - .combine(getFilesFromCommit(commit, version)) + .combine(fileActions) .combine( Utils.singletonCloseableIterator( new IndexedFile(version, DeltaSourceOffset.END_INDEX(), /* addFile= */ null))); @@ -867,7 +875,7 @@ private CloseableIterator getFilesFromCommit(CommitActions commit, } /** - * Validates a commit and fail the stream if it's invalid. Mimics + * Validates a commit, fail the stream if it's invalid and decides whether to skip it. Mimics * DeltaSource.validateCommitAndDecideSkipping in Scala. * * @param commit the CommitActions representing a single commit @@ -876,9 +884,10 @@ private CloseableIterator getFilesFromCommit(CommitActions commit, * @param tablePath the path to the Delta table * @param endOffsetOpt optional end offset for boundary checking * @param verifyMetadataAction Whether to verify metadata action compatibility + * @return true if the commit should be skipped (no AddFiles emitted), false otherwise * @throws RuntimeException if the commit is invalid. */ - private void validateCommit( + private boolean validateCommitAndDecideSkipping( CommitActions commit, long version, long batchStartVersion, @@ -890,26 +899,43 @@ private void validateCommit( DeltaSourceOffset endOffset = endOffsetOpt.get(); if (endOffset.reservoirVersion() == version && endOffset.index() == DeltaSourceOffset.BASE_INDEX()) { - return; + return false; } } - // TODO(#5319): Implement ignoreChanges & skipChangeCommits & ignoreDeletes (legacy) + + // TODO(#5319): Implement ignoreChanges & ignoreFileDeletion (deprecated) + // A check on the source table that disallows changes on the source data. + boolean shouldAllowChanges = skipChangeCommits; + // A check on the source table that disallows commits that only include deletes to the data. + boolean shouldAllowDeletes = shouldAllowChanges || options.ignoreDeletes(); + + boolean hasFileAdd = false; + boolean shouldSkipCommit = false; + Metadata metadataAction = null; + String removeFileActionPath = null; try (CloseableIterator actionsIter = commit.getActions()) { while (actionsIter.hasNext()) { ColumnarBatch batch = actionsIter.next(); int numRows = batch.getSize(); - Metadata metadataAction = null; for (int rowId = 0; rowId < numRows; rowId++) { - // RULE 1: If commit has RemoveFile(dataChange=true), fail this stream. + // Track AddFile(dataChange=true) + Optional addOpt = StreamingHelper.getAddFileWithDataChange(batch, rowId); + if (addOpt.isPresent()) { + hasFileAdd = true; + } + + // Track RemoveFile(dataChange=true) Optional removeOpt = StreamingHelper.getDataChangeRemove(batch, rowId); if (removeOpt.isPresent()) { - RemoveFile removeFile = removeOpt.get(); - throw (RuntimeException) - DeltaErrors.deltaSourceIgnoreDeleteError(version, removeFile.getPath(), tablePath); + // skip change commits include delete-only commits + shouldSkipCommit = skipChangeCommits; + if (removeFileActionPath == null) { + removeFileActionPath = removeOpt.get().getPath(); + } } - // RULE 2: If commit has Metadata, check read-incompatible schema changes. + // Track Metadata for read-incompatible schema changes. Optional metadataOpt = StreamingHelper.getMetadata(batch, rowId); if (metadataOpt.isPresent()) { Metadata metadata = metadataOpt.get(); @@ -934,6 +960,21 @@ private void validateCommit( } catch (IOException e) { throw new RuntimeException("Failed to process commit at version " + version, e); } + + if (removeFileActionPath != null) { + if (hasFileAdd && !shouldAllowChanges) { + // Commit contains data changes (adds + removes) and changes are disallowed. + // TODO(#5319): log CommitInfo action's operation instead of path + throw (RuntimeException) + DeltaErrors.deltaSourceIgnoreChangesError(version, removeFileActionPath, tablePath); + } else if (!hasFileAdd && !shouldAllowDeletes) { + // Commit contains only removes (deletes) and deletes are disallowed. + throw (RuntimeException) + DeltaErrors.deltaSourceIgnoreDeleteError(version, removeFileActionPath, tablePath); + } + } + + return shouldSkipCommit; } /** diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java index 1fbd2aec235..3d544b31420 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java @@ -63,12 +63,15 @@ public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntim DeltaOptions.STARTING_TIMESTAMP_OPTION(), DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION(), DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION(), + DeltaOptions.IGNORE_DELETES_OPTION(), + DeltaOptions.SKIP_CHANGE_COMMITS_OPTION(), DeltaOptions.EXCLUDE_REGEX_OPTION())); /** * Block list of DeltaOptions that are not supported for streaming in V2 connector. Only - * startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, and excludeRegex - * are supported. User-defined custom options (not in DeltaOptions) are allowed to pass through. + * startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, ignoreDeletes, + * skipChangeCommits, and excludeRegex are supported. User-defined custom options (not in + * DeltaOptions) are allowed to pass through. */ private static final Set UNSUPPORTED_STREAMING_OPTIONS = Collections.unmodifiableSet( @@ -76,8 +79,6 @@ public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntim Arrays.asList( DeltaOptions.IGNORE_FILE_DELETION_OPTION().toLowerCase(), DeltaOptions.IGNORE_CHANGES_OPTION().toLowerCase(), - DeltaOptions.IGNORE_DELETES_OPTION().toLowerCase(), - DeltaOptions.SKIP_CHANGE_COMMITS_OPTION().toLowerCase(), DeltaOptions.FAIL_ON_DATA_LOSS_OPTION().toLowerCase(), DeltaOptions.CDC_READ_OPTION().toLowerCase(), DeltaOptions.CDC_READ_OPTION_LEGACY().toLowerCase(), diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java index 6d21bb0abe0..7eccb585ee5 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java @@ -65,6 +65,13 @@ protected void createEmptyTestTable(String path, String tableName) { "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tableName, path)); } + protected void createEmptyPartitionedTestTable(String path, String tableName) { + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta PARTITIONED BY (name) LOCATION '%s'", + tableName, path)); + } + protected void createSchemaEvolutionTestTable(String path, String tableName) { spark.sql( String.format( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java index b8424d1c764..3584f9727c8 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java @@ -944,7 +944,7 @@ private static Stream removeFileScenarios() { String sourceTableName = "merge_source_" + System.nanoTime(); sql( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", - sourceTableName, tempDir.getAbsolutePath() + "_source"); + sourceTableName, new File(tempDir, "source").getAbsolutePath()); sql("INSERT INTO %s VALUES (2, 'UpdatedUser2'), (4, 'User4')", sourceTableName); // Perform MERGE operation @@ -956,7 +956,367 @@ private static Stream removeFileScenarios() { sql("DROP TABLE IF EXISTS %s", sourceTableName); }, - "MERGE: Matched (REMOVE+ADD) and not matched (ADD)")); + "MERGE: Matched (REMOVE+ADD) and not matched (ADD)"), + + // Full table delete (only RemoveFiles, no AddFiles) + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); + sql("DELETE FROM %s", tableName); + }, + "DELETE: Full table delete (removes only, no adds)")); + } + + // ================================================================================================ + // Tests for ignoreDeletes parity between DSv1 and DSv2 + // ================================================================================================ + + /** + * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 produce the same file changes for + * delete-only commits (commits with only RemoveFile actions, no AddFile actions). The delete-only + * commit should be silently skipped (only sentinels emitted, no data files). + */ + @ParameterizedTest + @MethodSource("deleteOnlyScenarios") + public void testGetFileChanges_withIgnoreDeletes_deleteOnlyParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = + "test_ignore_deletes_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); + createEmptyPartitionedTestTable(testTablePath, testTableName); + + scenarioSetup.setup(testTableName, tempDir); + + long fromVersion = 0L; + long fromIndex = DeltaSourceOffset.BASE_INDEX(); + DeltaOptions options = createDeltaOptions("ignoreDeletes", "true"); + + // DSv1 + DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); + DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); + Option endOffset = Option.empty(); + ClosableIterator deltaChanges = + deltaSource.getFileChanges( + fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); + List deltaFilesList = new ArrayList<>(); + while (deltaChanges.hasNext()) { + deltaFilesList.add(deltaChanges.next()); + } + deltaChanges.close(); + + // DSv2 + Configuration hadoopConf = new Configuration(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + SparkMicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, options); + try (CloseableIterator kernelChanges = + stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { + List kernelFilesList = new ArrayList<>(); + while (kernelChanges.hasNext()) { + kernelFilesList.add(kernelChanges.next()); + } + compareFileChanges(deltaFilesList, kernelFilesList); + } + } + + /** + * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 still throw on commits containing + * both adds and removes (e.g., UPDATE, MERGE), since ignoreDeletes only suppresses delete-only + * commits. + */ + @ParameterizedTest + @MethodSource("changeCommitScenarios") + public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = + "test_ignore_deletes_change_" + + Math.abs(testDescription.hashCode()) + + "_" + + System.nanoTime(); + createEmptyTestTable(testTablePath, testTableName); + + scenarioSetup.setup(testTableName, tempDir); + + long fromVersion = 0L; + long fromIndex = DeltaSourceOffset.BASE_INDEX(); + Option endOffset = Option.empty(); + DeltaOptions options = createDeltaOptions("ignoreDeletes", "true"); + + // DSv1 + DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); + DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); + + AtomicInteger dsv1SuccessfulCalls = new AtomicInteger(0); + assertThrows( + UnsupportedOperationException.class, + () -> { + ClosableIterator deltaChanges = + deltaSource.getFileChanges( + fromVersion, + fromIndex, + isInitialSnapshot, + endOffset, + /* verifyMetadataAction= */ true); + try { + while (deltaChanges.hasNext()) { + deltaChanges.next(); + dsv1SuccessfulCalls.incrementAndGet(); + } + } finally { + deltaChanges.close(); + } + }, + String.format( + "DSv1 should throw on change commit with ignoreDeletes for: %s", testDescription)); + + // DSv2 + Configuration hadoopConf = new Configuration(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + SparkMicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, options); + + AtomicInteger dsv2SuccessfulCalls = new AtomicInteger(0); + assertThrows( + UnsupportedOperationException.class, + () -> { + CloseableIterator kernelChanges = + stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty()); + try { + while (kernelChanges.hasNext()) { + kernelChanges.next(); + dsv2SuccessfulCalls.incrementAndGet(); + } + } finally { + kernelChanges.close(); + } + }, + String.format( + "DSv2 should throw on change commit with ignoreDeletes for: %s", testDescription)); + + assertEquals( + dsv1SuccessfulCalls.get(), + dsv2SuccessfulCalls.get(), + String.format( + "DSv1 and DSv2 should throw after the same number of next() calls for: %s. " + + "DSv1=%d, DSv2=%d", + testDescription, dsv1SuccessfulCalls.get(), dsv2SuccessfulCalls.get())); + } + + // ================================================================================================ + // Tests for skipChangeCommits parity between DSv1 and DSv2 + // ================================================================================================ + + /** + * Verifies that with skipChangeCommits=true, both DSv1 and DSv2 produce the same file changes for + * delete-only commits. Since skipChangeCommits suppresses all commits containing RemoveFile + * actions, these commits should be silently skipped (only sentinels emitted, no data files). + */ + @ParameterizedTest + @MethodSource("deleteOnlyScenarios") + public void testGetFileChanges_withSkipChangeCommits_deleteOnlyParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = + "test_skip_change_del_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); + createEmptyPartitionedTestTable(testTablePath, testTableName); + + scenarioSetup.setup(testTableName, tempDir); + + long fromVersion = 0L; + long fromIndex = DeltaSourceOffset.BASE_INDEX(); + DeltaOptions options = createDeltaOptions("skipChangeCommits", "true"); + + // DSv1 + DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); + DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); + Option endOffset = Option.empty(); + ClosableIterator deltaChanges = + deltaSource.getFileChanges( + fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); + List deltaFilesList = new ArrayList<>(); + while (deltaChanges.hasNext()) { + deltaFilesList.add(deltaChanges.next()); + } + deltaChanges.close(); + + // DSv2 + Configuration hadoopConf = new Configuration(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + SparkMicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, options); + try (CloseableIterator kernelChanges = + stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { + List kernelFilesList = new ArrayList<>(); + while (kernelChanges.hasNext()) { + kernelFilesList.add(kernelChanges.next()); + } + compareFileChanges(deltaFilesList, kernelFilesList); + } + } + + /** + * Verifies that with skipChangeCommits=true, both DSv1 and DSv2 silently skip commits containing + * both adds and removes (e.g., UPDATE, MERGE), instead of throwing. This is the key behavioral + * difference from ignoreDeletes, which throws on such commits. + */ + @ParameterizedTest + @MethodSource("changeCommitScenarios") + public void testGetFileChanges_withSkipChangeCommits_changeCommitParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = + "test_skip_change_chg_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); + createEmptyTestTable(testTablePath, testTableName); + + scenarioSetup.setup(testTableName, tempDir); + + long fromVersion = 0L; + long fromIndex = DeltaSourceOffset.BASE_INDEX(); + DeltaOptions options = createDeltaOptions("skipChangeCommits", "true"); + + // DSv1 + DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); + DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); + Option endOffset = Option.empty(); + ClosableIterator deltaChanges = + deltaSource.getFileChanges( + fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); + List deltaFilesList = new ArrayList<>(); + while (deltaChanges.hasNext()) { + deltaFilesList.add(deltaChanges.next()); + } + deltaChanges.close(); + + // DSv2 + Configuration hadoopConf = new Configuration(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + SparkMicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, options); + try (CloseableIterator kernelChanges = + stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { + List kernelFilesList = new ArrayList<>(); + while (kernelChanges.hasNext()) { + kernelFilesList.add(kernelChanges.next()); + } + compareFileChanges(deltaFilesList, kernelFilesList); + } + } + + // ================================================================================================ + // Shared scenario providers for ignoreDeletes and skipChangeCommits tests + // ================================================================================================ + + /** + * Provides delete-only scenarios: commits with only RemoveFile actions and no AddFile actions. + * Used by both ignoreDeletes and skipChangeCommits tests. + * + *

    Arguments: (ScenarioSetup, isInitialSnapshot, testDescription) + */ + private static Stream deleteOnlyScenarios() { + return Stream.of( + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); + sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); + sql("DELETE FROM %s", tableName); + }, + false, + "Full table delete"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); + sql("DELETE FROM %s", tableName); + sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); + }, + false, + "Insert-Delete-Insert: data resumes after delete"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'PartA'), (2, 'PartA'), (3, 'PartB')", tableName); + sql("DELETE FROM %s WHERE name = 'PartA'", tableName); + }, + false, + "Partitioned table: delete entire partition"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); + sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); + sql("DELETE FROM %s", tableName); + }, + true, + "Full table delete with initial snapshot")); + } + + /** + * Provides change-commit scenarios: commits containing both AddFile and RemoveFile actions (e.g., + * UPDATE, MERGE). Used by both ignoreDeletes and skipChangeCommits tests — ignoreDeletes expects + * these to throw, while skipChangeCommits expects them to be silently skipped. + * + *

    Arguments: (ScenarioSetup, isInitialSnapshot, testDescription) + */ + private static Stream changeCommitScenarios() { + return Stream.of( + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); + sql("UPDATE %s SET name = 'Updated' WHERE id = 1", tableName); + }, + false, + "UPDATE: AddFile + RemoveFile"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); + String sourceTableName = "merge_src_" + System.nanoTime(); + sql( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", + sourceTableName, new File(tempDir, "source").getAbsolutePath()); + sql("INSERT INTO %s VALUES (2, 'UpdatedUser2'), (4, 'User4')", sourceTableName); + sql( + "MERGE INTO %s AS target USING %s AS source ON target.id = source.id " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + + "WHEN NOT MATCHED THEN INSERT (id, name) " + + "VALUES (source.id, source.name)", + tableName, sourceTableName); + sql("DROP TABLE IF EXISTS %s", sourceTableName); + }, + false, + "MERGE: AddFile + RemoveFile"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); + sql("UPDATE %s SET name = 'Updated' WHERE id = 1", tableName); + }, + true, + "UPDATE with initial snapshot: AddFile + RemoveFile")); } @Test diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java index 39f3730ef20..fd61863bf44 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java @@ -453,7 +453,8 @@ public void testValidateStreamingOptions_UnsupportedOptions() { // Note: DeltaOptions uses CaseInsensitiveMap which lowercases keys during iteration assertEquals( "The following streaming options are not supported: [readchangefeed]. " - + "Supported options are: [startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, excludeRegex].", + + "Supported options are: [startingVersion, startingTimestamp, maxFilesPerTrigger, " + + "maxBytesPerTrigger, ignoreDeletes, skipChangeCommits, excludeRegex].", exception.getMessage()); } From 5ba861fa6d6999d7079706a307a7e293b59d4880 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Fri, 13 Mar 2026 08:59:39 +0530 Subject: [PATCH 24/80] Fix V3 remove action ignoring baseRowId and defaultRowCommitVersion (#6267) ## Problem `generateIcebergCompatWriterV3RemoveAction()` accepts `baseRowId` and `defaultRowCommitVersion` parameters but silently discards them, hardcoding `Optional.empty()` instead. This breaks row tracking for V3 writer compat remove actions. ## Fix Forward the caller-provided `baseRowId` and `defaultRowCommitVersion` to `convertRemoveDataFileStatus()` instead of passing `Optional.empty()`. ## Validation - Confirmed by reading the method: params accepted at lines 266-274, discarded at lines 308-310. - New test verifies non-empty values are preserved in the output Row. ## Regression Prevention New test case explicitly checks that baseRowId and defaultRowCommitVersion are forwarded through to the RemoveFile row. --------- Co-authored-by: foo.bar --- .../GenerateIcebergCompatActionUtils.java | 4 +- .../defaults/CommitIcebergActionSuite.scala | 71 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java index 01cd87291ec..f464e1adca4 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java @@ -306,8 +306,8 @@ public static Row generateIcebergCompatWriterV3RemoveAction( fileStatus, partitionValues, dataChange, - Optional.empty() /* baseRowId */, - Optional.empty() /* defaultRowCommitVersion */, + baseRowId, + defaultRowCommitVersion, deletionVectorDescriptor); return SingleAction.createRemoveFileSingleAction(removeFileRow); } diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala index 90d8afc759e..1020a843a73 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala @@ -54,6 +54,19 @@ trait AbstractCommitIcebergActionSuite extends AnyFunSuite { self: AbstractWrite private val tblPropertiesIcebergWriterCompatV3Enabled = Map( TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.getKey -> "true") + /** Helper to create a transaction, generate a single action, and commit it. */ + private def commitSingleAction( + engine: Engine, + tablePath: String, + actionFn: Transaction => Row): Unit = { + val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) + val action = actionFn(txn) + commitTransaction( + txn, + engine, + inMemoryIterable(toCloseableIterator(Seq(action).asJava.iterator()))) + } + private def createIcebergCompatAction( actionType: String, // "ADD" or "REMOVE" version: String, @@ -629,4 +642,62 @@ trait AbstractCommitIcebergActionSuite extends AnyFunSuite { self: AbstractWrite } } } + + test("V3: baseRowId and defaultRowCommitVersion are forwarded in remove actions") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable( + engine, + tablePath, + testSchema, + tableProperties = tblPropertiesIcebergWriterCompatV3Enabled) + + // Append 1 add so we have something to remove + commitSingleAction( + engine, + tablePath, + createIcebergCompatAction( + "ADD", + "V3", + _, + engine, + generateDataFileStatus(tablePath, "file1.parquet"), + true)) + + // Remove with explicit baseRowId and defaultRowCommitVersion + val expectedBaseRowId = 42L + val expectedDefaultRowCommitVersion = 7L + commitSingleAction( + engine, + tablePath, + txn => + GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV3RemoveAction( + txn.getTransactionState(engine), + generateDataFileStatus(tablePath, "file1.parquet"), + Collections.emptyMap(), + true, // dataChange + Optional.of[java.lang.Long](expectedBaseRowId), + Optional.of[java.lang.Long](expectedDefaultRowCommitVersion), + Optional.empty(), // deletionVectorDescriptor + Optional.of( + TransactionStateRow.getPhysicalSchema(txn.getTransactionState(engine))))) + + // Read back the remove action and verify the values are preserved + val rows = Table.forPath(engine, tablePath).asInstanceOf[TableImpl] + .getChanges(engine, 2, 2, Set(DeltaAction.REMOVE).asJava) + .toSeq + .flatMap(_.getRows.toSeq) + .filterNot(row => row.isNullAt(row.getSchema.indexOf("remove"))) + + assert(rows.size == 1) + val removeFile = new RemoveFile(rows.head.getStruct(rows.head.getSchema.indexOf("remove"))) + assert( + removeFile.getBaseRowId.isPresent, + "baseRowId should be present when caller provides it") + assert(removeFile.getBaseRowId.get == expectedBaseRowId) + assert( + removeFile.getDefaultRowCommitVersion.isPresent, + "defaultRowCommitVersion should be present when caller provides it") + assert(removeFile.getDefaultRowCommitVersion.get == expectedDefaultRowCommitVersion) + } + } } From c6bd29b4246230ead0cca8e6c374168f612da4c9 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Fri, 13 Mar 2026 10:43:35 -0700 Subject: [PATCH 25/80] [Spark] Fix timestamp overflow in dataskipping (#6260) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description During data-skipping, we add one millisecond to timestamp max stats to compensate for the fact that JSON serialization will truncate them to milliseconds. This has the consequence that if the data contains a timestamp within the range (Long.MaxValue - 1000, Long.MaxValue], then the addition will throw an overflow exception. This PR fixes this by adding a check for if the stats is within that range, then we saturate the stats to Long.MaxValue instead of adding 1000 to it. ## How was this patch tested? Unit test ## Does this PR introduce _any_ user-facing changes? No --- .../sql/delta/stats/DataSkippingReader.scala | 43 ++++--- .../delta/stats/DataSkippingDeltaTests.scala | 117 +++++++++++++----- 2 files changed, 107 insertions(+), 53 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala index 7ed7a76629a..1685e2bed64 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala @@ -232,6 +232,27 @@ private[delta] object DataSkippingReader { Option(ExpressionEncoder[java.lang.Long]()), Option(ExpressionEncoder[java.lang.Long]()), Option(ExpressionEncoder[java.lang.Long]())) + + /** + * For timestamps, JSON serialization will truncate to milliseconds. This means + * that we must adjust 1 millisecond upwards for max stats, or we will incorrectly skip + * records that differ only in microsecond precision. (For example, a file containing only + * 01:02:03.456789 will be written with min == max == 01:02:03.456, so we must consider it + * to contain the range from 01:02:03.456 to 01:02:03.457.) + * + * To avoid overflow when the timestamp is near Long.MAX_VALUE, we check if adding 1 + * millisecond would overflow. If so, we saturate to Long.MAX_VALUE to ensure the max stat + * is >= all actual values in the file while avoiding arithmetic overflow. + */ + def getAdjustedTimestamp(col: Column, tsType: DataType): Column = { + val maxTimestampLiteral = Literal(Long.MaxValue, tsType) + val overflowThresholdLiteral = Literal(Long.MaxValue - 1000, tsType) + val adjustedExpr = If( + GreaterThan(col.expr, overflowThresholdLiteral), + maxTimestampLiteral, + TimestampAdd("MILLISECOND", Literal(1L, LongType), col.expr)) + Column(Cast(adjustedExpr, tsType)) + } } /** @@ -1087,27 +1108,9 @@ trait DataSkippingReaderBase .filterNot(_._2.isInstanceOf[StructType]) .map { case (statCol, TimestampType, _) if pathToStatType.head == MAX => - // SC-22824: For timestamps, JSON serialization will truncate to milliseconds. This means - // that we must adjust 1 millisecond upwards for max stats, or we will incorrectly skip - // records that differ only in microsecond precision. (For example, a file containing only - // 01:02:03.456789 will be written with min == max == 01:02:03.456, so we must consider it - // to contain the range from 01:02:03.456 to 01:02:03.457.) - // - // There is a longer term task SC-22825 to fix the serialization problem that caused this. - // But we need the adjustment in any case to correctly read stats written by old versions. - // TimeAdd is removed in Spark 4.1, using TimestampAdd instead - Column(Cast(TimestampAdd( - "MILLISECOND", - new Literal(1L, LongType), - statCol.expr), TimestampType)) + getAdjustedTimestamp(statCol, TimestampType) case (statCol, TimestampNTZType, _) if pathToStatType.head == MAX => - // We also apply the same adjustment of max stats that was applied to Timestamp - // for TimestampNTZ because these 2 types have the same precision in terms of time. - // TimeAdd is removed in Spark 4.1, using TimestampAdd instead - Column(Cast(TimestampAdd( - "MILLISECOND", - new Literal(1L, LongType), - statCol.expr), TimestampNTZType)) + getAdjustedTimestamp(statCol, TimestampNTZType) case (statCol, _, _) => statCol } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala index 0c6f961a1ae..ed76da9a302 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala @@ -1671,41 +1671,92 @@ trait DataSkippingDeltaTestsBase extends QueryTest s"STRUCT(CAST(strTs AS $timestampType) AS ts) AS nested") val tempDir = Utils.createTempDir() - val r = DeltaLog.forTable(spark, tempDir) - df.coalesce(1).write.format("delta").save(r.dataPath.toString) + val log = DeltaLog.forTable(spark, tempDir) + df.coalesce(1).write.format("delta").save(log.dataPath.toString) + + checkSkipping( + log, + // Check to ensure that the value actually in the file is always in range queries. + hits = Seq( + s"""ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""nested.ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""nested.ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""TS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""nEstED.tS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)"""), + // Check the range of values that are far enough away to be data skipped. Note that the + // values are aligned with millisecond boundaries because of the JSON serialization + // truncation. + misses = Seq( + s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"""ts <= cast("2019-09-04 01:02:03.455999" AS $timestampType)""", + s"""nested.ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"""nested.ts <= cast("2019-09-09 01:02:03.455999" AS $timestampType)""", + s"""TS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"""nEstED.tS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)"""), + data = data, + checkEmptyUnusedFiltersForHits = false) + } + } - // Check to ensure that the value actually in the file is always in range queries. - val hits = Seq( - s"""ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""nested.ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""nested.ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""TS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""nEstED.tS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""") - - // Check the range of values that are far enough away to be data skipped. Note that the values - // are aligned with millisecond boundaries because of the JSON serialization truncation. - val misses = Seq( - s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", - s"""ts <= cast("2019-09-04 01:02:03.455999" AS $timestampType)""", - s"""nested.ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", - s"""nested.ts <= cast("2019-09-09 01:02:03.455999" AS $timestampType)""", - s"""TS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", - s"""nEstED.tS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""") - - hits.foreach { predicate => - Given(predicate) - if (filesRead(r, predicate) != 1) { - failPretty(s"Expected hit but got miss for $predicate", predicate, data) - } - } + for (timestampType <- Seq("TIMESTAMP", "TIMESTAMP_NTZ")) { + test(s"data skipping on $timestampType with Long.MaxValue") { + val maxVal = "294247-01-10 04:00:54.775807Z" + val tempDir = Utils.createTempDir() + val log = DeltaLog.forTable(spark, tempDir) + Seq(maxVal).toDF("strTs") + .selectExpr(s"CAST(strTs AS $timestampType) AS ts") + .coalesce(1) + .write + .format("delta") + .save(log.dataPath.toString) + + checkSkipping( + log, + hits = Seq( + s"""ts >= cast("$maxVal" AS $timestampType)""", + s"""ts >= "$maxVal"""", + s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + // This still hits because of JSON truncation to milliseconds + s"""ts < cast("$maxVal" AS $timestampType)""".stripMargin), + misses = Seq( + s"""ts <= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"""ts > cast("$maxVal" AS $timestampType)"""), + data = maxVal, + checkEmptyUnusedFiltersForHits = false) + } + } - misses.foreach { predicate => - Given(predicate) - if (filesRead(r, predicate) != 0) { - failPretty(s"Expected miss but got hit for $predicate", predicate, data) - } - } + for (timestampType <- Seq("TIMESTAMP", "TIMESTAMP_NTZ")) { + test(s"data skipping on $timestampType near Long.MaxValue") { + val tempDir = Utils.createTempDir() + val log = DeltaLog.forTable(spark, tempDir) + + val nearMaxMicros = Long.MaxValue - 999L + + // Create DataFrame with the near-max timestamp value + Seq(nearMaxMicros).toDF("microsSinceEpoch") + .selectExpr(s"TIMESTAMP_MICROS(microsSinceEpoch) AS ts") + .selectExpr(s"CAST(ts AS $timestampType) AS ts") + .coalesce(1) + .write + .format("delta") + .save(log.dataPath.toString) + + checkSkipping( + log, + // maxValue of the stats on ts will be saturated to Long.MaxValue instead + // of being added 1000 microseconds, which will cause a long overflow. + hits = Seq( + s"ts >= TIMESTAMP_MICROS($nearMaxMicros)", + s"ts >= TIMESTAMP_MICROS(${nearMaxMicros - 100})", + s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"ts >= TIMESTAMP_MICROS(${Long.MaxValue - 1000})", + s"ts < TIMESTAMP_MICROS($nearMaxMicros)"), + misses = Seq( + s"""ts <= cast("2019-09-09 01:02:03.457001" AS $timestampType)"""), + data = nearMaxMicros.toString, + checkEmptyUnusedFiltersForHits = false) } } From e49b4a56cc5c0a2d94d82f1e4de9c4caee66d139 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Fri, 13 Mar 2026 23:19:53 +0530 Subject: [PATCH 26/80] Add lifecycle management for IcebergRESTCatalogPlanningClient (#6268) ## Problem `IcebergRESTCatalogPlanningClient` implements `AutoCloseable` but the `ServerSidePlanningClient` trait does not extend it, and `ServerSidePlannedTable` never calls `close()`. This leaks HTTP client connections. ## Fix Add proper lifecycle management so the HTTP client is closed when no longer needed. ## Validation - Confirmed by reading the trait hierarchy and table construction code. - New test verifies the client is properly closed. ## Regression Prevention New test verifies client lifecycle management. --------- Co-authored-by: foo.bar --- .../IcebergRESTCatalogPlanningClient.scala | 2 +- .../ServerSidePlannedTable.scala | 26 ++++++++++----- .../ServerSidePlanningClient.scala | 8 ++++- .../ServerSidePlannedTableSuite.scala | 32 +++++++++++++++++++ .../TestServerSidePlanningClient.scala | 7 ++++ 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala index f800a2d8956..474c090a3f2 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala @@ -68,7 +68,7 @@ private case class CatalogConfigResponse( class IcebergRESTCatalogPlanningClient( baseUriRaw: String, catalogName: String, - token: String) extends ServerSidePlanningClient with AutoCloseable { + token: String) extends ServerSidePlanningClient { // Normalize baseUri to handle trailing slashes private val baseUri = baseUriRaw.stripSuffix("/") diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala index 54a449dea9b..e6e0057fcaf 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala @@ -179,7 +179,7 @@ class ServerSidePlannedTable( tableName: String, tableSchema: StructType, planningClient: ServerSidePlanningClient) - extends Table with SupportsRead with DeltaLogging { + extends Table with SupportsRead with AutoCloseable with DeltaLogging { // Returns fully qualified name (e.g., "catalog.database.table"). // The databaseName parameter receives ident.namespace().mkString(".") from DeltaCatalog, @@ -195,6 +195,10 @@ class ServerSidePlannedTable( override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { new ServerSidePlannedScanBuilder(spark, databaseName, tableName, tableSchema, planningClient) } + + override def close(): Unit = { + planningClient.close() + } } /** @@ -341,13 +345,19 @@ class ServerSidePlannedScan( } } - // Call the server-side planning API to get the scan plan with files AND credentials - private lazy val scanPlan: ScanPlan = planningClient.planScan( - databaseName, - tableName, - combinedFilter, - projectionColumnNames, - limit) + // Call the server-side planning API to get the scan plan with files AND credentials. + // Close the client after planning - the scan plan contains all data needed for partition + // creation and reading, so the client (and its HTTP connection) is no longer needed. + private lazy val scanPlan: ScanPlan = { + val plan = planningClient.planScan( + databaseName, + tableName, + combinedFilter, + projectionColumnNames, + limit) + planningClient.close() + plan + } // Explicitly signal that columnar is unsupported to prevent early enumeration of the partitions override def columnarSupportMode(): Scan.ColumnarSupportMode = diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala index 2ade4f0ba8f..2162e9b8b78 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala @@ -35,7 +35,7 @@ case class ScanFile( * representation for filter pushdown. This keeps the interface catalog-agnostic while allowing * each server-side planning catalog implementation to convert filters to their own native format. */ -trait ServerSidePlanningClient { +trait ServerSidePlanningClient extends AutoCloseable { /** * Plan a table scan and return the list of files to read. * @@ -61,6 +61,12 @@ trait ServerSidePlanningClient { * @return true if ALL filters can be converted, false if ANY filter cannot be converted */ def canConvertFilters(filters: Array[Filter]): Boolean + + /** + * Close any resources held by this client. + * Default implementation is a no-op for clients that don't hold resources. + */ + override def close(): Unit = {} } /** diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala index 771c7a148af..30533ec0643 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala @@ -485,4 +485,36 @@ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { assert(capturedProjection.isEmpty, "Should not fire a planTable request for EXPLAIN") } } + + test("ServerSidePlannedTable closes planning client on close") { + var clientClosed = false + val trackingClient = new ServerSidePlanningClient { + override def planScan( + databaseName: String, + table: String, + filterOption: Option[Filter], + projectionOption: Option[Seq[String]], + limitOption: Option[Int]): ScanPlan = ScanPlan(Seq.empty) + override def canConvertFilters(filters: Array[Filter]): Boolean = true + override def close(): Unit = { clientClosed = true } + } + + val table = new ServerSidePlannedTable( + spark, "test_db", "test_table", new org.apache.spark.sql.types.StructType(), trackingClient) + assert(!clientClosed, "Client should not be closed before table.close()") + table.close() + assert(clientClosed, "Client should be closed after table.close()") + } + + test("planning client is closed after scan completes") { + withPushdownCapturingEnabled { + assert(!TestServerSidePlanningClient.isClientClosed, + "Client should not be closed before query execution") + + sql("SELECT id FROM test_db.shared_test").collect() + + assert(TestServerSidePlanningClient.isClientClosed, + "Planning client should be closed after scan plan is obtained") + } + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala index 21ce5501ed1..11b149fd18d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala @@ -94,6 +94,10 @@ class TestServerSidePlanningClient(spark: SparkSession) extends ServerSidePlanni TestServerSidePlanningClient.filtersConvertible } + override def close(): Unit = { + TestServerSidePlanningClient.clientClosed = true + } + private def getFileFormat(path: Path): String = "parquet" } @@ -106,10 +110,12 @@ object TestServerSidePlanningClient { private var capturedProjection: Option[Seq[String]] = None private var capturedLimit: Option[Int] = None private var filtersConvertible: Boolean = true // Default: all filters convertible + private[serverSidePlanning] var clientClosed: Boolean = false def getCapturedFilter: Option[Filter] = capturedFilter def getCapturedProjection: Option[Seq[String]] = capturedProjection def getCapturedLimit: Option[Int] = capturedLimit + def isClientClosed: Boolean = clientClosed /** * Configure whether filters should be treated as convertible. @@ -124,6 +130,7 @@ object TestServerSidePlanningClient { capturedProjection = None capturedLimit = None filtersConvertible = true // Reset to default + clientClosed = false } } From 0d54da51e7eade47b8115d92aaf7be1e8e4c011f Mon Sep 17 00:00:00 2001 From: Chirag Singh <137233133+chirag-s-db@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:42:45 -0700 Subject: [PATCH 27/80] [Spark] Deflake PartitionLikeDataSkippingSuite (#6283) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Deflake PartitionLikeDataSkippingSuite by making result comparison insensitive to nondeterministic record ordering. ## How was this patch tested? See test change. ## Does this PR introduce _any_ user-facing changes? No. --- .../spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala index 2dd3d595694..ce0d83159e7 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala @@ -71,7 +71,7 @@ trait PartitionLikeDataSkippingSuiteBase minNumFilesToApply.toString) { // Execute the query with partition-like filters and validate that the result matches. val res = sql(query).collect() - assert(res.sameElements(baseResult)) + assert(res.sortBy(_.toString).sameElements(baseResult.sortBy(_.toString))) val predicates = sql(query).queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition From f328807d32bd37751227ad047f41c3353c5aba0e Mon Sep 17 00:00:00 2001 From: Yi Li Date: Fri, 13 Mar 2026 13:34:09 -0700 Subject: [PATCH 28/80] [Spark] Block metadata changes on UC-managed tables (#6243) #### Which Delta project/connector is this regarding? - [X] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Block metadata changes on UC-managed tables Without this, metadata mutations on a managed table succeed locally but leave the UC catalog unaware, causing metadata drift. Two kill switches in OptimisticTransaction: - updateMetadata(): blocks schema, partition, description, and configuration changes. Covers INSERT with autoMerge, MERGE with schema evolution, and DataFrame writes with mergeSchema. - commitLarge(): blocks clustering DomainMetadata changes for RESTORE TABLE, which bypasses updateMetadata(). ALTER TABLE is already blocked by UCSingleCatalog before reaching Delta. ## How was this patch tested? New tests added. ## Does this PR introduce _any_ user-facing changes? Yes. Table schema evolution with DML is blocked for UC managed tables. --------- Signed-off-by: Yi Li --- .../sql/delta/OptimisticTransaction.scala | 67 +++- .../CoordinatedCommitsUtils.scala | 2 +- .../UCCommitCoordinatorBuilder.scala | 5 +- .../delta/UCManagedTableKillSwitchSuite.scala | 91 +++++ .../UCDeltaTableBlockMetadataUpdateTest.java | 377 ++++++++++++++++++ .../UCDeltaTableDataFrameWriteTest.java | 29 +- .../UCDeltaTableIntegrationBaseTest.java | 41 +- 7 files changed, 601 insertions(+), 11 deletions(-) create mode 100644 spark/src/test/scala/org/apache/spark/sql/delta/UCManagedTableKillSwitchSuite.scala create mode 100644 spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableBlockMetadataUpdateTest.java diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index 031f0779ec9..db67d28d238 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.constraints.{Constraints, Invariants} -import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils, TableCommitCoordinatorClient} +import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils, TableCommitCoordinatorClient, UCCommitCoordinatorBuilder} import org.apache.spark.sql.delta.files._ import org.apache.spark.sql.delta.hooks.{CheckpointHook, ChecksumHook, GenerateSymlinkManifest, HudiConverterHook, IcebergConverterHook, PostCommitHook, UpdateCatalogFactory} import org.apache.spark.sql.delta.implicits.addFileEncoder @@ -57,11 +57,12 @@ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkException import org.apache.spark.internal.{MDC, MessageWithContext} import org.apache.spark.sql.{AnalysisException, Column, DataFrame, SaveMode, SparkSession} -import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.UnsetTableProperties import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, ResolveDefaultColumns} +import org.apache.spark.sql.delta.clustering.ClusteringMetadataDomain import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.util.{Clock, Utils} @@ -573,9 +574,62 @@ trait OptimisticTransactionImpl extends TransactionHelper "Cannot update the metadata in a transaction that has already written data.") assert(newMetadata.isEmpty, "Cannot change the metadata more than once in a transaction.") + // Temporary: block metadata changes on UC-managed CatalogOwned tables until Delta supports + // propagating metadata updates to UC. UC is identified by catalog implementation class (handles + // "spark_catalog" registration). New table creation is naturally excluded because + // isCatalogOwned is false until the first commit. REPLACE TABLE is currently also blocked + // here and will need to be explicitly allowed once UC supports metadata propagation. + // Intentionally conservative: configuration is compared as a whole map, which also + // catches Delta-internal additions (e.g. table-feature flags). This is acceptable for + // a temporary kill switch - once Delta supports propagating metadata updates to UC, + // this check will be removed entirely. + val existingMetadata = snapshot.metadata + if (isUCManagedTable && + (proposedNewMetadata.schemaString != existingMetadata.schemaString || + proposedNewMetadata.partitionColumns != existingMetadata.partitionColumns || + proposedNewMetadata.description != existingMetadata.description || + proposedNewMetadata.configuration != existingMetadata.configuration)) { + throw DeltaErrors.operationNotSupportedException( + "Metadata changes on Unity Catalog managed tables") + } updateMetadataInternal(proposedNewMetadata, ignoreDefaultProperties) } + /** + * True if this transaction targets a UC-managed CatalogOwned table. + * + * Computed once as a lazy val because catalogTable and SparkSession are immutable for + * the lifetime of a transaction. Visibility is protected[delta] (not private) to allow + * test subclasses to override without requiring UCSingleCatalog. + */ + protected[delta] lazy val isUCManagedTable: Boolean = { + snapshot.isCatalogOwned && + catalogTable.exists { ct => + ct.tableType == CatalogTableType.MANAGED && + CatalogOwnedTableUtils.getCatalogName(spark, ct.identifier) + .contains(UCCommitCoordinatorBuilder.COORDINATOR_NAME) + } + } + + /** + * Returns true if committing [[dm]] would change the clustering columns on a UC-managed + * CatalogOwned table and should therefore be blocked. + * + * Both a missing entry and a removed=true tombstone mean "no clustering", so the effective + * configuration is normalised to Option[String] before comparison. + */ + private def isClusteringChangedOnUCManagedTable(dm: DomainMetadata): Boolean = { + if (dm.domain != ClusteringMetadataDomain.domainName) return false + if (!isUCManagedTable) return false + val existingConfig = + snapshot.domainMetadata + .find(_.domain == ClusteringMetadataDomain.domainName) + .filterNot(_.removed) + .map(_.configuration) + val incomingConfig = if (dm.removed) None else Some(dm.configuration) + incomingConfig != existingConfig + } + /** * Can this transaction still update the metadata? * This is allowed only once per transaction. @@ -1924,6 +1978,15 @@ trait OptimisticTransactionImpl extends TransactionHelper newProtocol.toIterator allActions = allActions.map { action => action match { + case dm: DomainMetadata if isClusteringChangedOnUCManagedTable(dm) => + // Temporary: block clustering changes on UC-managed tables (commitLarge() path). + // commitLarge() bypasses prepareCommit(), so this guard is needed separately. + // The check is intentionally inside the lazy map: commitLarge streams actions to + // avoid materialising large sets, so an eager pre-scan is not practical. The + // exception is thrown before any data is written to the commit coordinator because + // the iterator is consumed first during serialisation. + throw DeltaErrors.operationNotSupportedException( + "Clustering column changes on Unity Catalog managed tables") case a: AddFile => assertDeletionVectorWellFormed(a) validateAddFileInvariants(a, notNullPartitionCols) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala index 1e01bf95ae0..0b3c438685d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala @@ -166,7 +166,7 @@ object CatalogOwnedTableUtils extends DeltaLogging { case spark.sessionState.analyzer.CatalogAndIdentifier(catalog, _) => if (catalog.getClass.getName == UCCommitCoordinatorBuilder.UNITY_CATALOG_CONNECTOR_CLASS) { // UC is the current commit coordinator. - Some("unity-catalog") + Some(UCCommitCoordinatorBuilder.COORDINATOR_NAME) } else { // Other catalog (e.g., `spark_catalog`) is the commit coordinator. Some(catalog.name) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala index 4dc34819946..0f92e15f870 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala @@ -43,6 +43,9 @@ import org.apache.spark.sql.SparkSession object UCCommitCoordinatorBuilder extends CatalogOwnedCommitCoordinatorBuilder with DeltaLogging { + /** The coordinator name used in table metadata to identify UC-managed tables. */ + final val COORDINATOR_NAME: String = "unity-catalog" + /** Prefix for Spark SQL catalog configurations. */ final private val SPARK_SQL_CATALOG_PREFIX = "spark.sql.catalog." @@ -64,7 +67,7 @@ object UCCommitCoordinatorBuilder // Use a var instead of val for ease of testing by injecting different UCClientFactory. private[delta] var ucClientFactory: UCClientFactory = UCTokenBasedRestClientFactory - override def getName: String = "unity-catalog" + override def getName: String = COORDINATOR_NAME override def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { val metastoreId = conf.getOrElse( diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/UCManagedTableKillSwitchSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/UCManagedTableKillSwitchSuite.scala new file mode 100644 index 00000000000..6a285829a13 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/UCManagedTableKillSwitchSuite.scala @@ -0,0 +1,91 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +import org.apache.spark.sql.delta.actions.DomainMetadata +import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite +import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumn} +import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} + +import org.apache.spark.sql.catalyst.TableIdentifier + +/** + * Unit tests for the kill switch that blocks clustering column changes on UC-managed + * CatalogOwned tables. These tests bypass UCSingleCatalog (only available in sparkUnityCatalog) + * by overriding [[OptimisticTransaction.isUCManagedTable]] in a test-local subclass. + * + * The kill switch fires in [[OptimisticTransaction.commitLarge]], which is used by RESTORE TABLE + * and bypasses prepareCommit. + */ +class UCManagedTableKillSwitchSuite + extends CatalogOwnedTestBaseSuite + with DeltaSQLTestUtils + with DeltaSQLCommandTest { + + // Enable CatalogOwned by default so every CREATE TABLE produces a CatalogOwned table. + override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) + + /** + * Test subclass that pretends it targets a UC-managed table, bypassing the real + * [[CatalogOwnedTableUtils.getCatalogName]] check that requires UCSingleCatalog. + */ + private class UCManagedTxn(log: DeltaLog, snap: Snapshot) + extends OptimisticTransaction(log, None, snap) { + override protected[delta] lazy val isUCManagedTable: Boolean = true + } + + private val clusteringOnId: Seq[DomainMetadata] = + Seq(ClusteredTableUtils.createDomainMetadata(Seq(ClusteringColumn(Seq("id"))))) + + private val clusteringOnName: Seq[DomainMetadata] = + Seq(ClusteredTableUtils.createDomainMetadata(Seq(ClusteringColumn(Seq("name"))))) + + /** Creates a CatalogOwned clustered-by-id table and returns its (log, snapshot). */ + private def createClusteredTable(tableName: String): (DeltaLog, Snapshot) = { + sql(s"CREATE TABLE $tableName (id INT, name STRING) USING delta " + + s"CLUSTER BY (id) TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported')") + sql(s"INSERT INTO $tableName VALUES (1, 'a')") + val (log, snap) = + DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) + assert(snap.isCatalogOwned, "table should be CatalogOwned") + (log, snap) + } + + test("commitLarge blocks clustering change on UC-managed CatalogOwned table") { + withTable("tbl") { + val (log, snap) = createClusteredTable("tbl") + val ex = intercept[DeltaAnalysisException] { + new UCManagedTxn(log, snap).commitLarge( + spark, + clusteringOnName.iterator, + newProtocolOpt = None, + op = DeltaOperations.Restore(Some(0L), None), + context = Map.empty, + metrics = Map.empty) + } + assert(ex.getMessage.contains("Clustering column changes on Unity Catalog managed tables")) + } + } + + test("commit with unchanged clustering is allowed on UC-managed CatalogOwned table") { + withTable("tbl") { + val (log, snap) = createClusteredTable("tbl") + // Commit the same clustering DomainMetadata that the snapshot already has - must not throw. + new UCManagedTxn(log, snap).commit(clusteringOnId, DeltaOperations.ManualUpdate) + } + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableBlockMetadataUpdateTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableBlockMetadataUpdateTest.java new file mode 100644 index 00000000000..0ffa51e5f23 --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableBlockMetadataUpdateTest.java @@ -0,0 +1,377 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +/** + * Tests that schema-changing and property-changing operations are blocked on Unity Catalog managed + * (CatalogOwned) tables — regardless of which layer does the blocking. + * + *

    There are two distinct layers of protection: + * + *

      + *
    1. Kill switch in {@code OptimisticTransaction.updateMetadata()}: blocks any + * commit that would change schema, partitions, description, or configuration on an existing + * CatalogOwned table. A second guard in {@code prepareCommit()} and {@code commitLarge()} + * blocks {@code delta.clustering} {@code DomainMetadata} changes (e.g. via RESTORE TABLE to a + * version written by an older client that had different clustering columns). + *
    2. UC catalog layer in {@code UCSingleCatalog}: {@code alterTable()} throws + * {@code UnsupportedOperationException} for all ALTER TABLE variants. INSERT OVERWRITE with + * {@code overwriteSchema=true} and {@code CREATE OR REPLACE TABLE} both route through REPLACE + * TABLE AS SELECT (RTAS) because {@code UCSingleCatalog} does not implement {@code + * StagingTableCatalog}; RTAS is not supported in OSS Delta. + *
    + * + *

    EXTERNAL tables are not CatalogOwned and are NOT affected by the kill switch; they continue to + * allow schema evolution as before. + */ +public class UCDeltaTableBlockMetadataUpdateTest extends UCDeltaTableIntegrationBaseTest { + + // Error produced by the kill switch in OptimisticTransaction.updateMetadata(). + private static final String KILL_SWITCH_ERROR = + "Metadata changes on Unity Catalog managed tables"; + + // Error produced by the clustering kill switch in commitLarge(). + private static final String CLUSTERING_KILL_SWITCH_ERROR = + "Clustering column changes on Unity Catalog managed tables"; + + // Error produced by UCSingleCatalog.alterTable() for all ALTER TABLE variants. + private static final String ALTER_TABLE_ERROR = "Altering a table is not supported yet"; + + // Error produced by OSS Delta when REPLACE TABLE AS SELECT (RTAS) is attempted. + // Triggered by CREATE OR REPLACE TABLE and DataFrame saveAsTable(overwrite+overwriteSchema) + // when the target catalog does not implement StagingTableCatalog. + private static final String RTAS_ERROR = "REPLACE TABLE AS SELECT (RTAS) is not supported"; + + // --------------------------------------------------------------------------- + // Kill-switch tests: operations blocked by OptimisticTransaction.updateMetadata() + // --------------------------------------------------------------------------- + + /** + * INSERT with {@code autoMerge=true} and MERGE INTO with schema evolution must be blocked by the + * kill switch in {@code updateMetadata()} on CatalogOwned tables. The kill switch covers all + * metadata fields (schema, partitions, description, properties); schema evolution via autoMerge + * is the primary user-facing path that reaches it without going through ALTER TABLE. + */ + @Test + public void testMetadataChangesViaWritesAreBlocked() throws Exception { + withNewTable( + "block_schema_evolution_target", + "id INT, name STRING", + TableType.MANAGED, + targetTable -> { + sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); + withNewTable( + "block_schema_evolution_source", + "id INT, name STRING, extra STRING", + TableType.EXTERNAL, + sourceTable -> { + sql("INSERT INTO %s VALUES (2, 'new', 'extra_value')", sourceTable); + sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); + try { + // INSERT with autoMerge introduces a new column. + assertThrowsWithCauseContaining( + KILL_SWITCH_ERROR, + () -> sql("INSERT INTO %s SELECT * FROM %s", targetTable, sourceTable)); + + // MERGE INTO with autoMerge introduces a new column from the source. + assertThrowsWithCauseContaining( + KILL_SWITCH_ERROR, + () -> + sql( + "MERGE INTO %s AS target " + + "USING %s AS source " + + "ON target.id = source.id " + + "WHEN NOT MATCHED THEN INSERT *", + targetTable, sourceTable)); + } finally { + sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); + } + }); + }); + } + + // --------------------------------------------------------------------------- + // UC catalog layer tests: operations blocked by UCSingleCatalog before reaching Delta + // --------------------------------------------------------------------------- + + /** + * All ALTER TABLE variants on a CatalogOwned table must be blocked by {@code + * UCSingleCatalog.alterTable()}, which throws {@code UnsupportedOperationException} for every + * table change regardless of the specific operation. + * + *

    Covered operations: SET TBLPROPERTIES (configuration change), ADD COLUMNS (schema change), + * and CLUSTER BY (clustering change). All share the same managed table and each throws before + * modifying anything. + */ + @Test + public void testAlterTableOperationsAreBlocked() throws Exception { + withNewTable( + "block_alter_table_test", + "id INT, name STRING", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'initial')", tableName); + + // ALTER TABLE SET TBLPROPERTIES would change configuration. + assertThrowsWithCauseContaining( + ALTER_TABLE_ERROR, + () -> sql("ALTER TABLE %s SET TBLPROPERTIES ('custom.key' = 'value')", tableName)); + + // ALTER TABLE ADD COLUMNS would change the schema. + assertThrowsWithCauseContaining( + ALTER_TABLE_ERROR, () -> sql("ALTER TABLE %s ADD COLUMNS (extra STRING)", tableName)); + + // ALTER TABLE CLUSTER BY would change clustering columns. + assertThrowsWithCauseContaining( + ALTER_TABLE_ERROR, () -> sql("ALTER TABLE %s CLUSTER BY (id)", tableName)); + }); + } + + /** + * RESTORE TABLE to a version with unchanged clustering must succeed. The clustering kill switch + * only fires when clustering actually changes. + */ + @Test + public void testRestoreTableWithUnchangedClusteringSucceeds() throws Exception { + String tableName = fullTableName("restore_unchanged_clustering_test"); + try { + sql( + "CREATE TABLE %s (id INT, name STRING) USING DELTA CLUSTER BY (id)" + + " TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + long versionAfterInsert = currentVersion(tableName); + // Restore to version 0 (before the insert): clustering is unchanged, must succeed. + sql("RESTORE TABLE %s TO VERSION AS OF %d", tableName, versionAfterInsert - 1); + check(tableName, List.of()); + } finally { + sql("DROP TABLE IF EXISTS %s", tableName); + } + } + + /** + * RESTORE TABLE to a version whose clustering differs from the current version must be blocked by + * the kill switch in {@code commitLarge()}. + * + *

    A table can have different clustering at different versions if an older client or another + * connector wrote a version before this guard was in place. This test simulates that by writing a + * Delta log entry with different clustering directly to the table's underlying storage (bypassing + * the kill switch), then attempting a RESTORE that would change clustering back. + * + *

    This test is local-only: it needs direct filesystem access to write the fake commit, which + * is only available when backed by {@code S3CredentialFileSystem} (local UC). + */ + @Test + public void testRestoreTableWithClusteringChangeIsBlocked() throws Exception { + Assumptions.assumeFalse( + isUCRemoteConfigured(), "Requires local filesystem access (local UC only)"); + + String tableName = fullTableName("restore_clustering_change_test"); + try { + sql( + "CREATE TABLE %s (id INT, name STRING) USING DELTA CLUSTER BY (id)" + + " TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + long insertVersion = currentVersion(tableName); + + // Simulate an older client writing a version with different clustering directly to the + // table's storage, bypassing the kill switch. This is how a real-world scenario could arise. + String s3Location = + sql("DESCRIBE FORMATTED %s", tableName).stream() + .filter(r -> r.size() >= 2 && "Location".equalsIgnoreCase(r.get(0).trim())) + .map(r -> r.get(1).trim()) + .findFirst() + .orElseThrow(); + // Convert URI to a local filesystem path: + // - S3CredentialFileSystem maps s3://fakeS3Bucket/abs/path → /abs/path + // - Local UC may return a file: URI directly + String localTablePath; + if (s3Location.startsWith("s3://")) { + localTablePath = s3Location.replaceAll("s3://[^/]+", ""); + } else if (s3Location.startsWith("file:")) { + localTablePath = s3Location.replaceAll("^file:/+", "/"); + } else { + localTablePath = s3Location; + } + long hackedVersion = insertVersion + 1; + Path hackedCommitFile = + Paths.get(localTablePath, "_delta_log", String.format("%020d.json", hackedVersion)); + // Write a minimal Delta commit with delta.clustering on 'name' instead of 'id'. + String clusteringOnName = "{\\\"clusteringColumns\\\":[[\\\"name\\\"]]}"; + Files.writeString( + hackedCommitFile, + "{\"commitInfo\":{\"timestamp\":1000000000000,\"inCommitTimestamp\":1000000000000," + + "\"operation\":\"MANUAL UPDATE\",\"operationParameters\":{},\"isBlindAppend\":false}}\n" + + "{\"domainMetadata\":{\"domain\":\"delta.clustering\"," + + "\"configuration\":\"" + + clusteringOnName + + "\",\"removed\":false}}\n"); + + // RESTORE to a version before the hacked commit: the current snapshot now shows 'name' + // clustering, so restoring to 'id' clustering fires the kill switch. + assertThrowsWithCauseContaining( + CLUSTERING_KILL_SWITCH_ERROR, + () -> sql("RESTORE TABLE %s TO VERSION AS OF %d", tableName, insertVersion - 1)); + } finally { + sql("DROP TABLE IF EXISTS %s", tableName); + } + } + + /** + * INSERT OVERWRITE with {@code overwriteSchema=true} that would replace the schema of an existing + * CatalogOwned table must be blocked. + * + *

    Because {@code UCSingleCatalog} does not implement {@code StagingTableCatalog}, Spark routes + * the overwrite-with-schema-change through REPLACE TABLE AS SELECT (RTAS), which OSS Delta does + * not support. + */ + @Test + public void testInsertOverwriteWithOverwriteSchemaIsBlocked() throws Exception { + withNewTable( + "block_overwrite_schema_target", + "id INT, name STRING", + TableType.MANAGED, + targetTable -> { + sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); + withNewTable( + "block_overwrite_schema_source", + "id INT, name STRING, extra STRING", + TableType.EXTERNAL, + sourceTable -> { + sql("INSERT INTO %s VALUES (2, 'new', 'extra_val')", sourceTable); + assertThrowsWithCauseContaining( + RTAS_ERROR, + () -> + spark() + .read() + .table(sourceTable) + .write() + .format("delta") + .mode("overwrite") + .option("overwriteSchema", "true") + .saveAsTable(targetTable)); + }); + }); + } + + /** + * {@code CREATE OR REPLACE TABLE} with a different schema on an existing CatalogOwned table must + * be blocked. + * + *

    Because {@code UCSingleCatalog} does not implement {@code StagingTableCatalog}, Spark routes + * {@code CREATE OR REPLACE TABLE} through REPLACE TABLE AS SELECT (RTAS), which OSS Delta does + * not support. + */ + @Test + public void testReplaceTableWithNewSchemaIsBlocked() throws Exception { + withNewTable( + "block_replace_schema_test", + "id INT, name STRING", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'initial')", tableName); + assertThrowsWithCauseContaining( + RTAS_ERROR, + () -> + sql( + "CREATE OR REPLACE TABLE %s (id INT, name STRING, extra STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", + tableName)); + }); + } + + // --------------------------------------------------------------------------- + // Positive tests: operations that must still succeed + // --------------------------------------------------------------------------- + + /** Normal INSERT with no metadata change must still succeed on CatalogOwned tables. */ + @Test + public void testNormalInsertSucceedsForManagedTable() throws Exception { + withNewTable( + "normal_insert_managed_test", + "id INT, name STRING", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'foo'), (2, 'bar')", tableName); + check(tableName, List.of(List.of("1", "foo"), List.of("2", "bar"))); + }); + } + + /** + * INSERT with {@code autoMerge=true} but no new columns must succeed -- {@code autoMerge} only + * triggers a schema update when the incoming data actually introduces extra columns. + */ + @Test + public void testInsertWithAutoMergeAndNoSchemaChangeSucceeds() throws Exception { + withNewTable( + "auto_merge_no_change_managed_test", + "id INT, name STRING", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'initial')", tableName); + sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); + try { + sql("INSERT INTO %s VALUES (2, 'second')", tableName); + check(tableName, List.of(List.of("1", "initial"), List.of("2", "second"))); + } finally { + sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); + } + }); + } + + /** + * Schema evolution via INSERT with {@code autoMerge=true} must still work on EXTERNAL (non- + * CatalogOwned) tables. The kill switch must not affect tables that are not CatalogOwned. + */ + @Test + public void testInsertWithMergeSchemaStillWorksForExternalTable() throws Exception { + withNewTable( + "merge_schema_external_target", + "id INT, name STRING", + TableType.EXTERNAL, + targetTable -> { + sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); + withNewTable( + "merge_schema_external_source", + "id INT, name STRING, extra STRING", + TableType.EXTERNAL, + sourceTable -> { + sql("INSERT INTO %s VALUES (2, 'new', 'extra_value')", sourceTable); + sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); + try { + // Should succeed: EXTERNAL tables are not CatalogOwned. + sql("INSERT INTO %s SELECT * FROM %s", targetTable, sourceTable); + // The target now has 3 columns; row 1 has null for 'extra'. + check( + targetTable, + List.of(List.of("1", "initial", "null"), List.of("2", "new", "extra_value"))); + } finally { + sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); + } + }); + }); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java index aeaff4e29cd..70a22ce669b 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java @@ -209,13 +209,36 @@ public void testWriteToCreateNewManagedTable() throws Exception { } } - // TODO: Enable testMergeSchema for UC_REMOTE=true once UC managed tables support schema - // evolution (mergeSchema). Currently, saveAsTable with mergeSchema triggers a schema change - // that requires UC to update the table metadata, which is not yet supported for managed tables. @TestAllTableTypes public void testMergeSchema(TableType tableType) throws Exception { Assumptions.assumeFalse( isUCRemoteConfigured(), "mergeSchema not yet supported for UC managed tables remotely"); + if (tableType == TableType.MANAGED) { + // mergeSchema triggers updateMetadata() with a new schema, which the kill switch blocks + // on CatalogOwned tables. Assert the failure rather than skipping. + withNewTable( + "merge_schema_blocked_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2)", tableName); + assertThrowsWithCauseContaining( + "Metadata changes on Unity Catalog", + () -> + spark() + .createDataFrame( + List.of(RowFactory.create(3, "extra")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("name", DataTypes.StringType)) + .write() + .format("delta") + .mode("append") + .option("mergeSchema", "true") + .saveAsTable(tableName)); + }); + return; + } withNewTable( "merge_schema_test", "id INT", diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java index 41ff9a9c723..0021ccf023b 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java @@ -16,10 +16,13 @@ package io.sparkuctest; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; @@ -32,11 +35,13 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; +import org.opentest4j.TestAbortedException; /** * Abstract base class for Unity Catalog + Delta Table integration tests. @@ -81,11 +86,14 @@ Stream allTableTypesTestsFactory() { () -> { try { method.invoke(this, tableType); - } catch (java.lang.reflect.InvocationTargetException e) { - // Unwrap so JUnit sees the original exception. Without this, - // TestAbortedException (thrown by Assumptions) gets wrapped and - // JUnit treats the test as failed instead of skipped. + } catch (InvocationTargetException e) { + // Unwrap so JUnit sees the original exception type. Without this, + // TestAbortedException (thrown by Assumptions) gets wrapped and JUnit + // treats the test as failed instead of skipped. Also unwrap + // RuntimeException/Error so assertThrows() in individual tests still + // matches the expected exception class rather than InvocationTargetException. Throwable cause = e.getCause(); + if (cause instanceof TestAbortedException) throw (TestAbortedException) cause; if (cause instanceof RuntimeException) throw (RuntimeException) cause; if (cause instanceof Error) throw (Error) cause; throw e; @@ -334,6 +342,31 @@ protected String currentTimestamp(String tableName) { return sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(1); } + /** + * Asserts that the given operation throws an exception whose cause chain contains {@code + * expectedMessage}. + */ + protected void assertThrowsWithCauseContaining( + String expectedMessage, ThrowingCallable operation) { + assertThatThrownBy(operation) + .satisfies( + e -> { + Throwable t = e; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains(expectedMessage)) { + return; + } + t = t.getCause(); + } + throw new AssertionError( + "Expected exception containing '" + + expectedMessage + + "' in cause chain, but none found. Top-level: " + + e, + e); + }); + } + /** Helper to build an expected row as a list of string values. */ protected static List row(String... values) { return List.of(values); From ae4cf5f7d429a53f5178809efd8a56e56693c3b8 Mon Sep 17 00:00:00 2001 From: Prakhar Jain Date: Fri, 13 Mar 2026 15:05:39 -0700 Subject: [PATCH 29/80] [SPARK] Write fileSizeHistogram (and not histogramOpt) as field name in CRC file (#6282) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Added a conf to make delta-spark write fileSizeHistogram (and not histogramOpt) as field name in CRC file. ## How was this patch tested? UTs ## Does this PR introduce _any_ user-facing changes? No --- .../org/apache/spark/sql/delta/Checksum.scala | 60 +++++++++++++- .../sql/delta/sources/DeltaSQLConf.scala | 9 +++ .../VersionChecksumHistogramCompatSuite.scala | 81 +++++++++++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala b/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala index de7d32d9a74..33a27cca4d6 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala @@ -86,6 +86,54 @@ case class VersionChecksum( @JsonAlias(Array("fileSizeHistogram")) histogramOpt: Option[FileSizeHistogram], deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram], + allFiles: Option[Seq[AddFile]]) { + + /** + * Converts to the protocol-compliant representation that serializes the histogram field as + * `fileSizeHistogram` (the Delta spec field name) instead of `histogramOpt`. + */ + def toProtocolCompliant: VersionChecksumProtocolCompliant = VersionChecksumProtocolCompliant( + txnId = txnId, + tableSizeBytes = tableSizeBytes, + numFiles = numFiles, + numDeletedRecordsOpt = numDeletedRecordsOpt, + numDeletionVectorsOpt = numDeletionVectorsOpt, + numMetadata = numMetadata, + numProtocol = numProtocol, + inCommitTimestampOpt = inCommitTimestampOpt, + setTransactions = setTransactions, + domainMetadata = domainMetadata, + metadata = metadata, + protocol = protocol, + fileSizeHistogram = histogramOpt, + deletedRecordCountsHistogramOpt = deletedRecordCountsHistogramOpt, + allFiles = allFiles + ) +} + +/** + * Protocol-compliant version of [[VersionChecksum]] that serializes the file size histogram + * using the Delta spec field name `fileSizeHistogram` instead of the legacy `histogramOpt`. + * Used only for CRC file writes when the protocol-compliant flag is enabled. + */ +case class VersionChecksumProtocolCompliant( + txnId: Option[String], + tableSizeBytes: Long, + numFiles: Long, + @JsonDeserialize(contentAs = classOf[Long]) + numDeletedRecordsOpt: Option[Long], + @JsonDeserialize(contentAs = classOf[Long]) + numDeletionVectorsOpt: Option[Long], + numMetadata: Long, + numProtocol: Long, + @JsonDeserialize(contentAs = classOf[Long]) + inCommitTimestampOpt: Option[Long], + setTransactions: Option[Seq[SetTransaction]], + domainMetadata: Option[Seq[DomainMetadata]], + metadata: Metadata, + protocol: Protocol, + fileSizeHistogram: Option[FileSizeHistogram], + deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram], allFiles: Option[Seq[AddFile]]) /** @@ -105,15 +153,23 @@ trait RecordChecksum extends DeltaLogging { return } - val version = snapshot.version val checksumWithoutTxnId = getChecksum(snapshot) val checksum = checksumWithoutTxnId.copy(txnId = Some(txnId)) + writeChecksumFile(snapshot.version, checksum) + } + + protected def writeChecksumFile(version: Long, checksum: VersionChecksum): Unit = { val eventData = mutable.Map[String, Any]("operationSucceeded" -> false) eventData("numAddFileActions") = checksum.allFiles.map(_.size).getOrElse(-1) eventData("numSetTransactionActions") = checksum.setTransactions.map(_.size).getOrElse(-1) val startTimeMs = System.currentTimeMillis() try { - val toWrite = JsonUtils.toJson(checksum) + "\n" + val toWrite = (if (spark.conf.get( + DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL)) { + JsonUtils.toJson(checksum.toProtocolCompliant) + } else { + JsonUtils.toJson(checksum) + }) + "\n" eventData("jsonSerializationTimeTakenMs") = System.currentTimeMillis() - startTimeMs eventData("checksumLength") = toWrite.length val stream = writer.createAtomic( diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index e95f0f1007b..92fb2f55f6d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -1377,6 +1377,15 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL = + buildConf("writeChecksumFile.histogramFollowsProtocol") + .internal() + .doc("""When true, writes the file size histogram to CRC files using the Delta spec field + |name "fileSizeHistogram". When false, uses the legacy Delta-Spark field name + |"histogramOpt".""".stripMargin) + .booleanConf + .createWithDefault(true) + private val FORCED_CHECKSUM_VALIDATION_INTERVAL_DEFAULT = 400 val FORCED_CHECKSUM_VALIDATION_INTERVAL = buildConf("versionChecksum.forcedValidationInterval") diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala index 69b5dc67402..e7725fa9431 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala @@ -16,6 +16,8 @@ package org.apache.spark.sql.delta +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.stats.FileSizeHistogram import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} @@ -267,4 +269,83 @@ class VersionChecksumHistogramCompatSuite "Scenario B: fileSizeHistogram (last in JSON) should win over histogramOpt") } } + + + test("writeChecksumFile writes correct field name based on conf") { + withTempDir { dir => + spark.range(10).write.format("delta").save(dir.getAbsolutePath) + val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) + val testHistogram = FileSizeHistogram( + sortedBinBoundaries = Vector(0L, 1024L, 10240L), + fileCounts = Array(5L, 10L, 15L), + totalBytes = Array(100L, 200L, 300L)) + val checksum = deltaLog.snapshot.checksumOpt.get.copy(histogramOpt = Some(testHistogram)) + + val currentSpark = spark + val currentLog = deltaLog + val writer = new RecordChecksum { + override val deltaLog: DeltaLog = currentLog + override protected def spark: org.apache.spark.sql.SparkSession = currentSpark + def writeChecksum(version: Long, cs: VersionChecksum): Unit = + writeChecksumFile(version, cs) + } + + // Write with flag OFF (default) -- should use histogramOpt + val versionOff = deltaLog.snapshot.version + 1 + withSQLConf(DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL.key -> "false") { + writer.writeChecksum(versionOff, checksum) + } + val crcJsonOff = + deltaLog.store.read(FileNames.checksumFile(deltaLog.logPath, versionOff)).head + assert(crcJsonOff.contains("\"histogramOpt\":"), + "Flag OFF: CRC should contain histogramOpt") + assert(!crcJsonOff.contains("\"fileSizeHistogram\":"), + "Flag OFF: CRC should not contain fileSizeHistogram") + + // Write with flag ON -- should use fileSizeHistogram + val versionOn = versionOff + 1 + withSQLConf(DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL.key -> "true") { + writer.writeChecksum(versionOn, checksum) + } + val crcJsonOn = + deltaLog.store.read(FileNames.checksumFile(deltaLog.logPath, versionOn)).head + assert(crcJsonOn.contains("\"fileSizeHistogram\":"), + "Flag ON: CRC should contain fileSizeHistogram") + assert(!crcJsonOn.contains("\"histogramOpt\":"), + "Flag ON: CRC should not contain histogramOpt") + + // Both CRCs should be readable and produce the same histogram + val checksumOff = JsonUtils.mapper.readValue[VersionChecksum](crcJsonOff) + val checksumOn = JsonUtils.mapper.readValue[VersionChecksum](crcJsonOn) + assert(checksumOff.histogramOpt.get === testHistogram, + "Flag OFF: read-back histogram should match the test histogram") + assert(checksumOn.histogramOpt.get === testHistogram, + "Flag ON: read-back histogram should match the test histogram") + } + } + + test("VersionChecksumProtocolCompliant fields match VersionChecksum") { + // Use reflection to ensure the two classes stay in sync. If someone adds a field to + // VersionChecksum but forgets VersionChecksumProtocolCompliant, this test will catch it. + val checksumFields = classOf[VersionChecksum].getDeclaredFields + .map(f => (f.getName, f.getType)).toSet + val protocolCompliantFields = classOf[VersionChecksumProtocolCompliant].getDeclaredFields + .map(f => (f.getName, f.getType)).toSet + + // The only difference should be histogramOpt vs fileSizeHistogram (same type) + val expectedOnlyInChecksum = Set(("histogramOpt", classOf[Option[_]])) + val expectedOnlyInProtocolCompliant = Set(("fileSizeHistogram", classOf[Option[_]])) + + val onlyInChecksum = checksumFields -- protocolCompliantFields + val onlyInProtocolCompliant = protocolCompliantFields -- checksumFields + + assert(onlyInChecksum === expectedOnlyInChecksum, + s"Unexpected fields only in VersionChecksum: $onlyInChecksum. " + + "Did you add a new field to VersionChecksum without updating " + + "VersionChecksumProtocolCompliant?") + assert(onlyInProtocolCompliant === expectedOnlyInProtocolCompliant, + s"Unexpected fields only in VersionChecksumProtocolCompliant: $onlyInProtocolCompliant. " + + "Did you add a new field to VersionChecksumProtocolCompliant without updating " + + "VersionChecksum?") + } } From 17d9734a704f2693df8f37dc5bee3d1b437e0219 Mon Sep 17 00:00:00 2001 From: Wei Luo <143362963+weiluo-db@users.noreply.github.com> Date: Fri, 13 Mar 2026 16:23:47 -0700 Subject: [PATCH 30/80] [Kernel] Add Delta-Spark compatibility for CRCInfo.fileSizeHistogram (#6281) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [x] Kernel - [ ] Other (fill in here) ## Description This PR updates Kernel's `CRCInfo` to **read** both `fileSizeHistogram` and `histogramOpt` fields to ensure backward compatibility with Delta-Spark. While the [Delta spec](https://github.com/delta-io/delta/blob/master/PROTOCOL.md#version-checksum-file-schema) officially uses `fileSizeHistogram` in the Version Checksum file, Delta-Spark has historically used histogramOpt ([code](https://github.com/delta-io/delta/blob/5ba861fa6d6999d7079706a307a7e293b59d4880/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala#L87)). Following [a recent fix](https://github.com/delta-io/delta/commit/021755fdb381be6b8a77794064c5cdd075a63bc4) that allows Delta-Spark to **read** both fields, this PR mirrors that compatibility in Kernel. This ensures smooth read operations as we fully transition Delta-Spark to write the correct `fileSizeHistogram` field. ## How was this patch tested? New unit tests. ## Does this PR introduce _any_ user-facing changes? No --- .../kernel/internal/checksum/CRCInfo.java | 12 + .../internal/checksum/ChecksumReader.java | 2 +- .../checksum/CRCInfoReadCompatSuite.scala | 219 ++++++++++++++++++ 3 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/CRCInfoReadCompatSuite.scala diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java index fbb69fcf160..a4ee453a69a 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java @@ -51,6 +51,7 @@ public class CRCInfo { private static final String TXN_ID = "txnId"; private static final String DOMAIN_METADATA = "domainMetadata"; private static final String FILE_SIZE_HISTOGRAM = "fileSizeHistogram"; + private static final String HISTOGRAM_OPT = "histogramOpt"; public static final StructType CRC_FILE_SCHEMA = new StructType() @@ -64,6 +65,10 @@ public class CRCInfo { .add(DOMAIN_METADATA, new ArrayType(DomainMetadata.FULL_SCHEMA, false), /*nullable*/ true) .add(FILE_SIZE_HISTOGRAM, FileSizeHistogram.FULL_SCHEMA, /*nullable*/ true); + // Used by ChecksumReader to support reading CRC files with the legacy "histogramOpt" field. + public static final StructType CRC_FILE_READ_SCHEMA = + CRC_FILE_SCHEMA.add(HISTOGRAM_OPT, FileSizeHistogram.FULL_SCHEMA, /*nullable*/ true); + public static Optional fromColumnarBatch( long version, ColumnarBatch batch, int rowId, String crcFilePath) { // Read required fields. @@ -89,6 +94,13 @@ public static Optional fromColumnarBatch( Optional fileSizeHistogram = FileSizeHistogram.fromColumnVector( batch.getColumnVector(getSchemaIndex(FILE_SIZE_HISTOGRAM)), rowId); + if (!fileSizeHistogram.isPresent()) { + int histogramOptIdx = batch.getSchema().indexOf(HISTOGRAM_OPT); + if (histogramOptIdx >= 0) { + fileSizeHistogram = + FileSizeHistogram.fromColumnVector(batch.getColumnVector(histogramOptIdx), rowId); + } + } ColumnVector domainMetadataVector = batch.getColumnVector(getSchemaIndex(DOMAIN_METADATA)); Optional> domainMetadata = domainMetadataVector.isNullAt(rowId) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java index 4a22a831f15..86faf00bc00 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java @@ -45,7 +45,7 @@ public static Optional tryReadChecksumFile(Engine engine, FileStatus ch .getJsonHandler() .readJsonFiles( singletonCloseableIterator(checkSumFile), - CRCInfo.CRC_FILE_SCHEMA, + CRCInfo.CRC_FILE_READ_SCHEMA, Optional.empty())) { // We do this instead of iterating through the rows or using `getSingularRow` so we // can use the existing fromColumnVector methods in Protocol, Metadata, Format etc diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/CRCInfoReadCompatSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/CRCInfoReadCompatSuite.scala new file mode 100644 index 00000000000..f07c6be77b7 --- /dev/null +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/CRCInfoReadCompatSuite.scala @@ -0,0 +1,219 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal.checksum + +import java.util +import java.util.{Collections, Optional} + +import scala.collection.JavaConverters._ + +import io.delta.kernel.data.{ColumnarBatch, ColumnVector, Row} +import io.delta.kernel.internal.actions.{DomainMetadata, Format, Metadata, Protocol} +import io.delta.kernel.internal.checksum.CRCInfo.{CRC_FILE_READ_SCHEMA, CRC_FILE_SCHEMA} +import io.delta.kernel.internal.data.GenericColumnVector +import io.delta.kernel.internal.stats.FileSizeHistogram +import io.delta.kernel.internal.types.DataTypeJsonSerDe +import io.delta.kernel.internal.util.VectorUtils +import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, stringStringMapValue} +import io.delta.kernel.test.VectorTestUtils +import io.delta.kernel.types.{DataType, StringType, StructType} + +import org.scalatest.funsuite.AnyFunSuite + +/** + * Tests that CRCInfo.fromColumnarBatch correctly reads the file size histogram from CRC files + * written using the legacy field name "histogramOpt" or the spec-compliant "fileSizeHistogram". + */ +class CRCInfoReadCompatSuite extends AnyFunSuite with VectorTestUtils { + + private val testProtocol = + new Protocol(1, 2, Collections.emptySet(), Collections.emptySet()) + + private val testMetadata = new Metadata( + "id", + Optional.of("name"), + Optional.of("description"), + new Format("parquet", Collections.emptyMap()), + DataTypeJsonSerDe.serializeDataType(new StructType()), + new StructType(), + buildArrayValue(util.Arrays.asList("c3"), StringType.STRING), + Optional.of(123), + stringStringMapValue(new util.HashMap[String, String]() { + put("delta.appendOnly", "true") + })) + + /** Creates a simple histogram with distinct values for identification in tests. */ + private def createTestHistogram(fileCount: Long): FileSizeHistogram = { + val boundaries = Array(0L, 1024L) + val counts = Array(fileCount, 0L) + val bytes = Array(fileCount * 100, 0L) + new FileSizeHistogram(boundaries, counts, bytes) + } + + /** + * Builds a ColumnVector for a FileSizeHistogram struct field. If histogram is None, the vector + * is null at row 0. + */ + private def histogramColumnVector( + histogram: Option[FileSizeHistogram]): ColumnVector = { + val rowValue: Row = histogram.map(_.toRow()).orNull + new GenericColumnVector( + util.Arrays.asList(rowValue), + FileSizeHistogram.FULL_SCHEMA) + } + + /** + * Build a ColumnarBatch with the given schema and histogram column vectors at the appropriate + * positions. + */ + private def buildBatch( + schema: StructType, + fileSizeHistogram: Option[FileSizeHistogram], + histogramOpt: Option[FileSizeHistogram]): ColumnarBatch = { + val protocolColVector = + new GenericColumnVector( + util.Arrays.asList(testProtocol.toRow()), + Protocol.FULL_SCHEMA) + val metadataColVector = + new GenericColumnVector( + util.Arrays.asList(testMetadata.toRow()), + Metadata.FULL_SCHEMA) + + new ColumnarBatch { + override def getSchema: StructType = schema + override def getSize: Int = 1 + + override def getColumnVector(ordinal: Int): ColumnVector = { + val fieldName = schema.at(ordinal).getName + fieldName match { + case "tableSizeBytes" => longVector(Seq(1000L)) + case "numFiles" => longVector(Seq(10L)) + case "numMetadata" => longVector(Seq(1L)) + case "numProtocol" => longVector(Seq(1L)) + case "metadata" => metadataColVector + case "protocol" => protocolColVector + case "txnId" => stringVector(Seq(null)) + case "domainMetadata" => nullColumnVector( + CRC_FILE_SCHEMA.get("domainMetadata").getDataType) + case "fileSizeHistogram" => histogramColumnVector(fileSizeHistogram) + case "histogramOpt" => histogramColumnVector(histogramOpt) + case _ => + throw new IllegalArgumentException(s"Unknown field: $fieldName") + } + } + } + } + + /** Creates a column vector that is null at every row. */ + private def nullColumnVector(dataType: DataType): ColumnVector = { + new ColumnVector { + override def getDataType: DataType = dataType + override def getSize: Int = 1 + override def close(): Unit = {} + override def isNullAt(rowId: Int): Boolean = true + } + } + + test("reads fileSizeHistogram when only spec-compliant field is present") { + val histogram = createTestHistogram(fileCount = 42) + val batch = buildBatch( + CRC_FILE_READ_SCHEMA, + fileSizeHistogram = Some(histogram), + histogramOpt = None) + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(crcInfo.get().getFileSizeHistogram.isPresent) + assert(crcInfo.get().getFileSizeHistogram.get() === histogram) + } + + test("reads histogramOpt when only legacy field is present") { + val histogram = createTestHistogram(fileCount = 99) + val batch = buildBatch( + CRC_FILE_READ_SCHEMA, + fileSizeHistogram = None, + histogramOpt = Some(histogram)) + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(crcInfo.get().getFileSizeHistogram.isPresent) + assert(crcInfo.get().getFileSizeHistogram.get() === histogram) + } + + test("prefers fileSizeHistogram when both fields are present") { + val specHistogram = createTestHistogram(fileCount = 10) + val legacyHistogram = createTestHistogram(fileCount = 20) + val batch = buildBatch( + CRC_FILE_READ_SCHEMA, + fileSizeHistogram = Some(specHistogram), + histogramOpt = Some(legacyHistogram)) + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(crcInfo.get().getFileSizeHistogram.isPresent) + assert(crcInfo.get().getFileSizeHistogram.get() === specHistogram) + } + + test("returns empty histogram when neither field is present") { + val batch = buildBatch( + CRC_FILE_READ_SCHEMA, + fileSizeHistogram = None, + histogramOpt = None) + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(!crcInfo.get().getFileSizeHistogram.isPresent) + } + + test("safely skips fallback when batch uses original CRC_FILE_SCHEMA") { + // When fromColumnarBatch is called with a batch using the original schema + // (without histogramOpt), the fallback should be safely skipped. + val protocolColVector = + new GenericColumnVector( + util.Arrays.asList(testProtocol.toRow()), + Protocol.FULL_SCHEMA) + val metadataColVector = + new GenericColumnVector( + util.Arrays.asList(testMetadata.toRow()), + Metadata.FULL_SCHEMA) + + val batch = new ColumnarBatch { + override def getSchema: StructType = CRC_FILE_SCHEMA + override def getSize: Int = 1 + override def getColumnVector(ordinal: Int): ColumnVector = { + val fieldName = CRC_FILE_SCHEMA.at(ordinal).getName + fieldName match { + case "tableSizeBytes" => longVector(Seq(1000L)) + case "numFiles" => longVector(Seq(10L)) + case "numMetadata" => longVector(Seq(1L)) + case "numProtocol" => longVector(Seq(1L)) + case "metadata" => metadataColVector + case "protocol" => protocolColVector + case "txnId" => stringVector(Seq(null)) + case "domainMetadata" => nullColumnVector( + CRC_FILE_SCHEMA.get("domainMetadata").getDataType) + case "fileSizeHistogram" => histogramColumnVector(None) + case _ => + throw new IllegalArgumentException(s"Unknown field: $fieldName") + } + } + } + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(!crcInfo.get().getFileSizeHistogram.isPresent) + } +} From 43f34cc01f87da67bd8087a00fd3440fcd887a16 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Mon, 16 Mar 2026 03:36:59 +0530 Subject: [PATCH 31/80] Add HTTP retry logic for transient failures in SSP client (#6274) ## Problem `IcebergRESTCatalogPlanningClient` has no retry logic for transient HTTP failures (acknowledged by a TODO in the code). Temporary network hiccups or 5xx server errors cause immediate query failure. ## Fix Configure Apache HTTP client with retry handler for transient failures (IOException, 5xx). 3 retries with backoff. 4xx errors (client errors) are not retried. ## Validation - Confirmed by reading the HTTP client construction and the TODO at line 372. - New tests verify retry behavior on 503 and no-retry on 404. ## Regression Prevention New test cases verify retry logic for transient vs permanent failures. --------- Co-authored-by: foo.bar --- .../IcebergRESTCatalogPlanningClient.scala | 47 ++++++++++- ...bergRESTCatalogAdapterWithPlanSupport.java | 57 +++++++++++++ .../iceberg/rest/IcebergRESTServer.java | 20 +++++ .../IcebergRESTServletWithPlanSupport.java | 17 ++++ ...cebergRESTCatalogPlanningClientSuite.scala | 80 +++++++++++++++++++ 5 files changed, 218 insertions(+), 3 deletions(-) diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala index 474c090a3f2..adad47eee9f 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala @@ -27,8 +27,10 @@ import org.apache.hadoop.conf.Configuration import org.apache.http.client.methods.{HttpGet, HttpPost} import org.apache.http.entity.{ContentType, StringEntity} import org.apache.http.util.EntityUtils -import org.apache.http.{HttpHeaders, HttpStatus} -import org.apache.http.impl.client.HttpClientBuilder +import org.apache.http.{HttpHeaders, HttpResponse, HttpStatus} +import org.apache.http.client.ServiceUnavailableRetryStrategy +import org.apache.http.impl.client.{DefaultHttpRequestRetryHandler, HttpClientBuilder} +import org.apache.http.protocol.HttpContext import org.apache.http.message.BasicHeader import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.sources.Filter @@ -309,9 +311,16 @@ class IcebergRESTCatalogPlanningClient( scala.util.Properties.versionNumberString } + // Maximum number of retries for transient HTTP failures (IOException, 5xx server errors) + private val HTTP_MAX_RETRIES = 3 + private lazy val httpClient = HttpClientBuilder.create() .setDefaultHeaders(httpHeaders) .setConnectionTimeToLive(30, java.util.concurrent.TimeUnit.SECONDS) + // requestSentRetryEnabled=true: safe to retry already-sent requests because + // planScan is a read-only operation (idempotent POST to /plan endpoint) + .setRetryHandler(new DefaultHttpRequestRetryHandler(HTTP_MAX_RETRIES, true)) + .setServiceUnavailableRetryStrategy(new ServerErrorRetryStrategy(HTTP_MAX_RETRIES)) .build() override def canConvertFilters(filters: Array[Filter]): Boolean = { @@ -369,7 +378,6 @@ class IcebergRESTCatalogPlanningClient( } val httpPost = new HttpPost(planTableScanUri) httpPost.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) - // TODO: Add retry logic for transient HTTP failures (e.g., connection timeouts, 5xx errors) val httpResponse = httpClient.execute(httpPost) // Only unpartitioned tables are supported. This map is used when parsing the response @@ -503,6 +511,39 @@ class IcebergRESTCatalogPlanningClient( } } + /** + * Retry strategy for server errors (5xx status codes) with exponential backoff. + * Retries up to maxRetries times with doubling intervals (1s, 2s, 4s, ...). + * Does NOT retry on client errors (4xx) since those indicate request-level issues. + * + * The ServiceUnavailableRetryStrategy interface calls retryRequest() first, then + * getRetryInterval(), so we capture the execution count in retryRequest() and + * use it to compute the backoff in getRetryInterval(). + */ + private class ServerErrorRetryStrategy(maxRetries: Int) + extends ServiceUnavailableRetryStrategy { + + // ThreadLocal so concurrent planScan calls each track their own retry attempt. + // The HTTP client is shared and thread-safe (see class doc), so multiple threads + // can be retrying independently through the same strategy instance. + private val lastExecutionCount = new ThreadLocal[Int] { + override def initialValue(): Int = 1 + } + + override def retryRequest( + response: HttpResponse, + executionCount: Int, + context: HttpContext): Boolean = { + lastExecutionCount.set(executionCount) + val statusCode = response.getStatusLine.getStatusCode + statusCode >= 500 && executionCount <= maxRetries + } + + // Exponential backoff: 1s, 2s, 4s, ... + override def getRetryInterval: Long = + java.util.concurrent.TimeUnit.SECONDS.toMillis(1L << (lastExecutionCount.get() - 1)) + } + private def parsePlanTableScanResponse( json: String, specsById: Map[Int, PartitionSpec], diff --git a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java index a94dca8cc43..04c4e94d674 100644 --- a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java +++ b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import shadedForDelta.org.apache.iceberg.BaseFileScanTask; @@ -81,6 +82,14 @@ class IcebergRESTCatalogAdapterWithPlanSupport extends RESTCatalogAdapter { // Volatile is used to guarantee correct cross-thread access (test thread and Jetty server thread). private static volatile String capturedPlanRequestPath = null; + // Failure injection fields for testing HTTP retry logic. + // planRequestFailCount: number of remaining /plan requests to fail before allowing success. + // planRequestFailStatusCode: HTTP status code to return for injected failures. + // planRequestCount: total number of /plan requests received (for verifying retry behavior). + private static final AtomicInteger planRequestFailCount = new AtomicInteger(0); + private static volatile int planRequestFailStatusCode = 503; + private static final AtomicInteger planRequestCount = new AtomicInteger(0); + IcebergRESTCatalogAdapterWithPlanSupport(Catalog catalog) { super(catalog); this.catalog = catalog; @@ -189,6 +198,54 @@ static void clearCaptured() { testCredentials = null; testResidual = null; capturedPlanRequestPath = null; + planRequestFailCount.set(0); + planRequestCount.set(0); + } + + /** + * Configure the server to fail the next N /plan requests with the specified HTTP status code. + * After N failures, subsequent requests proceed normally. + * Used for testing HTTP retry logic in the client. + * + * @param count Number of /plan requests to fail + * @param statusCode HTTP status code to return for injected failures (e.g., 503, 404) + */ + static void setFailNextPlanRequests(int count, int statusCode) { + planRequestFailCount.set(count); + planRequestFailStatusCode = statusCode; + } + + /** + * Atomically get and decrement the remaining failure count. + * Returns the value before decrement. If > 0, the request should be failed. + * Package-private for servlet access. + */ + static int getAndDecrementFailCount() { + return planRequestFailCount.getAndDecrement(); + } + + /** + * Get the HTTP status code to use for injected failures. + * Package-private for servlet access. + */ + static int getPlanRequestFailStatusCode() { + return planRequestFailStatusCode; + } + + /** + * Increment and return the total /plan request count. + * Package-private for servlet access. + */ + static void incrementPlanRequestCount() { + planRequestCount.incrementAndGet(); + } + + /** + * Get the total number of /plan requests received. + * Package-private for test access. + */ + static int getPlanRequestCount() { + return planRequestCount.get(); } @Override diff --git a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java index 5bbe5b2b95c..378475676b0 100644 --- a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java +++ b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java @@ -215,6 +215,26 @@ public void setTestResidual(shadedForDelta.org.apache.iceberg.expressions.Expres IcebergRESTCatalogAdapterWithPlanSupport.setTestResidual(residual); } + /** + * Configure the server to fail the next N /plan requests with the specified HTTP status code. + * After N failures, subsequent requests proceed normally. + * Used for testing HTTP retry logic in the client. + * + * @param count Number of /plan requests to fail + * @param statusCode HTTP status code to return for injected failures (e.g., 503, 404) + */ + public void setFailNextPlanRequests(int count, int statusCode) { + IcebergRESTCatalogAdapterWithPlanSupport.setFailNextPlanRequests(count, statusCode); + } + + /** + * Get the total number of /plan requests received since last clearCaptured(). + * Used for verifying retry behavior in tests. + */ + public int getPlanRequestCount() { + return IcebergRESTCatalogAdapterWithPlanSupport.getPlanRequestCount(); + } + /** * Clear captured filter and projection. Call between tests. */ diff --git a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java index 6f59cc35895..6617850401f 100644 --- a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java +++ b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java @@ -121,6 +121,23 @@ private void handleConfigRequest(HttpServletRequest req, HttpServletResponse res private void handlePlanRequest(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Track plan request count for testing retry behavior + IcebergRESTCatalogAdapterWithPlanSupport.incrementPlanRequestCount(); + + // Check if we should inject a failure for testing HTTP retry logic + int remainingFailures = IcebergRESTCatalogAdapterWithPlanSupport.getAndDecrementFailCount(); + if (remainingFailures > 0) { + int failStatusCode = IcebergRESTCatalogAdapterWithPlanSupport.getPlanRequestFailStatusCode(); + LOG.info("Injecting test failure: returning HTTP {} ({} failures remaining)", + failStatusCode, remainingFailures - 1); + resp.setStatus(failStatusCode); + resp.setContentType("application/json"); + resp.getWriter().write( + "{\"error\": {\"message\": \"Injected test failure\", \"type\": \"TestError\", \"code\": " + + failStatusCode + "}}"); + return; + } + try { // Extract request components String path = req.getPathInfo(); diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala index 2243abb843c..841ffd6333f 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala @@ -523,6 +523,86 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe IcebergRESTServerTestUtils.populateTestData(spark, tableName) } + test("retry on transient 503 server error") { + withTempTable("retryTest503") { table => + populateTestData(s"rest_catalog.${defaultNamespace}.retryTest503") + + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + try { + server.clearCaptured() + // Configure server to fail the first plan request with 503 + server.setFailNextPlanRequests(1, 503) + + // Client should retry and succeed on the second attempt + val scanPlan = client.planScan(defaultNamespace.toString, "retryTest503") + assert(scanPlan != null, "Scan plan should not be null after retry") + assert(scanPlan.files.nonEmpty, "Scan plan should have files after successful retry") + + // Verify 2 requests were made: 1 failed (503) + 1 success + assert(server.getPlanRequestCount() == 2, + s"Expected 2 plan requests (1 retry), got ${server.getPlanRequestCount()}") + } finally { + server.clearCaptured() + client.close() + } + } + } + + test("retries exhausted on persistent 503 server error") { + // No populateTestData needed: failure injection intercepts at the servlet level before + // table data is accessed, so we only need the table to exist for a valid URI. + withTempTable("retryTestExhausted") { table => + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + try { + server.clearCaptured() + // Configure server to fail more requests than the client will retry (max 3 retries = 4 + // total attempts). Setting 10 failures ensures all retries see 503. + server.setFailNextPlanRequests(10, 503) + + val exception = intercept[java.io.IOException] { + client.planScan(defaultNamespace.toString, "retryTestExhausted") + } + assert(exception.getMessage.contains("503"), + s"Error should mention 503 status code. Got: ${exception.getMessage}") + + // Verify 4 requests were made: 1 original + 3 retries (max retries = 3) + assert(server.getPlanRequestCount() == 4, + s"Expected 4 plan requests (1 + 3 retries), got ${server.getPlanRequestCount()}") + } finally { + server.clearCaptured() + client.close() + } + } + } + + test("no retry on 404 client error") { + // No populateTestData needed: failure injection intercepts at the servlet level before + // table data is accessed, so we only need the table to exist for a valid URI. + withTempTable("retryTest404") { table => + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + try { + server.clearCaptured() + // Configure server to fail all plan requests with 404 + // Using a high count ensures the test fails if the client retries + server.setFailNextPlanRequests(10, 404) + + // Client should NOT retry 404 and should throw immediately + val exception = intercept[java.io.IOException] { + client.planScan(defaultNamespace.toString, "retryTest404") + } + assert(exception.getMessage.contains("404"), + s"Error should mention 404 status code. Got: ${exception.getMessage}") + + // Verify only 1 request was made (no retry for 404) + assert(server.getPlanRequestCount() == 1, + s"Expected 1 plan request (no retry for 404), got ${server.getPlanRequestCount()}") + } finally { + server.clearCaptured() + client.close() + } + } + } + test("User-Agent header format") { val client = new IcebergRESTCatalogPlanningClient("http://localhost:8080", "test_catalog", "") try { From 33abe39fe1df0ee7410310b4d68e56fbb170b121 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Mon, 16 Mar 2026 03:46:31 +0530 Subject: [PATCH 32/80] Fix reference equality on String in DeletionVectorDescriptor.isInline() (#6265) ## Problem `isInline()` uses `==` (reference equality) instead of `.equals()` (value equality) to compare `storageType` against `INLINE_DV_MARKER`. The same file correctly uses `.equals()` in three other places (lines 194, 222, 230). Since `storageType` comes from `getString()` which returns dynamically constructed strings, `==` may fail. ## Fix Change `storageType == INLINE_DV_MARKER` to `INLINE_DV_MARKER.equals(storageType)` (null-safe pattern). ## Validation - Confirmed inconsistency by reading lines 194, 207, 222, 230 of DeletionVectorDescriptor.java. - New test verifies `isInline()` works with non-interned strings. ## Regression Prevention New test case explicitly constructs a DeletionVectorDescriptor with `new String("i")` and asserts `isInline()` returns true. --------- Co-authored-by: foo.bar --- .../actions/DeletionVectorDescriptor.java | 2 +- .../actions/DeletionVectorDescriptorSuite.scala | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java index 30d457a9f02..5691ec08bf9 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java @@ -204,7 +204,7 @@ public String serializeToBase64() { } public boolean isInline() { - return storageType == INLINE_DV_MARKER; + return INLINE_DV_MARKER.equals(storageType); } public boolean isOnDisk() { diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala index aa7aabe256d..9fb3623c465 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala @@ -21,7 +21,7 @@ import java.util.{Base64, Optional} import org.scalatest.funsuite.AnyFunSuite /** - * Tests for DeletionVectorDescriptor.serializeToBase64(). + * Tests for DeletionVectorDescriptor. */ class DeletionVectorDescriptorSuite extends AnyFunSuite { @@ -59,6 +59,21 @@ class DeletionVectorDescriptorSuite extends AnyFunSuite { } } + // Regression test: isInline() must use .equals() not == for String comparison. + // Using `new String(...)` creates non-interned Strings that would fail with ==. + testCases.foreach { case (storageType, pathOrInlineDv, offset, sizeInBytes, cardinality) => + test(s"isInline with non-interned string - $storageType storage type") { + val dv = new DeletionVectorDescriptor( + new String(storageType), // deliberately non-interned + pathOrInlineDv, + offset.map(Integer.valueOf).map(Optional.of[Integer]).getOrElse(Optional.empty[Integer]()), + sizeInBytes, + cardinality) + + assert(dv.isInline() === (storageType == "i")) + } + } + test("serializeToBase64 throws for non-inline DV without offset") { val ex = intercept[IllegalArgumentException] { val dv = new DeletionVectorDescriptor( From dbedd84ff7e31a5640a9d2666c5e032ddfbc8ef7 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Sun, 15 Mar 2026 20:46:22 -0700 Subject: [PATCH 33/80] Fix decimal type mismatch in V2 connector filter pushdown (#6285) ## Summary - When the V2 connector pushes down filters with decimal literals to the Kernel, the literal's intrinsic decimal type (e.g. `Decimal(5,2)` for `100.00`) can differ from the column's declared type (e.g. `Decimal(7,2)`). The Kernel's expression evaluator requires exact type match and throws `KernelEngineException` during data skipping. - Fix by widening decimal literals in `ExpressionUtils` to match the column's declared type from the table schema before converting to Kernel predicates. Column name lookup is case-insensitive. - If the literal cannot fit in the column's type (higher scale or exceeds precision), the filter is skipped for pushdown rather than causing an error. ## Test plan - [x] Added 14 unit tests in `ExpressionUtilsTest` covering: literal widening, scale widening, higher-scale rejection, precision overflow rejection, matching types, no-schema fallback, non-decimal columns, case-insensitive lookup, unknown columns, compound filters (AND/OR/NOT), EqualNullSafe, and all comparison operators - [x] All 62 tests pass in `ExpressionUtilsTest` --------- Co-authored-by: Claude Opus 4.6 --- .../spark/internal/v2/catalog/SparkTable.java | 5 + .../internal/v2/read/SparkScanBuilder.java | 6 +- .../internal/v2/utils/ExpressionUtils.java | 144 ++++++++++-- .../v2/read/SparkScanBuilderTest.java | 24 ++ .../v2/utils/ExpressionUtilsTest.java | 210 ++++++++++++++++++ 5 files changed, 373 insertions(+), 16 deletions(-) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java index 4ad34c54e29..75a1be09dd7 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java @@ -254,6 +254,7 @@ public ScanBuilder newScanBuilder(CaseInsensitiveStringMap scanOptions) { snapshotManager, schemaProvider.getDataSchema(), schemaProvider.getPartitionSchema(), + schemaProvider.getRawSchema(), catalogStats, merged); } @@ -409,6 +410,10 @@ StructType getPartitionSchema() { return withInit(() -> partitionSchema); } + StructType getRawSchema() { + return withInit(() -> rawSchema); + } + Column[] getColumns() { return withInit(() -> columns); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java index e470c579457..dcafbd3fecd 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java @@ -45,6 +45,7 @@ public class SparkScanBuilder private final DeltaSnapshotManager snapshotManager; private final StructType dataSchema; private final StructType partitionSchema; + private final StructType tableSchema; private final Optional catalogStats; private final CaseInsensitiveStringMap options; private final Set partitionColumnSet; @@ -65,6 +66,7 @@ public class SparkScanBuilder * @param snapshotManager the snapshot manager for this table * @param dataSchema the data schema (non-partition columns) * @param partitionSchema the partition schema + * @param tableSchema the full table schema (all columns) for filter type alignment * @param catalogStats optional V2 Statistics converted from catalog stats * @param options scan options */ @@ -74,6 +76,7 @@ public SparkScanBuilder( DeltaSnapshotManager snapshotManager, StructType dataSchema, StructType partitionSchema, + StructType tableSchema, Optional catalogStats, CaseInsensitiveStringMap options) { this.initialSnapshot = requireNonNull(initialSnapshot, "initialSnapshot is null"); @@ -81,6 +84,7 @@ public SparkScanBuilder( this.snapshotManager = requireNonNull(snapshotManager, "snapshotManager is null"); this.dataSchema = requireNonNull(dataSchema, "dataSchema is null"); this.partitionSchema = requireNonNull(partitionSchema, "partitionSchema is null"); + this.tableSchema = requireNonNull(tableSchema, "tableSchema is null"); this.catalogStats = requireNonNull(catalogStats, "catalogStats is null"); this.options = requireNonNull(options, "options is null"); this.requiredDataSchema = this.dataSchema; @@ -111,7 +115,7 @@ public Filter[] pushFilters(Filter[] filters) { for (Filter filter : filters) { ExpressionUtils.FilterClassificationResult classification = - ExpressionUtils.classifyFilter(filter, partitionColumnSet); + ExpressionUtils.classifyFilter(filter, partitionColumnSet, tableSchema); // Collect kernel predicates if supported if (classification.isKernelSupported) { convertedKernelPredicates.add(classification.kernelPredicate.get()); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java index 4753c650f51..ed1130551d3 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java @@ -24,6 +24,7 @@ import io.delta.kernel.expressions.Or; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.InternalUtils; +import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; import java.util.*; @@ -32,6 +33,8 @@ import org.apache.spark.sql.connector.expressions.LiteralValue; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.sources.*; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; @@ -65,7 +68,25 @@ public final class ExpressionUtils { * not supported, along with a boolean indicating whether the conversion was partial */ public static ConvertedPredicate convertSparkFilterToKernelPredicate(Filter filter) { - return convertSparkFilterToKernelPredicate(filter, true /*canPartialPushDown*/); + return convertSparkFilterToKernelPredicate(filter, true /*canPartialPushDown*/, null); + } + + /** + * Converts a Spark SQL filter to a Delta Kernel predicate with table schema for type alignment. + * + *

    When a table schema is provided, decimal literal types in comparison filters are widened to + * match the column's declared decimal type. This prevents type mismatch errors in the Kernel's + * expression evaluator during data skipping, where stats column types (from the table schema) + * must match the filter literal types exactly. + * + * @param filter the Spark SQL filter to convert + * @param tableSchema the table schema for looking up column types, may be null + * @return ConvertedPredicate containing the converted Kernel predicate, or empty if conversion is + * not supported, along with a boolean indicating whether the conversion was partial + */ + public static ConvertedPredicate convertSparkFilterToKernelPredicate( + Filter filter, StructType tableSchema) { + return convertSparkFilterToKernelPredicate(filter, true /*canPartialPushDown*/, tableSchema); } /** @@ -81,10 +102,23 @@ public static ConvertedPredicate convertSparkFilterToKernelPredicate(Filter filt @VisibleForTesting static ConvertedPredicate convertSparkFilterToKernelPredicate( Filter filter, boolean canPartialPushDown) { + return convertSparkFilterToKernelPredicate(filter, canPartialPushDown, null); + } + + /** + * Core implementation of filter-to-predicate conversion with partial pushdown control and + * optional table schema for decimal type alignment. + * + * @param filter the Spark SQL filter to convert + * @param canPartialPushDown whether partial pushdown is allowed for AND filters + * @param tableSchema the table schema for looking up column types, may be null + */ + private static ConvertedPredicate convertSparkFilterToKernelPredicate( + Filter filter, boolean canPartialPushDown, StructType tableSchema) { if (filter instanceof EqualTo) { EqualTo f = (EqualTo) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("=", kernelColumn(f.attribute()), l))); } if (filter instanceof EqualNullSafe) { @@ -94,31 +128,31 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( return new ConvertedPredicate( f.value() == null ? Optional.of(new Predicate("IS_NULL", kernelColumn(f.attribute()))) - : convertValueToKernelLiteral(f.value()) + : convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("=", kernelColumn(f.attribute()), l))); } if (filter instanceof GreaterThan) { GreaterThan f = (GreaterThan) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate(">", kernelColumn(f.attribute()), l))); } if (filter instanceof GreaterThanOrEqual) { GreaterThanOrEqual f = (GreaterThanOrEqual) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate(">=", kernelColumn(f.attribute()), l))); } if (filter instanceof LessThan) { LessThan f = (LessThan) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("<", kernelColumn(f.attribute()), l))); } if (filter instanceof LessThanOrEqual) { LessThanOrEqual f = (LessThanOrEqual) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("<=", kernelColumn(f.attribute()), l))); } if (filter instanceof IsNull) { @@ -139,8 +173,10 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( } if (filter instanceof org.apache.spark.sql.sources.And) { org.apache.spark.sql.sources.And f = (org.apache.spark.sql.sources.And) filter; - ConvertedPredicate left = convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown); - ConvertedPredicate right = convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown); + ConvertedPredicate left = + convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown, tableSchema); + ConvertedPredicate right = + convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown, tableSchema); boolean isPartial = left.isPartial() || right.isPartial(); if (left.isPresent() && right.isPresent()) { return new ConvertedPredicate(Optional.of(new And(left.get(), right.get())), isPartial); @@ -155,8 +191,10 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( } if (filter instanceof org.apache.spark.sql.sources.Or) { org.apache.spark.sql.sources.Or f = (org.apache.spark.sql.sources.Or) filter; - ConvertedPredicate left = convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown); - ConvertedPredicate right = convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown); + ConvertedPredicate left = + convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown, tableSchema); + ConvertedPredicate right = + convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown, tableSchema); // OR requires both operands to be convertible for correctness boolean isPartial = left.isPartial() || right.isPartial(); if (!left.isPresent() || !right.isPresent()) { @@ -185,7 +223,7 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( // NOT(age < 30) = NOT(true) = false → system excludes both row // We will return incorrect result, then. ConvertedPredicate child = - convertSparkFilterToKernelPredicate(f.child(), false /*canPartialPushDown*/); + convertSparkFilterToKernelPredicate(f.child(), false /*canPartialPushDown*/, tableSchema); return new ConvertedPredicate( child.getConvertedPredicate().map(c -> new Predicate("NOT", c)), child.isPartial()); } @@ -193,6 +231,66 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( return new ConvertedPredicate(Optional.empty()); } + /** + * Converts a filter literal value to a Kernel Literal, aligning decimal types with the column's + * declared type from the table schema when available. + * + *

    When the value is a {@link java.math.BigDecimal} and the table schema is provided, this + * method looks up the column's declared {@link org.apache.spark.sql.types.DecimalType} and widens + * the literal to match. This prevents type mismatch errors in the Kernel's expression evaluator + * during data skipping, where stats column types must match the filter literal types exactly. + * + *

    If the literal's scale exceeds the column's scale, or the widened value exceeds the column's + * precision, the filter cannot be safely pushed down and an empty Optional is returned. + * + * @param value the filter literal value + * @param attribute the column name referenced by the filter + * @param tableSchema the table schema for type lookup, may be null + * @return Optional containing the Kernel Literal with aligned type, or empty if conversion fails + */ + private static Optional convertComparisonLiteral( + Object value, String attribute, StructType tableSchema) { + if (value instanceof BigDecimal && tableSchema != null) { + BigDecimal bd = (BigDecimal) value; + Optional columnType = lookupColumnType(attribute, tableSchema); + if (columnType.isPresent() && columnType.get() instanceof DecimalType) { + DecimalType colDecimalType = (DecimalType) columnType.get(); + return widenDecimalLiteral(bd, colDecimalType.precision(), colDecimalType.scale()); + } + } + return convertValueToKernelLiteral(value); + } + + /** + * Widens a BigDecimal literal to match the target decimal precision and scale. Returns empty if + * the literal cannot be safely represented in the target type (e.g., the literal has more decimal + * digits than the target scale, or the widened value exceeds the target precision). + */ + private static Optional widenDecimalLiteral( + BigDecimal bd, int targetPrecision, int targetScale) { + if (bd.scale() <= targetScale) { + BigDecimal widened = bd.setScale(targetScale); + if (widened.precision() <= targetPrecision) { + return Optional.of(Literal.ofDecimal(widened, targetPrecision, targetScale)); + } + } + // Literal doesn't fit in column type or has higher scale - skip pushdown + return Optional.empty(); + } + + /** + * Looks up the data type of a top-level column in the table schema using case-insensitive name + * matching. Returns empty for nested columns or if the column is not found. + */ + private static Optional lookupColumnType(String attribute, StructType tableSchema) { + for (StructField field : tableSchema.fields()) { + if (field.name().equalsIgnoreCase(attribute)) { + return Optional.of(field.dataType()); + } + } + return Optional.empty(); + } + /** * Creates a Delta Kernel Column from a Spark SQL column attribute name. * @@ -274,9 +372,9 @@ static Optional convertValueToKernelLiteral(Object value) { Double d = (Double) value; return Optional.of(Literal.ofDouble(d)); } - if (value instanceof java.math.BigDecimal) { + if (value instanceof BigDecimal) { // Preserve precision and scale from the original BigDecimal - java.math.BigDecimal bd = (java.math.BigDecimal) value; + BigDecimal bd = (BigDecimal) value; return Optional.of(Literal.ofDecimal(bd, bd.precision(), bd.scale())); } if (value instanceof UTF8String) { @@ -380,9 +478,25 @@ public FilterClassificationResult( */ public static FilterClassificationResult classifyFilter( Filter filter, Set partitionColumnSet) { + return classifyFilter(filter, partitionColumnSet, null); + } + + /** + * Classifies a Spark Filter with table schema for decimal type alignment. + * + * @param filter the Spark Filter to classify + * @param partitionColumnSet a set of partition column names (in lower case) for identifying data + * filters + * @param tableSchema the table schema for aligning decimal literal types, may be null + * @return FilterClassificationResult containing classification details + */ + public static FilterClassificationResult classifyFilter( + Filter filter, Set partitionColumnSet, StructType tableSchema) { // try to convert Spark filter to Kernel Predicate ConvertedPredicate convertedPredicate = - ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + tableSchema != null + ? ExpressionUtils.convertSparkFilterToKernelPredicate(filter, tableSchema) + : ExpressionUtils.convertSparkFilterToKernelPredicate(filter); boolean isKernelSupported = convertedPredicate.isPresent(); boolean isPartialConversion = convertedPredicate.isPartial(); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java index 01f06aad9b0..56d00e05cf5 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java @@ -62,6 +62,13 @@ public void testBuild_returnsScanWithExpectedSchema(@TempDir File tempDir) { StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + StructType tableSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("name", DataTypes.StringType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + }); SparkScanBuilder builder = new SparkScanBuilder( tableName, @@ -69,6 +76,7 @@ public void testBuild_returnsScanWithExpectedSchema(@TempDir File tempDir) { snapshotManager, dataSchema, partitionSchema, + tableSchema, Optional.empty(), CaseInsensitiveStringMap.empty()); @@ -106,6 +114,13 @@ public void testToMicroBatchStream_returnsSparkMicroBatchStream(@TempDir File te StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + StructType tableSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("name", DataTypes.StringType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + }); SparkScanBuilder builder = new SparkScanBuilder( tableName, @@ -113,6 +128,7 @@ public void testToMicroBatchStream_returnsSparkMicroBatchStream(@TempDir File te snapshotManager, dataSchema, partitionSchema, + tableSchema, Optional.empty(), CaseInsensitiveStringMap.empty()); Scan scan = builder.build(); @@ -741,12 +757,20 @@ private SparkScanBuilder createTestScanBuilder(File tempDir) { StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + StructType tableSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("name", DataTypes.StringType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + }); return new SparkScanBuilder( tableName, snapshot, snapshotManager, dataSchema, partitionSchema, + tableSchema, Optional.empty(), CaseInsensitiveStringMap.empty()); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java index 8d69fb6f360..4dc822c07f7 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java @@ -629,6 +629,216 @@ public void testDsv2PredicateToCatalystExpression_UnsupportedPredicate() { assertFalse(result.isPresent(), "Unsupported predicates should return empty Optional"); } + // ===== Tests for decimal type alignment with table schema ===== + + private final org.apache.spark.sql.types.StructType decimalSchema = + new org.apache.spark.sql.types.StructType() + .add("price", new org.apache.spark.sql.types.DecimalType(7, 2), true) + .add("quantity", org.apache.spark.sql.types.DataTypes.IntegerType, true); + + @Test + public void testDecimalLiteralWidenedToColumnType() { + // Literal 100.00 has Decimal(5,2), column is Decimal(7,2) → should widen to Decimal(7,2) + EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Decimal filter should be converted with widened type"); + assertFalse(result.isPartial()); + Predicate pred = result.get(); + assertEquals("=", pred.getName()); + Literal literal = (Literal) pred.getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralScaleWidened() { + // Literal 100 has Decimal(3,0), column is Decimal(7,2) → should widen to Decimal(7,2) + EqualTo filter = new EqualTo("price", new BigDecimal("100")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Decimal filter with lower scale should be widened"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + assertEquals(new BigDecimal("100.00"), literal.getValue()); + } + + @Test + public void testDecimalLiteralHigherScaleThanColumn() { + // Literal 99.999 has scale=3, column is Decimal(7,2) → scale exceeds column, skip pushdown + GreaterThan filter = new GreaterThan("price", new BigDecimal("99.999")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertFalse( + result.isPresent(), + "Decimal literal with higher scale than column should not be pushed down"); + } + + @Test + public void testDecimalLiteralExceedsColumnPrecision() { + // Literal 123456.00 has 6 integral digits + 2 scale = precision 8, + // column is Decimal(7,2) which holds max 99999.99 → skip pushdown + LessThan filter = new LessThan("price", new BigDecimal("123456.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertFalse( + result.isPresent(), "Decimal literal exceeding column precision should not be pushed down"); + } + + @Test + public void testDecimalLiteralMatchingType() { + // Literal already matches column type Decimal(7,2) → no widening needed + EqualTo filter = new EqualTo("price", new BigDecimal("12345.67")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Decimal filter with matching type should be converted"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralWithoutSchema() { + // Without schema, decimal literal retains its intrinsic type + EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "Decimal filter without schema should still be converted"); + Literal literal = (Literal) result.get().getChildren().get(1); + // 100.00 has precision=5, scale=2 + assertEquals(new DecimalType(5, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralNonDecimalColumn() { + // Column "quantity" is IntegerType, not DecimalType → use default conversion + EqualTo filter = new EqualTo("quantity", new BigDecimal("42")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "BigDecimal for non-decimal column should use default type"); + Literal literal = (Literal) result.get().getChildren().get(1); + // Default BigDecimal conversion: precision=2, scale=0 + assertEquals(new DecimalType(2, 0), literal.getDataType()); + } + + @Test + public void testDecimalLiteralCaseInsensitiveColumnLookup() { + // Filter uses "PRICE" but schema has "price" → should still widen + EqualTo filter = new EqualTo("PRICE", new BigDecimal("100.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Case-insensitive column lookup should match"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralColumnNotInSchema() { + // Column "unknown" not in schema → use default conversion + EqualTo filter = new EqualTo("unknown", new BigDecimal("100.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Unknown column should fall back to default conversion"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(5, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralInCompoundFilter() { + // AND(price >= 100.00, price <= 200.00) with Decimal(7,2) column + GreaterThanOrEqual left = new GreaterThanOrEqual("price", new BigDecimal("100.00")); + LessThanOrEqual right = new LessThanOrEqual("price", new BigDecimal("200.00")); + org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(left, right); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, decimalSchema); + + assertTrue(result.isPresent(), "AND filter with decimal operands should be converted"); + assertFalse(result.isPartial()); + // Verify both literals are widened to Decimal(7,2) + io.delta.kernel.expressions.And andPred = (io.delta.kernel.expressions.And) result.get(); + Predicate leftPred = (Predicate) andPred.getChildren().get(0); + Predicate rightPred = (Predicate) andPred.getChildren().get(1); + assertEquals(new DecimalType(7, 2), ((Literal) leftPred.getChildren().get(1)).getDataType()); + assertEquals(new DecimalType(7, 2), ((Literal) rightPred.getChildren().get(1)).getDataType()); + } + + @Test + public void testDecimalLiteralInOrFilter() { + // OR(price >= 100.00, price >= 200.00) with Decimal(7,2) column + GreaterThanOrEqual left = new GreaterThanOrEqual("price", new BigDecimal("100.00")); + GreaterThanOrEqual right = new GreaterThanOrEqual("price", new BigDecimal("200.00")); + org.apache.spark.sql.sources.Or orFilter = new org.apache.spark.sql.sources.Or(left, right); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(orFilter, decimalSchema); + + assertTrue(result.isPresent(), "OR filter with decimal operands should be converted"); + } + + @Test + public void testDecimalLiteralInNotFilter() { + // NOT(price = 100.00) with Decimal(7,2) column + EqualTo eq = new EqualTo("price", new BigDecimal("100.00")); + Not notFilter = new Not(eq); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(notFilter, decimalSchema); + + assertTrue(result.isPresent(), "NOT filter with decimal operand should be converted"); + Predicate notPred = result.get(); + assertEquals("NOT", notPred.getName()); + Predicate innerPred = (Predicate) notPred.getChildren().get(0); + Literal literal = (Literal) innerPred.getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralWithEqualNullSafe() { + // EqualNullSafe(price, 100.00) with Decimal(7,2) column + EqualNullSafe filter = new EqualNullSafe("price", new BigDecimal("100.00")); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "EqualNullSafe with decimal should be converted"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralAllComparisonOperators() { + // Test all comparison operators widen decimals correctly + BigDecimal value = new BigDecimal("50.00"); // Decimal(4,2) → should widen to Decimal(7,2) + Filter[] filters = + new Filter[] { + new EqualTo("price", value), + new GreaterThan("price", value), + new GreaterThanOrEqual("price", value), + new LessThan("price", value), + new LessThanOrEqual("price", value), + }; + String[] expectedOps = new String[] {"=", ">", ">=", "<", "<="}; + + for (int i = 0; i < filters.length; i++) { + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filters[i], decimalSchema); + assertTrue(result.isPresent(), expectedOps[i] + " filter should be converted"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals( + new DecimalType(7, 2), + literal.getDataType(), + expectedOps[i] + " should widen decimal to column type"); + } + } + @Test public void testDsv2PredicateToCatalystExpression_ColumnNotFound() { NamedReference invalidRef = FieldReference.apply("nonexistent_column"); From bfee457b51edfabe36c8e2338e5e951bb0569ad5 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Mon, 16 Mar 2026 23:50:10 +0530 Subject: [PATCH 34/80] Add warning logging in fetchCatalogPrefix exception handler (#6273) ## Problem fetchCatalogPrefix() catches all exceptions silently, making debugging difficult. ## Fix Add logWarning to the catch block while preserving fallback behavior. ## Validation Confirmed by reading lines 188-208. New test verifies the fallback works. ## Regression Prevention New test verifies warnings are emitted on exceptions. --------- Co-authored-by: foo.bar --- .../IcebergRESTCatalogPlanningClient.scala | 10 ++++++--- ...cebergRESTCatalogPlanningClientSuite.scala | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala index adad47eee9f..ff782689d2b 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala @@ -32,6 +32,7 @@ import org.apache.http.client.ServiceUnavailableRetryStrategy import org.apache.http.impl.client.{DefaultHttpRequestRetryHandler, HttpClientBuilder} import org.apache.http.protocol.HttpContext import org.apache.http.message.BasicHeader +import org.apache.spark.internal.Logging import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType @@ -70,7 +71,7 @@ private case class CatalogConfigResponse( class IcebergRESTCatalogPlanningClient( baseUriRaw: String, catalogName: String, - token: String) extends ServerSidePlanningClient { + token: String) extends ServerSidePlanningClient with Logging { // Normalize baseUri to handle trailing slashes private val baseUri = baseUriRaw.stripSuffix("/") @@ -188,8 +189,8 @@ class IcebergRESTCatalogPlanningClient( * Returns None on any error or if no prefix is defined in the config. */ private def fetchCatalogPrefix(): Option[String] = { + val configUri = s"$baseUri/config?warehouse=$catalogName" try { - val configUri = s"$baseUri/config?warehouse=$catalogName" val httpGet = new HttpGet(configUri) val response = httpClient.execute(httpGet) try { @@ -205,7 +206,10 @@ class IcebergRESTCatalogPlanningClient( response.close() } } catch { - case _: Exception => None + case e: Exception => + logWarning(s"Failed to fetch catalog prefix from $configUri. " + + s"Falling back to base URI. Error: ${e.getMessage}") + None } } diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala index 841ffd6333f..b38019500e9 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala @@ -603,6 +603,27 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe } } + test("fetchCatalogPrefix falls back to baseUri on connection failure") { + // Use a port that's expected to have no listener. fetchCatalogPrefix() makes an HTTP GET + // to /config which will fail with a connection error. It should catch the exception, log a + // warning, and return None — causing icebergRestCatalogUriRoot to fall back to baseUri. + // The subsequent planScan HTTP POST will also fail (same unreachable host). + val unreachableUri = "http://localhost:1" + val client = new IcebergRESTCatalogPlanningClient(unreachableUri, "test_catalog", "") + try { + val ex = intercept[Exception] { + client.planScan("test_db", "test_table") + } + // Verify the exception is a connection error. This confirms fetchCatalogPrefix() + // did not throw a different exception type (e.g., NPE, parse error) and that the + // client progressed past the config fetch to attempt the plan HTTP POST. + assert(ex.getMessage != null, + "Expected a connection error with a message from the HTTP client") + } finally { + client.close() + } + } + test("User-Agent header format") { val client = new IcebergRESTCatalogPlanningClient("http://localhost:8080", "test_catalog", "") try { From 4ac45ccc6dea7a4a8095d03a63ff9556451ea8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ili=C4=87?= Date: Mon, 16 Mar 2026 19:47:52 +0100 Subject: [PATCH 35/80] [KERNEL][5821] Fix null pointer exception in `FieldMetadata.equals` (#5887) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [X] Kernel - [ ] Other (fill in here) ## Description Fix null pointer exception in `FieldMetadata.equals`. Previously, `e.getValue().getClass().isArray()` could raise null pointer exception, since `e.getValue()` can be null. ## How was this patch tested? New unit tests. ## Does this PR introduce _any_ user-facing changes? No. --- .../io/delta/kernel/types/FieldMetadata.java | 16 +- .../kernel/types/FieldMetadataSuite.scala | 140 ++++++++++++++++++ 2 files changed, 147 insertions(+), 9 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java b/kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java index f2dc9edb0e3..e7a17a4654d 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java @@ -172,17 +172,15 @@ public boolean equalsIgnoreKeys(FieldMetadata other, Set keys) { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - FieldMetadata that = (FieldMetadata) o; - if (this.metadata.size() != that.metadata.size()) return false; + FieldMetadata other = (FieldMetadata) o; + if (this.metadata.size() != other.metadata.size()) return false; return this.metadata.entrySet().stream() .allMatch( - e -> - Objects.equals(e.getValue(), that.metadata.get(e.getKey())) - || (e.getValue() != null - && e.getValue().getClass().isArray() - && that.metadata.get(e.getKey()).getClass().isArray() - && Arrays.equals( - (Object[]) e.getValue(), (Object[]) that.metadata.get(e.getKey())))); + e -> { + Object value = e.getValue(); + Object otherValue = other.metadata.get(e.getKey()); + return Objects.deepEquals(value, otherValue); + }); } @Override diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala index 81a3dc20ff2..35e4dc2b77d 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala @@ -326,4 +326,144 @@ class FieldMetadataSuite extends AnyFunSuite { } assert(e.getMessage == "keys must not be null") } + + test("equals validates all keys and values") { + val meta1 = FieldMetadata.builder() + .putString("collation", "UTF8_BINARY") + .putString("otherKey", "same") + .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) + .build() + + val meta2 = FieldMetadata.builder() + .putString("collation", "UTF8_BINARY") + .putString("otherKey", "same") + .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) + .build() + + val meta3 = FieldMetadata.builder() + .putString("collation", "EN_CI") // different -> should not be equal + .putString("otherKey", "same") + .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) + .build() + + assertThat(meta1.equals(meta2)).isTrue + assertThat(meta1.equals(meta3)).isFalse + } + + test("equals handles arrays properly") { + val longs: Seq[java.lang.Long] = Seq(1L, 2L, 3L) + val doubles: Seq[java.lang.Double] = Seq(1.0, 2.0, 3.0) + val booleans: Seq[java.lang.Boolean] = Seq(true, false, true) + val strings: Seq[java.lang.String] = Seq("x", "y", "z") + val stringsWithNulls: Seq[java.lang.String] = Seq("a", null, "b") + val inner1 = FieldMetadata.builder().putBoolean("k", true).build() + val inner2 = FieldMetadata.builder().putBoolean("k", true).build() + + val meta1 = FieldMetadata.builder() + .putLongArray("longArrayKey", longs.toArray) + .putDoubleArray("doubleArrayKey", doubles.toArray) + .putBooleanArray("booleanArrayKey", booleans.toArray) + .putStringArray("stringArrayKey", strings.toArray) + .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) + .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner1)) + .build() + + val meta2 = FieldMetadata.builder() + .putLongArray("longArrayKey", longs.toArray) + .putDoubleArray("doubleArrayKey", doubles.toArray) + .putBooleanArray("booleanArrayKey", booleans.toArray) + .putStringArray("stringArrayKey", strings.toArray) + .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) + .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner2)) + .build() + + assertThat(meta1.equals(meta2)).isTrue + + // Change one array element to ensure inequality is detected + val meta3 = FieldMetadata.builder() + .putLongArray("longArrayKey", Seq[java.lang.Long](1L, 2L, 99L).toArray) + .putDoubleArray("doubleArrayKey", doubles.toArray) + .putBooleanArray("booleanArrayKey", booleans.toArray) + .putStringArray("stringArrayKey", strings.toArray) + .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) + .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner2)) + .build() + + assertThat(meta1.equals(meta3)).isFalse + } + + test("equals handles case where only one side has the key") { + val meta1 = FieldMetadata.builder() + .putString("collation", "EN_CI") + .putString("common", "v") + .build() + val meta2 = FieldMetadata.builder() + .putString("common", "v") + .build() + + assertThat(meta1.equals(meta2)).isFalse + assertThat(meta2.equals(meta1)).isFalse + } + + test("equals handles entries with null value") { + val meta1 = FieldMetadata.builder() + .putString("nullableKey", null) + .putString("same", "x") + .build() + val meta2 = FieldMetadata.builder() + .putString("nullableKey", null) + .putString("same", "x") + .build() + val meta3 = FieldMetadata.builder() + .putString("nullableKey", "value") + .putString("same", "x") + .build() + + assertThat(meta1.equals(meta2)).isTrue + assertThat(meta1.equals(meta3)).isFalse + } + + test("equals handles entries with null key") { + val meta1 = FieldMetadata.builder() + .putString(null, "A") + .putString("same", "x") + .build() + val meta2 = FieldMetadata.builder() + .putString(null, "A") + .putString("same", "x") + .build() + val meta3 = FieldMetadata.builder() + .putString(null, "B") + .putString("same", "x") + .build() + val meta4 = FieldMetadata.builder() + .putString("same", "x") + .build() + + // same key/value pairs -> equal + assertThat(meta1.equals(meta2)).isTrue + // different values under null key -> unequal + assertThat(meta1.equals(meta3)).isFalse + // one side missing the null key -> unequal + assertThat(meta1.equals(meta4)).isFalse + } + + test("equals handles entry with null key and null value") { + val meta1 = FieldMetadata.builder() + .putString(null, null) + .putString("same", "x") + .build() + val meta2 = FieldMetadata.builder() + .putString(null, null) + .putString("same", "x") + .build() + + assertThat(meta1.equals(meta2)).isTrue + } + + test("equals returns false for null or different class") { + val meta = FieldMetadata.builder().putString("k", "v").build() + assertThat(meta.equals(null)).isFalse + assertThat(meta.equals("not-metadata")).isFalse + } } From 87a1b48ebe82c89e8b7bb9d93109e1d20d1b1149 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Tue, 17 Mar 2026 02:23:05 +0530 Subject: [PATCH 36/80] Reject collated StringType keys in MapType constructor (#6291) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Kernel's MapType constructor accepts collated StringType keys (e.g., UTF8_LCASE), but this should be disallowed per the Delta protocol spec and Spark's behavior. Fixes #5881. ## Fix Add validation in MapType constructors to reject non-default collation string types as keys. ## Validation - Confirmed by reading MapType.java constructor — no keyType validation exists. - New tests verify collated keys are rejected and default keys are accepted. ## Regression Prevention New test cases cover both collated (rejected) and default (accepted) StringType keys. --- .../java/io/delta/kernel/types/MapType.java | 21 +++++++++ .../types/DataTypeJsonSerDeSuite.scala | 31 +++++++++---- .../delta/kernel/types/StructFieldSuite.scala | 7 +-- .../io/delta/kernel/types/TypesSuite.scala | 46 +++++++++++++++++++ 4 files changed, 89 insertions(+), 16 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java b/kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java index b46fe327009..c1d4f2ab661 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java @@ -33,15 +33,36 @@ public class MapType extends DataType { public static final String MAP_VALUE_NAME = "value"; public MapType(DataType keyType, DataType valueType, boolean valueContainsNull) { + validateKeyType(keyType); this.keyField = new StructField(MAP_KEY_NAME, keyType, false); this.valueField = new StructField(MAP_VALUE_NAME, valueType, valueContainsNull); } public MapType(StructField keyField, StructField valueField) { + validateKeyType(keyField.getDataType()); this.keyField = keyField; this.valueField = valueField; } + /** + * The Delta protocol does not support collated string types as map keys. Only StringType with the + * default UTF8_BINARY collation is allowed. + * + * @see + * Collated String Type RFC + */ + private static void validateKeyType(DataType keyType) { + if (keyType instanceof StringType && !((StringType) keyType).isUTF8BinaryCollated()) { + throw new IllegalArgumentException( + String.format( + "MapType does not support collated string types as keys. " + + "Found key type '%s', but only StringType with default UTF8_BINARY " + + "collation is allowed.", + keyType)); + } + } + public StructField getKeyField() { return keyField; } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala index 700774bda1a..972f22bd8f7 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala @@ -253,6 +253,22 @@ class DataTypeJsonSerDeSuite extends AnyFunSuite { } } + test("deserialize: schema with collated map key throws IllegalArgumentException") { + // A JSON schema encoding a MapType with a collated StringType key should fail + // deserialization because MapType rejects collated keys. + val json = structTypeJson(Seq( + structFieldJson( + "m", + mapTypeJson("\"string\"", "\"integer\"", false), + true, + metadataJson = Some( + s"""{"$COLLATIONS_METADATA_KEY" : {"m.key" : "ICU.UNICODE_CI"}}""")))) + + intercept[IllegalArgumentException] { + parse(json) + } + } + test("serialize/deserialize: parsed and original struct" + " type differing just in StringType collation") { val json = structTypeJson(Seq( @@ -370,7 +386,6 @@ class DataTypeJsonSerDeSuite extends AnyFunSuite { metadataJson = Some( s"""{ |"$COLLATIONS_METADATA_KEY": { - | "tags.key": "SPARK.UTF8_LCASE", | "tags.value": "ICU.UNICODE" |}, |"delta.typeChanges": [ @@ -386,7 +401,7 @@ class DataTypeJsonSerDeSuite extends AnyFunSuite { .add( "tags", new MapType( - new StructField("key", new StringType("SPARK.UTF8_LCASE"), false), + new StructField("key", StringType.STRING, false), new StructField("value", new StringType("ICU.UNICODE"), false) .withTypeChanges(Seq(new TypeChange(BinaryType.BINARY, StringType.STRING)).asJava)), true) @@ -745,8 +760,7 @@ object DataTypeJsonSerDeSuite { false, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" - | : {"b2.key" : "ICU.UNICODE_CI", - | "b2.value" : "SPARK.UTF8_LCASE"}}""".stripMargin)), + | : {"b2.value" : "SPARK.UTF8_LCASE"}}""".stripMargin)), structFieldJson("b3", arrayTypeJson("\"string\"", false), true), structFieldJson("b4", mapTypeJson("\"string\"", "\"string\"", false), false))), true), @@ -772,7 +786,7 @@ object DataTypeJsonSerDeSuite { .add( "b2", new MapType( - new StringType("ICU.UNICODE_CI"), + StringType.STRING, new StringType("SPARK.UTF8_LCASE"), true), false) @@ -839,10 +853,7 @@ object DataTypeJsonSerDeSuite { | : {"b1" : "SPARK.UTF8_LCASE"}}""".stripMargin)))), false), false), - true, - metadataJson = Some( - s"""{"$COLLATIONS_METADATA_KEY" - | : {"a3.element.key" : "ICU.UNICODE_CI"}}""".stripMargin)), + true), structFieldJson( "a4", arrayTypeJson( @@ -894,7 +905,7 @@ object DataTypeJsonSerDeSuite { "a3", new ArrayType( new MapType( - new StringType("ICU.UNICODE_CI"), + StringType.STRING, new StructType() .add("b1", new StringType("SPARK.UTF8_LCASE"), false), false), diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala index 4385175cb66..832e8c3a2e8 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala @@ -266,18 +266,13 @@ class StructFieldSuite extends AnyFunSuite { new StructField( "mapField", new MapType( - new StructField("key", new StringType("ICU.DE_DE"), false), + new StructField("key", StringType.STRING, false), new StructField("value", LongType.LONG, true).withTypeChanges( Seq( new TypeChange(ShortType.SHORT, IntegerType.INTEGER), new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava)), false), FieldMetadata.builder() - .putFieldMetadata( - COLLATIONS_METADATA_KEY, - FieldMetadata.builder() - .putString("mapField.key", "ICU.DE_DE") - .build()) .putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, Array( diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala index 3989c46e62f..94372e058e4 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala @@ -64,4 +64,50 @@ class TypesSuite extends AnyFunSuite { assert(dataType.isNested(), s"Expected $dataType to be nested") } } + + test("MapType constructor throws IllegalArgumentException for collated StringType keys") { + // Test multiple collation providers to ensure all non-default collations are rejected + Seq("SPARK.UTF8_LCASE", "ICU.UNICODE_CI").foreach { collation => + val collatedString = new StringType(collation) + + // 3-arg constructor + val ex1 = intercept[IllegalArgumentException] { + new MapType(collatedString, IntegerType.INTEGER, false) + } + assert(ex1.getMessage.contains("does not support collated string types as keys")) + assert(ex1.getMessage.contains("UTF8_BINARY")) + assert( + ex1.getMessage.contains(collatedString.toString), + s"Error message should include the found type but was: ${ex1.getMessage}") + + // 2-arg StructField constructor + val ex2 = intercept[IllegalArgumentException] { + new MapType( + new StructField("key", collatedString, false), + new StructField("value", IntegerType.INTEGER, true)) + } + assert(ex2.getMessage.contains("does not support collated string types as keys")) + assert( + ex2.getMessage.contains(collatedString.toString), + s"Error message should include the found type but was: ${ex2.getMessage}") + } + } + + test("MapType allows default UTF8_BINARY StringType keys and non-string keys") { + val map1 = new MapType(StringType.STRING, IntegerType.INTEGER, false) + assert(map1.getKeyType === StringType.STRING) + + val utf8BinaryString = new StringType("SPARK.UTF8_BINARY") + val map2 = new MapType(utf8BinaryString, IntegerType.INTEGER, false) + assert(map2.getKeyType === utf8BinaryString) + + val map3 = new MapType(IntegerType.INTEGER, StringType.STRING, true) + assert(map3.getKeyType === IntegerType.INTEGER) + } + + test("MapType allows collated StringType as values") { + val collatedString = new StringType("SPARK.UTF8_LCASE") + val map = new MapType(StringType.STRING, collatedString, true) + assert(map.getValueType === collatedString) + } } From 6433d5543d590fc786971b4f51d5dfc7d9290173 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 17 Mar 2026 07:53:11 +0800 Subject: [PATCH 37/80] [Spark] Add In filter pushdown to ExpressionUtils (#6103) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Spark's In(attribute, values[]) filter was not converted to a Kernel predicate, meaning WHERE col IN (...) queries had no filter pushdown in the V2 connector. The Kernel's IN predicate accepts a column followed by one or more typed literals as children. The conversion builds the children list by calling convertValueToKernelLiteral on each value; if any value cannot be converted (null, unsupported type) or the values array is empty the whole filter is left for post-scan evaluation rather than producing an incorrect partially-pushed predicate. ## How was this patch tested? Unit test ## Does this PR introduce _any_ user-facing changes? No --------- Co-authored-by: Claude Sonnet 4.6 --- .../internal/v2/utils/ExpressionUtils.java | 24 ++++- .../v2/utils/ExpressionUtilsTest.java | 102 ++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java index ed1130551d3..3d0e71e5aa3 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java @@ -18,8 +18,10 @@ import static org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath; import com.google.common.annotations.VisibleForTesting; +import io.delta.kernel.expressions.AlwaysFalse; import io.delta.kernel.expressions.And; import io.delta.kernel.expressions.Column; +import io.delta.kernel.expressions.In; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Or; import io.delta.kernel.expressions.Predicate; @@ -59,8 +61,9 @@ public final class ExpressionUtils { *

  • Comparison: EqualTo, GreaterThan, LessThan, etc. *
  • Null tests: IsNull, IsNotNull *
  • Null-safe comparison: EqualNullSafe - *
  • Logical operators: And, Or, Not *
  • String prefix: StringStartsWith + *
  • Set membership: In + *
  • Logical operators: And, Or, Not * * * @param filter the Spark SQL filter to convert @@ -171,6 +174,25 @@ private static ConvertedPredicate convertSparkFilterToKernelPredicate( convertValueToKernelLiteral(f.value()) .map(l -> new Predicate("STARTS_WITH", kernelColumn(f.attribute()), l))); } + if (filter instanceof org.apache.spark.sql.sources.In) { + org.apache.spark.sql.sources.In f = (org.apache.spark.sql.sources.In) filter; + // An empty IN list can never match any row. Push ALWAYS_FALSE so the kernel skips + // all files entirely, rather than scanning every file only to discard every row. + if (f.values().length == 0) { + return new ConvertedPredicate(Optional.of(AlwaysFalse.ALWAYS_FALSE)); + } + List literals = new ArrayList<>(); + for (Object value : f.values()) { + Optional lit = convertValueToKernelLiteral(value); + if (!lit.isPresent()) { + // A value that can't be converted (e.g. null, unsupported type) makes the whole + // IN expression unsafe to push down; return empty to keep it for post-scan evaluation. + return new ConvertedPredicate(Optional.empty()); + } + literals.add(lit.get()); + } + return new ConvertedPredicate(Optional.of(new In(kernelColumn(f.attribute()), literals))); + } if (filter instanceof org.apache.spark.sql.sources.And) { org.apache.spark.sql.sources.And f = (org.apache.spark.sql.sources.And) filter; ConvertedPredicate left = diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java index 4dc822c07f7..ae1f47e6437 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java @@ -137,6 +137,108 @@ public void testStringStartsWithFilter_NullValue() { assertFalse(result.isPresent(), "StringStartsWith with null value should not be converted"); } + @Test + public void testInFilter_BasicConversion() { + In filter = new In("city", new Object[] {"hz", "sh", "bj"}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "In filter should be converted"); + assertFalse(result.isPartial(), "In filter should be fully converted"); + assertEquals("IN", result.get().getName()); + // Children: column + 3 literals + assertEquals(4, result.get().getChildren().size()); + } + + @Test + public void testInFilter_SingleValue() { + In filter = new In("id", new Object[] {42}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "In filter with single value should be converted"); + assertFalse(result.isPartial(), "In filter should be fully converted"); + assertEquals("IN", result.get().getName()); + // Children: column + 1 literal + assertEquals(2, result.get().getChildren().size()); + } + + @Test + public void testInFilter_EmptyValues() { + In filter = new In("city", new Object[] {}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + // Empty IN list always evaluates to FALSE; push ALWAYS_FALSE so the kernel skips all files. + assertTrue(result.isPresent(), "In filter with empty values should push ALWAYS_FALSE"); + assertEquals("ALWAYS_FALSE", result.get().getName()); + } + + @Test + public void testInFilter_WithNullValue() { + // null in the values array makes the IN expression unsafe to push down (SQL null semantics) + In filter = new In("city", new Object[] {"hz", null, "bj"}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertFalse(result.isPresent(), "In filter with null value should not be pushed down"); + } + + @Test + public void testInFilter_WithUnsupportedType() { + In filter = new In("col", new Object[] {42, new Object()}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertFalse(result.isPresent(), "In filter with unconvertible value should not be pushed down"); + } + + @Test + public void testInFilter_InAndFilter() { + // AND(In(...), EqualTo(...)) — both convertible, should be fully pushed down + In inFilter = new In("city", new Object[] {"hz", "sh"}); + EqualTo eqFilter = new EqualTo("part", 1); + org.apache.spark.sql.sources.And andFilter = + new org.apache.spark.sql.sources.And(inFilter, eqFilter); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter); + + assertTrue(result.isPresent(), "AND(In, EqualTo) should be converted"); + assertFalse(result.isPartial(), "AND(In, EqualTo) should be fully converted"); + assertTrue( + result.get() instanceof io.delta.kernel.expressions.And, + "Result should be an AND predicate"); + } + + @Test + public void testInFilter_NotInWithNullValue() { + // NOT(IN(col, 1, null)): null in the IN list causes IN to bail → NOT also bails + In inFilter = new In("city", new Object[] {"hz", null}); + Not notFilter = new Not(inFilter); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(notFilter); + + assertFalse( + result.isPresent(), "NOT(IN(..., null)) should not be pushed down due to null in IN list"); + } + + @Test + public void testInFilter_ORWithUnsupportedFilter() { + // OR(In(...), StringEndsWith(...)) — one branch unsupported, whole OR cannot be pushed + In inFilter = new In("city", new Object[] {"hz", "sh"}); + StringEndsWith endsWithFilter = new StringEndsWith("name", "foo"); + Or orFilter = new Or(inFilter, endsWithFilter); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(orFilter); + + assertFalse( + result.isPresent(), + "OR(In, unsupported) should not be pushed down when one branch is unsupported"); + } + // Test data provider for parameterized literal conversion tests static Stream valueTypesProvider() { return Stream.of( From 8504c90a4aa5e56ef1d8cf3ac9bcd6b2a00b77e3 Mon Sep 17 00:00:00 2001 From: yyanyy Date: Mon, 16 Mar 2026 17:45:05 -0700 Subject: [PATCH 38/80] [Spark] Implement SupportsReportPartitioning in DSv2 SparkScan (#6224) --- .../internal/v2/read/DeltaInputPartition.java | 94 +++++++ .../spark/internal/v2/read/SparkBatch.java | 58 ++++- .../internal/v2/read/SparkReaderFactory.java | 21 +- .../spark/internal/v2/read/SparkScan.java | 44 +++- .../internal/v2/utils/PartitionUtils.java | 6 + .../delta/spark/internal/v2/V2ReadTest.java | 48 ++++ .../spark/internal/v2/read/SparkScanTest.java | 237 ++++++++++++++++++ 7 files changed, 501 insertions(+), 7 deletions(-) create mode 100644 spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaInputPartition.java diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaInputPartition.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaInputPartition.java new file mode 100644 index 00000000000..11556b9e354 --- /dev/null +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaInputPartition.java @@ -0,0 +1,94 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.read; + +import java.util.Objects; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.read.HasPartitionKey; +import org.apache.spark.sql.connector.read.InputPartition; +import org.apache.spark.sql.execution.datasources.FilePartition; + +/** + * A Delta-specific InputPartition that wraps a FilePartition and implements HasPartitionKey. This + * enables Spark to leverage partition information for optimizations like shuffle elimination in + * joins and aggregations when using KeyGroupedPartitioning. + * + *

    Each DeltaInputPartition represents files from a single logical partition (i.e., all files + * share the same partition values). The partition key is the InternalRow containing the partition + * column values. + */ +public class DeltaInputPartition implements InputPartition, HasPartitionKey { + private final FilePartition filePartition; + private final InternalRow partitionKey; + + /** + * Creates a new DeltaInputPartition. + * + * @param filePartition The underlying FilePartition containing the files to read. + * @param partitionKey The partition key (partition column values) for all files in this + * partition. Must not be null. + */ + public DeltaInputPartition(FilePartition filePartition, InternalRow partitionKey) { + this.filePartition = Objects.requireNonNull(filePartition, "filePartition is null"); + this.partitionKey = Objects.requireNonNull(partitionKey, "partitionKey is null"); + } + + /** + * Returns the partition key (partition column values) associated with this partition. All files + * in this partition have the same partition key. + * + * @return The partition key as an InternalRow. + */ + @Override + public InternalRow partitionKey() { + return partitionKey; + } + + /** + * Returns the underlying FilePartition. + * + * @return The FilePartition containing the files to read. + */ + public FilePartition getFilePartition() { + return filePartition; + } + + @Override + public String[] preferredLocations() { + return filePartition.preferredLocations(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeltaInputPartition that = (DeltaInputPartition) o; + return Objects.equals(filePartition, that.filePartition) + && Objects.equals(partitionKey, that.partitionKey); + } + + @Override + public int hashCode() { + return Objects.hash(filePartition, partitionKey); + } + + @Override + public String toString() { + return String.format( + "DeltaInputPartition(partitionKey=%s, files=%d)", + partitionKey, filePartition.files().length); + } +} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java index fa13f84a800..e7b1493a250 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java @@ -20,10 +20,13 @@ import io.delta.spark.internal.v2.utils.PartitionUtils; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.Batch; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReaderFactory; @@ -86,10 +89,57 @@ public InputPartition[] planInputPartitions() { PartitionUtils.calculateMaxSplitBytes( sparkSession, totalBytes, partitionedFiles.size(), sqlConf); - scala.collection.Seq filePartitions = - FilePartition$.MODULE$.getFilePartitions( - sparkSession, JavaConverters.asScalaBuffer(partitionedFiles).toSeq(), maxSplitBytes); - return JavaConverters.seqAsJavaList(filePartitions).toArray(new InputPartition[0]); + // For non-partitioned tables, use simple file partitioning + if (partitionSchema.fields().length == 0) { + scala.collection.Seq filePartitions = + FilePartition$.MODULE$.getFilePartitions( + sparkSession, JavaConverters.asScalaBuffer(partitionedFiles).toSeq(), maxSplitBytes); + return JavaConverters.seqAsJavaList(filePartitions).toArray(new InputPartition[0]); + } + + // For partitioned tables, group files by partition values and wrap in DeltaInputPartition + // to support HasPartitionKey for KeyGroupedPartitioning optimizations + return planPartitionedInputPartitions(sparkSession, maxSplitBytes); + } + + /** + * Plans input partitions for partitioned tables by grouping files by their partition values. Each + * resulting DeltaInputPartition implements HasPartitionKey, enabling Spark to leverage partition + * information for optimizations like shuffle elimination. + */ + private InputPartition[] planPartitionedInputPartitions( + SparkSession sparkSession, long maxSplitBytes) { + // Note: Using InternalRow as map key relies on GenericInternalRow's value-based + // equals()/hashCode(), which is what PartitionUtils.getPartitionRow() returns. + Map> filesByPartition = new LinkedHashMap<>(); + for (PartitionedFile file : partitionedFiles) { + InternalRow partitionKey = file.partitionValues(); + filesByPartition.computeIfAbsent(partitionKey, k -> new ArrayList<>()).add(file); + } + + // Create DeltaInputPartitions for each partition group + List result = new ArrayList<>(); + int partitionIndex = 0; + + for (Map.Entry> entry : filesByPartition.entrySet()) { + InternalRow partitionKey = entry.getKey(); + List filesInPartition = entry.getValue(); + + // Split files within this partition based on maxSplitBytes + scala.collection.Seq filePartitions = + FilePartition$.MODULE$.getFilePartitions( + sparkSession, JavaConverters.asScalaBuffer(filesInPartition).toSeq(), maxSplitBytes); + + // Wrap each FilePartition in a DeltaInputPartition with the partition key. + // Re-index partitions with a global counter because getFilePartitions returns 0-based + // indices within each partition group, but we need unique indices across all groups. + for (FilePartition fp : JavaConverters.seqAsJavaList(filePartitions)) { + FilePartition reindexedPartition = new FilePartition(partitionIndex++, fp.files()); + result.add(new DeltaInputPartition(reindexedPartition, partitionKey)); + } + } + + return result.toArray(new InputPartition[0]); } @Override diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java index 4a951b98ab5..81ca328af39 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java @@ -37,7 +37,7 @@ public SparkReaderFactory( @Override public PartitionReader createColumnarReader(InputPartition partition) { - return new SparkPartitionReader(readFunc, (FilePartition) partition); + return new SparkPartitionReader(readFunc, extractFilePartition(partition)); } @Override @@ -47,6 +47,23 @@ public boolean supportColumnarReads(InputPartition partition) { @Override public PartitionReader createReader(InputPartition partition) { - return new SparkPartitionReader(readFunc, (FilePartition) partition); + return new SparkPartitionReader(readFunc, extractFilePartition(partition)); + } + + /** + * Extracts the FilePartition from the given InputPartition. Handles both DeltaInputPartition (for + * partitioned tables) and FilePartition (for non-partitioned tables). + */ + private FilePartition extractFilePartition(InputPartition partition) { + if (partition instanceof DeltaInputPartition) { + return ((DeltaInputPartition) partition).getFilePartition(); + } else if (partition instanceof FilePartition) { + return (FilePartition) partition; + } else { + throw new IllegalArgumentException( + "Unexpected partition type: " + + partition.getClass().getName() + + ". Expected DeltaInputPartition or FilePartition."); + } } } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java index 3d544b31420..85fdcc30a73 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java @@ -42,6 +42,9 @@ import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.read.*; import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.connector.read.partitioning.KeyGroupedPartitioning; +import org.apache.spark.sql.connector.read.partitioning.Partitioning; +import org.apache.spark.sql.connector.read.partitioning.UnknownPartitioning; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.execution.datasources.*; @@ -53,7 +56,11 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap; /** Spark DSV2 Scan implementation backed by Delta Kernel. */ -public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntimeV2Filtering { +public class SparkScan + implements Scan, + SupportsReportStatistics, + SupportsRuntimeV2Filtering, + SupportsReportPartitioning { /** Supported streaming options for the V2 connector. */ private static final List SUPPORTED_STREAMING_OPTIONS = @@ -529,6 +536,41 @@ static void validateStreamingOptions(DeltaOptions deltaOptions) { } } + /** + * Reports partition key expressions to Spark so it can recognize partition-aligned data layout. + * Called by V2ScanPartitioningAndOrdering during logical optimization to extract partition keys. + * Together with HasPartitionKey on DeltaInputPartition, this enables Spark to eliminate shuffles + * for joins and aggregations on partition columns. + * + *

    Note: This method triggers scan file materialization via {@link #ensurePlanned()} because + * {@code numPartitions} is derived from the planned file count. Since Spark calls this during + * logical optimization (before {@link #toBatch()}), this changes when planning occurs compared to + * the non-partitioned path. This is functionally correct as {@code ensurePlanned} is idempotent. + */ + @Override + public Partitioning outputPartitioning() { + // If no partition columns, return unknown partitioning + if (partitionSchema.fields().length == 0) { + return new UnknownPartitioning(0); + } + + ensurePlanned(); + + // Create partition key expressions from partition schema + org.apache.spark.sql.connector.expressions.Expression[] keys = + Arrays.stream(partitionSchema.fields()) + .map( + field -> + (org.apache.spark.sql.connector.expressions.Expression) + FieldReference.column(field.name())) + .toArray(org.apache.spark.sql.connector.expressions.Expression[]::new); + + // numPartitions is not used by Spark's KeyGroupedPartitioning handling (Spark derives + // partition count from the actual InputPartition[] with HasPartitionKey), so we use the + // file count as a reasonable upper-bound estimate. + return new KeyGroupedPartitioning(keys, partitionedFiles.size()); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java index f88a3c57223..b2e960dd0f8 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java @@ -93,6 +93,12 @@ public static long calculateMaxSplitBytes( *

    Note: Partition values in AddFile use physical column names as keys when column mapping is * enabled. This method uses DeltaColumnMapping.getPhysicalName to map from logical schema fields * to physical partition value keys. + * + * @implNote The returned {@link InternalRow} is a {@code GenericInternalRow} (via {@code + * InternalRow.fromSeq}), which has value-based {@code equals()}/{@code hashCode()}. Callers + * such as {@code SparkBatch.planPartitionedInputPartitions} rely on this for grouping files + * by partition key. Changing the return type to a different InternalRow subtype (e.g. {@code + * UnsafeRow}) may break that contract. */ public static InternalRow getPartitionRow( MapValue partitionValues, StructType partitionSchema, ZoneId zoneId) { diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java index 81d9d37bd6a..7565169b2b5 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java @@ -16,6 +16,7 @@ package io.delta.spark.internal.v2; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; @@ -102,4 +103,51 @@ public void testDeletionVectorRead(@TempDir File tempDir) throws Exception { // 500 odd numbers from 0-999: 1, 3, 5, ..., 999 assertTrue(count == 500, "Expected 500 rows after DV filtering, got " + count); } + + @Test + public void testPartitionedJoinEliminatesShuffle(@TempDir File tempDir) { + String tablePath = tempDir.getAbsolutePath(); + + // Create a partitioned Delta table via V1 + spark.sql( + str( + "CREATE TABLE delta.`%s` (id INT, data STRING, part INT) " + + "USING delta PARTITIONED BY (part)", + tablePath)); + spark.sql( + str( + "INSERT INTO delta.`%s` VALUES (1, 'a', 1), (2, 'b', 1), (3, 'c', 2), (4, 'd', 3)", + tablePath)); + + // Disable broadcast join so Spark must use shuffle or partition-aware join. + // Enable V2 bucketing so Spark recognizes KeyGroupedPartitioning from + // SupportsReportPartitioning (default is false in Spark 4.0, true in 4.1). + withSQLConf( + "spark.sql.autoBroadcastJoinThreshold", + "-1", + () -> + withSQLConf( + "spark.sql.sources.v2.bucketing.enabled", + "true", + () -> { + // Self-join on partition column via DSv2 catalog + String joinQuery = + str( + "SELECT a.id, b.data FROM dsv2.delta.`%s` a " + + "JOIN dsv2.delta.`%s` b ON a.part = b.part", + tablePath, tablePath); + + String explainOutput = + spark.sql(joinQuery).queryExecution().executedPlan().toString(); + + // With SupportsReportPartitioning + HasPartitionKey, Spark should recognize + // both sides are KeyGroupedPartitioned on 'part' and skip the shuffle + // (no Exchange node) + assertFalse( + explainOutput.contains("Exchange"), + "Expected no Exchange (shuffle) in plan for partition-aligned join, " + + "but found one.\nPlan:\n" + + explainOutput); + })); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java index fd61863bf44..c96dd4bd0d0 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java @@ -14,7 +14,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.TableIdentifier; import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; @@ -25,11 +27,18 @@ import org.apache.spark.sql.connector.expressions.LiteralValue; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.expressions.filter.Predicate; +import org.apache.spark.sql.connector.read.Batch; +import org.apache.spark.sql.connector.read.HasPartitionKey; +import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.ScanBuilder; import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.connector.read.partitioning.KeyGroupedPartitioning; +import org.apache.spark.sql.connector.read.partitioning.Partitioning; +import org.apache.spark.sql.connector.read.partitioning.UnknownPartitioning; import org.apache.spark.sql.delta.DeltaOptions; +import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; @@ -820,6 +829,234 @@ public void testNotEqualsWithAndWithoutRuntimeFilter() { assertNotEquals(scan1.hashCode(), scan2.hashCode()); } + // ================================================================================================ + // Tests for output partitioning (SupportsReportPartitioning) + // ================================================================================================ + + @Test + public void testOutputPartitioningForPartitionedTable() { + // Partitioned table should return KeyGroupedPartitioning + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + Partitioning partitioning = scan.outputPartitioning(); + + assertTrue( + partitioning instanceof KeyGroupedPartitioning, + "Partitioned table should return KeyGroupedPartitioning"); + + KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; + + // The partitioned table has 3 partition columns: date, city, part + Expression[] keys = kgp.keys(); + assertEquals(3, keys.length, "Should have 3 partition key expressions"); + + // Verify partition key names match partition schema + Set keyNames = new HashSet<>(); + for (Expression key : keys) { + assertTrue(key instanceof FieldReference, "Key should be a FieldReference"); + keyNames.add(((FieldReference) key).fieldNames()[0]); + } + assertTrue(keyNames.containsAll(Arrays.asList("date", "city", "part"))); + + // numPartitions returns partitionedFiles.size() (file count, not unique partition count). + // In this test data, each partition has one file, so file count equals partition count. + assertEquals(5, kgp.numPartitions(), "Should have 5 files (one per partition)"); + } + + /** Creates a non-partitioned table with sample data and returns a SparkScan for it. */ + private SparkScan createNonPartitionedScan(File tempDir, String tableName) { + spark.sql( + String.format( + "CREATE TABLE `%s` (id INT, name STRING) USING delta LOCATION '%s'", + tableName, tempDir.getAbsolutePath())); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob')", tableName)); + + SparkTable nonPartTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, tableName), + tempDir.getAbsolutePath(), + options); + + SparkScanBuilder builder = (SparkScanBuilder) nonPartTable.newScanBuilder(options); + return (SparkScan) builder.build(); + } + + @Test + public void testOutputPartitioningForNonPartitionedTable(@TempDir File tempDir) { + // Non-partitioned table should return UnknownPartitioning + SparkScan scan = createNonPartitionedScan(tempDir, "deltatbl_nonpartitioned"); + + Partitioning partitioning = scan.outputPartitioning(); + + assertTrue( + partitioning instanceof UnknownPartitioning, + "Non-partitioned table should return UnknownPartitioning"); + } + + @Test + public void testOutputPartitioningAfterRuntimeFilter() { + // Output partitioning should reflect filtered partition count + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + // Apply filter to only keep city=hz (2 rows: part=1/date=20180520 and part=1/date=20180718) + scan.filter(new Predicate[] {cityPredicate}); + + Partitioning partitioning = scan.outputPartitioning(); + assertTrue(partitioning instanceof KeyGroupedPartitioning); + + KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; + // numPartitions returns partitionedFiles.size() (file count); here each partition has one file. + assertEquals( + 2, + kgp.numPartitions(), + "After filtering to city=hz, should have 2 files (one per partition)"); + } + + // ================================================================================================ + // Tests for DeltaInputPartition in planInputPartitions + // ================================================================================================ + + @Test + public void testPlanInputPartitionsReturnsHasPartitionKeyForPartitionedTable() { + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + Batch batch = scan.toBatch(); + + InputPartition[] partitions = batch.planInputPartitions(); + + assertTrue(partitions.length > 0, "Should have at least one partition"); + for (InputPartition partition : partitions) { + assertTrue( + partition instanceof DeltaInputPartition, + "Partitioned table should return DeltaInputPartition instances"); + assertTrue( + partition instanceof HasPartitionKey, + "DeltaInputPartition should implement HasPartitionKey"); + + DeltaInputPartition deltaPartition = (DeltaInputPartition) partition; + assertNotNull(deltaPartition.partitionKey(), "Partition key should not be null"); + assertNotNull(deltaPartition.getFilePartition(), "FilePartition should not be null"); + } + } + + @Test + public void testPlanInputPartitionsGroupsFilesByPartition(@TempDir File tempDir) { + // Create a table with multiple files per partition to actually exercise the grouping logic + // in planPartitionedInputPartitions (the default test table has 1 file per partition, + // which would pass even without grouping). + String multiFileTableName = "deltatbl_multifile_partitioned"; + spark.sql( + String.format( + "CREATE TABLE `%s` (id INT, data STRING, part INT) USING delta " + + "LOCATION '%s' PARTITIONED BY (part)", + multiFileTableName, tempDir.getAbsolutePath())); + // Insert in separate statements to create multiple files per partition + spark.sql( + String.format("INSERT INTO `%s` VALUES (1, 'a', 1), (2, 'b', 2)", multiFileTableName)); + spark.sql( + String.format("INSERT INTO `%s` VALUES (3, 'c', 1), (4, 'd', 2)", multiFileTableName)); + spark.sql(String.format("INSERT INTO `%s` VALUES (5, 'e', 1)", multiFileTableName)); + // Now part=1 has 3 files, part=2 has 2 files + + SparkTable multiFileTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, multiFileTableName), + tempDir.getAbsolutePath(), + options); + + SparkScanBuilder builder = (SparkScanBuilder) multiFileTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + Batch batch = scan.toBatch(); + + InputPartition[] partitions = batch.planInputPartitions(); + + // Verify all partitions are DeltaInputPartition with partition keys + Map> partitionsByKey = new HashMap<>(); + for (InputPartition p : partitions) { + assertTrue(p instanceof DeltaInputPartition); + DeltaInputPartition dp = (DeltaInputPartition) p; + partitionsByKey.computeIfAbsent(dp.partitionKey(), k -> new ArrayList<>()).add(dp); + } + + // Should have exactly 2 unique partition keys (part=1 and part=2) + assertEquals(2, partitionsByKey.size(), "Should have 2 unique partition keys"); + + // Verify that the grouping actually produced multiple DeltaInputPartitions for a single + // partition key (since multiple files exist per partition and each gets its own + // FilePartition at default maxSplitBytes) + int totalPartitions = partitions.length; + assertTrue( + totalPartitions > 2, + "With 5 files across 2 partitions, should have more than 2 input partitions, " + + "got " + + totalPartitions); + + // Verify all DeltaInputPartitions with the same key share the same partition key instance + for (Map.Entry> entry : partitionsByKey.entrySet()) { + List group = entry.getValue(); + InternalRow expectedKey = group.get(0).partitionKey(); + for (DeltaInputPartition dp : group) { + assertEquals( + expectedKey, + dp.partitionKey(), + "All partitions in the same group should have equal partition keys"); + } + } + } + + @Test + public void testPlanInputPartitionsReturnsFilePartitionForNonPartitionedTable( + @TempDir File tempDir) { + SparkScan scan = createNonPartitionedScan(tempDir, "deltatbl_nonpartitioned_batch"); + Batch batch = scan.toBatch(); + + InputPartition[] partitions = batch.planInputPartitions(); + + assertTrue(partitions.length > 0, "Should have at least one partition"); + for (InputPartition partition : partitions) { + assertTrue( + partition instanceof FilePartition, + "Non-partitioned table should return FilePartition instances, not DeltaInputPartition"); + assertFalse( + partition instanceof DeltaInputPartition, + "Non-partitioned table should NOT return DeltaInputPartition"); + } + } + + @Test + public void testOutputPartitioningForEmptyPartitionedTable(@TempDir File tempDir) { + // Empty partitioned table should return KeyGroupedPartitioning with 0 partitions + String emptyTableName = "deltatbl_empty_partitioned"; + spark.sql( + String.format( + "CREATE TABLE `%s` (id INT, name STRING, part INT) USING delta " + + "LOCATION '%s' PARTITIONED BY (part)", + emptyTableName, tempDir.getAbsolutePath())); + + SparkTable emptyTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, emptyTableName), + tempDir.getAbsolutePath(), + options); + + SparkScanBuilder builder = (SparkScanBuilder) emptyTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + Partitioning partitioning = scan.outputPartitioning(); + assertTrue( + partitioning instanceof KeyGroupedPartitioning, + "Empty partitioned table should still return KeyGroupedPartitioning"); + + KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; + assertEquals(0, kgp.numPartitions(), "Empty table should have 0 partitions"); + + Batch batch = scan.toBatch(); + InputPartition[] partitions = batch.planInputPartitions(); + assertEquals(0, partitions.length, "Empty table should return 0 input partitions"); + } + // ================================================================================================ // Tests for catalog statistics propagation // ================================================================================================ From 3ac7b40767f1175c57d373e5010b43f7617663fb Mon Sep 17 00:00:00 2001 From: Ziya Mukhtarov Date: Tue, 17 Mar 2026 16:24:33 +0100 Subject: [PATCH 39/80] [Spark] Check NullType inside UDTs (#6299) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Follow-up to #6254. This PR improves the check for NullType in schema for streaming to also consider UDTs. ## How was this patch tested? New unit tests. ## Does this PR introduce _any_ user-facing changes? No. --- .../spark/sql/delta/schema/SchemaUtils.scala | 34 +++++++++++++------ .../spark/sql/delta/sources/DeltaSink.scala | 4 +-- .../spark/sql/delta/DeltaSinkSuite.scala | 25 ++++++++++++++ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala index f1229f4db37..5b4937cd9ff 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala @@ -109,6 +109,24 @@ object SchemaUtils extends DeltaLogging { Some(other).filter(f) } + /** + * Checks if a given data type contains a NullType, including inside UDTs. + * `typeExistsRecursively` does not recurse into UDT sqlTypes, so this method + * explicitly handles that case. + */ + def nullTypeExistsRecursively( + t: DataType + ): Boolean = { + typeExistsRecursively(t) { + case _: NullType => + true + case udt: UserDefinedType[_] => + nullTypeExistsRecursively(udt.sqlType) + case _ => + false + } + } + /** Turns the data types to nullable in a recursive manner for nested columns. */ def typeAsNullable(dt: DataType): DataType = dt match { case s: StructType => s.asNullable @@ -130,14 +148,8 @@ object SchemaUtils extends DeltaLogging { */ def dropNullTypeColumns(df: DataFrame): DataFrame = { val schema = df.schema - def hasNullType(t: DataType): Boolean = - typeExistsRecursively(t) { - case _: NullType => true - case udt: UserDefinedType[_] => hasNullType(udt.sqlType) - case _ => false - } + if (!nullTypeExistsRecursively(schema)) return df - if (!hasNullType(schema)) return df def generateSelectExpr(sf: StructField, nameStack: Seq[String]): Column = sf.dataType match { case st: StructType => val nested = st.fields.flatMap { f => @@ -151,17 +163,17 @@ object SchemaUtils extends DeltaLogging { when(col(colName).isNull, null) .otherwise(struct(nested: _*)) .alias(sf.name) - case a: ArrayType if hasNullType(a) => + case a: ArrayType if nullTypeExistsRecursively(a) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE", messageParameters = Array(colName, "ArrayType")) - case m: MapType if hasNullType(m) => + case m: MapType if nullTypeExistsRecursively(m) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE", messageParameters = Array(colName, "MapType")) - case udt: UserDefinedType[_] if hasNullType(udt.sqlType) => + case udt: UserDefinedType[_] if nullTypeExistsRecursively(udt.sqlType) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", @@ -275,7 +287,7 @@ object SchemaUtils extends DeltaLogging { return nullFields.headOption } - if (typeExistsRecursively(schema)(_.isInstanceOf[NullType])) { + if (nullTypeExistsRecursively(schema)) { findNullTypeColumnRec(schema, Seq.empty) } else { None diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala index d8a559ff32b..10b5037c17e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala @@ -42,7 +42,7 @@ import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.execution.streaming.Sink import org.apache.spark.sql.streaming.OutputMode -import org.apache.spark.sql.types.{ArrayType, DataType, MapType, NullType, StructType} +import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType} import org.apache.spark.util.Utils /** @@ -114,7 +114,7 @@ case class DeltaSink( val txn = deltaLog.startTransaction(catalogTable) assert(queryId != null) - if (SchemaUtils.typeExistsRecursively(data.schema)(_.isInstanceOf[NullType])) { + if (SchemaUtils.nullTypeExistsRecursively(data.schema)) { throw DeltaErrors.streamWriteNullTypeException } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala index 766e6ed9aec..9a78364b0c9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala @@ -655,6 +655,31 @@ class DeltaSinkSuite "deltaLog should not be initialized after constructor") } } + + test("DeltaSink rejects DataFrame with UDT containing NullType") { + failAfter(streamingTimeout) { + withTempDirs { (outputDir, checkpointDir) => + val inputData = MemoryStream[Int] + val ds = inputData.toDS() + val dsWriter = + ds.map(i => (i, new NullData())) + .toDF("id", "value") + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("delta") + + val wrapperException = intercept[StreamingQueryException] { + val q = dsWriter.start(outputDir.getCanonicalPath) + inputData.addData(42) + q.processAllAvailable() + } + assert(wrapperException.cause.isInstanceOf[AnalysisException]) + checkError( + wrapperException.cause.asInstanceOf[AnalysisException], + "DELTA_NULL_SCHEMA_IN_STREAMING_WRITE") + } + } + } } // Batch sizes 1, 2, and 100 exercise different backfill behaviors in the commit coordinator. From 7f55e7602e6c2e575fe8660df7a49f446beda3aa Mon Sep 17 00:00:00 2001 From: Timothy Wang Date: Tue, 17 Mar 2026 11:24:49 -0400 Subject: [PATCH 40/80] [Delta-Spark] Extend stagingCatalog for non-Spark session catalog (#6166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🥞 Stacked PR Use this [link](https://github.com/delta-io/delta/pull/6166/files) to review incremental changes. - [**atomic-rtas**](https://github.com/delta-io/delta/pull/6166) [[Files changed](https://github.com/delta-io/delta/pull/6166/files)] - [atomic-rtas-ci](https://github.com/delta-io/delta/pull/6233) [[Files changed](https://github.com/delta-io/delta/pull/6233/files/a85f6ff586dc587fa09bfc3f94b0b930631a40a9..0606404eb15233b4744f487a4cba428baf6a3118)] --------- #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description **Unity Catalog side PR:** https://github.com/unitycatalog/unitycatalog/pull/1373 Currently, `REPLACE TABLE` and `REPLACE TABLE AS SELECT` through the `stagingTable` only works for the default Spark Session Catalog (HMS for OSS). However, we want to make this REPLACE path be able to talk to non-Spark session catalogs such as UC. This PR enables this communication to non-Spark session catalogs. We configure the spark shell like this: ```sh "$SPARK_HOME/bin/spark-shell" --master "local[*]" \ --jars "$KERNEL_UC_JAR" \ --packages "$DELTA_PKGS" \ --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ --conf "spark.sql.catalog.main=io.unitycatalog.spark.UCSingleCatalog" \ --conf "spark.sql.catalog.main.uri=${UC_URI%/}" \ --conf "spark.sql.catalog.main.token=${UC_TOKEN}" \ --conf "spark.sql.defaultCatalog=main" \ ``` **note:** the `"spark.sql.catalog.main=io.unitycatalog.spark.UCSingleCatalog"` if we run the following REPLACE operation ```sql REPLACE TABLE main.demo_zh.t_rtas USING DELTA TBLPROPERTIES ('delta.feature.catalogManaged'='supported') AS SELECT 2 AS i, 'new' AS s; ``` this operation will fail because it is not looking at the specified catalog `main`. See here where it tries to get the existing table we want to replace: [AbstractDeltaCatalog.scala#L161](https://github.com/delta-io/delta/blob/master/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala#L161) ```scala val existingTableOpt = getExistingTableIfExists(id) ``` and it is asking `catalog: SessionCatalog` for this information: https://github.com/delta-io/delta/blob/master/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala#L991 ```scala protected lazy val catalog: SessionCatalog = spark.sessionState.catalog ``` So this PR ensures that if the `SessionCatalog` doesn't have the CatalogTable information of the table we want to replace, it will query `UCProxy` -> UC server for this information (containing tableId, location, credentials). > it asks the delegated catalog to materialize the existing target as a CatalogTable, and then uses that existing table metadata to drive replace. Then, with a CatalogTable filled with the properties like tableId, location, credentials, it can perform the remaining operations for REPLACE. **TLDR:** Enable staged Delta `REPLACE TABLE`, `REPLACE TABLE AS SELECT`, and `CREATE OR REPLACE TABLE` to resolve the existing target from delegated catalogs such as Unity Catalog when `SessionCatalog` is insufficient, and preserve existing UC table metadata/location during replace, including a narrow UC external Delta replace path. ## How was this patch tested? - Preserve UC-managed table identity across REPLACE, RTAS, and CREATE OR REPLACE. - Route existing-table replace flows through the delegated catalog instead of falling back to the V1 session catalog. - Block illegal UC-managed metadata changes while allowing same-metadata replace operations. - Preserve catalog-owned / coordinated-commit state that must survive managed replace. - Add repo-local coverage for managed replace atomicity, identity preservation, and kill-switch behavior. ## Does this PR introduce _any_ user-facing changes? Yes. Compared to the target branch, existing catalog-managed Delta tables can now support atomic data-only replace without redefining table metadata. Metadata-changing replace remains unsupported, and `CREATE OR REPLACE TABLE` on a missing catalog-managed table now fails instead of behaving like create. --- .../spark/sql/delta/DeltaAnalysis.scala | 6 +- .../sql/delta/OptimisticTransaction.scala | 47 +++++-- .../delta/catalog/AbstractDeltaCatalog.scala | 88 +++++++++++-- .../commands/CreateDeltaTableCommand.scala | 7 +- .../delta/commands/CreateDeltaTableLike.scala | 21 ++- .../CoordinatedCommitsUtils.scala | 12 ++ .../sql/delta/DeltaCreateTableLikeSuite.scala | 114 ++++++++++++++++- .../CatalogOwnedPropertySuite.scala | 120 +++++++++++++++++- .../UCCommitCoordinatorClient.java | 8 +- 9 files changed, 384 insertions(+), 39 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala index 1d82514fc20..96b7d4aaa9a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala @@ -251,7 +251,10 @@ class DeltaAnalysis(session: SparkSession) protocol } val newDeltaCatalog = new DeltaCatalogV1() - val existingTableOpt = newDeltaCatalog.getExistingTableIfExists(catalogTableTarget.identifier) + val existingTableOpt = newDeltaCatalog.getExistingTableIfExists( + catalogTableTarget.identifier, + identOpt = None, + operation = TableCreationModes.Create) val newTable = newDeltaCatalog .verifyTableAndSolidify( catalogTableTarget, @@ -1583,4 +1586,3 @@ case class DeltaDynamicPartitionOverwriteCommand( ).run(sparkSession) } } - diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index db67d28d238..272e56a80de 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -574,6 +574,7 @@ trait OptimisticTransactionImpl extends TransactionHelper "Cannot update the metadata in a transaction that has already written data.") assert(newMetadata.isEmpty, "Cannot change the metadata more than once in a transaction.") + updateMetadataInternal(proposedNewMetadata, ignoreDefaultProperties) // Temporary: block metadata changes on UC-managed CatalogOwned tables until Delta supports // propagating metadata updates to UC. UC is identified by catalog implementation class (handles // "spark_catalog" registration). New table creation is naturally excluded because @@ -583,16 +584,36 @@ trait OptimisticTransactionImpl extends TransactionHelper // catches Delta-internal additions (e.g. table-feature flags). This is acceptable for // a temporary kill switch - once Delta supports propagating metadata updates to UC, // this check will be removed entirely. - val existingMetadata = snapshot.metadata - if (isUCManagedTable && - (proposedNewMetadata.schemaString != existingMetadata.schemaString || - proposedNewMetadata.partitionColumns != existingMetadata.partitionColumns || - proposedNewMetadata.description != existingMetadata.description || - proposedNewMetadata.configuration != existingMetadata.configuration)) { + if (!isCreatingNewTable) { + throwIfUCManagedMetadataChanged(snapshot.metadata, context = "updateMetadata") + } + } + + /** + * Returns true if the proposed metadata differs from the existing metadata for a UC-managed + * table. + */ + private def hasUCManagedMetadataChange( + existingMetadata: Metadata, + proposedMetadata: Metadata): Boolean = { + proposedMetadata.schemaString != existingMetadata.schemaString || + proposedMetadata.partitionColumns != existingMetadata.partitionColumns || + proposedMetadata.description != existingMetadata.description || + proposedMetadata.configuration != existingMetadata.configuration + } + + private def throwIfUCManagedMetadataChanged( + existingMetadata: Metadata, + context: String): Unit = { + val proposedMetadata = newMetadata.getOrElse(existingMetadata) + if (isUCManagedTable && hasUCManagedMetadataChange(existingMetadata, proposedMetadata)) { + logWarning(log"Blocking UC-managed metadata update during " + + log"${MDC(DeltaLogKeys.OPERATION, context)} because metadata changed: " + + log"${MDC(DeltaLogKeys.METADATA_OLD, existingMetadata)} => " + + log"${MDC(DeltaLogKeys.METADATA_NEW, proposedMetadata)}") throw DeltaErrors.operationNotSupportedException( "Metadata changes on Unity Catalog managed tables") } - updateMetadataInternal(proposedNewMetadata, ignoreDefaultProperties) } /** @@ -928,6 +949,8 @@ trait OptimisticTransactionImpl extends TransactionHelper CoordinatedCommitsUtils.getExplicitCCConfigurations(snapshot.metadata.configuration) val existingICTConfs = CoordinatedCommitsUtils.getExplicitICTConfigurations(snapshot.metadata.configuration) + val existingQoLConfs = + CoordinatedCommitsUtils.getExplicitQoLConfigurations(snapshot.metadata.configuration) val oldMappingMode = snapshot.metadata.columnMappingMode val newMappingMode = metadata.columnMappingMode val shouldReuseColumnMetadataForReplaceTable = @@ -943,8 +966,11 @@ trait OptimisticTransactionImpl extends TransactionHelper // to remove them and retain the Coordinated Commits configurations from the existing table. val newConfsWithoutCC = newMetadata.get.configuration -- CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS - var newConfs: Map[String, String] = newConfsWithoutCC ++ existingCCConfs ++ - existingUCTableIdConf + val existingQoLConfsToRetain = existingQoLConfs.filterNot { case (key, _) => + newConfsWithoutCC.contains(key) + } + var newConfs: Map[String, String] = + newConfsWithoutCC ++ existingCCConfs ++ existingQoLConfsToRetain ++ existingUCTableIdConf // We also need to retain the existing ICT dependency configurations, but only when the // existing table does have Coordinated Commits configurations or Catalog-Owned enabled. // Otherwise, we treat the ICT configurations the same as any other configurations, @@ -956,6 +982,9 @@ trait OptimisticTransactionImpl extends TransactionHelper newConfs = newConfsWithoutICT ++ existingICTConfs } newMetadata = Some(newMetadata.get.copy(configuration = newConfs)) + throwIfUCManagedMetadataChanged( + snapshot.metadata, + context = "updateMetadataForNewTableInReplace") } /** diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala index c6e9d07e329..c49648db277 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala @@ -155,10 +155,20 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension Option(allTableProperties.get("location")) } val id = { - TableIdentifier(ident.name(), ident.namespace().lastOption) + // Preserve the catalog name in the V1 identifier for Unity Catalog because this `id` + // becomes `tableDesc.identifier` for the rest of the create/replace flow. Downstream + // catalog-update logic reads `table.identifier.catalog`; without it, the table looks like + // an unqualified V1 table and catalog updates route through the current/session catalog + // instead of the delegated Unity Catalog entry. + val base = TableIdentifier(ident.name(), ident.namespace().lastOption) + if (isUnityCatalog) { + base.copy(catalog = Some(name())) // `name()` here is the catalog name. + } else { + base + } } var locUriOpt = location.map(CatalogUtils.stringToURI) - val existingTableOpt = getExistingTableIfExists(id) + val existingTableOpt = getExistingTableIfExists(id, Some(ident), operation) // PROP_IS_MANAGED_LOCATION indicates that the table location is not user-specified but // system-generated. The table should be created as managed table in this case. val isManagedLocation = Option(allTableProperties.get(TableCatalog.PROP_IS_MANAGED_LOCATION)) @@ -558,8 +568,19 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension properties = validatedConfigurations) } - /** Checks if a table already exists for the provided identifier. */ - def getExistingTableIfExists(table: TableIdentifier): Option[CatalogTable] = { + /** + * Checks if a Delta table already exists for the provided identifier. + * + * This first goes through the legacy V1 SessionCatalog/HMS lookup using [[TableIdentifier]]. + * For operations that target an existing Unity Catalog table, it then falls back to the + * delegated V2 catalog plugin lookup (for example Unity Catalog) when the V1 lookup does not + * surface the table entry. + */ + def getExistingTableIfExists( + table: TableIdentifier, + identOpt: Option[Identifier], + operation: TableCreationModes.CreationMode) + : Option[CatalogTable] = { // If this is a path identifier, we cannot return an existing CatalogTable. The Create command // will check the file system itself if (isPathIdentifier(table)) return None @@ -573,6 +594,50 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension throw DeltaErrors.notADeltaTable(table.table) } Some(oldTable) + } else if (operation != TableCreationModes.Create) { + identOpt match { + case Some(ident) => getExistingTableFromDelegatedCatalog(ident) + case None => + logDebug(log"Delegated catalog lookup skipped because no V2 identifier was provided " + + log"for ${MDC(DeltaLogKeys.TABLE_NAME, table)} during ${MDC(DeltaLogKeys.OPERATION, + operation.toString)}.") + None + } + } else { + None + } + } + + /** + * Returns an existing Delta table by querying the delegated catalog for Unity Catalog paths + * where the V1 SessionCatalog lookup does not surface the existing table entry. + * + * [[getExistingTableIfExists]] first checks the V1 catalog using a [[TableIdentifier]]. For + * Unity Catalog, some staged create/replace paths need a delegated V2 catalog lookup on the + * original [[Identifier]] to recover the existing table metadata. + * + * Even though this goes through the delegated V2 catalog plugin, Spark can still surface the + * result as a [[V1Table]] wrapper when the delegated catalog is exposing V1-backed table + * metadata. In that case we unwrap the embedded [[CatalogTable]] and continue on the V1 Delta + * path. + */ + private def getExistingTableFromDelegatedCatalog(ident: Identifier): Option[CatalogTable] = { + if (isUnityCatalog) { + try { + super.loadTable(ident) match { + case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => + Some(v1.catalogTable) + case _ => + logDebug(log"Delegated catalog lookup for ${MDC(DeltaLogKeys.TABLE_NAME, ident)} " + + log"did not return a Delta table.") + None + } + } catch { + case _: NoSuchTableException => + logDebug(log"Delegated catalog lookup did not find an existing table for " + + log"${MDC(DeltaLogKeys.TABLE_NAME, ident)}.") + None + } } else { None } @@ -617,11 +682,11 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension } /** - * The UC table ID property was renamed from an old name. In a transition period we need to - * translate the old UC table ID property name set by caller to new one. And in case both the new - * and old properties are set, remove the old one. Later in UC server it might throw error if it - * sees both. - * TODO: clean up once callers are migrated. + * Normalizes the deprecated UC table ID property key to the canonical key. + * + * This is temporary compatibility for callers that still send the old key during the rename + * transition. If both keys are present, we drop the old one and keep the canonical key only. + * TODO(issue #6296): remove once all callers stop sending the deprecated key. */ private def translateUCTableIdProperty(props: util.Map[String, String]): Unit = { val oldTableIdProperty = Option(props.remove(UC_TABLE_ID_KEY_OLD)) @@ -654,6 +719,11 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension writeOptions = sqlWriteOptions } expandTableProps(props, writeOptions, conf) + if (isUnityCatalog) { + // Unity Catalog callers may still send the deprecated `ucTableId` property key. + // Normalize it here to the canonical `io.unitycatalog.tableId` key before create/replace. + translateUCTableIdProperty(props) + } createDeltaTable( ident, schema, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala index b471f787225..f23fd6937e7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala @@ -786,6 +786,11 @@ case class CreateDeltaTableCommand( newMetadata.configuration, txn.snapshot) newMetadata = newMetadata.copy(configuration = updatedConfig) + if (allowCatalogManaged && txn.snapshot.isCatalogOwned) { + // Preserve the existing Delta metadata id across REPLACE. This is distinct from the + // Unity Catalog table id stored in `io.unitycatalog.tableId`. + newMetadata = newMetadata.copy(id = txn.snapshot.metadata.id) + } txn.updateMetadataForNewTableInReplace(newMetadata) } } @@ -802,7 +807,7 @@ case class CreateDeltaTableCommand( deltaLog: DeltaLog, tableWithLocation: CatalogTable, snapshotOpt: Option[Snapshot] = None): OptimisticTransaction = { - val txn = deltaLog.startTransaction(None, snapshotOpt) + val txn = deltaLog.startTransaction(existingTableOpt, snapshotOpt) validatePrerequisitesForClusteredTable(txn.snapshot.protocol, txn.deltaLog) // During CREATE (not REPLACE/overwrites), we synchronously run conversion diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala index 2bed9506ac1..d0e38f95e65 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala @@ -107,15 +107,26 @@ trait CreateDeltaTableLike extends SQLConfHelper { } case TableCreationModes.Replace | TableCreationModes.CreateOrReplace if existingTableOpt.isDefined => - UpdateCatalogFactory.getUpdateCatalogHook(table, spark).updateSchema(spark, snapshot) + // Catalog-managed / CC tables are owned by the delegated V2 catalog plugin (for example + // Unity Catalog), so SessionCatalog's post-commit UpdateCatalogHook must not run. + if (!allowCatalogManaged) { + UpdateCatalogFactory.getUpdateCatalogHook(table, spark).updateSchema(spark, snapshot) + } case TableCreationModes.Replace => val ident = Identifier.of(table.identifier.database.toArray, table.identifier.table) throw DeltaErrors.cannotReplaceMissingTableException(ident) case TableCreationModes.CreateOrReplace => - spark.sessionState.catalog.createTable( - cleaned, - ignoreIfExists = false, - validateLocation = false) + createTableFunc match { + case Some(createFunc) => + // This is the new missing-table path where creation is delegated through the V2 + // catalog plugin (for example Unity Catalog) instead of SessionCatalog.createTable(). + createFunc(cleaned) + case None => + spark.sessionState.catalog.createTable( + cleaned, + ignoreIfExists = false, + validateLocation = false) + } } if (conf.getConf(DeltaSQLConf.HMS_FORCE_ALTER_TABLE_DATA_SCHEMA)) { spark.sessionState.catalog.alterTableDataSchema(cleaned.identifier, cleaned.schema) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala index 0b3c438685d..5e51c131104 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala @@ -771,6 +771,18 @@ object CoordinatedCommitsUtils extends DeltaLogging { properties.filter { case (k, _) => ICT_TABLE_PROPERTY_KEYS.contains(k) } } + /** + * Extracts the explicit QoL configurations from the provided properties. + * + * These are preserved across catalog-managed REPLACE when the existing table already has the + * QoL defaults materialized in metadata, so a no-op REPLACE does not accidentally drop them + * while rebuilding the configuration map. + */ + def getExplicitQoLConfigurations(properties: Map[String, String]): Map[String, String] = { + val qolKeys = CatalogOwnedTableUtils.QOL_TABLE_FEATURES_AND_PROPERTIES.map(_._2.key).toSet + properties.filter { case (k, _) => qolKeys.contains(k) } + } + /** * Fetches the SparkSession default configurations for ICT. The `withDefaultKey` * flag controls whether the keys in the returned map should have the default prefix or not. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala index ec1abc4a5ef..396822e2068 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala @@ -17,12 +17,15 @@ package org.apache.spark.sql.delta import java.io.File -import java.net.URI import java.util.UUID import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes} -import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.delta.commands.{ + CreateDeltaTableCommand, + CreateDeltaTableLike, + TableCreationModes +} +import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.scalatest.exceptions.TestFailedException import org.apache.spark.sql.QueryTest @@ -35,7 +38,8 @@ import org.apache.spark.sql.types.StructType class DeltaCreateTableLikeSuite extends QueryTest with SharedSparkSession - with DeltaSQLCommandTest { + with DeltaSQLCommandTest + with DeltaSQLTestUtils { def checkTableEmpty(tblName: String): Boolean = { val numRows = spark.sql(s"SELECT * FROM $tblName") @@ -289,7 +293,7 @@ class DeltaCreateTableLikeSuite extends QueryTest withTable("t") { def getCatalogTable: CatalogTable = { val storage = CatalogStorageFormat.empty.copy( - locationUri = Some(new URI(s"$dir/${UUID.randomUUID().toString}"))) + locationUri = Some(dir.toPath.resolve(UUID.randomUUID().toString).toUri)) val catalogTableTarget = CatalogTable( identifier = TableIdentifier("t"), tableType = CatalogTableType.MANAGED, @@ -320,6 +324,102 @@ class DeltaCreateTableLikeSuite extends QueryTest } } + test("catalog-managed CREATE OR REPLACE creates missing tables") { + withTempDir { dir => + withTable("t") { + val storage = CatalogStorageFormat.empty.copy( + locationUri = Some(dir.toPath.resolve(UUID.randomUUID().toString).toUri)) + val catalogTableTarget = CatalogTable( + identifier = TableIdentifier("t"), + tableType = CatalogTableType.MANAGED, + storage = storage, + provider = Some("delta"), + schema = new StructType().add("id", "long")) + val command = CreateDeltaTableCommand( + new DeltaCatalog().verifyTableAndSolidify( + tableDesc = catalogTableTarget, + query = None, + maybeClusterBySpec = None), + existingTableOpt = None, + mode = SaveMode.ErrorIfExists, + query = None, + operation = TableCreationModes.CreateOrReplace, + allowCatalogManaged = true, + createTableFunc = None) + + command.run(spark) + assert(spark.sessionState.catalog.tableExists(TableIdentifier("t"))) + } + } + } + + test("catalog-managed CREATE OR REPLACE skips catalog create callback " + + "when metadata is unchanged") { + withCatalogManagedTable(createTable = false) { tableName => + spark.sql( + s"""CREATE TABLE $tableName (id LONG) USING DELTA + |TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported') + |""".stripMargin) + + val existingTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) + val snapshot = DeltaLog.forTable(spark, existingTable).update() + var createCallbackCalls = 0 + + val command = new CreateDeltaTableLike { + override val table: CatalogTable = existingTable + override val existingTableOpt: Option[CatalogTable] = Some(existingTable) + override val operation: TableCreationModes.CreationMode = + TableCreationModes.CreateOrReplace + override val mode: SaveMode = SaveMode.ErrorIfExists + override val allowCatalogManaged: Boolean = true + + def runUpdateCatalog(): Unit = { + updateCatalog( + spark, + table, + snapshot, + query = None, + didNotChangeMetadata = true, + createTableFunc = Some((_: CatalogTable) => { + createCallbackCalls += 1 + })) + } + } + + command.runUpdateCatalog() + assert(createCallbackCalls === 0) + } + } + + test("catalog-managed CREATE OR REPLACE rejects query-derived nullable schema") { + withCatalogManagedTable(createTable = false) { tableName => + withTable("source") { + spark.sql( + s"""CREATE TABLE $tableName (id LONG NOT NULL) USING DELTA + |TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported') + |""".stripMargin) + spark.sql(s"INSERT INTO $tableName VALUES (1)") + spark.sql("CREATE TABLE source (id LONG) USING DELTA") + spark.sql("INSERT INTO source VALUES (2)") + + val existingTable = + spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) + val versionBefore = DeltaLog.forTable(spark, existingTable).update().version + val query = spark.sql("SELECT id FROM source").logicalPlan + val err = intercept[AssertionError] { + new DeltaCatalog().verifyTableAndSolidify( + tableDesc = existingTable.copy(schema = query.schema.asNullable), + query = Some(query), + maybeClusterBySpec = None) + } + + assert(err.getMessage.contains("Can't specify table schema in CTAS.")) + assert(DeltaLog.forTable(spark, existingTable).update().version === versionBefore) + checkAnswer(spark.sql(s"SELECT * FROM $tableName"), Seq(org.apache.spark.sql.Row(1L))) + } + } + } + test("CREATE TABLE LIKE where sourceTable is a json table") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" @@ -370,7 +470,7 @@ class DeltaCreateTableLikeSuite extends QueryTest test("CREATE TABLE LIKE where target table is a named external table") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" - withTempDir { dir => + withTempDir(prefix = "sparkdirprefix") { dir => withTable(srcTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl LOCATION '${dir.toURI.toString}'") @@ -385,7 +485,7 @@ class DeltaCreateTableLikeSuite extends QueryTest withTable(srcTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE delta.`${dir.toURI.toString}` LIKE $srcTbl") - checkTableCopyDelta(srcTbl, dir.toString, checkTargetTableByPath = true + checkTableCopyDelta(srcTbl, dir.toURI.toString, checkTargetTableByPath = true ) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala index 87324dc4db1..a1cc206e5ad 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala @@ -116,6 +116,11 @@ class CatalogOwnedPropertySuite extends QueryTest ucTableId.get } + private def getSnapshotVersion(tableName: String): Long = { + val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) + snapshot.version + } + private def validateInCommitTimestampTableFeature(tableName: String, expected: Boolean): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) val writerFeatureNames = snapshot.protocol.writerFeatureNames @@ -241,6 +246,7 @@ class CatalogOwnedPropertySuite extends QueryTest // CatalogManaged enabled table. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + val versionBefore = getSnapshotVersion(tableName = "t1") // Specifying CatalogManaged on an already CatalogManaged table should succeed. // The CatalogManaged properties are treated as a no-op. @@ -251,6 +257,7 @@ class CatalogOwnedPropertySuite extends QueryTest validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) } } @@ -280,6 +287,7 @@ class CatalogOwnedPropertySuite extends QueryTest // CatalogManaged enabled table. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + val versionBefore = getSnapshotVersion(tableName = "t1") // Source RTAS table. createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = false) @@ -294,6 +302,48 @@ class CatalogOwnedPropertySuite extends QueryTest validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) + } + } + + test("[RTAS] failed replace preserves existing catalog-managed table data and version") { + withTable("t1") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (1), (2)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + intercept[Exception] { + sql( + """REPLACE TABLE t1 USING delta AS + |SELECT IF(id = 2L, CAST(raise_error('boom') AS BIGINT), id) AS id + |FROM VALUES (1L), (2L) AS src(id) + |""".stripMargin) + } + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore) + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) + } + } + + test("[RTAS] replacing an existing catalog-managed table preserves UC identity") { + withTable("t1", "t2") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (10)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = false) + sql("INSERT INTO t2 VALUES (1), (2)") + + sql("REPLACE TABLE t1 USING delta AS SELECT * FROM t2") + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) } } @@ -319,6 +369,7 @@ class CatalogOwnedPropertySuite extends QueryTest createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (1)") + val versionBefore = getSnapshotVersion(tableName = "t1") // CREATE OR REPLACE with CatalogManaged on existing CatalogManaged table should succeed. sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + @@ -328,6 +379,65 @@ class CatalogOwnedPropertySuite extends QueryTest validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) + } + } + + test("[CREATE OR REPLACE] replacing an existing catalog-managed table is atomic") { + withTable("t1") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (1)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + sql("CREATE OR REPLACE TABLE t1 USING delta TBLPROPERTIES " + + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported') AS " + + "SELECT 2 AS id") + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(2))) + } + } + + test("[REPLACE] replacing an existing catalog-managed table preserves UC identity") { + withTable("t1") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (1)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + sql("REPLACE TABLE t1 (id LONG) USING delta") + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) + checkAnswer(sql("SELECT * FROM t1"), Seq.empty) + } + } + + test("[CREATE OR REPLACE] specifying table UUID for existing catalog-managed table " + + "should be blocked") { + withTable("t1") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (1)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + val error = intercept[DeltaUnsupportedOperationException] { + sql("CREATE OR REPLACE TABLE t1 USING delta TBLPROPERTIES " + + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported', " + + s"'${UCCommitCoordinatorClient.UC_TABLE_ID_KEY}' = '${UUID.randomUUID().toString}') " + + "AS SELECT 2 AS id") + } + checkError(error, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", "42939", + Map("prop" -> "io.unitycatalog.tableId")) + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore) + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1))) } } @@ -352,7 +462,7 @@ class CatalogOwnedPropertySuite extends QueryTest } } - test("[CREATE OR REPLACE] continuous CREATE OR REPLACE with CatalogManaged " + + test("[CREATE OR REPLACE] repeated same-schema CREATE OR REPLACE with CatalogManaged " + "should succeed repeatedly") { withTable("t1") { // First CREATE OR REPLACE creates the table. @@ -372,16 +482,16 @@ class CatalogOwnedPropertySuite extends QueryTest val ucTableIdSecond = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdFirst === ucTableIdSecond) - // Third CREATE OR REPLACE should also succeed. - sql("CREATE OR REPLACE TABLE t1 (id LONG, name STRING) USING delta TBLPROPERTIES " + + // Third CREATE OR REPLACE should also succeed when the metadata is unchanged. + sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdThird = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdFirst === ucTableIdThird) // Verify table is functional. - sql("INSERT INTO t1 VALUES (1, 'a'), (2, 'b')") - checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1, "a"), Row(2, "b"))) + sql("INSERT INTO t1 VALUES (2)") + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(2))) } } test("[CREATE LIKE] Specifying table UUID should be blocked") { diff --git a/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java b/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java index e991b17785c..b1fd4dbf270 100644 --- a/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java +++ b/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java @@ -71,6 +71,12 @@ public UCCommitCoordinatorClient(Map conf, UCClient ucClient) { /** Key used to identify the write version in protocol communications with the UC server. */ private static final String WRITE_VERSION_KEY = "writeVersion"; + /** + * Temporary kill switch for sending metadata updates through UC from the Spark path. + * TODO(issue #6296): remove once metadata updates are supported end-to-end. + */ + private static final boolean SHOULD_PASS_METADATA_TO_UC = false; + // Unity Catalog Identifiers /** * Key for identifying Unity Catalog table ID in `delta.coordinatedCommits.tableConf{-preview}`. @@ -429,7 +435,7 @@ protected CommitResponse commitImpl( Optional.of(commitTimestamp), Optional.of(lastKnownBackfilledVersion.get()), disown, - updatedActions.getNewMetadata() == updatedActions.getOldMetadata() ? + updatedActions.getNewMetadata() == updatedActions.getOldMetadata() || !SHOULD_PASS_METADATA_TO_UC ? Optional.empty() : Optional.of(updatedActions.getNewMetadata()), updatedActions.getNewProtocol() == updatedActions.getOldProtocol() ? From fdde3410abe9abf9d786ff8526264ede4d0f735e Mon Sep 17 00:00:00 2001 From: Zikang Han Date: Tue, 17 Mar 2026 17:32:46 -0700 Subject: [PATCH 41/80] [kernel-spark] Fix a potential resource leak in SparkMicroBatchStream (#6297) --- .../v2/read/SparkMicroBatchStream.java | 44 +++++- .../v2/read/SparkMicroBatchStreamTest.java | 138 ++++++++++++++++++ 2 files changed, 176 insertions(+), 6 deletions(-) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java index c37bd9f4c59..56872ce63b2 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java @@ -845,12 +845,18 @@ private CloseableIterator processCommitToIndexedFiles( shouldSkipCommit ? Utils.toCloseableIterator(Collections.emptyIterator()) : getFilesFromCommit(commit, version); - return Utils.singletonCloseableIterator( - new IndexedFile(version, DeltaSourceOffset.BASE_INDEX(), /* addFile= */ null)) - .combine(fileActions) - .combine( - Utils.singletonCloseableIterator( - new IndexedFile(version, DeltaSourceOffset.END_INDEX(), /* addFile= */ null))); + CloseableIterator inner = + Utils.singletonCloseableIterator( + new IndexedFile(version, DeltaSourceOffset.BASE_INDEX(), /* addFile= */ null)) + .combine(fileActions) + .combine( + Utils.singletonCloseableIterator( + new IndexedFile( + version, DeltaSourceOffset.END_INDEX(), /* addFile= */ null))); + + // Wrap the iterator so that closing it also closes the CommitActions, releasing its + // internal ActionsIterator and any associated file handles / parsed data. + return wrapIteratorWithCommitClose(inner, commit); } catch (Exception e) { // commit is not a CloseableIterator, we need to close it manually. Utils.closeCloseables(commit); @@ -858,6 +864,32 @@ private CloseableIterator processCommitToIndexedFiles( } } + /** + * Wraps an iterator so that closing it also closes the given {@link CommitActions}, releasing its + * internal resources. This is necessary because {@link CommitActions} is an {@link AutoCloseable} + * but not a {@link CloseableIterator}, so closing the iterator chain alone does not close the + * {@link CommitActions} that produced it. Package-private for testing. + */ + static CloseableIterator wrapIteratorWithCommitClose( + CloseableIterator inner, CommitActions commit) { + return new CloseableIterator() { + @Override + public boolean hasNext() { + return inner.hasNext(); + } + + @Override + public IndexedFile next() { + return inner.next(); + } + + @Override + public void close() throws IOException { + Utils.closeCloseables(inner, commit); + } + }; + } + private CloseableIterator getFilesFromCommit(CommitActions commit, long version) { // Assign each IndexedFile a unique index within the commit. We use a mutable array // because variables captured by a lambda must be effectively final (never reassigned). diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java index 3584f9727c8..388e576148c 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java @@ -18,6 +18,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; +import io.delta.kernel.CommitActions; +import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; @@ -33,6 +35,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -3640,4 +3643,139 @@ public void testFindClosedByInterruptCause() { "readJsonFile", new java.io.FileNotFoundException("missing")))) .isEmpty(); } + + /** Regression test: closing the wrapped iterator must also close CommitActions. */ + @Test + public void testWrapIteratorWithCommitClose_closesCommitOnIteratorClose() throws Exception { + AtomicBoolean commitClosed = new AtomicBoolean(false); + AtomicBoolean innerClosed = new AtomicBoolean(false); + + CloseableIterator inner = + new CloseableIterator() { + private boolean consumed = false; + + @Override + public boolean hasNext() { + return !consumed; + } + + @Override + public IndexedFile next() { + consumed = true; + return new IndexedFile(/* version= */ 1L, /* index= */ 0L, /* addFile= */ null); + } + + @Override + public void close() { + innerClosed.set(true); + } + }; + + try (CloseableIterator wrapped = + SparkMicroBatchStream.wrapIteratorWithCommitClose( + inner, newTrackingCommitActions(commitClosed))) { + while (wrapped.hasNext()) { + wrapped.next(); + } + } + + assertTrue(innerClosed.get(), "Inner iterator should be closed"); + assertTrue(commitClosed.get(), "CommitActions should be closed"); + } + + /** Closing the wrapped iterator before full consumption must still close CommitActions. */ + @Test + public void testWrapIteratorWithCommitClose_closesCommitOnEarlyClose() throws Exception { + AtomicBoolean commitClosed = new AtomicBoolean(false); + + CloseableIterator inner = + new CloseableIterator() { + private int remaining = 10; + + @Override + public boolean hasNext() { + return remaining > 0; + } + + @Override + public IndexedFile next() { + remaining--; + return new IndexedFile(/* version= */ 1L, /* index= */ remaining, /* addFile= */ null); + } + + @Override + public void close() {} + }; + + try (CloseableIterator wrapped = + SparkMicroBatchStream.wrapIteratorWithCommitClose( + inner, newTrackingCommitActions(commitClosed))) { + assertTrue(wrapped.hasNext()); + wrapped.next(); + // Intentionally don't consume the rest + } + + assertTrue(commitClosed.get(), "CommitActions should be closed on early termination"); + } + + /** If inner.close() throws, CommitActions must still be closed. */ + @Test + public void testWrapIteratorWithCommitClose_closesCommitEvenWhenInnerCloseThrows() + throws Exception { + AtomicBoolean commitClosed = new AtomicBoolean(false); + + CloseableIterator inner = + new CloseableIterator() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public IndexedFile next() { + throw new java.util.NoSuchElementException(); + } + + @Override + public void close() { + throw new RuntimeException("inner close failed"); + } + }; + + CloseableIterator wrapped = + SparkMicroBatchStream.wrapIteratorWithCommitClose( + inner, newTrackingCommitActions(commitClosed)); + try { + wrapped.close(); + } catch (RuntimeException e) { + // Expected — inner.close() threw + } + + assertTrue(commitClosed.get(), "CommitActions should be closed even when inner.close() throws"); + } + + /** Creates a CommitActions stub that sets {@code closedFlag} to true on close. */ + private static CommitActions newTrackingCommitActions(AtomicBoolean closedFlag) { + return new CommitActions() { + @Override + public long getVersion() { + return 1L; + } + + @Override + public long getTimestamp() { + return 0L; + } + + @Override + public CloseableIterator getActions() { + throw new UnsupportedOperationException("not needed for this test"); + } + + @Override + public void close() { + closedFlag.set(true); + } + }; + } } From 73a0edb8c1beca3644c738839a3807f5d7a54f89 Mon Sep 17 00:00:00 2001 From: Zhipeng Mao Date: Wed, 18 Mar 2026 19:38:45 +0100 Subject: [PATCH 42/80] [Spark] Enable MERGE and INSERT null expansion fix by default (#6311) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description It enables null expansion fix for MERGE and INSERT by default. ## How was this patch tested? Existing tests. ## Does this PR introduce _any_ user-facing changes? Currently, when you insert or merge a NULL struct into a Delta table with schema evolution enabled, the system expands that NULL struct into a non-NULL struct with all fields set to NULL. This behavior does not align with expected SQL semantics. With this change we will preserve NULL structs as-is in your target tables. This means a NULL struct will remain NULL rather than being converted to a non-NULL struct with NULL fields. --- .../suitegenerator/SuiteGeneratorConfig.scala | 10 +- .../sql/delta/sources/DeltaSQLConf.scala | 4 +- ...ergeIntoStructEvolutionNullnessSuite.scala | 22 +---- ...tedArrayStructEvolutionNullnessTests.scala | 97 +------------------ ...estedMapStructEvolutionNullnessTests.scala | 97 +------------------ ...toNestedStructEvolutionNullnessTests.scala | 97 +------------------ ...uctEvolutionNullnessMultiClauseTests.scala | 97 +------------------ ...velArrayStructEvolutionNullnessTests.scala | 97 +------------------ ...LevelMapStructEvolutionNullnessTests.scala | 97 +------------------ ...TopLevelStructEvolutionNullnessTests.scala | 97 +------------------ 10 files changed, 12 insertions(+), 703 deletions(-) diff --git a/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala b/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala index 568bef53330..cf18c009186 100644 --- a/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala +++ b/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala @@ -148,12 +148,6 @@ object SuiteGeneratorConfig { val DELETE_SCALA = DimensionMixin("DeleteScala", alias = Some("Scala")) val DELETE_SQL = DimensionMixin("DeleteSQL", alias = Some("SQL")) val DELETE_WITH_DVS = DimensionMixin("DeleteSQLWithDeletionVectors", alias = Some("DV")) - val STRUCT_EVOLUTION_PRESERVE_NULL_SOURCE = DimensionWithMultipleValues( - "StructEvolutionPreserveNullSource", - List("Disabled", "Enabled"), alias = Some("PreserveNullSource")) - val STRUCT_EVOLUTION_PRESERVE_NULL_SOURCE_UPDATE_STAR = DimensionWithMultipleValues( - "StructEvolutionPreserveNullSourceUpdateStar", - List("Disabled", "Enabled"), alias = Some("PreserveNullSourceUpdateStar")) } private object Tests { @@ -269,9 +263,7 @@ object SuiteGeneratorConfig { "MergeIntoStructEvolutionNullnessMultiClauseTests" :: Nil, List( List( - Dims.MERGE_SQL, Dims.NAME_BASED, Dims.COLUMN_MAPPING.asOptional, - Dims.STRUCT_EVOLUTION_PRESERVE_NULL_SOURCE, - Dims.STRUCT_EVOLUTION_PRESERVE_NULL_SOURCE_UPDATE_STAR + Dims.MERGE_SQL, Dims.NAME_BASED ) ) ) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index 92fb2f55f6d..c8ff9a9e123 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -752,7 +752,7 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { |and streaming inserts with struct type cast. |""".stripMargin) .booleanConf - .createWithDefault(DeltaUtils.isTesting) + .createWithDefault(true) val DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS_UPDATE_STAR = buildConf("merge.preserveNullSourceStructs.updateStar") @@ -775,7 +775,7 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { |type cast during INSERT operations. |""".stripMargin) .booleanConf - .createWithDefault(DeltaUtils.isTesting) + .createWithDefault(true) val DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED = buildConf("insert.byName.schemaEvolution.enabled") diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala index a9de828e024..5ffa13e24e0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala @@ -31,10 +31,10 @@ import org.apache.spark.sql.types.{ArrayType, IntegerType, StructType} trait MergeIntoStructEvolutionNullnessTestUtils extends MergeHelpers { /** Whether to preserve null source structs for struct evolution tests. */ - protected def preserveNullSourceStructs: Boolean + protected def preserveNullSourceStructs: Boolean = true /** Whether to preserve null source structs for UPDATE * specifically. */ - protected def preserveNullSourceStructsUpdateStar: Boolean + protected def preserveNullSourceStructsUpdateStar: Boolean = true /** Configurations for preserving null source structs. */ protected val preserveNullStructsConfs: Seq[(String, String)] = Seq( @@ -2239,21 +2239,3 @@ trait MergeIntoStructEvolutionNullnessMultiClauseTests expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } - -trait StructEvolutionPreserveNullSourceEnabled extends MergeIntoStructEvolutionNullnessTestUtils { - override protected def preserveNullSourceStructs: Boolean = true -} - -trait StructEvolutionPreserveNullSourceDisabled extends MergeIntoStructEvolutionNullnessTestUtils { - override protected def preserveNullSourceStructs: Boolean = false -} - -trait StructEvolutionPreserveNullSourceUpdateStarEnabled - extends MergeIntoStructEvolutionNullnessTestUtils { - override protected def preserveNullSourceStructsUpdateStar: Boolean = true -} - -trait StructEvolutionPreserveNullSourceUpdateStarDisabled - extends MergeIntoStructEvolutionNullnessTestUtils { - override protected def preserveNullSourceStructsUpdateStar: Boolean = false -} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala index 60499b46e2a..7c7218f1354 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreservG4SCW5ASuite +class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedArrayStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreservO4GSSLISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveEF65NXYSuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveUXQCW4ISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPrese5QFIOCISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPrese7LUJGJASuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreser2UCWDCASuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserY6M56VASuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceU6MQ3SIISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUEG2RWVYSuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpFQ7PINASuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpR36OX5ISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala index 32904042286..8b07e74679b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveN4NGDUBYSuite +class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedMapStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveND64DRCASuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNuG4OZSQISuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNuQSZI2SISuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreservUKDCWWISuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreservVUMRVOQSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveKMOZRLISuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveWJIHVZYSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpd7CQVRRQSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdI3MWPXASuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpda5FZ34QYSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdaH76RPFYSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala index cc5f3e5035a..2031b538824 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveNullXPVWNHQSuite +class MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveNullY6JU3TISuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNullS6EZ266ISuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNullSMTAVKOYSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreserveNu3TOZDYYSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreserveNu6WJICSYSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveNulF3SBMJASuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveNulS5ZNPDISuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdateStarOffSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdateStarOnSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdateStarOffSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdateStarOnSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala index 27d18cea045..4fd48cec149 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapIdModePreserveNullSourceOffPreserv23BA7GQSuite +class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedSuite extends MergeIntoStructEvolutionNullnessMultiClauseTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapIdModePreserveNullSourceOffPreservUYSKTYQSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapIdModePreserveNullSourceOnPreserveB5BD4IASuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapIdModePreserveNullSourceOnPreserveIO32QAASuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapNameModePreserveNullSourceOffPrese2A7DLQISuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapNameModePreserveNullSourceOffPreseY5PJPFYSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapNameModePreserveNullSourceOnPreser7ZAUKFYSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapNameModePreserveNullSourceOnPreserRU3Q5DYSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedPreserveNullSourceOffPreserveNullSourceUGATQTNQSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedPreserveNullSourceOffPreserveNullSourceUWC7MIPQSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpB5L64TISuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpUYKAD7YSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala index a8973377638..a26a2c966ac 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPrese53B5UFQSuite +class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelArrayStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreseMJWNT3ASuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserSDHOQ3YSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserWUUBTSQSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPrePFWPP2ISuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreQ33TTJISuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPres5MRQBEASuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPresT5HAGKASuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourcNGOSKFYSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourcVCDDUUISuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSource6R2VGSQSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceG5G5COQSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala index 73dca4593a5..f6eddd146a0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreservIVRBZBASuite +class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelMapStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreservKAQJEBYSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserve2CQMFJASuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveLRUOIYQSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPrese7ZKRFZQSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreseBJ226TYSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreser5E2PNOASuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreser6UFQNPQSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUMKC5U2ASuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUPRQCVZISuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUp7WIQ4DISuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpGZV4B5YSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala index c05b3f5a6a2..1ce93ba19d5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveNuHUY7GZQSuite +class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveNuP2BPJAASuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNul4SO3XDASuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNulTHLQ4DASuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreserveJI3LTLQSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreserveQXK26DISuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveN3E2WHXISuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveNK7XS44YSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpda565RLXQSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdaFOJBZMYSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdatAY5MSVASuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdateStarOnSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled From 4c3611f3d5c2829073eab69368ab1c8b74b55c90 Mon Sep 17 00:00:00 2001 From: Andrei Tserakhau Date: Wed, 18 Mar 2026 23:26:49 +0100 Subject: [PATCH 43/80] [Spark] Fix CDC non-constant argument detection for correlated subquery expressions (#6310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description When table_changes() is used inside a correlated subquery with an expression referencing an outer column (e.g. o.version + 0), the top-level expression node (Add) is not Unevaluable — but its child OuterReference is. The previous guard only checked isInstanceOf[Unevaluable] on the top-level node, so it passed through, and the subsequent .eval() call threw a SparkException with error class INTERNAL_ERROR instead of the expected user-facing error. This PR replaces the upfront Unevaluable check with a catch on SparkException(INTERNAL_ERROR), which correctly handles this case and re-throws it as DELTA_CDC_NON_CONSTANT_ARGUMENT. ## How was this patch tested? Added a new negative test in DeltaCDCSQLSuite — "negative case - table_changes in correlated subquery with OuterReference" — which verifies that the query throws DELTA_CDC_NON_CONSTANT_ARGUMENT with the correct parameters instead of an internal Spark error. ## Does this PR introduce _any_ user-facing changes? No --- .../sql/delta/DeltaTableValueFunctions.scala | 8 ++--- .../spark/sql/delta/DeltaCDCSQLSuite.scala | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala index c8e4ea9fd9b..f25479a4023 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala @@ -26,10 +26,11 @@ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.DeltaDataSource +import org.apache.spark.SparkException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.{FunctionRegistryBase, NamedRelation, TableFunctionRegistry, UnresolvedLeafNode, UnresolvedRelation} -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExpressionInfo, Literal, StringLiteral, Unevaluable} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExpressionInfo, Literal, StringLiteral} import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, UnaryNode} import org.apache.spark.sql.connector.catalog.V1Table import org.apache.spark.sql.execution.datasources.LogicalRelation @@ -114,12 +115,11 @@ trait CDCStatementBase extends DeltaTableValueFunction { val fakePlan = util.AnalysisHelper.FakeLogicalPlan(Seq(value), Nil) val timestampExpression = org.apache.spark.sql.catalyst.optimizer.ComputeCurrentTime(fakePlan).expressions.head - if (timestampExpression.isInstanceOf[Unevaluable]) { - throw DeltaErrors.cdcNonConstantArgument(fnName, keyPrefix, position, value) - } timestampExpression.eval().toString } catch { case _: NullPointerException => throw DeltaErrors.nullRangeBoundaryInCDCRead() + case e: SparkException if e.getErrorClass == "INTERNAL_ERROR" => + throw DeltaErrors.cdcNonConstantArgument(fnName, keyPrefix, position, value) } value.dataType match { // We dont need to explicitly handle ShortType as it is parsed as IntegerType. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala index 16d0fe00682..58a2947258c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala @@ -294,6 +294,36 @@ class DeltaCDCSQLSuite extends DeltaCDCSuiteBase with DeltaColumnMappingTestUtil } } + test("negative case - table_changes in correlated subquery with OuterReference") { + // When table_changes() is used inside a correlated subquery with an expression that + // wraps an OuterReference (e.g. `o.version + 0`), the top-level node is not Unevaluable + // (Add is evaluable) but its child OuterReference is. The old isInstanceOf[Unevaluable] + // check on the top-level expression misses this case and .eval() throws INTERNAL_ERROR. + // We instead catch that SparkException and re-throw as DELTA_CDC_NON_CONSTANT_ARGUMENT. + val tbl = "tbl" + val otherTbl = "other_tbl" + withTable(tbl, otherTbl) { + spark.range(10).write.format("delta").saveAsTable(tbl) + spark.range(5).toDF("version").write.format("delta").saveAsTable(otherTbl) + + val q = s""" + SELECT * FROM $otherTbl o WHERE EXISTS ( + SELECT 1 FROM table_changes('$tbl', o.version + 0) + ) + """ + checkErrorMatchPVals( + intercept[AnalysisException] { sql(q) }, + "DELTA_CDC_NON_CONSTANT_ARGUMENT", + parameters = Map( + "argumentName" -> "`starting`", + "pos" -> "2", + "functionName" -> "`table_changes`", + "sqlExpr" -> ".*" + ) + ) + } + } + test("resolve expression for timestamp function") { val tbl = "tbl" withDefaultTimeZone(UTC) { From d637549b63cfe86ed86d45a593274c0babe5470b Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Thu, 19 Mar 2026 13:47:28 -0700 Subject: [PATCH 44/80] [kernel-spark] Migrate DeltaSourceDeletionVectorsSuite to v2 (#6294) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Create DeltaV2SourceDeletionVectorsSuite to migrate DeltaSourceDeletionVectorsSuite to v2. Add test scenarios for both partial delete by DV and full delete by DV in SparkMicroBatchStreamTest ## How was this patch tested? Test only PR To verify the v2 connector is active, a runtime exception was deliberately thrown inside `latestOffset`. The resulting stack trace shows `SparkMicroBatchStream`, confirming that execution is going through the v2 connector code path as expected. Removed this exception after verification. ## Does this PR introduce _any_ user-facing changes? No --- .../DeltaV2SourceDeletionVectorsSuite.scala | 86 +++++++++++++++++++ .../DeltaSourceDeletionVectorsSuite.scala | 49 ++++++----- .../v2/read/SparkMicroBatchStreamTest.java | 29 +++++++ 3 files changed, 139 insertions(+), 25 deletions(-) create mode 100644 spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala new file mode 100644 index 00000000000..1a5b5f6ff5e --- /dev/null +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala @@ -0,0 +1,86 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.test + +import org.apache.spark.sql.delta.DeltaSourceDeletionVectorsSuite +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +/** + * Test suite that runs DeltaSourceDeletionVectorsSuite using the V2 connector + * (V2_ENABLE_MODE=STRICT). + */ +class DeltaV2SourceDeletionVectorsSuite + extends DeltaSourceDeletionVectorsSuite with V2ForceTest { + + override protected def useDsv2: Boolean = true + + /** + * Override executeDml to temporarily use V1 connector for DML operations. + * SparkTable (V2) is read-only and does not support writes, so DML must + * go through the V1 path. Only streaming reads use the V2 connector. + */ + override protected def executeDml(sqlText: String): Unit = { + withSQLConf(DeltaSQLConf.V2_ENABLE_MODE.key -> "NONE") { + sql(sqlText) + } + } + + private lazy val shouldPassTests = Set( + "allow to delete files before starting a streaming query", + "allow to delete files before staring a streaming query without checkpoint", + "multiple deletion vectors per file with initial snapshot", + "deleting files fails query if ignoreDeletes = false", + "allow to delete files after staring a streaming query when ignoreDeletes is true", + "updating the source table causes failure when ignoreChanges = false - using DELETE", + "updating source table when ignoreDeletes = true fails the query - using DELETE", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE - List()", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + + " - List((ignoreDeletes,true))", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + + " - List((skipChangeCommits,true))", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE - List()", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + + " - List((ignoreDeletes,true))", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + + " - List((skipChangeCommits,true))" + ) + + private lazy val shouldFailTests = Set( + // TODO(#5319): enable these tests after ignoreChanges/ignoreFileDeletion read options + // are supported by the V2 connector. + "allow to delete files after staring a streaming query when ignoreFileDeletion is true", + "allow to update the source table when ignoreChanges = true - using DELETE", + "deleting files when ignoreChanges = true doesn't fail the query", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + + " - List((ignoreChanges,true))", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + + " - List((ignoreChanges,true))", + "multiple deletion vectors per file - List((ignoreFileDeletion,true))", + "multiple deletion vectors per file - List((ignoreChanges,true))" + ) + + override protected def shouldFail(testName: String): Boolean = { + val inPassList = shouldPassTests.contains(testName) + val inFailList = shouldFailTests.contains(testName) + + assert(inPassList || inFailList, s"Test '$testName' not in shouldPassTests or shouldFailTests") + assert(!(inPassList && inFailList), + s"Test '$testName' in both shouldPassTests and shouldFailTests") + + inFailList + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala index daaca6b70c1..42b7e43d079 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala @@ -31,9 +31,17 @@ import org.apache.spark.sql.streaming.util.StreamManualClock trait DeltaSourceDeletionVectorTests extends StreamTest with DeletionVectorsTestUtils { + self: DeltaSourceConnectorTrait => import testImplicits._ + /** + * Executes a DML SQL statement (DELETE, INSERT, etc.). + * Overridable so that V2 suites can route DML through the V1 connector, + * since SparkTable (V2) is read-only and does not support writes. + */ + protected def executeDml(sqlText: String): Unit = sql(sqlText) + test("allow to delete files before starting a streaming query") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) @@ -41,7 +49,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } - sql(s"DELETE FROM delta.`$inputDir`") + executeDml(s"DELETE FROM delta.`$inputDir`") (5 until 10).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) @@ -49,9 +57,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest deltaLog.checkpoint() assert(deltaLog.readLastCheckpointFile().nonEmpty, "this test requires a checkpoint") - val df = spark.readStream - .format("delta") - .load(inputDir.getCanonicalPath) + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) testStream(df)( AssertOnQuery { q => @@ -69,16 +75,14 @@ trait DeltaSourceDeletionVectorTests extends StreamTest val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } - sql(s"DELETE FROM delta.`$inputDir`") + executeDml(s"DELETE FROM delta.`$inputDir`") (5 until 7).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } assert(deltaLog.readLastCheckpointFile().isEmpty, "this test requires no checkpoint") - val df = spark.readStream - .format("delta") - .load(inputDir.getCanonicalPath) + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) testStream(df)( AssertOnQuery { q => @@ -115,7 +119,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest Seq(i, i + 1).toDF().coalesce(1).write.format("delta").mode("append").save(inputDir) } - val df = spark.readStream.format("delta").options(sourceOptions.toMap).load(inputDir) + val df = loadStreamWithOptions(inputDir, sourceOptions.toMap) val expectDVs = commandShouldProduceDVs.getOrElse( sqlCommand.toUpperCase().startsWith("DELETE")) @@ -126,7 +130,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest }, CheckAnswer((0 until 10): _*), AssertOnQuery { q => - sql(sqlCommand) + executeDml(sqlCommand) deletionVectorsPresentIfExpected(inputDir, expectDVs) }) @@ -148,7 +152,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest } val log = DeltaLog.forTable(spark, inputDir) val commitVersionBeforeDML = log.update().version - val df = spark.readStream.format("delta").options(sourceOptions.toMap).load(inputDir) + val df = loadStreamWithOptions(inputDir, sourceOptions.toMap) def expectDVsInCommand(shouldProduceDVs: Option[Boolean], command: String): Boolean = { shouldProduceDVs.getOrElse(command.toUpperCase().startsWith("DELETE")) } @@ -177,11 +181,11 @@ trait DeltaSourceDeletionVectorTests extends StreamTest true }, AssertOnQuery { q => - sql(sqlCommand1) + executeDml(sqlCommand1) deletionVectorsPresentIfExpected(inputDir, expectDVsInCommand1) }, AssertOnQuery { q => - sql(sqlCommand2) + executeDml(sqlCommand2) deletionVectorsPresentIfExpected(inputDir, expectDVsInCommand2) }, AssertOnQuery { q => @@ -416,21 +420,19 @@ trait DeltaSourceDeletionVectorTests extends StreamTest (0 until 10).toDF("value").coalesce(1).write.format("delta").save(path) // V1: Delete row 0 - sql(s"DELETE FROM delta.`$path` WHERE value = 0") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 0") // V2: Delete row 1 - sql(s"DELETE FROM delta.`$path` WHERE value = 1") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 1") // V3: Delete row 2 - sql(s"DELETE FROM delta.`$path` WHERE value = 2") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 2") // Verify DVs are present assert(getFilesWithDeletionVectors(deltaLog).nonEmpty, "This test requires deletion vectors to be present") - val df = spark.readStream - .format("delta") - .load(path) + val df = loadStreamWithOptions(path, Map.empty) testStream(df)( // Process the initial snapshot @@ -457,10 +459,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest // V0: 10 rows in a single file (0 until 10).toDF("value").coalesce(1).write.format("delta").save(path) - val df = spark.readStream - .format("delta") - .options(sourceOptions.toMap) - .load(path) + val df = loadStreamWithOptions(path, sourceOptions.toMap) testStream(df)( AssertOnQuery { q => @@ -470,12 +469,12 @@ trait DeltaSourceDeletionVectorTests extends StreamTest CheckAnswer((0 until 10): _*), AssertOnQuery { q => // V1: Delete row 0 - creates first DV (version 1) - sql(s"DELETE FROM delta.`$path` WHERE value = 0") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 0") true }, AssertOnQuery { q => // V2: Delete row 1 - updates DV (version 2). DV is cumulative: {0, 1} - sql(s"DELETE FROM delta.`$path` WHERE value = 1") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 1") true }, AssertOnQuery { q => diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java index 388e576148c..54188294515 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java @@ -1265,6 +1265,20 @@ private static Stream deleteOnlyScenarios() { }, false, "Partitioned table: delete entire partition"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql( + "ALTER TABLE %s SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)", + tableName); + sql( + "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + + "(1, 'User1'), (2, 'User2'), (3, 'User3') AS t(id, name)", + tableName); + sql("DELETE FROM %s WHERE id >= 1", tableName); + }, + false, + "Full DELETE with DV: full file delete via WHERE clause"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { @@ -1312,6 +1326,21 @@ private static Stream changeCommitScenarios() { }, false, "MERGE: AddFile + RemoveFile"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql( + "ALTER TABLE %s SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)", + tableName); + // Coalesce to to ensure DV is partial delete + sql( + "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + + "(1, 'User1'), (2, 'User2'), (3, 'User3') AS t(id, name)", + tableName); + sql("DELETE FROM %s WHERE id = 1", tableName); + }, + false, + "Partial DELETE with DV: AddFile(with DV) + RemoveFile"), Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { From 47d08e30ceb01148b768eac34bb14b2602d811a4 Mon Sep 17 00:00:00 2001 From: Zihao Xu Date: Fri, 20 Mar 2026 01:58:41 -0500 Subject: [PATCH 45/80] Preserve exception cause chain when rethrowing wrapped exceptions (#6325) ## Description When `S3SingleDriverLogStore` catches Hadoop's `FileAlreadyExistsException` and converts it to Java's `FileAlreadyExistsException`, the original exception cause was being lost. This makes debugging harder because the stack trace does not show the original Hadoop exception. This change preserves the cause chain by calling `initCause(e)` on the converted exception before rethrowing. ## How was this patch tested? Existing tests cover this code path. The change is purely additive (preserving exception metadata that was previously discarded). --- .../spark/sql/delta/storage/S3SingleDriverLogStore.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala b/spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala index 528625cc573..cf2cceebafd 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala @@ -198,7 +198,9 @@ class S3SingleDriverLogStore( } catch { // Convert Hadoop's FileAlreadyExistsException to Java's FileAlreadyExistsException case e: org.apache.hadoop.fs.FileAlreadyExistsException => - throw new java.nio.file.FileAlreadyExistsException(e.getMessage) + val converted = new java.nio.file.FileAlreadyExistsException(e.getMessage) + converted.initCause(e) + throw converted } finally { releasePathLock(lockedPath) } From 1838f3073429b8cf574fd27dd913d6acd0c77088 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Fri, 20 Mar 2026 09:01:01 -0700 Subject: [PATCH 46/80] [kernel-spark] Override columnarSupportMode in DSv2 SparkScan (#6276) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Override `columnarSupportMode()` in `SparkScan` to return `SUPPORTED` or `UNSUPPORTED` based on `ParquetUtils.isBatchReadSupportedForSchema()`, which checks both `spark.sql.parquet.enableVectorizedReader` config and schema type compatibility. - Without this override, the default `PARTITION_DEFINED` mode causes Spark to eagerly call `planInputPartitions()` during query planning to check per-partition columnar support, triggering unnecessary early file enumeration. - Since columnar support is uniform across all partitions (determined by schema and config, not individual files), we declare it at the scan level to avoid this overhead. ## Test plan - [x] `testColumnarSupportModeReturnsSupported` — verifies `SUPPORTED` for batch-compatible schema (INT, STRING) - [x] `testColumnarSupportModeDoesNotTriggerPlanning` — verifies `columnarSupportMode()` does not set the `planned` flag (avoids early file enumeration) - [x] `testColumnarSupportModeWithUnsupportedSchema` — verifies `UNSUPPORTED` for MAP schema with nested vectorized reader disabled - [x] `testColumnarSupportModeWithVectorizedReaderDisabled` — verifies `UNSUPPORTED` when `spark.sql.parquet.enableVectorizedReader=false` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 --- .../spark/internal/v2/read/SparkScan.java | 35 +++++ .../internal/v2/utils/PartitionUtils.java | 22 ++- .../spark/internal/v2/DeltaV2TestBase.java | 14 ++ .../spark/internal/v2/read/SparkScanTest.java | 125 ++++++++++++++++++ 4 files changed, 190 insertions(+), 6 deletions(-) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java index 85fdcc30a73..fe81918b6f0 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java @@ -26,6 +26,7 @@ import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.data.ScanStateRow; import io.delta.kernel.utils.CloseableIterator; +import io.delta.spark.internal.v2.read.deletionvector.DeletionVectorSchemaContext; import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.utils.PartitionUtils; import io.delta.spark.internal.v2.utils.ScalaUtils; @@ -48,6 +49,7 @@ import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.execution.datasources.*; +import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; import org.apache.spark.sql.types.StringType; @@ -167,6 +169,39 @@ public StructType readSchema() { return new StructType(fields.toArray(new StructField[0])); } + /** + * Override columnarSupportMode to explicitly declare whether this scan supports columnar + * (vectorized) reading. Without this override, the default {@code PARTITION_DEFINED} mode causes + * Spark to eagerly call {@code planInputPartitions()} during query planning to check + * per-partition columnar support, triggering unnecessary early file enumeration. + * + *

    Since columnar support is uniform across all partitions (determined by schema compatibility + * and table features, not by individual files), we can declare it at the scan level to avoid this + * overhead. + * + *

    This must stay consistent with the vectorized reader decision in {@link + * PartitionUtils#createDeltaParquetReaderFactory}. In particular, deletion-vector-enabled tables + * augment the read schema with internal columns (e.g., {@code __delta_internal_is_row_deleted}), + * which changes the schema passed to the vectorized reader check. We replicate that logic here to + * ensure the scan-level declaration matches the per-partition reader behavior. + */ + @Override + public Scan.ColumnarSupportMode columnarSupportMode() { + // When the table supports deletion vectors, the reader factory augments the read schema + // with internal columns via DeletionVectorSchemaContext. Reuse the same class here so the + // batch-read check stays consistent — if DeletionVectorSchemaContext adds new fields in + // the future, this code path picks them up automatically. + StructType schemaForBatchCheck = + PartitionUtils.tableSupportsDeletionVectors(initialSnapshot) + ? new DeletionVectorSchemaContext(readDataSchema, partitionSchema) + .getSchemaWithDvColumn() + : readDataSchema; + + return ParquetUtils.isBatchReadSupportedForSchema(sqlConf, schemaForBatchCheck) + ? Scan.ColumnarSupportMode.SUPPORTED + : Scan.ColumnarSupportMode.UNSUPPORTED; + } + @Override public Batch toBatch() { ensurePlanned(); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java index b2e960dd0f8..6c6e067b340 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java @@ -60,6 +60,19 @@ public class PartitionUtils { private PartitionUtils() {} + /** + * Returns whether the given snapshot's table supports deletion vectors. A table supports DVs when + * its protocol includes the {@link TableFeatures#DELETION_VECTORS_RW_FEATURE} and the table + * format is Parquet. + */ + public static boolean tableSupportsDeletionVectors(Snapshot snapshot) { + SnapshotImpl snapshotImpl = (SnapshotImpl) snapshot; + Protocol protocol = snapshotImpl.getProtocol(); + Metadata metadata = snapshotImpl.getMetadata(); + return protocol.supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE) + && "parquet".equalsIgnoreCase(metadata.getFormat().getProvider()); + } + /** * Calculate the maximum split bytes for file partitioning, considering total bytes and file * count. This is used for optimal file splitting in both batch and streaming read. @@ -205,11 +218,8 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( String tablePath = snapshotImpl.getDataPath().toString(); // Create DV schema context if table supports deletion vectors - boolean isTableSupportDv = - protocol.supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE) - && "parquet".equalsIgnoreCase(metadata.getFormat().getProvider()); Optional dvSchemaContext = - isTableSupportDv + tableSupportsDeletionVectors(snapshot) ? Optional.of(new DeletionVectorSchemaContext(readDataSchema, partitionSchema)) : Optional.empty(); StructType finalReadDataSchema = @@ -226,11 +236,11 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( String.valueOf(enableVectorizedReader))); // TODO(https://github.com/delta-io/delta/issues/5859): Enable file splitting for DV tables - boolean optimizationsEnabled = !isTableSupportDv; + boolean optimizationsEnabled = !dvSchemaContext.isPresent(); // TODO(https://github.com/delta-io/delta/issues/5859): Support _metadata.row_index for DV Option useMetadataRowIndex = - isTableSupportDv ? Option.apply(Boolean.FALSE) : Option.empty(); + dvSchemaContext.isPresent() ? Option.apply(Boolean.FALSE) : Option.empty(); DeltaParquetFileFormatV2 deltaFormat = new DeltaParquetFileFormatV2( protocol, diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java index 7eccb585ee5..16dc480014a 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java @@ -114,6 +114,20 @@ protected void withSQLConf(String key, String value, ThrowingRunnable action) th } } + /** + * Runs the given action and drops the specified tables afterwards, similar to Scala's {@code + * withTable}. + */ + protected void withTable(String[] tableNames, ThrowingRunnable action) throws Exception { + try { + action.run(); + } finally { + for (String tableName : tableNames) { + spark.sql(String.format("DROP TABLE IF EXISTS %s", tableName)); + } + } + } + protected static void createPartitionedTable(String tableName, String path) { spark.sql( String.format( diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java index c96dd4bd0d0..f350b6d3ce8 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java @@ -112,6 +112,131 @@ public static void setupPartitionedTable(@TempDir File tempDir) { protected static final List allCities = Arrays.asList("city=hz", "city=sh", "city=bj", "city=sz"); + // =============================================================================================== + // Tests for columnarSupportMode + // =============================================================================================== + + @Test + public void testColumnarSupportModeReturnsSupported() { + // Table schema uses simple types (INT, STRING) which are batch-read-compatible + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + assertEquals( + Scan.ColumnarSupportMode.SUPPORTED, + scan.columnarSupportMode(), + "columnarSupportMode should return SUPPORTED for batch-compatible schema"); + } + + @Test + public void testColumnarSupportModeDoesNotTriggerPlanning() throws Exception { + // Calling columnarSupportMode() must NOT trigger file planning (the whole point of the + // override is to avoid the early planInputPartitions() call that PARTITION_DEFINED causes). + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + // Call columnarSupportMode before any planning + scan.columnarSupportMode(); + + // Verify the scan has not been planned yet + Field plannedField = SparkScan.class.getDeclaredField("planned"); + plannedField.setAccessible(true); + assertFalse( + (boolean) plannedField.get(scan), "columnarSupportMode() should not trigger file planning"); + } + + @Test + public void testColumnarSupportModeWithUnsupportedSchema(@TempDir File tempDir) throws Exception { + // Create a table with a MAP column and disable nested column vectorized reading + // to ensure the schema is not batch-read-compatible + String path = tempDir.getAbsolutePath(); + String mapTableName = "columnar_map_table"; + + withTable( + new String[] {mapTableName}, + () -> { + spark.sql( + String.format( + "CREATE TABLE %s (id INT, tags MAP) USING delta LOCATION '%s'", + mapTableName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, map('k', 'v'))", mapTableName)); + + withSQLConf( + "spark.sql.parquet.enableNestedColumnVectorizedReader", + "false", + () -> { + SparkTable mapTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, mapTableName), + path, + options); + SparkScanBuilder builder = (SparkScanBuilder) mapTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + assertEquals( + Scan.ColumnarSupportMode.UNSUPPORTED, + scan.columnarSupportMode(), + "columnarSupportMode should return UNSUPPORTED for schema with MAP type" + + " when nested column vectorized reader is disabled"); + }); + }); + } + + @Test + public void testColumnarSupportModeWithVectorizedReaderDisabled() throws Exception { + // When spark.sql.parquet.enableVectorizedReader is false, columnarSupportMode should + // return UNSUPPORTED even for batch-compatible schemas. + withSQLConf( + "spark.sql.parquet.enableVectorizedReader", + "false", + () -> { + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + assertEquals( + Scan.ColumnarSupportMode.UNSUPPORTED, + scan.columnarSupportMode(), + "columnarSupportMode should return UNSUPPORTED when vectorized reader is disabled"); + }); + } + + @Test + public void testColumnarSupportModeWithDeletionVectors(@TempDir File tempDir) throws Exception { + // For a DV-enabled table with a batch-compatible schema, columnarSupportMode should still + // return SUPPORTED because the DV internal column (__delta_internal_is_row_deleted, ByteType) + // is also batch-compatible. This verifies consistency with PartitionUtils reader factory. + String dvPath = tempDir.getAbsolutePath(); + String dvTableName = "columnar_dv_table"; + + withTable( + new String[] {dvTableName}, + () -> { + spark.sql( + String.format( + "CREATE TABLE %s (id INT, value STRING) USING delta " + + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true') " + + "LOCATION '%s'", + dvTableName, dvPath)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", dvTableName)); + + SparkTable dvTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, dvTableName), + dvPath, + options); + SparkScanBuilder builder = (SparkScanBuilder) dvTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + // DV-enabled table with simple types should still return SUPPORTED because the + // DV column (ByteType) is batch-compatible + assertEquals( + Scan.ColumnarSupportMode.SUPPORTED, + scan.columnarSupportMode(), + "columnarSupportMode should return SUPPORTED for DV-enabled table with" + + " batch-compatible schema"); + }); + } + // =============================================================================================== // Tests for getDataSchema, getPartitionSchema, getReadDataSchema, getOptions, getConfiguration // =============================================================================================== From cbabe3efc95cf35c53e4ff57e5a4fd73ce7ed8a8 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Fri, 20 Mar 2026 13:07:31 -0700 Subject: [PATCH 47/80] [kernel-spark] Simplify null check in decimal pushdown and add tests (#6286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Simplify unnecessary ternary null check in `classifyFilter()` — the 2-arg `convertSparkFilterToKernelPredicate(filter, tableSchema)` already handles null `tableSchema` correctly - Add unit test for partial pushdown when one side of AND fails decimal scale check: `AND(price > 99.999, price < 200.00)` where left has scale=3 exceeding column's scale=2 — only the right side is pushed - Add integration tests through `pushFilters()` → `classifyFilter()` → `convertComparisonLiteral()` exercising the decimal-widening path end-to-end ## Test plan - [x] `ExpressionUtilsTest` — 63 tests pass (including new `testDecimalPartialPushDownInAndFilter`) - [x] `SparkScanBuilderTest` — 23 tests pass (including new `testPushFilters_decimalWideningEndToEnd` and `testPushFilters_decimalRejectionPartialPushDown`) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 --- .../internal/v2/utils/ExpressionUtils.java | 4 +- .../v2/read/SparkScanBuilderTest.java | 116 ++++++++++++++++-- .../v2/utils/ExpressionUtilsTest.java | 47 +++++++ 3 files changed, 155 insertions(+), 12 deletions(-) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java index 3d0e71e5aa3..42cb805ed74 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java @@ -516,9 +516,7 @@ public static FilterClassificationResult classifyFilter( Filter filter, Set partitionColumnSet, StructType tableSchema) { // try to convert Spark filter to Kernel Predicate ConvertedPredicate convertedPredicate = - tableSchema != null - ? ExpressionUtils.convertSparkFilterToKernelPredicate(filter, tableSchema) - : ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, tableSchema); boolean isKernelSupported = convertedPredicate.isPresent(); boolean isPartialConversion = convertedPredicate.isPartial(); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java index 56d00e05cf5..f1884bfda44 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java @@ -27,9 +27,11 @@ import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; import java.lang.reflect.Field; +import java.math.BigDecimal; import java.util.Arrays; import java.util.HashSet; import java.util.Optional; +import java.util.Set; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.sources.*; @@ -739,15 +741,6 @@ private void checkSupportsPushDownFilters( } private SparkScanBuilder createTestScanBuilder(File tempDir) { - String path = tempDir.getAbsolutePath(); - String tableName = String.format("predicate_test_%d", System.currentTimeMillis()); - spark.sql( - String.format( - "CREATE OR REPLACE TABLE %s (id INT, name STRING, dep_id INT) USING delta PARTITIONED BY (dep_id) LOCATION '%s'", - tableName, path)); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); - Snapshot snapshot = snapshotManager.loadLatestSnapshot(); StructType dataSchema = DataTypes.createStructType( new StructField[] { @@ -764,6 +757,111 @@ private SparkScanBuilder createTestScanBuilder(File tempDir) { DataTypes.createStructField("name", DataTypes.StringType, true), DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) }); + return createScanBuilder(tempDir, dataSchema, partitionSchema, tableSchema); + } + + /** + * Integration test: decimal widening end-to-end through pushFilters() → classifyFilter() → + * convertComparisonLiteral(). Verifies that a decimal literal Decimal(5,2) is widened to match + * column type Decimal(7,2) when pushed through the full filter pushdown path. + */ + @Test + public void testPushFilters_decimalWideningEndToEnd(@TempDir File tempDir) throws Exception { + SparkScanBuilder builder = createDecimalTestScanBuilder(tempDir); + + // price = 100.00 where literal is Decimal(5,2) and column is Decimal(7,2) + Filter[] sparkFilter = new Filter[] {new EqualTo("price", new BigDecimal("100.00"))}; + Predicate kernelPredicate = + new Predicate("=", new Column("price"), Literal.ofDecimal(new BigDecimal("100.00"), 7, 2)); + + checkSupportsPushDownFilters( + builder, + sparkFilter, // input filters + sparkFilter, // expected post-scan filters (data filter, stays for row-level eval) + sparkFilter, // expected pushed filters + new Predicate[] {kernelPredicate}, // expected pushed kernel predicates (widened) + sparkFilter, // expected data filters + Optional.of(kernelPredicate)); // expected kernelScanBuilder.predicate + } + + /** + * Integration test: decimal literal with scale exceeding column scale is rejected during + * pushFilters(). The filter AND(price > 99.999, price < 200.00) should partially push only the + * right side because 99.999 has scale=3 exceeding column's Decimal(7,2) scale=2. + */ + @Test + public void testPushFilters_decimalRejectionPartialPushDown(@TempDir File tempDir) + throws Exception { + SparkScanBuilder builder = createDecimalTestScanBuilder(tempDir); + + // AND(price > 99.999, price < 200.00): left side has scale=3 exceeding column's scale=2 + Filter[] sparkFilter = + new Filter[] { + new And( + new GreaterThan("price", new BigDecimal("99.999")), + new LessThan("price", new BigDecimal("200.00"))) + }; + // Only the right side (price < 200.00) is pushed as a kernel predicate + Predicate kernelPredicate = + new Predicate("<", new Column("price"), Literal.ofDecimal(new BigDecimal("200.00"), 7, 2)); + + checkSupportsPushDownFilters( + builder, + sparkFilter, // input filters + sparkFilter, // expected post-scan filters (partial conversion, stays for row-level eval) + new Filter[] {}, // expected pushed filters (partial: Spark filter not added) + new Predicate[] {kernelPredicate}, // expected pushed kernel predicates + sparkFilter, // expected data filters + Optional.of(kernelPredicate)); // expected kernelScanBuilder.predicate + } + + private SparkScanBuilder createDecimalTestScanBuilder(File tempDir) { + StructType dataSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("price", DataTypes.createDecimalType(7, 2), true), + DataTypes.createStructField("quantity", DataTypes.IntegerType, true) + }); + StructType partitionSchema = + DataTypes.createStructType( + new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + StructType tableSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("price", DataTypes.createDecimalType(7, 2), true), + DataTypes.createStructField("quantity", DataTypes.IntegerType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + }); + return createScanBuilder(tempDir, dataSchema, partitionSchema, tableSchema); + } + + /** + * Shared helper for creating a SparkScanBuilder with the given schemas. Both + * createTestScanBuilder and createDecimalTestScanBuilder delegate to this method to avoid + * duplicating snapshot loading and builder instantiation logic. + */ + private SparkScanBuilder createScanBuilder( + File tempDir, StructType dataSchema, StructType partitionSchema, StructType tableSchema) { + String path = tempDir.getAbsolutePath(); + String tableName = String.format("test_%d", System.currentTimeMillis()); + // Build CREATE TABLE SQL from the tableSchema and partitionSchema + StringBuilder columns = new StringBuilder(); + Set partitionCols = new HashSet<>(); + for (StructField f : partitionSchema.fields()) { + partitionCols.add(f.name()); + } + for (StructField f : tableSchema.fields()) { + if (columns.length() > 0) columns.append(", "); + columns.append(f.name()).append(" ").append(f.dataType().sql()); + } + String partitionColNames = String.join(", ", partitionCols); + spark.sql( + String.format( + "CREATE OR REPLACE TABLE %s (%s) USING delta PARTITIONED BY (%s) LOCATION '%s'", + tableName, columns, partitionColNames, path)); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return new SparkScanBuilder( tableName, snapshot, diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java index ae1f47e6437..b9230a24bcd 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java @@ -22,7 +22,9 @@ import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.*; import java.math.BigDecimal; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; import org.apache.spark.sql.connector.expressions.FieldReference; @@ -852,6 +854,31 @@ public void testDecimalLiteralColumnNotInSchema() { assertEquals(new DecimalType(5, 2), literal.getDataType()); } + @Test + public void testDecimalPartialPushDownInAndFilter() { + // AND(price > 99.999, price < 200.00) where left has scale=3 exceeding column's scale=2. + // Only the right side should be pushed down (partial pushdown). + GreaterThan left = new GreaterThan("price", new BigDecimal("99.999")); + LessThan right = new LessThan("price", new BigDecimal("200.00")); + org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(left, right); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, decimalSchema); + + assertTrue(result.isPresent(), "AND filter should partially push down the valid side"); + assertTrue(result.isPartial(), "Result should be marked as partial conversion"); + // Only the right side (price < 200.00) should be pushed, not a compound AND + Predicate pred = result.get(); + assertNotEquals( + "AND", + pred.getName(), + "Left side (price > 99.999) should be dropped, not pushed as compound AND"); + assertEquals("<", pred.getName()); + Literal literal = (Literal) pred.getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + assertEquals(new BigDecimal("200.00"), literal.getValue()); + } + @Test public void testDecimalLiteralInCompoundFilter() { // AND(price >= 100.00, price <= 200.00) with Decimal(7,2) column @@ -941,6 +968,26 @@ public void testDecimalLiteralAllComparisonOperators() { } } + @Test + public void testClassifyFilterWithNullSchemaMatchesTwoArgOverload() { + // Directly validates that classifyFilter(filter, partitionColumns, null) produces + // the same result as the 2-arg classifyFilter(filter, partitionColumns). + Set partitionColumns = new HashSet<>(); + partitionColumns.add("dep_id"); + + EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); + + ExpressionUtils.FilterClassificationResult resultTwoArg = + ExpressionUtils.classifyFilter(filter, partitionColumns); + ExpressionUtils.FilterClassificationResult resultThreeArg = + ExpressionUtils.classifyFilter(filter, partitionColumns, null); + + assertEquals(resultTwoArg.isKernelSupported, resultThreeArg.isKernelSupported); + assertEquals(resultTwoArg.isPartialConversion, resultThreeArg.isPartialConversion); + assertEquals(resultTwoArg.isDataFilter, resultThreeArg.isDataFilter); + assertEquals(resultTwoArg.kernelPredicate, resultThreeArg.kernelPredicate); + } + @Test public void testDsv2PredicateToCatalystExpression_ColumnNotFound() { NamedReference invalidRef = FieldReference.apply("nonexistent_column"); From d1e59d8ba864272b4dac25ebb8318c9bb5b2fb60 Mon Sep 17 00:00:00 2001 From: seewishnew Date: Fri, 20 Mar 2026 13:33:31 -0700 Subject: [PATCH 48/80] [Spark] Block options.fs.* from being rendered with show/describe table props (#6300) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description `DeltaTableV2.properties()` currently copies all `v1Table.storage.properties` into the user-visible V2 table properties map with an `option.` prefix. When catalogs like Unity Catalog vend temporary cloud storage credentials at table-load time, they inject `fs.*` properties (e.g., `fs.s3a.init.access.key`, `fs.s3a.init.session.token`, `fs.unitycatalog.auth.token`, `fs.unitycatalog.uri`) into `CatalogTable.storage.properties`. These then appear verbatim in `DESCRIBE TABLE EXTENDED` and `SHOW TBLPROPERTIES` output. While Spark's default redaction regex (`(?i)secret|password|token|access[.]?key`) does redact actual credential values, it still leaks non-secret but sensitive metadata in plaintext — credential expiry timestamps, UC server endpoints, credential session identifiers, auth mechanism details, and S3 configuration such as path-style access settings. None of this information belongs in user-facing table property output. This PR filters out `fs.*`-prefixed keys in `DeltaTableV2.properties()` before they are copied into the V2 properties map. This is safe because no code path reads `option.fs.*` back from `properties()` for storage access — the actual data path goes through `DeltaTableV2.deltaLog`, which reads `catalogTable.storage.properties` directly and is completely unaffected by this change. An extensible `HIDDEN_STORAGE_PROPERTY_PREFIXES` constant is added to the `DeltaTableV2` companion object so additional prefixes can be excluded in the future. Resolves https://github.com/unitycatalog/unitycatalog/issues/1362 ## How was this patch tested? Added `testFsPropertiesHiddenFromTableProperties` in `UCDeltaUtilityTest`. The test creates a UC MANAGED Delta table with an explicit custom table property (`myCustomProp=myCustomValue`) and verifies: 1. `SHOW TBLPROPERTIES` contains **no** keys starting with `option.fs.` (UC-vended credentials/metadata are hidden). 2. `SHOW TBLPROPERTIES` **does** contain the user-set property (`myCustomProp`) and Delta properties (`delta.minReaderVersion`) — confirming the filter is selective and doesn't suppress other properties. 3. `DESCRIBE EXTENDED` Table Properties row does **not** contain `option.fs.` but **does** contain `myCustomProp=myCustomValue`. 4. Two `INSERT` operations followed by `SELECT` after each verify the data path still works — credentials still flow to the filesystem for actual storage access even though they are hidden from the display path. ## Does this PR introduce _any_ user-facing changes? Yes. `DESCRIBE TABLE EXTENDED` and `SHOW TBLPROPERTIES` for Delta tables will no longer display `option.fs.*` properties. Previously, these commands could expose internal filesystem configuration injected by catalogs (such as Unity Catalog), including: - `option.fs.s3a.init.access.key=*********(redacted)` - `option.fs.s3a.init.credential.expired.time=1771725415000` - `option.fs.s3a.aws.credentials.provider=*********(redacted)` - `option.fs.s3a.path.style.access=true` - `option.fs.unitycatalog.uri=https://...` - `option.fs.unitycatalog.credentials.uid=...` - `option.fs.unitycatalog.auth.type=static` After this change, none of these keys appear in the output. This is a security improvement — no previously visible *user-set* table properties are affected. --- .../sql/delta/catalog/DeltaTableV2.scala | 13 +++- .../apache/spark/sql/delta/DeltaSuite.scala | 53 +++++++++++++++ .../io/sparkuctest/UCDeltaUtilityTest.java | 66 +++++++++++++++++++ 3 files changed, 131 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala index a980d8983d1..bd4bb302fc8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala @@ -242,7 +242,9 @@ class DeltaTableV2 private( base.put(TableCatalog.PROP_OWNER, table.owner) } v1Table.storage.properties.foreach { case (key, value) => - base.put(TableCatalog.OPTION_PREFIX + key, value) + if (!DeltaTableV2.HIDDEN_STORAGE_PROPERTY_PREFIXES.exists(key.startsWith)) { + base.put(TableCatalog.OPTION_PREFIX + key, value) + } } if (v1Table.tableType == CatalogTableType.EXTERNAL) { base.put(TableCatalog.PROP_EXTERNAL, "true") @@ -434,6 +436,15 @@ class DeltaTableV2 private( } object DeltaTableV2 { + + /** + * Storage property key prefixes that should be excluded from the user-visible V2 table + * properties returned by [[DeltaTableV2.properties()]]. These are Hadoop filesystem + * configuration options that may contain sensitive credentials (access keys, session + * tokens, etc.) injected by catalogs at table-load time. + */ + private[delta] val HIDDEN_STORAGE_PROPERTY_PREFIXES: Seq[String] = Seq("fs.") + def unapply(deltaTable: DeltaTableV2): Option[( SparkSession, Path, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala index 7a75da71b0d..c84830cdfe8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger // scalastyle:off import.ordering.noEmptyLine import org.apache.spark.sql.delta.actions.{Action, TableFeatureProtocolUtils} +import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite} import org.apache.spark.sql.delta.files.TahoeLogFileIndex @@ -37,6 +38,8 @@ import org.apache.spark.SparkException import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.connector.catalog.TableCatalog import org.apache.spark.sql.catalyst.expressions.InSet import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.plans.logical.Filter @@ -3042,6 +3045,56 @@ class DeltaSuite extends QueryTest // Not properly supported: ambiguous without special handling for escaping. assert(parseTableAndAlias("'store sales'") === "'store" -> Some("sales'")) } + + test("DeltaTableV2.properties() filters fs.* storage properties injected by catalogs") { + withTempDir { dir => + spark.range(1).write.format("delta").save(dir.getAbsolutePath) + + val tablePath = new Path(dir.toURI) + + // Simulate catalog (e.g., Unity Catalog) injecting fs.* credentials and metadata + // into CatalogTable.storage.properties at table-load time. + val injectedFsProps = Map( + "fs.s3a.fake-endpoint" -> "s3.us-west-2.amazonaws.com", + "fs.unitycatalog.uri" -> "https://uc.example.com", + "fs.unitycatalog.auth.fake-token" -> "dapi_secret_token" + ) + val otherStorageProps = Map( + "nonFsProp" -> "visible_value", + "path" -> dir.getAbsolutePath + ) + val allStorageProps = injectedFsProps ++ otherStorageProps + + val catalogTable = CatalogTable( + identifier = TableIdentifier("test_fs_filter"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat( + locationUri = Some(dir.toURI), + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = allStorageProps + ), + schema = new StructType().add("id", "long"), + provider = Some("delta") + ) + + val deltaTable = DeltaTableV2(spark, tablePath, Some(catalogTable)) + val v2Props = deltaTable.properties() + + injectedFsProps.keys.foreach { fsKey => + assert(!v2Props.containsKey(TableCatalog.OPTION_PREFIX + fsKey), + s"DeltaTableV2.properties() should hide '${TableCatalog.OPTION_PREFIX}$fsKey'") + } + injectedFsProps.keys.foreach { fsKey => + assert(!v2Props.containsKey(fsKey), + s"DeltaTableV2.properties() should also hide '$fsKey'") + } + assert(v2Props.get(TableCatalog.OPTION_PREFIX + "nonFsProp") === "visible_value", + "Non-fs storage properties should remain visible in DeltaTableV2.properties()") + } + } } diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java index a8a11fc8e07..ac9d2b6dbe9 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java @@ -73,6 +73,72 @@ private void assertDescribeHistory(String tableName, List> expected Assertions.assertThat(prunedResults).isEqualTo(expected); } + @TestAllTableTypes + public void testFsPropertiesHiddenFromTableProperties(TableType tableType) throws Exception { + withNewTable( + "fs_props_hidden", + "id INT, name STRING", + null, // no partition + tableType, + "'myCustomProp'='myCustomValue'", + tableName -> { + // SHOW TBLPROPERTIES returns one row per property (key, value). + List> propRows = sql("SHOW TBLPROPERTIES %s", tableName); + List propKeys = new ArrayList<>(); + for (List row : propRows) { + propKeys.add(row.get(0)); + } + + // Verify no key starts with option.fs. — these are internal catalog-vended + // credentials/metadata that should not be user-visible. + for (String key : propKeys) { + Assertions.assertThat(key) + .as("SHOW TBLPROPERTIES should not expose option.fs.* keys") + .doesNotStartWith("option.fs."); + } + + // Verify that non-fs storage properties and user-set table properties ARE + // still present — confirming the filter is selective, not a blanket removal. + Assertions.assertThat(propKeys) + .as("User-set table properties should still be visible") + .contains("myCustomProp"); + Assertions.assertThat(propKeys) + .as("Delta table properties should still be visible") + .contains("delta.minReaderVersion"); + + // DESCRIBE EXTENDED returns a "Table Properties" row with all properties + // in a single string like "[key1=val1,key2=val2,...]". + boolean foundTableProperties = false; + List> descRows = sql("DESCRIBE EXTENDED %s", tableName); + for (List row : descRows) { + if (row.size() >= 2 && "Table Properties".equals(row.get(0))) { + foundTableProperties = true; + Assertions.assertThat(row.get(1)) + .as("DESCRIBE EXTENDED should not expose option.fs.* storage properties") + .doesNotContain("option.fs."); + Assertions.assertThat(row.get(1)) + .as("DESCRIBE EXTENDED should not expose fs.* storage properties either") + .doesNotContain("fs."); + Assertions.assertThat(row.get(1)) + .as("DESCRIBE EXTENDED should still show user-set properties") + .contains("myCustomProp=myCustomValue"); + } + } + Assertions.assertThat(foundTableProperties) + .as("DESCRIBE EXTENDED must include a 'Table Properties' row") + .isTrue(); + + // Verify the data path still works — credentials still flow to the filesystem + // via CatalogTable.storage.properties even though they are hidden from properties(). + sql("INSERT INTO %s VALUES (1, 'hello'), (2, 'world')", tableName); + check(tableName, List.of(List.of("1", "hello"), List.of("2", "world"))); + sql("INSERT INTO %s VALUES (3, 'foo')", tableName); + check( + tableName, + List.of(List.of("1", "hello"), List.of("2", "world"), List.of("3", "foo"))); + }); + } + @Test public void testMaintenanceOpsBlockedOnManagedTable() throws Exception { withNewTable( From 5dcc0990497b23bc8ac8dc53b22064c546cf8c08 Mon Sep 17 00:00:00 2001 From: Zihao Xu Date: Fri, 20 Mar 2026 17:39:05 -0500 Subject: [PATCH 49/80] Add logging when incremental commit is silently disabled (#6327) ## Description When incremental checksum validation fails, the code silently disables incremental commit for the session. This adds warning logs so operators can see when and why incremental commit was disabled, including the validation error and the snapshot version. ## How was this patch tested? Existing tests cover this code path. The change only adds logging to existing error handling branches. --- .../org/apache/spark/sql/delta/SnapshotManagement.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala b/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala index cc23282fa32..77c208dfd7b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala @@ -1407,13 +1407,19 @@ trait SnapshotManagement { self: DeltaLog => // NOTE: Validation is a no-op with incremental commit disabled. newSnapshot.validateChecksum(Map("context" -> checksumContext)) } catch { - case _: IllegalStateException if !DeltaUtils.isTesting => false + case e: IllegalStateException if !DeltaUtils.isTesting => + logWarning(log"Incremental checksum validation failed: " + + log"${MDC(DeltaLogKeys.ERROR, e.getMessage)}") + false } if (!crcIsValid) { // Create snapshot without incremental checksum. This will fallback to creating // a checksum based on state reconstruction. Disable incremental commit to avoid // further error triggers in this session. + logWarning(log"Disabling incremental commit for this session due to checksum " + + log"validation failure at version " + + log"${MDC(DeltaLogKeys.VERSION, newSnapshot.version)}") spark.sessionState.conf.setConf(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED, false) spark.sessionState.conf.setConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC, false) return createSnapshotWithCrc(checksumOpt = None) From b6f21a2dee75823ebbe808cc6d4d150f25c23802 Mon Sep 17 00:00:00 2001 From: richardc-db <87336575+richardc-db@users.noreply.github.com> Date: Fri, 20 Mar 2026 15:41:55 -0700 Subject: [PATCH 50/80] [KERNEL][VARIANT] Add variant GA table feature to delta kernel java (#6322) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [x] Kernel - [ ] Other (fill in here) ## Description title. Graduates the variantShredding table feature from preview to GA in the kernel - When delta.enableVariantShredding is set to true, the GA feature (variantShredding) is now automatically enabled instead of the preview feature (variantShredding-preview). - If a table's protocol already contains variantShredding-preview, the GA feature will not be auto-enabled — this preserves backward compatibility with older clients that only understand the preview feature. - The preview feature (variantShredding-preview) can no longer be auto-enabled via metadata. To use it on a new table, users must explicitly set delta.feature.variantShredding-preview=supported. - Both features can coexist in the same protocol if a user explicitly enables both. ## How was this patch tested? added UTs ## Does this PR introduce _any_ user-facing changes? yes, new table feature. --- .../io/delta/kernel/internal/TableConfig.java | 2 +- ...erCompatV3MetadataValidatorAndUpdater.java | 1 + .../internal/tablefeatures/TableFeatures.java | 49 ++++++++---- .../internal/actions/ProtocolSuite.scala | 5 ++ ...atV3MetadataValidatorAndUpdaterSuite.scala | 3 + .../tablefeatures/TableFeaturesSuite.scala | 60 ++++++++++----- .../defaults/DeltaTableFeaturesSuite.scala | 74 +++++++++++++++++++ 7 files changed, 160 insertions(+), 34 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java index ffb695526eb..253ba7da5ea 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java @@ -367,7 +367,7 @@ public static class UniversalFormats { /** * Table property that enables modifying the table in accordance with the Delta-Variant Shredding - * Preview protocol. + * protocol. * * @see diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java index 4fb71571eff..3586d5bcf2d 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java @@ -113,6 +113,7 @@ public static Optional validateAndUpdateIcebergWriterCompatV3Metadata( ICEBERG_WRITER_COMPAT_V3, DELETION_VECTORS_RW_FEATURE, VARIANT_RW_FEATURE, + VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, // Also allow writerV1 features for backward compatibility. diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java index 99f61f14450..e375f9b3be6 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java @@ -238,27 +238,49 @@ public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata me new VariantTypeTableFeatureBase("variantType-preview"); /* ---- End: variantType ---- */ - /* ---- Start: variantShredding-preview ---- */ - public static final TableFeature VARIANT_SHREDDING_PREVIEW_RW_FEATURE = - new VariantShreddingPreviewFeature(); - - private static class VariantShreddingPreviewFeature extends TableFeature.ReaderWriterFeature - implements FeatureAutoEnabledByMetadata { - VariantShreddingPreviewFeature() { - super("variantShredding-preview", /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); + /* ---- Start: variantShredding ---- */ + // Base class for variantShredding and variantShredding-preview features. Both features have + // identical behavior. Now that variantShredding has graduated to GA: + // + // - When `delta.enableVariantShredding` is set to true, the GA feature (`variantShredding`) + // is automatically enabled unless the table already has `variantShredding-preview` in its + // protocol (to avoid breaking clients that only understand the preview feature). + // - The preview feature (`variantShredding-preview`) is never auto-enabled. To use it on a + // new table, a user must explicitly set `delta.feature.variantShredding-preview=supported` + // in the table properties. + private static class VariantShreddingTableFeatureBase extends TableFeature.ReaderWriterFeature { + VariantShreddingTableFeatureBase(String featureName) { + super(featureName, /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } @Override - public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { - return TableConfig.VARIANT_SHREDDING_ENABLED.fromMetadata(metadata); + public Set requiredFeatures() { + return new HashSet<>(Arrays.asList(TableFeatures.VARIANT_RW_FEATURE)); + } + } + + private static class VariantShreddingTableFeature extends VariantShreddingTableFeatureBase + implements FeatureAutoEnabledByMetadata { + VariantShreddingTableFeature() { + super("variantShredding"); } @Override - public Set requiredFeatures() { - return new HashSet<>(Arrays.asList(TableFeatures.VARIANT_RW_FEATURE)); + public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { + return TableConfig.VARIANT_SHREDDING_ENABLED.fromMetadata(metadata) + && + // Don't automatically enable the stable feature if the preview feature is + // already supported, to avoid possibly breaking old clients that only + // support the preview feature. + !protocol.supportsFeature(VARIANT_SHREDDING_PREVIEW_RW_FEATURE); } } - /* ---- End: variantShredding-preview ---- */ + + public static final TableFeature VARIANT_SHREDDING_RW_FEATURE = + new VariantShreddingTableFeature(); + public static final TableFeature VARIANT_SHREDDING_PREVIEW_RW_FEATURE = + new VariantShreddingTableFeatureBase("variantShredding-preview"); + /* ---- End: variantShredding ---- */ public static final TableFeature DOMAIN_METADATA_W_FEATURE = new DomainMetadataFeature(); @@ -557,6 +579,7 @@ private static class MaterializePartitionColumnsFeature extends TableFeature.Wri VACUUM_PROTOCOL_CHECK_RW_FEATURE, VARIANT_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, + VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, ICEBERG_WRITER_COMPAT_V1, ICEBERG_WRITER_COMPAT_V3)); diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala index d90a5f9f62b..d96df5f2720 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala @@ -290,6 +290,11 @@ class ProtocolSuite extends AnyFunSuite with TestUtils { 7, Set[String]("variantType", "variantShredding-preview"), Set[String]("variantType", "variantShredding-preview")), + Set("variantShredding") -> ( + 3, + 7, + Set[String]("variantType", "variantShredding"), + Set[String]("variantType", "variantShredding")), Set("rowTracking") -> ( 1, 7, diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala index 330603f9509..9410747ac58 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala @@ -76,6 +76,7 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite ICEBERG_COMPAT_V3_W_FEATURE, DELETION_VECTORS_RW_FEATURE, VARIANT_RW_FEATURE, + VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, ROW_TRACKING_W_FEATURE, @@ -100,6 +101,7 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite "rowTracking", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview", tableFeature).asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) @@ -425,6 +427,7 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite "rowTracking", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview", "icebergCompatV2", "icebergWriterCompatV1", diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala index 6da7863e6ae..61b5d5ef1b5 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala @@ -50,6 +50,7 @@ class TableFeaturesSuite extends AnyFunSuite { "vacuumProtocolCheck", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview") val writerOnlyFeatures = Seq( @@ -132,11 +133,11 @@ class TableFeaturesSuite extends AnyFunSuite { ("rowTracking", testMetadata(tblProps = Map("delta.enableRowTracking" -> "true")), true), ("rowTracking", testMetadata(tblProps = Map("delta.enableRowTracking" -> "false")), false), ( - "variantShredding-preview", + "variantShredding", testMetadata(tblProps = Map("delta.enableVariantShredding" -> "true")), true), ( - "variantShredding-preview", + "variantShredding", testMetadata(tblProps = Map("delta.enableVariantShredding" -> "false")), false), ( @@ -205,7 +206,8 @@ class TableFeaturesSuite extends AnyFunSuite { "vacuumProtocolCheck", "clustering", "catalogManaged", - "allowColumnDefaults").foreach { + "allowColumnDefaults", + "variantShredding-preview").foreach { feature => test(s"doesn't support auto enable by metadata: $feature") { val tableFeature = TableFeatures.getTableFeature(feature) @@ -215,7 +217,10 @@ class TableFeaturesSuite extends AnyFunSuite { Seq( ("variantType", testMetadata(includeVariantTypeCol = true)), - ("typeWidening", testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true")))).foreach { + ("typeWidening", testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true"))), + ( + "variantShredding", + testMetadata(tblProps = Map("delta.enableVariantShredding" -> "true")))).foreach { case (feature, metadataEnablingFeature) => test("special handling of tables containing preview features: " + feature) { val protocolWithPreviewFeature = new Protocol(3, 7) @@ -242,6 +247,7 @@ class TableFeaturesSuite extends AnyFunSuite { "v2Checkpoint", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview", "typeWidening", "typeWidening-preview", @@ -287,6 +293,7 @@ class TableFeaturesSuite extends AnyFunSuite { "clustering", "variantType-preview", "variantType", + "variantShredding", "variantShredding-preview", "materializePartitionColumns") @@ -331,6 +338,7 @@ class TableFeaturesSuite extends AnyFunSuite { "catalogManaged", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview", "deletionVectors", "typeWidening", @@ -581,19 +589,20 @@ class TableFeaturesSuite extends AnyFunSuite { testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true"))) } - Seq("variantType", "variantType-preview", "variantShredding-preview").foreach { feature => - checkWriteSupported( - s"validateKernelCanWriteToTable: protocol 7 with $feature, " + - s"metadata doesn't contains $feature", - new Protocol(3, 7, singleton(feature), singleton(feature)), - testMetadata()) - - checkWriteSupported( - s"validateKernelCanWriteToTable: protocol 7 with $feature, " + - s"metadata contains $feature", - new Protocol(3, 7, singleton(feature), singleton(feature)), - testMetadata(includeVariantTypeCol = true)) - } + Seq("variantType", "variantType-preview", "variantShredding", "variantShredding-preview") + .foreach { feature => + checkWriteSupported( + s"validateKernelCanWriteToTable: protocol 7 with $feature, " + + s"metadata doesn't contains $feature", + new Protocol(3, 7, singleton(feature), singleton(feature)), + testMetadata()) + + checkWriteSupported( + s"validateKernelCanWriteToTable: protocol 7 with $feature, " + + s"metadata contains $feature", + new Protocol(3, 7, singleton(feature), singleton(feature)), + testMetadata(includeVariantTypeCol = true)) + } checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with vacuumProtocolCheck, " + @@ -1043,9 +1052,9 @@ class TableFeaturesSuite extends AnyFunSuite { set( "columnMapping", "deletionVectors", - "variantShredding-preview", + "variantShredding", "variantType")), - set("variantType", "variantShredding-preview"))).foreach { + set("variantType", "variantShredding"))).foreach { case (newMetadata, currentProtocol, expectedProtocol, expectedNewFeatures) => test(s"autoUpgradeProtocolBasedOnMetadata:" + s"$currentProtocol -> $expectedProtocol, $expectedNewFeatures") { @@ -1145,7 +1154,18 @@ class TableFeaturesSuite extends AnyFunSuite { 3, 7, set("columnMapping", "deletionVectors"), - set("columnMapping", "deletionVectors", "icebergCompatV2")))).foreach { + set("columnMapping", "deletionVectors", "icebergCompatV2"))), + ( + // Enable the variantShredding GA feature when the preview feature is already enabled. + // The GA feature should not be auto-enabled. + testMetadata( + tblProps = Map("delta.enableVariantShredding" -> "true"), + includeVariantTypeCol = true), + new Protocol( + 3, + 7, + set("variantType", "variantShredding-preview"), + set("variantType", "variantShredding-preview")))).foreach { case (newMetadata, currentProtocol) => test(s"autoUpgradeProtocolBasedOnMetadata: no-op upgrade: $currentProtocol") { val newProtocolAndNewFeaturesEnabled = diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala index f2c6d6b0e4a..c0bb7b878af 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala @@ -408,6 +408,80 @@ trait DeltaTableFeaturesSuiteBase extends AnyFunSuite with AbstractWriteUtils { } /* ---- End: type widening tests ---- */ + /* ---- Start: variant shredding tests ---- */ + test("only variantShredding feature is enabled when metadata supports it: new table") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable( + engine, + tablePath = tablePath, + schema = testSchema, + tableProperties = Map("delta.enableVariantShredding" -> "true")) + + val protocolV0 = getProtocol(engine, tablePath) + assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + assert(protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + + updateTableMetadata( + engine = engine, + tablePath = tablePath, + tableProperties = Map("delta.enableVariantShredding" -> "true")) + val protocolV1 = getProtocol(engine, tablePath) + assert(protocolV1 === protocolV0) + } + } + + test("only variantShredding feature is enabled when new metadata supports it: existing table") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable(engine, tablePath = tablePath, schema = testSchema) + val protocolV0 = getProtocol(engine, tablePath) + assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + + updateTableMetadata( + engine = engine, + tablePath = tablePath, + tableProperties = Map("delta.enableVariantShredding" -> "true")) + val protocolV1 = getProtocol(engine, tablePath) + assert(!protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + assert(protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + } + } + + test("variantShredding-preview in existing table is respected") { + withTempDirAndEngine { (tablePath, engine) => + spark.sql(s"CREATE TABLE delta.`$tablePath`(id INT) USING delta " + + s"TBLPROPERTIES ('delta.feature.variantShredding-preview' = 'supported')") + + val protocolV0 = getProtocol(engine, tablePath) + require(protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + require(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + + updateTableMetadata( + engine = engine, + tablePath = tablePath, + tableProperties = Map("delta.enableVariantShredding" -> "true")) + val protocolV1 = getProtocol(engine, tablePath) + assert(protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + assert(!protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + } + } + test("both variantShredding and variantShredding-preview can coexist") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable( + engine, + tablePath = tablePath, + schema = testSchema, + tableProperties = Map( + "delta.enableVariantShredding" -> "true", + "delta.feature.variantShredding-preview" -> "supported")) + + val protocol = getProtocol(engine, tablePath) + assert(protocol.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + assert(protocol.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + } + } + /* ---- End: variant shredding tests ---- */ + /* ---- Start: materialize partition columns tests ---- */ test("materialize partition columns in parquet schema when feature is enabled") { withTempDirAndEngine { (tablePath, engine) => From cd215cb6935341745e8cd72a3c34fec71fcb002e Mon Sep 17 00:00:00 2001 From: yyanyy Date: Fri, 20 Mar 2026 20:56:15 -0700 Subject: [PATCH 51/80] [Spark] Fix flaky testPlanInputPartitionsGroupsFilesByPartition (#6344) --- .../spark/internal/v2/read/SparkScanTest.java | 87 ++++++++++--------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java index f350b6d3ce8..f64265a21d9 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java @@ -1067,7 +1067,8 @@ public void testPlanInputPartitionsReturnsHasPartitionKeyForPartitionedTable() { } @Test - public void testPlanInputPartitionsGroupsFilesByPartition(@TempDir File tempDir) { + public void testPlanInputPartitionsGroupsFilesByPartition(@TempDir File tempDir) + throws Exception { // Create a table with multiple files per partition to actually exercise the grouping logic // in planPartitionedInputPartitions (the default test table has 1 file per partition, // which would pass even without grouping). @@ -1091,44 +1092,52 @@ public void testPlanInputPartitionsGroupsFilesByPartition(@TempDir File tempDir) tempDir.getAbsolutePath(), options); - SparkScanBuilder builder = (SparkScanBuilder) multiFileTable.newScanBuilder(options); - SparkScan scan = (SparkScan) builder.build(); - Batch batch = scan.toBatch(); - - InputPartition[] partitions = batch.planInputPartitions(); - - // Verify all partitions are DeltaInputPartition with partition keys - Map> partitionsByKey = new HashMap<>(); - for (InputPartition p : partitions) { - assertTrue(p instanceof DeltaInputPartition); - DeltaInputPartition dp = (DeltaInputPartition) p; - partitionsByKey.computeIfAbsent(dp.partitionKey(), k -> new ArrayList<>()).add(dp); - } - - // Should have exactly 2 unique partition keys (part=1 and part=2) - assertEquals(2, partitionsByKey.size(), "Should have 2 unique partition keys"); - - // Verify that the grouping actually produced multiple DeltaInputPartitions for a single - // partition key (since multiple files exist per partition and each gets its own - // FilePartition at default maxSplitBytes) - int totalPartitions = partitions.length; - assertTrue( - totalPartitions > 2, - "With 5 files across 2 partitions, should have more than 2 input partitions, " - + "got " - + totalPartitions); - - // Verify all DeltaInputPartitions with the same key share the same partition key instance - for (Map.Entry> entry : partitionsByKey.entrySet()) { - List group = entry.getValue(); - InternalRow expectedKey = group.get(0).partitionKey(); - for (DeltaInputPartition dp : group) { - assertEquals( - expectedKey, - dp.partitionKey(), - "All partitions in the same group should have equal partition keys"); - } - } + // Force maxPartitionBytes=1 so each file gets its own FilePartition, making the + // totalPartitions > 2 assertion deterministic regardless of default parallelism. + withSQLConf( + "spark.sql.files.maxPartitionBytes", + "1", + () -> { + SparkScanBuilder builder = (SparkScanBuilder) multiFileTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + Batch batch = scan.toBatch(); + + InputPartition[] partitions = batch.planInputPartitions(); + + // Verify all partitions are DeltaInputPartition with partition keys + Map> partitionsByKey = new HashMap<>(); + for (InputPartition p : partitions) { + assertTrue(p instanceof DeltaInputPartition); + DeltaInputPartition dp = (DeltaInputPartition) p; + partitionsByKey.computeIfAbsent(dp.partitionKey(), k -> new ArrayList<>()).add(dp); + } + + // Should have exactly 2 unique partition keys (part=1 and part=2) + assertEquals(2, partitionsByKey.size(), "Should have 2 unique partition keys"); + + // Verify that the grouping actually produced multiple DeltaInputPartitions for a + // single partition key (since multiple files exist per partition and each gets its + // own FilePartition when maxPartitionBytes=1) + int totalPartitions = partitions.length; + assertTrue( + totalPartitions > 2, + "With 5 files across 2 partitions, should have more than 2 input partitions, " + + "got " + + totalPartitions); + + // Verify all DeltaInputPartitions with the same key share the same partition key + for (Map.Entry> entry : + partitionsByKey.entrySet()) { + List group = entry.getValue(); + InternalRow expectedKey = group.get(0).partitionKey(); + for (DeltaInputPartition dp : group) { + assertEquals( + expectedKey, + dp.partitionKey(), + "All partitions in the same group should have equal partition keys"); + } + } + }); } @Test From 887ca8431b6ac76541ad8f04a48e54e3a0fbe9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ili=C4=87?= Date: Sat, 21 Mar 2026 15:05:06 +0100 Subject: [PATCH 52/80] [KERNEL] Add collations table feature (#5718) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [x] Kernel - [ ] Other (fill in here) ## Description Add collations table features, `collations-preview` and `collations`, as writer features. ## How was this patch tested? New test using golden table. ## Does this PR introduce _any_ user-facing changes? No. --- .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 20 bytes .../_delta_log/00000000000000000000.crc | 1 + .../_delta_log/00000000000000000000.json | 3 + .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 20 bytes .../_delta_log/00000000000000000000.crc | 1 + .../_delta_log/00000000000000000000.json | 3 + .../internal/tablefeatures/TableFeatures.java | 46 ++++++- ...atV1MetadataValidatorAndUpdaterSuite.scala | 5 +- ...atV3MetadataValidatorAndUpdaterSuite.scala | 2 +- .../tablefeatures/TableFeaturesSuite.scala | 26 +++- .../defaults/DeltaTableWritesSuite.scala | 126 ++++++++++++++++++ .../defaults/IcebergWriterCompatV1Suite.scala | 6 - 12 files changed, 203 insertions(+), 16 deletions(-) create mode 100644 connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/.00000000000000000000.json.crc create mode 100644 connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.crc create mode 100644 connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.json create mode 100644 connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/.00000000000000000000.json.crc create mode 100644 connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/00000000000000000000.crc create mode 100644 connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/00000000000000000000.json diff --git a/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/.00000000000000000000.json.crc b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..69ac0d941a97bd80e8500c2edb4fe3b4c1fb0aa1 GIT binary patch literal 20 bcmYc;N@ieSU}DIAllo?tM2eD6UGiA~Ky?Q- literal 0 HcmV?d00001 diff --git a/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.crc b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.crc new file mode 100644 index 00000000000..b499b99ebae --- /dev/null +++ b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.crc @@ -0,0 +1 @@ +{"txnId":"6132e880-0f3a-4db4-b882-1da039bffbad","tableSizeBytes":0,"numFiles":0,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"0eb3e007-b3cc-40e4-bca1-a5970d86b5a6","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_binary_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_lcase_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"utf8_lcase_col\":\"spark.UTF8_LCASE\"}}},{\"name\":\"unicode_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"unicode_col\":\"icu.UNICODE\"}}}]}","partitionColumns":[],"configuration":{},"createdTime":1773779518731},"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["domainMetadata","collations-preview","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"allFiles":[]} diff --git a/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.json b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000..d83ea327864 --- /dev/null +++ b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1773779521901,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","statsOnLoad":false,"partitionBy":"[]"},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"0","numOutputRows":"0","numOutputBytes":"0"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/","txnId":"6132e880-0f3a-4db4-b882-1da039bffbad"}} +{"metaData":{"id":"0eb3e007-b3cc-40e4-bca1-a5970d86b5a6","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_binary_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_lcase_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"utf8_lcase_col\":\"spark.UTF8_LCASE\"}}},{\"name\":\"unicode_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"unicode_col\":\"icu.UNICODE\"}}}]}","partitionColumns":[],"configuration":{},"createdTime":1773779518731}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["domainMetadata","collations-preview","appendOnly","invariants"]}} diff --git a/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/.00000000000000000000.json.crc b/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..e62bc23a8efbec38cfcbed20cafa7dc7f5eb7118 GIT binary patch literal 20 bcmYc;N@ieSU}Eq-`TOJ1{=","txnId":"534b1ff2-bfc1-4d6f-b435-08baef2d8dd0"}} +{"metaData":{"id":"dffc30c0-bb9a-4f95-91f9-3141788a8094","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_binary_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_lcase_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"utf8_lcase_col\":\"spark.UTF8_LCASE\"}}},{\"name\":\"unicode_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"unicode_col\":\"icu.UNICODE\"}}}]}","partitionColumns":[],"configuration":{},"createdTime":1773774637236}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["domainMetadata","collations","appendOnly","invariants"]}} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java index e375f9b3be6..6df63150af6 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java @@ -538,6 +538,34 @@ private static class MaterializePartitionColumnsFeature extends TableFeature.Wri } } + /** This feature was replaced with its stable version, `collations`. */ + public static final TableFeature COLLATIONS_PREVIEW_W_FEATURE = new CollationsPreview(); + + private static class CollationsPreview extends TableFeature.WriterFeature { + CollationsPreview() { + super("collations-preview", /* minWriterVersion = */ 7); + } + } + + /** The stable collation feature. */ + public static final TableFeature COLLATIONS_W_FEATURE = new Collations(); + + private static class Collations extends TableFeature.WriterFeature + implements FeatureAutoEnabledByMetadata { + Collations() { + super("collations", /* minWriterVersion = */ 7); + } + + @Override + public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { + return hasCollatedColumn(metadata.getSchema()) + && + // Don't automatically enable the stable feature if the preview feature is already + // supported, to avoid breaking old clients that only support the preview feature. + !protocol.supportsFeature(COLLATIONS_PREVIEW_W_FEATURE); + } + } + ///////////////////////////////////////////////////////////////////////////////// /// END: Define the {@link TableFeature}s /// ///////////////////////////////////////////////////////////////////////////////// @@ -582,7 +610,9 @@ private static class MaterializePartitionColumnsFeature extends TableFeature.Wri VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, ICEBERG_WRITER_COMPAT_V1, - ICEBERG_WRITER_COMPAT_V3)); + ICEBERG_WRITER_COMPAT_V3, + COLLATIONS_PREVIEW_W_FEATURE, + COLLATIONS_W_FEATURE)); public static final Map TABLE_FEATURE_MAP = Collections.unmodifiableMap( @@ -887,4 +917,18 @@ private static boolean hasTypeColumn(StructType tableSchema, DataType type) { return new SchemaIterable(tableSchema) .stream().anyMatch(element -> element.getField().getDataType().equals(type)); } + + /** + * Check if the table schema has any string column with a non-default (non-UTF8_BINARY) collation. + */ + static boolean hasCollatedColumn(StructType tableSchema) { + return new SchemaIterable(tableSchema) + .stream() + .anyMatch( + element -> { + DataType dataType = element.getField().getDataType(); + return dataType instanceof StringType + && !((StringType) dataType).isUTF8BinaryCollated(); + }); + } } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala index 75c52426e92..4f449c47679 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala @@ -389,7 +389,8 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite Seq( // "defaultColumns", add this to this test once we support defaultColumns - // "collations", add this to this test once we support collations + "collations", + "collations-preview", "variantType").foreach { incompatibleFeature => test(s"cannot enable with incompatible feature $incompatibleFeature") { checkUnsupportedOrIncompatibleFeature( @@ -399,7 +400,7 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite } } - Seq("collations", "defaultColumns").foreach { unsupportedIncompatibleFeature => + Seq("defaultColumns").foreach { unsupportedIncompatibleFeature => test(s"cannot enable with incompatible UNSUPPORTED feature $unsupportedIncompatibleFeature") { // We add this test here so that it will fail when we add Kernel support for these features // When this happens -> add the feature to the test above diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala index 9410747ac58..05e09e34e59 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala @@ -438,7 +438,7 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV3Metadata(false, metadata, protocol, Optional.empty()) } - Seq("collations", "defaultColumns").foreach { unsupportedIncompatibleFeature => + Seq("defaultColumns").foreach { unsupportedIncompatibleFeature => test(s"cannot enable with incompatible UNSUPPORTED feature $unsupportedIncompatibleFeature") { // We add this test here so that it will fail when we add Kernel support for these features // When this happens -> add the feature to the test above diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala index 61b5d5ef1b5..850fc32be94 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala @@ -70,7 +70,9 @@ class TableFeaturesSuite extends AnyFunSuite { "icebergWriterCompatV1", "icebergWriterCompatV3", "clustering", - "materializePartitionColumns") + "materializePartitionColumns", + "collations", + "collations-preview") val legacyFeatures = Seq( "appendOnly", @@ -150,6 +152,8 @@ class TableFeaturesSuite extends AnyFunSuite { false), ("timestampNtz", testMetadata(includeTimestampNtzTypeCol = true), true), ("timestampNtz", testMetadata(includeTimestampNtzTypeCol = false), false), + ("collations", testMetadata(includeCollatedStringTypeCol = true), true), + ("collations", testMetadata(includeCollatedStringTypeCol = false), false), ("v2Checkpoint", testMetadata(tblProps = Map("delta.checkpointPolicy" -> "v2")), true), ("v2Checkpoint", testMetadata(tblProps = Map("delta.checkpointPolicy" -> "classic")), false), ( @@ -207,7 +211,8 @@ class TableFeaturesSuite extends AnyFunSuite { "clustering", "catalogManaged", "allowColumnDefaults", - "variantShredding-preview").foreach { + "variantShredding-preview", + "collations-preview").foreach { feature => test(s"doesn't support auto enable by metadata: $feature") { val tableFeature = TableFeatures.getTableFeature(feature) @@ -220,7 +225,8 @@ class TableFeaturesSuite extends AnyFunSuite { ("typeWidening", testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true"))), ( "variantShredding", - testMetadata(tblProps = Map("delta.enableVariantShredding" -> "true")))).foreach { + testMetadata(tblProps = Map("delta.enableVariantShredding" -> "true"))), + ("collations", testMetadata(includeCollatedStringTypeCol = true))).foreach { case (feature, metadataEnablingFeature) => test("special handling of tables containing preview features: " + feature) { val protocolWithPreviewFeature = new Protocol(3, 7) @@ -295,7 +301,9 @@ class TableFeaturesSuite extends AnyFunSuite { "variantType", "variantShredding", "variantShredding-preview", - "materializePartitionColumns") + "materializePartitionColumns", + "collations", + "collations-preview") assert(results.map(_.featureName()).toSet == expected.toSet) } @@ -1256,13 +1264,15 @@ class TableFeaturesSuite extends AnyFunSuite { includeVariantTypeCol: Boolean = false, includeGeneratedColumn: Boolean = false, includeIdentityColumn: Boolean = false, + includeCollatedStringTypeCol: Boolean = false, tblProps: Map[String, String] = Map.empty): Metadata = { val testSchema = createTestSchema( includeInvariant, includeTimestampNtzTypeCol, includeVariantTypeCol, includeGeneratedColumn, - includeIdentityColumn) + includeIdentityColumn, + includeCollatedStringTypeCol) new Metadata( "id", Optional.of("name"), @@ -1290,7 +1300,8 @@ class TableFeaturesSuite extends AnyFunSuite { includeTimestampNtzTypeCol: Boolean = false, includeVariantTypeCol: Boolean = false, includeGeneratedColumn: Boolean = false, - includeIdentityColumn: Boolean = false): StructType = { + includeIdentityColumn: Boolean = false, + includeCollatedStringTypeCol: Boolean = false): StructType = { var structType = new StructType() .add("c1", IntegerType.INTEGER) .add("c2", StringType.STRING) @@ -1326,6 +1337,9 @@ class TableFeaturesSuite extends AnyFunSuite { .putBoolean("delta.identity.allowExplicitInsert", true) .build()) } + if (includeCollatedStringTypeCol) { + structType = structType.add("c8", new StringType("ICU.UNICODE.75")) + } structType } diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala index 92928d8c645..29d1d47c26e 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala @@ -1031,6 +1031,132 @@ abstract class AbstractDeltaTableWritesSuite extends AnyFunSuite with AbstractWr } } + test("create table with collated columns - collations writer feature in protocol") { + Seq( + new StructType() + .add("id", IntegerType.INTEGER) + .add("val", new StringType("ICU.UNICODE")), + new StructType() + .add("id", IntegerType.INTEGER) + .add("val", new StringType("ICU.SR_CYRL_SRB")), + new StructType() + .add("id", IntegerType.INTEGER) + .add( + "info", + new StructType() + .add("name", new StringType("ICU.UNICODE.74.1"))), + new StructType() + .add("id", IntegerType.INTEGER) + .add("tags", new ArrayType(new StringType("SPARK.UTF8_LCASE"), true)), + new StructType() + .add("id", IntegerType.INTEGER) + .add( + "props", + new MapType(StringType.STRING, new StringType("ICU.UNICODE"), true))).foreach { schema => + withTempDirAndEngine { (tblPath, engine) => + appendData( + engine, + tblPath, + isNewTable = true, + schema, + data = Seq.empty) + + val protocol = getProtocol(engine, tblPath) + assert(protocol.getWriterFeatures.contains("collations")) + } + } + } + + test("create table without collated columns - collations feature NOT in protocol") { + withTempDirAndEngine { (tblPath, engine) => + val schema = new StructType() + .add("id", IntegerType.INTEGER) + .add("name", StringType.STRING) // default UTF8_BINARY collation + + appendData( + engine, + tblPath, + isNewTable = true, + schema, + data = Seq.empty) + + val protocol = getProtocol(engine, tblPath) + assert( + !protocol.getWriterFeatures.contains("collations"), + "Should NOT have 'collations' writer feature when schema has no custom collations") + } + } + + test("write to golden table with collated columns (utf8_binary, utf8_lcase, unicode)") { + Seq( + ("collations-table", "collations"), + ("collations-preview-table", "collations-preview")).foreach { + case (collationTable, feature) => + withTempDirAndEngine { (tblPath, engine) => + copyTable(collationTable, tblPath) + + val utf8Lcase = new StringType("SPARK.UTF8_LCASE") + val unicode = new StringType("ICU.UNICODE") + + val schema = new StructType() + .add("id", IntegerType.INTEGER) + .add("utf8_binary_col", StringType.STRING) + .add("utf8_lcase_col", utf8Lcase) + .add("unicode_col", unicode) + + // Verify golden table schema has expected collations + val goldenMetadata = getMetadata(engine, tblPath) + val goldenSchema = + DataTypeJsonSerDe.deserializeStructType(goldenMetadata.getSchemaString()) + assert(goldenSchema === schema) + + // Verify protocol has the expected collations writer feature + val protocol = getProtocol(engine, tblPath) + assert(protocol.getWriterFeatures.contains(feature)) + + // First append: data batch uses matching collations + val data1 = generateData(schema, Seq.empty, Map.empty, batchSize = 10, numBatches = 1) + + val commitResult1 = appendData( + engine, + tblPath, + data = Seq(Map.empty[String, Literal] -> data1)) + + verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) + verifyCommitInfo(tblPath, version = 1, partitionCols = null) + verifyWrittenContent(tblPath, schema, data1.flatMap(_.toTestRows)) + + // Second append: data batch uses different collations than the table schema + val diffCollationSchema = new StructType() + .add("id", IntegerType.INTEGER) + .add("utf8_binary_col", new StringType("ICU.UNICODE.75.1")) + .add("utf8_lcase_col", new StringType("SPARK.UTF8_BINARY")) + .add("unicode_col", utf8Lcase) + + val data2 = + generateData(diffCollationSchema, Seq.empty, Map.empty, batchSize = 5, numBatches = 1) + + val commitResult2 = appendData( + engine, + tblPath, + data = Seq(Map.empty[String, Literal] -> data2)) + + verifyCommitResult(commitResult2, expVersion = 2, expIsReadyForCheckpoint = false) + verifyCommitInfo(tblPath, version = 2, partitionCols = null) + verifyWrittenContent(tblPath, schema, (data1 ++ data2).flatMap(_.toTestRows)) + + // Verify the schema is preserved with collations after writes + val metadata = getMetadata(engine, tblPath) + val parsedSchema = DataTypeJsonSerDe.deserializeStructType(metadata.getSchemaString()) + assert(parsedSchema === schema) + + // After writes, the feature flags must not have changed + val protocolAfterWrite = getProtocol(engine, tblPath) + assert(protocolAfterWrite.getWriterFeatures.contains(feature)) + } + } + } + /////////////////////////////////////////////////////////////////////////// // Create table and insert data tests (CTAS & INSERT) /////////////////////////////////////////////////////////////////////////// diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala index 02c64d0046b..ded5d42cc5a 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala @@ -565,12 +565,6 @@ trait IcebergWriterCompatV1SuiteBase tablePropertiesToEnable = Map("delta.feature.defaultColumns" -> "supported"), expectedErrorMessage = "Unsupported Delta table feature") - // collations is not added to Kernel yet --> throws an error on feature lookup - testIncompatibleUnsupportedTableFeature( - "collations inactive", - tablePropertiesToEnable = Map("delta.feature.collations" -> "supported"), - expectedErrorMessage = "Unsupported Delta table feature") - /* ----- Legacy incompatible features allowed if they are inactive ----- */ test("legacy table features allowed with icebergWriterCompatV1 if inactive") { From 829e63e075af0c3b9065bc8a0015bd01fbae13e9 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Mon, 23 Mar 2026 09:22:00 -0700 Subject: [PATCH 53/80] protocol: Clarify that checkpoints exclude tombstoned domain metadatas (#6337) Clarify in two places that checkpoints and reconciled snapshots should not contain `domainMetadata` actions with `removed=true`: - Action reconciliation: the `domainMetadata` collection excludes tombstones. - Checkpoint contents: the Domain Metadata bullet excludes removed entries. --- PROTOCOL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PROTOCOL.md b/PROTOCOL.md index 1c989a18948..c9015a57f06 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -893,7 +893,7 @@ A given snapshot of the table can be computed by replaying the events committed - A single `protocol` action - A single `metaData` action - A collection of `txn` actions with unique `appId`s - - A collection of `domainMetadata` actions with unique `domain`s. + - A collection of `domainMetadata` actions with unique `domain`s, excluding tombstones (i.e. actions with `removed=true`). - A collection of `add` actions with unique path keys, corresponding to the newest (path, deletionVector.uniqueId) pair encountered for each path. - A collection of `remove` actions with unique `(path, deletionVector.uniqueId)` keys. The intersection of the primary keys in the `add` collection and `remove` collection must be empty. That means a logical file cannot exist in both the `remove` and `add` collections at the same time; however, the same *data file* can exist with *different* DVs in the `remove` collection, as logically they represent different content. The `remove` actions act as _tombstones_, and only exist for the benefit of the VACUUM command. Snapshot reads only return `add` actions on the read path. @@ -2090,7 +2090,7 @@ Each row in the checkpoint corresponds to a single action. The checkpoint **must * Files that have been [added](#Add-File-and-Remove-File) and not yet removed * Files that were recently [removed](#Add-File-and-Remove-File) and have not yet expired * [Transaction identifiers](#Transaction-Identifiers) - * [Domain Metadata](#Domain-Metadata) + * [Domain Metadata](#Domain-Metadata) that have not been removed (i.e. excluding tombstones with `removed=true`) * [Checkpoint Metadata](#checkpoint-metadata) - Requires [V2 checkpoints](#v2-spec) * [Sidecar File](#sidecar-files) - Requires [V2 checkpoints](#v2-spec) From e8cffee7b45701910d0a6bd7a9edb26e6fcaae8c Mon Sep 17 00:00:00 2001 From: richardc-db <87336575+richardc-db@users.noreply.github.com> Date: Mon, 23 Mar 2026 11:05:43 -0700 Subject: [PATCH 54/80] [KERNEL][VARIANT] make kernel shredding test not use spark (#6350) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [x] Kernel - [ ] Other (fill in here) ## Description test only follow up to #6322. revises a test to not be reliant on delta-spark versions. ## How was this patch tested? test only. test still passes ## Does this PR introduce _any_ user-facing changes? no --- .../io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala index c0bb7b878af..28fe7d300f0 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala @@ -449,8 +449,11 @@ trait DeltaTableFeaturesSuiteBase extends AnyFunSuite with AbstractWriteUtils { test("variantShredding-preview in existing table is respected") { withTempDirAndEngine { (tablePath, engine) => - spark.sql(s"CREATE TABLE delta.`$tablePath`(id INT) USING delta " + - s"TBLPROPERTIES ('delta.feature.variantShredding-preview' = 'supported')") + createEmptyTable( + engine, + tablePath = tablePath, + schema = testSchema, + tableProperties = Map("delta.feature.variantShredding-preview" -> "supported")) val protocolV0 = getProtocol(engine, tablePath) require(protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) From f50715459ce5252deb6a3efddfaa1f269668b7f5 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Mon, 23 Mar 2026 14:47:32 -0700 Subject: [PATCH 55/80] [Spark] Fix decimal IN predicate crash when BigDecimal precision < scale (#6354) ## Summary - Fix `ExpressionUtils.convertValueToKernelLiteral` crash when a `BigDecimal` has precision less than scale - Normalize precision to `Math.max(bd.precision(), bd.scale() + 1)` before calling `Literal.ofDecimal` - Add unit test for the edge case ## Problem A query with a decimal `IN` predicate containing `0.00` crashes when using the V2 connector: ```sql SELECT * FROM delta_table WHERE dec10_2 IN (0.00, 100.00) ``` This throws an `IllegalArgumentException: Invalid precision and scale combo` from Kernel's `Literal.ofDecimal`. **Root cause:** Java's `BigDecimal("0.00")` reports `precision()=1` and `scale()=2`. This violates the invariant that `precision >= scale` required by Kernel's `Literal.ofDecimal`. The V2 code in `ExpressionUtils.convertValueToKernelLiteral` was passing these values through without normalization. ## Fix Before constructing the Kernel literal, normalize precision: ```java int precision = Math.max(bd.precision(), bd.scale() + 1); ``` For `BigDecimal("0.00")`, this yields `precision=3, scale=2` (i.e., `DECIMAL(3,2)`), which correctly represents the value. ## Test plan - [x] New unit test `testConvertValueToKernelLiteral_DecimalWithScaleExceedingPrecision` passes - [x] All 73 `ExpressionUtilsTest` tests pass --- .../spark/internal/v2/utils/ExpressionUtils.java | 6 ++++-- .../internal/v2/utils/ExpressionUtilsTest.java | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java index 42cb805ed74..b5a6dea697c 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java @@ -395,9 +395,11 @@ static Optional convertValueToKernelLiteral(Object value) { return Optional.of(Literal.ofDouble(d)); } if (value instanceof BigDecimal) { - // Preserve precision and scale from the original BigDecimal BigDecimal bd = (BigDecimal) value; - return Optional.of(Literal.ofDecimal(bd, bd.precision(), bd.scale())); + // BigDecimal precision can be less than scale (e.g. BigDecimal("0.00") has + // precision=1, scale=2). Kernel requires precision >= scale, so normalize. + int precision = Math.max(bd.precision(), bd.scale() + 1); + return Optional.of(Literal.ofDecimal(bd, precision, bd.scale())); } if (value instanceof UTF8String) { UTF8String s = (UTF8String) value; diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java index b9230a24bcd..07f85c7a33d 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java @@ -451,6 +451,22 @@ public void testConvertValueToKernelLiteral( "DataType should match expected type for " + typeName); } + @Test + public void testConvertValueToKernelLiteral_DecimalWithScaleExceedingPrecision() { + // BigDecimal("0.00") has precision=1, scale=2 which violates precision >= scale. + // The conversion should normalize precision to max(precision, scale + 1). + BigDecimal bd = new BigDecimal("0.00"); + assertEquals(1, bd.precision(), "Precondition: precision of 0.00 should be 1"); + assertEquals(2, bd.scale(), "Precondition: scale of 0.00 should be 2"); + + Optional result = ExpressionUtils.convertValueToKernelLiteral(bd); + assertTrue(result.isPresent(), "BigDecimal 0.00 should be convertible"); + + Literal literal = result.get(); + // Normalized precision should be max(1, 2+1) = 3 + assertEquals(new DecimalType(3, 2), literal.getDataType()); + } + @Test public void testConvertValueToKernelLiteral_NullValue() { Optional result = ExpressionUtils.convertValueToKernelLiteral(null); From 4875065a1e41c7a22ba342faa6cefc22d6dedda5 Mon Sep 17 00:00:00 2001 From: openinx Date: Mon, 23 Mar 2026 15:32:45 -0700 Subject: [PATCH 56/80] [Spark] Fix misleading error message when UniForm Iceberg is enabled without a supported IcebergCompat version (#6352) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description When a user enables `delta.universalFormat.enabledFormats = 'iceberg'` alongside an unrecognized IcebergCompat property (e.g. `delta.enableIcebergCompatV3`), the error message currently says: > To enable IcebergCompatV2, set the table property 'delta.enableIcebergCompatV2' = 'true'. This is misleading because: 1. Spark Delta silently ignores the unrecognized V3 property (only V1 and V2 are in `IcebergCompat.knownVersions`) 2. The error then directs the user to V2 with no indication that V3 is not supported The fix updates the message to explicitly state that the supported versions are IcebergCompatV1 and IcebergCompatV2, making clear to users that higher versions are not yet supported in this Spark Delta release. Resolves #6351 ## How was this patch tested? The change is limited to an error message string. Manual reproduction: ```sql SET spark.databricks.delta.allowArbitraryProperties.enabled=true; CREATE TABLE demo.icebergCompatV3 (i INT, s STRING) USING DELTA TBLPROPERTIES ( 'delta.columnMapping.mode' = 'name', 'delta.enableIcebergCompatV3' = 'true', 'delta.enableDeletionVectors' = 'false', 'delta.universalFormat.enabledFormats' = 'iceberg' ); ``` **Before:** `"To enable IcebergCompatV2, set the table property 'delta.enableIcebergCompatV2' = 'true'."` **After:** `"Supported versions are IcebergCompatV1 and IcebergCompatV2."` ## Does this PR introduce _any_ user-facing changes? Yes. The error message for `DELTA_UNIVERSAL_FORMAT_VIOLATION` (when UniForm Iceberg is enabled without a recognized IcebergCompat version) now explicitly lists the supported IcebergCompat versions (V1 and V2), instead of directing the user to enable V2. This helps users who set an unsupported version (e.g. V3) understand why the error occurred. --------- Signed-off-by: openinx --- .../main/scala/org/apache/spark/sql/delta/DeltaErrors.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala index 2c9a81a73f3..aa7a96cef17 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala @@ -3542,8 +3542,8 @@ trait DeltaErrorsBase messageParameters = Array( UniversalFormat.ICEBERG_FORMAT, "Requires IcebergCompat to be explicitly enabled in order for Universal Format (Iceberg) " + - "to be enabled on an existing table. To enable IcebergCompatV2, set the table property " + - "'delta.enableIcebergCompatV2' = 'true'." + "to be enabled on an existing table. Supported versions are IcebergCompatV1 and " + + "IcebergCompatV2." ) ) } From 914dd23d2a6f5de85a8b3e152dbf32d91683c34c Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Mon, 23 Mar 2026 19:45:27 -0700 Subject: [PATCH 57/80] [Spark] Add regression test for streaming data loss with coordinated commits (#6338) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description This PR adds a regression test that exposes a **data loss bug** in Delta streaming when used with Coordinated Commits and non-trivial backfill batch sizes. Related issue: #6339 ### Observed Behavior The test writes 100 sequential single-row commits to a Delta table while a streaming query is running, then verifies all 100 values appear in the sink. - **batchSize = 1**: Test passes consistently. All 100 values present. - **batchSize = 2**: Test fails. ~4 out of 100 commits are lost on average. **Only odd-versioned commits are lost** (these are the versions that are not immediately backfilled). - **batchSize = 3**: Test fails. The loss pattern follows the backfill cycle: - `v % 3 == 0`: no loss (these versions trigger backfill) - `v % 3 == 1`: v may be lost, and if v is lost, v+1 is always lost together (they get backfilled as a pair) - `v % 3 == 2`: v may be lost independently The pattern strongly correlates with which commits are sitting unbackfilled in the coordinator at any given time, suggesting the bug is related to the interaction between backfill and commit listing. ### Hypothesized Root Cause We suspect the issue is a race condition in `CoordinatedCommitsUtils.commitFilesIterator`, which is used by `DeltaSource.getFileChanges` during both `latestOffset` and `getBatch`. This method lists commits in two lazy, sequential steps: 1. List backfilled commits from the filesystem (`listedDeltas`) 2. Query the coordinator for unbackfilled commits (`tailFromSnapshot`) These two steps are **not atomic**. A possible scenario (example with batchSize = 3): 1. **During `latestOffset`**: filesystem has `[0.json]`, coordinator has `[1, 2]`. `latestOffset` correctly computes endOffset covering through version 2. 2. **During `getBatch`**: the filesystem listing iterator runs and sees `[0.json]`. 3. **Between the two iterators**: a concurrent write creates version 3, triggering `backfillToVersion(3)`. This writes versions 1, 2, 3 to the filesystem and **removes them from the coordinator** via `registerBackfill`. 4. **The coordinator query runs**: returns empty — versions 1 and 2 have been removed. 5. **Result**: `getBatch` misses versions 1 and 2. The next batch starts from version 3, so they are never re-read. ## How was this patch tested? Added a new test `"streaming processes 100 sequential single-value commits and contains all values 0 to 99"` that: - Creates a Delta table and starts a streaming query - Appends 100 single-row commits while the stream is running - Verifies all 100 values appear in the sink The test passes for batchSize = 1 but fails for batchSize = 2 and 3 due to the data loss. ## Does this PR introduce _any_ user-facing changes? No. This PR only adds a test to demonstrate the existing bug. A fix will follow in a subsequent PR. --- .../sql/delta/test/DeltaV2SourceSuite.scala | 1 + .../spark/sql/delta/DeltaSourceSuite.scala | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index 986e4e951db..0cd4a8ba20e 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -135,6 +135,7 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { "fail on data loss - gaps of files", "fail on data loss - starting from missing files with option off", "fail on data loss - gaps of files with option off", + "streaming processes 100 sequential single-value commits and contains all values 0 to 99", // === Misc === // TODO(#5900): fix exception mismatch diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala index 24da9a2f6cc..a8e46da414f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala @@ -2948,6 +2948,47 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } } + + test("streaming processes 100 sequential single-value commits and contains all values 0 to 99") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + // TODO(#6339): enable batch size 2 after fix PR merged + assume(!catalogOwnedCoordinatorBackfillBatchSize.contains(2), + "Test cannot pass with batch size 2 due to issue #6339") + // Write the first value to initialize the Delta table + Seq(0).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions(inputDir.toString, Map.empty[String, String]) + + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + + try { + // Process the initial commit (value 0) + q.processAllAvailable() + + // Append values 1 through 99, one commit each + (1 until 100).foreach { i => + Seq(i).toDF("x") + .write + .format("delta") + .mode("append") + .save(inputDir.toString) + } + + q.processAllAvailable() + + // Verify the output contains exactly all values from 0 to 99 + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + (0 until 100).map(Row(_)) + ) + } finally { + q.stop() + } + } + } } /** From 9642c3ab823ebc1dba92a49932f56cf9956ddfde Mon Sep 17 00:00:00 2001 From: You Zhou <98635051+PorridgeSwim@users.noreply.github.com> Date: Mon, 23 Mar 2026 21:33:51 -0700 Subject: [PATCH 58/80] [kernel-spark] Support ignoreChanges read option in dsv2 (#6249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🥞 Stacked PR Use this [link](https://github.com/delta-io/delta/pull/6249/files/0d54da51e7eade47b8115d92aaf7be1e8e4c011f..7e400f87189bef892d3b0022c0aede238a0c84de) to review incremental changes. - [stack/ignoreDeletesV2](https://github.com/delta-io/delta/pull/6245) [[Files changed](https://github.com/delta-io/delta/pull/6245/files)] - [stack/skipChangeCommitsV2](https://github.com/delta-io/delta/pull/6246) [[Files changed](https://github.com/delta-io/delta/pull/6246/files/6e9962d4a30a63ed14786830bae2b668004a76c9..2e111cf6ac9d1e5f84d83d94412b05486f543613)] - [**stack/ignoreChangesV2**](https://github.com/delta-io/delta/pull/6249) [[Files changed](https://github.com/delta-io/delta/pull/6249/files/0d54da51e7eade47b8115d92aaf7be1e8e4c011f..7e400f87189bef892d3b0022c0aede238a0c84de)] - [stack/ignoreFileDeletionV2](https://github.com/delta-io/delta/pull/6250) [[Files changed](https://github.com/delta-io/delta/pull/6250/files/7e400f87189bef892d3b0022c0aede238a0c84de..af230590d0d9c1a5c37cc6a7b6af404025a6d415)] --------- #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Support ignoreChanges read option in DSv2, which skip all remove file actions but keep the add file actions in the same commit. ## How was this patch tested? Unit tests that test parity between v1 and v2 connector on both pure deletes commit (only remove) and change commit (add + remove). Integration tests: - streaming with ignoreChanges = true allows both delete and change commits ## Does this PR introduce _any_ user-facing changes? No --- .../DeltaV2SourceDeletionVectorsSuite.scala | 18 +- .../sql/delta/test/DeltaV2SourceSuite.scala | 1 + .../spark/sql/delta/DeltaSourceSuite.scala | 51 +++++ .../UCDeltaTableDataFrameStreamingTest.java | 8 - .../v2/read/SparkMicroBatchStream.java | 11 +- .../spark/internal/v2/read/SparkScan.java | 8 +- .../v2/read/SparkMicroBatchStreamTest.java | 208 +++++++++--------- .../spark/internal/v2/read/SparkScanTest.java | 2 +- 8 files changed, 179 insertions(+), 128 deletions(-) diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala index 1a5b5f6ff5e..205297d57f6 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala @@ -44,33 +44,33 @@ class DeltaV2SourceDeletionVectorsSuite "allow to delete files before staring a streaming query without checkpoint", "multiple deletion vectors per file with initial snapshot", "deleting files fails query if ignoreDeletes = false", + "deleting files when ignoreChanges = true doesn't fail the query", "allow to delete files after staring a streaming query when ignoreDeletes is true", "updating the source table causes failure when ignoreChanges = false - using DELETE", + "allow to update the source table when ignoreChanges = true - using DELETE", "updating source table when ignoreDeletes = true fails the query - using DELETE", "subsequent DML commands are processed correctly in a batch - DELETE->DELETE - List()", "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + " - List((ignoreDeletes,true))", "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + " - List((skipChangeCommits,true))", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + + " - List((ignoreChanges,true))", "subsequent DML commands are processed correctly in a batch - INSERT->DELETE - List()", "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + " - List((ignoreDeletes,true))", "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + - " - List((skipChangeCommits,true))" + " - List((skipChangeCommits,true))", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + + " - List((ignoreChanges,true))", + "multiple deletion vectors per file - List((ignoreChanges,true))" ) private lazy val shouldFailTests = Set( // TODO(#5319): enable these tests after ignoreChanges/ignoreFileDeletion read options // are supported by the V2 connector. "allow to delete files after staring a streaming query when ignoreFileDeletion is true", - "allow to update the source table when ignoreChanges = true - using DELETE", - "deleting files when ignoreChanges = true doesn't fail the query", - "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + - " - List((ignoreChanges,true))", - "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + - " - List((ignoreChanges,true))", - "multiple deletion vectors per file - List((ignoreFileDeletion,true))", - "multiple deletion vectors per file - List((ignoreChanges,true))" + "multiple deletion vectors per file - List((ignoreFileDeletion,true))" ) override protected def shouldFail(testName: String): Boolean = { diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index 0cd4a8ba20e..df87b30ba87 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -72,6 +72,7 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { "streaming with ignoreDeletes = true skips delete-only commits", "streaming with ignoreDeletes = true still fails on change commits", "streaming with skipChangeCommits = true skips both delete and change commits", + "streaming with ignoreChanges = true allows both delete and change commits", // ========== startingVersion option tests ========== "startingVersion", diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala index a8e46da414f..cf21375299f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala @@ -2019,6 +2019,57 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } + test("streaming with ignoreChanges = true allows both delete and change commits") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions( + inputDir.toString, Map(DeltaOptions.IGNORE_CHANGES_OPTION -> "true")) + + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3).map(Row(_))) + + // Delete all rows: delete-only commit, allowed by ignoreChanges + io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() + + // The delete commit is skipped (no AddFiles) + Seq(4, 5).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + q.processAllAvailable() + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3, 4, 5).map(Row(_))) + + // Overwrite produces both AddFile and RemoveFile actions (change commit). + // Unlike skipChangeCommits which skips the commit, ignoreChanges lets the + // AddFiles through (potentially duplicating data). + Seq(6, 7, 8).toDF("x") + .write + .mode("overwrite") + .format("delta") + .save(inputDir.toString) + + Seq(9, 10).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + + q.processAllAvailable() + + // The overwrite commit's AddFiles ARE emitted (unlike skipChangeCommits). The + // result includes the overwrite's new rows. + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).map(Row(_))) + } finally { + q.stop() + } + } + } + test("fail on data loss - starting from missing files") { withTempDirs { (srcData, targetData, chkLocation) => def addData(): Unit = { diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java index 96078c94714..c246aee7dec 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java @@ -347,14 +347,6 @@ public void testStreamingInvalidOptions(TableType tableType) throws Exception { assertInvalidStreamOption( tableName, r -> r.option("startingVersion", -1), "startingVersion"); assertInvalidStreamOption(tableName, r -> r.option("startingVersion", 999999), "999999"); - - if (tableType == TableType.MANAGED) { - assertInvalidStreamOption( - tableName, - r -> r.option("ignoreChanges", "true"), - "not supported", - "ignoreChanges"); - } }); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java index 56872ce63b2..a22b6a7c274 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java @@ -935,10 +935,13 @@ private boolean validateCommitAndDecideSkipping( } } - // TODO(#5319): Implement ignoreChanges & ignoreFileDeletion (deprecated) - // A check on the source table that disallows changes on the source data. - boolean shouldAllowChanges = skipChangeCommits; - // A check on the source table that disallows commits that only include deletes to the data. + // TODO(#5319): Implement ignoreFileDeletion (deprecated) + // "Changes" are commits that contain both AddFile and RemoveFile actions (e.g. UPDATE, + // MERGE that rewrites data files). Allowing changes means these commits won't cause failures. + boolean shouldAllowChanges = options.ignoreChanges() || skipChangeCommits; + // "Deletes" are commits that contain only RemoveFile actions without any AddFile actions + // (e.g. DELETE that purely removes data). This is a subset of changes, so allowing changes + // implies allowing deletes. boolean shouldAllowDeletes = shouldAllowChanges || options.ignoreDeletes(); boolean hasFileAdd = false; diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java index fe81918b6f0..4593cd441b5 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java @@ -72,22 +72,22 @@ public class SparkScan DeltaOptions.STARTING_TIMESTAMP_OPTION(), DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION(), DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION(), + DeltaOptions.IGNORE_CHANGES_OPTION(), DeltaOptions.IGNORE_DELETES_OPTION(), DeltaOptions.SKIP_CHANGE_COMMITS_OPTION(), DeltaOptions.EXCLUDE_REGEX_OPTION())); /** * Block list of DeltaOptions that are not supported for streaming in V2 connector. Only - * startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, ignoreDeletes, - * skipChangeCommits, and excludeRegex are supported. User-defined custom options (not in - * DeltaOptions) are allowed to pass through. + * startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, ignoreChanges, + * ignoreDeletes, skipChangeCommits, and excludeRegex are supported. User-defined custom options + * (not in DeltaOptions) are allowed to pass through. */ private static final Set UNSUPPORTED_STREAMING_OPTIONS = Collections.unmodifiableSet( new HashSet<>( Arrays.asList( DeltaOptions.IGNORE_FILE_DELETION_OPTION().toLowerCase(), - DeltaOptions.IGNORE_CHANGES_OPTION().toLowerCase(), DeltaOptions.FAIL_ON_DATA_LOSS_OPTION().toLowerCase(), DeltaOptions.CDC_READ_OPTION().toLowerCase(), DeltaOptions.CDC_READ_OPTION_LEGACY().toLowerCase(), diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java index 54188294515..e86291de0bc 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java @@ -972,32 +972,35 @@ private static Stream removeFileScenarios() { } // ================================================================================================ - // Tests for ignoreDeletes parity between DSv1 and DSv2 + // Shared helpers for ignoreDeletes / skipChangeCommits / ignoreChanges parity tests // ================================================================================================ /** - * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 produce the same file changes for - * delete-only commits (commits with only RemoveFile actions, no AddFile actions). The delete-only - * commit should be silently skipped (only sentinels emitted, no data files). + * Runs a parity test: verifies that DSv1 and DSv2 produce identical file changes for the given + * option and scenario. */ - @ParameterizedTest - @MethodSource("deleteOnlyScenarios") - public void testGetFileChanges_withIgnoreDeletes_deleteOnlyParity( + private void runFileChangeParityTest( ScenarioSetup scenarioSetup, boolean isInitialSnapshot, String testDescription, - @TempDir File tempDir) + File tempDir, + String optionName, + boolean usePartitionedTable) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = - "test_ignore_deletes_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); - createEmptyPartitionedTestTable(testTablePath, testTableName); + "test_" + optionName + "_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); + if (usePartitionedTable) { + createEmptyPartitionedTestTable(testTablePath, testTableName); + } else { + createEmptyTestTable(testTablePath, testTableName); + } scenarioSetup.setup(testTableName, tempDir); long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); - DeltaOptions options = createDeltaOptions("ignoreDeletes", "true"); + DeltaOptions options = createDeltaOptions(optionName, "true"); // DSv1 DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); @@ -1029,24 +1032,19 @@ public void testGetFileChanges_withIgnoreDeletes_deleteOnlyParity( } /** - * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 still throw on commits containing - * both adds and removes (e.g., UPDATE, MERGE), since ignoreDeletes only suppresses delete-only - * commits. + * Runs a throws-parity test: verifies that both DSv1 and DSv2 throw UnsupportedOperationException + * after the same number of next() calls. */ - @ParameterizedTest - @MethodSource("changeCommitScenarios") - public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( + private void runFileChangeThrowsParityTest( ScenarioSetup scenarioSetup, boolean isInitialSnapshot, String testDescription, - @TempDir File tempDir) + File tempDir, + String optionName) throws Exception { String testTablePath = tempDir.getAbsolutePath(); String testTableName = - "test_ignore_deletes_change_" - + Math.abs(testDescription.hashCode()) - + "_" - + System.nanoTime(); + "test_" + optionName + "_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); createEmptyTestTable(testTablePath, testTableName); scenarioSetup.setup(testTableName, tempDir); @@ -1054,7 +1052,7 @@ public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( long fromVersion = 0L; long fromIndex = DeltaSourceOffset.BASE_INDEX(); Option endOffset = Option.empty(); - DeltaOptions options = createDeltaOptions("ignoreDeletes", "true"); + DeltaOptions options = createDeltaOptions(optionName, "true"); // DSv1 DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); @@ -1081,7 +1079,7 @@ public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( } }, String.format( - "DSv1 should throw on change commit with ignoreDeletes for: %s", testDescription)); + "DSv1 should throw on change commit with %s for: %s", optionName, testDescription)); // DSv2 Configuration hadoopConf = new Configuration(); @@ -1106,7 +1104,7 @@ public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( } }, String.format( - "DSv2 should throw on change commit with ignoreDeletes for: %s", testDescription)); + "DSv2 should throw on change commit with %s for: %s", optionName, testDescription)); assertEquals( dsv1SuccessfulCalls.get(), @@ -1117,6 +1115,44 @@ public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( testDescription, dsv1SuccessfulCalls.get(), dsv2SuccessfulCalls.get())); } + // ================================================================================================ + // Tests for ignoreDeletes parity between DSv1 and DSv2 + // ================================================================================================ + + /** + * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 produce the same file changes for + * delete-only commits (commits with only RemoveFile actions, no AddFile actions). The delete-only + * commit should be silently skipped (only sentinels emitted, no data files). + */ + @ParameterizedTest + @MethodSource("deleteOnlyScenarios") + public void testGetFileChanges_withIgnoreDeletes_deleteOnlyParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "ignoreDeletes", true); + } + + /** + * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 still throw on commits containing + * both adds and removes (e.g., UPDATE, MERGE), since ignoreDeletes only suppresses delete-only + * commits. + */ + @ParameterizedTest + @MethodSource("changeCommitScenarios") + public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeThrowsParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "ignoreDeletes"); + } + // ================================================================================================ // Tests for skipChangeCommits parity between DSv1 and DSv2 // ================================================================================================ @@ -1134,44 +1170,8 @@ public void testGetFileChanges_withSkipChangeCommits_deleteOnlyParity( String testDescription, @TempDir File tempDir) throws Exception { - String testTablePath = tempDir.getAbsolutePath(); - String testTableName = - "test_skip_change_del_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); - createEmptyPartitionedTestTable(testTablePath, testTableName); - - scenarioSetup.setup(testTableName, tempDir); - - long fromVersion = 0L; - long fromIndex = DeltaSourceOffset.BASE_INDEX(); - DeltaOptions options = createDeltaOptions("skipChangeCommits", "true"); - - // DSv1 - DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); - DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); - Option endOffset = Option.empty(); - ClosableIterator deltaChanges = - deltaSource.getFileChanges( - fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); - List deltaFilesList = new ArrayList<>(); - while (deltaChanges.hasNext()) { - deltaFilesList.add(deltaChanges.next()); - } - deltaChanges.close(); - - // DSv2 - Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); - SparkMicroBatchStream stream = - createTestStreamWithDefaults(snapshotManager, hadoopConf, options); - try (CloseableIterator kernelChanges = - stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { - List kernelFilesList = new ArrayList<>(); - while (kernelChanges.hasNext()) { - kernelFilesList.add(kernelChanges.next()); - } - compareFileChanges(deltaFilesList, kernelFilesList); - } + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "skipChangeCommits", true); } /** @@ -1187,53 +1187,56 @@ public void testGetFileChanges_withSkipChangeCommits_changeCommitParity( String testDescription, @TempDir File tempDir) throws Exception { - String testTablePath = tempDir.getAbsolutePath(); - String testTableName = - "test_skip_change_chg_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); - createEmptyTestTable(testTablePath, testTableName); - - scenarioSetup.setup(testTableName, tempDir); + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "skipChangeCommits", false); + } - long fromVersion = 0L; - long fromIndex = DeltaSourceOffset.BASE_INDEX(); - DeltaOptions options = createDeltaOptions("skipChangeCommits", "true"); + // ================================================================================================ + // Tests for ignoreChanges parity between DSv1 and DSv2 + // ================================================================================================ - // DSv1 - DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); - DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); - Option endOffset = Option.empty(); - ClosableIterator deltaChanges = - deltaSource.getFileChanges( - fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); - List deltaFilesList = new ArrayList<>(); - while (deltaChanges.hasNext()) { - deltaFilesList.add(deltaChanges.next()); - } - deltaChanges.close(); + /** + * Verifies that with ignoreChanges=true, both DSv1 and DSv2 produce the same file changes for + * delete-only commits. Since ignoreChanges implies shouldAllowDeletes, these commits should be + * silently skipped (only sentinels emitted, no data files). + */ + @ParameterizedTest + @MethodSource("deleteOnlyScenarios") + public void testGetFileChanges_withIgnoreChanges_deleteOnlyParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "ignoreChanges", true); + } - // DSv2 - Configuration hadoopConf = new Configuration(); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(testTablePath, hadoopConf); - SparkMicroBatchStream stream = - createTestStreamWithDefaults(snapshotManager, hadoopConf, options); - try (CloseableIterator kernelChanges = - stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { - List kernelFilesList = new ArrayList<>(); - while (kernelChanges.hasNext()) { - kernelFilesList.add(kernelChanges.next()); - } - compareFileChanges(deltaFilesList, kernelFilesList); - } + /** + * Verifies that with ignoreChanges=true, both DSv1 and DSv2 produce the same file changes for + * change commits (commits containing both AddFile and RemoveFile actions). Unlike ignoreDeletes, + * ignoreChanges allows these commits through and emits their AddFiles. + */ + @ParameterizedTest + @MethodSource("changeCommitScenarios") + public void testGetFileChanges_withIgnoreChanges_changeCommitParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "ignoreChanges", false); } + // TODO(#5319): test the combinations of ignoreDeletes, skipChangeCommits, and ignoreChanges // ================================================================================================ - // Shared scenario providers for ignoreDeletes and skipChangeCommits tests + // Shared scenario providers for ignoreDeletes, skipChangeCommits, and ignoreChanges tests // ================================================================================================ /** * Provides delete-only scenarios: commits with only RemoveFile actions and no AddFile actions. - * Used by both ignoreDeletes and skipChangeCommits tests. + * Used by ignoreDeletes, skipChangeCommits, and ignoreChanges tests. * *

    Arguments: (ScenarioSetup, isInitialSnapshot, testDescription) */ @@ -1292,8 +1295,9 @@ private static Stream deleteOnlyScenarios() { /** * Provides change-commit scenarios: commits containing both AddFile and RemoveFile actions (e.g., - * UPDATE, MERGE). Used by both ignoreDeletes and skipChangeCommits tests — ignoreDeletes expects - * these to throw, while skipChangeCommits expects them to be silently skipped. + * UPDATE, MERGE). Used by ignoreDeletes, skipChangeCommits, and ignoreChanges tests — + * ignoreDeletes expects these to throw, skipChangeCommits expects them to be silently skipped, + * and ignoreChanges expects them to pass through with AddFiles emitted. * *

    Arguments: (ScenarioSetup, isInitialSnapshot, testDescription) */ diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java index f64265a21d9..2c452aa5461 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java @@ -588,7 +588,7 @@ public void testValidateStreamingOptions_UnsupportedOptions() { assertEquals( "The following streaming options are not supported: [readchangefeed]. " + "Supported options are: [startingVersion, startingTimestamp, maxFilesPerTrigger, " - + "maxBytesPerTrigger, ignoreDeletes, skipChangeCommits, excludeRegex].", + + "maxBytesPerTrigger, ignoreChanges, ignoreDeletes, skipChangeCommits, excludeRegex].", exception.getMessage()); } From 54ddfab89c2588949f8f90d7b0f0f60b1062283e Mon Sep 17 00:00:00 2001 From: ChengJi Date: Tue, 24 Mar 2026 08:57:16 -0700 Subject: [PATCH 59/80] [UniForm] Write atomic-supported property to Iceberg compact tables (#6335) ## Description Write atomic-supported property to Iceberg compact tables. This is part of project to support write to UC managed UniForm tables ## How was this patch tested? Add UTs ``` build/sbt -DsparkVersion=4.0 "iceberg/testOnly org.apache.spark.sql.delta.uniform.UniversalFormatSuite" ``` --- .../apache/spark/sql/delta/DeltaConfig.scala | 12 ++++ .../spark/sql/delta/IcebergCompat.scala | 67 ++++++++++++++----- .../sql/delta/UniversalFormatSuiteBase.scala | 41 +++++++++++- 3 files changed, 102 insertions(+), 18 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala index 5c9830dbf3c..2d20f4d1cd8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala @@ -850,6 +850,18 @@ trait DeltaConfigsBase extends DeltaLogging { validationFunction = _ => true, helpMessage = "needs to be a boolean." ) + /** + * Guard property automatically set when a new IcebergCompat table is created + * Atomic UniForm Iceberg conversion requires this property to be present + */ + val ICEBERG_ATOMIC_CONVERSION_SUPPORTED = buildConfig[Boolean]( + "universalFormat.iceberg.atomicConversion.supported", + "false", + _.toBoolean, + _ => true, + "needs to be a boolean.", + userConfigurable = true + ) val CAST_ICEBERG_TIME_TYPE = buildConfig[Boolean]( key = "castIcebergTimeType", diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala b/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala index f71222a68f1..f99d0c26c22 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala @@ -46,7 +46,7 @@ object IcebergCompatV1 extends IcebergCompatBase( icebergFormatVersion = 2, config = DeltaConfigs.ICEBERG_COMPAT_V1_ENABLED, tableFeature = IcebergCompatV1TableFeature, - requiredTableProperties = Seq(RequireColumnMapping), + relatedTableProperties = Seq(RequireColumnMapping, OptionalAtomicEligible), incompatibleTableFeatures = Set(DeletionVectorsTableFeature), checks = Seq( CheckOnlySingleVersionEnabled, @@ -63,7 +63,7 @@ object IcebergCompatV2 extends IcebergCompatBase( icebergFormatVersion = 2, config = DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED, tableFeature = IcebergCompatV2TableFeature, - requiredTableProperties = Seq(RequireColumnMapping), + relatedTableProperties = Seq(RequireColumnMapping, OptionalAtomicEligible), incompatibleTableFeatures = Set(DeletionVectorsTableFeature), checks = Seq( CheckOnlySingleVersionEnabled, @@ -95,7 +95,7 @@ case class IcebergCompatBase( icebergFormatVersion: Int, config: DeltaConfig[Option[Boolean]], tableFeature: TableFeature, - requiredTableProperties: Seq[RequiredDeltaTableProperty[_<:Any]], + relatedTableProperties: Seq[DeltaTableProperty], incompatibleTableFeatures: Set[TableFeature] = Set.empty, checks: Seq[IcebergCompatCheck]) extends DeltaLogging { def isEnabled(metadata: Metadata): Boolean = config.fromMetaData(metadata).getOrElse(false) @@ -156,8 +156,10 @@ case class IcebergCompatBase( } } - // Check we have all required delta table properties - requiredTableProperties.foreach { + // Go through all relatedProperties: + // 1. Check we have all required delta table properties + // 2. Enable related non-required delta properties if needed + relatedTableProperties.foreach { case RequiredDeltaTableProperty( deltaConfig, validator, autoSetValue, autoEnableOnExistingTable) => val newestValue = deltaConfig.fromMetaData(newestMetadata) @@ -179,6 +181,24 @@ case class IcebergCompatBase( deltaConfig.key, newestValue.toString, autoSetValue) } } + case OptionalDeltaTableProperty( + deltaConfig, + autoEnableOnExistingTables, + autoSetValue, + shouldExistingValuesBeingPreserved + ) => + val newestValueExplicitlySet = newestMetadata.configuration.contains(deltaConfig.key) + if (!newestValueExplicitlySet && + (autoEnableOnExistingTables || isCreatingOrReorgTable) + ) { + val oldValueExplicitlySet = prevMetadata.configuration.contains(deltaConfig.key) + val setValue = if (shouldExistingValuesBeingPreserved && oldValueExplicitlySet) { + prevMetadata.configuration(deltaConfig.key) + } else { + autoSetValue + } + tblPropertyUpdates += deltaConfig.key -> setValue + } } // Update Protocol and Metadata if necessary @@ -196,7 +216,7 @@ case class IcebergCompatBase( val newConfiguration = newestMetadata.configuration ++ tblPropertyUpdates.toMap var tmpNewMetadata = newestMetadata.copy(configuration = newConfiguration) - requiredTableProperties.foreach { tp => + relatedTableProperties.foreach { tp => tmpNewMetadata = tp.postProcess(prevMetadata, tmpNewMetadata, isCreatingOrReorgTable) } @@ -302,6 +322,16 @@ object IcebergCompat extends IcebergCompatVersionBase( ) with DeltaLogging +trait DeltaTableProperty { + /** + * A callback after all properties are added to the new metadata. + * @return Updated metadata. None if no change + */ + def postProcess( + prevMetadata: Metadata, + newMetadata: Metadata, + isCreatingNewTable: Boolean) : Metadata = newMetadata +} /** * Wrapper class for table property validation @@ -316,16 +346,14 @@ case class RequiredDeltaTableProperty[T]( deltaConfig: DeltaConfig[T], validator: T => Boolean, autoSetValue: String, - autoEnableOnExistingTable: Boolean = false) { - /** - * A callback after all required properties are added to the new metadata. - * @return Updated metadata. None if no change - */ - def postProcess( - prevMetadata: Metadata, - newMetadata: Metadata, - isCreatingNewTable: Boolean) : Metadata = newMetadata -} + autoEnableOnExistingTable: Boolean = false) extends DeltaTableProperty + +case class OptionalDeltaTableProperty[T]( + deltaConfig: DeltaConfig[T], + autoEnableOnExistingTables: Boolean, + autoSetValue: String, + shouldExistingValuesBeingPreserved: Boolean) extends DeltaTableProperty + class RequireColumnMapping(allowedModes: Seq[DeltaColumnMappingMode]) extends RequiredDeltaTableProperty( @@ -355,6 +383,13 @@ class RequireColumnMapping(allowedModes: Seq[DeltaColumnMappingMode]) object RequireColumnMapping extends RequireColumnMapping(Seq(NameMapping, IdMapping)) +object OptionalAtomicEligible extends OptionalDeltaTableProperty( + deltaConfig = DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED, + autoEnableOnExistingTables = false, + autoSetValue = "true", + shouldExistingValuesBeingPreserved = true +) + case class IcebergCompatContext( spark: SparkSession, diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala index c81dec7a2f8..625ad4a77a8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala @@ -445,12 +445,14 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal ) updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) - assert(updatedConfiguration.size == 5) + assert(updatedConfiguration.size == 6) assert(updatedConfiguration("dummykey") == "dummyvalue") assert(updatedConfiguration("delta.universalFormat.enabledFormats") == "iceberg") assert(updatedConfiguration("delta.columnMapping.mode") == "name") assert(updatedConfiguration(s"delta.enableIcebergCompatV$icv") == "true") assert(updatedConfiguration("delta.columnMapping.maxColumnId") == "1") + assert(updatedConfiguration( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key) == "true") configurationUnderTest = Map( s"delta.enableIcebergCompatV$icv" -> "true", @@ -459,7 +461,10 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal "delta.columnMapping.mode" -> "id" ) updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) - assert(updatedConfiguration.size == 4) + assert(updatedConfiguration.size == 6) + assert(updatedConfiguration("delta.columnMapping.maxColumnId") == "1") + assert(updatedConfiguration( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key) == "true") assert(updatedConfiguration("dummykey") == "dummyvalue") assert(updatedConfiguration("delta.columnMapping.mode") == "id") assert(updatedConfiguration("delta.universalFormat.enabledFormats") == "iceberg") @@ -468,6 +473,38 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal } } + test("enforceDependenciesInConfiguration preserves existing atomic guard property") { + withTempTableAndDir { case (id, _) => + executeSql(s"CREATE TABLE $id (id INT) USING DELTA") + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(id)) + + def getConfig: Map[String, String] = { + val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(id)) + UniversalFormat.enforceDependenciesInConfiguration( + spark, catalogTable, + Map(DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED.key -> "true"), + snapshot + ) + } + + // Case 1: snapshot has no guard -> auto-set to "true" + assert(getConfig.get( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key).contains("true")) + + // Case 2: snapshot has guard = false -> preserve false + executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + + s"('${DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key}' = 'false')") + assert(getConfig.get( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key).contains("false")) + + // Case 3: snapshot has guard = true -> preserve true + executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + + s"('${DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key}' = 'true')") + assert(getConfig.get( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key).contains("true")) + } + } + test("UniForm config validation") { Seq("ICEBERG", "iceberg,iceberg", "iceber", "paimon").foreach { invalidConf => withTempTableAndDir { case (id, loc) => From 3cbc5bf3f8a21baaca2beb42952806552787c90e Mon Sep 17 00:00:00 2001 From: Tathagata Das Date: Tue, 24 Mar 2026 15:44:26 -0400 Subject: [PATCH 60/80] Pinning github actions to shas instead of versions (#6372) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [x] Other (fill in here) ## Description As the title says ## How was this patch tested? Will be merged later ## Does this PR introduce _any_ user-facing changes? No --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/flink_test.yaml | 6 +++--- .github/workflows/iceberg_test.yaml | 6 +++--- .github/workflows/kernel_docs.yaml | 10 +++++----- .github/workflows/kernel_test.yaml | 10 +++++----- .github/workflows/kernel_unitycatalog_test.yaml | 4 ++-- .github/workflows/publish_docs.yaml | 16 ++++++++-------- .github/workflows/spark_examples_test.yaml | 10 +++++----- .github/workflows/spark_python_test.yaml | 10 +++++----- .github/workflows/spark_test.yaml | 12 ++++++------ .github/workflows/unidoc.yaml | 4 ++-- 11 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 207817a51bd..8bee17570e0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,21 +13,21 @@ jobs: name: "Build Test" runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Install Java 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 with: python-version: '3.9' - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt diff --git a/.github/workflows/flink_test.yaml b/.github/workflows/flink_test.yaml index 487afd8976e..71fd9df2111 100644 --- a/.github/workflows/flink_test.yaml +++ b/.github/workflows/flink_test.yaml @@ -27,16 +27,16 @@ jobs: echo "Runner OS: ${{ runner.os }}" echo "Runner arch: ${{ runner.arch }}" - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17. - name: install java - uses: actions/setup-java@v4 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 with: distribution: "zulu" java-version: "17" - name: Cache SBT and dependencies id: cache-sbt - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.sbt diff --git a/.github/workflows/iceberg_test.yaml b/.github/workflows/iceberg_test.yaml index 9750ca7d6dc..1a451ea18ac 100644 --- a/.github/workflows/iceberg_test.yaml +++ b/.github/workflows/iceberg_test.yaml @@ -19,14 +19,14 @@ jobs: env: SCALA_VERSION: ${{ matrix.scala }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt diff --git a/.github/workflows/kernel_docs.yaml b/.github/workflows/kernel_docs.yaml index 885c2c3047c..f82db0b46eb 100644 --- a/.github/workflows/kernel_docs.yaml +++ b/.github/workflows/kernel_docs.yaml @@ -26,9 +26,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "11" @@ -40,12 +40,12 @@ jobs: cp -r kernel/kernel-api/target/javaunidoc/. kernel/docs/snapshot/kernel-api/java/ cp -r kernel/kernel-defaults/target/javaunidoc/. kernel/docs/snapshot/kernel-defaults/java/ - name: Setup Pages - uses: actions/configure-pages@v3 + uses: actions/configure-pages@b8130d9ab958b325bbde9786d62f2c97a9885a0e # v3.0.7 - name: Upload artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@84bb4cd4b733d5c320c9c9cfbc354937524f4d64 # v1.0.10 with: # Upload kernel docs path: kernel/docs - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v2 + uses: actions/deploy-pages@de14547edc9944350dc0481aa5b7afb08e75f254 # v2.0.5 diff --git a/.github/workflows/kernel_test.yaml b/.github/workflows/kernel_test.yaml index e0e2f5bdba7..0e104311d70 100644 --- a/.github/workflows/kernel_test.yaml +++ b/.github/workflows/kernel_test.yaml @@ -44,16 +44,16 @@ jobs: echo "Runner OS: ${{ runner.os }}" echo "Runner arch: ${{ runner.arch }}" - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17. - name: install java - uses: actions/setup-java@v4 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 with: distribution: "zulu" java-version: "17" - name: Cache SBT and dependencies id: cache-sbt - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.sbt @@ -76,10 +76,10 @@ jobs: name: "DK: Integration" runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 # Run integration tests with JDK 11, as they have no Spark dependency - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "11" diff --git a/.github/workflows/kernel_unitycatalog_test.yaml b/.github/workflows/kernel_unitycatalog_test.yaml index f71722112d5..72830cfa3cb 100644 --- a/.github/workflows/kernel_unitycatalog_test.yaml +++ b/.github/workflows/kernel_unitycatalog_test.yaml @@ -25,9 +25,9 @@ jobs: env: SCALA_VERSION: 2.13.16 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" diff --git a/.github/workflows/publish_docs.yaml b/.github/workflows/publish_docs.yaml index 15b56750855..87694fe3ef7 100644 --- a/.github/workflows/publish_docs.yaml +++ b/.github/workflows/publish_docs.yaml @@ -29,18 +29,18 @@ jobs: out_dir: docs/apis/_site/api steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: repository: delta-io/delta ref: ${{ matrix.version.ref }} - - uses: actions/setup-java@v3 + - uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: ${{ matrix.version.java }} - name: Setup python environment - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3.3.0 with: activate-environment: delta_docs environment-file: docs/environment.yml @@ -71,7 +71,7 @@ jobs: done - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ github.run_id }}-apidocs-${{ matrix.version.name }} path: ${{ matrix.version.out_dir }} @@ -82,14 +82,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: docs/.nvmrc - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 name: Install pnpm with: package_json_file: docs/package.json @@ -99,7 +99,7 @@ jobs: working-directory: docs - name: Download API docs artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: ${{ github.run_id }}-apidocs-* path: docs/public/api diff --git a/.github/workflows/spark_examples_test.yaml b/.github/workflows/spark_examples_test.yaml index a7cf25c5011..3748f91c661 100644 --- a/.github/workflows/spark_examples_test.yaml +++ b/.github/workflows/spark_examples_test.yaml @@ -17,9 +17,9 @@ jobs: outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" @@ -44,7 +44,7 @@ jobs: env: SCALA_VERSION: ${{ matrix.scala }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Get Spark version details id: spark-details run: | @@ -59,12 +59,12 @@ jobs: echo "spark_full_version=$SPARK_FULL_VERSION" >> $GITHUB_OUTPUT echo "Using JVM $JVM_VERSION for Spark $SPARK_FULL_VERSION, package suffix: '$SPARK_PACKAGE_SUFFIX', support iceberg: '$SUPPORT_ICEBERG'" - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt diff --git a/.github/workflows/spark_python_test.yaml b/.github/workflows/spark_python_test.yaml index 7fd21e9e37a..b248ccefe70 100644 --- a/.github/workflows/spark_python_test.yaml +++ b/.github/workflows/spark_python_test.yaml @@ -17,9 +17,9 @@ jobs: outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" @@ -45,7 +45,7 @@ jobs: SCALA_VERSION: ${{ matrix.scala }} SPARK_VERSION: ${{ matrix.spark_version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Get Spark version details id: spark-details run: | @@ -56,12 +56,12 @@ jobs: echo "spark_full_version=$FULL_VERSION" >> $GITHUB_OUTPUT echo "Using JVM $JVM_VERSION for Spark ${{ matrix.spark_version }} (full: $FULL_VERSION)" - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt diff --git a/.github/workflows/spark_test.yaml b/.github/workflows/spark_test.yaml index 20bb9ad01a5..40dc0e292d4 100644 --- a/.github/workflows/spark_test.yaml +++ b/.github/workflows/spark_test.yaml @@ -17,9 +17,9 @@ jobs: outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" @@ -51,7 +51,7 @@ jobs: # Important: This must be the same as the length of shards in matrix NUM_SHARDS: 8 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Get Spark version details id: spark-details run: | @@ -60,12 +60,12 @@ jobs: echo "jvm_version=$JVM_VERSION" >> $GITHUB_OUTPUT echo "Using JVM version: $JVM_VERSION for Spark ${{ matrix.spark_version }}" - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt @@ -131,7 +131,7 @@ jobs: TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group spark --shard ${{ matrix.shard }} --spark-version ${{ matrix.spark_version }} - name: Upload test reports if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: test-reports-spark${{ matrix.spark_version }}-shard${{ matrix.shard }} path: "**/target/test-reports/*.xml" diff --git a/.github/workflows/unidoc.yaml b/.github/workflows/unidoc.yaml index 735bdf806d4..c9c5cc525db 100644 --- a/.github/workflows/unidoc.yaml +++ b/.github/workflows/unidoc.yaml @@ -10,10 +10,10 @@ scala: [2.13.16] steps: - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: generate unidoc run: build/sbt "++ ${{ matrix.scala }}" unidoc From cca7c70ad1e283a92e80de6b3ec88358f9bd1ffa Mon Sep 17 00:00:00 2001 From: giovanni-sorice <103190482+giovanni-sorice@users.noreply.github.com> Date: Tue, 24 Mar 2026 20:59:43 +0100 Subject: [PATCH 61/80] =?UTF-8?q?Allow=20writes=20to=20CDF=20tables=20for?= =?UTF-8?q?=20add-only,=20remove-only,=20and=20non-data-ch=E2=80=A6=20(#55?= =?UTF-8?q?67)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Coauthor One andreAmorimF Co-authored-by: Coauthor Two aryadneguardieiro Co-authored-by: Coauthor Three yurimantchev-sudo #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [x] Kernel - [ ] Other (fill in here) ## Description Allow writes to CDF tables for add-only, remove-only, and non-data-change transactions. More context: https://delta-users.slack.com/archives/C04TRPG3LHZ/p1763402127485709?thread_ts=1763400653.848629&cid=C04TRPG3LHZ ## How was this patch tested? ## Does this PR introduce _any_ user-facing changes? --------- Co-authored-by: André Fonseca <1077309+andreAmorimF@users.noreply.github.com> Co-authored-by: OussamaSaoudi <45303303+OussamaSaoudi@users.noreply.github.com> --- .../io/delta/kernel/internal/DeltaErrors.java | 11 ++ .../kernel/internal/TransactionImpl.java | 23 ++++ .../internal/tablefeatures/TableFeatures.java | 6 +- .../tablefeatures/TableFeaturesSuite.scala | 6 +- .../defaults/DeltaReplaceTableSuite.scala | 4 + .../defaults/DeltaTableWritesSuite.scala | 106 +++++++++++++++++- .../defaults/IcebergWriterCompatV1Suite.scala | 4 +- 7 files changed, 154 insertions(+), 6 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java index 5849ea8e1e0..ca3d5a5ec36 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java @@ -536,6 +536,17 @@ public static KernelException cannotModifyAppendOnlyTable(String tablePath) { tablePath, TableConfig.APPEND_ONLY_ENABLED.getKey())); } + public static KernelException cdfMixedAddRemoveNotSupported(String tablePath) { + return new KernelException( + String.format( + "Cannot add and remove data in the same transaction when Change Data Feed is enabled " + + "on table %s. This would require writing CDC files for DML operations, which is " + + "not yet supported by Delta Kernel. You can perform add-only operations (like " + + "INSERT or CREATE TABLE), remove-only operations (like DELETE), or mixed " + + "operations with dataChange=false (like OPTIMIZE).", + tablePath)); + } + public static KernelException rowTrackingMetadataMissingInFile(String entry, String filePath) { return new KernelException( String.format("Required metadata key %s is not present in scan file %s.", entry, filePath)); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java index f85f156bf9b..0851dc7d480 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java @@ -68,6 +68,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -542,6 +543,13 @@ private Tuple2> doCommit( } boolean isAppendOnlyTable = APPEND_ONLY_ENABLED.fromMetadata(metadata); + boolean isCdfEnabled = TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(metadata); + + // Track CDF validation state: whether we've seen adds and removes + // with enableChangeDataFeed=true + // This is wrapped in an AtomicBoolean to allow modification from within the lambda + final AtomicBoolean hasAddWithDataChange = new AtomicBoolean(false); + final AtomicBoolean hasRemoveWithDataChange = new AtomicBoolean(false); // Create a new CloseableIterator that will return the metadata actions followed by the // data actions. @@ -556,6 +564,21 @@ private Tuple2> doCommit( if (isAppendOnlyTable && removeFile.getDataChange()) { throw DeltaErrors.cannotModifyAppendOnlyTable(dataPath.toString()); } + if (isCdfEnabled && removeFile.getDataChange()) { + hasRemoveWithDataChange.set(true); + if (hasAddWithDataChange.get()) { + throw DeltaErrors.cdfMixedAddRemoveNotSupported(dataPath.toString()); + } + } + } + if (!action.isNullAt(ADD_FILE_ORDINAL)) { + AddFile addFile = new AddFile(action.getStruct(ADD_FILE_ORDINAL)); + if (isCdfEnabled && addFile.getDataChange()) { + hasAddWithDataChange.set(true); + if (hasRemoveWithDataChange.get()) { + throw DeltaErrors.cdfMixedAddRemoveNotSupported(dataPath.toString()); + } + } } return action; }); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java index 6df63150af6..28cd7008440 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java @@ -143,8 +143,10 @@ private static class ChangeDataFeedFeature extends TableFeature.LegacyWriterFeat @Override public boolean hasKernelWriteSupport(Metadata metadata) { - // writable if change data feed is disabled - return !TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(metadata); + // Kernel now supports writes to CDF-enabled tables with restrictions. + // Validation is performed at commit time to check for unsupported mixed operations + // (add+remove with dataChange=true) which would require CDC file generation. + return true; } @Override diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala index 850fc32be94..a9550a74563 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala @@ -456,7 +456,8 @@ class TableFeaturesSuite extends AnyFunSuite { new Protocol(1, 4), testMetadata(includeGeneratedColumn = true)) - checkWriteUnsupported( + // Supported if append-only or remove-only + checkWriteSupported( "validateKernelCanWriteToTable: protocol 4 with changeDataFeed", new Protocol(1, 4), testMetadata(tblProps = Map("delta.enableChangeDataFeed" -> "true"))) @@ -523,7 +524,8 @@ class TableFeaturesSuite extends AnyFunSuite { new Protocol(1, 7, Set().asJava, singleton("changeDataFeed")), testMetadata()) - checkWriteUnsupported( + // Supported if append-only or remove-only + checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with changeDataFeed, " + "metadata contains changeDataFeed", new Protocol(1, 7, Set().asJava, singleton("changeDataFeed")), diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala index 244e6041903..63f42d4f595 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala @@ -120,6 +120,7 @@ abstract class DeltaReplaceTableSuite extends DeltaReplaceTableSuiteBase { } } + /* TODO: re-enable when other table-features are unsupported test("Cannot enable a feature that Kernel does not support") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) @@ -132,7 +133,9 @@ abstract class DeltaReplaceTableSuite extends DeltaReplaceTableSuiteBase { }.getMessage.contains("Unsupported Delta writer feature")) } } + */ + /* TODO: re-enable when other table-features are unsupported test("Cannot replace a table with a protocol Kernel does not support") { withTempDirAndEngine { (tablePath, engine) => spark.sql( @@ -148,6 +151,7 @@ abstract class DeltaReplaceTableSuite extends DeltaReplaceTableSuiteBase { }.getMessage.contains("Unsupported Delta writer feature")) } } + */ test("Must provide a schema for replace table transaction") { withTempDirAndEngine { (tablePath, engine) => diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala index 29d1d47c26e..89475a850c5 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala @@ -17,7 +17,7 @@ package io.delta.kernel.defaults import java.io.File import java.nio.file.Files -import java.util.{Locale, Optional} +import java.util.{Locale, Optional, UUID} import scala.collection.immutable.Seq import scala.jdk.CollectionConverters._ @@ -37,10 +37,12 @@ import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.expressions.Literal._ import io.delta.kernel.internal.{ScanImpl, SnapshotImpl, TableConfig} import io.delta.kernel.internal.checkpoints.CheckpointerSuite.selectSingleElement +import io.delta.kernel.internal.data.GenericRow import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.internal.types.DataTypeJsonSerDe import io.delta.kernel.internal.util.{Clock, JsonUtils} import io.delta.kernel.internal.util.SchemaUtils.casePreservingPartitionColNames +import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.shaded.com.fasterxml.jackson.databind.node.ObjectNode import io.delta.kernel.transaction.DataLayoutSpec import io.delta.kernel.types._ @@ -1959,6 +1961,108 @@ abstract class AbstractDeltaTableWritesSuite extends AnyFunSuite with AbstractWr } } + /////////////////////////////////////////////////////////////////////////// + // Change Data Feed (CDF) tests + /////////////////////////////////////////////////////////////////////////// + + private def createAddFileRow( + path: String = s"part-${UUID.randomUUID()}.parquet", + dataChange: Boolean = true): Row = { + import io.delta.kernel.internal.actions.{AddFile, SingleAction} + import io.delta.kernel.internal.util.PartitionUtils + + val partitionValues = PartitionUtils.serializePartitionMap( + java.util.Collections.emptyMap[String, Literal]()) + + val addFileRow = AddFile.createAddFileRow( + testSchema, + path, + partitionValues, + 100L, // size + System.currentTimeMillis(), // modificationTime + dataChange, + Optional.empty(), // deletionVector + Optional.empty(), // tags + Optional.empty(), // baseRowId + Optional.empty(), // defaultRowCommitVersion + Optional.empty() // stats + ) + + SingleAction.createAddFileSingleAction(addFileRow) + } + + // Helper to create a mock RemoveFile action row + private def createRemoveFileRow( + path: String, + dataChange: Boolean = true): Row = { + import io.delta.kernel.internal.actions.{RemoveFile, SingleAction} + + val removeFileRow = new GenericRow( + RemoveFile.FULL_SCHEMA, + Map[Integer, Object]( + Integer.valueOf(RemoveFile.FULL_SCHEMA.indexOf("path")) -> path, + Integer.valueOf(RemoveFile.FULL_SCHEMA.indexOf("deletionTimestamp")) -> Long.box( + System.currentTimeMillis()), + Integer.valueOf(RemoveFile.FULL_SCHEMA.indexOf("dataChange")) -> Boolean.box(dataChange), + Integer.valueOf(RemoveFile.FULL_SCHEMA.indexOf("size")) -> Long.box(100L)).asJava) + + SingleAction.createRemoveFileSingleAction(removeFileRow) + } + + // Test cases: (description, actions, shouldSucceed) + val cdfTestCases: Seq[(String, Seq[Row], Boolean)] = Seq( + ("add with dataChange=true", Seq(createAddFileRow(dataChange = true)), true), + ("add with dataChange=false", Seq(createAddFileRow(dataChange = false)), true), + ( + "multiple adds with dataChange=true", + Seq(createAddFileRow(dataChange = true), createAddFileRow(dataChange = true)), + true), + ( + "multiple removes with dataChange=true", { + val path1 = "file1.parquet" + val path2 = "file2.parquet" + Seq( + createAddFileRow(path1, dataChange = false), + createAddFileRow(path2, dataChange = false), + createRemoveFileRow(path1, dataChange = true), + createRemoveFileRow(path2, dataChange = true)) + }, + true), + ( + "add dataChange=true, remove dataChange=false", { + val path = "file1.parquet" + Seq( + createAddFileRow(path, dataChange = true), + createRemoveFileRow(path, dataChange = false)) + }, + true), + ( + "add dataChange=true, remove dataChange=true", { + val path = "file1.parquet" + Seq(createAddFileRow(path, dataChange = true), createRemoveFileRow(path, dataChange = true)) + }, + false)) + + cdfTestCases.foreach { case (desc, actions, shouldSucceed) => + test(s"CDF-enabled table: $desc - ${if (shouldSucceed) "succeeds" else "fails"}") { + withTempDirAndEngine { (tablePath, engine) => + val tableProps = Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true") + val txn = getCreateTxn(engine, tablePath, testSchema, tableProperties = tableProps) + + val actionsIterable = inMemoryIterable(toCloseableIterator(actions.asJava.iterator())) + + if (shouldSucceed) { + val result = commitTransaction(txn, engine, actionsIterable) + assert(result.getVersion === 0) + } else { + intercept[KernelException] { + commitTransaction(txn, engine, actionsIterable) + } + } + } + } + } + def removeTimestampNtzTypeColumns(structType: StructType): StructType = { def process(dataType: DataType): Option[DataType] = dataType match { case a: ArrayType => diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala index ded5d42cc5a..a9e784ed477 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala @@ -470,7 +470,9 @@ trait IcebergWriterCompatV1SuiteBase testIncompatibleUnsupportedTableFeature( "changeDataFeed", - tablePropertiesToEnable = Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true")) + tablePropertiesToEnable = Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true"), + expectedErrorMessage = + "Table features [changeDataFeed] are incompatible with icebergWriterCompatV1") testIncompatibleUnsupportedTableFeature( "invariants", From 6c6f720480c75731edbb07fa4d873f23c041cdae Mon Sep 17 00:00:00 2001 From: seewishnew Date: Wed, 25 Mar 2026 17:17:58 -0700 Subject: [PATCH 62/80] [CI] Disable python venv centric github action workflows and remove pipenv in spark_test workflow (#6390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [x] Other (CI/GitHub Actions) ## Description This PR hardens the GitHub Actions CI pipelines against supply chain attacks targeting the Python/PyPI package ecosystem. It reduces the attack surface by eliminating unnecessary `pip install` invocations from workflows where they are not required for the tests being run. ### Changes **`spark_test.yaml` — Remove redundant Python environment setup** The `spark_test` workflow runs Scala/Java tests via `build/sbt sparkGroup/test`, but previously set up a full Python environment (pyenv, pipenv, and 20+ `pipenv run pip install` commands) that was not consumed by any test step. This PR removes the `Install Job dependencies` step entirely and switches the test invocation from `pipenv run python run-tests.py` to `python3 run-tests.py`, which requires only the system Python (stdlib-only — `run-tests.py` has no third-party imports). This change is consistent with #6384, which independently identified the same redundancy. **`iceberg_test.yaml`, `spark_python_test.yaml`, `publish_docs.yaml` — Disable workflows with heavy PyPI dependency chains** These three workflows install packages from PyPI at runtime via `pip`/`pipenv`/`conda` and cannot function without those installs. Rather than deleting them, they are renamed to `disabled_*.yaml` to preserve them for future re-enablement once the dependency installation strategy is hardened (e.g., via hash-pinned lockfiles or reproducible resolution with `UV_EXCLUDE_NEWER`). | Workflow | Reason for disabling | |---|---| | `disabled_spark_python_test.yaml` | Installs 20+ PyPI packages via pyenv/pipenv including pyspark, pandas, pyarrow, grpcio | | `disabled_iceberg_test.yaml` | Installs pyenv/pipenv and runs tests through `pipenv run python` | | `disabled_publish_docs.yaml` | Sets up a full conda environment (`docs/environment.yml`) with 50+ packages for Sphinx doc generation | **Workflows left unchanged (no PyPI exposure)** The following workflows use only system Python (stdlib-only scripts) or no Python at all, and are unaffected by this change: - `build.yaml` — stdlib Python only (`test_cross_spark_publish.py`), no pip - `kernel_test.yaml` — stdlib Python test runner (`run-tests.py`), no pip - `spark_examples_test.yaml` — stdlib Python for version info, tests are pure SBT - `flink_test.yaml` — pure SBT - `kernel_unitycatalog_test.yaml` — pure SBT - `kernel_docs.yaml` — pure SBT - `unidoc.yaml` — pure SBT ### Additional context All 52 GitHub Actions `uses:` references across all workflows are already pinned to full 40-character commit SHAs with version comments. An audit confirmed that 14 of 15 unique pinned SHAs are older than 30 days; the single exception (`pnpm/action-setup@v5.0.0`, 12 days old) is in the now-disabled `publish_docs.yaml`. ## How was this patch tested? 1. **CI run on fork**: The changes were pushed to a sandboxed public fork ([seewishnew/sandboxed-delta](https://github.com/seewishnew/sandboxed-delta)) and all enabled workflows executed on GitHub Actions. The `spark_test`, `build`, `spark_examples_test`, `flink_test`, `kernel_unitycatalog_test`, `kernel_docs`, and `unidoc` workflows ran without issues related to the CI changes. 2. **SHA pin audit**: All `uses:` references were verified to be pinned to full commit SHAs with correct version comments using a custom scanner. ## Does this PR introduce _any_ user-facing changes? No. This PR only modifies CI workflow configuration. No source code, build definitions, APIs, or runtime behavior are changed. Three CI jobs (`spark_python_test`, `iceberg_test`, `publish_docs`) are temporarily disabled and can be re-enabled once a hardened dependency installation strategy is in place. --- .github/CODEOWNERS | 7 +++ ...g_test.yaml => disabled_iceberg_test.yaml} | 16 +++---- ...h_docs.yaml => disabled_publish_docs.yaml} | 18 ++++--- ...t.yaml => disabled_spark_python_test.yaml} | 17 ++++--- .github/workflows/spark_test.yaml | 47 +------------------ 5 files changed, 31 insertions(+), 74 deletions(-) rename .github/workflows/{iceberg_test.yaml => disabled_iceberg_test.yaml} (83%) rename .github/workflows/{publish_docs.yaml => disabled_publish_docs.yaml} (88%) rename .github/workflows/{spark_python_test.yaml => disabled_spark_python_test.yaml} (92%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ac8a3f63300..7b7fac76c6c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,6 +8,13 @@ /project/ @tdas /version.sbt @tdas +# Python client, examples, and tooling +/python/ @tdas + +# Iceberg / UniForm integration and shaded Iceberg sources +/iceberg/ @tdas +/icebergShaded/ @tdas + # Spark V2 and Unified modules /spark/v2/ @tdas @huan233usc @TimothyW553 @raveeram-db @murali-db /spark-unified/ @tdas @huan233usc @TimothyW553 @raveeram-db @murali-db diff --git a/.github/workflows/iceberg_test.yaml b/.github/workflows/disabled_iceberg_test.yaml similarity index 83% rename from .github/workflows/iceberg_test.yaml rename to .github/workflows/disabled_iceberg_test.yaml index 1a451ea18ac..3f6b19fe81c 100644 --- a/.github/workflows/iceberg_test.yaml +++ b/.github/workflows/disabled_iceberg_test.yaml @@ -1,15 +1,13 @@ -name: "Delta Iceberg Latest" +name: "Delta Iceberg Latest [DISABLED]" +# SECURITY: All Python/PySpark workflows disabled due to active supply chain attack +# targeting OSS package ecosystems (PyPI). C2 domains: models.litellm.cloud, checkmarx.zone +# Date disabled: 2026-03-25 +# To re-enable: remove 'if: false' from all jobs and restore original triggers on: - push: - paths-ignore: - - '**.md' - - '**.txt' - pull_request: - paths-ignore: - - '**.md' - - '**.txt' + workflow_dispatch: # manual-only, auto triggers removed jobs: test: + if: false # SECURITY: disabled - supply chain attack mitigation name: "DIL: Scala ${{ matrix.scala }}" runs-on: ubuntu-24.04 strategy: diff --git a/.github/workflows/publish_docs.yaml b/.github/workflows/disabled_publish_docs.yaml similarity index 88% rename from .github/workflows/publish_docs.yaml rename to .github/workflows/disabled_publish_docs.yaml index 87694fe3ef7..b5c2d03e600 100644 --- a/.github/workflows/publish_docs.yaml +++ b/.github/workflows/disabled_publish_docs.yaml @@ -1,18 +1,15 @@ -name: Publish Docs +name: "Publish Docs [DISABLED]" +# SECURITY: All Python/PySpark workflows disabled due to active supply chain attack +# targeting OSS package ecosystems (PyPI). C2 domains: models.litellm.cloud, checkmarx.zone +# Date disabled: 2026-03-25 +# To re-enable: remove 'if: false' from all jobs and restore original triggers on: - push: - branches: - - master - paths: - - docs/** - release: - types: - - published - workflow_dispatch: + workflow_dispatch: # manual-only, auto triggers removed jobs: build_api_docs: + if: false # SECURITY: disabled - supply chain attack mitigation name: Build API docs (${{ matrix.version.name }}) runs-on: ubuntu-latest @@ -77,6 +74,7 @@ jobs: path: ${{ matrix.version.out_dir }} build_site: + if: false # SECURITY: disabled - supply chain attack mitigation name: Build site needs: build_api_docs runs-on: ubuntu-latest diff --git a/.github/workflows/spark_python_test.yaml b/.github/workflows/disabled_spark_python_test.yaml similarity index 92% rename from .github/workflows/spark_python_test.yaml rename to .github/workflows/disabled_spark_python_test.yaml index b248ccefe70..b57d068cac4 100644 --- a/.github/workflows/spark_python_test.yaml +++ b/.github/workflows/disabled_spark_python_test.yaml @@ -1,17 +1,15 @@ -name: "Delta Spark Python" +name: "Delta Spark Python [DISABLED]" +# SECURITY: All Python/PySpark workflows disabled due to active supply chain attack +# targeting OSS package ecosystems (PyPI). C2 domains: models.litellm.cloud, checkmarx.zone +# Date disabled: 2026-03-25 +# To re-enable: remove 'if: false' from all jobs and restore original triggers on: - push: - paths-ignore: - - '**.md' - - '**.txt' - pull_request: - paths-ignore: - - '**.md' - - '**.txt' + workflow_dispatch: # manual-only, auto triggers removed jobs: # Generate Spark versions matrix from CrossSparkVersions.scala # This workflow tests against released versions only (no snapshots) generate-matrix: + if: false # SECURITY: disabled - supply chain attack mitigation name: "Generate Released Spark Versions Matrix" runs-on: ubuntu-24.04 outputs: @@ -32,6 +30,7 @@ jobs: echo "Generated released Spark versions: $SPARK_VERSIONS" test: + if: false # SECURITY: disabled - supply chain attack mitigation name: "DSP (${{ matrix.spark_version }})" runs-on: ubuntu-24.04 needs: generate-matrix diff --git a/.github/workflows/spark_test.yaml b/.github/workflows/spark_test.yaml index 40dc0e292d4..8f846c6b930 100644 --- a/.github/workflows/spark_test.yaml +++ b/.github/workflows/spark_test.yaml @@ -76,59 +76,14 @@ jobs: # just use the cache. The cache is immutable so we need to use a new key when trying to # cache new stuff. key: delta-sbt-cache-spark${{ matrix.spark_version }}-scala${{ matrix.scala }} - - name: Install Job dependencies - run: | - sudo apt-get update - sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git - sudo apt install libedit-dev - curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz - mkdir -p ~/buf - tar -xvzf buf-Linux-x86_64.tar.gz -C ~/buf --strip-components 1 - rm buf-Linux-x86_64.tar.gz - sudo apt install python3-pip --fix-missing - sudo pip3 install pipenv==2024.4.1 - curl https://pyenv.run | bash - export PATH="~/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - pyenv install 3.9 - pyenv global system 3.9 - pipenv --python 3.9 install - # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version - # available. From version 24.1, `pip` doesn't allow installing python packages - # with version string containing `-`. In Delta-Spark case, the pypi package generated has - # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from - # the`version.sbt` file. - pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0 - pipenv run pip install flake8==3.9.0 - pipenv run pip install black==23.12.1 - pipenv run pip install importlib_metadata==3.10.0 - pipenv run pip install mypy==1.8.0 - pipenv run pip install mypy-protobuf==3.3.0 - pipenv run pip install cryptography==37.0.4 - pipenv run pip install twine==4.0.1 - pipenv run pip install wheel==0.33.4 - pipenv run pip install setuptools==41.1.0 - pipenv run pip install pydocstyle==3.0.0 - pipenv run pip install pandas==2.2.0 - pipenv run pip install pyarrow==11.0.0 - pipenv run pip install pypandoc==1.3.3 - pipenv run pip install numpy==1.22.4 - pipenv run pip install grpcio==1.67.0 - pipenv run pip install grpcio-status==1.67.0 - pipenv run pip install googleapis-common-protos==1.65.0 - pipenv run pip install protobuf==5.29.1 - pipenv run pip install googleapis-common-protos-stubs==2.2.0 - pipenv run pip install grpc-stubs==1.24.11 - name: Scala structured logging style check run: | if [ -f ./dev/spark_structured_logging_style.py ]; then python3 ./dev/spark_structured_logging_style.py fi - name: Run Scala/Java tests - # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_python_test.yaml run: | - TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group spark --shard ${{ matrix.shard }} --spark-version ${{ matrix.spark_version }} + TEST_PARALLELISM_COUNT=4 python3 run-tests.py --group spark --shard ${{ matrix.shard }} --spark-version ${{ matrix.spark_version }} - name: Upload test reports if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 From fc7d92cf740c96a955537a7b262ce578ce47c40e Mon Sep 17 00:00:00 2001 From: Milan Stefanovic <150366084+milastdbx@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:00:40 +0100 Subject: [PATCH 63/80] [KERNEL] Add GeoSpatial Table feature (#6235) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [X] Kernel - [ ] Other (fill in here) ## Description In this PR we are adding geospatial table feature as per spec: https://github.com/delta-io/delta/pull/4726 ## How was this patch tested? Tests ## Does this PR introduce _any_ user-facing changes? Adds new table feature --- .../internal/tablefeatures/TableFeatures.java | 26 +++++++- .../tablefeatures/TableFeaturesSuite.scala | 61 +++++++++++++++++-- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java index 28cd7008440..ff52106c0ec 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java @@ -205,6 +205,20 @@ public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata me } } + static final TableFeature GEOSPATIAL_RW_FEATURE = new GeoSpatialTableFeature(); + + private static class GeoSpatialTableFeature extends TableFeature.ReaderWriterFeature + implements FeatureAutoEnabledByMetadata { + GeoSpatialTableFeature() { + super("geospatial", /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); + } + + @Override + public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { + return hasGeospatial(metadata); + } + } + /* ---- Start: variantType ---- */ // Base class for variantType and variantType-preview features. Both features are same in terms // of behavior and given the feature is graduated, we will enable the `variantType` by default @@ -614,7 +628,8 @@ public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata me ICEBERG_WRITER_COMPAT_V1, ICEBERG_WRITER_COMPAT_V3, COLLATIONS_PREVIEW_W_FEATURE, - COLLATIONS_W_FEATURE)); + COLLATIONS_W_FEATURE, + GEOSPATIAL_RW_FEATURE)); public static final Map TABLE_FEATURE_MAP = Collections.unmodifiableMap( @@ -912,6 +927,15 @@ private static Set getDependencyFeatures(Set feature } } + public static boolean hasGeospatial(Metadata metadata) { + return new SchemaIterable(metadata.getSchema()) + .stream() + .anyMatch( + element -> + element.getField().getDataType() instanceof GeometryType + || element.getField().getDataType() instanceof GeographyType); + } + /** * Check if the table schema has a column of type. Caution: works only for the primitive types. */ diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala index a9550a74563..1360d55472f 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala @@ -51,7 +51,8 @@ class TableFeaturesSuite extends AnyFunSuite { "variantType", "variantType-preview", "variantShredding", - "variantShredding-preview") + "variantShredding-preview", + "geospatial") val writerOnlyFeatures = Seq( "allowColumnDefaults", @@ -195,7 +196,10 @@ class TableFeaturesSuite extends AnyFunSuite { ( "icebergWriterCompatV3", testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "false")), - false)).foreach({ case (feature, metadata, expected) => + false), + ("geospatial", testMetadata(includeGeometryTypeCol = true), true), + ("geospatial", testMetadata(includeGeographyTypeCol = true), true), + ("geospatial", testMetadata(), false)).foreach({ case (feature, metadata, expected) => test(s"metadataRequiresFeatureToBeEnabled - $feature - $metadata") { val tableFeature = TableFeatures.getTableFeature(feature) assert(tableFeature.isInstanceOf[FeatureAutoEnabledByMetadata]) @@ -259,7 +263,8 @@ class TableFeaturesSuite extends AnyFunSuite { "typeWidening-preview", "deletionVectors", "timestampNtz", - "vacuumProtocolCheck") + "vacuumProtocolCheck", + "geospatial") assert(results.map(_.featureName()).toSet == expected.toSet) } @@ -303,7 +308,8 @@ class TableFeaturesSuite extends AnyFunSuite { "variantShredding-preview", "materializePartitionColumns", "collations", - "collations-preview") + "collations-preview", + "geospatial") assert(results.map(_.featureName()).toSet == expected.toSet) } @@ -355,7 +361,8 @@ class TableFeaturesSuite extends AnyFunSuite { "v2Checkpoint", "vacuumProtocolCheck", "allowColumnDefaults", - "columnMapping").foreach { feature => + "columnMapping", + "geospatial").foreach { feature => test(s"validateKernelCanReadTheTable: protocol 3 with $feature") { val protocol = new Protocol(3, 1, singleton(feature), Set().asJava) validateKernelCanReadTheTable(protocol, "/test/table") @@ -671,6 +678,18 @@ class TableFeaturesSuite extends AnyFunSuite { new Protocol(3, 7, emptySet(), singleton("materializePartitionColumns")), testMetadata()) + checkWriteSupported( + "validateKernelCanWriteToTable: protocol 7 with geospatial, " + + "metadata doesn't contain geospatial", + new Protocol(3, 7, singleton("geospatial"), singleton("geospatial")), + testMetadata()) + + checkWriteSupported( + "validateKernelCanWriteToTable: protocol 7 with geospatial, " + + "metadata contains geospatial", + new Protocol(3, 7, singleton("geospatial"), singleton("geospatial")), + testMetadata(includeGeometryTypeCol = true)) + checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with multiple features supported", new Protocol( @@ -1064,7 +1083,25 @@ class TableFeaturesSuite extends AnyFunSuite { "deletionVectors", "variantShredding", "variantType")), - set("variantType", "variantShredding"))).foreach { + set("variantType", "variantShredding")), + ( + testMetadata(includeGeometryTypeCol = true), + new Protocol(1, 1), + new Protocol( + 3, + 7, + set("geospatial"), + set("geospatial")), + set("geospatial")), + ( + testMetadata(includeGeographyTypeCol = true), + new Protocol(1, 2), + new Protocol( + 3, + 7, + set("geospatial"), + set("geospatial", "appendOnly", "invariants")), + set("geospatial"))).foreach { case (newMetadata, currentProtocol, expectedProtocol, expectedNewFeatures) => test(s"autoUpgradeProtocolBasedOnMetadata:" + s"$currentProtocol -> $expectedProtocol, $expectedNewFeatures") { @@ -1266,6 +1303,8 @@ class TableFeaturesSuite extends AnyFunSuite { includeVariantTypeCol: Boolean = false, includeGeneratedColumn: Boolean = false, includeIdentityColumn: Boolean = false, + includeGeometryTypeCol: Boolean = false, + includeGeographyTypeCol: Boolean = false, includeCollatedStringTypeCol: Boolean = false, tblProps: Map[String, String] = Map.empty): Metadata = { val testSchema = createTestSchema( @@ -1274,6 +1313,8 @@ class TableFeaturesSuite extends AnyFunSuite { includeVariantTypeCol, includeGeneratedColumn, includeIdentityColumn, + includeGeometryTypeCol, + includeGeographyTypeCol, includeCollatedStringTypeCol) new Metadata( "id", @@ -1303,6 +1344,8 @@ class TableFeaturesSuite extends AnyFunSuite { includeVariantTypeCol: Boolean = false, includeGeneratedColumn: Boolean = false, includeIdentityColumn: Boolean = false, + includeGeometryTypeCol: Boolean = false, + includeGeographyTypeCol: Boolean = false, includeCollatedStringTypeCol: Boolean = false): StructType = { var structType = new StructType() .add("c1", IntegerType.INTEGER) @@ -1339,6 +1382,12 @@ class TableFeaturesSuite extends AnyFunSuite { .putBoolean("delta.identity.allowExplicitInsert", true) .build()) } + if (includeGeometryTypeCol) { + structType = structType.add("c8", GeometryType.ofDefault()) + } + if (includeGeographyTypeCol) { + structType = structType.add("c9", GeographyType.ofDefault()) + } if (includeCollatedStringTypeCol) { structType = structType.add("c8", new StringType("ICU.UNICODE.75")) } From 5b6af653b025ba497055a6c0ce79bc985da954bc Mon Sep 17 00:00:00 2001 From: Zihao Xu Date: Mon, 30 Mar 2026 01:29:07 -0500 Subject: [PATCH 64/80] Fix race condition in commitFilesIterator (#6353) ## Description `commitFilesIterator` discovers commit files in two phases: Phase 1 lists backfilled files from the filesystem, Phase 2 queries the coordinator for unbackfilled commits. If all unbackfilled commits get concurrently backfilled between the two phases (e.g., a concurrent writer triggers `backfillToVersion` which copies staged files to `_delta_log/` and removes them from the coordinator via `registerBackfill`), Phase 2 finds no `UnbackfilledDeltaFile` entries and the existing gap check has no anchor (`headOption` returns `None`), returning `Iterator.empty`. Versions between `maxVersionSeen + 1` and `endSnapshot.version` are silently dropped. This race manifests when a streaming reader and a concurrent writer run on different processes (different DeltaLog instances). The reader's Phase 1 lists the filesystem and finds nothing new. The writer commits and triggers backfill. The reader's Phase 2 does a fresh `deltaLog.update()`, which classifies the now-backfilled files as `BackfilledDeltaFile` and gets empty `getCommits()`. The gap check fails and data is lost. Related issue: #6339 ## How was this patch tested? Added `DeltaSourceWithCoordinatedCommitsBatch2Suite` and `DeltaSourceWithCoordinatedCommitsBatch3Suite` to exercise batch-size-2 and batch-size-3 backfill boundaries. ## Does this PR introduce _any_ user-facing changes? No. This is a bug fix for an internal race condition in commit file discovery. --- .../CoordinatedCommitsUtils.scala | 49 ++++++++++++++++--- .../sql/delta/sources/DeltaSQLConf.scala | 9 ++++ 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala index 5e51c131104..a6c2a26d7e7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.delta.actions.{Metadata, Protocol, TableFeatureProto import org.apache.spark.sql.delta.commands.CloneTableCommand import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.sql.delta.util.FileNames.{BackfilledDeltaFile, CompactedDeltaFile, DeltaFile, UnbackfilledDeltaFile} import io.delta.storage.LogStore @@ -565,14 +566,48 @@ object CoordinatedCommitsUtils extends DeltaLogging { case UnbackfilledDeltaFile(fileStatus, version, _) if version > maxVersionSeen => (fileStatus, version) } - // Check for a gap between listing and commit files in the logsegment - val gapListing = unbackfilledDeltas.headOption match { - case Some((_, version)) if maxVersionSeen + 1 < version => - listDeltas(maxVersionSeen + 1, Some(version)) - // no gap before - case _ => Iterator.empty + val backfillGapFixEnabled = SparkSession.active.sessionState.conf.getConf( + DeltaSQLConf.COMMIT_FILES_ITERATOR_BACKFILL_GAP_FIX_ENABLED) + if (backfillGapFixEnabled) { + // This fixes two bugs in the gap listing between Phase 1 (filesystem) and + // Phase 2 (coordinator/snapshot): + // + // Bug 1 - data loss: between Phase 1 and Phase 2, + // a concurrent writer backfills ALL remaining commits. unbackfilledDeltas is empty, + // headOption returns None, and the old code falls through to Iterator.empty, + // silently dropping versions [maxVersionSeen+1, endSnapshot.version]. + // Fix: fall back to filesystem listing up to endSnapshot.version. + // + // Bug 2 - duplicate entries (Some case): listDeltas uses an inclusive upper bound + // (takeWhile { version <= endVersion }). If the first unbackfilled version is also + // backfilled on filesystem (either it happened after we contacted UC, or UC did not + // know), the gap listing includes it, producing a duplicate with unbackfilledDeltas. + // Example: prev snapshot v100, latest v110. update() returns v105-v110 as + // unbackfilled, but v105 is backfilled on filesystem. Old code calls + // listDeltas(101, Some(105)), which includes 105.json. gapListing=[v101..v105], + // unbackfilledDeltas=[v105..v110]. v105 appears twice. + // Fix: use version - 1 as exclusive upper bound. listDeltas(101, Some(104)) stops + // before v105, eliminating the overlap. + val highestGapVersion = unbackfilledDeltas.headOption match { + case Some((_, version)) => version - 1 + case None => endSnapshot.version + } + val gapListing = if (maxVersionSeen < highestGapVersion) { + listDeltas(maxVersionSeen + 1, Some(highestGapVersion)) + } else { + Iterator.empty + } + gapListing ++ unbackfilledDeltas + } else { + // Check for a gap between listing and commit files in the logsegment + val gapListing = unbackfilledDeltas.headOption match { + case Some((_, version)) if maxVersionSeen + 1 < version => + listDeltas(maxVersionSeen + 1, Some(version)) + // no gap before + case _ => Iterator.empty + } + gapListing ++ unbackfilledDeltas } - gapListing ++ unbackfilledDeltas } // We want to avoid invoking `tailFromSnapshot()` as it internally calls deltaLog.update() diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index c8ff9a9e123..f86266fb1da 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -909,6 +909,15 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .intConf .createWithDefault(5) + val COMMIT_FILES_ITERATOR_BACKFILL_GAP_FIX_ENABLED = + buildConf("coordinatedCommits.commitFilesIterator.backfillGapFix.enabled") + .internal() + .doc("When enabled, commitFilesIterator falls back to filesystem listing when all " + + "unbackfilled commits are concurrently backfilled between Phase 1 (filesystem listing) " + + "and Phase 2 (coordinator query), preventing silent data loss.") + .booleanConf + .createWithDefault(true) + ////////////////////////////////////////////// // DynamoDB Commit Coordinator-specific configs end ///////////////////////////////////////////// From 0cf0dd201f4b71c5d4712ef57b855e3ba958beed Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Mon, 30 Mar 2026 21:42:19 +0530 Subject: [PATCH 65/80] [ServerSidePlanning] Add OAuth support (#6360) ## Summary Replace the static bearer token in `IcebergRESTCatalogPlanningClient` with a per-request `tokenSupplier` backed by UC's `TokenProvider`, enabling OAuth client credentials flow. ## Auth config **Static token (existing, unchanged):** ``` spark.sql.catalog..token = "dapi..." ``` **OAuth client credentials (new):** ``` spark.sql.catalog..auth.type = oauth spark.sql.catalog..auth.oauth.uri = https:///token spark.sql.catalog..auth.oauth.clientId = spark.sql.catalog..auth.oauth.clientSecret = ``` Follows UC's `AuthConfigUtils` pattern. Error if both `.token` and `.auth.token` are set. ## Key changes - `ServerSidePlanningMetadata`: `authToken` -> `tokenSupplier` (per-request function) - `UnityCatalogMetadata`: Extract `auth.*` configs, create `TokenProvider`. Legacy `.token` fallback with conflict check. - `IcebergRESTCatalogPlanningClient`: `token: String` -> `tokenSupplier: () => String`. `HttpRequestInterceptor` for per-request Bearer token injection. ## Test plan - [ ] `extractAuthConfig` tests: OAuth, static, legacy, conflict error, legacy wins, empty - [ ] Existing SSP tests updated for new constructor signatures - [ ] CI on delta-io/delta This pull request was AI-assisted by Isaac. --- .../IcebergRESTCatalogPlanningClient.scala | 50 ++++++----- ...bergRESTCatalogPlanningClientFactory.scala | 6 +- ...cebergRESTCatalogPlanningClientSuite.scala | 88 +++++++++++++++---- .../ServerSidePlanningCredentialsSuite.scala | 4 +- .../ServerSidePlanningMetadata.scala | 8 +- .../UnityCatalogMetadata.scala | 63 ++++++++++--- .../ServerSidePlannedTableSuite.scala | 74 +++++++++++++++- 7 files changed, 239 insertions(+), 54 deletions(-) diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala index ff782689d2b..3df29756b19 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala @@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.http.client.methods.{HttpGet, HttpPost} import org.apache.http.entity.{ContentType, StringEntity} import org.apache.http.util.EntityUtils -import org.apache.http.{HttpHeaders, HttpResponse, HttpStatus} +import org.apache.http.{HttpHeaders, HttpRequest, HttpRequestInterceptor, HttpResponse, HttpStatus} import org.apache.http.client.ServiceUnavailableRetryStrategy import org.apache.http.impl.client.{DefaultHttpRequestRetryHandler, HttpClientBuilder} import org.apache.http.protocol.HttpContext @@ -66,12 +66,14 @@ private case class CatalogConfigResponse( * @param baseUriRaw Base URI of the Iceberg REST catalog up to /v1, e.g., * "http:///iceberg/v1". Trailing slashes are handled automatically. * @param catalogName Name of the catalog for config endpoint query parameter. - * @param token Authentication token for the catalog server. + * @param tokenSupplier Supplier of auth tokens, called per-request to support OAuth. + * Returns empty string if no auth is needed. */ class IcebergRESTCatalogPlanningClient( baseUriRaw: String, catalogName: String, - token: String) extends ServerSidePlanningClient with Logging { + tokenSupplier: () => String +) extends ServerSidePlanningClient with Logging { // Normalize baseUri to handle trailing slashes private val baseUri = baseUriRaw.stripSuffix("/") @@ -213,19 +215,13 @@ class IcebergRESTCatalogPlanningClient( } } + // Default headers without auth -- auth is injected per-request via HttpRequestInterceptor private val httpHeaders = { - val baseHeaders = Map( + Map( HttpHeaders.ACCEPT -> ContentType.APPLICATION_JSON.getMimeType, HttpHeaders.CONTENT_TYPE -> ContentType.APPLICATION_JSON.getMimeType, HttpHeaders.USER_AGENT -> buildUserAgent() - ) - // Add Bearer token authentication if token is provided - val headersWithAuth = if (token.nonEmpty) { - baseHeaders + (HttpHeaders.AUTHORIZATION -> s"Bearer $token") - } else { - baseHeaders - } - headersWithAuth.map { case (k, v) => new BasicHeader(k, v) }.toSeq.asJava + ).map { case (k, v) => new BasicHeader(k, v) }.toSeq.asJava } /** @@ -318,14 +314,28 @@ class IcebergRESTCatalogPlanningClient( // Maximum number of retries for transient HTTP failures (IOException, 5xx server errors) private val HTTP_MAX_RETRIES = 3 - private lazy val httpClient = HttpClientBuilder.create() - .setDefaultHeaders(httpHeaders) - .setConnectionTimeToLive(30, java.util.concurrent.TimeUnit.SECONDS) - // requestSentRetryEnabled=true: safe to retry already-sent requests because - // planScan is a read-only operation (idempotent POST to /plan endpoint) - .setRetryHandler(new DefaultHttpRequestRetryHandler(HTTP_MAX_RETRIES, true)) - .setServiceUnavailableRetryStrategy(new ServerErrorRetryStrategy(HTTP_MAX_RETRIES)) - .build() + private lazy val httpClient = { + val builder = HttpClientBuilder.create() + .setDefaultHeaders(httpHeaders) + .setConnectionTimeToLive(30, java.util.concurrent.TimeUnit.SECONDS) + // requestSentRetryEnabled=true: safe to retry already-sent requests because + // planScan is a read-only operation (idempotent POST to /plan endpoint) + .setRetryHandler(new DefaultHttpRequestRetryHandler(HTTP_MAX_RETRIES, true)) + .setServiceUnavailableRetryStrategy(new ServerErrorRetryStrategy(HTTP_MAX_RETRIES)) + + // Per-request interceptor: calls tokenSupplier() to get the current token. + // The token provider implementation handles caching as needed. + builder.addInterceptorFirst(new HttpRequestInterceptor { + override def process(request: HttpRequest, context: HttpContext): Unit = { + val token = tokenSupplier() + if (token != null && token.nonEmpty) { + request.setHeader(HttpHeaders.AUTHORIZATION, s"Bearer $token") + } + } + }) + + builder.build() + } override def canConvertFilters(filters: Array[Filter]): Boolean = { // Check if all filters can be converted to Iceberg expressions diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala index aaf0eb35b26..f072b04a625 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala @@ -28,9 +28,11 @@ class IcebergRESTCatalogPlanningClientFactory extends ServerSidePlanningClientFa metadata: ServerSidePlanningMetadata): ServerSidePlanningClient = { val baseUri = metadata.planningEndpointUri - val token = metadata.authToken.getOrElse("") val catalogName = metadata.catalogName + val supplier: () => String = + metadata.tokenSupplier.getOrElse(() => "") - new IcebergRESTCatalogPlanningClient(baseUri, catalogName, token) + new IcebergRESTCatalogPlanningClient( + baseUri, catalogName, supplier) } } diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala index b38019500e9..af96b017ebb 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala @@ -78,7 +78,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe // Tests that the REST /plan endpoint returns 0 files for an empty table. test("basic plan table scan via IcebergRESTCatalogPlanningClient") { withTempTable("testTable") { table => - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { val scanPlan = client.planScan(defaultNamespace.toString, "testTable") assert(scanPlan != null, "Scan plan should not be null") @@ -106,7 +106,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe .map(row => (new Path(row.getString(0)).getName, row.getLong(1))) .toMap - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { val scanPlan = client.planScan(defaultNamespace.toString, "tableWithData") assert(scanPlan != null, "Scan plan should not be null") @@ -145,7 +145,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("testTable") { table => // Client expects baseUri to include the /v1 path (per Iceberg REST spec) - val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", () => "") try { // Make a call that will trigger the lazy initialization of icebergRestCatalogUriRoot // which internally calls fetchCatalogPrefix() @@ -173,7 +173,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("testTable") { table => // Client expects baseUri to include the /v1 path (per Iceberg REST spec) - val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", () => "") try { // Make a call that will trigger the lazy initialization val scanPlan = client.planScan(defaultNamespace.toString, "testTable") @@ -200,7 +200,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("filterTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.filterTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { val testCases = Seq( (EqualTo("longCol", 2L), "EqualTo numeric (long)"), @@ -310,7 +310,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe Set("`address.city`")) ) - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { testCases.foreach { testCase => // Clear previous captured projection @@ -343,7 +343,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe val tableName = s"rest_catalog.${defaultNamespace}.limitTest" populateTestData(tableName) - val client = new IcebergRESTCatalogPlanningClient(serverUri, null, "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, null, () => "") try { // Test different limit values val testCases = Seq( @@ -378,7 +378,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe val tableName = s"rest_catalog.${defaultNamespace}.filterProjectionLimitTest" populateTestData(tableName) - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { // Note: Filter types are already tested in "filter sent to IRC server" test. // Here we verify filter, projection, AND limit are sent together correctly. @@ -456,7 +456,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("caseSensitiveTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.caseSensitiveTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, null, "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, null, () => "") try { server.clearCaptured() @@ -481,7 +481,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("residualTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.residualTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { // Verify that a trivial (alwaysTrue) residual is accepted server.setTestResidual(Expressions.alwaysTrue()) @@ -527,7 +527,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("retryTest503") { table => populateTestData(s"rest_catalog.${defaultNamespace}.retryTest503") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { server.clearCaptured() // Configure server to fail the first plan request with 503 @@ -552,7 +552,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe // No populateTestData needed: failure injection intercepts at the servlet level before // table data is accessed, so we only need the table to exist for a valid URI. withTempTable("retryTestExhausted") { table => - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { server.clearCaptured() // Configure server to fail more requests than the client will retry (max 3 retries = 4 @@ -579,7 +579,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe // No populateTestData needed: failure injection intercepts at the servlet level before // table data is accessed, so we only need the table to exist for a valid URI. withTempTable("retryTest404") { table => - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { server.clearCaptured() // Configure server to fail all plan requests with 404 @@ -609,7 +609,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe // warning, and return None — causing icebergRestCatalogUriRoot to fall back to baseUri. // The subsequent planScan HTTP POST will also fail (same unreachable host). val unreachableUri = "http://localhost:1" - val client = new IcebergRESTCatalogPlanningClient(unreachableUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(unreachableUri, "test_catalog", () => "") try { val ex = intercept[Exception] { client.planScan("test_db", "test_table") @@ -625,7 +625,8 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe } test("User-Agent header format") { - val client = new IcebergRESTCatalogPlanningClient("http://localhost:8080", "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient( + "http://localhost:8080", "test_catalog", () => "") try { val userAgent = client.getUserAgent() @@ -664,4 +665,61 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe } } + test("e2e: OAuth token supplier with mock token server") { + // Start a mock OAuth token endpoint using JDK HttpServer. + // OAuthTokenProvider sends: POST with Basic auth + grant_type=client_credentials + val tokenRequestCount = new java.util.concurrent.atomic.AtomicInteger(0) + val mockToken = "mock-oauth-access-token" + val oauthServer = com.sun.net.httpserver.HttpServer.create( + new java.net.InetSocketAddress("localhost", 0), 0) + oauthServer.createContext("/token", (exchange: com.sun.net.httpserver.HttpExchange) => { + try { + tokenRequestCount.incrementAndGet() + val responseJson = + s"""{"access_token":"$mockToken","token_type":"Bearer","expires_in":3600}""" + val bytes = responseJson.getBytes(java.nio.charset.StandardCharsets.UTF_8) + exchange.getResponseHeaders.add("Content-Type", "application/json") + exchange.sendResponseHeaders(200, bytes.length) + exchange.getResponseBody.write(bytes) + } finally { + exchange.close() + } + }) + oauthServer.start() + + try { + val oauthUri = s"http://localhost:${oauthServer.getAddress.getPort}/token" + val authConfig = Map( + "type" -> "oauth", + "oauth.uri" -> oauthUri, + "oauth.clientId" -> "test-client-id", + "oauth.clientSecret" -> "test-client-secret") + val tokenProvider = io.unitycatalog.client.auth.TokenProvider.create( + authConfig.asJava.asInstanceOf[java.util.Map[String, String]]) + val supplier: () => String = () => tokenProvider.accessToken() + + withTempTable("oauthE2eTest") { table => + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", supplier) + try { + // First planScan triggers token fetch from mock server + val scanPlan = client.planScan(defaultNamespace.toString, "oauthE2eTest") + assert(scanPlan != null) + + // OAuthTokenProvider should have fetched exactly 1 token (cached for reuse) + assert(tokenRequestCount.get() == 1, + s"Expected 1 token request, got ${tokenRequestCount.get()}") + + // Second planScan reuses cached token (not expired) + client.planScan(defaultNamespace.toString, "oauthE2eTest") + assert(tokenRequestCount.get() == 1, + s"Token should be cached. Expected 1 request, got ${tokenRequestCount.get()}") + } finally { + client.close() + } + } + } finally { + oauthServer.stop(0) + } + } + } diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala index a8808aa06ac..e02cd9b4a9a 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala @@ -70,7 +70,7 @@ class ServerSidePlanningCredentialsSuite extends QueryTest with SharedSparkSessi withTempTable("credentialsTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.credentialsTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { // Covers the successful credential extraction and Hadoop configuration injection cases. val testCases: Seq[CredentialTestCase] = Seq( @@ -147,7 +147,7 @@ class ServerSidePlanningCredentialsSuite extends QueryTest with SharedSparkSessi withTempTable("incompleteCredsTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.incompleteCredsTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { // Test cases for incomplete credentials that should throw errors val errorTestCases = Seq( diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala index a1b8e4c889d..ab7905b4577 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala @@ -32,9 +32,11 @@ private[serverSidePlanning] trait ServerSidePlanningMetadata { def planningEndpointUri: String /** - * Authentication token for the planning endpoint. + * Supplier of authentication tokens for the planning endpoint. + * Called per-request to get the current token (supports OAuth). + * Returns None if no authentication is configured. */ - def authToken: Option[String] + def tokenSupplier: Option[() => String] /** * Catalog name for configuration lookups. @@ -56,7 +58,7 @@ private[serverSidePlanning] case class DefaultMetadata( catalogName: String, tableProps: Map[String, String] = Map.empty) extends ServerSidePlanningMetadata { override def planningEndpointUri: String = "" - override def authToken: Option[String] = None + override def tokenSupplier: Option[() => String] = None override def tableProperties: Map[String, String] = tableProps } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala index 7406bf4edd9..04c175111c8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala @@ -16,29 +16,28 @@ package org.apache.spark.sql.delta.serverSidePlanning +import scala.jdk.CollectionConverters._ + +import io.unitycatalog.client.auth.TokenProvider + import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.{Identifier, Table} /** * Metadata for Unity Catalog tables. - * Provides base Iceberg REST endpoint for server-side planning. + * Provides base Iceberg REST endpoint and authentication for server-side planning. */ case class UnityCatalogMetadata( catalogName: String, ucUri: String, - ucToken: String, + override val tokenSupplier: Option[() => String], tableProps: Map[String, String]) extends ServerSidePlanningMetadata { override def planningEndpointUri: String = { - // Return base Iceberg REST path up to /v1/ - // The IcebergRESTCatalogPlanningClient will call /v1/config to get the prefix - // and construct the full URL according to the Iceberg REST catalog spec val base = if (ucUri.endsWith("/")) ucUri.dropRight(1) else ucUri s"$base/api/2.1/unity-catalog/iceberg-rest/v1" } - override def authToken: Option[String] = Some(ucToken) - override def tableProperties: Map[String, String] = tableProps } @@ -58,11 +57,55 @@ object UnityCatalogMetadata { // Read UC configuration from Spark conf val ucUri = spark.conf.get(s"spark.sql.catalog.$catalogName.uri", "") - val ucToken = spark.conf.get(s"spark.sql.catalog.$catalogName.token", "") + val authConfig = extractAuthConfig(spark, catalogName) + + val supplier: Option[() => String] = if (authConfig.nonEmpty) { + val tp = TokenProvider.create(authConfig.asJava) + Some(() => tp.accessToken()) + } else { + None + } - // Table properties currently unused, may be needed in future val tableProps = Map.empty[String, String] + UnityCatalogMetadata(catalogName, ucUri, supplier, tableProps) + } - UnityCatalogMetadata(catalogName, ucUri, ucToken, tableProps) + /** + * Extract authentication config for the planning endpoint. + * + * Follows the same pattern as UC's AuthConfigUtils: + * 1. Scan spark.sql.catalog..auth.* keys, strip prefix + * 2. If legacy .token exists: + * - Error if auth.token also set (configured twice) + * - Otherwise convert to Map("type" -> "static", ...) + * 3. Result passed to TokenProvider.create() + */ + private[serverSidePlanning] def extractAuthConfig( + spark: SparkSession, + catalogName: String): Map[String, String] = { + val catalogPrefix = s"spark.sql.catalog.$catalogName." + val authPrefix = s"${catalogPrefix}auth." + + // Extract all auth.* configs, stripping the prefix + val newFormatConfig = spark.conf.getAll + .filterKeys(_.startsWith(authPrefix)) + .map { case (fullKey, value) => (fullKey.stripPrefix(authPrefix), value) } + .toMap + + // Check legacy token + val legacyToken = spark.conf.get(s"${catalogPrefix}token", "") + + if (legacyToken.nonEmpty) { + // Conflict: both legacy .token and auth.token set + require(!newFormatConfig.contains("token"), + s"Static token configured twice. Use either " + + s"'spark.sql.catalog.$catalogName.token' (legacy) or " + + s"'spark.sql.catalog.$catalogName.auth.token' (new), not both.") + + // Legacy token: convert to static format. Matches UC AuthConfigUtils behavior. + Map("type" -> "static", "token" -> legacyToken) + } else { + newFormatConfig + } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala index 30533ec0643..231ea7493ed 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala @@ -265,7 +265,7 @@ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { // Verify the metadata has expected defaults assert(metadata.catalogName == "my_catalog") assert(metadata.planningEndpointUri == "") - assert(metadata.authToken.isEmpty) + assert(metadata.tokenSupplier.isEmpty) assert(metadata.tableProperties.isEmpty) } @@ -274,7 +274,7 @@ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { val metadata = UnityCatalogMetadata( catalogName = "test_catalog", ucUri = ucUri, - ucToken = "test-token", + tokenSupplier = Some(() => "test-token"), tableProps = Map.empty ) @@ -287,6 +287,76 @@ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { assert(metadata.planningEndpointUri == expectedEndpoint) } + test("extractAuthConfig: OAuth config") { + val cat = "oauthCat" + withSQLConf( + s"spark.sql.catalog.$cat.auth.type" -> "oauth", + s"spark.sql.catalog.$cat.auth.oauth.uri" -> + "https://oauth.example.com/token", + s"spark.sql.catalog.$cat.auth.oauth.clientId" -> + "client123", + s"spark.sql.catalog.$cat.auth.oauth.clientSecret" -> + "secret456" + ) { + val config = UnityCatalogMetadata + .extractAuthConfig(spark, cat) + assert(config("type") == "oauth") + assert(config("oauth.uri") == + "https://oauth.example.com/token") + assert(config("oauth.clientId") == "client123") + assert(config.size == 4) + } + } + + test("extractAuthConfig: legacy token as static") { + val cat = "legacyCat" + withSQLConf( + s"spark.sql.catalog.$cat.token" -> "dapi-token" + ) { + val config = UnityCatalogMetadata + .extractAuthConfig(spark, cat) + assert(config("type") == "static") + assert(config("token") == "dapi-token") + assert(config.size == 2) + } + } + + test("extractAuthConfig: error if both token formats") { + val cat = "conflictCat" + withSQLConf( + s"spark.sql.catalog.$cat.auth.token" -> "new", + s"spark.sql.catalog.$cat.token" -> "old" + ) { + val e = intercept[IllegalArgumentException] { + UnityCatalogMetadata.extractAuthConfig(spark, cat) + } + assert(e.getMessage.contains("configured twice")) + } + } + + test("extractAuthConfig: legacy token wins over auth.*") { + val cat = "legacyWinsCat" + withSQLConf( + s"spark.sql.catalog.$cat.auth.type" -> "oauth", + s"spark.sql.catalog.$cat.auth.oauth.uri" -> + "https://oauth.example.com", + s"spark.sql.catalog.$cat.token" -> "dapi-legacy" + ) { + val config = UnityCatalogMetadata + .extractAuthConfig(spark, cat) + // Legacy .token always wins (converted to static) + assert(config("type") == "static") + assert(config("token") == "dapi-legacy") + assert(config.size == 2) + } + } + + test("extractAuthConfig: empty when no auth") { + val config = UnityCatalogMetadata + .extractAuthConfig(spark, "nonExistentCat") + assert(config.isEmpty) + } + test("simple EqualTo filter pushed to planning client") { withPushdownCapturingEnabled { sql("SELECT id, name, value FROM test_db.shared_test WHERE id = 2").collect() From 95f05480673ae04159a9c198a9a7ef9fe09af6c5 Mon Sep 17 00:00:00 2001 From: Leon Windheuser Date: Mon, 30 Mar 2026 18:21:59 +0200 Subject: [PATCH 66/80] [Spark] Introduce ANALYSIS phase for fuzz tester (#6369) #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description Introduce an `ANALYSIS` phase for the fuzz tester. ## How was this patch tested? Fuzz tester ## Does this PR introduce _any_ user-facing changes? No --- .../spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala | 3 +++ .../fuzzer/PhaseLockingTransactionExecutionObserver.scala | 5 ++++- .../org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala | 1 + .../delta/concurrency/TransactionExecutionTestMixin.scala | 3 +++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala b/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala index 4fc42e666ce..0c452183afb 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.delta.fuzzer case class OptimisticTransactionPhases( + analysisPhase: ExecutionPhaseLock, initialPhase: ExecutionPhaseLock, preparePhase: ExecutionPhaseLock, commitPhase: ExecutionPhaseLock, @@ -27,6 +28,7 @@ object OptimisticTransactionPhases { private final val PREFIX = "TXN_" + final val ANALYSIS_PHASE_LABEL = PREFIX + "ANALYSIS" final val INITIAL_PHASE_LABEL = PREFIX + "INIT" final val PREPARE_PHASE_LABEL = PREFIX + "PREPARE" final val COMMIT_PHASE_LABEL = PREFIX + "COMMIT" @@ -39,6 +41,7 @@ object OptimisticTransactionPhases { txnName + "-" + phaseLabel OptimisticTransactionPhases( + analysisPhase = ExecutionPhaseLock(toTxnPhaseLabel(ANALYSIS_PHASE_LABEL)), initialPhase = ExecutionPhaseLock(toTxnPhaseLabel(INITIAL_PHASE_LABEL)), preparePhase = ExecutionPhaseLock(toTxnPhaseLabel(PREPARE_PHASE_LABEL)), commitPhase = ExecutionPhaseLock(toTxnPhaseLabel(COMMIT_PHASE_LABEL)), diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala b/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala index 62110fc0634..9b2d71502db 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala @@ -24,6 +24,7 @@ private[delta] class PhaseLockingTransactionExecutionObserver( with PhaseLockingExecutionObserver { override val phaseLocks: Seq[ExecutionPhaseLock] = Seq( + phases.analysisPhase, phases.initialPhase, phases.preparePhase, phases.commitPhase, @@ -46,8 +47,10 @@ private[delta] class PhaseLockingTransactionExecutionObserver( */ @volatile protected var autoAdvanceNextObserver: Boolean = false - override def startingTransaction(f: => OptimisticTransaction): OptimisticTransaction = + override def startingTransaction(f: => OptimisticTransaction): OptimisticTransaction = { + phases.analysisPhase.leave() phases.initialPhase.execute(f) + } override def preparingCommit[T](f: => T): T = phases.preparePhase.execute(f) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala index 29594434e4f..b0fb2eb0394 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala @@ -171,6 +171,7 @@ class DeltaAllFilesInCrcSuite try { val query = s"DELETE from delta.`${tempDir.getAbsolutePath}` WHERE id >= 0" val (observer, future) = runQueryWithObserver(name = "A", executor, query) + observer.phases.analysisPhase.entryBarrier.unblock() observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() // Make sure that delete query has run the actual computation and has reached diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala b/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala index 059314aa072..2b1a83e6850 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala @@ -53,6 +53,7 @@ trait TransactionExecutionTestMixin { spark.withActive( try { TransactionExecutionObserver.withObserver(observer) { + observer.phases.analysisPhase.waitToEnter() fn() } } catch { @@ -114,6 +115,7 @@ trait TransactionExecutionTestMixin { /** Unblocks all phases before the `commitPhase` for [[TransactionObserver]] */ def unblockUntilPreCommit(observer: TransactionObserver): Unit = { + observer.phases.analysisPhase.entryBarrier.unblock() observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() } @@ -129,6 +131,7 @@ trait TransactionExecutionTestMixin { /** Unblocks all phases for [[TransactionObserver]] so that corresponding query can finish. */ def unblockAllPhases(observer: TransactionObserver): Unit = { + observer.phases.analysisPhase.entryBarrier.unblock() observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() observer.phases.commitPhase.entryBarrier.unblock() From 8e997d8b85265e34fc2cac3f16d9a866b5e2e2f8 Mon Sep 17 00:00:00 2001 From: Thang Long Vu <107926660+longvu-db@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:34:53 +0200 Subject: [PATCH 67/80] [Spark] Add Delta Option useNullIntolerantEqualityWithDPO, to not overwrite partitions that contain NULL in the table (#6375) Co-authored-by: Isaac #### Which Delta project/connector is this regarding? - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description When users specify this option with DPO Mode, DPO will not overwrite partitions that contain NULL in the table. ## How was this patch tested? Added UTs. ## Does this PR introduce _any_ user-facing changes? Adding a new `DeltaOption` `useNullIntolerantEqualityWithDPO`, to ignore overwriting partitions that contain NULL in Dynamic Partition Overwrite mode. --- .../apache/spark/sql/delta/DeltaOptions.scala | 12 ++++ .../sql/delta/commands/WriteIntoDelta.scala | 26 +++++++- .../spark/sql/delta/DeltaOptionSuite.scala | 26 +++++++- .../apache/spark/sql/delta/DeltaSuite.scala | 65 +++++++++++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala index f42afd5c9ed..de2792546e2 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala @@ -53,6 +53,11 @@ trait DeltaWriteOptions val replaceWhere: Option[String] = options.get(REPLACE_WHERE_OPTION) val userMetadata: Option[String] = options.get(USER_METADATA_OPTION) + val useNullIntolerantEqualityWithDPO: Option[Boolean] = + options + .get(USE_NULL_INTOLERANT_EQUALITY_WITH_DPO) + .map(toBoolean(_, USE_NULL_INTOLERANT_EQUALITY_WITH_DPO)) + /** * Whether to add an adaptive shuffle before writing out the files to break skew, and coalesce * data into chunkier files. @@ -259,6 +264,12 @@ object DeltaOptions extends DeltaLogging { /** An option to specify user-defined metadata in commitInfo */ val USER_METADATA_OPTION = "userMetadata" + /** + * An option to ignore overwriting partitions that contain NULL in + * Dynamic Partition Overwrite, used by INSERT INTO ... REPLACE USING (...). + */ + val USE_NULL_INTOLERANT_EQUALITY_WITH_DPO = "useNullIntolerantEqualityWithDPO" + val PARTITION_OVERWRITE_MODE_OPTION = "partitionOverwriteMode" val PARTITION_OVERWRITE_MODE_DYNAMIC = "DYNAMIC" val PARTITION_OVERWRITE_MODE_STATIC = "STATIC" @@ -325,6 +336,7 @@ object DeltaOptions extends DeltaLogging { EXCLUDE_REGEX_OPTION, OVERWRITE_SCHEMA_OPTION, USER_METADATA_OPTION, + USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, PARTITION_OVERWRITE_MODE_OPTION, MAX_FILES_PER_TRIGGER_OPTION, IGNORE_FILE_DELETION_OPTION, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala index f2775c39cb8..bd6229eb262 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala @@ -333,11 +333,33 @@ case class WriteIntoDelta( txn, data, options ) val addFiles = newFiles.collect { case a: AddFile => a } + + if (!useDynamicPartitionOverwriteMode && + options.useNullIntolerantEqualityWithDPO.isDefined) { + throw DeltaErrors.illegalDeltaOptionException( + name = DeltaOptions.USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, + input = options.useNullIntolerantEqualityWithDPO.get.toString, + explain = "This option should be specified only in Dynamic Partition Overwrite mode.") + } + val deletedFiles = if (useDynamicPartitionOverwriteMode) { // with dynamic partition overwrite for any partition that is being written to all // existing data in that partition will be deleted. - // the selection what to delete is on the next two lines - val updatePartitions = addFiles.map(_.partitionValues).toSet + // the selection what to delete is determined by `updatePartitions`. + + // Dynamic Partition Overwrite (DPO) uses null-tolerant equality, meaning NULL partitions + // in the table will be overwritten if there are matching NULL values in the query. + // This option simulates null-intolerant equality by not including partitions with + // NULL values in the set of partitions to be overwritten. + val updatePartitions = + if (options.useNullIntolerantEqualityWithDPO.contains(true)) { + addFiles.collect { case addFile + if addFile.partitionValues.forall { case (_, value) => value != null } + => addFile.partitionValues + }.toSet + } else { + addFiles.map(_.partitionValues).toSet + } txn.filterFiles(updatePartitions).map(_.remove) } else { txn.filterFiles().map(_.remove) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala index 8ea7a1b36f9..0e629582a99 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.delta import java.util.Locale // scalastyle:off import.ordering.noEmptyLine -import org.apache.spark.sql.delta.DeltaOptions.{OVERWRITE_SCHEMA_OPTION, PARTITION_OVERWRITE_MODE_OPTION} +import org.apache.spark.sql.delta.DeltaOptions.{OVERWRITE_SCHEMA_OPTION, PARTITION_OVERWRITE_MODE_OPTION, USE_NULL_INTOLERANT_EQUALITY_WITH_DPO} import org.apache.spark.sql.delta.actions.{Action, FileAction} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest @@ -312,6 +312,30 @@ class DeltaOptionSuite extends QueryTest } } + test("useNullIntolerantEqualityWithDPO should never be specified when " + + "Dynamic Partition Overwrite is not enabled") { + withTable("temp") { + checkError( + exception = intercept[DeltaIllegalArgumentException] { + Seq(1, 2, 3).toDF + .withColumn("part", $"value" % 2) + .write + .mode("overwrite") + .format("delta") + .partitionBy("part") + .option(USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, "true") + .saveAsTable("temp") + }, + condition = "DELTA_ILLEGAL_OPTION", + sqlState = Some("42616"), + parameters = Map( + "name" -> USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, + "input" -> "true", + "explain" -> "This option should be specified only in Dynamic Partition Overwrite mode.") + ) + } + } + test("overwriteSchema=true should be invalid with partitionOverwriteMode=dynamic, " + "saveAsTable") { withTable("temp") { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala index c84830cdfe8..68a5d4ba57f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala @@ -20,6 +20,8 @@ import java.io.{File, FileNotFoundException} import java.util.concurrent.atomic.AtomicInteger // scalastyle:off import.ordering.noEmptyLine +import org.apache.spark.sql.delta.DeltaOptions +import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{Action, TableFeatureProtocolUtils} import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader @@ -3356,6 +3358,69 @@ class DeltaNameColumnMappingSuite extends DeltaSuite insertedDF.filter(col("id") >= 6).union(otherDF)) } } + + for (useNullIntolerantEqualityWithDPO <- BOOLEAN_DOMAIN) { + test(s"useNullIntolerantEqualityWithDPO=$useNullIntolerantEqualityWithDPO " + + "in Dynamic Partition Overwrite") { + withTempDir { tempDir => + Seq[(Option[Int], Option[Int], String)]( + (None, None, "target"), + (Some(1), None, "target"), + (Some(1), Some(2), "target"), + (Some(1), Some(3), "target")) + .toDF("part1", "part2", "row_origin") + .write + .format("delta") + .partitionBy("part1", "part2") + .mode("append") + .save(tempDir.getCanonicalPath) + + Seq[(Option[Int], Option[Int], String)]( + (None, None, "source"), + (Some(1), None, "source"), + (None, Some(1), "source"), + (Some(1), Some(2), "source"), + (Some(2), Some(3), "source")) + .toDF("part1", "part2", "row_origin") + .write + .format("delta") + .partitionBy("part1", "part2") + .mode("overwrite") + .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") + .option(DeltaOptions.USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, + useNullIntolerantEqualityWithDPO.toString) + .save(tempDir.getCanonicalPath) + + // When this option is set to true, partitions that contain null values are inserted, + // not overwritten. + if (useNullIntolerantEqualityWithDPO) { + checkAnswer( + spark.read.format("delta").load(tempDir.toString), + Row(null, null, "source") :: + Row(null, null, "target") :: + Row(1, null, "source") :: + Row(1, null, "target") :: + Row(null, 1, "source") :: + Row(1, 2, "source") :: + Row(1, 3, "target") :: + Row(2, 3, "source") :: + Nil + ) + } else { + checkAnswer( + spark.read.format("delta").load(tempDir.toString), + Row(null, null, "source") :: + Row(1, null, "source") :: + Row(null, 1, "source") :: + Row(1, 2, "source") :: + Row(1, 3, "target") :: + Row(2, 3, "source") :: + Nil + ) + } + } + } + } } class DeltaWithCatalogOwnedBatch1Suite extends DeltaSuite { From eeb138d72b08b9f516886960da9133e8a2b53568 Mon Sep 17 00:00:00 2001 From: littlegrasscao <45191106+littlegrasscao@users.noreply.github.com> Date: Mon, 30 Mar 2026 12:35:21 -0700 Subject: [PATCH 68/80] DeltaFormatSharingSource only finish current version when startOffset is from Legacy (#6392) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [X] Other (Sharing) ## Description When DeltaFormatSharingSource restarts from a legacy DeltaSharingSourceOffset checkpoint that is mid-version (index != -1), the stream must finish the current version before transitioning to the new DeltaSourceOffset format. - Lucky case (legacy offset at a version boundary, index == -1): transition immediately to DeltaSourceOffset, fetch file IDs with SHA256, and continue with normal multi-version batches. - Unlucky case (legacy offset mid-version): restrict the fetch to a single version using MD5 file IDs (matching the legacy hash), and convert the end offset back to legacy format until the stream reaches a version boundary. - Propagate fileIdHash through getTableFileChanges, DeltaSharingUtils.getRefresherForGetFiles, getRefresherForGetFilesWithStartingVersion, and getRefresherForGetCDFFiles so the server returns file IDs with the correct hash algorithm. ## How was this patch tested? New test cases ## Does this PR introduce _any_ user-facing changes? No --------- Co-authored-by: Sun Cao Co-authored-by: Rakesh Veeramacheneni <173086727+raveeram-db@users.noreply.github.com> --- .../spark/DeltaFormatSharingSource.scala | 73 ++- .../sharing/spark/DeltaSharingUtils.scala | 15 +- .../spark-4.0/SharingStreamingTestShims.scala | 2 + .../spark-4.1/SharingStreamingTestShims.scala | 4 +- .../spark-4.2/SharingStreamingTestShims.scala | 4 +- .../spark/DeltaFormatSharingSourceSuite.scala | 465 ++++++++++++++++++ .../TestClientForDeltaFormatSharing.scala | 19 + 7 files changed, 556 insertions(+), 26 deletions(-) diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala index 6404a36666f..98942851420 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.delta.sources.{ } import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils -import io.delta.sharing.client.DeltaSharingClient +import io.delta.sharing.client.{DeltaSharingClient, DeltaSharingRestClient} import io.delta.sharing.client.util.ConfUtils import io.delta.sharing.client.model.{Table => DeltaSharingTable} @@ -409,7 +409,7 @@ case class DeltaFormatSharingSource( return null } - maybeGetLatestFileChangesFromServer(deltaSourceOffset) + maybeGetLatestFileChangesFromServer(deltaSourceOffset, wasConvertedFromLegacy) maybeMoveToNextVersion(deltaSource.latestOffset(startDeltaSourceOffsetOpt.orNull, limit)) } @@ -471,17 +471,43 @@ case class DeltaFormatSharingSource( * 2 usages: 1) used to compare with latestTableVersionInLocalDeltaLogOpt to * check whether new files are needed. 2) used for getTableFileChanges, * check more details in the function header. + * @param wasConvertedFromLegacy true when the startingOffset was converted from a legacy + * DeltaSharingSourceOffset checkpoint. */ - private def maybeGetLatestFileChangesFromServer(startingOffset: DeltaSourceOffset): Unit = { + private def maybeGetLatestFileChangesFromServer( + startingOffset: DeltaSourceOffset, + wasConvertedFromLegacy: Boolean): Unit = { // Use a local variable to avoid a difference in the two usages below. val latestTableVersion = getOrUpdateLatestTableVersion if (needNewFilesFromServer(startingOffset, latestTableVersion)) { - val endingVersionForQuery = + // Lucky case: offset conversion lands on a version + // boundary -- fetch file IDs with sha256 and pull up to + // maxVersionsPerRpc versions as a regular batch. + // + // Unlucky case: offset conversion is mid-version -- fetch + // file IDs with md5 and restrict to 1 version so the + // batch finishes the version and lands on a boundary. + val isLegacyMidVersion = wasConvertedFromLegacy && + startingOffset.index != DeltaSourceOffset.BASE_INDEX + val endingVersionForQuery = if (isLegacyMidVersion) { + startingOffset.reservoirVersion + } else { getEndingVersionForRpc(startingOffset, latestTableVersion) + } + + val fileIdHash: Option[String] = if (isLegacyMidVersion) { + Some(DeltaSharingRestClient.FILEIDHASH_MD5) + } else { + Some(DeltaSharingRestClient.FILEIDHASH_SHA256) + } + + logInfo(s"Legacy transition state: wasConvertedFromLegacy=$wasConvertedFromLegacy, " + + s"isLegacyMidVersion=$isLegacyMidVersion, fileIdHash=$fileIdHash, " + + s"endingVersionForQuery=$endingVersionForQuery") if (startingOffset.isInitialSnapshot || !options.readChangeFeed) { - getTableFileChanges(startingOffset, endingVersionForQuery) + getTableFileChanges(startingOffset, endingVersionForQuery, fileIdHash) } else { throw new UnsupportedOperationException("CDF Streaming is not supported yet.") } @@ -499,10 +525,12 @@ case class DeltaFormatSharingSource( * it will only load files since startingOffset.reservoirVersion. * @param endingVersionForQuery The ending version used for the query, always smaller than * the latest table version on server. + * @param fileIdHash Optional hash algorithm to use for file ID hashing. */ private def getTableFileChanges( startingOffset: DeltaSourceOffset, - endingVersionForQuery: Long): Unit = { + endingVersionForQuery: Long, + fileIdHash: Option[String] = None): Unit = { logInfo( s"Fetching files with table version(${startingOffset.reservoirVersion}), " + s"index(${startingOffset.index}), isInitialSnapshot(${startingOffset.isInitialSnapshot})," + @@ -521,7 +549,7 @@ case class DeltaFormatSharingSource( timestampAsOf = None, jsonPredicateHints = None, refreshToken = None, - fileIdHash = None + fileIdHash = fileIdHash ) val refreshFunc = DeltaSharingUtils.getRefresherForGetFiles( client = client, @@ -531,7 +559,8 @@ case class DeltaFormatSharingSource( versionAsOf = Some(startingOffset.reservoirVersion), timestampAsOf = None, jsonPredicateHints = None, - useRefreshToken = false + useRefreshToken = false, + fileIdHash = fileIdHash ) logInfo( s"Fetched ${tableFiles.lines.size} lines for table version ${tableFiles.version} from" + @@ -545,13 +574,14 @@ case class DeltaFormatSharingSource( table = table, startingVersion = startingOffset.reservoirVersion, endingVersion = Some(endingVersionForQuery), - fileIdHash = None + fileIdHash = fileIdHash ) val refreshFunc = DeltaSharingUtils.getRefresherForGetFilesWithStartingVersion( client = client, table = table, startingVersion = startingOffset.reservoirVersion, - endingVersion = Some(endingVersionForQuery) + endingVersion = Some(endingVersionForQuery), + fileIdHash = fileIdHash ) logInfo( s"Fetched ${tableFiles.lines.size} lines from startingVersion " + @@ -608,20 +638,27 @@ case class DeltaFormatSharingSource( // specifies the starting version. On restart, this Source // is instantiated, so we convert legacy offsets to // DeltaSourceOffset. - val endOffset = forceToDeltaSourceOffset(end)._1 + val (endOffset, endConvertedFromLegacy) = forceToDeltaSourceOffset(end) // When the query is past the initial snapshot, // startOffsetOption is defined and contains the starting // version. Convert from legacy offset if needed. - val startDeltaOffsetOption = - startOffsetOption.map(o => forceToDeltaSourceOffset(o)._1) + val (startDeltaOffsetOption, startConvertedFromLegacy) = + startOffsetOption.map(forceToDeltaSourceOffset) match { + case Some((offset, fromLegacy)) => (Some(offset), fromLegacy) + case None => (None, false) + } + // Possible legacy conversion scenarios: + // 1. start=None, end=legacy: initial snapshot restart. + // 2. start=legacy, end=new: post-snapshot, version + // boundary reached in one batch. + val wasConvertedFromLegacy = startConvertedFromLegacy || endConvertedFromLegacy val startingOffset = getStartingOffset(startDeltaOffsetOption, Some(endOffset)) - // Files should already be fetched in latestOffset; this - // is a safeguard in case files for startOffset are not - // present. Whether startOffset was converted from legacy - // does not matter here. - maybeGetLatestFileChangesFromServer(startingOffset = startingOffset) + // For most cases, files should already be fetched in latestOffset. When latestOffset + // succeeded but getBatch failed, on restart, Spark engine skips latestOffset, so we need to + // fetch files here. + maybeGetLatestFileChangesFromServer(startingOffset, wasConvertedFromLegacy) // Reset latestProcessedEndOffsetOption only when endOffset is larger. // Because with microbatch pipelining, we may get getBatch requests out of order. if (latestProcessedEndOffsetOption.isEmpty || diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala index 2f7e9b46e71..ebd7999e54f 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala @@ -186,7 +186,8 @@ object DeltaSharingUtils extends Logging { versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - useRefreshToken: Boolean): RefresherFunction = { refreshTokenOpt => + useRefreshToken: Boolean, + fileIdHash: Option[String] = None): RefresherFunction = { refreshTokenOpt => { // If versionAsOf is specified, ignore refresh token (e.g., in streaming queries) val tableFiles = client @@ -198,7 +199,7 @@ object DeltaSharingUtils extends Logging { timestampAsOf = timestampAsOf, jsonPredicateHints = jsonPredicateHints, refreshToken = if (useRefreshToken) refreshTokenOpt else None, - fileIdHash = None + fileIdHash = fileIdHash ) getTableRefreshResult(tableFiles) } @@ -213,13 +214,14 @@ object DeltaSharingUtils extends Logging { def getRefresherForGetCDFFiles( client: DeltaSharingClient, table: Table, - cdfOptions: Map[String, String]): RefresherFunction = { (_: Option[String]) => + cdfOptions: Map[String, String], + fileIdHash: Option[String] = None): RefresherFunction = { (_: Option[String]) => { val tableFiles = client.getCDFFiles( table = table, cdfOptions = cdfOptions, includeHistoricalMetadata = true, - fileIdHash = None + fileIdHash = fileIdHash ) getTableRefreshResult(tableFiles) } @@ -235,14 +237,15 @@ object DeltaSharingUtils extends Logging { client: DeltaSharingClient, table: Table, startingVersion: Long, - endingVersion: Option[Long]): RefresherFunction = { (_: Option[String]) => + endingVersion: Option[Long], + fileIdHash: Option[String] = None): RefresherFunction = { (_: Option[String]) => { val tableFiles = client .getFiles( table = table, startingVersion = startingVersion, endingVersion = endingVersion, - fileIdHash = None + fileIdHash = fileIdHash ) getTableRefreshResult(tableFiles) } diff --git a/sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala b/sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala index 790d4a51fea..2b67fb154ef 100644 --- a/sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala +++ b/sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala @@ -19,6 +19,7 @@ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.{ CheckpointFileManager => CheckpointFileManagerShim, CommitMetadata => CommitMetadataShim, + OffsetSeqLog => OffsetSeqLogShim, SerializedOffset => SerializedOffsetShim, StreamMetadata => StreamMetadataShim } @@ -33,6 +34,7 @@ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim + type OffsetSeqLog = OffsetSeqLogShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim diff --git a/sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala b/sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala index 8700b3b2977..eaa199abf81 100644 --- a/sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala +++ b/sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala @@ -18,7 +18,8 @@ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.checkpointing.{ CheckpointFileManager => CheckpointFileManagerShim, - CommitMetadata => CommitMetadataShim + CommitMetadata => CommitMetadataShim, + OffsetSeqLog => OffsetSeqLogShim } import org.apache.spark.sql.execution.streaming.runtime.{ SerializedOffset => SerializedOffsetShim, @@ -34,6 +35,7 @@ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim + type OffsetSeqLog = OffsetSeqLogShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim val StreamingCheckpointConstants: StreamingCheckpointConstantsShim.type = diff --git a/sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala b/sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala index 53057098891..ff6f23e9c73 100644 --- a/sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala +++ b/sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala @@ -18,7 +18,8 @@ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.checkpointing.{ CheckpointFileManager => CheckpointFileManagerShim, - CommitMetadata => CommitMetadataShim + CommitMetadata => CommitMetadataShim, + OffsetSeqLog => OffsetSeqLogShim } import org.apache.spark.sql.execution.streaming.runtime.{ SerializedOffset => SerializedOffsetShim, @@ -34,6 +35,7 @@ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim + type OffsetSeqLog = OffsetSeqLogShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim val StreamingCheckpointConstants: StreamingCheckpointConstantsShim.type = diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala index 6cd9ac3add8..a99c6730513 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala @@ -37,6 +37,7 @@ import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import io.delta.sharing.spark.test.shims.SharingStreamingTestShims.{ CheckpointFileManager, CommitMetadata, + OffsetSeqLog, SerializedOffset, StreamingCheckpointConstants, StreamMetadata @@ -1742,4 +1743,468 @@ class DeltaFormatSharingSourceSuite } } } + + // E2E test: Legacy checkpoint at a version boundary (index=-1, i.e., the + // "lucky case") with isStartingVersion=true. The stream should transition + // fully to DeltaSourceOffset and use SHA256 file IDs. + test("E2E: legacy checkpoint at version boundary (lucky case) transitions cleanly") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_lucky_case" + withTable(deltaTableName) { + sql(s"""CREATE TABLE $deltaTableName (value STRING) + |USING DELTA""".stripMargin) + sql(s"INSERT INTO $deltaTableName VALUES ('a'), ('b')") + sql(s"INSERT INTO $deltaTableName VALUES ('c'), ('d')") + val tableId = DeltaLog.forTable( + spark, new TableIdentifier(deltaTableName)) + .update().metadata.id + val sharedTableName = "shared_lucky_case" + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + + s"#share1.default.$sharedTableName" + spark.sessionState.conf.setConfString( + "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s") + + // Build checkpoint: legacy offset at version boundary (index=-1) + val checkpointPath = new Path(checkpointDir.getCanonicalPath) + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) + val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS + val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS + val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA + fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) + fileManager.mkdirs(new Path(checkpointPath, commitsDir)) + val metadataPath = new Path(checkpointPath, metaDir) + val streamId = java.util.UUID.randomUUID.toString + StreamMetadata.write(StreamMetadata(streamId), metadataPath, hadoopConf) + val offsetMetadataJson = + """{"batchWatermarkMs":0,"batchTimestampMs":0,"conf":{},"sourceMetadataInfo":{}}""" + + // Batch 0: version 1, at boundary (index=-1), initial snapshot + val legacyOffset0Json = + "{\"sourceVersion\":1," + + s""""tableId":"$tableId",""" + + "\"tableVersion\":1," + + "\"index\":-1," + + "\"isStartingVersion\":true}" + val offset0Content = + s"v1\n$offsetMetadataJson\n$legacyOffset0Json" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val offset0Path = new Path(new Path(checkpointPath, offsetsDir), "0") + val offset0Out = fileManager.createAtomic(offset0Path, overwriteIfPossible = true) + offset0Out.write(offset0Content) + offset0Out.close() + val commit0Content = s"v1\n${CommitMetadata(0).json}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val commit0Path = new Path(new Path(checkpointPath, commitsDir), "0") + val commit0Out = fileManager.createAtomic(commit0Path, overwriteIfPossible = true) + commit0Out.write(commit0Content) + commit0Out.close() + + val autoResolveKey = DeltaSQLConf + .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT.key + withSQLConf( + (getDeltaSharingClassesSQLConf ++ Seq( + autoResolveKey -> "true" + )).toSeq: _* + ) { + prepareMockedClientMetadata(deltaTableName, sharedTableName) + // getBatch(None, offset_0) with isInitialSnapshot=true uses + // snapshot at the same version. + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName, versionAsOf = Some(1L)) + // Since the legacy offset is at a version boundary (index=-1 -> BASE_INDEX), + // this is the "lucky case": the stream should use SHA256 file IDs and + // fetch multiple versions normally. + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 1L, 2L) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 2L) + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) + + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + // Replayed batch 0 produces v1 snapshot, then version 2 + checkAnswer( + spark.read.format("delta").load(outputDir.getCanonicalPath), + Seq("a", "b", "c", "d").toDF()) + + // Validate the final offset is in DeltaSourceOffset format + // (has reservoirVersion, not tableVersion). + val offsetLog = new OffsetSeqLog( + spark, s"${checkpointDir.getCanonicalPath}/offsets") + val (latestBatchId, latestOffsetSeq) = offsetLog.getLatest().get + val offsetJson = latestOffsetSeq.offsets.head.get.json() + assert(offsetJson.contains("reservoirVersion"), + s"Expected DeltaSourceOffset (reservoirVersion) but got: $offsetJson") + assert(!offsetJson.contains("tableVersion"), + s"Expected no legacy tableVersion in final offset but got: $offsetJson") + } + } + } + } + + // E2E: Legacy checkpoint mid-version (index != -1, the "unlucky case") + // with isStartingVersion=false. The stream should restrict to 1 version + // with MD5 file IDs, then transition after reaching a boundary. + test("E2E: legacy checkpoint mid-version (unlucky case) " + + "finishes version before transitioning") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_unlucky_case" + withTable(deltaTableName) { + sql(s"""CREATE TABLE $deltaTableName (value STRING) + |USING DELTA""".stripMargin) + sql(s"INSERT INTO $deltaTableName VALUES ('v1a'), ('v1b')") + // Version 2: exactly 2 files (1 row each) for the mid-version case. + // parallelize with numSlices=2 places each row in its own partition. + spark.createDataFrame( + spark.sparkContext.parallelize(Seq(Row("v2a"), Row("v2b")), 2), + new StructType().add("value", StringType) + ).write.mode("append").insertInto(deltaTableName) + sql(s"INSERT INTO $deltaTableName VALUES ('v3a'), ('v3b')") + val tableId = DeltaLog.forTable( + spark, new TableIdentifier(deltaTableName)) + .update().metadata.id + val sharedTableName = "shared_unlucky_case" + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + + s"#share1.default.$sharedTableName" + spark.sessionState.conf.setConfString( + "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s") + + // Build checkpoint: legacy offset mid-version (index=0, not -1) + val checkpointPath = new Path(checkpointDir.getCanonicalPath) + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) + val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS + val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS + val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA + fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) + fileManager.mkdirs(new Path(checkpointPath, commitsDir)) + val metadataPath = new Path(checkpointPath, metaDir) + val streamId = java.util.UUID.randomUUID.toString + StreamMetadata.write(StreamMetadata(streamId), metadataPath, hadoopConf) + val offsetMetadataJson = + """{"batchWatermarkMs":0,"batchTimestampMs":0,"conf":{},"sourceMetadataInfo":{}}""" + + // Batch 0: mid-version at version 2, index 0 (processed 1 of 2 files) + val legacyOffset0Json = + "{\"sourceVersion\":1," + + s""""tableId":"$tableId",""" + + "\"tableVersion\":2," + + "\"index\":0," + + "\"isStartingVersion\":false}" + val offset0Content = + s"v1\n$offsetMetadataJson\n$legacyOffset0Json" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val offset0Path = new Path(new Path(checkpointPath, offsetsDir), "0") + val offset0Out = fileManager.createAtomic(offset0Path, overwriteIfPossible = true) + offset0Out.write(offset0Content) + offset0Out.close() + val commit0Content = s"v1\n${CommitMetadata(0).json}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val commit0Path = new Path(new Path(checkpointPath, commitsDir), "0") + val commit0Out = fileManager.createAtomic(commit0Path, overwriteIfPossible = true) + commit0Out.write(commit0Content) + commit0Out.close() + + val autoResolveKey = DeltaSQLConf + .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT.key + withSQLConf( + (getDeltaSharingClassesSQLConf ++ Seq( + autoResolveKey -> "true" + )).toSeq: _* + ) { + prepareMockedClientMetadata(deltaTableName, sharedTableName) + // getBatch(None, offset_0) computes starting offset at + // version 1 as initial snapshot, so we need this mock. + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName, versionAsOf = Some(1L)) + // For mid-version case, the stream should restrict to version 2 + // only and use MD5 file IDs. + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 2L) + // After transitioning at version boundary, stream fetches + // remaining versions normally. + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 3L) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 3L, 3L) + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) + + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + // The legacy checkpoint was at version 2, index 0 (1 of 2 files + // processed). The stream finishes the remaining file in version 2, + // transitions to DeltaSourceOffset at the version boundary, then + // processes version 3 normally. + val outputValues = spark.read.format("delta") + .load(outputDir.getCanonicalPath) + .collect().map(_.getString(0)).toSet + assert(outputValues.size == 3, + s"Expected 3 rows (1 remaining v2 + 2 v3) but got: $outputValues") + assert(outputValues.contains("v3a") && outputValues.contains("v3b"), + s"Expected v3a and v3b in output but got: $outputValues") + assert(outputValues.intersect(Set("v2a", "v2b")).size == 1, + s"Expected exactly one v2 row from the unprocessed file but got: $outputValues") + + // Validate the final offset has transitioned to DeltaSourceOffset format. + val offsetLog = new OffsetSeqLog( + spark, s"${checkpointDir.getCanonicalPath}/offsets") + val (latestBatchId, latestOffsetSeq) = offsetLog.getLatest().get + val offsetJson = latestOffsetSeq.offsets.head.get.json() + assert(offsetJson.contains("reservoirVersion"), + s"Expected DeltaSourceOffset (reservoirVersion) but got: $offsetJson") + assert(!offsetJson.contains("tableVersion"), + s"Expected no legacy tableVersion in final offset but got: $offsetJson") + } + } + } + } + + // Test that fileIdHash is correctly passed to the server: SHA256 for + // version-boundary legacy offsets (lucky case), MD5 for mid-version + // legacy offsets (unlucky case), and SHA256 after transition. + test("fileIdHash: SHA256 for boundary legacy offset") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_fileidhash" + withTable(deltaTableName) { + sql(s"""CREATE TABLE $deltaTableName (value STRING) + |USING DELTA""".stripMargin) + sql(s"INSERT INTO $deltaTableName VALUES ('a'), ('b')") + sql(s"INSERT INTO $deltaTableName VALUES ('c'), ('d')") + val tableId = DeltaLog.forTable( + spark, new TableIdentifier(deltaTableName)) + .update().metadata.id + val sharedTableName = "shared_fileidhash" + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + + s"#share1.default.$sharedTableName" + spark.sessionState.conf.setConfString( + "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s") + + val checkpointPath = new Path(checkpointDir.getCanonicalPath) + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) + val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS + val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS + val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA + fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) + fileManager.mkdirs(new Path(checkpointPath, commitsDir)) + val metadataPath = new Path(checkpointPath, metaDir) + val streamId = java.util.UUID.randomUUID.toString + StreamMetadata.write(StreamMetadata(streamId), metadataPath, hadoopConf) + val offsetMetadataJson = + """{"batchWatermarkMs":0,"batchTimestampMs":0,"conf":{},"sourceMetadataInfo":{}}""" + + // Version boundary (lucky case) should use SHA256 + val boundaryOffsetJson = + "{\"sourceVersion\":1," + + s""""tableId":"$tableId",""" + + "\"tableVersion\":1," + + "\"index\":-1," + + "\"isStartingVersion\":true}" + val offset0Content = + s"v1\n$offsetMetadataJson\n$boundaryOffsetJson" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val offset0Path = new Path(new Path(checkpointPath, offsetsDir), "0") + val offset0Out = fileManager.createAtomic(offset0Path, overwriteIfPossible = true) + offset0Out.write(offset0Content) + offset0Out.close() + val commit0Content = s"v1\n${CommitMetadata(0).json}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val commit0Path = new Path(new Path(checkpointPath, commitsDir), "0") + val commit0Out = fileManager.createAtomic(commit0Path, overwriteIfPossible = true) + commit0Out.write(commit0Content) + commit0Out.close() + + val autoResolveKey = DeltaSQLConf + .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT.key + withSQLConf( + (getDeltaSharingClassesSQLConf ++ Seq( + autoResolveKey -> "true" + )).toSeq: _* + ) { + prepareMockedClientMetadata(deltaTableName, sharedTableName) + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName, versionAsOf = Some(1L)) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 1L, 2L) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 2L) + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) + + TestClientForDeltaFormatSharing.clearFileIdHashHistory() + + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + + // All streaming getFiles calls should use SHA256 for the + // lucky case (version boundary). + val history = TestClientForDeltaFormatSharing.getFileIdHashHistory + val streamingCalls = history.filter { case (name, qt, _) => + name == sharedTableName && qt.startsWith("getFiles_streaming") + } + assert(streamingCalls.nonEmpty, "Expected at least one streaming getFiles call") + streamingCalls.foreach { case (_, queryType, fileIdHash) => + assert(fileIdHash.contains(DeltaSharingRestClient.FILEIDHASH_SHA256), + s"Expected SHA256 for boundary legacy offset but got $fileIdHash in $queryType") + } + } + } + } + } + + test("fileIdHash: MD5 for mid-version legacy offset") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_fileidhash_mid" + withTable(deltaTableName) { + sql(s"""CREATE TABLE $deltaTableName (value STRING) + |USING DELTA""".stripMargin) + sql(s"INSERT INTO $deltaTableName VALUES ('e'), ('f')") + sql(s"INSERT INTO $deltaTableName VALUES ('g'), ('h')") + sql(s"INSERT INTO $deltaTableName VALUES ('i'), ('j')") + val tableId = DeltaLog.forTable( + spark, new TableIdentifier(deltaTableName)) + .update().metadata.id + val sharedTableName = "shared_fileidhash_mid" + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + + s"#share1.default.$sharedTableName" + spark.sessionState.conf.setConfString( + "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s") + + val checkpointPath = new Path(checkpointDir.getCanonicalPath) + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) + val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS + val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS + val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA + fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) + fileManager.mkdirs(new Path(checkpointPath, commitsDir)) + val metadataPath = new Path(checkpointPath, metaDir) + val streamId = java.util.UUID.randomUUID.toString + StreamMetadata.write(StreamMetadata(streamId), metadataPath, hadoopConf) + val offsetMetadataJson = + """{"batchWatermarkMs":0,"batchTimestampMs":0,"conf":{},"sourceMetadataInfo":{}}""" + + // Mid-version offset: version 2, index 0 (not at boundary) + val midVersionOffsetJson = + "{\"sourceVersion\":1," + + s""""tableId":"$tableId",""" + + "\"tableVersion\":2," + + "\"index\":0," + + "\"isStartingVersion\":false}" + val offset0Content = + s"v1\n$offsetMetadataJson\n$midVersionOffsetJson" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val offset0Path = new Path(new Path(checkpointPath, offsetsDir), "0") + val offset0Out = fileManager.createAtomic(offset0Path, overwriteIfPossible = true) + offset0Out.write(offset0Content) + offset0Out.close() + val commit0Content = s"v1\n${CommitMetadata(0).json}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val commit0Path = new Path(new Path(checkpointPath, commitsDir), "0") + val commit0Out = fileManager.createAtomic(commit0Path, overwriteIfPossible = true) + commit0Out.write(commit0Content) + commit0Out.close() + + val autoResolveKey = DeltaSQLConf + .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT.key + withSQLConf( + (getDeltaSharingClassesSQLConf ++ Seq( + autoResolveKey -> "true" + )).toSeq: _* + ) { + prepareMockedClientMetadata(deltaTableName, sharedTableName) + // getBatch recovery loads snapshot at version 2-1=1 + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName, versionAsOf = Some(1L)) + // Mid-version: restricted to version 2 only (MD5) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 2L) + // After transition: fetches from version 2 onwards normally (SHA256) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 3L) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 3L, 3L) + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) + + TestClientForDeltaFormatSharing.clearFileIdHashHistory() + + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + + val history = TestClientForDeltaFormatSharing.getFileIdHashHistory + val streamingCalls = history.filter { case (name, qt, _) => + name == sharedTableName && qt.startsWith("getFiles_streaming") + } + assert(streamingCalls.nonEmpty, "Expected at least one streaming getFiles call") + // The first streaming call should use MD5 (mid-version legacy), + // subsequent calls after transition should use SHA256. + val firstCall = streamingCalls.head + assert(firstCall._3.contains(DeltaSharingRestClient.FILEIDHASH_MD5), + s"Expected MD5 for mid-version legacy offset but got ${firstCall._3}") + if (streamingCalls.size > 1) { + streamingCalls.tail.foreach { case (_, queryType, fileIdHash) => + assert(fileIdHash.contains(DeltaSharingRestClient.FILEIDHASH_SHA256), + s"Expected SHA256 after transition but got $fileIdHash in $queryType") + } + } + } + } + } + } } diff --git a/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala b/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala index 1180ad2aa85..d944cbf091e 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala @@ -154,6 +154,10 @@ private[spark] class TestClientForDeltaFormatSharing( val tableFullName = s"${table.share}.${table.schema}.${table.name}" limit.foreach(lim => TestClientForDeltaFormatSharing.limits.put(tableFullName, lim)) TestClientForDeltaFormatSharing.requestedFormat.put(tableFullName, responseFormat) + TestClientForDeltaFormatSharing.fileIdHashHistory.synchronized { + TestClientForDeltaFormatSharing.fileIdHashHistory += + ((table.name, "getFiles_snapshot", fileIdHash)) + } jsonPredicateHints.foreach(p => TestClientForDeltaFormatSharing.jsonPredicateHints.put(tableFullName, p)) @@ -220,6 +224,10 @@ private[spark] class TestClientForDeltaFormatSharing( ) val tableFullName = s"${table.share}.${table.schema}.${table.name}" TestClientForDeltaFormatSharing.requestedFormat.put(tableFullName, responseFormat) + TestClientForDeltaFormatSharing.fileIdHashHistory.synchronized { + TestClientForDeltaFormatSharing.fileIdHashHistory += + ((table.name, s"getFiles_streaming_${startingVersion}_${endingVersion.get}", fileIdHash)) + } val iterator = SparkEnv.get.blockManager .get[String](getBlockId(table.name, s"getFiles_${startingVersion}_${endingVersion.get}")) .map(_.data.asInstanceOf[Iterator[String]]) @@ -318,4 +326,15 @@ object TestClientForDeltaFormatSharing { val requestedFormat = scala.collection.mutable.Map[String, String]() val jsonPredicateHints = scala.collection.mutable.Map[String, String]() @volatile var lastCallerOrg: String = "" + + // Captures (tableName, queryType, fileIdHash) for each getFiles call. + val fileIdHashHistory = scala.collection.mutable.ArrayBuffer[(String, String, Option[String])]() + + def clearFileIdHashHistory(): Unit = fileIdHashHistory.synchronized { + fileIdHashHistory.clear() + } + + def getFileIdHashHistory: Seq[(String, String, Option[String])] = fileIdHashHistory.synchronized { + fileIdHashHistory.toSeq + } } From e67af7e6bcc0c48af08e2b490d5520a2dbdd638b Mon Sep 17 00:00:00 2001 From: Milan Stefanovic <150366084+milastdbx@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:37:21 +0200 Subject: [PATCH 69/80] [Kernel] Geospatial stats parsing: handle geometry/geography as WKT strings (#6301) #### Which Delta project/connector is this regarding? - [ ] Spark - [ ] Standalone - [ ] Flink - [X] Kernel - [ ] Other (fill in here) ## Description This is **PR 1 of 3** for geospatial data skipping support. It teaches the stats system (JSON parsing, schema validation, DataFileStatistics) to handle geometry/geography columns as WKT string values, and adds GeometryUtils for WKT POINT parsing/formatting. Delta spec: https://github.com/delta-io/delta/pull/4726 ### Changes - **StatsSchemaHelper**: Mark `GeometryType`/`GeographyType` as eligible for min/max stats - **JsonUtils**: Handle geometry/geography as WKT string in `parseJsonValueToLiteral()` - **SchemaUtils**: Add geometry/geography to supported types in `validateSupportedType()` - **DataFileStatistics**: Validate geometry/geography stats accept string literals - **DefaultJsonRow**: Decode geometry/geography JSON values as strings - **GeometryUtils** (new): WKT POINT parsing (`parsePointXY`) and formatting (`formatPointWKT`) ### PR dependency graph ``` master | +-- PR A (this): Stats Parsing + GeometryUtils | | | +-- PR B: Expressions + Predicate Translation | | | +-- PR C: Parquet / Data Layer ``` ## How was this patch tested? - `GeometryUtilsSuite`: 15 tests covering WKT parsing/formatting, round-trips, error cases - `DefaultJsonHandlerSuite`: pending test placeholder for geometry/geography parsing ## Does this PR introduce _any_ user-facing changes? Adds new table feature --------- Co-authored-by: Uros Bojanic --- .../io/delta/kernel/expressions/Literal.java | 21 + .../internal/skipping/StatsSchemaHelper.java | 6 +- .../kernel/internal/util/GeometryUtils.java | 178 +++++++ .../delta/kernel/internal/util/JsonUtils.java | 13 + .../kernel/internal/util/SchemaUtils.java | 4 +- .../kernel/statistics/DataFileStatistics.java | 7 + .../util/DataFileStatisticsSuite.scala | 496 ++++++++++++++++++ .../internal/util/GeometryUtilsSuite.scala | 297 +++++++++++ .../internal/data/DefaultJsonRow.java | 5 + .../engine/DefaultJsonHandlerSuite.scala | 57 ++ .../delta/kernel/defaults/utils/TestRow.scala | 2 + 11 files changed, 1084 insertions(+), 2 deletions(-) create mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/GeometryUtils.java create mode 100644 kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/GeometryUtilsSuite.scala diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java index 4f111affdff..5dac2ef78d7 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java @@ -193,6 +193,25 @@ public static Literal ofDecimal(BigDecimal value, int precision, int scale) { return new Literal(valueToStore, new DecimalType(precision, scale)); } + /** + * Create a geospatial Well-Known Text (WKT) literal with the given data type. This factory only + * enforces the type contract (GeometryType or GeographyType). Callers are responsible for WKT + * content validation, as different engines may support different geometry types beyond POINT. + * + * @param wkt WKT string value + * @param dataType must be GeometryType or GeographyType + * @return a Literal with the given geospatial type and String value + */ + public static Literal ofGeospatialWKT(String wkt, DataType dataType) { + Objects.requireNonNull( + wkt, "WKT string cannot be null; use Literal.ofNull()" + " for null values"); + checkArgument( + dataType instanceof GeometryType || dataType instanceof GeographyType, + "dataType must be GeometryType or GeographyType, got: %s", + dataType); + return new Literal(wkt, dataType); + } + /** * Create {@code null} value literal. * @@ -232,6 +251,8 @@ private Literal(Object value, DataType dataType) { *

  • TIMESTAMP: {@link Long} represents the microseconds since epoch in UTC *
  • TIMESTAMP_NTZ: {@link Long} represents the microseconds since epoch with no timezone *
  • DECIMAL: {@link BigDecimal}.Use {@link #getDataType()} to find the precision and scale + *
  • GEOMETRY: {@link String} WKT (Well-Known Text) representation + *
  • GEOGRAPHY: {@link String} WKT (Well-Known Text) representation * * * @return Literal value. diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java index 804f9280192..1cc6d6c0bcf 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java @@ -69,7 +69,11 @@ public static boolean isSkippingEligibleDataType(DataType dataType) { || // StringType is eligible, but since its string can include collation info, it needs to // be matched separately. - dataType instanceof StringType; + dataType instanceof StringType + // Geometry/geography types are eligible but need separate matching + // due to SRID (and algorithm) parameters + || dataType instanceof GeometryType + || dataType instanceof GeographyType; } /** diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/GeometryUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/GeometryUtils.java new file mode 100644 index 00000000000..c784f1b6e63 --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/GeometryUtils.java @@ -0,0 +1,178 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal.util; + +import java.util.Objects; +import java.util.OptionalDouble; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Utility for parsing and formatting WKT POINT strings for geometry/geography stats. */ +public class GeometryUtils { + private GeometryUtils() {} + + // Matches: POINT [ZM|Z|M] (coords) - (case-insensitive, flexible whitespace) + private static final Pattern POINT_WKT_PATTERN = + Pattern.compile("^POINT\\s*(ZM|Z|M)?\\s*\\(([^)]+)\\)$", Pattern.CASE_INSENSITIVE); + + // Matches: POINT [Z|M|ZM] EMPTY - (case-insensitive, flexible whitespace) + private static final Pattern POINT_EMPTY_PATTERN = + Pattern.compile("^POINT\\s*(ZM|Z|M)?\\s+EMPTY$", Pattern.CASE_INSENSITIVE); + + /** + * Formats x/y (and optional z/m) coordinates as a WKT POINT string. Uses standard OGC WKT format: + * "POINT(1.0 2.0)", "POINT Z (1.0 2.0 3.0)", "POINT ZM (1.0 2.0 3.0 4.0)". + */ + public static String formatPointWKT(double x, double y, OptionalDouble z, OptionalDouble m) { + boolean hasZ = z.isPresent(); + boolean hasM = m.isPresent(); + if (Double.isNaN(x) && Double.isNaN(y)) { + StringBuilder sb = new StringBuilder("POINT"); + if (hasZ == hasM) { + if (hasZ) sb.append(" ZM"); + } else { + sb.append(hasZ ? " Z" : " M"); + } + return sb.append(" EMPTY").toString(); + } + StringBuilder sb = new StringBuilder("POINT"); + if (hasZ == hasM) { + if (hasZ) sb.append(" ZM "); + } else { + sb.append(hasZ ? " Z " : " M "); + } + sb.append("(").append(x).append(" ").append(y); + z.ifPresent(v -> sb.append(" ").append(v)); + m.ifPresent(v -> sb.append(" ").append(v)); + return sb.append(")").toString(); + } + + /** + * Validates that the given string is a well-formed WKT POINT. Throws KernelException if the + * string is null or not a valid POINT WKT. + */ + public static void validatePointWKT(String wkt) { + try { + parsePointXY(wkt); + } catch (IllegalArgumentException e) { + throw new io.delta.kernel.exceptions.KernelException( + String.format("Geospatial stats must be a valid POINT WKT but got: %s", wkt), e); + } + } + + /** + * Parses only the x and y coordinates from a WKT POINT string. Returns double[]{x, y}. Z and M + * coordinates are validated but not returned because stats bounding boxes only use 2D + * coordinates. + * + *

    Supported formats: POINT(x y), POINT Z (x y z), POINT M (x y m), POINT ZM (x y z m), POINT + * EMPTY, POINT Z EMPTY, POINT M EMPTY, POINT ZM EMPTY. + * + *

    All EMPTY variants return {NaN, NaN}. + * + * @throws IllegalArgumentException if not a valid POINT WKT + */ + public static double[] parsePointXY(String wkt) { + Objects.requireNonNull(wkt, "WKT POINT string cannot be null"); + String trimmed = wkt.trim(); + + if (POINT_EMPTY_PATTERN.matcher(trimmed).matches()) { + return new double[] {Double.NaN, Double.NaN}; + } + + Matcher matcher = POINT_WKT_PATTERN.matcher(trimmed); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid WKT POINT string: " + wkt); + } + + String modifier = matcher.group(1); + String coordStr = matcher.group(2).trim(); + String[] parts = coordStr.split("\\s+"); + + String mod = (modifier == null) ? "" : modifier.toUpperCase(); + int expectedCount; + switch (mod) { + case "ZM": + expectedCount = 4; + break; + case "Z": + case "M": + expectedCount = 3; + break; + default: + expectedCount = 2; + break; + } + + if (parts.length != expectedCount) { + throw new IllegalArgumentException( + String.format( + "POINT %s expects %d coordinates but got %d: %s", + mod, expectedCount, parts.length, wkt)); + } + + try { + double x = Double.parseDouble(parts[0]); + double y = Double.parseDouble(parts[1]); + if (!Double.isFinite(x) || !Double.isFinite(y)) { + throw new IllegalArgumentException("POINT coordinates must be finite numbers: " + wkt); + } + for (int i = 2; i < parts.length; i++) { + double v = Double.parseDouble(parts[i]); + if (!Double.isFinite(v)) { + throw new IllegalArgumentException("POINT coordinates must be finite numbers: " + wkt); + } + } + return new double[] {x, y}; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid coordinate in WKT POINT string: " + wkt, e); + } + } + + /** + * Validates that the given WKT POINT is a valid geography point with coordinates in range: + * longitude in [-180, 180] and latitude in [-90, 90]. Throws KernelException if invalid. + */ + public static void validateGeographyPointWKT(String wkt) { + double[] xy; + try { + xy = parsePointXY(wkt); + } catch (IllegalArgumentException e) { + throw new io.delta.kernel.exceptions.KernelException( + String.format("Geospatial stats must be a valid POINT WKT" + " but got: %s", wkt), e); + } + validateGeographyCoordinates(xy[0], xy[1], wkt); + } + + /** + * Validates that longitude is in [-180, 180] and latitude is in [-90, 90]. Throws KernelException + * if out of range. + */ + public static void validateGeographyCoordinates(double lon, double lat, String wkt) { + // POINT EMPTY produces NaN — skip range check + if (Double.isNaN(lon) && Double.isNaN(lat)) { + return; + } + if (lon < -180 || lon > 180 || lat < -90 || lat > 90) { + throw new io.delta.kernel.exceptions.KernelException( + String.format( + "Geography coordinates out of range: longitude" + + " must be in [-180, 180], latitude must be" + + " in [-90, 90], but got: %s", + wkt)); + } + } +} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java index 3104c57f585..968df6b2b2c 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java @@ -258,6 +258,19 @@ public static Literal parseJsonValueToLiteral(JsonNode valueNode, DataType dataT } String textValue = valueNode.asText(); return Literal.ofString(textValue); + } else if (dataType instanceof GeometryType || dataType instanceof GeographyType) { + if (!valueNode.isTextual()) { + String typeName = dataType instanceof GeometryType ? "geometry" : "geography"; + throw new KernelException( + String.format("Expected %s as WKT string but got: %s", typeName, valueNode)); + } + String wkt = valueNode.asText(); + if (dataType instanceof GeographyType) { + GeometryUtils.validateGeographyPointWKT(wkt); + } else { + GeometryUtils.validatePointWKT(wkt); + } + return Literal.ofGeospatialWKT(wkt, dataType); } else { throw unsupportedStatsDataType(dataType); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java index 4aca1ae22ec..0e500265c67 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java @@ -853,7 +853,9 @@ protected static void validateSupportedType(DataType dataType) { || dataType instanceof DateType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType - || dataType instanceof VariantType) { + || dataType instanceof VariantType + || dataType instanceof GeometryType + || dataType instanceof GeographyType) { // supported types return; } else if (dataType instanceof StructType) { diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java b/kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java index 84e77904f33..f09cda1c665 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java @@ -25,6 +25,7 @@ import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.skipping.StatsSchemaHelper; +import io.delta.kernel.internal.util.GeometryUtils; import io.delta.kernel.internal.util.JsonUtils; import io.delta.kernel.types.*; import java.io.IOException; @@ -437,6 +438,12 @@ private void writeJsonValue(JsonGenerator generator, Literal literal) throws IOE LocalDateTime localDateTime = ChronoUnit.MICROS.addTo(EPOCH, epochMicros).toLocalDateTime(); LocalDateTime truncated = localDateTime.truncatedTo(ChronoUnit.MILLIS); generator.writeString(truncated.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); + } else if (type instanceof GeometryType) { + GeometryUtils.validatePointWKT((String) value); + generator.writeString((String) value); + } else if (type instanceof GeographyType) { + GeometryUtils.validateGeographyPointWKT((String) value); + generator.writeString((String) value); } else { throw unsupportedStatsDataType(type); } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala index 088929b6029..91bac545bd6 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala @@ -1027,4 +1027,500 @@ class DataFileStatisticsSuite extends AnyFunSuite with Matchers { assert(exception.getMessage === expectedMessage) } + test("serialization with geometry and geography types") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val minValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (1 2)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (10 20)", + GeographyType.ofDefault())).asJava + + val maxValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (3 4)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (30 40)", + GeographyType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(1L), + new Column("geog") -> java.lang.Long.valueOf(2L)).asJava + + val stats = new DataFileStatistics( + 100, + minValues, + maxValues, + nullCount, + Optional.of(true)) + + val json = stats.serializeAsJson(schema) + val expectedJson = + """{ + | "numRecords": 100, + | "minValues": { + | "geom": "POINT (1 2)", + | "geog": "POINT (10 20)" + | }, + | "maxValues": { + | "geom": "POINT (3 4)", + | "geog": "POINT (30 40)" + | }, + | "nullCount": { + | "geom": 1, + | "geog": 2 + | }, + | "tightBounds": true + |}""".stripMargin + + assert(areJsonNodesEqual(json, expectedJson)) + } + + test("deserialization with geometry and geography types") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val json = + """{ + | "numRecords": 50, + | "minValues": { + | "geom": "POINT (1 2)", + | "geog": "POINT (10 20)" + | }, + | "maxValues": { + | "geom": "POINT (3 4)", + | "geog": "POINT (30 40)" + | }, + | "nullCount": { + | "geom": 3, + | "geog": 0 + | }, + | "tightBounds": false + |}""".stripMargin + + val result = DataFileStatistics.deserializeFromJson(json, schema) + assert(result.isPresent) + + val stats = result.get() + assert(stats.getNumRecords == 50) + + val minValues = stats.getMinValues + assert(minValues.get(new Column("geom")).getValue == "POINT (1 2)") + assert(minValues.get(new Column("geom")).getDataType.isInstanceOf[GeometryType]) + assert(minValues.get(new Column("geog")).getValue == "POINT (10 20)") + assert(minValues.get(new Column("geog")).getDataType.isInstanceOf[GeographyType]) + + val maxValues = stats.getMaxValues + assert(maxValues.get(new Column("geom")).getValue == "POINT (3 4)") + assert(maxValues.get(new Column("geog")).getValue == "POINT (30 40)") + + assert(stats.getNullCount.get(new Column("geom")) == 3L) + assert(stats.getNullCount.get(new Column("geog")) == 0L) + assert(stats.getTightBounds.isPresent && !stats.getTightBounds.get) + } + + test("round-trip serialization for geometry and geography types") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val minValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (1.5 2.5)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (10.5 20.5)", + GeographyType.ofDefault())).asJava + + val maxValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (100 200)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (50 60)", + GeographyType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(0L), + new Column("geog") -> java.lang.Long.valueOf(5L)).asJava + + val originalStats = new DataFileStatistics( + 200, + minValues, + maxValues, + nullCount, + Optional.of(true)) + + val json = originalStats.serializeAsJson(schema) + val deserialized = DataFileStatistics.deserializeFromJson(json, schema) + assert(deserialized.isPresent) + + val stats = deserialized.get() + assert(stats.getNumRecords == 200) + assert(stats.getMinValues.get(new Column("geom")).getValue == + "POINT (1.5 2.5)") + assert(stats.getMinValues.get(new Column("geom")) + .getDataType.isInstanceOf[GeometryType]) + assert(stats.getMinValues.get(new Column("geog")).getValue == + "POINT (10.5 20.5)") + assert(stats.getMinValues.get(new Column("geog")) + .getDataType.isInstanceOf[GeographyType]) + assert(stats.getMaxValues.get(new Column("geom")).getValue == + "POINT (100 200)") + assert(stats.getMaxValues.get(new Column("geog")).getValue == + "POINT (50 60)") + assert(stats.getNullCount.get(new Column("geom")) == 0L) + assert(stats.getNullCount.get(new Column("geog")) == 5L) + } + + test("serialization with null geospatial stats literal") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val minValues = Map[Column, Literal]( + new Column("geom") -> Literal.ofNull(GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (1 2)", + GeographyType.ofDefault())).asJava + + val maxValues = Map[Column, Literal]( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (5 6)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofNull(GeographyType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(10L), + new Column("geog") -> java.lang.Long.valueOf(10L)).asJava + + val stats = new DataFileStatistics( + 10, + minValues, + maxValues, + nullCount, + Optional.empty()) + + val json = stats.serializeAsJson(schema) + val expectedJson = + """{ + | "numRecords": 10, + | "minValues": { + | "geom": null, + | "geog": "POINT (1 2)" + | }, + | "maxValues": { + | "geom": "POINT (5 6)", + | "geog": null + | }, + | "nullCount": { + | "geom": 10, + | "geog": 10 + | } + |}""".stripMargin + assert(areJsonNodesEqual(json, expectedJson)) + } + + test("round-trip with POINT Z and POINT ZM variants") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val minValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT Z (1.0 2.0 3.0)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT ZM (10.0 20.0 30.0 40.0)", + GeographyType.ofDefault())).asJava + + val maxValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT Z (4.0 5.0 6.0)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT ZM (40.0 50.0 60.0 70.0)", + GeographyType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(0L), + new Column("geog") -> java.lang.Long.valueOf(0L)).asJava + + val original = new DataFileStatistics( + 50, + minValues, + maxValues, + nullCount, + Optional.of(true)) + + val json = original.serializeAsJson(schema) + val deserialized = + DataFileStatistics.deserializeFromJson(json, schema) + assert(deserialized.isPresent) + + val stats = deserialized.get() + assert(stats.getMinValues.get(new Column("geom")).getValue == + "POINT Z (1.0 2.0 3.0)") + assert(stats.getMinValues.get(new Column("geog")).getValue == + "POINT ZM (10.0 20.0 30.0 40.0)") + assert(stats.getMaxValues.get(new Column("geom")).getValue == + "POINT Z (4.0 5.0 6.0)") + assert(stats.getMaxValues.get(new Column("geog")).getValue == + "POINT ZM (40.0 50.0 60.0 70.0)") + } + + test("round-trip preserves full double precision (17 digits) in POINT WKT") { + // 17 significant digits is the max needed to distinguish IEEE 754 doubles + val minWkt = "POINT (-122.41941550123456 37.774929501234567)" + val maxWkt = "POINT Z (179.99999999999997 89.999999999999986 12345.678901234567)" + + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geom3d", GeometryType.ofDefault()) + + val minValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + minWkt, + GeometryType.ofDefault()), + new Column("geom3d") -> Literal.ofGeospatialWKT( + maxWkt, + GeometryType.ofDefault())).asJava + + val maxValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + minWkt, + GeometryType.ofDefault()), + new Column("geom3d") -> Literal.ofGeospatialWKT( + maxWkt, + GeometryType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(0L), + new Column("geom3d") -> java.lang.Long.valueOf(0L)).asJava + + val original = new DataFileStatistics( + 1, + minValues, + maxValues, + nullCount, + Optional.of(true)) + + val json = original.serializeAsJson(schema) + val deserialized = + DataFileStatistics.deserializeFromJson(json, schema) + assert(deserialized.isPresent) + + val stats = deserialized.get() + assert(stats.getMinValues.get(new Column("geom")).getValue === minWkt) + assert(stats.getMaxValues.get(new Column("geom")).getValue === minWkt) + assert(stats.getMinValues.get(new Column("geom3d")).getValue === + maxWkt) + assert(stats.getMaxValues.get(new Column("geom3d")).getValue === + maxWkt) + } + + test("deserializing invalid geospatial stats throws KernelException") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + + val invalidJson = + """|{ + | "numRecords": 10, + | "minValues": { + | "geom": 1234 + | } + |}""".stripMargin + + val exception = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(invalidJson, schema) + } + assert(exception.getMessage.contains("Expected geometry as WKT string but got")) + } + + test("serialize throws when geospatial literal type mismatches schema") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + + val minValues = Map[Column, Literal]( + new Column("geom") -> Literal.ofString("POINT (1 2)")).asJava + + val stats = new DataFileStatistics( + 10, + minValues, + Collections.emptyMap[Column, Literal](), + Collections.emptyMap[Column, java.lang.Long](), + Optional.empty()) + + val exception = intercept[KernelException] { + stats.serializeAsJson(schema) + } + assert(exception.getMessage.contains("Type mismatch")) + assert(exception.getMessage.contains("geom")) + } + + test("deserializing non-POINT geospatial stats throws KernelException") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + + val lineStringJson = + """|{ + | "numRecords": 10, + | "minValues": { + | "geom": "LINESTRING (0 0, 1 1)" + | } + |}""".stripMargin + + val ex1 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(lineStringJson, schema) + } + assert(ex1.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + + val polygonJson = + """|{ + | "numRecords": 10, + | "minValues": { + | "geom": "POLYGON ((0 0, 1 0, 1 1, 0 0))" + | } + |}""".stripMargin + + val ex2 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(polygonJson, schema) + } + assert(ex2.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + } + + test("serializing non-POINT geospatial literal throws KernelException") { + val schema = new StructType() + .add("geog", GeographyType.ofDefault()) + + val minValues = Map[Column, Literal]( + new Column("geog") -> Literal.ofGeospatialWKT( + "LINESTRING (0 0, 1 1)", + GeographyType.ofDefault())).asJava + + val stats = new DataFileStatistics( + 10, + minValues, + Collections.emptyMap[Column, Literal](), + Collections.emptyMap[Column, java.lang.Long](), + Optional.empty()) + + val exception = intercept[KernelException] { + stats.serializeAsJson(schema) + } + assert(exception.getMessage.contains( + "Geospatial stats must be a valid POINT WKT")) + } + + test("constructing geospatial literal with null WKT throws") { + intercept[NullPointerException] { + Literal.ofGeospatialWKT(null, GeometryType.ofDefault()) + } + } + + test("deserializing malformed POINT WKT throws KernelException") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + + // POINT with wrong coordinate count + val wrongCoordCount = + """|{ + | "numRecords": 10, + | "minValues": { "geom": "POINT (1 2 3)" } + |}""".stripMargin + val ex1 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(wrongCoordCount, schema) + } + assert(ex1.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + + // POINT with non-numeric coordinates + val nonNumeric = + """|{ + | "numRecords": 10, + | "minValues": { "geom": "POINT (abc def)" } + |}""".stripMargin + val ex2 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(nonNumeric, schema) + } + assert(ex2.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + + // POINT with missing parens + val missingParens = + """|{ + | "numRecords": 10, + | "minValues": { "geom": "POINT 1 2" } + |}""".stripMargin + val ex3 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(missingParens, schema) + } + assert(ex3.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + } + + test("deserialization with missing geospatial stats in JSON") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + .add("id", IntegerType.INTEGER) + + // Stats JSON has no min/max for geospatial columns + val json = + """{ + | "numRecords": 100, + | "minValues": { "id": 1 }, + | "maxValues": { "id": 99 }, + | "nullCount": { "id": 0 } + |}""".stripMargin + + val result = DataFileStatistics.deserializeFromJson(json, schema) + assert(result.isPresent) + val stats = result.get() + assert(stats.getNumRecords == 100) + assert(!stats.getMinValues.containsKey(new Column("geom"))) + assert(!stats.getMinValues.containsKey(new Column("geog"))) + assert(stats.getMinValues.get(new Column("id")).getValue == 1) + } + + test("geography coordinate range validation on deserialization") { + val schema = new StructType() + .add("geog", GeographyType.ofDefault()) + + val outOfRange = + """|{ + | "numRecords": 10, + | "minValues": { "geog": "POINT (200 0)" } + |}""".stripMargin + val ex = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(outOfRange, schema) + } + assert(ex.getMessage.contains("out of range")) + } + + test("geography coordinate range validation on serialization") { + val schema = new StructType() + .add("geog", GeographyType.ofDefault()) + + val minValues = Map[Column, Literal]( + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (200 0)", + GeographyType.ofDefault())).asJava + + val stats = new DataFileStatistics( + 10, + minValues, + Collections.emptyMap[Column, Literal](), + Collections.emptyMap[Column, java.lang.Long](), + Optional.empty()) + + val ex = intercept[KernelException] { + stats.serializeAsJson(schema) + } + assert(ex.getMessage.contains("out of range")) + } + } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/GeometryUtilsSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/GeometryUtilsSuite.scala new file mode 100644 index 00000000000..9e19b604e9f --- /dev/null +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/GeometryUtilsSuite.scala @@ -0,0 +1,297 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal.util + +import java.util.OptionalDouble + +import io.delta.kernel.internal.util.GeometryUtils.{formatPointWKT, parsePointXY} + +import org.scalatest.funsuite.AnyFunSuite + +class GeometryUtilsSuite extends AnyFunSuite { + + private def checkXY(wkt: String, x: Double, y: Double): Unit = { + val xy = parsePointXY(wkt) + assert(xy(0) === x) + assert(xy(1) === y) + } + + private def checkInvalid(wkt: String, msgFragment: String): Unit = { + withClue(s"Expected exception for input: $wkt") { + val e = intercept[IllegalArgumentException](parsePointXY(wkt)) + assert(e.getMessage.contains(msgFragment)) + } + } + + test("POINT (x y) - basic 2D") { + checkXY("POINT (1.0 2.0)", 1.0, 2.0) + checkXY("POINT (0.0 0.0)", 0.0, 0.0) + checkXY("POINT (-1.5 3.7)", -1.5, 3.7) + } + + test("POINT Z(x y z) - 3D") { + checkXY("POINT Z(1.0 2.0 3.0)", 1.0, 2.0) + checkXY("POINT Z(-1.0 -2.0 -3.0)", -1.0, -2.0) + } + + test("POINT M(x y m) - measured") { + checkXY("POINT M(1.0 2.0 4.0)", 1.0, 2.0) + } + + test("POINT ZM(x y z m) - 4D") { + checkXY("POINT ZM(1.0 2.0 3.0 4.0)", 1.0, 2.0) + checkXY("POINT ZM(0.0 0.0 0.0 0.0)", 0.0, 0.0) + } + + test("POINT EMPTY variants - returns NaN") { + for ( + wkt <- Seq( + "POINT EMPTY", + "POINT Z EMPTY", + "POINT M EMPTY", + "POINT ZM EMPTY", + "point empty", + "point z empty", + "point m empty", + "point zm empty", + "POINT EMPTY", + "POINT Z EMPTY", + "POINT ZM EMPTY") + ) { + withClue(s"$wkt should parse as NaN: ") { + val xy = parsePointXY(wkt) + assert(xy(0).isNaN) + assert(xy(1).isNaN) + } + } + } + + test("whitespace variations") { + // no space before paren + checkXY("POINT(1.0 2.0)", 1.0, 2.0) + checkXY("POINT Z(1.0 2.0 3.0)", 1.0, 2.0) + checkXY("POINT M(1.0 2.0 4.0)", 1.0, 2.0) + checkXY("POINT ZM(1.0 2.0 3.0 4.0)", 1.0, 2.0) + // extra spaces before paren + checkXY("POINT (1.0 2.0)", 1.0, 2.0) + checkXY("POINT Z (1.0 2.0 3.0)", 1.0, 2.0) + checkXY("POINT M (1.0 2.0 4.0)", 1.0, 2.0) + checkXY("POINT ZM (1.0 2.0 3.0 4.0)", 1.0, 2.0) + // extra spaces between modifier and paren + checkXY("POINT Z (1.0 2.0 3.0)", 1.0, 2.0) + checkXY("POINT M (1.0 2.0 4.0)", 1.0, 2.0) + checkXY("POINT ZM (1.0 2.0 3.0 4.0)", 1.0, 2.0) + // extra spaces around/between coordinates + checkXY("POINT ( 1.0 2.0 )", 1.0, 2.0) + checkXY("POINT Z ( 1.0 2.0 3.0 )", 1.0, 2.0) + checkXY("POINT ZM ( 1.0 2.0 3.0 4.0 )", 1.0, 2.0) + } + + test("case insensitivity") { + checkXY("point (1.0 2.0)", 1.0, 2.0) + checkXY("POINT zm(1.0 2.0 3.0 4.0)", 1.0, 2.0) + checkXY("Point Z(1.0 2.0 3.0)", 1.0, 2.0) + } + + test("scientific notation") { + checkXY("POINT (1.5e2 2.5E2)", 150.0, 250.0) + checkXY("POINT (-1.5e2 2.5e-1)", -150.0, 0.25) + checkXY("POINT (1e3 2e3)", 1000.0, 2000.0) + checkXY("POINT (-1E3 -2E3)", -1000.0, -2000.0) + checkXY("POINT (1.23456789e10 -9.87654321e-5)", 1.23456789e10, -9.87654321e-5) + checkXY("POINT Z (1e2 2e2 3e2)", 100.0, 200.0) + } + + test("integer coordinates (no decimal point)") { + checkXY("POINT (1 2)", 1.0, 2.0) + checkXY("POINT (-100 200)", -100.0, 200.0) + checkXY("POINT (0 0)", 0.0, 0.0) + checkXY("POINT Z (1 2 3)", 1.0, 2.0) + checkXY("POINT ZM (1 2 3 4)", 1.0, 2.0) + } + + test("null input throws") { + val e = intercept[NullPointerException](parsePointXY(null)) + assert(e.getMessage.contains("cannot be null")) + } + + test("missing parens throws") { + checkInvalid("POINT 1.0 2.0", "Invalid WKT POINT") + checkInvalid("POINT", "Invalid WKT POINT") + } + + test("wrong coordinate count throws") { + checkInvalid("POINT (1.0 2.0 3.0)", "expects 2 coordinates but got 3") + checkInvalid("POINT Z(1.0 2.0)", "expects 3 coordinates but got 2") + checkInvalid("POINT M(1.0 2.0)", "expects 3 coordinates but got 2") + checkInvalid("POINT ZM(1.0 2.0 3.0)", "expects 4 coordinates but got 3") + } + + test("excess coordinates throws") { + checkInvalid("POINT Z(1.0 2.0 3.0 4.0)", "expects 3 coordinates but got 4") + checkInvalid("POINT M(1.0 2.0 3.0 4.0)", "expects 3 coordinates but got 4") + checkInvalid("POINT ZM(1.0 2.0 3.0 4.0 5.0)", "expects 4 coordinates but got 5") + } + + test("non-space separators are rejected") { + // comma/semicolon create 1 token instead of 2 + checkInvalid("POINT (1.0,2.0)", "expects 2 coordinates but got 1") + checkInvalid("POINT (1.0;2.0)", "expects 2 coordinates but got 1") + } + + test("non-numeric coordinate throws") { + checkInvalid("POINT (1.0 foo)", "Invalid coordinate") + checkInvalid("POINT ZM(1.0 2.0 3.0 bar)", "Invalid coordinate") + } + + test("NaN and Infinity coordinates throw") { + checkInvalid("POINT (NaN 1.0)", "finite numbers") + checkInvalid("POINT (1.0 NaN)", "finite numbers") + checkInvalid("POINT (Infinity 1.0)", "finite numbers") + checkInvalid("POINT (1.0 -Infinity)", "finite numbers") + checkInvalid("POINT Z(1.0 2.0 NaN)", "finite numbers") + } + + test("formatPointWKT - 2D") { + val wkt = formatPointWKT( + 1.0, + 2.0, + OptionalDouble.empty(), + OptionalDouble.empty()) + assert(wkt === "POINT(1.0 2.0)") + val xy = parsePointXY(wkt) + assert(xy(0) === 1.0) + assert(xy(1) === 2.0) + } + + test("formatPointWKT - Z round-trip") { + val wkt = formatPointWKT( + 1.0, + 2.0, + OptionalDouble.of(3.0), + OptionalDouble.empty()) + assert(wkt === "POINT Z (1.0 2.0 3.0)") + val xy = parsePointXY(wkt) + assert(xy(0) === 1.0) + assert(xy(1) === 2.0) + } + + test("formatPointWKT - M round-trip") { + val wkt = formatPointWKT( + 1.0, + 2.0, + OptionalDouble.empty(), + OptionalDouble.of(4.0)) + assert(wkt === "POINT M (1.0 2.0 4.0)") + val xy = parsePointXY(wkt) + assert(xy(0) === 1.0) + assert(xy(1) === 2.0) + } + + test("formatPointWKT - ZM round-trip") { + val wkt = formatPointWKT( + 1.0, + 2.0, + OptionalDouble.of(3.0), + OptionalDouble.of(4.0)) + assert(wkt === "POINT ZM (1.0 2.0 3.0 4.0)") + val xy = parsePointXY(wkt) + assert(xy(0) === 1.0) + assert(xy(1) === 2.0) + } + + test("formatPointWKT - EMPTY round-trips") { + val empty2d = formatPointWKT( + Double.NaN, + Double.NaN, + OptionalDouble.empty(), + OptionalDouble.empty()) + assert(empty2d === "POINT EMPTY") + val xy = parsePointXY(empty2d) + assert(xy(0).isNaN) + assert(xy(1).isNaN) + + val emptyZ = formatPointWKT( + Double.NaN, + Double.NaN, + OptionalDouble.of(Double.NaN), + OptionalDouble.empty()) + assert(emptyZ === "POINT Z EMPTY") + assert(parsePointXY(emptyZ)(0).isNaN) + + val emptyM = formatPointWKT( + Double.NaN, + Double.NaN, + OptionalDouble.empty(), + OptionalDouble.of(Double.NaN)) + assert(emptyM === "POINT M EMPTY") + assert(parsePointXY(emptyM)(0).isNaN) + + val emptyZM = formatPointWKT( + Double.NaN, + Double.NaN, + OptionalDouble.of(Double.NaN), + OptionalDouble.of(Double.NaN)) + assert(emptyZM === "POINT ZM EMPTY") + assert(parsePointXY(emptyZM)(0).isNaN) + } + + test("formatPointWKT - round-trip negative coords") { + val wkt = formatPointWKT( + -180.0, + -90.0, + OptionalDouble.empty(), + OptionalDouble.empty()) + val xy = parsePointXY(wkt) + assert(xy(0) === -180.0) + assert(xy(1) === -90.0) + } + + // Geography coordinate range validation tests + + test("validateGeographyPointWKT - valid coordinates") { + GeometryUtils.validateGeographyPointWKT("POINT (0 0)") + GeometryUtils.validateGeographyPointWKT("POINT (-180 -90)") + GeometryUtils.validateGeographyPointWKT("POINT (180 90)") + GeometryUtils.validateGeographyPointWKT("POINT Z (10 20 100)") + GeometryUtils.validateGeographyPointWKT("POINT EMPTY") + } + + test("validateGeographyPointWKT - longitude out of range") { + val e = intercept[io.delta.kernel.exceptions.KernelException] { + GeometryUtils.validateGeographyPointWKT("POINT (181 0)") + } + assert(e.getMessage.contains("out of range")) + + val e2 = intercept[io.delta.kernel.exceptions.KernelException] { + GeometryUtils.validateGeographyPointWKT("POINT (-181 0)") + } + assert(e2.getMessage.contains("out of range")) + } + + test("validateGeographyPointWKT - latitude out of range") { + val e = intercept[io.delta.kernel.exceptions.KernelException] { + GeometryUtils.validateGeographyPointWKT("POINT (0 91)") + } + assert(e.getMessage.contains("out of range")) + + val e2 = intercept[io.delta.kernel.exceptions.KernelException] { + GeometryUtils.validateGeographyPointWKT("POINT (0 -91)") + } + assert(e2.getMessage.contains("out of range")) + } + +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java index fd807992fa9..fd9324da195 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java @@ -350,6 +350,11 @@ public ColumnVector getValues() { }; } + if (dataType instanceof GeometryType || dataType instanceof GeographyType) { + throwIfTypeMismatch("string", jsonValue.isTextual(), jsonValue); + return jsonValue.asText(); + } + throw new UnsupportedOperationException( String.format("Unsupported DataType %s for RootNode %s", dataType, jsonValue)); } diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala index a25a0187955..9b793f75d87 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala @@ -452,6 +452,63 @@ class DefaultJsonHandlerSuite extends AnyFunSuite with TestUtils with DefaultVec } } + test("parse geometry type as WKT string") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + testJsonParserWithSchema( + """{"geom": "POINT (1.0 2.0)"}""", + schema, + TestRow("POINT (1.0 2.0)")) + } + + test("parse geography type as WKT string") { + val schema = new StructType() + .add("geog", GeographyType.ofDefault()) + testJsonParserWithSchema( + """{"geog": "POINT (10.5 20.5)"}""", + schema, + TestRow("POINT (10.5 20.5)")) + } + + test("parse geometry/geography with null values") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault(), true) + .add("geog", GeographyType.ofDefault(), true) + testJsonParserWithSchema( + """{"geom": null, "geog": null}""", + schema, + TestRow(null, null)) + } + + test("parse geometry POINT variants (Z, M, ZM)") { + val schema = new StructType() + .add("col1", GeometryType.ofDefault()) + testJsonParserWithSchema( + """{"col1": "POINT Z(1.0 2.0 3.0)"}""", + schema, + TestRow("POINT Z(1.0 2.0 3.0)")) + testJsonParserWithSchema( + """{"col1": "POINT M(1.0 2.0 4.0)"}""", + schema, + TestRow("POINT M(1.0 2.0 4.0)")) + testJsonParserWithSchema( + """{"col1": "POINT ZM(1.0 2.0 3.0 4.0)"}""", + schema, + TestRow("POINT ZM(1.0 2.0 3.0 4.0)")) + } + + test("parse geometry with non-string value throws") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + val e = intercept[RuntimeException] { + testJsonParserWithSchema( + """{"geom": 1234}""", + schema, + TestRow()) + } + assert(e.getMessage.contains("string")) + } + test("parse diverse type values in a map[string, string]") { val input = """ diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala index 79facaead07..adfeed87a1a 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala @@ -110,6 +110,8 @@ object TestRow { case _: ArrayType => arrayValueToScalaSeq(row.getArray(i)) case _: MapType => mapValueToScalaMap(row.getMap(i)) case _: StructType => TestRow(row.getStruct(i)) + case _: GeometryType => row.getString(i) + case _: GeographyType => row.getString(i) case _ => throw new UnsupportedOperationException("unrecognized data type") } }.toSeq) From 71bcdbd2092f18ce69cc8c8f83143287ec20ef23 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Mon, 30 Mar 2026 16:35:01 -0700 Subject: [PATCH 70/80] [Spark] Add AlwaysTrue/AlwaysFalse filter pushdown to V2 connector (#6355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add support for converting Spark's `AlwaysTrue` and `AlwaysFalse` filters to Kernel `ALWAYS_TRUE` and `ALWAYS_FALSE` predicates in the V2 connector's `ExpressionUtils`. - `AlwaysFalse` pushdown allows the Kernel to skip all files entirely, while `AlwaysTrue` is a correct no-op passthrough. - These were the only two Spark filter types with direct Kernel equivalents that were missing from the V2 connector's filter conversion. ## Test plan - [x] Added `testAlwaysTrueFilter` and `testAlwaysFalseFilter` unit tests in `ExpressionUtilsTest` - [x] All 74 tests pass in `ExpressionUtilsTest` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 --- .../internal/v2/utils/ExpressionUtils.java | 8 +++++++ .../v2/utils/ExpressionUtilsTest.java | 22 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java index b5a6dea697c..38674f38e1b 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import io.delta.kernel.expressions.AlwaysFalse; +import io.delta.kernel.expressions.AlwaysTrue; import io.delta.kernel.expressions.And; import io.delta.kernel.expressions.Column; import io.delta.kernel.expressions.In; @@ -250,6 +251,13 @@ private static ConvertedPredicate convertSparkFilterToKernelPredicate( child.getConvertedPredicate().map(c -> new Predicate("NOT", c)), child.isPartial()); } + if (filter instanceof org.apache.spark.sql.sources.AlwaysTrue) { + return new ConvertedPredicate(Optional.of(AlwaysTrue.ALWAYS_TRUE)); + } + if (filter instanceof org.apache.spark.sql.sources.AlwaysFalse) { + return new ConvertedPredicate(Optional.of(AlwaysFalse.ALWAYS_FALSE)); + } + return new ConvertedPredicate(Optional.empty()); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java index 07f85c7a33d..b9f6d46d6b3 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java @@ -271,6 +271,28 @@ static Stream valueTypesProvider() { TimestampType.TIMESTAMP)); } + @Test + public void testAlwaysTrueFilter() { + Filter filter = new org.apache.spark.sql.sources.AlwaysTrue(); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "AlwaysTrue should be converted"); + assertFalse(result.isPartial()); + assertEquals("ALWAYS_TRUE", result.get().getName()); + } + + @Test + public void testAlwaysFalseFilter() { + Filter filter = new org.apache.spark.sql.sources.AlwaysFalse(); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "AlwaysFalse should be converted"); + assertFalse(result.isPartial()); + assertEquals("ALWAYS_FALSE", result.get().getName()); + } + @Test public void testUnsupportedFilter() { // Create an unsupported filter (StringContains is not implemented in our conversion method) From 0f1a087ac8e0c4c5c4504485aad74980bec302a7 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Tue, 31 Mar 2026 05:38:40 +0530 Subject: [PATCH 71/80] [UC Commit Metrics] Add skeleton transport wiring and smoke tests (#6155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🥞 Stacked PR Use this [link](https://github.com/delta-io/delta/pull/6155/files) to review all changes. **Stack:** - **[[UC Commit Metrics] Add skeleton transport wiring and smoke tests](https://github.com/delta-io/delta/pull/6155)** [[Files changed](https://github.com/delta-io/delta/pull/6155/files)] ⬅️ _This PR_ - [[UC Commit Metrics] Add full payload construction and schema tests](https://github.com/delta-io/delta/pull/6156) [[Files changed](https://github.com/delta-io/delta/pull/6156/files/4b5eadb28ab152d9ac180423bfd51720b6219b5b..1f39302986a7b6138cd358f9515386fc283810a8)] - [[UC Commit Metrics] Add feature flag and async dispatch](https://github.com/delta-io/delta/pull/6333) [[Files changed](https://github.com/delta-io/delta/pull/6333/files/1f39302986a7b6138cd358f9515386fc283810a8..7b6d633980419060404203551bb1577ccfe922aa)] --- ## Summary - Self-contained post-commit hook in `hooks/metrics/UpdateMetricsHook.scala` (case class + companion object with payload builder, HTTP client, and constants). - Registered only for UC-managed tables (`CatalogTableUtils.isUnityCatalogManagedTable` check at registration time). - Derives endpoint from `spark.sql.catalog..uri`, authenticates via `UCCommitCoordinatorBuilder` + `TokenProvider`. - Best-effort: failures logged as warnings, never fail the commit. ## Test plan - [x] `build/sbt "spark/testOnly *metrics.UpdateMetricsHookSuite"` -- 6 tests pass --------- Signed-off-by: Murali Ramanujam Signed-off-by: foo.bar Co-authored-by: foo.bar --- .../sql/delta/OptimisticTransaction.scala | 5 + .../hooks/metrics/UpdateMetricsHook.scala | 209 ++++++++ .../metrics/UpdateMetricsHookSuite.scala | 459 ++++++++++++++++++ 3 files changed, 673 insertions(+) create mode 100644 spark/src/main/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHook.scala create mode 100644 spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index 272e56a80de..9cfcbecc39d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -38,6 +38,8 @@ import org.apache.spark.sql.delta.constraints.{Constraints, Invariants} import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils, TableCommitCoordinatorClient, UCCommitCoordinatorBuilder} import org.apache.spark.sql.delta.files._ import org.apache.spark.sql.delta.hooks.{CheckpointHook, ChecksumHook, GenerateSymlinkManifest, HudiConverterHook, IcebergConverterHook, PostCommitHook, UpdateCatalogFactory} +import org.apache.spark.sql.delta.hooks.metrics.UpdateMetricsHook +import org.apache.spark.sql.delta.util.CatalogTableUtils import org.apache.spark.sql.delta.implicits.addFileEncoder import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging @@ -449,6 +451,9 @@ trait OptimisticTransactionImpl extends TransactionHelper registerPostCommitHook(ChecksumHook) catalogTable.foreach { ct => registerPostCommitHook(UpdateCatalogFactory.getUpdateCatalogHook(ct, spark)) + if (CatalogTableUtils.isUnityCatalogManagedTable(ct)) { + registerPostCommitHook(UpdateMetricsHook(Some(ct))) + } } // The CheckpointHook will only checkpoint if necessary, so always register it to run. registerPostCommitHook(CheckpointHook) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHook.scala b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHook.scala new file mode 100644 index 00000000000..07a259ee8f1 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHook.scala @@ -0,0 +1,209 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.hooks.metrics + +// scalastyle:off import.ordering.noEmptyLine +import scala.collection.JavaConverters._ + +import com.fasterxml.jackson.annotation.JsonProperty +import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient + +import org.apache.spark.sql.delta.CommittedTransaction +import org.apache.spark.sql.delta.actions.Action +import org.apache.spark.sql.delta.coordinatedcommits.UCCommitCoordinatorBuilder +import org.apache.spark.sql.delta.hooks.PostCommitHook +import org.apache.spark.sql.delta.logging.DeltaLogKeys +import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.util.{CatalogTableUtils, JsonUtils} + +import io.unitycatalog.client.auth.TokenProvider +import org.apache.http.HttpHeaders +import org.apache.http.client.config.RequestConfig +import org.apache.http.client.methods.HttpPost +import org.apache.http.entity.{ContentType, StringEntity} +import org.apache.http.impl.client.{CloseableHttpClient, HttpClientBuilder} +import org.apache.http.util.EntityUtils +import org.apache.spark.internal.MDC +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +// Payload case classes (JSON-serialized via Jackson @JsonProperty) + +case class ReportDeltaMetricsRequest( + @JsonProperty("table_id") tableId: String, + @JsonProperty("report") report: CommitReportEnvelope) + +case class CommitReportEnvelope( + @JsonProperty("commit_report") commitReport: CommitReport) + +case class CommitReport() + +/** + * Post-commit hook that sends commit metrics to Unity Catalog + * for UC-managed Delta tables. + * + * Best-effort: failures are logged as warnings and never fail + * the commit. + * + * @param catalogTable catalog metadata for UC-managed detection + */ +case class UpdateMetricsHook(catalogTable: Option[CatalogTable]) + extends PostCommitHook with DeltaLogging { + + override val name: String = "Update catalog metrics" + + override def run( + spark: SparkSession, + txn: CommittedTransaction): Unit = { + val ct = catalogTable.orNull + if (ct == null || + !CatalogTableUtils.isUnityCatalogManagedTable(ct)) { + return + } + val tableId = ct.storage.properties( + UCCommitCoordinatorClient.UC_TABLE_ID_KEY) + + try { + val request = UpdateMetricsHook.buildRequest( + tableId, txn.committedActions, txn.committedVersion) + val catalogName = ct.identifier.catalog + UpdateMetricsHook.sendMetrics( + spark, request, catalogName = catalogName) + + logDebug( + log"Successfully sent UC metrics for table " + + log"${MDC(DeltaLogKeys.PATH, txn.deltaLog.logPath)} " + + log"version " + + log"${MDC(DeltaLogKeys.VERSION, txn.committedVersion)}") + + } catch { + case e: Exception => + logWarning( + log"Failed to send UC metrics for table " + + log"${MDC(DeltaLogKeys.PATH, txn.deltaLog.logPath)} " + + log"version " + + log"${MDC(DeltaLogKeys.VERSION, txn.committedVersion)}" + + log": ${MDC(DeltaLogKeys.ERROR, e.getMessage)}", e) + } + } +} + +object UpdateMetricsHook { + + // -- Payload builder -- + + private[metrics] def buildRequest( + tableId: String, + _committedActions: Seq[Action], + _committedVersion: Long): ReportDeltaMetricsRequest = { + ReportDeltaMetricsRequest( + tableId = tableId, + report = CommitReportEnvelope(CommitReport()) + ) + } + + // -- HTTP client -- + + private val CATALOG_CONF_PREFIX = "spark.sql.catalog" + private val CATALOG_URI_CONF_SUFFIX = "uri" + private val METRICS_ENDPOINT_PATH = + "/api/2.1/unity-catalog/delta/preview/metrics" + private val HTTP_TIMEOUT_MS = 5000L + + private[metrics] def sendMetrics( + spark: SparkSession, + request: ReportDeltaMetricsRequest, + catalogName: Option[String] = None): Unit = { + val catalog = catalogName.getOrElse( + throw new IllegalArgumentException( + "Catalog name required for UC metrics; " + + "endpoint URI is read from " + + "spark.sql.catalog..uri")) + val endpointUrl = getEndpointUrl(spark, catalog) + val authToken = getAuthToken(spark, catalog) + + val requestConfig = RequestConfig.custom() + .setConnectTimeout(HTTP_TIMEOUT_MS.toInt) + .setSocketTimeout(HTTP_TIMEOUT_MS.toInt) + .setConnectionRequestTimeout(HTTP_TIMEOUT_MS.toInt) + .build() + + val httpClient: CloseableHttpClient = + HttpClientBuilder.create() + .setDefaultRequestConfig(requestConfig) + .build() + + try { + val httpPost = new HttpPost(endpointUrl) + httpPost.setHeader( + HttpHeaders.AUTHORIZATION, s"Bearer $authToken") + + val jsonPayload = JsonUtils.toJson(request) + httpPost.setEntity( + new StringEntity( + jsonPayload, ContentType.APPLICATION_JSON)) + + val response = httpClient.execute(httpPost) + try { + val statusCode = response.getStatusLine.getStatusCode + if (statusCode < 200 || statusCode >= 300) { + val responseBody = Option(response.getEntity) + .map(EntityUtils.toString) + .getOrElse("") + throw new RuntimeException( + "UC metrics endpoint returned error " + + s"status $statusCode: $responseBody") + } + } finally { + response.close() + } + } finally { + httpClient.close() + } + } + + private def getEndpointUrl( + spark: SparkSession, + catalogName: String): String = { + val uriKey = s"$CATALOG_CONF_PREFIX.$catalogName" + + s".$CATALOG_URI_CONF_SUFFIX" + spark.conf.getOption(uriKey) match { + case Some(uri) if uri.nonEmpty => + s"${uri.stripSuffix("/")}$METRICS_ENDPOINT_PATH" + case _ => + throw new IllegalArgumentException( + s"UC catalog base URI not configured. " + + s"Set $uriKey") + } + } + + private def getAuthToken( + spark: SparkSession, + catalogName: String): String = { + val configMap = + UCCommitCoordinatorBuilder.getCatalogConfigMap(spark) + val config = configMap.get(catalogName).getOrElse( + throw new IllegalArgumentException( + "Unity Catalog configuration not found for " + + s"catalog '$catalogName'. Configure " + + "spark.sql.catalog..uri and auth " + + "(auth.type/auth.token or legacy .token).")) + val tokenProvider = + TokenProvider.create(config.authConfig.asJava) + tokenProvider.accessToken() + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala new file mode 100644 index 00000000000..229805603ae --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala @@ -0,0 +1,459 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.hooks.metrics + +import java.io.{BufferedReader, InputStreamReader, PrintWriter} +import java.net.{ServerSocket, Socket} +import java.util.concurrent.atomic.AtomicInteger + +import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient + +import org.apache.spark.sql.delta.{CommittedTransaction, DeltaLog} +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.delta.util.{CatalogTableUtils, JsonUtils} + +import org.apache.spark.SparkConf +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.StructType + +class UpdateMetricsHookSuite extends QueryTest + with SharedSparkSession + with DeltaSQLCommandTest { + private val TEST_CATALOG_NAME = "test_catalog" + private val TEST_UC_TABLE_ID = "uc-table-id-abc" + + private def ucStorageFormat(ucTableId: String = TEST_UC_TABLE_ID): CatalogStorageFormat = { + CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map( + UCCommitCoordinatorClient.UC_TABLE_ID_KEY -> ucTableId, + "delta.feature.catalogManaged" -> "supported" + ) + ) + } + + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.databricks.delta.properties.defaults.enableChangeDataFeed", "false") + } + + test("CatalogTableUtils.isUnityCatalogManagedTable: detection cases") { + val ucManaged = CatalogTable( + identifier = TableIdentifier("t", Some("default"), Some("uc_catalog")), + tableType = CatalogTableType.MANAGED, + storage = ucStorageFormat(), + schema = new StructType() + ) + assert(CatalogTableUtils.isUnityCatalogManagedTable(ucManaged), + "UC-managed table with correct storage properties should be detected") + + val noUCTableId = CatalogTable( + identifier = TableIdentifier("t", Some("default"), Some("uc_catalog")), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map("delta.feature.catalogManaged" -> "supported") + ), + schema = new StructType() + ) + assert(!CatalogTableUtils.isUnityCatalogManagedTable(noUCTableId), + "table without UC table ID should not be detected") + + val noCatalogFeature = CatalogTable( + identifier = TableIdentifier("t", Some("default"), Some("uc_catalog")), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map( + UCCommitCoordinatorClient.UC_TABLE_ID_KEY -> "some-id" + ) + ), + schema = new StructType() + ) + assert(!CatalogTableUtils.isUnityCatalogManagedTable(noCatalogFeature), + "table with UC ID but no catalog feature flag should not be detected") + + val emptyStorage = CatalogTable( + identifier = TableIdentifier("t", Some("default"), Some("uc_catalog")), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat.empty, + schema = new StructType() + ) + assert(!CatalogTableUtils.isUnityCatalogManagedTable(emptyStorage), + "table with empty storage properties should not be detected") + } + + test("sendMetrics: sends payload with Authorization header") { + val mockServer = new SimpleMockServer(0) + try { + mockServer.setResponseCode(200) + mockServer.start() + + spark.conf.set( + s"spark.sql.catalog.$TEST_CATALOG_NAME", + "io.unitycatalog.spark.UCSingleCatalog") + spark.conf.set( + s"spark.sql.catalog.$TEST_CATALOG_NAME.uri", + s"http://localhost:${mockServer.getPort()}") + spark.conf.set( + s"spark.sql.catalog.$TEST_CATALOG_NAME.token", + "test-token-123") + + val request = ReportDeltaMetricsRequest( + tableId = "abc-123", + report = CommitReportEnvelope(CommitReport()) + ) + UpdateMetricsHook.sendMetrics( + spark, request, catalogName = Some(TEST_CATALOG_NAME)) + + assert(mockServer.getRequestCount() == 1, "Expected 1 HTTP request") + + val authHeader = mockServer.getLastHeaders().get("Authorization") + assert(authHeader.isDefined, "Authorization header should be present") + assert(authHeader.get == "Bearer test-token-123", + "Auth token must match") + + val body = mockServer.getLastRequestBody() + val actualPayload = + JsonUtils.fromJson[Map[String, Any]](body) + val expectedPayload = + JsonUtils.fromJson[Map[String, Any]](JsonUtils.toJson(request)) + assert(actualPayload == expectedPayload, + "request JSON should match expected payload") + } finally { + mockServer.stop() + } + } + + test("sendMetrics: auth.type=static with auth.token") { + val authStaticCatalog = "auth_static_catalog" + val mockServer = new SimpleMockServer(0) + try { + mockServer.setResponseCode(200) + mockServer.start() + + spark.conf.set( + s"spark.sql.catalog.$authStaticCatalog", + "io.unitycatalog.spark.UCSingleCatalog") + spark.conf.set( + s"spark.sql.catalog.$authStaticCatalog.uri", + s"http://localhost:${mockServer.getPort()}") + spark.conf.set( + s"spark.sql.catalog.$authStaticCatalog.auth.type", + "static") + spark.conf.set( + s"spark.sql.catalog.$authStaticCatalog.auth.token", + "auth-static-token-456") + + val request = ReportDeltaMetricsRequest( + tableId = "auth-test-id", + report = CommitReportEnvelope(CommitReport()) + ) + UpdateMetricsHook.sendMetrics( + spark, request, catalogName = Some(authStaticCatalog)) + + assert(mockServer.getRequestCount() == 1, "Expected 1 HTTP request") + val authHeader = mockServer.getLastHeaders().get("Authorization") + assert(authHeader.isDefined, + "Authorization header should be present") + assert(authHeader.get == "Bearer auth-static-token-456", + "Auth token from auth.token must match") + } finally { + mockServer.stop() + } + } + + test("run(): fires HTTP POST for UC-managed table") { + val mockServer = new SimpleMockServer(0) + try { + mockServer.setResponseCode(200) + mockServer.start() + + withTempDir { dir => + spark.range(10).write.format("delta").save(dir.getCanonicalPath) + val deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) + val snapshot = deltaLog.snapshot + + spark.conf.set( + "spark.sql.catalog.spark_catalog", + "io.unitycatalog.spark.UCSingleCatalog") + spark.conf.set( + "spark.sql.catalog.spark_catalog.uri", + s"http://localhost:${mockServer.getPort()}") + spark.conf.set( + "spark.sql.catalog.spark_catalog.token", "smoke-token") + + val catalogTable = CatalogTable( + identifier = TableIdentifier( + "t", Some("default"), Some("spark_catalog")), + tableType = CatalogTableType.MANAGED, + storage = ucStorageFormat(), + schema = new StructType() + ) + + val txn = CommittedTransaction( + txnId = "smoke-txn", + deltaLog = deltaLog, + catalogTable = Some(catalogTable), + readSnapshot = snapshot, + committedVersion = 0L, + committedActions = Seq.empty, + postCommitSnapshot = snapshot, + postCommitHooks = Seq.empty, + txnExecutionTimeMs = 0L, + needsCheckpoint = false, + partitionsAddedToOpt = None, + isBlindAppend = true + ) + + UpdateMetricsHook(Some(catalogTable)).run(spark, txn) + + assert(mockServer.getRequestCount() == 1, + "Expected exactly 1 HTTP POST") + val body = mockServer.getLastRequestBody() + val expectedRequest = UpdateMetricsHook.buildRequest( + TEST_UC_TABLE_ID, Seq.empty, 0L) + val actualPayload = + JsonUtils.fromJson[Map[String, Any]](body) + val expectedPayload = + JsonUtils.fromJson[Map[String, Any]]( + JsonUtils.toJson(expectedRequest)) + assert(actualPayload == expectedPayload, + "smoke test payload should match expected JSON") + } + } finally { + mockServer.stop() + } + } + + test("run(): hook does not crash on HTTP 5xx (best-effort)") { + val mockServer = new SimpleMockServer(0) + try { + mockServer.setResponseCode(500) + mockServer.start() + + withTempDir { dir => + spark.range(10).write.format("delta") + .save(dir.getCanonicalPath) + val deltaLog = DeltaLog.forTable( + spark, dir.getCanonicalPath) + val snapshot = deltaLog.snapshot + + spark.conf.set( + "spark.sql.catalog.spark_catalog", + "io.unitycatalog.spark.UCSingleCatalog") + spark.conf.set( + "spark.sql.catalog.spark_catalog.uri", + s"http://localhost:${mockServer.getPort()}") + spark.conf.set( + "spark.sql.catalog.spark_catalog.token", + "error-token") + + val catalogTable = CatalogTable( + identifier = TableIdentifier( + "t", Some("default"), Some("spark_catalog")), + tableType = CatalogTableType.MANAGED, + storage = ucStorageFormat(), + schema = new StructType() + ) + + val txn = CommittedTransaction( + txnId = "error-txn", + deltaLog = deltaLog, + catalogTable = Some(catalogTable), + readSnapshot = snapshot, + committedVersion = 0L, + committedActions = Seq.empty, + postCommitSnapshot = snapshot, + postCommitHooks = Seq.empty, + txnExecutionTimeMs = 0L, + needsCheckpoint = false, + partitionsAddedToOpt = None, + isBlindAppend = true + ) + + UpdateMetricsHook(Some(catalogTable)).run(spark, txn) + + assert(mockServer.getRequestCount() == 1, + "request should still be sent on 5xx") + } + } finally { + mockServer.stop() + } + } + + test("run(): skips metrics when table ID not in storage properties") { + withTempDir { dir => + spark.range(1).write.format("delta").save(dir.getCanonicalPath) + val deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) + val snapshot = deltaLog.snapshot + + val catalogTable = CatalogTable( + identifier = TableIdentifier( + "t", Some("default"), Some("spark_catalog")), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map( + "delta.feature.catalogManaged" -> "supported" + ) + ), + schema = new StructType() + ) + + val txn = CommittedTransaction( + txnId = "skip-txn", + deltaLog = deltaLog, + catalogTable = Some(catalogTable), + readSnapshot = snapshot, + committedVersion = 0L, + committedActions = Seq.empty, + postCommitSnapshot = snapshot, + postCommitHooks = Seq.empty, + txnExecutionTimeMs = 0L, + needsCheckpoint = false, + partitionsAddedToOpt = None, + isBlindAppend = true + ) + + UpdateMetricsHook(Some(catalogTable)).run(spark, txn) + } + } +} + +class SimpleMockServer(port: Int) { + private var serverSocket: ServerSocket = _ + private var serverThread: Thread = _ + @volatile private var running = false + private var responseCode = 200 + private val requestCount = new AtomicInteger(0) + @volatile private var lastRequestBody = "" + @volatile private var lastHeaders = Map[String, String]() + private var actualPort = port + + def setResponseCode(code: Int): Unit = { responseCode = code } + + def getPort(): Int = actualPort + + def getRequestCount(): Int = requestCount.get() + + def getLastRequestBody(): String = lastRequestBody + + def getLastHeaders(): Map[String, String] = lastHeaders + + def start(): Unit = { + serverSocket = new ServerSocket(port) + actualPort = serverSocket.getLocalPort + running = true + + serverThread = new Thread(new Runnable { + override def run(): Unit = { + while (running) { + try { + val clientSocket = serverSocket.accept() + handleRequest(clientSocket) + } catch { + case _: java.net.SocketException if !running => + case _: Exception => + } + } + } + }) + serverThread.setDaemon(true) + serverThread.start() + } + + private def handleRequest(clientSocket: Socket): Unit = { + try { + val in = new BufferedReader( + new InputStreamReader(clientSocket.getInputStream)) + val out = new PrintWriter(clientSocket.getOutputStream, true) + + in.readLine() + + val headers = scala.collection.mutable.Map[String, String]() + var contentLength = 0 + var line = in.readLine() + while (line != null && line.nonEmpty) { + val parts = line.split(":", 2) + if (parts.length == 2) { + val key = parts(0).trim + val value = parts(1).trim + headers(key) = value + if (key.equalsIgnoreCase("Content-Length")) { + contentLength = value.toInt + } + } + line = in.readLine() + } + + val body = new Array[Char](contentLength) + if (contentLength > 0) in.read(body, 0, contentLength) + + requestCount.incrementAndGet() + lastRequestBody = new String(body) + lastHeaders = headers.toMap + + out.write( + s"HTTP/1.1 $responseCode ${statusMessage(responseCode)}\r\n") + out.write("Content-Type: application/json\r\n") + out.write("Content-Length: 2\r\n") + out.write("\r\n") + out.write("{}") + out.flush() + + clientSocket.close() + } catch { + case _: java.io.IOException => + } + } + + private def statusMessage(code: Int): String = code match { + case 200 => "OK" + case 400 => "Bad Request" + case 500 => "Internal Server Error" + case _ => "Unknown" + } + + def stop(): Unit = { + running = false + if (serverSocket != null && !serverSocket.isClosed) serverSocket.close() + if (serverThread != null) { + serverThread.interrupt() + serverThread.join(1000) + } + } +} From c6bb6f04ff257f3f6e184784381391ab05c41aa0 Mon Sep 17 00:00:00 2001 From: Timothy Wang Date: Mon, 30 Mar 2026 20:09:30 -0700 Subject: [PATCH 72/80] Add DDLContext POJO and prerequisite infra for DSv2 CREATE TABLE (#6377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🥞 Stacked PR Use this [link](https://github.com/delta-io/delta/pull/6377/files) to review incremental changes. - [**stack/ddl-1-foundations**](https://github.com/delta-io/delta/pull/6377) [[Files changed](https://github.com/delta-io/delta/pull/6377/files)] - [stack/ddl-2-builder](https://github.com/delta-io/delta/pull/6378) [[Files changed](https://github.com/delta-io/delta/pull/6378/files/b8c478ac19da5afb5aea0d8ce25dfa744b689e4f..a652586290caadbe79201d594ecf0e95f129b328)] - [stack/ddl-2b-builder](https://github.com/delta-io/delta/pull/6442) [[Files changed](https://github.com/delta-io/delta/pull/6442/files/a652586290caadbe79201d594ecf0e95f129b328..88c17741e4c8f17a56c4a0523c1b744c36789c95)] - [stack/ddl-3-wiring](https://github.com/delta-io/delta/pull/6379) [[Files changed](https://github.com/delta-io/delta/pull/6379/files/88c17741e4c8f17a56c4a0523c1b744c36789c95..113ebc006446bd7645be6a75c900104b0b0ed8a4)] --------- #### Which Delta project/connector is this regarding? ## Description In this stack of PRs we are implementing the foundation for DDL operations (metadata-only CREATE, CTAS, RTAS). For context, CREATE TABLE in this DSv2 world is 3 main steps: 1. pre-register the table to the catalog (UCProxy.createTable or HMS.createTable) 2. build a transaction for table creation (Delta commit 0) 3. commit (actually commit the transaction above) **We also want to make implementing future DDL operations easier in the future**. That's why in this PR we create a unified DDL request POJO. This POJO/request will be fed into the transaction builder. **Significance**: CreateTableBuilder.prepare() does all the Spark → Kernel conversion - DDLContext is a normalized request for downstream Kernel operations. > By the time DDLContext is constructed, everything is Kernel-native. buildTransaction() just passes it straight through to TableManager or UCCatalogManagedClient with no further conversion. ``` SQL: CREATE TABLE t (id INT, name STRING) │ ▼ ┌──────────────────────────────────────────┐ │ UCSingleCatalog.createTable( │ │ ident, schema, partitions, props) │ │ │ │ Spark's UC catalog plugin. Delegates │ │ to DeltaCatalog for Delta tables. │ └─────────────────────┬────────────────────┘ │ ▼ ┌──────────────────────────────────────────────────────────────────────────────────────────┐ │ DeltaCatalog.createTable(ident, schema, partitions, properties) │ │ │ │ 1. DeltaV2Mode routing check │ │ STRICT → always Kernel path │ │ AUTO → Kernel only if UC + catalogManaged feature flag │ │ NONE → V1 path (super.createTable) │ │ │ │ 2. If UC: preRegisterIfNeeded(ident, schema, parts, props) │ │ └→ Optional │ │ then injectStorageCredentials(ct, hadoopConf) │ │ then UCUtils.extractTableInfo(ct, spark) → Optional │ │ │ │ 3. CreateTableBuilder.prepare(ident, schema, parts, props, hadoopConf, ucTableInfo) │ │ └→ DDLContext │ │ │ │ 4. CreateTableBuilder.buildTransaction(ddlContext) → Transaction │ │ │ │ 5. txn.commit(engine, emptyIterable()) — metadata-only │ │ │ │ 6. return new SparkTable(ident, tablePath) — V2 connector │ └──────┬─────────────────────┬──────────────────────────┬──────────────────────┬───────────┘ │ │ │ │ ▼ ▼ ▼ ▼ STEP 2 STEP 3 STEP 4 STEP 5 Pre-register Prepare Build Txn Commit UC-only. Converts Spark Routes to Kernel txn.commit( Calls delegate types to Kernel: API based on engine, .createTable() ucTableInfo: emptyIterable() via UCProxy. sparkSchema → ) kernelSchema ┌── UC present? ───┐ UC assigns: (SchemaUtils) │ │ Kernel handles • tableId │ YES │ conflict • managed Transform[] → │ UCCatalog │ resolution, location DataLayoutSpec │ ManagedClient │ protocol, • ucUri (identity or │ .buildCreate │ and log write. • authConfig cluster-by) │ TableTxn( │ │ tableId, │ For UC: commit Returns properties → │ path, │ goes through Optional filtered props │ schema, │ UC committer. (strip DSv2 keys: │ engineInfo) │ location, provider, │ + UC committer │ For path-based: Non-UC: returns comment, owner, │ + UC properties │ writes directly Optional.empty() external, path, │ │ to filesystem. option.path) │ NO │ On failure: │ TableManager │ Returns drops pre- hadoopConf → │ .buildCreate │ TransactionCommit registered Engine │ TableTxn( │ Result with: catalog entry (DefaultEngine) │ path, │ • version (0) │ schema, │ • postCommit path resolved: │ engineInfo) │ Snapshot UC → ucTableInfo └───────┬──────────┘ .tablePath non-UC → props Then applies: ["location"] • .withTableProperties(filtered) (throws if • .withDataLayoutSpec(layout) none found) • .build(engine) → Transaction Returns: DDLContext POJO (ident, tablePath, kernelSchema, filteredProps, comment, dataLayoutSpec, engine, ucTableInfo, engineInfo) ``` ## How was this patch tested? ## Does this PR introduce _any_ user-facing changes? --- .../internal/v2/ddl/DDLRequestContext.java | 132 ++++++++++++++++++ .../v2/ddl/DDLRequestContextTest.java | 112 +++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/DDLRequestContext.java create mode 100644 spark/v2/src/test/java/io/delta/spark/internal/v2/ddl/DDLRequestContextTest.java diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/DDLRequestContext.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/DDLRequestContext.java new file mode 100644 index 00000000000..6269d0b1e7f --- /dev/null +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/DDLRequestContext.java @@ -0,0 +1,132 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.ddl; + +import static java.util.Objects.requireNonNull; + +import io.delta.kernel.engine.Engine; +import io.delta.kernel.transaction.DataLayoutSpec; +import io.delta.kernel.types.StructType; +import io.delta.spark.internal.v2.snapshot.unitycatalog.UCTableInfo; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.spark.sql.connector.catalog.Identifier; + +/** + * Immutable POJO capturing all inputs for a DDL operation (CREATE TABLE, CTAS, RTAS). + * + *

    Spark-level types (schema, partitions) are pre-converted to Kernel types by {@link + * CreateTableBuilder#prepare} so downstream code deals only with Kernel abstractions. The Spark + * {@link Identifier} is retained for logging and catalog-level identification only — it is never + * passed to Kernel APIs. + * + *

    All collection fields are defensively copied; accessors return unmodifiable views. + */ +public final class DDLRequestContext { + + private final Identifier ident; + private final String tablePath; + private final StructType kernelSchema; + private final Map properties; + private final Optional comment; + private final DataLayoutSpec dataLayoutSpec; + private final Engine engine; + private final Optional ucTableInfo; + private final String engineInfo; + + /** + * @param ident Spark catalog identifier (namespace + name); used for logging, not Kernel APIs + * @param tablePath resolved filesystem path for the Delta table + * @param kernelSchema Kernel-typed schema converted from Spark StructType + * @param properties user and Delta table properties with DSv2-internal keys already stripped + * @param comment table description from {@code CREATE TABLE ... COMMENT 'x'}, if provided; not + * yet written to the Delta log (see {@link #comment()}) + * @param dataLayoutSpec partitioning / clustering specification + * @param engine Kernel engine instance (typically {@code DefaultEngine}) + * @param ucTableInfo Unity Catalog metadata when the table is UC-managed, empty otherwise + * @param engineInfo version string for commit provenance (e.g. "Delta-Spark-DSv2/3.4.0") + */ + DDLRequestContext( + Identifier ident, + String tablePath, + StructType kernelSchema, + Map properties, + Optional comment, + DataLayoutSpec dataLayoutSpec, + Engine engine, + Optional ucTableInfo, + String engineInfo) { + this.ident = requireNonNull(ident); + this.tablePath = requireNonNull(tablePath); + this.kernelSchema = requireNonNull(kernelSchema); + this.properties = Collections.unmodifiableMap(new HashMap<>(requireNonNull(properties))); + this.comment = requireNonNull(comment); + this.dataLayoutSpec = requireNonNull(dataLayoutSpec); + this.engine = requireNonNull(engine); + this.ucTableInfo = requireNonNull(ucTableInfo); + this.engineInfo = requireNonNull(engineInfo); + } + + public Identifier ident() { + return ident; + } + + public String tablePath() { + return tablePath; + } + + public StructType kernelSchema() { + return kernelSchema; + } + + /** Returns an unmodifiable view of the table properties. */ + public Map properties() { + return properties; + } + + /** + * Table description from {@code COMMENT 'x'}, if provided. Not yet written to the Delta log + * because Kernel's {@code CreateTableTransactionBuilder} does not yet expose a {@code + * withDescription()} method. Preserved here for when that API becomes available. + */ + public Optional comment() { + return comment; + } + + public DataLayoutSpec dataLayoutSpec() { + return dataLayoutSpec; + } + + /** + * Kernel engine instance created via {@code DefaultEngine.create(hadoopConf)}. A new engine is + * created per DDL operation. {@code DefaultEngine} does not hold closeable resources so explicit + * shutdown is not required. See #5675 + * for centralizing engine creation. + */ + public Engine engine() { + return engine; + } + + public Optional ucTableInfo() { + return ucTableInfo; + } + + public String engineInfo() { + return engineInfo; + } +} diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/ddl/DDLRequestContextTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/ddl/DDLRequestContextTest.java new file mode 100644 index 00000000000..f3923c86e92 --- /dev/null +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/ddl/DDLRequestContextTest.java @@ -0,0 +1,112 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.ddl; + +import static org.junit.jupiter.api.Assertions.*; + +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.expressions.Column; +import io.delta.kernel.transaction.DataLayoutSpec; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampType; +import io.delta.spark.internal.v2.snapshot.unitycatalog.UCTableInfo; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.hadoop.conf.Configuration; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.junit.jupiter.api.Test; + +/** Unit tests for the {@link DDLRequestContext} POJO. */ +public class DDLRequestContextTest { + + private static final Engine ENGINE = DefaultEngine.create(new Configuration()); + + @Test + public void testConstructionWithAllFields() { + Identifier ident = Identifier.of(new String[] {"prod_catalog", "analytics"}, "page_views"); + StructType schema = + new StructType() + .add(new StructField("user_id", IntegerType.INTEGER, false)) + .add(new StructField("url", StringType.STRING, false)) + .add(new StructField("event_ts", TimestampType.TIMESTAMP, false)); + DataLayoutSpec clustering = + DataLayoutSpec.clustered(List.of(new Column("user_id"), new Column("event_ts"))); + UCTableInfo ucInfo = + new UCTableInfo( + "uc-table-id-123", + "/managed/prod_catalog/analytics/page_views", + "https://uc.example.com", + Map.of("token", "fake-token")); + + DDLRequestContext request = + new DDLRequestContext( + ident, + "/managed/prod_catalog/analytics/page_views", + schema, + Map.of("delta.appendOnly", "true", "delta.logRetentionDuration", "interval 30 days"), + Optional.of("Tracks page view events from the analytics pipeline"), + clustering, + ENGINE, + Optional.of(ucInfo), + "Delta-Spark-DSv2/4.0.0"); + + assertEquals("page_views", request.ident().name()); + assertArrayEquals(new String[] {"prod_catalog", "analytics"}, request.ident().namespace()); + assertEquals("/managed/prod_catalog/analytics/page_views", request.tablePath()); + assertEquals(3, request.kernelSchema().length()); + assertEquals("true", request.properties().get("delta.appendOnly")); + assertEquals( + Optional.of("Tracks page view events from the analytics pipeline"), request.comment()); + assertTrue(request.dataLayoutSpec().hasClustering()); + assertEquals(2, request.dataLayoutSpec().getClusteringColumns().size()); + assertTrue(request.ucTableInfo().isPresent()); + assertEquals("uc-table-id-123", request.ucTableInfo().get().getTableId()); + assertEquals("Delta-Spark-DSv2/4.0.0", request.engineInfo()); + } + + @Test + public void testPropertiesAreImmutableAndDefensivelyCopied() { + Identifier ident = Identifier.of(new String[] {"db"}, "tbl"); + StructType schema = new StructType().add(new StructField("id", IntegerType.INTEGER, false)); + + HashMap original = new HashMap<>(); + original.put("k", "v"); + DDLRequestContext request = + new DDLRequestContext( + ident, + "/p", + schema, + original, + /* comment = */ Optional.empty(), + DataLayoutSpec.noDataLayout(), + ENGINE, + /* ucTableInfo = */ Optional.empty(), + "e"); + + // Mutating the original map must not leak into the DDLRequestContext + original.put("injected", "bad"); + assertNull(request.properties().get("injected")); + + // The returned map must be unmodifiable + assertThrows(UnsupportedOperationException.class, () -> request.properties().put("x", "y")); + } +} From 7a035989fb1fbf8d883f72bb5d8437142b3f0a4d Mon Sep 17 00:00:00 2001 From: Tathagata Das Date: Tue, 31 Mar 2026 00:59:24 -0400 Subject: [PATCH 73/80] [SPARK] Add DeltaV2TableManager trait for Kernel migration (#6438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Add `DeltaV2TableManager` trait to `v2/interop/` that abstracts `DeltaLog`'s role as the entry point for Delta table operations. `DeltaLog` extends this trait. This is the foundation for migrating Delta protocol reading from `DeltaLog` to Kernel. The trait defines the minimal surface that commands and `OptimisticTransaction` need from the table manager: - Table identity: `logPath`, `dataPath`, `clock`, `options`, `tableId` - Configuration: `newDeltaHadoopConf()` - Snapshot access: `snapshot`, `unsafeVolatileSnapshot`, `tableExists` - Setup: `createLogDirectoriesIfNotExists()` With this trait in place, a future `KernelTableManager` can implement the same interface via Kernel instead of log replay, and all existing business logic (commands, `OptimisticTransaction`) works with either backend. **This PR has zero behavior change** — `DeltaLog` already implements every method defined in the trait, so `with DeltaV2TableManager` is a no-op. ## How was this patch tested? All existing `DeltaLog` tests exercise the new trait. No test changes needed. ## Does this PR introduce _any_ user-facing changes? No. --- .../org/apache/spark/sql/delta/DeltaLog.scala | 4 +- .../v2/interop/DeltaV2TableManager.scala | 67 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala index d6b6d19cd69..2f10c4bf371 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala @@ -28,6 +28,7 @@ import scala.util.Try import scala.util.control.NonFatal import com.databricks.spark.util.TagDefinitions._ +import org.apache.spark.sql.delta.v2.interop.DeltaV2TableManager import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions._ @@ -88,7 +89,8 @@ class DeltaLog private( with SnapshotManagement with DeltaFileFormat with ProvidesUniFormConverters - with ReadChecksum { + with ReadChecksum + with DeltaV2TableManager { import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.delta.util.FileNames._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala b/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala new file mode 100644 index 00000000000..73354dfb7d0 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala @@ -0,0 +1,67 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.v2.interop + +import org.apache.spark.sql.delta.{DeltaLog, Snapshot} +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path + +import org.apache.spark.util.Clock + +/** + * Interface for accessing and managing a Delta table. Mirrors DeltaLog's role as the entry + * point for all Delta table operations. + * + * Two implementations: + * - [[DeltaLog]] (existing): reads the Delta protocol via log replay + * - KernelTableManager (future): reads the Delta protocol via Kernel + * + * Commands, OptimisticTransaction, and other business logic code against this interface + * so they work with either backend. + */ +trait DeltaV2TableManager { + + /** The path of the Delta log directory. */ + def logPath: Path + + /** The path of the data directory of the Delta table. */ + def dataPath: Path + + /** Clock used for timestamping operations. */ + def clock: Clock + + /** Hadoop configuration options for this table. */ + def options: Map[String, String] + + /** Create a new Hadoop Configuration for this table. */ + def newDeltaHadoopConf(): Configuration + + /** Get the current snapshot of the table, updating if necessary. */ + def snapshot: Snapshot + + /** Get the latest snapshot without checking for updates. */ + def unsafeVolatileSnapshot: Snapshot + + /** Whether the table exists (version >= 0). */ + def tableExists: Boolean + + /** A unique identifier for this table. */ + def tableId: String + + /** Create the log directories if they do not exist. */ + def createLogDirectoriesIfNotExists(): Unit +} From 18019e54db807e7667f09c29a6fe6885cb4b0853 Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Tue, 31 Mar 2026 10:31:50 +0530 Subject: [PATCH 74/80] [CI Improvements] Fix flaky DeltaRetentionWithCatalogOwnedBatch1Suite test (#6348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Fixes a flaky test in `DeltaRetentionWithCatalogOwnedBatch1Suite` that fails intermittently with: ``` 20 did not equal 0 Delta files before the last checkpoint version should have been deleted (DeltaRetentionSuite.scala:152) ``` ### Root Cause The test `"log files being already deleted shouldn't fail log deletion job"` fails **only when executed between 00:00-01:00 UTC**. The mechanism: 1. `getStartTimeForRetentionTest()` returns today at 01:00 UTC — which is a **future** timestamp if the test runs before 01:00 UTC. 2. Delta files get `setLastModified` to this future time (01:00:xx UTC). 3. V2 checkpoint files (introduced by CatalogOwned in #5841) use UUID-based names (`*.checkpoint..json`) that don't match `checkpointFileSingular` format (`*.checkpoint.parquet`), so the test never sets their timestamps — they keep their real wallclock time (~00:04 UTC). 4. `BufferingLogDeletionIterator` checks `needsTimeAdjustment()` **before** `isCheckpointFile()`. When a V2 checkpoint (modTime=00:04) follows a delta file (modTime=01:01), the checkpoint's earlier real timestamp triggers time adjustment, causing it to be treated as a regular file instead of a checkpoint boundary. The buffer is never flushed, and zero files get deleted. ### Fix Replace the `checkpointFileSingular` lookup (lines 133-136) with the `getCheckpointFiles` + `checkpointVersion` pattern already used and working correctly elsewhere in the same file (lines 724-734). This finds all checkpoint file formats including V2. ### Verified Reproduced the failure by forcing `getStartTimeForRetentionTest` to return a future timestamp: - **Without fix**: `FAILED` — `20 did not equal 0` (identical to CI failures) - **With fix**: `PASSED` Confirmed all CI failures occurred in the 00:00-01:00 UTC window: 00:04:33, 00:13:10, 00:21:48, 00:04:47 UTC. ## How was this patch tested? Existing test `DeltaRetentionWithCatalogOwnedBatch1Suite."log files being already deleted shouldn't fail log deletion job"` — this PR fixes it. ## Does this PR introduce _any_ user-facing changes? No. Test-only change. This pull request was AI-assisted by Isaac. --- .../org/apache/spark/sql/delta/DeltaRetentionSuite.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala index 2552ffabf52..506e882fbf0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala @@ -130,10 +130,9 @@ class DeltaRetentionSuite extends QueryTest deltaFile.setLastModified(clock.getTimeMillis() + i * 10000) val crcFile = new File(FileNames.checksumFile(log.logPath, version).toUri) crcFile.setLastModified(clock.getTimeMillis() + i * 10000) - val chk = new File(FileNames.checkpointFileSingular(log.logPath, version).toUri) - if (chk.exists()) { - chk.setLastModified(clock.getTimeMillis() + i * 10000) - } + getCheckpointFiles(logPath) + .filter(f => FileNames.checkpointVersion(new Path(f.getCanonicalPath)) == version) + .foreach(_.setLastModified(clock.getTimeMillis() + i * 10000)) } // delete some files in the middle From c9291bd3634ea0218ca868e5207422fabfe33d8c Mon Sep 17 00:00:00 2001 From: Murali Ramanujam Date: Tue, 31 Mar 2026 12:15:17 +0530 Subject: [PATCH 75/80] [DSv2] Extract shared buildFileName from DelayedCommitProtocol (#6443) Extract the file name construction logic from DelayedCommitProtocol.getFileName into a shared companion object method DelayedCommitProtocol.buildFileName. This enables reuse of the standard Delta file naming convention (part-NNNNN-uuid.ext / cdc-NNNNN-uuid.ext) by both the V1 write path (DelayedCommitProtocol) and the DSv2 write path (DataWriter), ensuring consistent file naming across both code paths. Changes: - Move file name building logic to DelayedCommitProtocol.buildFileName companion method - Simplify getFileName to delegate to the new shared method - Preserve all existing behavior (test prefix, CDC prefix, split formatting) --- .../delta/files/DelayedCommitProtocol.scala | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala b/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala index 23df32bea14..881e8ffdcf8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala @@ -109,22 +109,17 @@ class DelayedCommitProtocol( addedFiles = new ArrayBuffer[(Map[String, String], String)] } - /** Prefix added in testing mode to all filenames to test special chars that need URL-encoding. */ - val FILE_NAME_PREFIX = SQLConf.get.getConf(DeltaSQLConf.TEST_FILE_NAME_PREFIX) - + // The file name looks like part-r-00000-2dd664f9-d2c4-4ffe-878f-c6c70c1fb0cb_00003.gz.parquet + // Note that %05d does not truncate the split number, so if we have more than 100000 tasks, + // the file name is fine and won't overflow. + // CDC files (CDC_PARTITION_COL = true) are named with "cdc-..." instead of "part-...". protected def getFileName( taskContext: TaskAttemptContext, ext: String, partitionValues: Map[String, String]): String = { - // The file name looks like part-r-00000-2dd664f9-d2c4-4ffe-878f-c6c70c1fb0cb_00003.gz.parquet - // Note that %05d does not truncate the split number, so if we have more than 100000 tasks, - // the file name is fine and won't overflow. val split = taskContext.getTaskAttemptID.getTaskID.getId - val uuid = UUID.randomUUID.toString - // CDC files (CDC_PARTITION_COL = true) are named with "cdc-..." instead of "part-...". - val typePrefix = - if (partitionValues.get(CDC_PARTITION_COL).contains("true")) "cdc-" else "part-" - f"${FILE_NAME_PREFIX}${typePrefix}${split}%05d-${uuid}${ext}" + val isCdc = partitionValues.get(CDC_PARTITION_COL).contains("true") + DelayedCommitProtocol.buildFileName(split, ext, isCdc) } protected def parsePartitions( @@ -282,3 +277,23 @@ class DelayedCommitProtocol( // TODO: we can also try delete the addedFiles as a best-effort cleanup. } } + +object DelayedCommitProtocol { + /** + * Builds a Delta data file name following the standard naming convention: + * {testPrefix}{typePrefix}{splitId%05d}-{uuid}{ext} + * + * Shared between V1 (DelayedCommitProtocol.getFileName) and DSv2 (DataWriter). + * + * @param splitId the task split/partition ID + * @param ext file extension including dot (e.g. ".snappy.parquet") + * @param isCdc true for CDC files (uses "cdc-" prefix instead of "part-") + * @return the file name string + */ + def buildFileName(splitId: Int, ext: String, isCdc: Boolean): String = { + val fileNamePrefix = SQLConf.get.getConf(DeltaSQLConf.TEST_FILE_NAME_PREFIX) + val uuid = UUID.randomUUID.toString + val typePrefix = if (isCdc) "cdc-" else "part-" + f"${fileNamePrefix}${typePrefix}${splitId}%05d-${uuid}${ext}" + } +} From 3f10cb46fc69ae1e6e4a6098519f6c77c257eec7 Mon Sep 17 00:00:00 2001 From: emu Date: Tue, 17 Mar 2026 13:58:37 +0000 Subject: [PATCH 76/80] Geospatial expressions: StIntersectsBoxes predicate and data skipping translation Add the public StIntersectsBoxes expression and its internal translation to ST_INTERSECTS_BOXES_ON_STATS for geospatial data skipping: - StIntersectsBoxes: public predicate taking column + WKT query bounds - DataSkippingUtils: translate to stats-based predicate with min/max columns - ExpressionVisitor: dispatch ST_INTERSECTS_BOXES_ON_STATS - DefaultExpressionEvaluator: transform (type validation) and eval (AABB intersection via WKT parsing) - GeometryDataSkippingSuite: end-to-end tests with 4-quadrant layout, null stats, and combined AND predicates Co-authored-by: Isaac --- .../kernel/expressions/StIntersectsBoxes.java | 65 ++++ .../internal/skipping/DataSkippingUtils.java | 25 ++ .../DefaultExpressionEvaluator.java | 66 ++++ .../expressions/ExpressionVisitor.java | 4 + .../defaults/GeometryDataSkippingSuite.scala | 346 ++++++++++++++++++ 5 files changed, 506 insertions(+) create mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StIntersectsBoxes.java create mode 100644 kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StIntersectsBoxes.java b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StIntersectsBoxes.java new file mode 100644 index 00000000000..8dbf2eee16f --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StIntersectsBoxes.java @@ -0,0 +1,65 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.expressions; + +import static io.delta.kernel.internal.util.Preconditions.checkArgument; + +import io.delta.kernel.types.StringType; +import java.util.Arrays; + +/** + * A predicate that tests whether the bounding box of a geometry column intersects a given query + * bounding box. + * + *

    This expression is not directly evaluatable. It is transformed to an internal + * ST_INTERSECTS_BOXES_ON_STATS predicate during data-skipping filter construction, which evaluates + * the intersection of the column's min/max bounding box against the query box. + * + * @since 4.0.0 + */ +public class StIntersectsBoxes extends Predicate { + + public static final String NAME = "ST_INTERSECTS_BOXES"; + + /** + * @param column the geometry or geography column to test + * @param queryMin lower-left corner of the query bounding box as a WKT POINT string literal + * @param queryMax upper-right corner of the query bounding box as a WKT POINT string literal + */ + public StIntersectsBoxes(Column column, Literal queryMin, Literal queryMax) { + super(NAME, Arrays.asList(column, queryMin, queryMax)); + checkArgument( + queryMin.getDataType() instanceof StringType, + "queryMin must be a StringType WKT literal, got: %s", + queryMin.getDataType()); + checkArgument( + queryMax.getDataType() instanceof StringType, + "queryMax must be a StringType WKT literal, got: %s", + queryMax.getDataType()); + } + + public Column getColumn() { + return (Column) getChildren().get(0); + } + + public Literal getQueryMin() { + return (Literal) getChildren().get(1); + } + + public Literal getQueryMax() { + return (Literal) getChildren().get(2); + } +} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java index 11d01a5e36c..a9fee2e5a86 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java @@ -286,6 +286,31 @@ private static Optional constructDataSkippingFilter( return constructNotDataSkippingFilters( asPredicate(getUnaryChild(dataFilters)), schemaHelper); + case StIntersectsBoxes.NAME: + if (dataFilters instanceof StIntersectsBoxes) { + StIntersectsBoxes stExpr = (StIntersectsBoxes) dataFilters; + Column col = stExpr.getColumn(); + if (schemaHelper.isSkippingEligibleMinMaxColumn(col)) { + Column minCol = schemaHelper.getMinColumn(col, Optional.empty())._1; + Column maxCol = schemaHelper.getMaxColumn(col, Optional.empty())._1; + Literal queryMin = stExpr.getQueryMin(); + Literal queryMax = stExpr.getQueryMax(); + Set refCols = + new HashSet() { + { + add(minCol); + add(maxCol); + } + }; + return Optional.of( + new DataSkippingPredicate( + "ST_INTERSECTS_BOXES_ON_STATS", + Arrays.asList(minCol, maxCol, queryMin, queryMax), + refCols)); + } + } + break; + // TODO more expressions } return Optional.empty(); diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index ab5937afbb7..b713f9a4737 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -30,6 +30,7 @@ import io.delta.kernel.defaults.internal.data.vector.DefaultConstantVector; import io.delta.kernel.engine.ExpressionHandler; import io.delta.kernel.expressions.*; +import io.delta.kernel.internal.util.GeometryUtils; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Collectors; @@ -361,6 +362,39 @@ ExpressionTransformResult visitIn(In in) { return new ExpressionTransformResult(transformedExpression, BooleanType.BOOLEAN); } + @Override + ExpressionTransformResult visitStIntersectsBoxesOnStats(Predicate predicate) { + List children = + predicate.getChildren().stream().map(this::visit).collect(Collectors.toList()); + checkArgument( + children.size() == 4, + "ST_INTERSECTS_BOXES_ON_STATS expects 4 children but got %d", + children.size()); + // Children 0,1 are stats columns (GeometryType/GeographyType). + // Children 2,3 are WKT string literals from the query bounds. + for (int i = 0; i < 2; i++) { + DataType dt = children.get(i).outputType; + checkArgument( + dt instanceof GeometryType || dt instanceof GeographyType, + "ST_INTERSECTS_BOXES_ON_STATS child %d must be " + "geometry/geography type, got %s", + i, + dt); + } + for (int i = 2; i < 4; i++) { + DataType dt = children.get(i).outputType; + checkArgument( + dt instanceof StringType, + "ST_INTERSECTS_BOXES_ON_STATS child %d must be " + "string (WKT), got %s", + i, + dt); + } + return new ExpressionTransformResult( + new Predicate( + "ST_INTERSECTS_BOXES_ON_STATS", + children.stream().map(c -> c.expression).collect(Collectors.toList())), + BooleanType.BOOLEAN); + } + private Predicate validateIsPredicate( Expression baseExpression, ExpressionTransformResult result) { checkArgument( @@ -747,6 +781,38 @@ ColumnVector visitIn(In in) { in.getChildren().stream().map(this::visit).collect(toList())); } + @Override + ColumnVector visitStIntersectsBoxesOnStats(Predicate predicate) { + List children = predicate.getChildren(); + ColumnVector leftMin = visit(children.get(0)); + ColumnVector leftMax = visit(children.get(1)); + ColumnVector rightMin = visit(children.get(2)); + ColumnVector rightMax = visit(children.get(3)); + int numRows = input.getSize(); + boolean[] result = new boolean[numRows]; + boolean[] nullability = new boolean[numRows]; + for (int rowId = 0; rowId < numRows; rowId++) { + if (leftMin.isNullAt(rowId) + || leftMax.isNullAt(rowId) + || rightMin.isNullAt(rowId) + || rightMax.isNullAt(rowId)) { + nullability[rowId] = true; + continue; + } + double[] lMin = GeometryUtils.parsePointXY(leftMin.getString(rowId)); + double[] lMax = GeometryUtils.parsePointXY(leftMax.getString(rowId)); + double[] rMin = GeometryUtils.parsePointXY(rightMin.getString(rowId)); + double[] rMax = GeometryUtils.parsePointXY(rightMax.getString(rowId)); + result[rowId] = boxesIntersect(lMin, lMax, rMin, rMax); + } + return new DefaultBooleanVector(numRows, Optional.of(nullability), result); + } + + private static boolean boxesIntersect( + double[] lMin, double[] lMax, double[] rMin, double[] rMax) { + return lMax[0] >= rMin[0] && rMax[0] >= lMin[0] && lMax[1] >= rMin[1] && rMax[1] >= lMin[1]; + } + /** * Utility method to evaluate inputs to the binary input expression. Also validates the * evaluated expression result {@link ColumnVector}s are of the same size. diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java index 468dd70225d..2dd360c24e4 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java @@ -74,6 +74,8 @@ abstract class ExpressionVisitor { abstract R visitIn(In in); + abstract R visitStIntersectsBoxesOnStats(Predicate predicate); + final R visit(Expression expression) { if (expression instanceof PartitionValueExpression) { return visitPartitionValue((PartitionValueExpression) expression); @@ -144,6 +146,8 @@ private R visitScalarExpression(ScalarExpression expression) { } else { return visitIn(new In(children.get(0), children.subList(1, children.size()))); } + case "ST_INTERSECTS_BOXES_ON_STATS": + return visitStIntersectsBoxesOnStats(createPredicate(name, children, collationIdentifier)); default: throw new UnsupportedOperationException( String.format("Scalar expression `%s` is not supported.", name)); diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala new file mode 100644 index 00000000000..313176c83ac --- /dev/null +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala @@ -0,0 +1,346 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults + +import java.util.{Collections, Optional} + +import scala.collection.JavaConverters._ +import scala.collection.immutable.Seq + +import io.delta.kernel._ +import io.delta.kernel.defaults.utils.WriteUtils +import io.delta.kernel.expressions.{Column, Literal, StIntersectsBoxes} +import io.delta.kernel.internal.util.Utils.toCloseableIterator +import io.delta.kernel.statistics.DataFileStatistics +import io.delta.kernel.types.{DataType, GeographyType, GeometryType, StructType} +import io.delta.kernel.utils.CloseableIterable.inMemoryIterable +import io.delta.kernel.utils.DataFileStatus + +import org.scalatest.funsuite.AnyFunSuite + +/** + * End-to-end tests for data skipping with geometry and geography column + * types using the StIntersectsBoxes predicate. + * + * Tests bypass the Parquet writer by injecting DataFileStatistics with + * geometry min/max literals directly into the Delta log. + */ +class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { + + // 4-quadrant layout: + // y=10 +----------+----------+ + // | NW file2 | NE file1 | + // y=7 + + + + // | x:[0,3] | x:[7,10] | + // y=3 +----------+----------+ + // | SW file0 | SE file3 | + // y=0 +----------+----------+ + // x=0 x=3 x=7 x=10 + private val fileExtents: Seq[(Double, Double, Double, Double)] = Seq( + (0.0, 0.0, 3.0, 3.0), // SW - file 0 + (7.0, 7.0, 10.0, 10.0), // NE - file 1 + (0.0, 7.0, 3.0, 10.0), // NW - file 2 + (7.0, 0.0, 10.0, 3.0) // SE - file 3 + ) + + for ( + colType <- Seq[DataType]( + new GeometryType("OGC:CRS84"), + new GeographyType("OGC:CRS84", "spherical")) + ) { + val typeName = colType.getClass.getSimpleName + + test(s"StIntersectsBoxes data skipping on $typeName column") { + withTempDirAndEngine { (tablePath, engine) => + val schema = new StructType().add("geom", colType) + writeFilesWithGeometryStats( + tablePath, + engine, + schema, + fileExtents) + + val snapshot = latestSnapshot(tablePath) + + def filesHit( + qMinX: Double, + qMinY: Double, + qMaxX: Double, + qMaxY: Double): Int = { + val pred = new StIntersectsBoxes( + new Column("geom"), + Literal.ofString(s"POINT ($qMinX $qMinY)"), + Literal.ofString(s"POINT ($qMaxX $qMaxY)")) + collectScanFileRows( + snapshot.getScanBuilder().withFilter(pred).build()).size + } + + // SW query - intersects only SW file + assert(filesHit(1.0, 1.0, 4.0, 4.0) == 1) + // NE query - intersects only NE file + assert(filesHit(8.0, 8.0, 11.0, 11.0) == 1) + // Center gap - intersects nothing + assert(filesHit(4.0, 4.0, 6.0, 6.0) == 0) + // West strip [1,1]-[4,9] - intersects SW and NW + assert(filesHit(1.0, 1.0, 4.0, 9.0) == 2) + // Top strip [0,8]-[11,11] - intersects NE and NW + assert(filesHit(0.0, 8.0, 11.0, 11.0) == 2) + // Global - intersects all 4 files + assert(filesHit(0.0, 0.0, 11.0, 11.0) == 4) + } + } + + test(s"StIntersectsBoxes: file with null $typeName stats is never skipped") { + withTempDirAndEngine { (tablePath, engine) => + val schema = new StructType().add("geom", colType) + val geomCol = new Column("geom") + + // File 0: SW quadrant with geometry stats + // File 1: no geometry stats -> must always be included + val extentsOpt: Seq[Option[(Double, Double, Double, Double)]] = + Seq(Some((0.0, 0.0, 3.0, 3.0)), None) + + extentsOpt.zipWithIndex.foreach { + case (extentOpt, idx) => + val txn = + if (idx == 0) + getCreateTxn(engine, tablePath, schema) + else getUpdateTxn(engine, tablePath) + + val txnState = txn.getTransactionState(engine) + val writeContext = Transaction.getWriteContext( + engine, + txnState, + Collections.emptyMap()) + + val stats = extentOpt match { + case Some((minX, minY, maxX, maxY)) => + new DataFileStatistics( + 10, + Map(geomCol -> Literal.ofString( + s"POINT ($minX $minY)")).asJava, + Map(geomCol -> Literal.ofString( + s"POINT ($maxX $maxY)")).asJava, + Map(geomCol -> (0L: java.lang.Long)).asJava, + Optional.empty()) + case None => + new DataFileStatistics( + 10, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Optional.empty()) + } + + val filePath = + engine.getFileSystemClient.resolvePath( + writeContext.getTargetDirectory + s"/part-$idx.parquet") + val fileStatus = new DataFileStatus( + filePath, + 1000, + 0L, + Optional.of(stats)) + + val actions = Transaction.generateAppendActions( + engine, + txnState, + toCloseableIterator(Seq(fileStatus).iterator.asJava), + writeContext) + commitTransaction( + txn, + engine, + inMemoryIterable(actions)) + } + + val snapshot = latestSnapshot(tablePath) + + def filesHit( + qMinX: Double, + qMinY: Double, + qMaxX: Double, + qMaxY: Double): Int = { + val pred = new StIntersectsBoxes( + geomCol, + Literal.ofString(s"POINT ($qMinX $qMinY)"), + Literal.ofString(s"POINT ($qMaxX $qMaxY)")) + collectScanFileRows( + snapshot.getScanBuilder().withFilter(pred).build()).size + } + + // NE query: SW file is provably non-overlapping, but + // null-stats file must be included + assert(filesHit(8.0, 8.0, 11.0, 11.0) == 1) + // SW query: SW file matches, null-stats file also included + assert(filesHit(1.0, 1.0, 4.0, 4.0) == 2) + } + } + } + + test("StIntersectsBoxes combined with AND predicate on second column") { + withTempDirAndEngine { (tablePath, engine) => + val schema = new StructType() + .add("id", io.delta.kernel.types.IntegerType.INTEGER) + .add("geom", new GeometryType("OGC:CRS84")) + + val idCol = new Column("id") + val geomCol = new Column("geom") + + // File 0 (SW): id in [1,5], geom bbox [0,0]-[3,3] + // File 1 (NE): id in [10,20], geom bbox [7,7]-[10,10] + val files = Seq( + (1, 5, 0.0, 0.0, 3.0, 3.0), + (10, 20, 7.0, 7.0, 10.0, 10.0)) + + files.zipWithIndex.foreach { + case ((idMin, idMax, gMinX, gMinY, gMaxX, gMaxY), idx) => + val txn = + if (idx == 0) + getCreateTxn(engine, tablePath, schema) + else getUpdateTxn(engine, tablePath) + + val txnState = txn.getTransactionState(engine) + val writeContext = Transaction.getWriteContext( + engine, + txnState, + Collections.emptyMap()) + + val stats = new DataFileStatistics( + 10, + Map( + idCol -> Literal.ofInt(idMin), + geomCol -> Literal.ofString( + s"POINT ($gMinX $gMinY)")).asJava, + Map( + idCol -> Literal.ofInt(idMax), + geomCol -> Literal.ofString( + s"POINT ($gMaxX $gMaxY)")).asJava, + Map( + idCol -> (0L: java.lang.Long), + geomCol -> (0L: java.lang.Long)).asJava, + Optional.empty()) + + val filePath = + engine.getFileSystemClient.resolvePath( + writeContext.getTargetDirectory + s"/part-$idx.parquet") + val fileStatus = new DataFileStatus( + filePath, + 1000, + 0L, + Optional.of(stats)) + + val actions = Transaction.generateAppendActions( + engine, + txnState, + toCloseableIterator(Seq(fileStatus).iterator.asJava), + writeContext) + commitTransaction( + txn, + engine, + inMemoryIterable(actions)) + } + + val snapshot = latestSnapshot(tablePath) + + // SW geometry + id <= 5: only File 0 survives both filters + val geoAndId = new io.delta.kernel.expressions.And( + new StIntersectsBoxes( + geomCol, + Literal.ofString("POINT (1.0 1.0)"), + Literal.ofString("POINT (4.0 4.0)")), + new io.delta.kernel.expressions.Predicate( + "<=", + idCol, + Literal.ofInt(5))) + + val files1 = collectScanFileRows( + snapshot.getScanBuilder().withFilter(geoAndId).build()) + assert(files1.size == 1) + + // NE geometry + id >= 15: only File 1 survives both filters + val geoAndId2 = new io.delta.kernel.expressions.And( + new StIntersectsBoxes( + geomCol, + Literal.ofString("POINT (8.0 8.0)"), + Literal.ofString("POINT (11.0 11.0)")), + new io.delta.kernel.expressions.Predicate( + ">=", + idCol, + Literal.ofInt(15))) + + val files2 = collectScanFileRows( + snapshot.getScanBuilder().withFilter(geoAndId2).build()) + assert(files2.size == 1) + + // Center geometry + any id: geometry miss skips both files + val geoCenterAny = new StIntersectsBoxes( + geomCol, + Literal.ofString("POINT (4.0 4.0)"), + Literal.ofString("POINT (6.0 6.0)")) + + val files3 = collectScanFileRows( + snapshot.getScanBuilder().withFilter(geoCenterAny).build()) + assert(files3.isEmpty) + } + } + + private def writeFilesWithGeometryStats( + tablePath: String, + engine: io.delta.kernel.engine.Engine, + schema: StructType, + extents: Seq[(Double, Double, Double, Double)]): Unit = { + val geomCol = new Column("geom") + extents.zipWithIndex.foreach { + case ((minX, minY, maxX, maxY), idx) => + val txn = + if (idx == 0) + getCreateTxn(engine, tablePath, schema) + else getUpdateTxn(engine, tablePath) + + val txnState = txn.getTransactionState(engine) + val writeContext = Transaction.getWriteContext( + engine, + txnState, + Collections.emptyMap()) + + val stats = new DataFileStatistics( + 10, + Map(geomCol -> Literal.ofString( + s"POINT ($minX $minY)")).asJava, + Map(geomCol -> Literal.ofString( + s"POINT ($maxX $maxY)")).asJava, + Map(geomCol -> (0L: java.lang.Long)).asJava, + Optional.empty()) + + val filePath = + engine.getFileSystemClient.resolvePath( + writeContext.getTargetDirectory + s"/part-$idx.parquet") + val fileStatus = new DataFileStatus( + filePath, + 1000, + 0L, + Optional.of(stats)) + + val actions = Transaction.generateAppendActions( + engine, + txnState, + toCloseableIterator(Seq(fileStatus).iterator.asJava), + writeContext) + commitTransaction( + txn, + engine, + inMemoryIterable(actions)) + } + } +} From b9d83463b03d18260609bb8d1e2c4741e3bc14b5 Mon Sep 17 00:00:00 2001 From: emu Date: Thu, 19 Mar 2026 17:18:40 +0000 Subject: [PATCH 77/80] eval delta --- .../DefaultExpressionEvaluator.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index b713f9a4737..d95d0a96b2d 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -370,23 +370,18 @@ ExpressionTransformResult visitStIntersectsBoxesOnStats(Predicate predicate) { children.size() == 4, "ST_INTERSECTS_BOXES_ON_STATS expects 4 children but got %d", children.size()); - // Children 0,1 are stats columns (GeometryType/GeographyType). - // Children 2,3 are WKT string literals from the query bounds. - for (int i = 0; i < 2; i++) { - DataType dt = children.get(i).outputType; - checkArgument( - dt instanceof GeometryType || dt instanceof GeographyType, - "ST_INTERSECTS_BOXES_ON_STATS child %d must be " + "geometry/geography type, got %s", - i, - dt); - } - for (int i = 2; i < 4; i++) { - DataType dt = children.get(i).outputType; + DataType child0Type = children.get(0).outputType; + checkArgument( + child0Type instanceof GeometryType || child0Type instanceof GeographyType, + "ST_INTERSECTS_BOXES_ON_STATS children must be " + "geometry/geography type, got %s", + child0Type); + for (int i = 1; i < 4; i++) { checkArgument( - dt instanceof StringType, - "ST_INTERSECTS_BOXES_ON_STATS child %d must be " + "string (WKT), got %s", + child0Type.equals(children.get(i).outputType), + "ST_INTERSECTS_BOXES_ON_STATS child %d type %s " + "doesn't match child 0 type %s", i, - dt); + children.get(i).outputType, + child0Type); } return new ExpressionTransformResult( new Predicate( From ff0a9cdbb814291f888318dde63031678d2497fb Mon Sep 17 00:00:00 2001 From: emu Date: Thu, 19 Mar 2026 18:01:15 +0000 Subject: [PATCH 78/80] only geometry --- ...xes.java => StGeometryBoxesIntersect.java} | 12 +- .../internal/skipping/DataSkippingUtils.java | 8 +- .../DefaultExpressionEvaluator.java | 29 +- .../expressions/ExpressionVisitor.java | 7 +- .../defaults/GeometryDataSkippingSuite.scala | 260 +++++++++--------- 5 files changed, 159 insertions(+), 157 deletions(-) rename kernel/kernel-api/src/main/java/io/delta/kernel/expressions/{StIntersectsBoxes.java => StGeometryBoxesIntersect.java} (80%) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StIntersectsBoxes.java b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java similarity index 80% rename from kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StIntersectsBoxes.java rename to kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java index 8dbf2eee16f..41e412e314a 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StIntersectsBoxes.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java @@ -25,21 +25,21 @@ * bounding box. * *

    This expression is not directly evaluatable. It is transformed to an internal - * ST_INTERSECTS_BOXES_ON_STATS predicate during data-skipping filter construction, which evaluates - * the intersection of the column's min/max bounding box against the query box. + * ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS predicate during data-skipping filter construction, which + * evaluates the intersection of the column's min/max bounding box against the query box. * * @since 4.0.0 */ -public class StIntersectsBoxes extends Predicate { +public class StGeometryBoxesIntersect extends Predicate { - public static final String NAME = "ST_INTERSECTS_BOXES"; + public static final String NAME = "ST_GEOMETRY_BOXES_INTERSECT"; /** - * @param column the geometry or geography column to test + * @param column the geometry column to test * @param queryMin lower-left corner of the query bounding box as a WKT POINT string literal * @param queryMax upper-right corner of the query bounding box as a WKT POINT string literal */ - public StIntersectsBoxes(Column column, Literal queryMin, Literal queryMax) { + public StGeometryBoxesIntersect(Column column, Literal queryMin, Literal queryMax) { super(NAME, Arrays.asList(column, queryMin, queryMax)); checkArgument( queryMin.getDataType() instanceof StringType, diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java index a9fee2e5a86..5ea124b8e26 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java @@ -286,9 +286,9 @@ private static Optional constructDataSkippingFilter( return constructNotDataSkippingFilters( asPredicate(getUnaryChild(dataFilters)), schemaHelper); - case StIntersectsBoxes.NAME: - if (dataFilters instanceof StIntersectsBoxes) { - StIntersectsBoxes stExpr = (StIntersectsBoxes) dataFilters; + case StGeometryBoxesIntersect.NAME: + if (dataFilters instanceof StGeometryBoxesIntersect) { + StGeometryBoxesIntersect stExpr = (StGeometryBoxesIntersect) dataFilters; Column col = stExpr.getColumn(); if (schemaHelper.isSkippingEligibleMinMaxColumn(col)) { Column minCol = schemaHelper.getMinColumn(col, Optional.empty())._1; @@ -304,7 +304,7 @@ private static Optional constructDataSkippingFilter( }; return Optional.of( new DataSkippingPredicate( - "ST_INTERSECTS_BOXES_ON_STATS", + "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS", Arrays.asList(minCol, maxCol, queryMin, queryMax), refCols)); } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index d95d0a96b2d..ff70275d731 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -363,29 +363,36 @@ ExpressionTransformResult visitIn(In in) { } @Override - ExpressionTransformResult visitStIntersectsBoxesOnStats(Predicate predicate) { + ExpressionTransformResult visitStGeometryBoxesIntersectsOnStats(Predicate predicate) { List children = predicate.getChildren().stream().map(this::visit).collect(Collectors.toList()); checkArgument( children.size() == 4, - "ST_INTERSECTS_BOXES_ON_STATS expects 4 children but got %d", + "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS expects 4 children but got %d", children.size()); + // Children 0,1 are stats columns (must be GeometryType). + // Children 2,3 are WKT query literals (must be StringType). DataType child0Type = children.get(0).outputType; checkArgument( - child0Type instanceof GeometryType || child0Type instanceof GeographyType, - "ST_INTERSECTS_BOXES_ON_STATS children must be " + "geometry/geography type, got %s", + child0Type instanceof GeometryType, + "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS child 0 must be " + "geometry type, got %s", child0Type); - for (int i = 1; i < 4; i++) { + checkArgument( + child0Type.equals(children.get(1).outputType), + "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS child 1 type %s " + + "doesn't match child 0 type %s", + children.get(1).outputType, + child0Type); + for (int i = 2; i < 4; i++) { checkArgument( - child0Type.equals(children.get(i).outputType), - "ST_INTERSECTS_BOXES_ON_STATS child %d type %s " + "doesn't match child 0 type %s", + children.get(i).outputType instanceof StringType, + "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS child %d must be " + "string (WKT), got %s", i, - children.get(i).outputType, - child0Type); + children.get(i).outputType); } return new ExpressionTransformResult( new Predicate( - "ST_INTERSECTS_BOXES_ON_STATS", + "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS", children.stream().map(c -> c.expression).collect(Collectors.toList())), BooleanType.BOOLEAN); } @@ -777,7 +784,7 @@ ColumnVector visitIn(In in) { } @Override - ColumnVector visitStIntersectsBoxesOnStats(Predicate predicate) { + ColumnVector visitStGeometryBoxesIntersectsOnStats(Predicate predicate) { List children = predicate.getChildren(); ColumnVector leftMin = visit(children.get(0)); ColumnVector leftMax = visit(children.get(1)); diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java index 2dd360c24e4..21bfb8a1824 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java @@ -74,7 +74,7 @@ abstract class ExpressionVisitor { abstract R visitIn(In in); - abstract R visitStIntersectsBoxesOnStats(Predicate predicate); + abstract R visitStGeometryBoxesIntersectsOnStats(Predicate predicate); final R visit(Expression expression) { if (expression instanceof PartitionValueExpression) { @@ -146,8 +146,9 @@ private R visitScalarExpression(ScalarExpression expression) { } else { return visitIn(new In(children.get(0), children.subList(1, children.size()))); } - case "ST_INTERSECTS_BOXES_ON_STATS": - return visitStIntersectsBoxesOnStats(createPredicate(name, children, collationIdentifier)); + case "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS": + return visitStGeometryBoxesIntersectsOnStats( + createPredicate(name, children, collationIdentifier)); default: throw new UnsupportedOperationException( String.format("Scalar expression `%s` is not supported.", name)); diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala index 313176c83ac..86c7c5226cf 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala @@ -22,18 +22,18 @@ import scala.collection.immutable.Seq import io.delta.kernel._ import io.delta.kernel.defaults.utils.WriteUtils -import io.delta.kernel.expressions.{Column, Literal, StIntersectsBoxes} +import io.delta.kernel.expressions.{Column, Literal, StGeometryBoxesIntersect} import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.statistics.DataFileStatistics -import io.delta.kernel.types.{DataType, GeographyType, GeometryType, StructType} +import io.delta.kernel.types.{GeometryType, StructType} import io.delta.kernel.utils.CloseableIterable.inMemoryIterable import io.delta.kernel.utils.DataFileStatus import org.scalatest.funsuite.AnyFunSuite /** - * End-to-end tests for data skipping with geometry and geography column - * types using the StIntersectsBoxes predicate. + * End-to-end tests for data skipping with geometry column types using + * the StGeometryBoxesIntersect predicate. * * Tests bypass the Parquet writer by injecting DataFileStatistics with * geometry min/max literals directly into the Delta log. @@ -56,139 +56,133 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { (7.0, 0.0, 10.0, 3.0) // SE - file 3 ) - for ( - colType <- Seq[DataType]( - new GeometryType("OGC:CRS84"), - new GeographyType("OGC:CRS84", "spherical")) - ) { - val typeName = colType.getClass.getSimpleName - - test(s"StIntersectsBoxes data skipping on $typeName column") { - withTempDirAndEngine { (tablePath, engine) => - val schema = new StructType().add("geom", colType) - writeFilesWithGeometryStats( - tablePath, - engine, - schema, - fileExtents) - - val snapshot = latestSnapshot(tablePath) - - def filesHit( - qMinX: Double, - qMinY: Double, - qMaxX: Double, - qMaxY: Double): Int = { - val pred = new StIntersectsBoxes( - new Column("geom"), - Literal.ofString(s"POINT ($qMinX $qMinY)"), - Literal.ofString(s"POINT ($qMaxX $qMaxY)")) - collectScanFileRows( - snapshot.getScanBuilder().withFilter(pred).build()).size - } - - // SW query - intersects only SW file - assert(filesHit(1.0, 1.0, 4.0, 4.0) == 1) - // NE query - intersects only NE file - assert(filesHit(8.0, 8.0, 11.0, 11.0) == 1) - // Center gap - intersects nothing - assert(filesHit(4.0, 4.0, 6.0, 6.0) == 0) - // West strip [1,1]-[4,9] - intersects SW and NW - assert(filesHit(1.0, 1.0, 4.0, 9.0) == 2) - // Top strip [0,8]-[11,11] - intersects NE and NW - assert(filesHit(0.0, 8.0, 11.0, 11.0) == 2) - // Global - intersects all 4 files - assert(filesHit(0.0, 0.0, 11.0, 11.0) == 4) + private val colType = new GeometryType("OGC:CRS84") + + test("StGeometryBoxesIntersect data skipping on GeometryType column") { + withTempDirAndEngine { (tablePath, engine) => + val schema = new StructType().add("geom", colType) + writeFilesWithGeometryStats( + tablePath, + engine, + schema, + fileExtents) + + val snapshot = latestSnapshot(tablePath) + + def filesHit( + qMinX: Double, + qMinY: Double, + qMaxX: Double, + qMaxY: Double): Int = { + val pred = new StGeometryBoxesIntersect( + new Column("geom"), + Literal.ofString(s"POINT ($qMinX $qMinY)"), + Literal.ofString(s"POINT ($qMaxX $qMaxY)")) + collectScanFileRows( + snapshot.getScanBuilder().withFilter(pred).build()).size } + + // SW query - intersects only SW file + assert(filesHit(1.0, 1.0, 4.0, 4.0) == 1) + // NE query - intersects only NE file + assert(filesHit(8.0, 8.0, 11.0, 11.0) == 1) + // Center gap - intersects nothing + assert(filesHit(4.0, 4.0, 6.0, 6.0) == 0) + // West strip [1,1]-[4,9] - intersects SW and NW + assert(filesHit(1.0, 1.0, 4.0, 9.0) == 2) + // Top strip [0,8]-[11,11] - intersects NE and NW + assert(filesHit(0.0, 8.0, 11.0, 11.0) == 2) + // Global - intersects all 4 files + assert(filesHit(0.0, 0.0, 11.0, 11.0) == 4) } + } + + test("StGeometryBoxesIntersect: file with null geometry stats is never skipped") { + withTempDirAndEngine { (tablePath, engine) => + val schema = new StructType().add("geom", colType) + val geomCol = new Column("geom") + + // File 0: SW quadrant with geometry stats + // File 1: no geometry stats -> must always be included + val extentsOpt: Seq[Option[(Double, Double, Double, Double)]] = + Seq(Some((0.0, 0.0, 3.0, 3.0)), None) + + extentsOpt.zipWithIndex.foreach { + case (extentOpt, idx) => + val txn = + if (idx == 0) + getCreateTxn(engine, tablePath, schema) + else getUpdateTxn(engine, tablePath) + + val txnState = txn.getTransactionState(engine) + val writeContext = Transaction.getWriteContext( + engine, + txnState, + Collections.emptyMap()) + + val stats = extentOpt match { + case Some((minX, minY, maxX, maxY)) => + new DataFileStatistics( + 10, + Map(geomCol -> Literal.ofString( + s"POINT ($minX $minY)")).asJava, + Map(geomCol -> Literal.ofString( + s"POINT ($maxX $maxY)")).asJava, + Map(geomCol -> (0L: java.lang.Long)).asJava, + Optional.empty()) + case None => + new DataFileStatistics( + 10, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Optional.empty()) + } - test(s"StIntersectsBoxes: file with null $typeName stats is never skipped") { - withTempDirAndEngine { (tablePath, engine) => - val schema = new StructType().add("geom", colType) - val geomCol = new Column("geom") - - // File 0: SW quadrant with geometry stats - // File 1: no geometry stats -> must always be included - val extentsOpt: Seq[Option[(Double, Double, Double, Double)]] = - Seq(Some((0.0, 0.0, 3.0, 3.0)), None) - - extentsOpt.zipWithIndex.foreach { - case (extentOpt, idx) => - val txn = - if (idx == 0) - getCreateTxn(engine, tablePath, schema) - else getUpdateTxn(engine, tablePath) - - val txnState = txn.getTransactionState(engine) - val writeContext = Transaction.getWriteContext( - engine, - txnState, - Collections.emptyMap()) - - val stats = extentOpt match { - case Some((minX, minY, maxX, maxY)) => - new DataFileStatistics( - 10, - Map(geomCol -> Literal.ofString( - s"POINT ($minX $minY)")).asJava, - Map(geomCol -> Literal.ofString( - s"POINT ($maxX $maxY)")).asJava, - Map(geomCol -> (0L: java.lang.Long)).asJava, - Optional.empty()) - case None => - new DataFileStatistics( - 10, - Collections.emptyMap(), - Collections.emptyMap(), - Collections.emptyMap(), - Optional.empty()) - } - - val filePath = - engine.getFileSystemClient.resolvePath( - writeContext.getTargetDirectory + s"/part-$idx.parquet") - val fileStatus = new DataFileStatus( - filePath, - 1000, - 0L, - Optional.of(stats)) - - val actions = Transaction.generateAppendActions( - engine, - txnState, - toCloseableIterator(Seq(fileStatus).iterator.asJava), - writeContext) - commitTransaction( - txn, - engine, - inMemoryIterable(actions)) - } - - val snapshot = latestSnapshot(tablePath) - - def filesHit( - qMinX: Double, - qMinY: Double, - qMaxX: Double, - qMaxY: Double): Int = { - val pred = new StIntersectsBoxes( - geomCol, - Literal.ofString(s"POINT ($qMinX $qMinY)"), - Literal.ofString(s"POINT ($qMaxX $qMaxY)")) - collectScanFileRows( - snapshot.getScanBuilder().withFilter(pred).build()).size - } - - // NE query: SW file is provably non-overlapping, but - // null-stats file must be included - assert(filesHit(8.0, 8.0, 11.0, 11.0) == 1) - // SW query: SW file matches, null-stats file also included - assert(filesHit(1.0, 1.0, 4.0, 4.0) == 2) + val filePath = + engine.getFileSystemClient.resolvePath( + writeContext.getTargetDirectory + s"/part-$idx.parquet") + val fileStatus = new DataFileStatus( + filePath, + 1000, + 0L, + Optional.of(stats)) + + val actions = Transaction.generateAppendActions( + engine, + txnState, + toCloseableIterator(Seq(fileStatus).iterator.asJava), + writeContext) + commitTransaction( + txn, + engine, + inMemoryIterable(actions)) } + + val snapshot = latestSnapshot(tablePath) + + def filesHit( + qMinX: Double, + qMinY: Double, + qMaxX: Double, + qMaxY: Double): Int = { + val pred = new StGeometryBoxesIntersect( + geomCol, + Literal.ofString(s"POINT ($qMinX $qMinY)"), + Literal.ofString(s"POINT ($qMaxX $qMaxY)")) + collectScanFileRows( + snapshot.getScanBuilder().withFilter(pred).build()).size + } + + // NE query: SW file is provably non-overlapping, but + // null-stats file must be included + assert(filesHit(8.0, 8.0, 11.0, 11.0) == 1) + // SW query: SW file matches, null-stats file also included + assert(filesHit(1.0, 1.0, 4.0, 4.0) == 2) } } - test("StIntersectsBoxes combined with AND predicate on second column") { + test("StGeometryBoxesIntersect combined with AND predicate on second column") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType() .add("id", io.delta.kernel.types.IntegerType.INTEGER) @@ -255,7 +249,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { // SW geometry + id <= 5: only File 0 survives both filters val geoAndId = new io.delta.kernel.expressions.And( - new StIntersectsBoxes( + new StGeometryBoxesIntersect( geomCol, Literal.ofString("POINT (1.0 1.0)"), Literal.ofString("POINT (4.0 4.0)")), @@ -270,7 +264,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { // NE geometry + id >= 15: only File 1 survives both filters val geoAndId2 = new io.delta.kernel.expressions.And( - new StIntersectsBoxes( + new StGeometryBoxesIntersect( geomCol, Literal.ofString("POINT (8.0 8.0)"), Literal.ofString("POINT (11.0 11.0)")), @@ -284,7 +278,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { assert(files2.size == 1) // Center geometry + any id: geometry miss skips both files - val geoCenterAny = new StIntersectsBoxes( + val geoCenterAny = new StGeometryBoxesIntersect( geomCol, Literal.ofString("POINT (4.0 4.0)"), Literal.ofString("POINT (6.0 6.0)")) From 9f1e274e4531fcf856792850965dd14db4d7e44c Mon Sep 17 00:00:00 2001 From: emu Date: Tue, 24 Mar 2026 14:28:18 +0000 Subject: [PATCH 79/80] fix --- ...esIntersect.java => StBoxesIntersect.java} | 29 ++++---- .../internal/skipping/DataSkippingUtils.java | 8 +-- .../data/vector/DefaultGenericVector.java | 7 +- .../DefaultExpressionEvaluator.java | 35 +++++----- .../expressions/ExpressionVisitor.java | 7 +- .../defaults/GeometryDataSkippingSuite.scala | 70 ++++++++++--------- 6 files changed, 83 insertions(+), 73 deletions(-) rename kernel/kernel-api/src/main/java/io/delta/kernel/expressions/{StGeometryBoxesIntersect.java => StBoxesIntersect.java} (60%) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StBoxesIntersect.java similarity index 60% rename from kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java rename to kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StBoxesIntersect.java index 41e412e314a..aa8b1fadb42 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StBoxesIntersect.java @@ -1,5 +1,5 @@ /* - * Copyright (2023) The Delta Lake Project Authors. + * Copyright (2026) The Delta Lake Project Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,8 @@ import static io.delta.kernel.internal.util.Preconditions.checkArgument; -import io.delta.kernel.types.StringType; +import io.delta.kernel.types.GeographyType; +import io.delta.kernel.types.GeometryType; import java.util.Arrays; /** @@ -25,29 +26,31 @@ * bounding box. * *

    This expression is not directly evaluatable. It is transformed to an internal - * ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS predicate during data-skipping filter construction, which - * evaluates the intersection of the column's min/max bounding box against the query box. + * ST_BOXES_INTERSECT_ON_STATS predicate during data-skipping filter construction, which evaluates + * the intersection of the column's min/max bounding box against the query box. * * @since 4.0.0 */ -public class StGeometryBoxesIntersect extends Predicate { +public class StBoxesIntersect extends Predicate { - public static final String NAME = "ST_GEOMETRY_BOXES_INTERSECT"; + public static final String NAME = "ST_BOXES_INTERSECT"; /** * @param column the geometry column to test - * @param queryMin lower-left corner of the query bounding box as a WKT POINT string literal - * @param queryMax upper-right corner of the query bounding box as a WKT POINT string literal + * @param queryMin lower-left corner of the query bounding box as a geospatial WKT POINT literal + * @param queryMax upper-right corner of the query bounding box as a geospatial WKT POINT literal */ - public StGeometryBoxesIntersect(Column column, Literal queryMin, Literal queryMax) { + public StBoxesIntersect(Column column, Literal queryMin, Literal queryMax) { super(NAME, Arrays.asList(column, queryMin, queryMax)); checkArgument( - queryMin.getDataType() instanceof StringType, - "queryMin must be a StringType WKT literal, got: %s", + queryMin.getDataType() instanceof GeometryType + || queryMin.getDataType() instanceof GeographyType, + "queryMin must be a GeometryType or GeographyType WKT literal, got: %s", queryMin.getDataType()); checkArgument( - queryMax.getDataType() instanceof StringType, - "queryMax must be a StringType WKT literal, got: %s", + queryMax.getDataType() instanceof GeometryType + || queryMax.getDataType() instanceof GeographyType, + "queryMax must be a GeometryType or GeographyType WKT literal, got: %s", queryMax.getDataType()); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java index 5ea124b8e26..bbb6efef566 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java @@ -286,9 +286,9 @@ private static Optional constructDataSkippingFilter( return constructNotDataSkippingFilters( asPredicate(getUnaryChild(dataFilters)), schemaHelper); - case StGeometryBoxesIntersect.NAME: - if (dataFilters instanceof StGeometryBoxesIntersect) { - StGeometryBoxesIntersect stExpr = (StGeometryBoxesIntersect) dataFilters; + case StBoxesIntersect.NAME: + if (dataFilters instanceof StBoxesIntersect) { + StBoxesIntersect stExpr = (StBoxesIntersect) dataFilters; Column col = stExpr.getColumn(); if (schemaHelper.isSkippingEligibleMinMaxColumn(col)) { Column minCol = schemaHelper.getMinColumn(col, Optional.empty())._1; @@ -304,7 +304,7 @@ private static Optional constructDataSkippingFilter( }; return Optional.of( new DataSkippingPredicate( - "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS", + "ST_BOXES_INTERSECT_ON_STATS", Arrays.asList(minCol, maxCol, queryMin, queryMax), refCols)); } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java index 132e6cd66d6..fcb3ae4889d 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java @@ -117,7 +117,12 @@ public double getDouble(int rowId) { @Override public String getString(int rowId) { assertValidRowId(rowId); - throwIfUnsafeAccess(StringType.class, "string"); + if (!(dataType instanceof StringType + || dataType instanceof GeometryType + || dataType instanceof GeographyType)) { + throw new UnsupportedOperationException( + String.format("Trying to access a `string` value from vector of type `%s`", dataType)); + } return (String) rowIdToValueAccessor.apply(rowId); } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index ff70275d731..9e13cf1dd80 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -363,36 +363,31 @@ ExpressionTransformResult visitIn(In in) { } @Override - ExpressionTransformResult visitStGeometryBoxesIntersectsOnStats(Predicate predicate) { + ExpressionTransformResult visitStBoxesIntersectOnStats(Predicate predicate) { List children = predicate.getChildren().stream().map(this::visit).collect(Collectors.toList()); checkArgument( children.size() == 4, - "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS expects 4 children but got %d", + "ST_BOXES_INTERSECT_ON_STATS expects 4 children but got %d", children.size()); - // Children 0,1 are stats columns (must be GeometryType). - // Children 2,3 are WKT query literals (must be StringType). + // All 4 children must be geometry/geography type. + // Children 0,1 are stats columns, children 2,3 are query literals. DataType child0Type = children.get(0).outputType; checkArgument( - child0Type instanceof GeometryType, - "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS child 0 must be " + "geometry type, got %s", + child0Type instanceof GeometryType || child0Type instanceof GeographyType, + "ST_BOXES_INTERSECT_ON_STATS child 0 must be " + "geometry/geography type, got %s", child0Type); - checkArgument( - child0Type.equals(children.get(1).outputType), - "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS child 1 type %s " - + "doesn't match child 0 type %s", - children.get(1).outputType, - child0Type); - for (int i = 2; i < 4; i++) { + for (int i = 1; i < 4; i++) { checkArgument( - children.get(i).outputType instanceof StringType, - "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS child %d must be " + "string (WKT), got %s", + child0Type.equals(children.get(i).outputType), + "ST_BOXES_INTERSECT_ON_STATS child %d type %s " + "doesn't match child 0 type %s", i, - children.get(i).outputType); + children.get(i).outputType, + child0Type); } return new ExpressionTransformResult( new Predicate( - "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS", + "ST_BOXES_INTERSECT_ON_STATS", children.stream().map(c -> c.expression).collect(Collectors.toList())), BooleanType.BOOLEAN); } @@ -590,7 +585,9 @@ ColumnVector visitLiteral(Literal literal) { || dataType instanceof DecimalType || dataType instanceof DateType || dataType instanceof TimestampType - || dataType instanceof TimestampNTZType) { + || dataType instanceof TimestampNTZType + || dataType instanceof GeometryType + || dataType instanceof GeographyType) { return new DefaultConstantVector(dataType, input.getSize(), literal.getValue()); } @@ -784,7 +781,7 @@ ColumnVector visitIn(In in) { } @Override - ColumnVector visitStGeometryBoxesIntersectsOnStats(Predicate predicate) { + ColumnVector visitStBoxesIntersectOnStats(Predicate predicate) { List children = predicate.getChildren(); ColumnVector leftMin = visit(children.get(0)); ColumnVector leftMax = visit(children.get(1)); diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java index 21bfb8a1824..4ddf4722783 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java @@ -74,7 +74,7 @@ abstract class ExpressionVisitor { abstract R visitIn(In in); - abstract R visitStGeometryBoxesIntersectsOnStats(Predicate predicate); + abstract R visitStBoxesIntersectOnStats(Predicate predicate); final R visit(Expression expression) { if (expression instanceof PartitionValueExpression) { @@ -146,9 +146,8 @@ private R visitScalarExpression(ScalarExpression expression) { } else { return visitIn(new In(children.get(0), children.subList(1, children.size()))); } - case "ST_GEOMETRY_BOXES_INTERSECTS_ON_STATS": - return visitStGeometryBoxesIntersectsOnStats( - createPredicate(name, children, collationIdentifier)); + case "ST_BOXES_INTERSECT_ON_STATS": + return visitStBoxesIntersectOnStats(createPredicate(name, children, collationIdentifier)); default: throw new UnsupportedOperationException( String.format("Scalar expression `%s` is not supported.", name)); diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala index 86c7c5226cf..4df00c86bd5 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala @@ -22,7 +22,7 @@ import scala.collection.immutable.Seq import io.delta.kernel._ import io.delta.kernel.defaults.utils.WriteUtils -import io.delta.kernel.expressions.{Column, Literal, StGeometryBoxesIntersect} +import io.delta.kernel.expressions.{Column, Literal, StBoxesIntersect} import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.statistics.DataFileStatistics import io.delta.kernel.types.{GeometryType, StructType} @@ -33,7 +33,7 @@ import org.scalatest.funsuite.AnyFunSuite /** * End-to-end tests for data skipping with geometry column types using - * the StGeometryBoxesIntersect predicate. + * the StBoxesIntersect predicate. * * Tests bypass the Parquet writer by injecting DataFileStatistics with * geometry min/max literals directly into the Delta log. @@ -58,7 +58,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { private val colType = new GeometryType("OGC:CRS84") - test("StGeometryBoxesIntersect data skipping on GeometryType column") { + test("StBoxesIntersect data skipping on GeometryType column") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType().add("geom", colType) writeFilesWithGeometryStats( @@ -74,10 +74,10 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { qMinY: Double, qMaxX: Double, qMaxY: Double): Int = { - val pred = new StGeometryBoxesIntersect( + val pred = new StBoxesIntersect( new Column("geom"), - Literal.ofString(s"POINT ($qMinX $qMinY)"), - Literal.ofString(s"POINT ($qMaxX $qMaxY)")) + Literal.ofGeospatialWKT(s"POINT ($qMinX $qMinY)", colType), + Literal.ofGeospatialWKT(s"POINT ($qMaxX $qMaxY)", colType)) collectScanFileRows( snapshot.getScanBuilder().withFilter(pred).build()).size } @@ -97,7 +97,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { } } - test("StGeometryBoxesIntersect: file with null geometry stats is never skipped") { + test("StBoxesIntersect: file with null geometry stats is never skipped") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType().add("geom", colType) val geomCol = new Column("geom") @@ -124,10 +124,12 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { case Some((minX, minY, maxX, maxY)) => new DataFileStatistics( 10, - Map(geomCol -> Literal.ofString( - s"POINT ($minX $minY)")).asJava, - Map(geomCol -> Literal.ofString( - s"POINT ($maxX $maxY)")).asJava, + Map(geomCol -> Literal.ofGeospatialWKT( + s"POINT ($minX $minY)", + colType)).asJava, + Map(geomCol -> Literal.ofGeospatialWKT( + s"POINT ($maxX $maxY)", + colType)).asJava, Map(geomCol -> (0L: java.lang.Long)).asJava, Optional.empty()) case None => @@ -166,10 +168,10 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { qMinY: Double, qMaxX: Double, qMaxY: Double): Int = { - val pred = new StGeometryBoxesIntersect( + val pred = new StBoxesIntersect( geomCol, - Literal.ofString(s"POINT ($qMinX $qMinY)"), - Literal.ofString(s"POINT ($qMaxX $qMaxY)")) + Literal.ofGeospatialWKT(s"POINT ($qMinX $qMinY)", colType), + Literal.ofGeospatialWKT(s"POINT ($qMaxX $qMaxY)", colType)) collectScanFileRows( snapshot.getScanBuilder().withFilter(pred).build()).size } @@ -182,7 +184,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { } } - test("StGeometryBoxesIntersect combined with AND predicate on second column") { + test("StBoxesIntersect combined with AND predicate on second column") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType() .add("id", io.delta.kernel.types.IntegerType.INTEGER) @@ -214,12 +216,14 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { 10, Map( idCol -> Literal.ofInt(idMin), - geomCol -> Literal.ofString( - s"POINT ($gMinX $gMinY)")).asJava, + geomCol -> Literal.ofGeospatialWKT( + s"POINT ($gMinX $gMinY)", + colType)).asJava, Map( idCol -> Literal.ofInt(idMax), - geomCol -> Literal.ofString( - s"POINT ($gMaxX $gMaxY)")).asJava, + geomCol -> Literal.ofGeospatialWKT( + s"POINT ($gMaxX $gMaxY)", + colType)).asJava, Map( idCol -> (0L: java.lang.Long), geomCol -> (0L: java.lang.Long)).asJava, @@ -249,10 +253,10 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { // SW geometry + id <= 5: only File 0 survives both filters val geoAndId = new io.delta.kernel.expressions.And( - new StGeometryBoxesIntersect( + new StBoxesIntersect( geomCol, - Literal.ofString("POINT (1.0 1.0)"), - Literal.ofString("POINT (4.0 4.0)")), + Literal.ofGeospatialWKT("POINT (1.0 1.0)", colType), + Literal.ofGeospatialWKT("POINT (4.0 4.0)", colType)), new io.delta.kernel.expressions.Predicate( "<=", idCol, @@ -264,10 +268,10 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { // NE geometry + id >= 15: only File 1 survives both filters val geoAndId2 = new io.delta.kernel.expressions.And( - new StGeometryBoxesIntersect( + new StBoxesIntersect( geomCol, - Literal.ofString("POINT (8.0 8.0)"), - Literal.ofString("POINT (11.0 11.0)")), + Literal.ofGeospatialWKT("POINT (8.0 8.0)", colType), + Literal.ofGeospatialWKT("POINT (11.0 11.0)", colType)), new io.delta.kernel.expressions.Predicate( ">=", idCol, @@ -278,10 +282,10 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { assert(files2.size == 1) // Center geometry + any id: geometry miss skips both files - val geoCenterAny = new StGeometryBoxesIntersect( + val geoCenterAny = new StBoxesIntersect( geomCol, - Literal.ofString("POINT (4.0 4.0)"), - Literal.ofString("POINT (6.0 6.0)")) + Literal.ofGeospatialWKT("POINT (4.0 4.0)", colType), + Literal.ofGeospatialWKT("POINT (6.0 6.0)", colType)) val files3 = collectScanFileRows( snapshot.getScanBuilder().withFilter(geoCenterAny).build()) @@ -310,10 +314,12 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { val stats = new DataFileStatistics( 10, - Map(geomCol -> Literal.ofString( - s"POINT ($minX $minY)")).asJava, - Map(geomCol -> Literal.ofString( - s"POINT ($maxX $maxY)")).asJava, + Map(geomCol -> Literal.ofGeospatialWKT( + s"POINT ($minX $minY)", + colType)).asJava, + Map(geomCol -> Literal.ofGeospatialWKT( + s"POINT ($maxX $maxY)", + colType)).asJava, Map(geomCol -> (0L: java.lang.Long)).asJava, Optional.empty()) From 04dd0647498bb12946be9e6d43add557995c0fe9 Mon Sep 17 00:00:00 2001 From: emu Date: Tue, 24 Mar 2026 15:27:47 +0000 Subject: [PATCH 80/80] fixing --- ...ect.java => StGeometryBoxesIntersect.java} | 26 +++++----- .../internal/skipping/DataSkippingUtils.java | 8 ++-- .../DefaultExpressionEvaluator.java | 17 +++---- .../expressions/ExpressionVisitor.java | 7 +-- .../defaults/GeometryDataSkippingSuite.scala | 47 ++++++++++--------- 5 files changed, 54 insertions(+), 51 deletions(-) rename kernel/kernel-api/src/main/java/io/delta/kernel/expressions/{StBoxesIntersect.java => StGeometryBoxesIntersect.java} (64%) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StBoxesIntersect.java b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java similarity index 64% rename from kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StBoxesIntersect.java rename to kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java index aa8b1fadb42..afceea94e38 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StBoxesIntersect.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java @@ -17,7 +17,6 @@ import static io.delta.kernel.internal.util.Preconditions.checkArgument; -import io.delta.kernel.types.GeographyType; import io.delta.kernel.types.GeometryType; import java.util.Arrays; @@ -26,31 +25,30 @@ * bounding box. * *

    This expression is not directly evaluatable. It is transformed to an internal - * ST_BOXES_INTERSECT_ON_STATS predicate during data-skipping filter construction, which evaluates - * the intersection of the column's min/max bounding box against the query box. + * ST_GEOMETRY_BOXES_INTERSECT_ON_STATS predicate during data-skipping filter construction, which + * evaluates the intersection of the column's min/max bounding box against the query box. * * @since 4.0.0 */ -public class StBoxesIntersect extends Predicate { +public class StGeometryBoxesIntersect extends Predicate { - public static final String NAME = "ST_BOXES_INTERSECT"; + public static final String NAME = "ST_GEOMETRY_BOXES_INTERSECT"; /** * @param column the geometry column to test - * @param queryMin lower-left corner of the query bounding box as a geospatial WKT POINT literal - * @param queryMax upper-right corner of the query bounding box as a geospatial WKT POINT literal + * @param queryMin lower-left corner of the query bounding box as a GeometryType WKT POINT literal + * @param queryMax upper-right corner of the query bounding box as a GeometryType WKT POINT + * literal */ - public StBoxesIntersect(Column column, Literal queryMin, Literal queryMax) { + public StGeometryBoxesIntersect(Column column, Literal queryMin, Literal queryMax) { super(NAME, Arrays.asList(column, queryMin, queryMax)); checkArgument( - queryMin.getDataType() instanceof GeometryType - || queryMin.getDataType() instanceof GeographyType, - "queryMin must be a GeometryType or GeographyType WKT literal, got: %s", + queryMin.getDataType() instanceof GeometryType, + "queryMin must be a GeometryType WKT literal, got: %s", queryMin.getDataType()); checkArgument( - queryMax.getDataType() instanceof GeometryType - || queryMax.getDataType() instanceof GeographyType, - "queryMax must be a GeometryType or GeographyType WKT literal, got: %s", + queryMax.getDataType() instanceof GeometryType, + "queryMax must be a GeometryType WKT literal, got: %s", queryMax.getDataType()); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java index bbb6efef566..37a644d1091 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java @@ -286,9 +286,9 @@ private static Optional constructDataSkippingFilter( return constructNotDataSkippingFilters( asPredicate(getUnaryChild(dataFilters)), schemaHelper); - case StBoxesIntersect.NAME: - if (dataFilters instanceof StBoxesIntersect) { - StBoxesIntersect stExpr = (StBoxesIntersect) dataFilters; + case StGeometryBoxesIntersect.NAME: + if (dataFilters instanceof StGeometryBoxesIntersect) { + StGeometryBoxesIntersect stExpr = (StGeometryBoxesIntersect) dataFilters; Column col = stExpr.getColumn(); if (schemaHelper.isSkippingEligibleMinMaxColumn(col)) { Column minCol = schemaHelper.getMinColumn(col, Optional.empty())._1; @@ -304,7 +304,7 @@ private static Optional constructDataSkippingFilter( }; return Optional.of( new DataSkippingPredicate( - "ST_BOXES_INTERSECT_ON_STATS", + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS", Arrays.asList(minCol, maxCol, queryMin, queryMax), refCols)); } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index 9e13cf1dd80..f886bddb05f 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -363,31 +363,32 @@ ExpressionTransformResult visitIn(In in) { } @Override - ExpressionTransformResult visitStBoxesIntersectOnStats(Predicate predicate) { + ExpressionTransformResult visitStGeometryBoxesIntersectOnStats(Predicate predicate) { List children = predicate.getChildren().stream().map(this::visit).collect(Collectors.toList()); checkArgument( children.size() == 4, - "ST_BOXES_INTERSECT_ON_STATS expects 4 children but got %d", + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS expects 4 children but got %d", children.size()); - // All 4 children must be geometry/geography type. + // All 4 children must be GeometryType. // Children 0,1 are stats columns, children 2,3 are query literals. DataType child0Type = children.get(0).outputType; checkArgument( - child0Type instanceof GeometryType || child0Type instanceof GeographyType, - "ST_BOXES_INTERSECT_ON_STATS child 0 must be " + "geometry/geography type, got %s", + child0Type instanceof GeometryType, + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS child 0 must be " + "geometry type, got %s", child0Type); for (int i = 1; i < 4; i++) { checkArgument( child0Type.equals(children.get(i).outputType), - "ST_BOXES_INTERSECT_ON_STATS child %d type %s " + "doesn't match child 0 type %s", + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS child %d type %s " + + "doesn't match child 0 type %s", i, children.get(i).outputType, child0Type); } return new ExpressionTransformResult( new Predicate( - "ST_BOXES_INTERSECT_ON_STATS", + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS", children.stream().map(c -> c.expression).collect(Collectors.toList())), BooleanType.BOOLEAN); } @@ -781,7 +782,7 @@ ColumnVector visitIn(In in) { } @Override - ColumnVector visitStBoxesIntersectOnStats(Predicate predicate) { + ColumnVector visitStGeometryBoxesIntersectOnStats(Predicate predicate) { List children = predicate.getChildren(); ColumnVector leftMin = visit(children.get(0)); ColumnVector leftMax = visit(children.get(1)); diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java index 4ddf4722783..3daac18409f 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java @@ -74,7 +74,7 @@ abstract class ExpressionVisitor { abstract R visitIn(In in); - abstract R visitStBoxesIntersectOnStats(Predicate predicate); + abstract R visitStGeometryBoxesIntersectOnStats(Predicate predicate); final R visit(Expression expression) { if (expression instanceof PartitionValueExpression) { @@ -146,8 +146,9 @@ private R visitScalarExpression(ScalarExpression expression) { } else { return visitIn(new In(children.get(0), children.subList(1, children.size()))); } - case "ST_BOXES_INTERSECT_ON_STATS": - return visitStBoxesIntersectOnStats(createPredicate(name, children, collationIdentifier)); + case "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS": + return visitStGeometryBoxesIntersectOnStats( + createPredicate(name, children, collationIdentifier)); default: throw new UnsupportedOperationException( String.format("Scalar expression `%s` is not supported.", name)); diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala index 4df00c86bd5..6fbad7b3bf9 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala @@ -22,7 +22,7 @@ import scala.collection.immutable.Seq import io.delta.kernel._ import io.delta.kernel.defaults.utils.WriteUtils -import io.delta.kernel.expressions.{Column, Literal, StBoxesIntersect} +import io.delta.kernel.expressions.{Column, Literal, StGeometryBoxesIntersect} import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.statistics.DataFileStatistics import io.delta.kernel.types.{GeometryType, StructType} @@ -33,7 +33,7 @@ import org.scalatest.funsuite.AnyFunSuite /** * End-to-end tests for data skipping with geometry column types using - * the StBoxesIntersect predicate. + * the StGeometryBoxesIntersect predicate. * * Tests bypass the Parquet writer by injecting DataFileStatistics with * geometry min/max literals directly into the Delta log. @@ -58,7 +58,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { private val colType = new GeometryType("OGC:CRS84") - test("StBoxesIntersect data skipping on GeometryType column") { + test("StGeometryBoxesIntersect data skipping on GeometryType column") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType().add("geom", colType) writeFilesWithGeometryStats( @@ -74,7 +74,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { qMinY: Double, qMaxX: Double, qMaxY: Double): Int = { - val pred = new StBoxesIntersect( + val pred = new StGeometryBoxesIntersect( new Column("geom"), Literal.ofGeospatialWKT(s"POINT ($qMinX $qMinY)", colType), Literal.ofGeospatialWKT(s"POINT ($qMaxX $qMaxY)", colType)) @@ -97,7 +97,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { } } - test("StBoxesIntersect: file with null geometry stats is never skipped") { + test("StGeometryBoxesIntersect: file with null geometry stats is never skipped") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType().add("geom", colType) val geomCol = new Column("geom") @@ -109,10 +109,11 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { extentsOpt.zipWithIndex.foreach { case (extentOpt, idx) => - val txn = - if (idx == 0) - getCreateTxn(engine, tablePath, schema) - else getUpdateTxn(engine, tablePath) + val txn = if (idx == 0) { + getCreateTxn(engine, tablePath, schema) + } else { + getUpdateTxn(engine, tablePath) + } val txnState = txn.getTransactionState(engine) val writeContext = Transaction.getWriteContext( @@ -168,7 +169,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { qMinY: Double, qMaxX: Double, qMaxY: Double): Int = { - val pred = new StBoxesIntersect( + val pred = new StGeometryBoxesIntersect( geomCol, Literal.ofGeospatialWKT(s"POINT ($qMinX $qMinY)", colType), Literal.ofGeospatialWKT(s"POINT ($qMaxX $qMaxY)", colType)) @@ -184,7 +185,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { } } - test("StBoxesIntersect combined with AND predicate on second column") { + test("StGeometryBoxesIntersect combined with AND predicate on second column") { withTempDirAndEngine { (tablePath, engine) => val schema = new StructType() .add("id", io.delta.kernel.types.IntegerType.INTEGER) @@ -201,10 +202,11 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { files.zipWithIndex.foreach { case ((idMin, idMax, gMinX, gMinY, gMaxX, gMaxY), idx) => - val txn = - if (idx == 0) - getCreateTxn(engine, tablePath, schema) - else getUpdateTxn(engine, tablePath) + val txn = if (idx == 0) { + getCreateTxn(engine, tablePath, schema) + } else { + getUpdateTxn(engine, tablePath) + } val txnState = txn.getTransactionState(engine) val writeContext = Transaction.getWriteContext( @@ -253,7 +255,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { // SW geometry + id <= 5: only File 0 survives both filters val geoAndId = new io.delta.kernel.expressions.And( - new StBoxesIntersect( + new StGeometryBoxesIntersect( geomCol, Literal.ofGeospatialWKT("POINT (1.0 1.0)", colType), Literal.ofGeospatialWKT("POINT (4.0 4.0)", colType)), @@ -268,7 +270,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { // NE geometry + id >= 15: only File 1 survives both filters val geoAndId2 = new io.delta.kernel.expressions.And( - new StBoxesIntersect( + new StGeometryBoxesIntersect( geomCol, Literal.ofGeospatialWKT("POINT (8.0 8.0)", colType), Literal.ofGeospatialWKT("POINT (11.0 11.0)", colType)), @@ -282,7 +284,7 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { assert(files2.size == 1) // Center geometry + any id: geometry miss skips both files - val geoCenterAny = new StBoxesIntersect( + val geoCenterAny = new StGeometryBoxesIntersect( geomCol, Literal.ofGeospatialWKT("POINT (4.0 4.0)", colType), Literal.ofGeospatialWKT("POINT (6.0 6.0)", colType)) @@ -301,10 +303,11 @@ class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { val geomCol = new Column("geom") extents.zipWithIndex.foreach { case ((minX, minY, maxX, maxY), idx) => - val txn = - if (idx == 0) - getCreateTxn(engine, tablePath, schema) - else getUpdateTxn(engine, tablePath) + val txn = if (idx == 0) { + getCreateTxn(engine, tablePath, schema) + } else { + getUpdateTxn(engine, tablePath) + } val txnState = txn.getTransactionState(engine) val writeContext = Transaction.getWriteContext(