From fe68ba8fe21c1de6e4c24d54d235e6602f3d0153 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi <45303303+OussamaSaoudi@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:20:14 +0000 Subject: [PATCH 1/2] Add Delta acceptance workloads for boundary cases and table operations --- .../tables/ReadinessCheckpointSuite.scala | 559 +++++ .../tables/ReadinessColumnMappingSuite.scala | 403 ++++ .../tables/ReadinessPredicatesSuite.scala | 779 +++++++ .../ReadinessSchemaReplacementSuite.scala | 494 +++++ .../tables/ReadinessStatisticsSuite.scala | 1011 +++++++++ .../ReadinessTableOperationsSuite.scala | 1354 ++++++++++++ .../tables/ReadinessTimestampSuite.scala | 1942 +++++++++++++++++ .../tables/ReadinessTypeWideningSuite.scala | 278 +++ 8 files changed, 6820 insertions(+) create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessCheckpointSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessColumnMappingSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessPredicatesSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessSchemaReplacementSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessStatisticsSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTableOperationsSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTimestampSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTypeWideningSuite.scala diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessCheckpointSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessCheckpointSuite.scala new file mode 100644 index 00000000..b3724720 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessCheckpointSuite.scala @@ -0,0 +1,559 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.spark.sql.Row +import org.apache.spark.sql.functions.{col, lit, map => sparkMap, when} +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.WorkloadTestSuite +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{Action, AddFile, CheckpointMetadata, CommitLog, Metadata, Protocol, + SidecarFile} +import io.delta.workload.model._ + +class ReadinessCheckpointSuite extends WorkloadTestSuite("readiness_checkpoint") { + + private def snapshotResult(protocol: Protocol, metadata: Metadata): SnapshotResult = + SnapshotResult( + ProtocolInfo(protocol.minReaderVersion, protocol.minWriterVersion, + protocol.readerFeatures.map(_.toSeq.sorted), + protocol.writerFeatures.map(_.toSeq.sorted)), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat(metadata.format.provider, metadata.format.options), + metadata.schemaString, metadata.partitionColumns, metadata.configuration, + metadata.createdTime)) + + private def assertReadRows( + spec: ReadSpec, + tablePath: Path, + schema: StructType, + rows: Seq[Row], + name: String): Unit = { + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, tablePath, + spec.query.version, spec.query.timestamp), spec.query.predicate, spec.query.columns) + assert(actual.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, schema), actual, name) + } + + Seq( + ("json_inline", "json", "v2", 1000, 0), + ("json_manifest", "json", "v2", 1000, 1), + ("parquet_single_part", "parquet", "classic", 1000, 0), + ("parquet_manifest", "parquet", "v2", 1000, 1), + ("parquet_sidecars", "parquet", "v2", 1, 2) + ).foreach { case (name, format, policy, partSize, sidecarCount) => + test(name) { + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> format, + "spark.databricks.delta.checkpoint.partSize" -> partSize.toString) + val previous = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap + settings.foreach { case (key, value) => spark.conf.set(key, value) } + try { + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 'a'), (2, CAST(NULL AS STRING)), (2, CAST(NULL AS STRING)) AS v(id, value)""") + sql("INSERT INTO tbl VALUES (3, 'c')") + forceCheckpoint("tbl") + sql("INSERT INTO tbl VALUES (4, 'd')") + val t = registerTable("tbl") + // Read the original commit actions before mutating the copied fixture. + val actions = (0L to 2L).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 2) + info("Original protocol: " + protocol.toJson) + info("Original metadata: " + metadata.toJson) + adds.foreach(a => info("Original add: " + a.toJson)) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + val expectedSnapshot = snapshotResult(protocol, metadata) + val checkpointRows = Seq(Row(1, "a"), Row(2, null), Row(2, null), Row(3, "c")) + val latestRows = checkpointRows :+ Row(4, "d") + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + assert(checkpoints.size == 1) + val checkpoint = checkpoints.head + if (policy == "classic") { + assert(checkpoint.getFileName.toString == "00000000000000000002.checkpoint.parquet") + val frame = spark.read.parquet(checkpoint.toString) + assert(frame.filter("add IS NOT NULL").count() == adds.size) + assert(frame.filter("metaData IS NOT NULL").count() == 1) + assert(frame.filter("protocol IS NOT NULL").count() == 1) + assert(!Files.exists(log.resolve("_sidecars"))) + } else { + assert(checkpoint.getFileName.toString.matches( + s"00000000000000000002\\.checkpoint\\.[^.]+\\.$format")) + if (name == "json_inline") { + // V2 allows inline file actions; this writer normally emits sidecars. + val checkpointAdds = adds.map(_.copy(dataChange = false)) + val inlineActions: Seq[Action] = + Seq(protocol, metadata, CheckpointMetadata(2L)) ++ checkpointAdds + Files.write(checkpoint, inlineActions.map(_.toJson).asJava, UTF_8) + val sidecars = Files.list(log.resolve("_sidecars")) + try sidecars.iterator().asScala.toVector.foreach(Files.delete) + finally sidecars.close() + Files.delete(log.resolve("_sidecars")) + val content = Files.readAllLines(checkpoint, UTF_8).asScala.map(Action.parse).toVector + assert(content.collect { case a: AddFile => a } == checkpointAdds) + assert(content.collect { case c: CheckpointMetadata => c.version } == Seq(2L)) + assert(!content.exists(_.isInstanceOf[SidecarFile])) + } else { + val manifest = if (format == "json") { + Files.readAllLines(checkpoint, UTF_8).asScala.map(Action.parse).toVector + } else { + spark.read.parquet(checkpoint.toString).toJSON.collect().toVector.map(Action.parse) + } + val references = manifest.collect { case s: SidecarFile => s } + assert(references.size == sidecarCount) + assert(!manifest.exists(_.isInstanceOf[AddFile])) + assert(manifest.collect { case c: CheckpointMetadata => c.version } == Seq(2L)) + assert(manifest.collect { case m: Metadata => m } == Seq(metadata)) + assert(manifest.collect { case p: Protocol => p } == Seq(protocol)) + val sidecarPaths = references.map { ref => + val path = log.resolve("_sidecars").resolve(ref.path) + assert(ref.path.endsWith(".parquet")) + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + path + } + val sidecarListing = Files.list(log.resolve("_sidecars")) + try assert(sidecarListing.iterator().asScala.toSet == sidecarPaths.toSet) + finally sidecarListing.close() + val content = spark.read.parquet(sidecarPaths.map(_.toString): _*) + assert(content.count() == adds.size) + assert(content.filter("add IS NOT NULL").count() == adds.size) + assert(content.filter("remove IS NOT NULL").count() == 0) + assert(content.select("add.path").collect().map(_.getString(0)).toSet == + adds.map(_.path).toSet) + } + } + // Keep same-version JSON commit 2 for the OSS historical snapshot lookup. + // This tests checkpoint plus commit 2, then version 3 tail replay. + // Checkpoint-only historical snapshots remain a separate failing repro. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 2L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + Seq( + ("read_all", ReadQuery(), latestRows, ReadResult(5L, 3, 0L)), + ("read_v2", ReadQuery(version = Some(2L)), checkpointRows, ReadResult(4L, 2, 0L)), + ("read_filtered", ReadQuery(predicate = Some("id >= 3")), + Seq(Row(3, "c"), Row(4, "d")), ReadResult(2L, 2, 1L)) + ).foreach { case (readName, query, rows, expected) => + readSpec(t, name = Some(readName), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, readName) + } + } + Seq(None, Some(2L)).foreach { version => + snapshotSpec(t, version = version.map(Long.box).orNull).assert { spec => + assert(spec.query.version.contains(version.getOrElse(3L))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + } finally { + previous.foreach { + case (key, Some(value)) => spark.conf.set(key, value) + case (key, None) => spark.conf.unset(key) + } + } + } + } + + // stats: OSS exposes structured checkpoint stats, but no product metadata-clustering + // control. These fixtures check shape and filtered correctness, never row groups READ. + Seq( + ("e2e_07_single_part", "classic", 1000, 1, 0), + ("e2e_07_multipart", "classic", 2, 3, 0), + ("e2e_07_sidecars", "v2", 2, 1, 3) + ).foreach { case (name, policy, partSize, checkpointCount, sidecarCount) => + test(name) { + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> "parquet", + "spark.databricks.delta.checkpoint.partSize" -> partSize.toString, + "spark.databricks.delta.statsAsStructInCheckpoint.forcedDisabled" -> "false") + val previous = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap + settings.foreach { case (key, value) => spark.conf.set(key, value) } + try { + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpoint.writeStatsAsStruct' = 'true')""") + val rowsByFile = (0 until 6).map { band => + val low = band * 10 + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + ($low, 'band_$band'), (${low + 1}, CAST(NULL AS STRING)), + (${low + 1}, CAST(NULL AS STRING)) AS v(id, value)""") + Seq(Row(low, s"band_$band"), Row(low + 1, null), Row(low + 1, null)) + } + forceCheckpoint("tbl") + val t = registerTable("tbl") + val actions = (0L to 6L).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 6 && adds.map(_.path).distinct.size == 6) + info("stats original protocol: " + protocol.toJson) + info("stats original metadata: " + metadata.toJson) + adds.foreach(a => info("stats original add: " + a.toJson)) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration("delta.checkpointPolicy") == policy) + assert(metadata.configuration("delta.checkpoint.writeStatsAsStruct") == "true") + val expectedSnapshot = snapshotResult(protocol, metadata) + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + assert(checkpoints.size == checkpointCount) + if (name == "e2e_07_single_part") { + assert(checkpoints.head.getFileName.toString == + "00000000000000000006.checkpoint.parquet") + } else if (policy == "classic") { + val expectedNames = (1 to 3).map { part => + f"00000000000000000006.checkpoint.$part%010d.0000000003.parquet" + }.toSet + assert(checkpoints.map(_.getFileName.toString).toSet == expectedNames) + } else { + assert(checkpoints.head.getFileName.toString.matches( + "00000000000000000006\\.checkpoint\\.[^.]+\\.parquet")) + } + val top = spark.read.parquet(checkpoints.map(_.toString): _*) + val topActions = top.toJSON.collect().toVector.map(Action.parse) + assert(topActions.collect { case m: Metadata => m } == Seq(metadata)) + assert(topActions.collect { case p: Protocol => p } == Seq(protocol)) + val sidecarPaths = if (policy == "v2") { + val refs = topActions.collect { case s: SidecarFile => s } + assert(refs.size == sidecarCount && refs.map(_.path).distinct.size == sidecarCount) + assert(topActions.collect { case c: CheckpointMetadata => c.version } == Seq(6L)) + assert(!topActions.exists(_.isInstanceOf[AddFile])) + val paths = refs.map { ref => + val path = log.resolve("_sidecars").resolve(ref.path) + assert(ref.path.endsWith(".parquet")) + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + path + } + val listing = Files.list(log.resolve("_sidecars")) + try assert(listing.iterator().asScala.toSet == paths.toSet) + finally listing.close() + paths + } else { + assert(!Files.exists(log.resolve("_sidecars"))) + Vector.empty[Path] + } + val fileActions = if (sidecarPaths.isEmpty) top else + spark.read.parquet(sidecarPaths.map(_.toString): _*) + assert(fileActions.filter("add IS NOT NULL").count() == 6) + assert(fileActions.filter("remove IS NOT NULL").count() == 0) + if (sidecarPaths.nonEmpty) assert(fileActions.count() == 6) + assert(fileActions.select("add.path").collect().filterNot(_.isNullAt(0)) + .map(_.getString(0)).toSet == adds.map(_.path).toSet) + val stats = fileActions.filter("add IS NOT NULL").select( + "add.stats_parsed.numRecords", "add.stats_parsed.minValues.id", + "add.stats_parsed.maxValues.id", "add.stats_parsed.nullCount.value").collect().toSeq + assert(stats.toSet == (0 until 6).map { band => + Row(3L, band * 10, band * 10 + 1, 2L) + }.toSet) + // Footer row groups are physical shape evidence, not reader IO counters. + val footerRows = (checkpoints ++ sidecarPaths).map { path => + val reader = ParquetFileReader.open(HadoopInputFile.fromPath( + new HadoopPath(path.toUri), spark.sparkContext.hadoopConfiguration)) + try { + val groups = reader.getFooter.getBlocks.asScala + info(s"stats footer: ${path.getFileName} groups=${groups.size} " + + s"rows=${groups.map(_.getRowCount).sum}") + assert(groups.size <= 1) + groups.map(_.getRowCount).sum + } finally reader.close() + } + assert(footerRows.sum == (if (policy == "v2") 12L else 8L)) + // Retain same-version JSON 6 for OSS version lookup, with no post-checkpoint tail. + // Missing commits 0..5 make the checkpoint required even though commit 6 exists. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 6L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + Seq( + ("read_all", None, rowsByFile.flatten, ReadResult(18L, 6, 0L)), + ("read_high", Some("id >= 40"), rowsByFile.drop(4).flatten, ReadResult(6L, 2, 4L)), + ("read_boundary", Some("id = 41"), Seq(Row(41, null), Row(41, null)), + ReadResult(2L, 1, 5L)), + ("read_empty", Some("id >= 100"), Seq.empty[Row], ReadResult(0L, 0, 6L)) + ).foreach { case (readName, predicate, rows, expected) => + readSpec(t, name = Some(readName), predicate = predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, readName) + } + } + snapshotSpec(t).assert { spec => + assert(spec.query.version.contains(6L)) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } finally { + previous.foreach { + case (key, Some(value)) => spark.conf.set(key, value) + case (key, None) => spark.conf.unset(key) + } + } + } + } + + // size: 64 KiB is a bounded fixture boundary, not a product dispatch threshold. + // These shapes assert portable read correctness, never distributed execution. + Seq("manifest", "json", "sidecars", "single_part", "multipart", "crc_all_files") + .foreach { shape => + test(s"e2e_43_$shape") { + val isCrc = shape == "crc_all_files" + val policy = if (Set("single_part", "multipart", "crc_all_files")(shape)) + "classic" else "v2" + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> "json", + "spark.databricks.delta.checkpoint.partSize" -> + (if (shape == "multipart") "2" else "1000"), + "spark.databricks.delta.allFilesInCrc.enabled" -> "true", + "spark.databricks.delta.allFilesInCrc.thresholdNumFiles" -> "100") + val previous = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap + settings.foreach { case (key, value) => spark.conf.set(key, value) } + try { + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false')""") + val rowsByFile = (0 until 4).map { band => + val low = band * 10 + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + ($low, 'band_$band'), (${low + 1}, CAST(NULL AS STRING)), + (${low + 1}, CAST(NULL AS STRING)) AS v(id, value)""") + Seq(Row(low, s"band_$band"), Row(low + 1, null), Row(low + 1, null)) + } + forceCheckpoint("tbl") + // A complete-file removal distinguishes active files from historical Adds in CRC. + if (isCrc) sql("DELETE FROM tbl WHERE id < 10") + sql("INSERT INTO tbl VALUES (99, 'tail')") + val t = registerTable("tbl") + val baseVersion = if (isCrc) 5L else 4L + val latestVersion = baseVersion + 1L + val actions = (0L to latestVersion).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 5 && adds.map(_.path).distinct.size == 5) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration("delta.checkpointPolicy") == policy) + assert(metadata.configuration("delta.enableDeletionVectors") == "false") + val expectedSnapshot = snapshotResult(protocol, metadata) + val baseRows = (if (isCrc) rowsByFile.drop(1) else rowsByFile).flatten + val latestRows = baseRows :+ Row(99, "tail") + info(s"size $shape protocol: " + protocol.toJson) + info(s"size $shape metadata: " + metadata.toJson) + adds.foreach(a => info(s"size $shape original add: " + a.toJson)) + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + .sortBy(_.getFileName.toString) + assert(checkpoints.size == (if (shape == "multipart") 2 else 1)) + if (policy == "classic") { + val names = if (shape == "multipart") (1 to 2).map { part => + f"00000000000000000004.checkpoint.$part%010d.0000000002.parquet" + }.toSet else Set("00000000000000000004.checkpoint.parquet") + assert(checkpoints.map(_.getFileName.toString).toSet == names) + } else { + assert(checkpoints.head.getFileName.toString.matches( + "00000000000000000004\\.checkpoint\\.[^.]+\\.json")) + } + val manifest = if (policy == "v2") { + Files.readAllLines(checkpoints.head, UTF_8).asScala.map(Action.parse).toVector + } else Vector.empty[Action] + val refs = manifest.collect { case s: SidecarFile => s } + assert(refs.size == (if (policy == "v2") 1 else 0)) + val sidecars = refs.map(ref => log.resolve("_sidecars").resolve(ref.path)) + refs.zip(sidecars).foreach { case (ref, path) => + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + } + if (policy == "v2") { + assert(!manifest.exists(_.isInstanceOf[AddFile])) + assert(manifest.collect { case c: CheckpointMetadata => c.version } == Seq(4L)) + assert(manifest.collect { case m: Metadata => m } == Seq(metadata)) + assert(manifest.collect { case p: Protocol => p } == Seq(protocol)) + } else assert(!Files.exists(log.resolve("_sidecars"))) + val boundary = 64L * 1024L + // Deterministic incompressible padding keeps Parquet fixtures small but >64 KiB. + val bytes = new Array[Byte](96 * 1024) + new scala.util.Random(43L).nextBytes(bytes) + val padding = java.util.Base64.getEncoder.encodeToString(bytes) + val paddedFiles = shape match { + case "sidecars" => sidecars + case "single_part" | "multipart" => checkpoints + case _ => Vector.empty[Path] + } + paddedFiles.foreach { path => + val frame = spark.read.parquet(path.toString) + val padded = frame.withColumn("add", when(col("add").isNotNull, + col("add").withField("tags", sparkMap(lit("e2e43.padding"), lit(padding)))) + .otherwise(col("add"))) + val temp = Files.createTempDirectory("e2e43-checkpoint-") + try { + padded.coalesce(1).write.mode("overwrite").parquet(temp.toString) + val parts = Files.list(temp) + val part = try parts.iterator().asScala + .filter(_.getFileName.toString.endsWith(".parquet")).toVector + finally parts.close() + assert(part.size == 1) + Files.move(part.head, path, java.nio.file.StandardCopyOption.REPLACE_EXISTING) + } finally org.apache.commons.io.FileUtils.deleteDirectory(temp.toFile) + } + if (shape == "manifest" || shape == "sidecars") { + val updated = manifest.map { + case c: CheckpointMetadata if shape == "manifest" => + c.copy(tags = Some(Map("e2e43.padding" -> padding))) + case ref: SidecarFile => + val path = log.resolve("_sidecars").resolve(ref.path) + ref.copy(sizeInBytes = Files.size(path), + modificationTime = Files.getLastModifiedTime(path).toMillis) + case action => action + } + Files.write(checkpoints.head, updated.map(_.toJson).asJava, UTF_8) + } + if (shape == "json") { + val lines = CommitLog.readRaw(tableDir, latestVersion).map { line => + val node = JsonUtil.mapper.readTree(line) + if (node.has("commitInfo")) { + val commitInfo = node.get("commitInfo") + .asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + commitInfo.put("userMetadata", padding) + } + node.toString + } + CommitLog.writeRaw(tableDir, latestVersion, lines) + } + val fileActions = spark.read.parquet( + (if (policy == "v2") sidecars else checkpoints).map(_.toString): _*) + val checkpointAdds = fileActions.toJSON.collect().toVector.map(Action.parse) + .collect { case a: AddFile => a } + val expectedAdds = adds.take(4).map { add => + add.copy(dataChange = false, tags = if (paddedFiles.nonEmpty) + Some(Map("e2e43.padding" -> padding)) else add.tags) + } + assert(checkpointAdds.sortBy(_.path) == expectedAdds.sortBy(_.path)) + if (policy == "classic") { + val top = fileActions.toJSON.collect().toVector.map(Action.parse) + assert(top.collect { case m: Metadata => m } == Seq(metadata)) + assert(top.collect { case p: Protocol => p } == Seq(protocol)) + } + if (isCrc) { + Seq(baseVersion, latestVersion).foreach { version => + val crcPath = log.resolve(f"$version%020d.crc") + assert(Files.exists(crcPath)) + val crc = JsonUtil.mapper.readTree(Files.readString(crcPath, UTF_8)) + assert(crc.has("allFiles") && crc.get("allFiles").isArray) + val active = (if (version == baseVersion) adds.slice(1, 4) else adds.drop(1)) + .map(_.copy(dataChange = false)) + val encoded = crc.get("allFiles").elements().asScala.map { node => + Action.parse("{\"add\":" + node.toString + "}").asInstanceOf[AddFile] + }.toVector + // The optional tags map can round-trip through Parquet as empty, not absent. + val normalized = encoded.map(a => a.copy(tags = a.tags.filter(_.nonEmpty))) + assert(normalized.sortBy(_.path) == active.sortBy(_.path)) + assert(crc.get("numFiles").asLong() == active.size) + assert(crc.get("tableSizeBytes").asLong() == active.map(_.size).sum) + info(s"size CRC v$version bytes=${Files.size(crcPath)} " + + s"allFiles=${encoded.size} activeBytes=${active.map(_.size).sum}") + } + } else { + // JSON 4 supports historical version lookup; absent 0..3 requires checkpoint 4. + // JSON 5 is a real one-commit replay tail with one new AddFile. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 4L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + val jsonPaths = files.filter(p => Files.exists(p) && + p.getFileName.toString.matches("[0-9]{20}\\.json")) + (checkpoints ++ sidecars ++ jsonPaths).foreach { path => + val large = paddedFiles.contains(path) || + (shape == "manifest" && checkpoints.contains(path)) || + (shape == "json" && path == CommitLog.commitFile(tableDir, latestVersion)) + val size = Files.size(path) + assert(if (large) size > boundary else size < boundary) + info(s"size $shape component=${log.relativize(path)} bytes=$size " + + s"large=$large boundary=$boundary") + } + } + Seq( + ("read_base", Some(baseVersion), None, baseRows, + ReadResult(baseRows.size.toLong, if (isCrc) 3 else 4, 0L)), + ("read_latest", None, None, latestRows, + ReadResult(latestRows.size.toLong, if (isCrc) 4 else 5, 0L)), + ("read_duplicates", None, Some("id = 21"), Seq(Row(21, null), Row(21, null)), + ReadResult(2L, 1, if (isCrc) 3L else 4L)) + ).foreach { case (name, version, predicate, rows, expected) => + readSpec(t, name = Some(name), version = version.map(Long.box).orNull, + predicate = predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, s"size $shape $name") + } + } + Seq(Some(baseVersion), None).foreach { version => + snapshotSpec(t, version = version.map(Long.box).orNull).assert { spec => + assert(spec.query.version.contains(version.getOrElse(latestVersion))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + if (isCrc) crcSpec(t, version = baseVersion, name = "crc_complete_active") + } finally { + previous.foreach { + case (key, Some(value)) => spark.conf.set(key, value) + case (key, None) => spark.conf.unset(key) + } + } + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessColumnMappingSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessColumnMappingSuite.scala new file mode 100644 index 00000000..1f328c3a --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessColumnMappingSuite.scala @@ -0,0 +1,403 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.nio.file.Files + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog, Metadata => LogMetadata, Protocol, RemoveFile} +import io.delta.workload.model.Succeeded + +class ReadinessColumnMappingSuite extends WorkloadTestSuite("readiness_column_mapping") { + + // One primary scenario covers parsed statistics and parsed partition values together. + // The existing checkpoints_partitioned fixture is the no-mapping parsed-key control. + for (mode <- Seq("name", "id")) { + test(s"parsed_physical_names_$mode") { + val properties = Map( + "delta.columnMapping.mode" -> mode, + "delta.checkpointInterval" -> "1000", + "delta.checkpoint.writeStatsAsStruct" -> "true", + "delta.checkpoint.writeStatsAsJson" -> "false") + val propertySql = properties.toSeq.sortBy(_._1) + .map { case (key, value) => s"'$key' = '$value'" }.mkString(", ") + sql(s"""CREATE TABLE tbl ( + id INT, old_value INT, payload STRUCT, old_part STRING) + USING delta PARTITIONED BY (old_part) TBLPROPERTIES ($propertySql)""") + sql("""INSERT INTO tbl VALUES + (1, 10, named_struct('old_leaf', -10), 'east'), + (1, 10, named_struct('old_leaf', -10), 'east'), + (2, 20, named_struct('old_leaf', -20), 'west'), + (3, NULL, named_struct('old_leaf', CAST(NULL AS INT)), NULL), + (4, 40, NULL, 'east')""") + sql("ALTER TABLE tbl RENAME COLUMN old_value TO value") + sql("ALTER TABLE tbl RENAME COLUMN payload.old_leaf TO leaf") + sql("ALTER TABLE tbl RENAME COLUMN old_part TO region") + sql("""INSERT INTO tbl VALUES + (5, 100, named_struct('leaf', -100), 'north'), + (6, NULL, named_struct('leaf', CAST(NULL AS INT)), NULL), + (7, 200, named_struct('leaf', -200), 'east')""") + forceCheckpoint("tbl") + // A real v6 tail is replayed after the v5 checkpoint; v5 JSON is only version lookup. + sql("INSERT INTO tbl VALUES (8, 300, named_struct('leaf', -300), 'west')") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val actions = (0L to 6L).map { version => + version -> CommitLog.readRaw(t.sourcePath, version).map(JsonUtil.mapper.readTree) + }.toMap + val beforeMetadata = actions(0L).find(_.has("metaData")).get.get("metaData") + val metadata = actions(4L).find(_.has("metaData")).get.get("metaData") + val before = DataType.fromJson(beforeMetadata.get("schemaString").asText) + .asInstanceOf[StructType] + val mapped = DataType.fromJson(metadata.get("schemaString").asText) + .asInstanceOf[StructType] + val physicalKey = "delta.columnMapping.physicalName" + val idKey = "delta.columnMapping.id" + val oldFields = before.fields.toSeq :+ + before("payload").dataType.asInstanceOf[StructType]("old_leaf") + val fields = mapped.fields.toSeq :+ + mapped("payload").dataType.asInstanceOf[StructType]("leaf") + assert(mapped.fieldNames.toSeq == Seq("id", "value", "payload", "region")) + assert(fields.map(_.metadata.getLong(idKey)).toSet == Set(1L, 2L, 3L, 4L, 5L)) + assert(fields.map(_.metadata) == oldFields.map(_.metadata)) + assert(fields.forall(field => field.metadata.getString(physicalKey) != field.name)) + val physical = fields.map(_.metadata.getString(physicalKey)) + assert(physical.distinct.size == 5) + val partitionName = physical(3) + val statsNames = physical.take(3).toSet + val originals = actions.values.flatten.filter(_.has("add")) + .map(_.get("add")).map(add => add.get("path").asText -> add).toMap + + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + for (version <- 0L to 6L; line <- CommitLog.readRaw(tableDir, version)) { + println(s"Physical-name source action: $mode v$version $line") + } + val checkpointPath = logDir.resolve("00000000000000000005.checkpoint.parquet") + assert(Files.exists(checkpointPath)) + val checkpoint = spark.read.parquet(checkpointPath.toString) + val addSchema = checkpoint.schema("add").dataType.asInstanceOf[StructType] + assert(!addSchema.fieldNames.contains("stats")) + val partition = addSchema("partitionValues_parsed").dataType.asInstanceOf[StructType] + assert(partition.fieldNames.toSeq == Seq(partitionName)) + assert(partition.fields.head.dataType == StringType) + val stats = addSchema("stats_parsed").dataType.asInstanceOf[StructType] + for (kind <- Seq("minValues", "maxValues", "nullCount")) { + val leaves = stats(kind).dataType.asInstanceOf[StructType] + assert(leaves.fieldNames.toSet == statsNames) + assert(leaves(physical(2)).dataType.asInstanceOf[StructType] + .fieldNames.toSeq == Seq(physical(4))) + } + val adds = checkpoint.filter("add IS NOT NULL").select("add").toJSON.collect() + assert(adds.nonEmpty) + for (line <- adds) { + val add = JsonUtil.mapper.readTree(line).get("add") + val original = originals(add.get("path").asText) + assert(add.get("partitionValues") == original.get("partitionValues")) + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val parsedPartition = add.get("partitionValues_parsed").get(partitionName) + val rawPartition = original.get("partitionValues").get(partitionName) + assert(Option(parsedPartition).filterNot(_.isNull).map(_.asText) == + Option(rawPartition).filterNot(_.isNull).map(_.asText)) + val rawStats = JsonUtil.mapper.readTree(original.get("stats").asText) + val parsedStats = add.get("stats_parsed") + assert(parsedStats.get("numRecords") == rawStats.get("numRecords")) + for (kind <- Seq("minValues", "maxValues", "nullCount"); + path <- Seq(Seq(physical(0)), Seq(physical(1)), Seq(physical(2), physical(4)))) { + val parsed = path.foldLeft(parsedStats.path(kind))((node, key) => node.path(key)) + val raw = path.foldLeft(rawStats.path(kind))((node, key) => node.path(key)) + assert(Option(parsed).filterNot(n => n.isNull || n.isMissingNode) == + Option(raw).filterNot(n => n.isNull || n.isMissingNode)) + } + } + // Earlier protocol/metadata/file history is absent: v5 checkpoint remains required. + // Keep v5 for version lookup and v6 as the actual post-checkpoint replay tail. + val stream = Files.list(logDir) + val files = try stream.iterator().asScala.toSeq finally stream.close() + files.filter { path => + val name = path.getFileName.toString + name.endsWith(".crc") || + (name.matches("[0-9]{20}\\.json") && name < "00000000000000000005.json") + }.foreach(Files.delete) + println(s"Physical-name checkpoint/layout oracle passed: $mode") + } + + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("payload", new StructType().add("leaf", IntegerType)).add("region", StringType) + val rows = Seq( + Row(1, 10, Row(-10), "east"), Row(1, 10, Row(-10), "east"), + Row(2, 20, Row(-20), "west"), Row(3, null, Row(null), null), + Row(4, 40, null, "east"), Row(5, 100, Row(-100), "north"), + Row(6, null, Row(null), null), Row(7, 200, Row(-200), "east"), + Row(8, 300, Row(-300), "west")) + // Row selections are literal bag oracles, independent of Spark's predicate evaluation. + val reads = Seq( + ("read_all", null, rows.indices, None, None), + ("read_checkpoint", null, 0 until 8, Some(5L), None), + ("read_old_file", "value = 10", Seq(0, 1), None, None), + ("read_new_file", "value = 100", Seq(5), None, None), + ("read_tail", "value = 300", Seq(8), None, None), + ("read_missing", "value = 999", Seq.empty[Int], None, None), + ("read_null", "value IS NULL", Seq(3, 6), None, None), + ("read_nested_old", "payload.leaf = -10", Seq(0, 1), None, None), + ("read_nested_new", "payload.leaf = -100", Seq(5), None, None), + ("read_nested_null", "payload.leaf IS NULL", Seq(3, 4, 6), None, None), + ("read_partition_old", "region = 'east'", Seq(0, 1, 4, 7), None, None), + ("read_partition_new", "region = 'north'", Seq(5), None, None), + ("read_partition_null", "region IS NULL", Seq(3, 6), None, None), + ("read_projection", null, rows.indices, None, Some(Seq("region", "value", "payload")))) + for ((name, predicate, indices, version, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, + version = version.map(Long.box).orNull, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"$mode/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"$mode/$name literal bag") + assert(captured.schema.fields.forall(_.nullable)) + println(s"Physical-name literal row/schema oracle passed: $mode/$name") + } + } + for (version <- Seq(5L, 6L)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.get("schemaString").asText) + assert(result.metadata.partitionColumns == Seq("region")) + assert(result.metadata.configuration == + properties + ("delta.columnMapping.maxColumnId" -> "5")) + case other => fail(s"$mode snapshot v$version must succeed: $other") + } + println(s"Physical-name snapshot mapping oracle passed: $mode/v$version") + } + } + } + } + + test("metadata_partition_values_ignore_directories") { + sql("""CREATE TABLE tbl (id INT, payload STRING, part STRING) USING delta + PARTITIONED BY (part) TBLPROPERTIES ('delta.checkpointInterval' = '1000')""") + sql("""INSERT INTO tbl VALUES + (1, 'duplicate', 'east'), (1, 'duplicate', 'east'), + (2, 'conflicting', 'west'), (3, 'null_partition', NULL), + (4, 'no_hive_directory', 'plain')""") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val schema = new StructType().add("id", IntegerType).add("payload", StringType) + .add("part", StringType) + + mutateTable(t) { tableDir => + val originals = CommitLog.read(tableDir, 1L).collect { case add: AddFile => add } + assert(originals.nonEmpty) + assert(originals.map(_.partitionValues("part")).toSet == Set("east", "west", null, "plain")) + CommitLog.mutate(tableDir, 1L) { actions => + actions.map { + case add: AddFile => + val directory = if (add.partitionValues("part") == "plain") "opaque/nested" + else "part=directory_lie" + val original = tableDir.resolve(add.path) + val relative = s"$directory/${original.getFileName}" + val target = tableDir.resolve(relative) + Files.createDirectories(target.getParent) + Files.move(original, target) + assert(!Files.exists(original) && Files.isRegularFile(target)) + assert(Files.size(target) == add.size) + add.copy(path = relative) + case other => other + } + } + val moved = CommitLog.read(tableDir, 1L).collect { case add: AddFile => add } + assert(moved.map(_.partitionValues) == originals.map(_.partitionValues)) + assert(moved.zip(originals).forall { case (after, before) => + after.copy(path = before.path) == before + }) + assert(moved.forall(add => Files.isRegularFile(tableDir.resolve(add.path)))) + assert(moved.forall(add => !originals.exists(_.path == add.path))) + // Require JSON replay: no checkpoint or Delta CRC may retain the original file paths. + val stream = Files.list(tableDir.resolve("_delta_log")) + val files = try stream.iterator().asScala.toSeq finally stream.close() + assert(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + println("Metadata-partition layout oracle passed: conflicting and non-Hive directories") + } + + val rows = Seq(Row(1, "duplicate", "east"), Row(1, "duplicate", "east"), + Row(2, "conflicting", "west"), Row(3, "null_partition", null), + Row(4, "no_hive_directory", "plain")) + val reads = Seq( + ("read_all", null, rows.indices, None), + ("read_metadata_east", "part = 'east'", Seq(0, 1), None), + ("read_metadata_west", "part = 'west'", Seq(2), None), + ("read_directory_lie", "part = 'directory_lie'", Seq.empty[Int], None), + ("read_null", "part IS NULL", Seq(3), None), + ("read_no_hive_directory", "part = 'plain'", Seq(4), None), + ("read_projection", null, rows.indices, Some(Seq("part", "id"))), + ("read_null_projection", "part IS NULL", Seq(3), Some(Seq("part", "payload")))) + for ((name, predicate, indices, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"metadata partition/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"metadata partition/$name literal bag") + assert(captured.schema == oracle.schema) + println(s"Metadata-partition literal row/schema oracle passed: $name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(DataType.fromJson(result.metadata.schemaString) == schema) + assert(result.metadata.partitionColumns == Seq("part")) + assert(result.metadata.configuration == Map("delta.checkpointInterval" -> "1000")) + case other => fail(s"metadata partition snapshot must succeed: $other") + } + println("Metadata-partition snapshot schema/partition oracle passed") + } + } + + test("enable_name_mapping_with_full_rewrite") { + sql("""CREATE TABLE tbl (id INT, value INT, payload STRING) USING delta + TBLPROPERTIES ('delta.columnMapping.mode' = 'none', + 'delta.checkpointInterval' = '1000')""") + sql("INSERT INTO tbl VALUES (1, 10, 'duplicate'), (1, 10, 'duplicate'), (2, NULL, 'nullable')") + sql("INSERT INTO tbl VALUES (3, 30, NULL), (NULL, NULL, NULL), (NULL, NULL, NULL)") + // A separately materialized source prevents replacement from reading its own target. + // Both SQL-created tables are tracked by the normal WorkloadContext cleanup. + sql("CREATE TABLE rewrite_source USING delta AS SELECT id, value, payload FROM tbl") + sql("""CREATE OR REPLACE TABLE tbl USING delta TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', 'delta.minReaderVersion' = '2', + 'delta.minWriterVersion' = '5', 'delta.checkpointInterval' = '1000') + AS SELECT id, value, payload FROM rewrite_source""") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val before = (0L to 2L).flatMap(CommitLog.read(t.sourcePath, _)) + val rewrite = CommitLog.read(t.sourcePath, 3L) + val oldAdds = before.collect { case add: AddFile => add } + val newAdds = rewrite.collect { case add: AddFile => add } + val removed = rewrite.collect { case remove: RemoveFile => remove } + assert(!before.exists(_.isInstanceOf[RemoveFile])) + assert(oldAdds.nonEmpty && newAdds.nonEmpty) + val oldPaths = oldAdds.map(_.path).toSet + val newPaths = newAdds.map(_.path).toSet + assert(oldPaths.size == oldAdds.size && newPaths.size == newAdds.size) + assert(removed.map(_.path).toSet == oldPaths && removed.size == oldPaths.size) + assert(newPaths.intersect(oldPaths).isEmpty) + assert((oldPaths -- removed.map(_.path)) ++ newPaths == newPaths) + assert((oldAdds ++ newAdds).forall(add => + Files.isRegularFile(t.sourcePath.resolve(add.path)) && + Files.size(t.sourcePath.resolve(add.path)) == add.size)) + + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("payload", StringType) + val oldMetadata = before.collect { case metadata: LogMetadata => metadata }.last + val newMetadata = rewrite.collect { case metadata: LogMetadata => metadata }.last + assert(DataType.fromJson(oldMetadata.schemaString) == schema) + assert(oldMetadata.configuration == Map( + "delta.columnMapping.mode" -> "none", "delta.checkpointInterval" -> "1000")) + val mapped = DataType.fromJson(newMetadata.schemaString).asInstanceOf[StructType] + assert(mapped.fields.map(f => (f.name, f.dataType, f.nullable)).toSeq == + schema.fields.map(f => (f.name, f.dataType, f.nullable)).toSeq) + val idKey = "delta.columnMapping.id" + val physicalKey = "delta.columnMapping.physicalName" + assert(mapped.fields.map(_.metadata.getLong(idKey)).toSet == Set(1L, 2L, 3L)) + val physicalNames = mapped.fields.map(_.metadata.getString(physicalKey)).toSeq + assert(physicalNames.distinct.size == 3) + assert(mapped.fields.forall(f => f.metadata.getString(physicalKey) != f.name)) + assert(newMetadata.configuration == Map( + "delta.columnMapping.mode" -> "name", "delta.columnMapping.maxColumnId" -> "3", + "delta.checkpointInterval" -> "1000")) + assert(oldMetadata.partitionColumns.isEmpty && newMetadata.partitionColumns.isEmpty) + assert(before.collect { case p: Protocol => p }.last == Protocol(1, 2)) + assert(rewrite.collect { case p: Protocol => p }.last == Protocol(2, 5)) + for (add <- oldAdds) { + assert(spark.read.parquet(t.sourcePath.resolve(add.path).toString).schema == schema) + } + for (add <- newAdds) { + val physical = spark.read.parquet(t.sourcePath.resolve(add.path).toString).schema + assert(physical.fieldNames.toSeq == physicalNames) + assert(physical.fields.map(_.dataType).toSeq == schema.fields.map(_.dataType).toSeq) + assert(physical.fields.map(_.metadata.getLong("parquet.field.id")).toSeq == + mapped.fields.map(_.metadata.getLong(idKey)).toSeq) + } + println(s"Full-rewrite action/physical-name oracle passed: " + + s"removed=${oldPaths.size}, fresh=${newPaths.size}") + + // Every selection is a literal bag, including both copies of each duplicate/null row. + val rows = Seq(Row(1, 10, "duplicate"), Row(1, 10, "duplicate"), + Row(2, null, "nullable"), Row(3, 30, null), Row(null, null, null), Row(null, null, null)) + val reads = Seq( + ("read_before", null, rows.indices, Some(2L), None), + ("read_all", null, rows.indices, None, None), + ("read_rewritten", null, rows.indices, Some(3L), None), + ("read_projection", null, rows.indices, None, Some(Seq("payload", "id"))), + ("read_hit", "value = 10", Seq(0, 1), None, None), + ("read_miss", "value = 999", Seq.empty[Int], None, None), + ("read_null_value", "value IS NULL", Seq(2, 4, 5), None, None), + ("read_null_payload", "payload IS NULL", Seq(3, 4, 5), None, None), + ("read_null_id_projection", "id IS NULL", Seq(4, 5), None, Some(Seq("id", "value")))) + for ((name, predicate, indices, version, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, + version = version.map(Long.box).orNull, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"full rewrite/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"full rewrite/$name literal bag") + assert(captured.schema == oracle.schema) + println(s"Full-rewrite literal row/schema oracle passed: $name") + } + } + for ((version, metadata, reader, writer) <- Seq( + (2L, oldMetadata, 1, 2), (3L, newMetadata, 2, 5))) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.schemaString) + assert(result.metadata.configuration == metadata.configuration) + assert(result.metadata.partitionColumns.isEmpty) + assert(result.protocol.minReaderVersion == reader) + assert(result.protocol.minWriterVersion == writer) + case other => fail(s"full rewrite snapshot v$version must succeed: $other") + } + println(s"Full-rewrite snapshot transition oracle passed: v$version") + } + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessPredicatesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessPredicatesSuite.scala new file mode 100644 index 00000000..208d081f --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessPredicatesSuite.scala @@ -0,0 +1,779 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types.{DataType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.model._ + +/** + * Portable row semantics for read predicates, without a pruning assertion. + * Capture and replay use UTC. Signed-zero requires Parquet filter pushdown disabled; three + * narrowing CAST cases require ANSI disabled. Use `startswith`; `starts_with` is unsupported. + */ +class ReadinessPredicatesSuite extends WorkloadTestSuite("readiness_predicates") { + + for (partitioned <- Seq(false, true)) { + val layout = if (partitioned) "partitioned" else "unpartitioned" + test(s"nondeterministic_values_$layout") { + val partition = if (partitioned) "PARTITIONED BY (p)" else "" + sql(s"CREATE TABLE tbl (id INT, value INT, p INT) USING delta $partition") + sql("INSERT INTO tbl VALUES (1, 10, 1), (2, 20, 1), (2, 20, 1)") + sql("INSERT INTO tbl VALUES (3, 20, 2), (4, 30, 2), (5, 40, 3)") + sql("INSERT INTO tbl VALUES (6, NULL, 1), (7, 50, NULL), (8, NULL, NULL)") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + + // Literal bags are independent of the filtered Delta reads, including duplicate/NULL rows. + val all = Seq( + "(1, 10, 1)", + "(2, 20, 1)", + "(2, 20, 1)", + "(3, 20, 2)", + "(4, 30, 2)", + "(5, 40, 3)", + "(6, NULL, 1)", + "(7, 50, NULL)", + "(8, NULL, NULL)" + ) + val equality = Seq("(2, 20, 1)", "(2, 20, 1)", "(3, 20, 2)") + val range = equality :+ "(4, 30, 2)" + val partitionBag = Seq("(1, 10, 1)", "(2, 20, 1)", "(2, 20, 1)", "(6, NULL, 1)") + val cases = Seq( + ("read_all", null, all), + ("random_all", "rand() >= 0", all), + ("random_none", "rand() < 0", Seq.empty[String]), + ("equality_control", "value = 20", equality), + ("range_control", "value >= 20 AND value < 40", range), + ("and_equality", "rand() >= 0 AND value = 20", equality), + ("and_range", "rand() >= 0 AND (value >= 20 AND value < 40)", range), + ("and_none", "rand() < 0 AND value = 20", Seq.empty[String]), + ("or_equality", "rand() < 0 OR value = 20", equality), + ("or_range", "rand() < 0 OR (value >= 20 AND value < 40)", range), + ("or_all", "rand() >= 0 OR value = 20", all), + ("partition_control", "p = 1", partitionBag), + ("and_partition", "p = 1 AND rand() >= 0", partitionBag), + ("or_partition", "p = 1 OR rand() < 0", partitionBag), + // Spark can simplify the domain-only controls. Referencing the row's small exact INT id + // keeps a nondeterministic expression at read time: adding [0, 1) cannot decrease id. + ("row_random_all", "rand() + id >= id", all), + ("row_random_none", "rand() + id < id", Seq.empty[String]), + ("row_and_equality", "rand() + id >= id AND value = 20", equality), + ("row_and_range", "rand() + id >= id AND (value >= 20 AND value < 40)", range), + ("row_or_equality", "rand() + id < id OR value = 20", equality), + ("row_or_range", "rand() + id < id OR (value >= 20 AND value < 40)", range), + ("row_and_partition", "p = 1 AND rand() + id >= id", partitionBag), + ("row_or_partition", "p = 1 OR rand() + id < id", partitionBag) + ) + + // rand() stays in the serialized read predicate. Its [0, 1) domain gives a stable bag + // across RNG seeds, partition counts and replays; random values are never stored on INSERT. + for ((label, predicate, bag) <- cases) { + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(_) => + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + s"SELECT CAST(id AS INT) AS id, CAST(value AS INT) AS value, " + + s"CAST(p AS INT) AS p FROM VALUES ${bag.mkString(", ")} AS expected(id, value, p)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS INT) AS value, " + + "CAST(NULL AS INT) AS p WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, value INT, p INT")) + require( + result.metadata.partitionColumns == + (if (partitioned) Seq("p") else Seq.empty) + ) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + } + + test("signed_zero") { + require( + spark.conf.get("spark.sql.parquet.filterPushdown") == "false", + "The signed-zero bag is validated with Parquet filter pushdown disabled" + ) + sql("CREATE TABLE tbl (id INT, f FLOAT, d DOUBLE) USING delta") + // Separate writes keep the two zero signs and nonzero ranges in distinct files. + sql("""INSERT INTO tbl VALUES + |(1, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE)), + |(2, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE)), + |(2, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE))""".stripMargin) + sql("""INSERT INTO tbl VALUES + |(3, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE)), + |(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE)), + |(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE))""".stripMargin) + sql("INSERT INTO tbl VALUES (5, -1, -1), (6, -2, -2)") + sql("INSERT INTO tbl VALUES (7, 1, 1), (8, 2, 2)") + sql("INSERT INTO tbl VALUES (9, NULL, NULL)") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + + // Literal bags do not filter the source table. IDs distinguish the original zero signs; + // exact duplicates check multiplicity. Row equality need not retain a zero's sign bit. + val zeros = Seq("(1, 0, 0)", "(2, 0, 0)", "(2, 0, 0)", "(3, 0, 0)", "(4, 0, 0)", "(4, 0, 0)") + val negative = Seq("(5, -1, -1)", "(6, -2, -2)") + val positive = Seq("(7, 1, 1)", "(8, 2, 2)") + val nulls = Seq("(9, NULL, NULL)") + val nonzero = negative ++ positive + val all = zeros ++ nonzero ++ nulls + val cases = Seq(("read_all", null, all)) ++ + Seq(("f", "FLOAT"), ("d", "DOUBLE")).flatMap { + case (column, dataType) => + val plus = s"CAST('0.0' AS $dataType)" + val minus = s"CAST('-0.0' AS $dataType)" + val one = s"CAST(1 AS $dataType)" + val negativeOne = s"CAST(-1 AS $dataType)" + val typedNull = s"CAST(NULL AS $dataType)" + val comparisons = for { + (sign, zero) <- Seq(("positive", plus), ("negative", minus)) + (label, operator, bag) <- Seq( + ("eq", "=", zeros), + ("null_safe_eq", "<=>", zeros), + ("ne", "!=", nonzero), + ("lt", "<", negative), + ("le", "<=", negative ++ zeros), + ("gt", ">", positive), + ("ge", ">=", zeros ++ positive) + ) + } yield (s"${column}_${label}_$sign", s"$column $operator $zero", bag) + val inLists = Seq( + ("in_positive", plus, zeros), + ("in_negative", minus, zeros), + ("in_both", s"$plus, $minus", zeros), + ("in_positive_null", s"$plus, $typedNull", zeros), + ("in_negative_null", s"$minus, $typedNull", zeros), + ("in_both_null", s"$plus, $minus, $typedNull", zeros), + ("in_positive_nonzero", s"$plus, $one", zeros :+ "(7, 1, 1)"), + ("in_negative_nonzero", s"$minus, $negativeOne", zeros :+ "(5, -1, -1)"), + ( + "in_both_null_nonzero", + s"$plus, $minus, $typedNull, $negativeOne, $one", + zeros ++ Seq("(5, -1, -1)", "(7, 1, 1)") + ), + ("in_nonzero_null", s"$negativeOne, $one, $typedNull", Seq("(5, -1, -1)", "(7, 1, 1)")), + ("in_null", typedNull, Seq.empty[String]), + ("in_miss", s"CAST(9 AS $dataType), $typedNull", Seq.empty[String]) + ) + comparisons ++ Seq((s"${column}_null_safe_null", s"$column <=> $typedNull", nulls)) ++ + inLists.map { + case (label, literals, bag) => + (s"${column}_$label", s"$column IN ($literals)", bag) + } + } + + for ((label, predicate, bag) <- cases) { + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + "SELECT CAST(id AS INT) AS id, CAST(f AS FLOAT) AS f, CAST(d AS DOUBLE) AS d " + + s"FROM VALUES ${bag.mkString(", ")} AS expected(id, f, d)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS FLOAT) AS f, " + + "CAST(NULL AS DOUBLE) AS d WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, f FLOAT, d DOUBLE")) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + // Explicit predicate casts, precision boundaries and legacy narrowing. + // Expected IDs are derived independently of the CAST predicates. Source duplicates stay intact. + for ((subcase, sourceType, sourceRows, checks, legacy) <- Seq( + ( + "float_double", + "FLOAT", + Seq( + (1, "-0.1"), + (2, "0.1"), + (2, "0.1"), + (3, "0.10000001"), + (4, "0.2"), + (5, "1.0"), + (6, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6)), + ("read_hit", "CAST(value AS DOUBLE) = CAST('0.10000000149011612' AS DOUBLE)", Seq(2)), + ( + "decimal_spelling_miss", + "CAST(value AS DOUBLE) = CAST('0.1' AS DOUBLE)", + Seq.empty[Int] + ), + ("above_decimal", "CAST(value AS DOUBLE) > CAST('0.1' AS DOUBLE)", Seq(2, 3, 4, 5)), + ( + "adjacent_range", + "CAST(value AS DOUBLE) >= CAST('0.10000000149011612' AS DOUBLE) AND " + + "CAST(value AS DOUBLE) < CAST('0.10000000894069672' AS DOUBLE)", + Seq(2) + ), + ( + "negative_hit", + "CAST(value AS DOUBLE) = CAST('-0.10000000149011612' AS DOUBLE)", + Seq(1) + ), + ("read_miss", "CAST(value AS DOUBLE) = CAST('2' AS DOUBLE)", Seq.empty[Int]) + ), + false + ), + ( + "double_2p53", + "BIGINT", + Seq( + (1, "9007199254740991"), + (2, "9007199254740992"), + (3, "9007199254740993"), + (3, "9007199254740993"), + (4, "9007199254740994"), + (5, "9007199254740995"), + (6, "-9007199254740993"), + (7, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7)), + ("read_hit", "CAST(value AS DOUBLE) = CAST('9007199254740992' AS DOUBLE)", Seq(2, 3)), + ("adjacent_hit", "CAST(value AS DOUBLE) = CAST('9007199254740994' AS DOUBLE)", Seq(4)), + ( + "adjacent_range", + "CAST(value AS DOUBLE) > CAST('9007199254740992' AS DOUBLE) AND " + + "CAST(value AS DOUBLE) < CAST('9007199254740996' AS DOUBLE)", + Seq(4) + ), + ( + "below_boundary", + "CAST(value AS DOUBLE) < CAST('9007199254740992' AS DOUBLE)", + Seq(1, 6) + ), + ( + "read_miss", + "CAST(value AS DOUBLE) = CAST('9007199254740998' AS DOUBLE)", + Seq.empty[Int] + ) + ), + false + ), + ( + "float_2p24", + "INT", + Seq( + (1, "16777215"), + (2, "16777216"), + (3, "16777217"), + (3, "16777217"), + (4, "16777218"), + (5, "16777219"), + (6, "-16777217"), + (7, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7)), + ("read_hit", "CAST(value AS FLOAT) = CAST('16777216' AS FLOAT)", Seq(2, 3)), + ("adjacent_hit", "CAST(value AS FLOAT) = CAST('16777218' AS FLOAT)", Seq(4)), + ( + "adjacent_range", + "CAST(value AS FLOAT) > CAST('16777216' AS FLOAT) AND " + + "CAST(value AS FLOAT) < CAST('16777220' AS FLOAT)", + Seq(4) + ), + ("below_boundary", "CAST(value AS FLOAT) < CAST('16777216' AS FLOAT)", Seq(1, 6)), + ("read_miss", "CAST(value AS FLOAT) = CAST('16777222' AS FLOAT)", Seq.empty[Int]) + ), + false + ), + ( + "long_int_wrap", + "BIGINT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "4294967301"), + (3, "4294967301"), + (4, "4294967300"), + (5, "4294967302"), + (6, "-4294967291"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS INT) = CAST('5' AS INT)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS INT) = CAST('4' AS INT)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS INT) = CAST('6' AS INT)", Seq(5, 9)), + ("below_five", "CAST(value AS INT) < CAST('5' AS INT)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS INT) > CAST('5' AS INT)", Seq(5, 9)), + ("read_miss", "CAST(value AS INT) = CAST('7' AS INT)", Seq.empty[Int]) + ), + true + ), + ( + "to_short_wrap", + "INT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "65541"), + (3, "65541"), + (4, "65540"), + (5, "65542"), + (6, "-65531"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS SHORT) = CAST('5' AS SHORT)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS SHORT) = CAST('4' AS SHORT)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS SHORT) = CAST('6' AS SHORT)", Seq(5, 9)), + ("below_five", "CAST(value AS SHORT) < CAST('5' AS SHORT)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS SHORT) > CAST('5' AS SHORT)", Seq(5, 9)), + ("read_miss", "CAST(value AS SHORT) = CAST('7' AS SHORT)", Seq.empty[Int]) + ), + true + ), + ( + "int_byte_wrap", + "INT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "261"), + (3, "261"), + (4, "260"), + (5, "262"), + (6, "-251"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS BYTE) = CAST('5' AS BYTE)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS BYTE) = CAST('4' AS BYTE)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS BYTE) = CAST('6' AS BYTE)", Seq(5, 9)), + ("below_five", "CAST(value AS BYTE) < CAST('5' AS BYTE)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS BYTE) > CAST('5' AS BYTE)", Seq(5, 9)), + ("read_miss", "CAST(value AS BYTE) = CAST('7' AS BYTE)", Seq.empty[Int]) + ), + true + ) + )) { + test(s"cast_filters_$subcase") { + // ReadQuery has no session config field. Supply the same legacy session on every replay. + if (legacy) { + require( + spark.conf.get("spark.sql.ansi.enabled") == "false", + "Narrowing CAST fixtures require an existing non-ANSI session for capture and replay" + ) + } + sql(s"CREATE TABLE tbl (id INT, value $sourceType) USING delta") + // Separate writes leave ordinary and boundary values in multiple stats-bearing files. + for (batch <- sourceRows.grouped(2)) { + val values = batch.map { case (id, value) => s"($id, $value)" }.mkString(", ") + sql(s"INSERT INTO tbl VALUES $values") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + for ((label, predicate, expectedIds) <- checks) { + val bag = sourceRows.filter { case (id, _) => expectedIds.contains(id) } + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + // Only source-type casts construct these literal bags; no predicate CAST is reused. + val oracle = if (bag.nonEmpty) { + val values = bag.map { case (id, value) => s"($id, $value)" }.mkString(", ") + s"SELECT CAST(id AS INT) AS id, CAST(value AS $sourceType) AS value " + + s"FROM VALUES $values AS expected(id, value)" + } else { + s"SELECT CAST(NULL AS INT) AS id, CAST(NULL AS $sourceType) AS value WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL(s"id INT, value $sourceType")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + if (subcase == "float_double") { + val compositionCases = Seq( + ("direct_nonnegative", "CAST(value AS DOUBLE) >= 0", Seq(2, 3, 4, 5)), + ( + "compound_hit", + "(CAST(value AS DOUBLE) > 0.15 AND id < 5) OR " + + "(CAST(value AS DOUBLE) < 0 AND id = 1)", + Seq(1, 4) + ), + ( + "compound_miss", + "(CAST(value AS DOUBLE) > 2 AND id < 5) OR " + + "(CAST(value AS DOUBLE) < -2 AND id = 1)", + Seq.empty[Int] + ) + ) + for ((label, predicate, ids) <- compositionCases) { + val bag = sourceRows.filter { case (id, _) => ids.contains(id) } + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + val values = bag.map { case (id, value) => s"($id, $value)" }.mkString(", ") + s"SELECT CAST(id AS INT) AS id, CAST(value AS FLOAT) AS value " + + s"FROM VALUES $values AS expected(id, value)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS FLOAT) AS value WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + } + + } + } + + // Complex filter row semantics. File counts do not prove filter pushdown. + test("complex_filters_month_timestamp") { + // Read specs have no timezone field; use UTC through queued capture and every replay. + require(spark.conf.get("spark.sql.session.timeZone") == "UTC") + sql("CREATE TABLE tbl (id INT, ts TIMESTAMP) USING delta") + val rows = Seq( + "(1, TIMESTAMP'2023-12-31 23:59:59.999999')", + "(2, TIMESTAMP'2024-01-01 00:00:00')", + "(3, TIMESTAMP'2024-01-31 23:59:59.999999')", + "(4, TIMESTAMP'2024-02-01 00:00:00')", + "(5, TIMESTAMP'2024-02-29 23:59:59.999999')", + "(6, TIMESTAMP'2024-03-01 00:00:00')", + "(7, TIMESTAMP'2024-12-31 23:59:59.999999')", + "(8, TIMESTAMP'2025-01-01 00:00:00')", + "(8, TIMESTAMP'2025-01-01 00:00:00')", + "(9, NULL)" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("january", "month(ts) = 1", Seq(2, 3, 8)), + ("february", "month(ts) = 2", Seq(4, 5)), + ("december", "month(ts) = 12", Seq(1, 7)), + ("miss_june", "month(ts) = 6", Seq.empty[Int]), + ("null_month", "month(ts) IS NULL", Seq(9)) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(ts AS TIMESTAMP) AS ts" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, ts)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, ts TIMESTAMP")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + test("complex_filters_strings") { + sql("CREATE TABLE tbl (id INT, name STRING, s STRUCT) USING delta") + val rows = Seq( + "(1, 'apple', named_struct('name', 'apple'))", + "(2, 'banana', named_struct('name', 'banana'))", + "(3, 'alice', named_struct('name', 'alice'))", + "(4, 'bob', named_struct('name', 'bob'))", + "(5, 'charlie', named_struct('name', 'charlie'))", + "(6, 'diana', named_struct('name', 'diana'))", + "(7, 'Apple', named_struct('name', 'APPLE'))", + "(8, 'ALICE', named_struct('name', 'Alice'))", + "(8, 'ALICE', named_struct('name', 'Alice'))", + "(9, '', named_struct('name', ''))", + "(10, NULL, named_struct('name', NULL))", + "(11, 'apPle', named_struct('name', NULL))" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)), + ("prefix_hit", "startswith(name, 'app')", Seq(1)), + ("prefix_a", "startswith(name, 'a')", Seq(1, 3, 11)), + ("prefix_nested", "startswith(s.name, 'a')", Seq(1, 3)), + ("prefix_miss", "startswith(name, 'z')", Seq.empty[Int]), + ("prefix_empty", "startswith(name, '')", Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 11)), + ("prefix_null", "startswith(name, CAST(NULL AS STRING))", Seq.empty[Int]), + ("prefix_null_result", "startswith(name, 'a') IS NULL", Seq(10)), + ("upper_alice", "upper(name) = 'ALICE'", Seq(3, 8)), + ("upper_apple", "upper(name) = 'APPLE'", Seq(1, 7, 11)), + ("upper_miss", "upper(name) = 'ZEBRA'", Seq.empty[Int]), + ("upper_null", "upper(name) IS NULL", Seq(10)), + ("upper_selective", "upper(name) = 'APPLE' AND id > 1", Seq(7, 11)) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(name AS STRING) AS name, " + + "CAST(s AS STRUCT) AS s" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, name, s)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, name STRING, s STRUCT")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + test("complex_filters_deeply_nested") { + sql( + "CREATE TABLE tbl (id INT, s STRUCT) USING delta" + ) + val rows = Seq( + "(1, named_struct('a', 1, 'b', 1, 'c', NULL, 'd', NULL, 'e', NULL, 'f', NULL, " + + "'g', NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(2, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 1, 'e', NULL, 'f', NULL, 'g', " + + "NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(3, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 1, 'g', NULL, " + + "'h', NULL, 'i', NULL, 'j', NULL))", + "(4, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "1, 'i', NULL, 'j', NULL))", + "(5, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 0))", + "(5, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 0))", + "(6, named_struct('a', 0, 'b', 1, 'c', 0, 'd', 1, 'e', 1, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))", + "(7, named_struct('a', 1, 'b', 0, 'c', 1, 'd', 1, 'e', 1, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))", + "(8, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 1))", + "(9, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', NULL))", + "(10, named_struct('a', NULL, 'b', 1, 'c', 0, 'd', 1, 'e', 1, 'f', 1, 'g', 0, " + + "'h', 1, 'i', 1, 'j', 0))", + "(11, named_struct('a', NULL, 'b', NULL, 'c', NULL, 'd', NULL, 'e', NULL, 'f', " + + "NULL, 'g', NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(12, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 0, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)), + ( + "deep_hit", + "s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))", + Seq(1, 2, 3, 4, 5) + ), + ( + "deep_false", + "NOT (s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND " + + "(s.f = 1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1))))))))))", + Seq(6, 7, 8, 12) + ), + ( + "deep_null", + "(s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))) IS " + + "NULL", + Seq(9, 10, 11) + ), + ( + "deep_miss", + "(s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))) AND " + + "id = 99", + Seq.empty[Int] + ) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(s AS STRUCT) AS s" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, s)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require( + schema == StructType.fromDDL( + "id INT, s STRUCT" + ) + ) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessSchemaReplacementSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessSchemaReplacementSuite.scala new file mode 100644 index 00000000..fdfe4326 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessSchemaReplacementSuite.scala @@ -0,0 +1,494 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.types.{ByteType, DataType, IntegerType, StringType, StructType} + +import io.delta.workload.WorkloadTestSuite +import io.delta.workload.engine.RowComparison +import io.delta.workload.model.Succeeded + +class ReadinessSchemaReplacementSuite extends WorkloadTestSuite("readiness_schema_replacement") { + + // Shared by the stats-column and partition-column variants of the same replacement scenario. + private def replaceSameName( + caseName: String, + partitioned: Boolean, + fromType: DataType, + toType: DataType, + seedValues: String, + replacementValues: String, + beforeRows: Seq[Row], + afterRows: Seq[Row], + matchingPredicate: String, + missingPredicate: String): Unit = { + val partitionClause = if (partitioned) "PARTITIONED BY (c)" else "" + val beforeSchema = new StructType().add("id", IntegerType) + .add("c", fromType).add("payload", StringType) + val afterSchema = new StructType().add("id", IntegerType) + .add("c", toType).add("payload", StringType) + + sql(s"CREATE TABLE tbl (${beforeSchema.toDDL}) USING delta $partitionClause") + sql(s"INSERT INTO tbl VALUES $seedValues") + // This is a REPLACE TABLE commit, rather than DROP/ADD COLUMN or an append with evolution. + sql(s"CREATE OR REPLACE TABLE tbl (${afterSchema.toDDL}) USING delta $partitionClause") + sql(s"INSERT INTO tbl VALUES $replacementValues") + val t = registerTable("tbl") + val table = io.delta.workload.WorkloadContext.current.getTableSpec(t) + + // Literal rows form an oracle independent of reading the generated Delta table. + def frame(rows: Seq[Row], schema: StructType): DataFrame = + spark.createDataFrame(rows.asJava, schema) + + def checkRead( + name: String, + rows: Seq[Row], + schema: StructType, + version: java.lang.Long = null, + predicate: String = null, + columns: Option[Seq[String]] = None): Unit = { + val oracle = frame(rows, schema) + val projectedOracle = columns.map(cs => oracle.selectExpr(cs: _*)).getOrElse(oracle) + readSpec(t, name = Some(name), version = version, predicate = predicate, + columns = columns).assert { spec => + val workloadName = table.outputName + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$workloadName/$name must succeed: $other") + } + val expectedPath = outputDir.resolve(workloadName).resolve("expected") + .resolve(s"${workloadName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expectedPath.toString) + assert(captured.schema == projectedOracle.schema) + RowComparison.assertRowsEqual( + projectedOracle, captured, s"$workloadName/$name literal oracle") + println(s"Independent row oracle passed: $workloadName/$name (${rows.size} rows)") + } + } + + checkRead("read_all", afterRows, afterSchema) + checkRead("read_before", beforeRows, beforeSchema, version = 1L) + checkRead("read_replaced_empty", Seq.empty, afterSchema, version = 2L) + checkRead("read_match", afterRows.take(2), afterSchema, predicate = matchingPredicate) + checkRead("read_missing", Seq.empty, afterSchema, predicate = missingPredicate) + checkRead("read_null", afterRows.slice(3, 4), afterSchema, predicate = "c IS NULL") + checkRead("read_non_null", afterRows.take(3) ++ afterRows.drop(4), afterSchema, + predicate = "c IS NOT NULL") + checkRead("read_projection", afterRows, afterSchema, columns = Some(Seq("c", "payload"))) + checkRead("read_before_projection", beforeRows, beforeSchema, version = 1L, + columns = Some(Seq("c", "payload"))) + checkRead("read_before_null", beforeRows.takeRight(1), beforeSchema, version = 1L, + predicate = "c IS NULL") + + for (v <- 0L to 3L) { + val expectedSchema = if (v < 2L) beforeSchema else afterSchema + snapshotSpec(t, version = v).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + assert(result.metadata.partitionColumns == (if (partitioned) Seq("c") else Seq.empty)) + case other => fail(s"${table.outputName} snapshot v$v must succeed: $other") + } + } + } + } + + test("string_to_int_stats") { + replaceSameName( + "string_to_int_stats", partitioned = false, StringType, IntegerType, + seedValues = """(1, 'not-an-int', 'old'), (1, 'not-an-int', 'old'), """ + + """(2, '007', 'old-padded'), (3, NULL, 'old-null')""", + replacementValues = """(101, 7, 'new-seven'), (101, 7, 'new-seven'), """ + + """(102, -5, 'new-negative'), (103, NULL, 'new-null'), (104, 0, 'new-zero')""", + beforeRows = Seq( + Row(1, "not-an-int", "old"), + Row(1, "not-an-int", "old"), + Row(2, "007", "old-padded"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, 7, "new-seven"), + Row(101, 7, "new-seven"), + Row(102, -5, "new-negative"), + Row(103, null, "new-null"), + Row(104, 0, "new-zero")), + matchingPredicate = "c = 7", missingPredicate = "c = 999") + } + + test("int_to_string_stats") { + replaceSameName( + "int_to_string_stats", partitioned = false, IntegerType, StringType, + seedValues = """(1, 7, 'old'), (1, 7, 'old'), (2, -5, 'old-negative'), """ + + """(3, NULL, 'old-null')""", + replacementValues = """(101, 'seven', 'new-word'), (101, 'seven', 'new-word'), """ + + """(102, '007', 'new-padded'), (103, NULL, 'new-null'), (104, '', 'new-empty')""", + beforeRows = Seq( + Row(1, 7, "old"), + Row(1, 7, "old"), + Row(2, -5, "old-negative"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "seven", "new-word"), + Row(101, "seven", "new-word"), + Row(102, "007", "new-padded"), + Row(103, null, "new-null"), + Row(104, "", "new-empty")), + matchingPredicate = "c = 'seven'", missingPredicate = "c = 'missing'") + } + + test("byte_to_string_stats") { + replaceSameName( + "byte_to_string_stats", partitioned = false, ByteType, StringType, + seedValues = """(1, CAST(-128 AS BYTE), 'old-min'), (1, CAST(-128 AS BYTE), 'old-min'), """ + + """(2, CAST(127 AS BYTE), 'old-max'), (3, CAST(NULL AS BYTE), 'old-null')""", + replacementValues = """(101, 'byte-min', 'new-word'), (101, 'byte-min', 'new-word'), """ + + """(102, '127', 'new-numeric'), (103, NULL, 'new-null'), (104, '', 'new-empty')""", + beforeRows = Seq( + Row(1, -128.toByte, "old-min"), + Row(1, -128.toByte, "old-min"), + Row(2, 127.toByte, "old-max"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "byte-min", "new-word"), + Row(101, "byte-min", "new-word"), + Row(102, "127", "new-numeric"), + Row(103, null, "new-null"), + Row(104, "", "new-empty")), + matchingPredicate = "c = 'byte-min'", missingPredicate = "c = 'missing'") + } + + test("string_to_int_partition") { + replaceSameName( + "string_to_int_partition", partitioned = true, StringType, IntegerType, + seedValues = """(1, 'not-an-int', 'old'), (1, 'not-an-int', 'old'), """ + + """(2, '007', 'old-padded'), (3, NULL, 'old-null')""", + replacementValues = """(101, 7, 'new-seven'), (101, 7, 'new-seven'), """ + + """(102, -5, 'new-negative'), (103, NULL, 'new-null'), (104, 0, 'new-zero')""", + beforeRows = Seq( + Row(1, "not-an-int", "old"), + Row(1, "not-an-int", "old"), + Row(2, "007", "old-padded"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, 7, "new-seven"), + Row(101, 7, "new-seven"), + Row(102, -5, "new-negative"), + Row(103, null, "new-null"), + Row(104, 0, "new-zero")), + matchingPredicate = "c = 7", missingPredicate = "c = 999") + } + + test("int_to_string_partition") { + replaceSameName( + "int_to_string_partition", partitioned = true, IntegerType, StringType, + seedValues = """(1, 7, 'old'), (1, 7, 'old'), (2, -5, 'old-negative'), """ + + """(3, NULL, 'old-null')""", + replacementValues = """(101, 'seven', 'new-word'), (101, 'seven', 'new-word'), """ + + """(102, '007', 'new-padded'), (103, NULL, 'new-null'), (104, 'zero', 'new-zero')""", + beforeRows = Seq( + Row(1, 7, "old"), + Row(1, 7, "old"), + Row(2, -5, "old-negative"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "seven", "new-word"), + Row(101, "seven", "new-word"), + Row(102, "007", "new-padded"), + Row(103, null, "new-null"), + Row(104, "zero", "new-zero")), + matchingPredicate = "c = 'seven'", missingPredicate = "c = 'missing'") + } + + test("byte_to_string_partition") { + replaceSameName( + "byte_to_string_partition", partitioned = true, ByteType, StringType, + seedValues = """(1, CAST(-128 AS BYTE), 'old-min'), (1, CAST(-128 AS BYTE), 'old-min'), """ + + """(2, CAST(127 AS BYTE), 'old-max'), (3, CAST(NULL AS BYTE), 'old-null')""", + replacementValues = """(101, 'byte-min', 'new-word'), (101, 'byte-min', 'new-word'), """ + + """(102, '127', 'new-numeric'), (103, NULL, 'new-null'), (104, 'zero', 'new-zero')""", + beforeRows = Seq( + Row(1, -128.toByte, "old-min"), + Row(1, -128.toByte, "old-min"), + Row(2, 127.toByte, "old-max"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "byte-min", "new-word"), + Row(101, "byte-min", "new-word"), + Row(102, "127", "new-numeric"), + Row(103, null, "new-null"), + Row(104, "zero", "new-zero")), + matchingPredicate = "c = 'byte-min'", missingPredicate = "c = 'missing'") + } + + // Changed partition/file sets and fresh mapped identities share the same replacement oracle. + import java.nio.file.Files + + import org.apache.spark.sql.types.{DoubleType, Metadata} + + import io.delta.workload.TableHandle + import io.delta.workload.json.JsonUtil + import io.delta.workload.log.CommitLog + + private def replacementRead( + t: TableHandle, + name: String, + rows: Seq[Row], + schema: StructType, + version: java.lang.Long = null, + predicate: String = null, + columns: Option[Seq[String]] = None): Unit = { + val table = io.delta.workload.WorkloadContext.current.getTableSpec(t) + val ref = readSpec(t, name = Some(name), version = version, predicate = predicate, + columns = columns) + ref.assert { spec => + val workloadName = table.outputName + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$workloadName/$name must succeed: $other") + } + val literal = spark.createDataFrame(rows.asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val path = outputDir.resolve(workloadName).resolve("expected") + .resolve(s"${workloadName}_${name}").resolve("expected_data") + val captured = spark.read.parquet(path.toString) + // RowComparison checks exact names/types and duplicate-sensitive row bags. + RowComparison.assertRowsEqual(oracle, captured, s"$workloadName/$name literal oracle") + assert(captured.schema.fields.forall(_.nullable)) + println(s"Independent row oracle passed: $workloadName/$name (${rows.size} rows)") + } + } + + private def replacementLayout( + caseName: String, + layout: String, + mappingMode: Option[String] = None): Unit = { + val mapped = mappingMode.isDefined + val parsed = layout == "checkpoint_parsed" + val checkpointed = layout != "commits" + val beforeNames = if (mapped) Seq("id", "label", "score", "region") + else Seq("id", "old_label", "old_score", "old_region") + val afterNames = Seq("id", "label", "score", "region") + def logicalSchema(names: Seq[String]): StructType = new StructType() + .add(names(0), IntegerType).add(names(1), StringType) + .add(names(2), DoubleType).add(names(3), StringType) + val beforeSchema = logicalSchema(beforeNames) + val afterSchema = logicalSchema(afterNames) + val beforeRows = Seq( + Row(1, "old", 1000.0, "old-east"), Row(1, "old", 1000.0, "old-east"), + Row(2, "old-negative", -1000.0, "old-west"), Row(3, null, null, null)) + val afterRows = Seq( + Row(101, "new-a", 7.5, "east"), Row(101, "new-a", 7.5, "east"), + Row(102, "new-b", -2.5, "west"), Row(103, null, null, null), + Row(104, "new-zero", 0.0, "east")) + val properties = Map( + "delta.checkpointInterval" -> "1000", + "delta.checkpoint.writeStatsAsJson" -> (!parsed).toString, + "delta.checkpoint.writeStatsAsStruct" -> parsed.toString) ++ + mappingMode.map("delta.columnMapping.mode" -> _).toMap + val propertySql = properties.toSeq.sortBy(_._1) + .map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + sql(s"""CREATE TABLE tbl (${beforeSchema.toDDL}) USING delta + PARTITIONED BY (${beforeNames.last}) TBLPROPERTIES ($propertySql)""") + sql("""INSERT INTO tbl VALUES + (1, 'old', 1000D, 'old-east'), (1, 'old', 1000D, 'old-east'), + (2, 'old-negative', -1000D, 'old-west'), (3, NULL, NULL, NULL)""") + + // The setting exists in OSS Delta 4.1.0 as well as Runtime. Restore after the DDL; + // it controls authoring only, and is not needed while queued read specs are captured. + val reuseKey = "spark.databricks.delta.columnMapping.reuseColumnMetadataDuringReplace" + val oldReuse = spark.conf.getOption(reuseKey) + try { + if (mapped) spark.conf.set(reuseKey, "false") + sql(s"""CREATE OR REPLACE TABLE tbl (${afterSchema.toDDL}) USING delta + PARTITIONED BY (region) TBLPROPERTIES ($propertySql)""") + } finally { + if (mapped) oldReuse match { + case Some(value) => spark.conf.set(reuseKey, value) + case None => spark.conf.unset(reuseKey) + } + } + sql("""INSERT INTO tbl VALUES + (101, 'new-a', 7.5D, 'east'), (101, 'new-a', 7.5D, 'east'), + (102, 'new-b', -2.5D, 'west'), (103, NULL, NULL, NULL), + (104, 'new-zero', 0D, 'east')""") + if (checkpointed) forceCheckpoint("tbl") + val t = registerTable("tbl") + val rawBefore = CommitLog.readRaw(t.sourcePath, 0L).map(JsonUtil.mapper.readTree) + .find(_.has("metaData")).get.get("metaData") + val rawAfter = CommitLog.readRaw(t.sourcePath, 2L).map(JsonUtil.mapper.readTree) + .find(_.has("metaData")).get.get("metaData") + val mappedBefore = DataType.fromJson(rawBefore.get("schemaString").asText) + .asInstanceOf[StructType] + val mappedAfter = DataType.fromJson(rawAfter.get("schemaString").asText) + .asInstanceOf[StructType] + def withoutMapping(schema: StructType): StructType = + StructType(schema.fields.map(_.copy(metadata = Metadata.empty))) + assert(withoutMapping(mappedBefore) == beforeSchema) + assert(withoutMapping(mappedAfter) == afterSchema) + if (mapped) { + val idKey = "delta.columnMapping.id" + val physicalKey = "delta.columnMapping.physicalName" + assert(mappedBefore.fields.map(_.metadata.getLong(idKey)).toSeq == Seq(1L, 2L, 3L, 4L)) + assert(mappedAfter.fields.map(_.metadata.getLong(idKey)).toSeq == Seq(5L, 6L, 7L, 8L)) + val oldPhysical = mappedBefore.fields.map(_.metadata.getString(physicalKey)).toSet + val newPhysical = mappedAfter.fields.map(_.metadata.getString(physicalKey)).toSet + assert(oldPhysical.size == 4 && newPhysical.size == 4) + assert(oldPhysical.intersect(newPhysical).isEmpty) + assert(newPhysical.intersect(afterNames.toSet).isEmpty) + } + val physicalNames = mappedAfter.fields.map { field => + if (mapped) field.metadata.getString("delta.columnMapping.physicalName") else field.name + }.toSeq + val statsNames = physicalNames.init.toSet + val partitionName = physicalNames.last + // Inspect new commit actions directly too, including the commit-only mapping variants. + val newAdds = CommitLog.readRaw(t.sourcePath, 3L).map(JsonUtil.mapper.readTree) + .filter(_.has("add")).map(_.get("add")) + assert(newAdds.nonEmpty) + for (add <- newAdds) { + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val stats = JsonUtil.mapper.readTree(add.get("stats").asText) + assert(stats.get("nullCount").fieldNames().asScala.toSet == statsNames) + for (kind <- Seq("minValues", "maxValues")) { + assert(stats.get(kind).fieldNames().asScala.toSet.subsetOf(statsNames)) + } + } + + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + // Retain exact pre-truncation actions in command evidence, including operation/removes. + for (v <- 0L to 3L; line <- CommitLog.readRaw(tableDir, v)) { + println(s"Replacement source action: $caseName v$v $line") + } + val stream = Files.list(logDir) + val files = try stream.iterator().asScala.toSeq finally stream.close() + if (checkpointed) { + val checkpoints = files.filter(_.getFileName.toString.endsWith(".checkpoint.parquet")) + assert(checkpoints.size == 1) + assert(checkpoints.head.getFileName.toString == "00000000000000000003.checkpoint.parquet") + val checkpoint = spark.read.parquet(checkpoints.head.toString) + val addSchema = checkpoint.schema("add").dataType.asInstanceOf[StructType] + assert(addSchema.fieldNames.contains("stats") == !parsed) + assert(addSchema.fieldNames.contains("stats_parsed") == parsed) + assert(addSchema.fieldNames.contains("partitionValues_parsed") == parsed) + if (parsed) { + val partitions = addSchema("partitionValues_parsed").dataType.asInstanceOf[StructType] + assert(partitions.fieldNames.toSeq == Seq(partitionName)) + assert(partitions.fields.head.dataType == StringType) + val stats = addSchema("stats_parsed").dataType.asInstanceOf[StructType] + for (kind <- Seq("minValues", "maxValues", "nullCount")) { + assert(stats(kind).dataType.asInstanceOf[StructType].fieldNames.toSet == statsNames) + } + } + val adds = checkpoint.filter("add IS NOT NULL").select("add").toJSON.collect() + assert(adds.nonEmpty) + for (json <- adds) { + val add = JsonUtil.mapper.readTree(json).get("add") + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val stats = if (parsed) add.get("stats_parsed") + else JsonUtil.mapper.readTree(add.get("stats").asText) + assert(stats.get("nullCount").fieldNames().asScala.toSet == statsNames) + for (kind <- Seq("minValues", "maxValues")) { + assert(stats.get(kind).fieldNames().asScala.toSet.subsetOf(statsNames)) + } + } + // Keep only the checkpoint-era JSON for version discovery. Earlier metadata and file + // state must come from the checkpoint; remove protocol CRC shortcuts as well. + files.filter { path => + val name = path.getFileName.toString + name.endsWith(".crc") || + (name.matches("[0-9]{20}\\.json") && name != "00000000000000000003.json") + }.foreach(Files.delete) + } else { + assert(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + } + println(s"Replacement layout oracle passed: $caseName ($layout)") + } + + replacementRead(t, "read_all", afterRows, afterSchema) + replacementRead(t, "read_projection", afterRows, afterSchema, + columns = Some(Seq("region", "score", "label"))) + replacementRead(t, "read_partition", afterRows.take(2) ++ afterRows.drop(4), + afterSchema, predicate = "region = 'east'") + replacementRead(t, "read_stats", afterRows.take(2), afterSchema, + predicate = "score > 7D") + replacementRead(t, "read_missing", Seq.empty, afterSchema, + predicate = "region = 'missing' OR score > 999D") + replacementRead(t, "read_null", afterRows.slice(3, 4), afterSchema, + predicate = "region IS NULL AND score IS NULL AND label IS NULL") + replacementRead(t, "read_old_stats", Seq.empty, afterSchema, + predicate = "score = 1000D OR label = 'old'") + if (!checkpointed) { + replacementRead(t, "read_before", beforeRows, beforeSchema, version = 1L) + replacementRead(t, "read_replaced_empty", Seq.empty, afterSchema, version = 2L) + } + for (v <- (if (checkpointed) Seq(3L) else Seq(1L, 2L, 3L))) { + val metadata = if (v < 2L) rawBefore else rawAfter + val expectedProperties = properties ++ mappingMode.map { mode => + Map("delta.columnMapping.mode" -> mode, + "delta.columnMapping.maxColumnId" -> (if (v < 2L) "4" else "8")) + }.getOrElse(Map.empty[String, String]) + snapshotSpec(t, version = v).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.get("schemaString").asText) + assert(result.metadata.partitionColumns == + Seq(if (v < 2L) beforeNames.last else "region")) + assert(result.metadata.configuration == expectedProperties) + case other => fail(s"$caseName snapshot v$v must succeed: $other") + } + println(s"Independent metadata oracle passed: $caseName v$v") + } + } + } + + test("changed_sets_checkpoint_json") { + replacementLayout("changed_sets_checkpoint_json", "checkpoint_json") + } + + test("changed_sets_checkpoint_parsed") { + replacementLayout("changed_sets_checkpoint_parsed", "checkpoint_parsed") + } + + test("mapped_name_commits") { + replacementLayout("mapped_name_commits", "commits", Some("name")) + } + + test("mapped_name_checkpoint_json") { + replacementLayout("mapped_name_checkpoint_json", "checkpoint_json", Some("name")) + } + + test("mapped_name_checkpoint_parsed") { + replacementLayout("mapped_name_checkpoint_parsed", "checkpoint_parsed", Some("name")) + } + + test("mapped_id_commits") { + replacementLayout("mapped_id_commits", "commits", Some("id")) + } + + test("mapped_id_checkpoint_json") { + replacementLayout("mapped_id_checkpoint_json", "checkpoint_json", Some("id")) + } + + test("mapped_id_checkpoint_parsed") { + replacementLayout("mapped_id_checkpoint_parsed", "checkpoint_parsed", Some("id")) + } + +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessStatisticsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessStatisticsSuite.scala new file mode 100644 index 00000000..cd7200d1 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessStatisticsSuite.scala @@ -0,0 +1,1011 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.nio.file.{Files, Path} +import java.sql.Timestamp +import java.time.Instant + +import scala.collection.JavaConverters._ + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.spark.sql.Row +import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.types.{ArrayType, BooleanType, DataType, DecimalType, IntegerType} +import org.apache.spark.sql.types.{StringType, StructType, TimestampType} +import org.apache.spark.unsafe.types.UTF8String + +import io.delta.workload.WorkloadTestSuite +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog, Metadata, RemoveFile} +import io.delta.workload.model.Succeeded + +/** Portable read specifications for incomplete or selected file statistics. */ +class ReadinessStatisticsSuite extends WorkloadTestSuite("readiness_statistics") { + test("missing_add_stats_predicates") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("flag", BooleanType).add("items", ArrayType(IntegerType, containsNull = true)) + val nullArray = Row(1, 1, true, null) + val emptyArray = Row(2, 2, false, Seq.empty[Integer]) + val nullElement = Row(3, null, null, Seq[Integer](null)) + val duplicate = Row(4, 3, true, Seq[Integer](1, 7)) + val nonmatching = Row(5, 4, false, Seq[Integer](2, 3)) + val matchingWithNull = Row(6, 5, null, Seq[Integer](7, null)) + val nonmatchingWithNull = Row(7, 2, true, Seq[Integer](null, 2)) + val matching = Row(8, null, false, Seq[Integer](7)) + val batches = Seq(Seq(nullArray, emptyArray, nullElement, duplicate, duplicate), + Seq(nonmatching, matchingWithNull, nonmatchingWithNull, matching)) + val rows = batches.flatten + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT, flag BOOLEAN, items ARRAY) USING delta + TBLPROPERTIES ('delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '100')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 1, TRUE, CAST(NULL AS ARRAY)), (2, 2, FALSE, CAST(array() AS ARRAY)), + (3, NULL, NULL, array(CAST(NULL AS INT))), (4, 3, TRUE, array(1, 7)), + (4, 3, TRUE, array(1, 7)) AS v(id, value, flag, items)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (5, 4, FALSE, array(2, 3)), (6, 5, NULL, array(7, CAST(NULL AS INT))), + (7, 2, TRUE, array(CAST(NULL AS INT), 2)), (8, NULL, FALSE, array(7)) + AS v(id, value, flag, items)""") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val metadata = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + batches.zipWithIndex.foreach { case (batch, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.stats.isDefined) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"predicate_input_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + // Arrays have no scalar min/max statistics; their predicates require residual filtering. + require(!stats.path("minValues").has("items") && !stats.path("maxValues").has("items")) + } + val stream = Files.list(dir.resolve("_delta_log")) + val names = try stream.iterator().asScala.map(_.getFileName.toString).toVector + finally stream.close() + require(!names.exists(_.contains("checkpoint"))) + // A cached allFiles set in a version CRC must not hide the missing AddFile statistics. + names.filter(_.matches("[0-9]{20}\\.crc")) + .foreach(name => Files.delete(dir.resolve("_delta_log").resolve(name))) + } + for (version <- 1L to 2L) { + modifyCommitActions(t, version) { actions => + actions.map { + case a: AddFile => a.copy(stats = None) + case other => other + } + } + } + mutateTable(t) { dir => + val actions = (0L to 2L).flatMap(v => CommitLog.read(dir, v)) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 2 && adds.map(_.path).distinct.size == 2) + require(adds.forall(_.stats.isEmpty), "Every persisted AddFile must omit statistics") + require(!actions.exists(_.isInstanceOf[RemoveFile])) + } + + // Expected bags come from literal values and SQL three-valued logic, without a Delta scan. + val casePredicate = "CASE WHEN value IN (1, 3) THEN TRUE " + + "WHEN value = 2 THEN FALSE ELSE CAST(NULL AS BOOLEAN) END" + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("ne", "value <> 2", Seq(nullArray, duplicate, duplicate, nonmatching, matchingWithNull)), + ("in", "value IN (1, 3, 99)", Seq(nullArray, duplicate, duplicate)), + ("in_with_null", "value IN (2, NULL)", Seq(emptyArray, nonmatchingWithNull)), + ("in_miss", "value IN (99, NULL)", Seq.empty), + ("is_not_true", "flag IS NOT TRUE", + Seq(emptyArray, nullElement, nonmatching, matchingWithNull, matching)), + ("array_contains", "array_contains(items, 7)", + Seq(duplicate, duplicate, matchingWithNull, matching)), + ("array_contains_miss", "array_contains(items, 99)", Seq.empty), + ("array_contains_is_not_true", "array_contains(items, 7) IS NOT TRUE", + Seq(nullArray, emptyArray, nullElement, nonmatching, nonmatchingWithNull)), + ("array_contains_not", "NOT array_contains(items, 7)", Seq(emptyArray, nonmatching)), + ("case_when", casePredicate, Seq(nullArray, duplicate, duplicate)), + ("case_when_is_not_true", s"($casePredicate) IS NOT TRUE", + Seq(emptyArray, nullElement, nonmatching, matchingWithNull, nonmatchingWithNull, matching))) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful missing-stat $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_missing_stats_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful missing-stat snapshot, got $other") + } + } + } + + for (variant <- Seq("no_null_count", "no_num_records", "neither_count", "partial_null_count")) { + test(s"partial_counts_$variant") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + val batches = Seq( + Seq(Row(1, 10), Row(2, null), Row(3, 30), Row(3, 30)), + Seq(Row(4, 40), Row(5, 50)), + Seq(Row(6, null), Row(7, null)), + Seq(Row(8, 80), Row(9, null))) + val rows = batches.flatten + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + // One mixed/all-null/non-null file per commit; duplicate rows must survive reads. + val values = batch.map { r => + s"(${r.getInt(0)}, ${if (r.isNullAt(1)) "NULL" else r.getInt(1).toString})" + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS v(id, value)") + } + val t = registerTable("tbl") + var copiedTable: Path = null + + // Check real AddFiles and their row-derived stats, then mutate only the copied table. + mutateTable(t) { dir => + copiedTable = dir + val initial = CommitLog.read(dir, 0) + val originalMetadata = initial.collect { case m: Metadata => m }.head + require(DataType.fromJson(originalMetadata.schemaString) == schema) + require(originalMetadata.configuration == properties) + require(originalMetadata.partitionColumns.isEmpty) + val logFiles = Files.list(dir.resolve("_delta_log")) + try { + val names = logFiles.iterator().asScala.map(_.getFileName.toString).toSeq + require(!names.exists(_.contains("checkpoint")), "Commit replay must be required") + // Version CRCs can contain a cached allFiles set with the original stats. + names.filter(_.matches("[0-9]{20}\\.crc")).foreach { name => + Files.delete(dir.resolve("_delta_log").resolve(name)) + } + } finally logFiles.close() + + batches.zipWithIndex.foreach { case (batch, index) => + val version = index.toLong + 1 + val actions = CommitLog.read(dir, version) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 1, s"Expected one AddFile at version $version") + val add = adds.head + require(add.deletionVector.isEmpty && add.baseRowId.isEmpty && + add.defaultRowCommitVersion.isEmpty) + val oracle = spark.createDataFrame(batch.asJava, schema) + RowComparison.assertRowsEqual(oracle, + spark.read.parquet(dir.resolve(add.path).toString), s"input_v$version") + val stats = JsonUtil.mapper.readTree(add.stats.get).asInstanceOf[ObjectNode] + require(stats.path("numRecords").asLong(-1) == batch.size) + for ((column, ordinal) <- Seq("id" -> 0, "value" -> 1)) { + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + if (values.nonEmpty) { + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + } + val before = stats.deepCopy() + // Keep the last file intact as a control for per-file partial statistics. + if (version < 4) { + variant match { + case "no_null_count" => stats.remove("nullCount") + case "no_num_records" => stats.remove("numRecords") + case "neither_count" => + stats.remove("nullCount") + stats.remove("numRecords") + case "partial_null_count" => + if (version == 3) stats.remove("nullCount") + else stats.get("nullCount").asInstanceOf[ObjectNode] + .remove(if (version == 1) "value" else "id") + } + require(stats != before, s"Mutation must change stats at version $version") + } else require(stats == before) + require(stats.get("minValues") == before.get("minValues")) + require(stats.get("maxValues") == before.get("maxValues")) + val changed = add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + CommitLog.write(dir, version, actions.map { + case _: AddFile => changed + case other => other + }) + val reread = CommitLog.read(dir, version).collect { case a: AddFile => a } + require(reread == Seq(changed), s"Persisted mutation mismatch at version $version") + require(JsonUtil.mapper.readTree(reread.head.stats.get) == stats) + } + } + + // These bags are derived directly from the literal input, without a Delta scan or stats. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("value_is_null", "value IS NULL", rows.filter(_.isNullAt(1))), + ("value_is_not_null", "value IS NOT NULL", rows.filterNot(_.isNullAt(1))), + ("value_eq_30", "value = 30", Seq(Row(3, 30), Row(3, 30))), + ("value_gt_35", "value > 35", Seq(Row(4, 40), Row(5, 50), Row(8, 80))), + ("value_eq_999", "value = 999", Seq.empty), + ("id_is_null", "id IS NULL", Seq.empty), + ("id_is_not_null", "id IS NOT NULL", rows)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful snapshot, got $other") + } + } + } + } + + test("remove_stats_without_active_add_stats") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + val batches = Seq( + Seq(Row(1, 10), Row(5, null)), + Seq(Row(2, 20), Row(3, 30), Row(3, 30), Row(4, null)), + Seq(Row(100, 1000), Row(101, 1010))) + val rewrittenRows = Seq(Row(3, 30), Row(3, 30), Row(4, null)) + val survivors = batches.head ++ rewrittenRows + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { r => + s"(${r.getInt(0)}, ${if (r.isNullAt(1)) "NULL" else r.getInt(1).toString})" + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS v(id, value)") + } + // Rewrite one file, remove another entirely, and leave the first file untouched. + sql("DELETE FROM tbl WHERE id = 2 OR id >= 100") + val t = registerTable("tbl") + var copiedTable: Path = null + + mutateTable(t) { dir => + copiedTable = dir + val commits = (0L to 4L).map(v => v -> CommitLog.read(dir, v)).toMap + val originalMetadata = commits(0).collect { case m: Metadata => m }.head + require(DataType.fromJson(originalMetadata.schemaString) == schema) + require(originalMetadata.configuration == properties) + require(originalMetadata.partitionColumns.isEmpty) + require(commits(0).collect { case a: AddFile => a }.isEmpty) + val logFiles = Files.list(dir.resolve("_delta_log")) + try { + val names = logFiles.iterator().asScala.map(_.getFileName.toString).toSeq + require(!names.exists(_.contains("checkpoint")), "Commit replay must be required") + // Prevent a version CRC's cached allFiles from hiding the stats mutation. + names.filter(_.matches("[0-9]{20}\\.crc")).foreach { name => + Files.delete(dir.resolve("_delta_log").resolve(name)) + } + } finally logFiles.close() + + val inputAdds = batches.zipWithIndex.map { case (batch, index) => + val adds = commits(index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1, s"Expected one input AddFile for batch $index") + require(!commits(index.toLong + 1).exists(_.isInstanceOf[RemoveFile])) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"original_batch_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for ((column, ordinal) <- Seq("id" -> 0, "value" -> 1)) { + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + add + } + val allAdds = commits.values.toSeq.flatten.collect { case a: AddFile => a } + val addsByPath = allAdds.map(a => a.path -> a).toMap + require(addsByPath.size == allAdds.size, "Each path must have one original Add") + require(allAdds.forall(a => a.stats.isDefined && a.partitionValues.isEmpty && + a.deletionVector.isEmpty && a.baseRowId.isEmpty && a.defaultRowCommitVersion.isEmpty)) + val removes = commits(4).collect { case r: RemoveFile => r } + val removedPaths = removes.map(_.path).toSet + require(removes.size == 2 && removedPaths == inputAdds.tail.map(_.path).toSet) + removes.foreach { r => + val original = addsByPath(r.path) + require(r.dataChange && r.deletionTimestamp.isDefined) + require(r.extendedFileMetadata.contains(true)) + require(r.partitionValues.contains(original.partitionValues)) + require(r.size.contains(original.size) && r.deletionVector.isEmpty) + require(r.baseRowId.isEmpty && r.defaultRowCommitVersion.isEmpty) + require(r.stats.forall(s => JsonUtil.mapper.readTree(s) == + JsonUtil.mapper.readTree(original.stats.get))) + } + val rewritten = commits(4).collect { case a: AddFile => a } + require(rewritten.nonEmpty) + RowComparison.assertRowsEqual(spark.createDataFrame(rewrittenRows.asJava, schema), + rewritten.map(a => spark.read.parquet(dir.resolve(a.path).toString)).reduce(_.union(_)), + "rewritten_survivors") + val activePaths = addsByPath.keySet -- removedPaths + require(activePaths == (rewritten.map(_.path).toSet + inputAdds.head.path)) + + for (version <- 1L to 4L) { + val changed = commits(version).map { + case a: AddFile if activePaths.contains(a.path) => a.copy(stats = None) + // Some writers omit Remove stats. Supply the exact, row-checked original Add stats. + case r: RemoveFile => r.copy(stats = addsByPath(r.path).stats) + case other => other + } + CommitLog.write(dir, version, changed) + require(CommitLog.read(dir, version) == changed, + s"Persisted action mutation mismatch at version $version") + } + val persisted = (0L to 4L).flatMap(v => CommitLog.read(dir, v)) + val persistedAdds = persisted.collect { case a: AddFile => a } + val persistedRemoves = persisted.collect { case r: RemoveFile => r } + require(persistedAdds.filter(a => activePaths.contains(a.path)).forall(_.stats.isEmpty)) + require(persistedAdds.filter(a => removedPaths.contains(a.path)).forall { a => + a == addsByPath(a.path) + }) + require(persistedRemoves.size == 2 && persistedRemoves.forall { r => + r.stats.isDefined && r.stats == addsByPath(r.path).stats + }) + } + + // Literal row bags, independent of Delta replay and statistics; duplicates are intentional. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, survivors), + ("hit_untouched", "id = 1", Seq(Row(1, 10))), + ("hit_rewritten", "id = 3", Seq(Row(3, 30), Row(3, 30))), + ("range", "value >= 30", Seq(Row(3, 30), Row(3, 30))), + ("null", "value IS NULL", Seq(Row(5, null), Row(4, null))), + ("miss", "id = 999", Seq.empty), + ("deleted_partial", "id = 2", Seq.empty), + ("deleted_file", "id >= 100", Seq.empty), + ("mixed", "id IN (2, 3, 100)", Seq(Row(3, 30), Row(3, 30)))) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful snapshot, got $other") + } + } + } + + test("unicode_string_stats_predicates") { + val schema = new StructType().add("id", IntegerType).add("s", StringType) + val head = "a" * 31 + // Exact source rows from WriteStatsSuite.utf8_boundary_string_min_max; no NULL rows. + // This independent read fixture avoids depending on another suite's generated output. + val rows = Seq(Row(1, head + "étail"), Row(2, head + "中tail"), Row(3, head + "😀")) + sql("CREATE TABLE tbl (id INT, s STRING) USING delta") + rows.foreach { row => + // Separate files give the three Unicode boundaries distinct min/max statistics. + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (${row.getInt(0)}, '${row.getString(1)}') AS v(id, s)""") + } + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + rows.zipWithIndex.foreach { case (row, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(Seq(row).asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"unicode_input_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == 1) + require(stats.path("nullCount").path("s").asLong(-1) == 0) + val value = row.getString(1) + val minimum = stats.path("minValues").path("s").asText() + val maximum = stats.path("maxValues").path("s").asText() + require(minimum.startsWith(head) && minimum.codePointAt(31) == value.codePointAt(31)) + require(UTF8String.fromString(minimum).compareTo(UTF8String.fromString(value)) <= 0) + require(UTF8String.fromString(value).compareTo(UTF8String.fromString(maximum)) <= 0) + } + } + + // Literal bags use UTF-8 binary ordering: e-acute < CJK < supplementary emoji. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("equal_2byte", s"s = '${head}étail'", Seq(rows(0))), + ("equal_3byte", s"s = '${head}中tail'", Seq(rows(1))), + ("equal_4byte", s"s = '${head}😀'", Seq(rows(2))), + ("range_2_to_3byte", s"s >= '${head}étail' AND s < '${head}中tail'", Seq(rows(0))), + ("range_3_to_4byte", s"s >= '${head}中tail' AND s < '${head}😀'", Seq(rows(1))), + ("range_from_4byte", s"s >= '${head}😀'", Seq(rows(2))), + ("range_through_4byte", s"s > '${head}étail' AND s <= '${head}😀'", rows.tail), + ("miss_same_prefix", s"s = '${head}étail_missing'", Seq.empty), + ("miss_above_4byte", s"s > '${head}😀'", Seq.empty), + ("is_null", "s IS NULL", Seq.empty), + ("is_not_null", "s IS NOT NULL", rows)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful Unicode $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_unicode_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + case other => fail(s"Expected successful Unicode snapshot, got $other") + } + } + } + + for (bounds <- Seq("true", "false", "absent")) { + test(s"tight_bounds_$bounds") { + val schema = new StructType().add("id", IntegerType).add("all_null", IntegerType) + .add("mixed_null", IntegerType).add("non_null", IntegerType) + val physical = Seq(Row(1, null, 10, 10), Row(2, null, null, 20), + Row(3, null, 30, 30), Row(3, null, 30, 30), Row(4, null, null, 40)) + val live = Seq(Row(2, null, null, 20), Row(3, null, 30, 30), Row(3, null, 30, 30)) + val control = physical.map(r => Row(r.getInt(0) + 100, r.get(1), r.get(2), r.get(3))) + // Existing cases cover non-DV partial counts; explicit bounds need controls here. + val batches = Seq.fill(3)(physical) ++ + (if (bounds == "absent") Seq.empty else Seq.fill(2)(control)) + val rows = Seq.fill(3)(live).flatten ++ + (if (bounds == "absent") Seq.empty else Seq.fill(2)(control).flatten) + val properties = Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, all_null INT, mixed_null INT, non_null INT) + USING delta TBLPROPERTIES ('delta.enableDeletionVectors' = 'true', + 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { row => + row.toSeq.map(v => if (v == null) "NULL" else v.toString).mkString("(", ", ", ")") + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values " + + "AS v(id, all_null, mixed_null, non_null)") + } + sql("DELETE FROM tbl WHERE id = 1 OR id = 4") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val lastVersion = batches.size.toLong + 1 + val commits = (0L to lastVersion).map(v => v -> CommitLog.read(dir, v)).toMap + val metadata = commits(0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + val originals = batches.indices.map { index => + val adds = commits(index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batches(index).asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"physical_$bounds/$index") + add + } + val deleted = commits(lastVersion) + val dvAdds = deleted.collect { case a: AddFile => a } + val removes = deleted.collect { case r: RemoveFile => r } + require(dvAdds.size == 3 && removes.size == 3) + require(dvAdds.map(_.path).toSet == originals.take(3).map(_.path).toSet) + require(removes.map(_.path).toSet == dvAdds.map(_.path).toSet) + require(dvAdds.forall(a => a.deletionVector.exists(_.cardinality == 2))) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, schema), + spark.read.format("delta").load(dir.toString), s"live_before_mutation_$bounds") + + val stream = Files.list(dir.resolve("_delta_log")) + val files = try stream.iterator().asScala.toVector finally stream.close() + require(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.matches("[0-9]{20}\\.crc")).foreach(Files.delete) + for (version <- 1L to lastVersion) { + val changed = commits(version).map { + case add: AddFile if add.deletionVector.isDefined || + originals.drop(3).exists(_.path == add.path) => + val index = originals.indexWhere(_.path == add.path) + require(index >= 0 && add.partitionValues.isEmpty) + val valid = if (index < 3) live else control + val basis = if (bounds == "false") batches(index) else valid + val stats = JsonUtil.mapper.createObjectNode() + // DV numRecords is always physical, including when tight null counts are live. + stats.put("numRecords", batches(index).size) + if (bounds != "absent") stats.put("tightBounds", bounds == "true") + val minimum = stats.putObject("minValues") + val maximum = stats.putObject("maxValues") + val nulls = stats.putObject("nullCount") + schema.fieldNames.zipWithIndex.foreach { case (column, ordinal) => + val values = basis.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + if (values.nonEmpty) { + minimum.put(column, values.min) + maximum.put(column, values.max) + } + nulls.put(column, basis.count(_.isNullAt(ordinal))) + } + index match { + case 1 => stats.remove("nullCount") + case 2 => + nulls.remove("all_null") + nulls.remove("mixed_null") + case 3 => stats.remove("numRecords") + case 4 => + stats.remove("numRecords") + stats.remove("nullCount") + case _ => + } + require(stats.has("tightBounds") == (bounds != "absent")) + require(stats.has("numRecords") == (index < 3)) + if (index < 3) require(stats.path("numRecords").asInt() == 5) + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + CommitLog.write(dir, version, changed) + require(CommitLog.read(dir, version) == changed, + s"Persisted bounds mutation mismatch at version $version") + } + } + + // SQL three-valued predicates applied to literal row bags, independent of file stats. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("all_null", "all_null IS NULL", rows), + ("all_null_not_null", "all_null IS NOT NULL", Seq.empty), + ("all_null_equal", "all_null = 10", Seq.empty), + ("mixed_null", "mixed_null IS NULL", rows.filter(_.isNullAt(2))), + ("mixed_not_null", "mixed_null IS NOT NULL", rows.filterNot(_.isNullAt(2))), + ("mixed_equal", "mixed_null = 30", rows.filter(r => !r.isNullAt(2) && + r.getInt(2) == 30)), + ("non_null_is_null", "non_null IS NULL", Seq.empty), + ("non_null_not_null", "non_null IS NOT NULL", rows), + ("surviving_duplicates", "id = 3", Seq.fill(3)(live.tail).flatten), + ("deleted_lower_bound", "id = 1", Seq.empty), + ("deleted_upper_bound", "id = 4", Seq.empty), + ("deleted_mixed_value", "id < 100 AND mixed_null = 10", Seq.empty), + ("surviving_range", "non_null >= 20 AND non_null <= 30", + rows.filter(r => r.getInt(3) >= 20 && r.getInt(3) <= 30)), + ("outside_range", "non_null > 40", Seq.empty)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $bounds/$name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$bounds/$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful $bounds snapshot, got $other") + } + } + } + } + + test("indexed_prefix_beyond_32") { + val columns = (0 until 36).map(i => f"c$i%02d") + val schema = columns.foldLeft(new StructType())((s, c) => s.add(c, IntegerType)) + // The first 32 fields are a fixed prefix; c32/c33 are selected, c34/c35 are not. + val prefix = 0 until 32 + val duplicate = Row.fromSeq(prefix ++ Seq(320, 330, 340, 350)) + val mixed = Row.fromSeq(prefix ++ Seq(321, 331, 341, 351)) + val nullLate = Row.fromSeq(prefix ++ Seq(null, null, null, null)) + val batches = Seq(Seq(duplicate, duplicate), Seq(mixed, nullLate)) + val rows = batches.flatten + val properties = Map("delta.dataSkippingNumIndexedCols" -> "34", + "delta.enableDeletionVectors" -> "false", "delta.checkpointInterval" -> "100") + val ddl = columns.map(c => s"$c INT").mkString(", ") + sql(s"""CREATE TABLE tbl ($ddl) USING delta TBLPROPERTIES ( + 'delta.dataSkippingNumIndexedCols' = '34', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { row => + row.toSeq.map(v => if (v == null) "NULL" else v.toString).mkString("(", ", ", ")") + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values " + + s"AS v(${columns.mkString(", ")})") + } + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val metadata = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + batches.zipWithIndex.foreach { case (batch, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"wide_input_$index") + // Verify the writer's physical AddFile stats, without assuming a reader's stats schema. + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + require(stats.path(field).fieldNames().asScala.toSet == columns.take(34).toSet) + } + columns.take(34).zipWithIndex.foreach { case (column, ordinal) => + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + } + } + + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("selected_c32_equal", "c32 = 320", Seq(duplicate, duplicate)), + ("selected_c33_equal", "c33 = 330", Seq(duplicate, duplicate)), + ("selected_c33_range", "c33 >= 330 AND c33 < 331", Seq(duplicate, duplicate)), + ("selected_c33_null", "c33 IS NULL", Seq(nullLate)), + ("selected_c33_miss", "c33 = 999", Seq.empty), + ("unselected_c34_equal", "c34 = 340", Seq(duplicate, duplicate)), + ("unselected_c34_range", "c34 > 340", Seq(mixed)), + ("unselected_c34_null", "c34 IS NULL", Seq(nullLate)), + ("unselected_c34_miss", "c34 = 999", Seq.empty)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful wide $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_wide_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful wide snapshot, got $other") + } + } + } + + test("alter_selected_stats_columns") { + val schema = new StructType().add("event_ts", TimestampType) + .add("customer", new StructType().add("country", StringType).add("segment", StringType)) + .add("amount", DecimalType(10, 2)).add("note", StringType) + val oldDuplicate = Row(Timestamp.from(Instant.parse("2024-01-01T00:00:00Z")), + Row("US", "retail"), BigDecimal("10.50").bigDecimal, "alpha") + val oldOther = Row(Timestamp.from(Instant.parse("2024-01-02T00:00:00Z")), + Row("CA", "wholesale"), BigDecimal("20.00").bigDecimal, "beta") + val newDuplicate = Row(Timestamp.from(Instant.parse("2024-01-03T00:00:00Z")), + Row("FR", "retail"), BigDecimal("30.25").bigDecimal, "gamma") + val newOther = Row(Timestamp.from(Instant.parse("2024-01-04T00:00:00Z")), + Row("US", "business"), BigDecimal("-5.00").bigDecimal, "alpha") + val nullRow = Row(null, Row(null, null), null, null) + val before = Seq(oldDuplicate, oldDuplicate, oldOther) + val after = Seq(newDuplicate, newDuplicate, newOther, nullRow) + val rows = before ++ after + val property = "delta.dataSkippingStatsColumns" + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100", property -> "note,customer.segment") + val selected = "event_ts,customer.country,amount" + sql("""CREATE TABLE tbl (event_ts TIMESTAMP, customer STRUCT, + amount DECIMAL(10, 2), note STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100', + 'delta.dataSkippingStatsColumns' = 'note,customer.segment')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (TIMESTAMP '2024-01-01T00:00:00Z', named_struct('country','US','segment','retail'), + 10.50, 'alpha'), + (TIMESTAMP '2024-01-01T00:00:00Z', named_struct('country','US','segment','retail'), + 10.50, 'alpha'), + (TIMESTAMP '2024-01-02T00:00:00Z', named_struct('country','CA','segment','wholesale'), + 20.00, 'beta') AS v(event_ts, customer, amount, note)""") + sql(s"ALTER TABLE tbl SET TBLPROPERTIES ('$property' = '$selected')") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (TIMESTAMP '2024-01-03T00:00:00Z', named_struct('country','FR','segment','retail'), + 30.25, 'gamma'), + (TIMESTAMP '2024-01-03T00:00:00Z', named_struct('country','FR','segment','retail'), + 30.25, 'gamma'), + (TIMESTAMP '2024-01-04T00:00:00Z', named_struct('country','US','segment','business'), + -5.00, 'alpha'), + (NULL, named_struct('country',NULL,'segment',NULL), NULL, NULL) + AS v(event_ts, customer, amount, note)""") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val initial = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + val changed = CommitLog.read(dir, 2) + val metadata = changed.collect { case m: Metadata => m }.head + require(initial.configuration == properties) + require(metadata.configuration == properties.updated(property, selected)) + require(initial.id == metadata.id && initial.schemaString == metadata.schemaString) + require(DataType.fromJson(metadata.schemaString) == schema) + require(initial.partitionColumns.isEmpty && metadata.partitionColumns.isEmpty) + require(!changed.exists(a => a.isInstanceOf[AddFile] || a.isInstanceOf[RemoveFile]), + "ALTER must change metadata without rewriting files or backfilling statistics") + for ((version, batch) <- Seq(1L -> before, 3L -> after)) { + val actions = CommitLog.read(dir, version) + require(!actions.exists(_.isInstanceOf[RemoveFile])) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"selected_stats_input_v$version") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + val expectedFields = if (version == 1) Set("customer", "note") + else Set("event_ts", "customer", "amount") + require(stats.path(field).fieldNames().asScala.toSet == expectedFields) + require(stats.path(field).path("customer").fieldNames().asScala.toSet == + (if (version == 1) Set("segment") else Set("country"))) + } + } + } + val queries = Seq[(String, java.lang.Long, String, Seq[Row])]( + ("before_all", 1L, null, before), + ("before_timestamp", 1L, "event_ts = TIMESTAMP '2024-01-01T00:00:00Z'", + Seq(oldDuplicate, oldDuplicate)), + ("all", null, null, rows), + ("selected_timestamp_equal", null, "event_ts = TIMESTAMP '2024-01-01T00:00:00Z'", + Seq(oldDuplicate, oldDuplicate)), + ("selected_timestamp_range", null, "event_ts >= TIMESTAMP '2024-01-03T00:00:00Z'", + Seq(newDuplicate, newDuplicate, newOther)), + ("selected_country_equal", null, "customer.country = 'US'", + Seq(oldDuplicate, oldDuplicate, newOther)), + ("selected_country_null", null, "customer.country IS NULL", Seq(nullRow)), + ("selected_amount_equal", null, "amount = 30.25", Seq(newDuplicate, newDuplicate)), + ("selected_amount_range", null, "amount >= 10.50 AND amount <= 20.00", before), + ("selected_amount_null", null, "amount IS NULL", Seq(nullRow)), + ("selected_country_miss", null, "customer.country = 'ZZ'", Seq.empty), + ("excluded_note_equal", null, "note = 'alpha'", Seq(oldDuplicate, oldDuplicate, newOther)), + ("excluded_segment_equal", null, "customer.segment = 'retail'", + Seq(oldDuplicate, oldDuplicate, newDuplicate, newDuplicate)), + ("excluded_note_null", null, "note IS NULL", Seq(nullRow)), + ("selected_combined", null, + "event_ts >= TIMESTAMP '2024-01-03T00:00:00Z' AND " + + "customer.country = 'FR' AND amount = 30.25", Seq(newDuplicate, newDuplicate))) + queries.foreach { case (name, version, predicate, expectedRows) => + readSpec(t, version = version, predicate = predicate, name = Some(s"read_$name")) + .assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful selected-stats $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_selected_stats_$name") + } + } + for (version <- Seq[java.lang.Long](1L, null)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + // This asserts the table schema and properties, not Kernel's internal stats schema. + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == + (if (version == null) properties.updated(property, selected) else properties)) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful selected-stats snapshot, got $other") + } + } + } + } + + // One checkpoint matrix serves both missing evolved-column stats and nullable evolution. + // Commit/time-travel coverage stays in DataSkippingSuite and SchemaEvolutionSuite. + for (parsed <- Seq(false, true)) { + val variant = if (parsed) "parsed" else "json_only" + test(s"nullable_evolution_checkpoint_$variant") { + val schema = new StructType().add("id", IntegerType).add("part", IntegerType) + .add("value", IntegerType) + val oldRows = Seq(Row(1, 0, null), Row(2, 1, null), Row(2, 1, null)) + val newRows = Seq(Row(3, 0, 30), Row(3, 0, 30), Row(4, 1, null), Row(5, 1, 50)) + val rows = oldRows ++ newRows + sql(s"""CREATE TABLE tbl (id INT, part INT) USING delta PARTITIONED BY (part) + TBLPROPERTIES ('delta.checkpointInterval' = '100', + 'delta.enableDeletionVectors' = 'false', + 'delta.checkpoint.writeStatsAsJson' = '${!parsed}', + 'delta.checkpoint.writeStatsAsStruct' = '$parsed')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 0), (2, 1), (2, 1) AS v(id, part)""") + sql("ALTER TABLE tbl ADD COLUMN value INT") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (3, 0, 30), (3, 0, 30), (4, 1, NULL), (5, 1, 50) AS v(id, part, value)""") + forceCheckpoint("tbl") + val t = registerTable("tbl") + var copiedTable: Path = null + + mutateTable(t) { dir => + copiedTable = dir + val commits = (0L to 3L).map(v => v -> CommitLog.read(dir, v)).toMap + require(!commits.values.flatten.exists(_.isInstanceOf[RemoveFile])) + val evolved = commits(2).collect { case m: Metadata => m }.head + require(DataType.fromJson(evolved.schemaString) == schema) + require(evolved.partitionColumns == Seq("part")) + val oldAdds = commits(1).collect { case a: AddFile => a } + val newAdds = commits(3).collect { case a: AddFile => a } + val adds = oldAdds ++ newAdds + require(oldAdds.size == 2 && newAdds.size == 2) + require(adds.map(_.path).distinct.size == 4, "All four files must remain active") + for ((batch, files) <- Seq(oldRows -> oldAdds, newRows -> newAdds)) { + require(files.map(_.partitionValues).toSet == + Set(Map("part" -> "0"), Map("part" -> "1"))) + files.foreach { add => + val part = add.partitionValues("part").toInt + val raw = spark.read.parquet(dir.resolve(add.path).toString) + require(raw.columns.contains("value") == (batch == newRows)) + val withValue = if (batch == oldRows) raw.withColumn("value", lit(null) + .cast(IntegerType)) else raw + RowComparison.assertRowsEqual( + spark.createDataFrame(batch.filter(_.getInt(1) == part).asJava, schema), + withValue.withColumn("part", lit(part)).select("id", "part", "value"), + s"input_$variant/${add.path}") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.count(_.getInt(1) == part)) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + require(stats.path(field).has("value") == (batch == newRows), + s"$field must omit the evolved column only for old files") + } + } + } + + val logDir = dir.resolve("_delta_log") + val checkpoint = logDir.resolve("00000000000000000003.checkpoint.parquet") + require(Files.isRegularFile(checkpoint), "Expected the final classic checkpoint") + val cp = spark.read.parquet(checkpoint.toString) + val addSchema = cp.schema("add").dataType.asInstanceOf[StructType] + require(addSchema.fieldNames.contains("stats") == !parsed) + require(addSchema.fieldNames.contains("stats_parsed") == parsed) + require(addSchema.fieldNames.contains("partitionValues_parsed") == parsed) + val cpAdds = cp.filter("add IS NOT NULL").select("add.*").collect() + require(cpAdds.length == adds.size, "Checkpoint must contain every active AddFile") + require(cpAdds.map(_.getAs[String]("path")).toSet == adds.map(_.path).toSet) + require(cp.filter("remove IS NOT NULL").count() == 0) + cpAdds.foreach { action => + val original = adds.find(_.path == action.getAs[String]("path")).get + val stats = JsonUtil.mapper.readTree(original.stats.get) + if (parsed) { + val partition = action.getAs[Row]("partitionValues_parsed") + require(partition.schema("part").dataType == IntegerType) + require(partition.getAs[Int]("part") == original.partitionValues("part").toInt) + val actual = action.getAs[Row]("stats_parsed") + require(actual.getAs[Long]("numRecords") == stats.path("numRecords").asLong()) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + val values = actual.getAs[Row](field) + require(values.getAs[Any]("id").toString == stats.path(field).path("id").asText()) + if (oldAdds.exists(_.path == original.path)) { + require(values.isNullAt(values.fieldIndex("value")), + s"Old file $field must have no parsed evolved-column statistic") + } else { + require(values.getAs[Any]("value").toString == + stats.path(field).path("value").asText()) + } + } + } else { + require(JsonUtil.mapper.readTree(action.getAs[String]("stats")) == stats) + } + } + val lastCheckpoint = JsonUtil.mapper.readTree( + Files.readAllBytes(logDir.resolve("_last_checkpoint"))) + require(lastCheckpoint.path("version").asLong(-1) == 3) + require(lastCheckpoint.path("size").asLong(-1) == cp.count()) + + // Keep the checkpoint-version commit for timestamp resolution, but remove its entire + // preceding history and every version CRC. Reconstruction now requires this checkpoint. + for (v <- 0L until 3L) Files.delete(CommitLog.commitFile(dir, v)) + val stream = Files.list(logDir) + val logFiles = try stream.iterator().asScala.toVector finally stream.close() + logFiles.filter(_.getFileName.toString.matches("[0-9]{20}\\.crc")) + .foreach(Files.delete) + require((0L until 3L).forall(v => !Files.exists(CommitLog.commitFile(dir, v)))) + require(Files.exists(CommitLog.commitFile(dir, 3))) + require(Files.exists(checkpoint) && adds.forall(a => Files.exists(dir.resolve(a.path)))) + } + + val queries = Seq[(String, String, Option[Seq[String]], Seq[Row])]( + ("all", null, None, rows), + ("projection", null, Some(Seq("value")), + Seq(Row(null), Row(null), Row(null), Row(30), Row(30), Row(null), Row(50))), + ("equal", "value = 30", None, Seq(Row(3, 0, 30), Row(3, 0, 30))), + ("range", "value >= 30 AND value < 50", None, Seq(Row(3, 0, 30), Row(3, 0, 30))), + ("is_null", "value IS NULL", None, oldRows :+ Row(4, 1, null)), + ("is_not_null", "value IS NOT NULL", None, + Seq(Row(3, 0, 30), Row(3, 0, 30), Row(5, 1, 50))), + ("partition_null", "part = 1 AND value IS NULL", None, + Seq(Row(2, 1, null), Row(2, 1, null), Row(4, 1, null)))) + queries.foreach { case (name, predicate, columns, expectedRows) => + readSpec(t, predicate = predicate, columns = columns, name = Some(s"read_$name")) + .assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $variant/$name read, got $other") + } + val expectedSchema = if (columns.isDefined) { + new StructType().add("value", IntegerType) + } else schema + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual( + spark.createDataFrame(expectedRows.asJava, expectedSchema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$variant/$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.partitionColumns == Seq("part")) + require(result.metadata.configuration("delta.checkpoint.writeStatsAsStruct") == + parsed.toString) + require(result.metadata.configuration("delta.checkpoint.writeStatsAsJson") == + (!parsed).toString) + case other => fail(s"Expected successful $variant snapshot, got $other") + } + } + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTableOperationsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTableOperationsSuite.scala new file mode 100644 index 00000000..a64a4672 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTableOperationsSuite.scala @@ -0,0 +1,1354 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.io.DataInputStream +import java.nio.file.{Files, Path, Paths} + +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.deltaharness.{DeltaHarness, Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{Action, AddFile, CommitLog, DeletionVector, DomainMetadata, Metadata, + Protocol, RemoveFile, Txn} +import io.delta.workload.model._ + +class ReadinessTableOperationsSuite extends WorkloadTestSuite("readiness_table_operations") { + + private def descriptor(dv: DeletionVector): DeletionVectorDescriptor = { + // At this DAT pin USE_LONG_FOR_INTS boxes Option[Int] offsets as Long. Normalize + // only this suite's descriptor input, without changing the shared mapper or DV bytes. + val offset = dv.offset.asInstanceOf[Option[Number]].map(n => Math.toIntExact(n.longValue())) + DeletionVectorDescriptor(dv.storageType, dv.pathOrInlineDv, offset, + dv.sizeInBytes, dv.cardinality) + } + + private def sidecar(table: Path, dv: DeletionVector): Path = + Paths.get(descriptor(dv).absolutePath(new HadoopPath(table.toUri)).toUri.getPath) + + private def payload(table: Path, dv: DeletionVector): Array[Byte] = { + if (dv.storageType == "i") descriptor(dv).inlineData + else { + assert(dv.storageType == "u") + val file = sidecar(table, dv) + assert(file.normalize().startsWith(table.normalize())) + val offset = descriptor(dv).offset.get + assert(offset > 0) + assert(Files.size(file) >= offset.toLong + 4L + dv.sizeInBytes + 4L) + val input = new DataInputStream(Files.newInputStream(file)) + try { + assert(input.readByte() == DeletionVectorStore.DV_FILE_FORMAT_VERSION_ID_V1) + input.skipNBytes(offset.toLong - 1L) + // This OSS reader checks both the payload length and checksum. + DeletionVectorStore.readRangeFromStream(input, dv.sizeInBytes) + } finally input.close() + } + } + + Seq("inline", "uuid_relative").foreach { form => + test(s"e2e_42_$form") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'remove'), (0, 'keep'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (9, 'remove'), (12, 'keep'), (18, CAST(NULL AS STRING)), (23, 'edge'), + (31, 'keep'), (44, CAST(NULL AS STRING)), (57, 'remove'), (88, 'last') AS v(id, value)""") + sql("DELETE FROM tbl WHERE id IN (-7, 9, 57)") + sql("DELETE FROM tbl WHERE id = 4") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val baseRows = Seq(Row(-7, "remove"), Row(0, "keep"), Row(4, null), Row(4, null), + Row(9, "remove"), Row(12, "keep"), Row(18, null), Row(23, "edge"), + Row(31, "keep"), Row(44, null), Row(57, "remove"), Row(88, "last")) + val firstRows = baseRows.filterNot(r => Set(-7, 9, 57).contains(r.getInt(0))) + val latestRows = firstRows.filterNot(_.getInt(0) == 4) + val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = original(0L).collect { case p: Protocol => p }.ensuring(_.size == 1).head + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + assert(protocol == Protocol(3, 7, Some(Set("deletionVectors")), + Some(Set("appendOnly", "deletionVectors", "invariants")))) + info("DV original protocol: " + protocol.toJson) + info("DV original metadata: " + metadata.toJson) + val adds = (1L to 3L).map { version => + val found = original(version).collect { case a: AddFile => a } + assert(found.size == 1) + info(s"DV original add v$version: " + found.head.toJson) + version -> found.head + }.toMap + assert(adds.values.map(_.path).toSet.size == 1) // Deletes must retain the data file. + assert(adds(1L).deletionVector.isEmpty) + val physical = spark.read.parquet(table.sourcePath.resolve(adds(1L).path).toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(baseRows.asJava, schema), physical, + s"$form physical input") + val physicalRows = physical.collect().toSeq + val sourceDvs = (2L to 3L).map(v => v -> adds(v).deletionVector.get).toMap + sourceDvs.foreach { case (version, dv) => + assert(dv.storageType == "u") + assert(dv.cardinality == (if (version == 2L) 3L else 5L)) + val expectedPositions = physicalRows.zipWithIndex.collect { + case (row, index) if Set(-7, 9, 57).contains(row.getInt(0)) || + (version == 3L && row.getInt(0) == 4) => index.toLong + } + assert(RoaringBitmapArray.readFrom(payload(table.sourcePath, dv)).toArray.toSeq == + expectedPositions) + } + val converted = sourceDvs.values.map { dv => + val bytes = payload(table.sourcePath, dv) + // Use the verified OSS helper with exact engine-generated payload bytes. + val inline = DeletionVectorDescriptor.inlineInLog(bytes, dv.cardinality) + assert(inline.storageType == "i" && inline.offset.isEmpty) + assert(inline.sizeInBytes == bytes.length && inline.cardinality == dv.cardinality) + assert(inline.inlineData.sameElements(bytes)) + dv -> DeletionVector(inline.storageType, inline.pathOrInlineDv, inline.offset, + inline.sizeInBytes, inline.cardinality) + }.toMap + if (form == "inline") { + (2L to 3L).foreach { version => + modifyCommitActions(table, version) { actions => + actions.map { + case add: AddFile => add.copy(deletionVector = add.deletionVector.map(converted)) + // The second delete removes the first DV identity; convert that tombstone too. + case remove: RemoveFile => + remove.copy(deletionVector = remove.deletionVector.map(converted)) + case other => other + } + } + } + } + val expectedAdds = adds.map { case (version, add) => + version -> (if (form == "inline") { + add.copy(deletionVector = add.deletionVector.map(converted)) + } else add) + } + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + val sidecars = sourceDvs.values.map(dv => sidecar(dir, dv)).toSet + if (form == "inline") sidecars.foreach(Files.delete) + else sidecars.foreach(p => assert(Files.isRegularFile(p))) + // Version checksums embed the original descriptors and become stale after conversion. + val logs = Files.list(dir.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + (1L to 3L).foreach { version => + val actions = CommitLog.read(dir, version) + assert(actions.collect { case a: AddFile => a } == Seq(expectedAdds(version))) + expectedAdds(version).deletionVector.foreach { dv => + assert(dv.storageType == (if (form == "inline") "i" else "u")) + assert(dv.cardinality == (if (version == 2L) 3L else 5L)) + assert(payload(dir, dv).sameElements(payload(table.sourcePath, sourceDvs(version)))) + assert(RoaringBitmapArray.readFrom(payload(dir, dv)).cardinality == dv.cardinality) + } + } + val oldIdentity = CommitLog.read(dir, 3L).collect { case r: RemoveFile => r } + assert(oldIdentity.size == 1) + assert(oldIdentity.head.path == adds(1L).path) + assert(oldIdentity.head.deletionVector == expectedAdds(2L).deletionVector) + assert(Files.size(dir.resolve(adds(1L).path)) == adds(1L).size) + if (form == "inline") assert(sidecars.forall(p => !Files.exists(p))) + } + val snapshot = SnapshotResult( + ProtocolInfo(3, 7, Some(Seq("deletionVectors")), + Some(Seq("appendOnly", "deletionVectors", "invariants"))), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), metadata.schemaString, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](1L, 2L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(snapshot)) + } + } + val projected = new StructType().add("value", StringType) + val reordered = projected.add("id", IntegerType) + val filter = "id IN (4, 12, 18, 57)" + val cases = Seq( + ("read_base", ReadQuery(version = Some(1L)), schema, baseRows), + ("read_first_delete", ReadQuery(version = Some(2L)), schema, firstRows), + ("read_latest", ReadQuery(), schema, latestRows), + ("read_v2_filtered_projected", ReadQuery(version = Some(2L), + predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered, + Seq(Row(null, 4), Row(null, 4), Row("keep", 12), Row(null, 18))), + ("read_v3_filtered_projected", ReadQuery(version = Some(3L), + predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered, + Seq(Row("keep", 12), Row(null, 18))), + ("read_projected", ReadQuery(columns = Some(Seq("value"))), projected, + Seq(Row("keep"), Row("keep"), Row(null), Row("edge"), Row("keep"), Row(null), + Row("last"))), + ("read_deleted_only", ReadQuery(predicate = Some("id IN (-7, 4, 9, 57)")), schema, + Seq.empty[Row])) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + // All predicates overlap physical min/max; the single data AddFile remains scanned. + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, 1, 0L))) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, expectedSchema), actual, + s"$form $name") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_metadata").toString) + val scanned = captured.collect().toSeq.map { row => + val node = JsonUtil.mapper.readTree(row.getString(0)) + val wrapped = if (node.has("add")) node.toString else s"""{"add":$node}""" + Action.parse(wrapped) + } + // OSS InMemoryLogReplay stores active AddFiles with dataChange=false; + // ReadCapture obtains these snapshot actions through DeltaSparkHarness.allFiles. + assert(scanned == Seq(expectedAdds(query.version.getOrElse(3L)) + .copy(dataChange = false))) + } + } + } + } + + test("e2e_54_truncated_history") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-1, 'old'), (2, 'old'), (2, 'old'), (CAST(NULL AS INT), 'old') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (7, 'keep'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (9, 'boundary'), (7, 'keep') AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val boundaryRows = Seq(Row(7, "keep"), Row(3, null), Row(3, null), Row(null, "missing"), + Row(9, "boundary"), Row(7, "keep")) + val latestRows = boundaryRows ++ Seq(Row(3, null), Row(11, "after"), Row(null, "after")) + val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + assert(original(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inputs = (1L to 3L).map { version => + val adds = original(version).collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.deletionVector.isEmpty) + version -> adds.head + }.toMap + assert(inputs.values.map(_.path).toSet.size == 3) + assert(original(2L).collect { case r: RemoveFile => r.path } == Seq(inputs(1L).path)) + DeltaHarness.get.openLog(spark, table.sourcePath.toString).checkpoint() + val checkpointName = "00000000000000000003.checkpoint.parquet" + val checkpointBytes = Files.readAllBytes(table.sourcePath.resolve("_delta_log") + .resolve(checkpointName)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + val logDir = dir.resolve("_delta_log") + (0L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + assert(Files.readAllBytes(logDir.resolve(checkpointName)).sameElements(checkpointBytes)) + // Retain v2 and v3 JSON for OSS historical lookup, plus the oldest checkpoint at v3. + // The missing prefix makes v1 unavailable; no table data is deleted. + (0L to 1L).foreach { version => + val commit = logDir.resolve(f"$version%020d.json") + assert(Files.isRegularFile(commit)) + Files.delete(commit) + assert(!Files.exists(commit)) + } + (2L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + inputs.values.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + val boundary = SnapshotResolver.buildDeltaReader(spark, dir, Some(3L), None) + assert(boundary.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(boundaryRows.asJava, schema), boundary, + "history boundary after JSON removal, before append") + // Append to the already truncated copy: v4 must be replayed after the v3 checkpoint. + spark.sql(s"""INSERT INTO delta.`$dir` SELECT /*+ COALESCE(1) */ * FROM VALUES + (3, CAST(NULL AS STRING)), (11, 'after'), (CAST(NULL AS INT), 'after') AS v(id, value)""") + .collect() + // The generator strips Hadoop checksum sidecars while copying. The append's LogStore + // may recreate this sidecar; keep the same portable fixture form after the mutation. + Files.deleteIfExists(logDir.resolve(".00000000000000000004.json.crc")) + val appended = CommitLog.read(dir, 4L) + val adds = appended.collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.deletionVector.isEmpty) + assert(!inputs.values.exists(_.path == adds.head.path)) + assert(appended.forall(a => !a.isInstanceOf[RemoveFile] && !a.isInstanceOf[Metadata] && + !a.isInstanceOf[Protocol])) + val commitInfo = appended.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "WRITE") + assert(commitInfo.get("operationParameters").get("mode").asText() == "Append") + assert(Files.size(dir.resolve(adds.head.path)) == adds.head.size) + assert(Files.readAllBytes(logDir.resolve(checkpointName)).sameElements(checkpointBytes)) + val files = Files.list(logDir) + try { + val names = files.iterator().asScala.map(_.getFileName.toString).toSeq + assert(names.filter(_.endsWith(".checkpoint.parquet")) == Seq(checkpointName)) + assert(names.filter(_.endsWith(".json")).sorted == (2L to 4L).map(v => f"$v%020d.json")) + } finally files.close() + assert(JsonUtil.mapper.readTree(Files.readAllBytes(logDir.resolve("_last_checkpoint"))) + .get("version").asLong() == 3L) + (2L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + inputs.values.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + info("history removed JSON v0/v1; retained v2/v3 and checkpoint v3; real append v4: " + + commitInfo.toString) + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](3L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(4L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val cases = Seq( + ("read_boundary_v3", ReadQuery(version = Some(3L)), boundaryRows, 2), + ("read_latest_after_append", ReadQuery(), latestRows, 3)) + cases.foreach { case (name, query, rows, fileCount) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull) + .assert { spec => + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, fileCount, 0L))) + val expected = spark.createDataFrame(rows.asJava, schema) + val actual = SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, None) + assert(actual.schema == schema) + RowComparison.assertRowsEqual(expected, actual, s"history $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == schema) + RowComparison.assertRowsEqual(expected, captured, s"history $name expected parquet") + } + } + // No checkpoint or JSON exists at or before v1. DAT resolves getSnapshotAt first; + // OSS raises DELTA_LOG_FILE_NOT_FOUND for that empty log segment, before scanning data. + readSpec(table, name = Some("read_unavailable_v1"), version = 1L, + expectError = ErrorCode("DELTA_LOG_FILE_NOT_FOUND")).assert { spec => + assert(spec.query == ReadQuery(version = Some(1L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode == "DELTA_LOG_FILE_NOT_FOUND") + assert(error.errorMessage.contains("version 1")) + info("history typed unavailable-history read: " + error.toString) + case other => fail(s"history unavailable-history read expected Failed, got $other") + } + } + snapshotSpec(table, version = 1L, expectError = ErrorCode("DELTA_LOG_FILE_NOT_FOUND")) + .assert { spec => + assert(spec.query == SnapshotQuery(version = Some(1L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode == "DELTA_LOG_FILE_NOT_FOUND") + assert(error.errorMessage.contains("version 1")) + case other => fail(s"history unavailable-history snapshot expected Failed, got $other") + } + } + } + + test("e2e_50_actual_vacuum") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (8, 'keep'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-2, 'early'), (5, 'middle'), (8, 'keep'), (12, CAST(NULL AS STRING)) AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val rows = Seq(Row(8, "keep"), Row(3, null), Row(3, null), Row(null, "missing"), + Row(-2, "early"), Row(5, "middle"), Row(8, "keep"), Row(12, null)) + val literal = spark.createDataFrame(rows.asJava, schema) + val before = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + assert(before(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inputs = (1L to 2L).flatMap(v => before(v).collect { case a: AddFile => a }) + assert(inputs.size == 2 && inputs.map(_.path).distinct.size == 2) + assert(inputs.forall(_.deletionVector.isEmpty)) + RowComparison.assertRowsEqual(literal, + SnapshotResolver.buildDeltaReader(spark, table.sourcePath, Some(2L), None), + "vacuum historical rows before pruning") + sql("OPTIMIZE tbl") + val rewrite = CommitLog.read(table.sourcePath, 3L) + val active = rewrite.collect { case a: AddFile => a } + val obsolete = rewrite.collect { case r: RemoveFile => r } + assert(active.nonEmpty && obsolete.size == 2) + assert(obsolete.map(_.path).toSet == inputs.map(_.path).toSet) + assert((active.map(_.path).toSet intersect obsolete.map(_.path).toSet).isEmpty) + assert(active.forall(a => !a.dataChange && a.deletionVector.isEmpty)) + assert(obsolete.forall(r => !r.dataChange && r.deletionVector.isEmpty)) + assert(rewrite.collect { case m: Metadata => m }.isEmpty) + assert(rewrite.collect { case p: Protocol => p }.isEmpty) + val optimizeInfo = rewrite.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(optimizeInfo.get("operation").asText() == "OPTIMIZE") + (inputs ++ active).foreach { add => + assert(Files.isRegularFile(table.sourcePath.resolve(add.path))) + assert(Files.size(table.sourcePath.resolve(add.path)) == add.size) + } + info("vacuum pre-VACUUM obsolete paths physically present: " + + obsolete.map(_.path).mkString(",")) + val retentionCheck = "spark.databricks.delta.retentionDurationCheck.enabled" + val originalRetentionCheck = spark.conf.getOption(retentionCheck) + try { + spark.conf.set(retentionCheck, "false") + sql("VACUUM tbl RETAIN 0 HOURS") + } finally { + originalRetentionCheck match { + case Some(value) => spark.conf.set(retentionCheck, value) + case None => spark.conf.unset(retentionCheck) + } + } + assert(spark.conf.getOption(retentionCheck) == originalRetentionCheck) + obsolete.foreach(r => assert(!Files.exists(table.sourcePath.resolve(r.path)))) + active.foreach(a => assert(Files.size(table.sourcePath.resolve(a.path)) == a.size)) + // VACUUM emits audit commits; they must not change the active file set or metadata. + val vacuum = (4L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val vacuumInfo = vacuum.toSeq.sortBy(_._1).flatMap { case (_, actions) => + assert(actions.forall(a => !a.isInstanceOf[AddFile] && !a.isInstanceOf[RemoveFile] && + !a.isInstanceOf[Metadata] && !a.isInstanceOf[Protocol])) + actions.map(a => JsonUtil.mapper.readTree(a.toJson)).filter(_.has("commitInfo")) + .map(_.get("commitInfo")) + } + assert(vacuumInfo.map(_.get("operation").asText()) == Seq("VACUUM START", "VACUUM END")) + vacuumInfo.foreach(n => info("vacuum actual vacuum commit: " + n.toString)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + (before ++ Map(3L -> rewrite) ++ vacuum).foreach { case (version, actions) => + assert(CommitLog.read(dir, version) == actions) + } + obsolete.foreach(r => assert(!Files.exists(dir.resolve(r.path)))) + active.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + // Force row materialization, independently of DAT's typed error capture. + val thrown = intercept[Exception] { + SnapshotResolver.buildDeltaReader(spark, dir, Some(2L), None).collect() + } + val causes = Iterator.iterate(thrown: Throwable)(_.getCause).takeWhile(_ != null).toSeq + assert(causes.exists(_.isInstanceOf[java.io.FileNotFoundException])) + assert(causes.exists(e => obsolete.exists(r => + Option(e.getMessage).exists(_.contains(r.path))))) + info("vacuum independent historical missing-file cause: " + + causes.map(e => e.getClass.getName + ": " + e.getMessage).mkString(" <- ")) + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](3L, 5L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(5L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_latest", ReadQuery(), schema, rows), + ("read_rewritten_v3", ReadQuery(version = Some(3L)), schema, rows), + ("read_vacuumed_v5", ReadQuery(version = Some(5L)), schema, rows), + ("read_latest_filtered", ReadQuery(predicate = Some("id = 3 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 3), Row(null, 3), Row("missing", null)))) + cases.foreach { case (name, query, expectedSchema, expectedRows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == expectedRows.size.toLong) + case other => fail(s"vacuum $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(expectedRows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"vacuum $name copied table") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"vacuum $name expected parquet") + } + } + readSpec(table, name = Some("read_pruned_v2"), version = 2L, + expectError = AnyError).assert { spec => + assert(spec.query == ReadQuery(version = Some(2L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode.nonEmpty) + assert(obsolete.exists(r => error.errorMessage.contains(r.path))) + val message = error.errorMessage.toLowerCase(java.util.Locale.ROOT) + assert(message.contains("not exist") || message.contains("not found")) + info("vacuum typed historical error: " + error.toString) + case other => fail(s"vacuum pruned historical read expected Failed, got $other") + } + // An error at the historical version must leave the current table readable. + RowComparison.assertRowsEqual(literal, + SnapshotResolver.buildDeltaReader(spark, copiedTable, None, None), + "vacuum current rows after historical failure") + } + } + + test("e2e_51_stats_refresh") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'drop'), (0, 'low'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (12, 'middle'), (18, CAST(NULL AS STRING)), (31, 'last'), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("DELETE FROM tbl WHERE id = -7") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val physicalRows = Seq(Row(-7, "drop"), Row(0, "low"), Row(4, null), Row(4, null), + Row(12, "middle"), Row(18, null), Row(31, "last"), Row(null, "missing")) + val rows = Seq(Row(0, "low"), Row(4, null), Row(4, null), Row(12, "middle"), + Row(18, null), Row(31, "last"), Row(null, "missing")) + val original = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = Protocol(3, 7, Some(Set("deletionVectors")), + Some(Set("appendOnly", "deletionVectors", "invariants"))) + assert(original(0L).collect { case p: Protocol => p } == Seq(protocol)) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inserted = original(1L).collect { case a: AddFile => a }.ensuring(_.size == 1).head + val deleted = original(2L).collect { case a: AddFile => a }.ensuring(_.size == 1).head + assert(inserted.path == deleted.path && inserted.deletionVector.isEmpty) + val dv = deleted.deletionVector.get + assert(dv.storageType == "u" && dv.cardinality == 1L) + val physical = spark.read.parquet(table.sourcePath.resolve(deleted.path).toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(physicalRows.asJava, schema), physical, + "replay literal physical rows") + val positions = physical.collect().toSeq.zipWithIndex.collect { + case (row, index) if !row.isNullAt(0) && row.getInt(0) == -7 => index.toLong + } + val dvBytes = payload(table.sourcePath, dv) + assert(RoaringBitmapArray.readFrom(dvBytes).toArray.toSeq == positions) + val fileBytes = Files.readAllBytes(table.sourcePath.resolve(deleted.path)) + // Physical stats include the deleted row. These bounds remain conservative for the DV. + // Model a partial-stats writer followed by a stats-only update, not an autostats service. + val partialStats = """{"numRecords":8,"tightBounds":false}""" + val refreshedStats = """{"numRecords":8,"minValues":{"id":-7,"value":"drop"}, + "maxValues":{"id":31,"value":"missing"},"nullCount":{"id":1,"value":3}, + "tightBounds":false}""" + val partial = deleted.copy(stats = Some(partialStats)) + val refreshed = partial.copy(stats = Some(refreshedStats), dataChange = false) + assert(refreshed.copy(stats = deleted.stats, dataChange = deleted.dataChange) == deleted) + assert(refreshed.path == deleted.path && refreshed.deletionVector == Some(dv)) + assert(refreshed.baseRowId == deleted.baseRowId) + assert(refreshed.defaultRowCommitVersion == deleted.defaultRowCommitVersion) + // Row tracking is not enabled: preserve absent row-ID fields as well as the real DV. + assert(deleted.baseRowId.isEmpty && deleted.defaultRowCommitVersion.isEmpty) + modifyCommitActions(table, 2L) { actions => + actions.map { case _: AddFile => partial; case other => other } + } + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + assert(CommitLog.read(dir, 0L) == original(0L)) + assert(CommitLog.read(dir, 1L) == original(1L)) + assert(CommitLog.read(dir, 2L) == original(2L).map { + case _: AddFile => partial; case other => other + }) + assert(!Files.exists(CommitLog.commitFile(dir, 3L))) + // One AddFile per identity per version; no conflicting same-version remove/add pair. + CommitLog.write(dir, 3L, Seq(refreshed)) + val logs = Files.list(dir.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + DeltaHarness.get.openLog(spark, dir.toString).checkpoint(3L) + val checkpoint = dir.resolve("_delta_log/00000000000000000003.checkpoint.parquet") + assert(Files.isRegularFile(checkpoint)) + val checkpointAdds = spark.read.parquet(checkpoint.toString) + .filter("add IS NOT NULL").select("add.*").toJSON.collect().toSeq + .map(json => Action.parse(s"""{"add":$json}""")) + assert(checkpointAdds == Seq(refreshed)) + val lastCheckpoint = JsonUtil.mapper.readTree( + Files.readAllBytes(dir.resolve("_delta_log/_last_checkpoint"))) + assert(lastCheckpoint.get("version").asLong() == 3L) + // Checkpoint creation can write fresh Hadoop sidecars; retain only protocol artifacts. + val checkpointLogs = Files.list(dir.resolve("_delta_log")) + try checkpointLogs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach { file => + assert(file.getFileName.toString.startsWith(".")) + Files.delete(file) + } + finally checkpointLogs.close() + // A transaction marker provides a genuine JSON tail without changing the row bag. + CommitLog.write(dir, 4L, Seq(Txn("readiness_e2e_51_tail", 1L))) + assert(CommitLog.read(dir, 3L) == Seq(refreshed)) + assert(CommitLog.read(dir, 4L) == Seq(Txn("readiness_e2e_51_tail", 1L))) + assert(Files.readAllBytes(dir.resolve(deleted.path)).sameElements(fileBytes)) + assert(payload(dir, dv).sameElements(dvBytes)) + DeltaLog.clearCache() + val replay = DeltaLog.forTable(spark, dir.toString).update() + assert(replay.version == 4L) + assert(replay.logSegment.checkpointProvider.version == 3L) + assert(replay.logSegment.deltas.map(_.getPath.getName) == Seq(f"${4L}%020d.json")) + (2L to 4L).foreach { version => + val snapshot = DeltaHarness.get.openLog(spark, dir.toString).getSnapshotAt(version) + val active = snapshot.allFiles.select("json").collect().toSeq + .map(r => Action.parse(r.getString(0))) + // Delta's active-file replay normalizes dataChange; commit flags were checked above. + val expectedAdd = (if (version == 2L) partial else refreshed).copy(dataChange = false) + assert(active == Seq(expectedAdd)) + } + val remaining = Files.list(dir.resolve("_delta_log")) + try assert(!remaining.iterator().asScala.exists(_.getFileName.toString.endsWith(".crc"))) + finally remaining.close() + info("replay checkpoint v3 + JSON tail v4 selected; physical file and DV unchanged") + } + val expectedSnapshot = SnapshotResult( + ProtocolInfo(3, 7, Some(Seq("deletionVectors")), + Some(Seq("appendOnly", "deletionVectors", "invariants"))), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](2L, 3L, 4L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + val resolved = Option(version).fold(4L)(_.toLong) + assert(spec.query == SnapshotQuery(version = Some(resolved))) + assert(spec.expectation == Succeeded(if (resolved == 4L) { + expectedSnapshot.copy(setTransactions = Some(Seq(AppTxn("readiness_e2e_51_tail", 1L)))) + } else expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val selective = "id IN (4, 12) OR id IS NULL" + val selected = Seq(Row(null, 4), Row(null, 4), Row("middle", 12), Row("missing", null)) + val versions = Seq("before" -> Some(2L), "refreshed" -> Some(3L), + "tail" -> Some(4L), "latest" -> None) + versions.foreach { case (label, version) => + val cases = Seq( + (s"read_$label", ReadQuery(version = version), schema, rows), + (s"read_${label}_selective", ReadQuery(version = version, + predicate = Some(selective), columns = Some(Seq("value", "id"))), projected, selected), + (s"read_${label}_miss", ReadQuery(version = version, predicate = Some("id > 100")), + schema, Seq.empty[Row])) + cases.foreach { case (name, query, expectedSchema, expectedRows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == expectedRows.size.toLong) + case other => fail(s"replay $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(expectedRows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"replay $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"replay $name expected parquet") + } + } + } + } + + Seq("zorder", "liquid").foreach { mode => + test(s"e2e_49_$mode") { + val liquid = mode == "liquid" + val clusterBy = if (liquid) "CLUSTER BY (id, value)" else "" + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta $clusterBy TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (8, 'late'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-2, 'early'), (5, 'middle'), (8, 'late'), (12, CAST(NULL AS STRING)) AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + // Independent literal bag: includes duplicate whole rows and nulls in both columns. + val firstRows = Seq(Row(8, "late"), Row(3, null), Row(3, null), Row(null, "missing")) + val allRows = firstRows ++ Seq(Row(-2, "early"), Row(5, "middle"), Row(8, "late"), + Row(12, null)) + val before = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = if (liquid) { + Protocol(1, 7, None, Some(Set("appendOnly", "invariants", "clustering", "domainMetadata"))) + } else Protocol(1, 2) + assert(before(0L).collect { case p: Protocol => p } == Seq(protocol)) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val domains = before(0L).collect { case d: DomainMetadata => d } + if (liquid) { + assert(domains.size == 1 && domains.head.domain == "delta.clustering") + assert(!domains.head.removed) + val configuration = JsonUtil.mapper.readTree(domains.head.configuration) + assert(configuration.get("clusteringColumns") == + JsonUtil.mapper.readTree("""[["id"],["value"]]""")) + } else assert(domains.isEmpty) + val literal = spark.createDataFrame(allRows.asJava, schema) + val unoptimized = SnapshotResolver.buildDeltaReader(spark, table.sourcePath, + Some(2L), None) + assert(unoptimized.schema == schema) + RowComparison.assertRowsEqual(literal, unoptimized, s"$mode before OPTIMIZE") + sql(if (liquid) "OPTIMIZE tbl" else "OPTIMIZE tbl ZORDER BY (id, value)") + val optimized = CommitLog.read(table.sourcePath, 3L) + val adds = optimized.collect { case a: AddFile => a } + val removes = optimized.collect { case r: RemoveFile => r } + // Check actual rewrite actions, without promising a file count or layout effectiveness. + assert(adds.nonEmpty && removes.nonEmpty) + assert(adds.forall(!_.dataChange) && removes.forall(!_.dataChange)) + val inputAdds = before.values.flatten.collect { case a: AddFile => a }.toSeq + assert(removes.map(_.path).toSet == inputAdds.map(_.path).toSet) + assert(adds.forall(_.clusteringProvider == (if (liquid) Some("liquid") else None))) + assert(optimized.collect { case m: Metadata => m }.isEmpty) + assert(optimized.collect { case p: Protocol => p }.isEmpty) + assert(optimized.collect { case d: DomainMetadata => d }.isEmpty) + val commitInfo = optimized.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "OPTIMIZE") + val parameters = commitInfo.get("operationParameters") + val columns = JsonUtil.mapper.readTree("""["id","value"]""") + val empty = JsonUtil.mapper.readTree("[]") + assert(JsonUtil.mapper.readTree(parameters.get("zOrderBy").asText()) == + (if (liquid) empty else columns)) + assert(JsonUtil.mapper.readTree(parameters.get("clusterBy").asText()) == + (if (liquid) columns else empty)) + info(s"cluster $mode protocol: " + protocol.toJson) + info(s"cluster $mode metadata: " + metadata.toJson) + info(s"cluster $mode optimize: " + commitInfo.toString) + domains.foreach(d => info(s"cluster $mode domain: " + d.toJson)) + adds.foreach(a => info(s"cluster $mode optimized add: " + a.toJson)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + (0L to 2L).foreach(v => assert(CommitLog.read(dir, v) == before(v))) + assert(CommitLog.read(dir, 3L) == optimized) + } + val expectedSnapshot = SnapshotResult( + ProtocolInfo(protocol.minReaderVersion, protocol.minWriterVersion, + writerFeatures = protocol.writerFeatures.map(_.toSeq.sorted)), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime), + domainMetadata = if (liquid) Some(domains.map(d => + AddDomainMetadata(d.domain, d.configuration))) else None) + Seq[java.lang.Long](0L, 2L, 3L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val predicate = "(id >= 3 AND id <= 8) OR id IS NULL" + val firstFiltered = Seq(Row("late", 8), Row(null, 3), Row(null, 3), Row("missing", null)) + val allFiltered = firstFiltered ++ Seq(Row("middle", 5), Row("late", 8)) + val cases = Seq( + ("read_latest", ReadQuery(), schema, allRows), + ("read_first_insert", ReadQuery(version = Some(1L)), schema, firstRows), + ("read_before_optimize", ReadQuery(version = Some(2L)), schema, allRows), + ("read_optimized_v3", ReadQuery(version = Some(3L)), schema, allRows), + ("read_first_filtered", ReadQuery(version = Some(1L), predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, firstFiltered), + ("read_latest_filtered", ReadQuery(predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, allFiltered), + ("read_optimized_filtered", ReadQuery(version = Some(3L), predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, allFiltered)) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$mode $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"$mode $name copied table") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"$mode $name expected parquet") + } + } + } + } + + test("e2e_55_restore") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'before'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing'), (12, 'keep'), (12, 'keep') AS v(id, value)""") + val table = registerTable("tbl") + val before = (0L to 1L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (99, 'mutated'), (99, 'mutated'), (CAST(NULL AS INT), 'changed'), + (6, CAST(NULL AS STRING)) AS v(id, value)""") + val mutated = CommitLog.read(table.sourcePath, 2L) + sql("RESTORE TABLE tbl TO VERSION AS OF 1") + val restored = CommitLog.read(table.sourcePath, 3L) + assert(!Files.exists(CommitLog.commitFile(table.sourcePath, 4L))) + (0L to 1L).foreach(v => assert(CommitLog.read(table.sourcePath, v) == before(v))) + assert(CommitLog.read(table.sourcePath, 2L) == mutated) + val originalAdds = before(1L).collect { case a: AddFile => a } + val mutatedAdds = mutated.collect { case a: AddFile => a } + val restoredAdds = restored.collect { case a: AddFile => a } + assert(originalAdds.nonEmpty && mutatedAdds.nonEmpty) + assert(originalAdds.map(_.path).toSet.intersect(mutatedAdds.map(_.path).toSet).isEmpty) + assert(mutated.collect { case r: RemoveFile => r.path }.toSet == + originalAdds.map(_.path).toSet) + assert(restoredAdds.toSet == originalAdds.map(_.copy(dataChange = true)).toSet) + val restoredRemoves = restored.collect { case r: RemoveFile => r } + assert(restoredRemoves.map(_.path).toSet == mutatedAdds.map(_.path).toSet) + assert(restoredAdds.forall(_.dataChange) && restoredRemoves.forall(_.dataChange)) + val commitInfo = restored.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "RESTORE") + assert(commitInfo.get("operationParameters").get("version").asText() == "1") + info("restore OSS Delta 4.1.0 RESTORE commit: " + commitInfo.toString) + + val schema = new StructType().add("id", IntegerType).add("value", StringType) + // Literal bags are independent of both the captured result and the Delta reader. + val initialRows = Seq(Row(-7, "before"), Row(4, null), Row(4, null), + Row(null, "missing"), Row(12, "keep"), Row(12, "keep")) + val mutatedRows = Seq(Row(99, "mutated"), Row(99, "mutated"), + Row(null, "changed"), Row(6, null)) + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(before(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + (0L to 1L).foreach(v => assert(CommitLog.read(dir, v) == before(v))) + assert(CommitLog.read(dir, 2L) == mutated) + assert(CommitLog.read(dir, 3L) == restored) + (originalAdds ++ mutatedAdds).foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + } + Seq[java.lang.Long](1L, 2L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_before_v1", ReadQuery(version = Some(1L)), schema, initialRows), + ("read_mutated_v2", ReadQuery(version = Some(2L)), schema, mutatedRows), + ("read_restored_v3", ReadQuery(version = Some(3L)), schema, initialRows), + ("read_latest", ReadQuery(), schema, initialRows), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("missing", null)))) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"restore $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"restore $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"restore $name expected parquet") + } + } + } + test("e2e_56_minor_log_compaction") { + import org.apache.spark.sql.delta.actions.{Action => SparkAction, InMemoryLogReplay} + import org.apache.spark.sql.delta.util.FileNames + + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'initial'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing'), (12, 'keep'), (12, 'keep') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (6, 'replacement'), (6, 'replacement'), (CAST(NULL AS INT), 'changed'), + (8, CAST(NULL AS STRING)) AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), (23, 'end') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (31, 'tail'), (31, 'tail'), (CAST(NULL AS INT), 'tail-null'), + (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)) AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (44, 'last'), (44, 'last') AS v(id, value)""") + val table = registerTable("tbl") + val original = (0L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(original(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val adds = (1L to 5L).map { v => + v -> original(v).collect { case a: AddFile => a }.ensuring(_.size == 1).head + }.toMap + assert(adds.values.map(_.path).toSet.size == 5) + assert(original(2L).collect { case r: RemoveFile => r.path } == Seq(adds(1L).path)) + assert(original(4L).collect { case r: RemoveFile => r.path }.toSet == + Set(adds(2L).path, adds(3L).path)) + + // Follow the pinned OSS DeltaLogMinorCompactionSuite: reconcile [0, 3], with + // no retention pruning, then serialize the existing OSS checkpoint iterator. + // Concatenating commits would retain cancelled adds and commitInfo, which is invalid. + val replay = new InMemoryLogReplay(None, None) + (0L to 3L).foreach { v => + val lines = Files.readAllLines(table.sourcePath.resolve(f"_delta_log/$v%020d.json")) + replay.append(v, lines.asScala.iterator.map(SparkAction.fromJson)) + } + val sourceLog = DeltaLog.forTable(spark, table.sourcePath.toString) + val compactedPath = FileNames.compactedDeltaFile(sourceLog.logPath, 0L, 3L) + val compactedName = "00000000000000000000.00000000000000000003.compacted.json" + assert(compactedPath.getName == compactedName) + sourceLog.store.write(compactedPath, replay.checkpoint.map(_.json), + overwrite = false, sourceLog.newDeltaHadoopConf()) + val compactedBytes = Files.readAllBytes(table.sourcePath.resolve(s"_delta_log/$compactedName")) + val compacted = Files.readAllLines(table.sourcePath.resolve(s"_delta_log/$compactedName")) + .asScala.map(Action.parse).toSeq + assert(compacted.size == 5) + assert(compacted.collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(compacted.collect { case m: Metadata => m } == Seq(metadata)) + // OSS Action.fromJson materializes absent optional tags as an empty map. + // FileAction.tagsOrEmpty defines their equivalence; preserve every nonempty tag. + val expectedAdds = Set(adds(2L), adds(3L)).map(a => + a.copy(dataChange = false, tags = a.tags.filter(_.nonEmpty))) + assert(compacted.collect { case a: AddFile => a.copy(tags = a.tags.filter(_.nonEmpty)) } + .toSet == expectedAdds) + assert(compacted.collect { case r: RemoveFile => r.copy(tags = r.tags.filter(_.nonEmpty)) } == + original(2L).collect { case r: RemoveFile => + r.copy(dataChange = false, tags = r.tags.filter(_.nonEmpty)) }) + info("compact compacted [0,3] actions: " + compacted.map(_.toJson).mkString("\n")) + + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + assert(Files.readAllBytes(dir.resolve(s"_delta_log/$compactedName")) + .sameElements(compactedBytes)) + // Compactions are optional accelerators. OSS validates original commit continuity; + // neither compaction nor protocol cleanup authorizes removing this interval. + (0L to 5L).foreach { v => + assert(CommitLog.read(dir, v) == original(v)) + assert(Files.readAllBytes(dir.resolve(f"_delta_log/$v%020d.json")) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(f"_delta_log/$v%020d.json")))) + } + adds.values.foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + val listing = Files.list(dir.resolve("_delta_log")) + try assert(!listing.iterator().asScala.exists(_.getFileName.toString.contains("checkpoint"))) + finally listing.close() + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + Seq[java.lang.Long](0L, 1L, 2L, 3L, 4L, 5L, null).foreach { version => + val v = Option(version).fold(5L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + val selected = snapshot.logSegment.deltas.map(_.getPath.getName) + val expected = if (v < 3L) (0L to v).map(n => f"$n%020d.json") + else Seq(compactedName) ++ (4L to v).map(n => f"$n%020d.json") + assert(selected == expected) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + info(s"compact v$v selected log files: " + selected.mkString(",")) + } + } + val initialRows = Seq(Row(-7, "initial"), Row(4, null), Row(4, null), + Row(null, "missing"), Row(12, "keep"), Row(12, "keep")) + val replacementRows = Seq(Row(6, "replacement"), Row(6, "replacement"), + Row(null, "changed"), Row(8, null)) + val endRows = replacementRows ++ Seq(Row(4, null), Row(4, null), Row(23, "end")) + val tailRows = Seq(Row(31, "tail"), Row(31, "tail"), Row(null, "tail-null"), + Row(4, null), Row(4, null)) + val latestRows = tailRows ++ Seq(Row(44, "last"), Row(44, "last")) + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_start_v0", ReadQuery(version = Some(0L)), schema, Seq.empty[Row]), + ("read_inside_v1", ReadQuery(version = Some(1L)), schema, initialRows), + ("read_inside_v2", ReadQuery(version = Some(2L)), schema, replacementRows), + ("read_end_v3", ReadQuery(version = Some(3L)), schema, endRows), + ("read_tail_v4", ReadQuery(version = Some(4L)), schema, tailRows), + ("read_tail_v5", ReadQuery(version = Some(5L)), schema, latestRows), + ("read_latest", ReadQuery(), schema, latestRows), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("tail-null", null)))) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"compact $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"compact $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"compact $name expected parquet") + } + } + } + test("e2e_57_auto_compaction") { + // These settings affect fixture generation only. Ordinary SQL inserts invoke + // OSS AutoCompact's post-commit hook; no explicit compaction command is called. + val settings = Map( + "spark.databricks.delta.autoCompact.enabled" -> "true", + "spark.databricks.delta.autoCompact.minNumFiles" -> "3", + "spark.databricks.delta.autoCompact.minFileSize" -> "1048576", + "spark.databricks.delta.autoCompact.maxFileSize" -> "2097152") + val saved = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap + try { + settings.foreach { case (key, value) => sql(s"SET $key=$value") } + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.autoOptimize.autoCompact' = 'true', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '1000', 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'initial'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (12, 'keep'), (12, 'keep'), (23, 'end') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (44, 'last'), (44, 'last'), (CAST(NULL AS INT), CAST(NULL AS STRING)), + (88, 'edge') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (99, 'tail'), (99, 'tail') AS v(id, value)""") + } finally { + saved.foreach { + case (key, Some(value)) => spark.conf.set(key, value) + case (key, None) => spark.conf.unset(key) + } + } + val table = registerTable("tbl") + val commits = (0L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + assert(!Files.exists(CommitLog.commitFile(table.sourcePath, 6L))) + val inputAdds = (1L to 3L).flatMap { v => + val adds = commits(v).collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.dataChange) + assert(adds.head.size > 0L && adds.head.size < 1048576L) + assert(commits(v).collect { case r: RemoveFile => r }.isEmpty) + adds + } + val rewritten = commits(4L).collect { case a: AddFile => a } + val removed = commits(4L).collect { case r: RemoveFile => r } + assert(rewritten.size == 1 && !rewritten.head.dataChange) + assert(removed.size == 3 && removed.forall(r => !r.dataChange)) + assert(removed.map(_.path).toSet == inputAdds.map(_.path).toSet) + assert(!inputAdds.exists(_.path == rewritten.head.path)) + val tail = commits(5L).collect { case a: AddFile => a } + assert(tail.size == 1 && tail.head.dataChange) + assert(commits(5L).collect { case r: RemoveFile => r }.isEmpty) + val history = (0L to 5L).map { v => + val info = commits(v).map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + v -> info + }.toMap + assert(history(0L).get("operation").asText() == "CREATE TABLE") + Seq(1L, 2L, 3L, 5L).foreach { v => + assert(history(v).get("operation").asText() == "WRITE") + } + assert(history(4L).get("operation").asText() == "OPTIMIZE") + assert(history(4L).get("operationParameters").get("auto").asText() == "true") + info("auto automatic compaction commit v4: " + history(4L).toString) + + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val metadata = commits(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val configuration = Map("delta.autoOptimize.autoCompact" -> "true", + "delta.enableDeletionVectors" -> "false", "delta.checkpointInterval" -> "1000", + "delta.checkpointPolicy" -> "classic") + assert(commits(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + (0L to 5L).foreach { v => + assert(CommitLog.read(dir, v) == commits(v)) + assert(Files.readAllBytes(CommitLog.commitFile(dir, v)) + .sameElements(Files.readAllBytes(CommitLog.commitFile(table.sourcePath, v)))) + } + (inputAdds ++ rewritten ++ tail).foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + val logs = Files.list(dir.resolve("_delta_log")) + try assert(!logs.iterator().asScala.exists(_.getFileName.toString.contains("checkpoint"))) + finally logs.close() + } + val activePaths = Map(0L -> Set.empty[String], 1L -> Set(inputAdds(0).path), + 2L -> inputAdds.take(2).map(_.path).toSet, 3L -> inputAdds.map(_.path).toSet, + 4L -> Set(rewritten.head.path), 5L -> Set(rewritten.head.path, tail.head.path)) + Seq[java.lang.Long](0L, 1L, 2L, 3L, 4L, 5L, null).foreach { version => + val v = Option(version).fold(5L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + assert(snapshot.allFiles.collect().map(_.path).toSet == activePaths(v)) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + } + } + // Literal, duplicate-sensitive bags are independent of captured expectations. + val firstRows = Seq(Row(-7, "initial"), Row(4, null), Row(4, null), Row(null, "missing")) + val secondRows = firstRows ++ Seq(Row(12, "keep"), Row(12, "keep"), Row(23, "end")) + val beforeRows = secondRows ++ Seq(Row(44, "last"), Row(44, "last"), + Row(null, null), Row(88, "edge")) + val latestRows = beforeRows ++ Seq(Row(99, "tail"), Row(99, "tail")) + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_first_v1", ReadQuery(version = Some(1L)), schema, firstRows, 1, 0L), + ("read_below_threshold_v2", ReadQuery(version = Some(2L)), schema, secondRows, 2, 0L), + ("read_before_v3", ReadQuery(version = Some(3L)), schema, beforeRows, 3, 0L), + ("read_post_v4", ReadQuery(version = Some(4L)), schema, beforeRows, 1, 0L), + ("read_tail_v5", ReadQuery(version = Some(5L)), schema, latestRows, 2, 0L), + ("read_latest", ReadQuery(), schema, latestRows, 2, 0L), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("missing", null), Row(null, null)), 1, 1L)) + cases.foreach { case (name, query, expectedSchema, rows, fileCount, skipped) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, fileCount, skipped))) + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"auto $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"auto $name expected parquet") + } + } + } + + test("e2e_58_unicode_路径") { + // The generator copies contents into /delta. Relative log paths must + // survive both the Unicode source root and that different destination root. + val parent = Files.createTempDirectory("paths-source-") + val source = parent.resolve("源_café_表_😀") + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta LOCATION '$source' + TBLPROPERTIES ('delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '1000', 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'base'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (4, CAST(NULL AS STRING)), (23, 'end'), + (CAST(NULL AS INT), CAST(NULL AS STRING)) AS v(id, value)""") + // Canonically equivalent NFC/NFD basenames must remain distinct byte names. + val basenames = Seq("données_café_表_😀.parquet", "données_cafe\u0301_表_😀.parquet") + assert(basenames.distinct.size == 2) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val first = Seq(Row(-7, "base"), Row(4, null), Row(4, null), Row(null, "missing")) + val second = Seq(Row(4, null), Row(23, "end"), Row(null, null)) + val inputRows = Seq(first, second) + val renamedAdds = (1L to 2L).map { v => + val add = CommitLog.read(source, v).collect { case a: AddFile => a } + .ensuring(_.size == 1).head + assert(!add.path.contains("/") && add.deletionVector.isEmpty) + val basename = basenames((v - 1).toInt) + val file = source.resolve(basename) + Files.move(source.resolve(add.path), file) + // Change only the file reference in raw JSON; preserve every other action field. + val commit = CommitLog.commitFile(source, v) + val lines = Files.readAllLines(commit).asScala.map { line => + val node = JsonUtil.mapper.readTree(line) + if (node.has("add")) { + node.get("add").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("path", new HadoopPath(basename).toUri.toString) + } + node.toString + } + Files.write(commit, lines.asJava) + val physical = spark.read.parquet(file.toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(inputRows((v - 1).toInt).asJava, + schema), physical, s"paths source physical v$v") + val renamed = CommitLog.read(source, v).collect { case a: AddFile => a }.head + assert(new HadoopPath(renamed.path).toUri.getPath == basename) + assert(Files.size(file) == renamed.size) + renamed + } + // Version checksums can embed the old paths; do not transport stale checksums. + val logs = Files.list(source.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + val commits = (0L to 2L).map(v => v -> Files.readAllBytes( + CommitLog.commitFile(source, v))).toMap + val metadata = CommitLog.read(source, 0L).collect { case m: Metadata => m } + .ensuring(_.size == 1).head + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(CommitLog.read(source, 0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + val table = registerTableFromPath(source.toString) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir != source && dir.getParent.getFileName.toString.endsWith("路径")) + commits.foreach { case (v, bytes) => + assert(Files.readAllBytes(CommitLog.commitFile(dir, v)).sameElements(bytes)) + assert(!new String(bytes, java.nio.charset.StandardCharsets.UTF_8) + .contains(source.toString)) + } + basenames.foreach { basename => + val file = dir.resolve(basename) + assert(file.getFileName.toString.getBytes(java.nio.charset.StandardCharsets.UTF_8) + .sameElements(basename.getBytes(java.nio.charset.StandardCharsets.UTF_8))) + assert(Files.readAllBytes(file).sameElements(Files.readAllBytes(source.resolve(basename)))) + } + // Remove the entire source before any snapshot/read capture. An absolute or + // stale reference to it cannot make the relocated fixture pass by accident. + org.apache.commons.io.FileUtils.deleteDirectory(parent.toFile) + assert(!Files.exists(parent) && !Files.exists(source)) + DeltaLog.clearCache() + info(s"paths source removed: $source; relocated root: $dir") + } + Seq[java.lang.Long](0L, 1L, 2L, null).foreach { version => + val v = Option(version).fold(2L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(!Files.exists(source)) + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + assert(snapshot.allFiles.collect().map(_.path).toSet == + renamedAdds.take(v.toInt).map(_.path).toSet) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_v1", ReadQuery(version = Some(1L)), schema, first, 1), + ("read_v2", ReadQuery(version = Some(2L)), schema, first ++ second, 2), + ("read_latest", ReadQuery(), schema, first ++ second, 2), + ("read_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row(null, 4), Row("missing", null), Row(null, null)), 2)) + cases.foreach { case (name, query, expectedSchema, rows, files) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(!Files.exists(source)) + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, files, 0L))) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + val paths = actual.inputFiles.map(p => Paths.get(java.net.URI.create(p))).toSet + val expectedFiles = basenames.take(files).map(copiedTable.resolve).toSet + assert(paths == expectedFiles && paths.forall(_.startsWith(copiedTable))) + assert(actual.schema == expectedSchema) + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"paths $name relocated table") + val fixture = copiedTable.getParent + val captured = spark.read.parquet(fixture.resolve( + s"expected/${fixture.getFileName}_$name/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"paths $name expected parquet") + } + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTimestampSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTimestampSuite.scala new file mode 100644 index 00000000..768c30d9 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTimestampSuite.scala @@ -0,0 +1,1942 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.nio.{ByteBuffer, ByteOrder} +import java.nio.file.{Files, Path, StandardCopyOption} +import java.sql.Timestamp +import java.time.{Instant, LocalDate} + +import scala.jdk.CollectionConverters._ + +import org.apache.commons.io.FileUtils +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.example.data.Group +import org.apache.parquet.example.data.simple.{NanoTime, SimpleGroupFactory} +import org.apache.parquet.hadoop.{ParquetFileReader, ParquetReader} +import org.apache.parquet.hadoop.example.{ExampleParquetWriter, GroupReadSupport} +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.parquet.schema.{LogicalTypeAnnotation, PrimitiveType} +import org.apache.spark.sql.Row +import org.apache.spark.sql.functions.{col, lit, when} +import org.apache.spark.sql.types.{DataType, IntegerType, StructType, TimestampNTZType, + TimestampType} + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog} +import io.delta.workload.model.Succeeded + +/** Timestamp stats encodings are checked on disk, independently of the table reader. */ +class ReadinessTimestampSuite extends WorkloadTestSuite("readiness_timestamp") { + private val first = "2024-01-01T00:00:00.123456Z" + private val last = "2024-01-01T00:00:00.123789Z" + private val nanoLower = "2024-01-01T00:00:00.123455111Z" + private val nanoUpper = "2024-01-01T00:00:00.123789999Z" + private val schema = new StructType().add("id", IntegerType).add("ts", TimestampType) + + // Literal instants, not values read back from the table, are the row oracle. + private def expectedRows: Seq[Row] = Seq( + Row(1, Timestamp.from(Instant.parse(first))), + Row(2, Timestamp.from(Instant.parse(last))), + Row(2, Timestamp.from(Instant.parse(last)))) + + private def createTable(): TableHandle = { + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP) USING delta TBLPROPERTIES ( + 'delta.minReaderVersion' = '1', 'delta.minWriterVersion' = '2', + 'delta.checkpointPolicy' = 'classic', + 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, TIMESTAMP '2024-01-01 00:00:00.123456'), + (2, TIMESTAMP '2024-01-01 00:00:00.123789'), + (2, TIMESTAMP '2024-01-01 00:00:00.123789') AS rows(id, ts)""") + val t = registerTable("tbl") + val adds = CommitLog.read(t.sourcePath, 1).collect { case a: AddFile => a } + require(adds.size == 1, "The timestamp bounds must describe one file with all three rows") + t + } + + private def checkSpecs(t: TableHandle, checkJsonStats: Boolean = false): Unit = { + val workloadName = WorkloadContext.current.workloadName + val tableOutputDir = outputDir.resolve(workloadName) + val queries = Seq( + ("all", null, expectedRows), + ("lower_edge", "ts <= TIMESTAMP '2024-01-01 00:00:00.123456'", expectedRows.take(1)), + ("upper_edge", "ts >= TIMESTAMP '2024-01-01 00:00:00.123789'", expectedRows.drop(1)), + ("outside", "ts < TIMESTAMP '2024-01-01 00:00:00.123456' OR " + + "ts > TIMESTAMP '2024-01-01 00:00:00.123789'", Seq.empty[Row])) + + for (version <- Seq[java.lang.Long](null, 1L)) { + val suffix = if (version == null) "latest" else "v1" + for ((label, predicate, rows) <- queries) { + val name = s"${label}_$suffix" + readSpec(t, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, schema) + val expectedDir = tableOutputDir.resolve("expected") + .resolve(s"${workloadName}_$name") + val captured = expectedDir.resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + if (checkJsonStats && label == "all") { + val scanned = spark.read.parquet( + expectedDir.resolve("expected_metadata").toString).collect() + require(scanned.length == 1) + val scanStats = JsonUtil.mapper.readTree(scanned.head.getString(0)) + .at("/add/stats").asText() + val tableDir = tableOutputDir.resolve("delta") + val rawStats = CommitLog.read(tableDir, 1) + .collectFirst { case a: AddFile => a.stats.get }.get + require(scanStats == rawStats, "Scanned stats must retain the injected precision") + println(s"Verified $name scanned AddFile stats match nanosecond JSON") + } + } + } + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + case other => fail(s"$suffix snapshot must succeed: $other") + } + } + } + } + + test("nanosecond_stats_json") { + val t = createTable() + // The bounds enclose the microsecond rows. All of digits 7-9 are nonzero. + val lower = nanoLower + val upper = nanoUpper + require(Instant.parse(lower).isBefore(Instant.parse(first))) + require(Instant.parse(upper).isAfter(Instant.parse(last))) + modifyCommitActions(t, version = 1) { actions => + actions.map { + case a: AddFile => + val stats = JsonUtil.mapper.readTree(a.stats.get) + stats.get("minValues").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", lower) + stats.get("maxValues").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", upper) + a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + mutateTable(t) { tableDir => + val adds = CommitLog.read(tableDir, 1).collect { case a: AddFile => a } + require(adds.size == 1) + val raw = JsonUtil.mapper.readTree(adds.head.stats.get) + require(raw.at("/minValues/ts").asText() == lower) + require(raw.at("/maxValues/ts").asText() == upper) + require(raw.get("numRecords").asLong() == 3) + // A copied checksum can contain the old AddFile stats and bypass JSON parsing. + val files = Files.list(tableDir.resolve("_delta_log")) + try files.iterator().asScala.filter(_.toString.endsWith(".crc")) + .foreach(Files.delete(_)) finally files.close() + println(s"Verified JSON stats: min=$lower max=$upper records=3") + } + checkSpecs(t, checkJsonStats = true) + } + + for ((caseName, encoding, lower, upper) <- Seq( + ("millisecond_stats_checkpoint", "TIMESTAMP_MILLIS", + "2024-01-01T00:00:00.123Z", "2024-01-01T00:00:00.124Z"), + ("microsecond_stats_checkpoint", "TIMESTAMP_MICROS", first, last), + ("int96_stats_checkpoint", "INT96", first, last))) { + test(caseName) { + val t = createTable() + require(!Instant.parse(lower).isAfter(Instant.parse(first))) + require(!Instant.parse(upper).isBefore(Instant.parse(last))) + forceCheckpoint("tbl") + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val checkpoint = logDir.resolve("00000000000000000001.checkpoint.parquet") + require(Files.isRegularFile(checkpoint), "Expected one classic checkpoint") + val rewriteDir = logDir.resolve("timestamp-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, encoding) + val df = spark.read.parquet(checkpoint.toString) + val stats = col("add.stats_parsed") + .withField("minValues", col("add.stats_parsed.minValues") + .withField("ts", lit(Timestamp.from(Instant.parse(lower))))) + .withField("maxValues", col("add.stats_parsed.maxValues") + .withField("ts", lit(Timestamp.from(Instant.parse(upper))))) + // Disable the JSON fallback: these reads must consume parsed checkpoint stats. + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewriteDir.toString) + val files = Files.list(rewriteDir) + val part = try { + val parts = files.iterator().asScala.filter(_.toString.endsWith(".parquet")).toSeq + require(parts.size == 1) + parts.head + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewriteDir)) FileUtils.deleteDirectory(rewriteDir.toFile) + } + checkCheckpoint(checkpoint, encoding, lower, upper) + if (encoding == "INT96") { + // Spark's writer only emits microsecond-aligned INT96. Preserve every checkpoint + // field with the existing Parquet Group API, replacing just the two stats cells. + rewriteInt96Bounds(checkpoint) + checkCheckpoint(checkpoint, encoding, nanoLower, nanoUpper) + } + // Retain v1 JSON as a discovery/metadata tail alongside the checkpoint. Removing v0 + // prevents full JSON replay, so v1 and latest require checkpoint bootstrap. + val files = Files.list(logDir) + try files.iterator().asScala.filter { p => + p.getFileName.toString == "00000000000000000000.json" || + p.toString.endsWith(".crc") + }.foreach(Files.delete(_)) finally files.close() + } + checkSpecs(t) + } + } + + private def rewriteInt96Bounds(checkpoint: Path): Unit = { + import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ + + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + val metadata = try footer.getFooter.getFileMetaData finally footer.close() + val parquetSchema = metadata.getSchema + val factory = new SimpleGroupFactory(parquetSchema) + // 2024-01-01 is Julian day 2460311. These nanoseconds after midnight enclose the rows. + val replacements = Map( + "add.stats_parsed.minValues.ts" -> new NanoTime(2460311, 123455111L).toBinary, + "add.stats_parsed.maxValues.ts" -> new NanoTime(2460311, 123789999L).toBinary) + + def copyGroup(source: Group, target: Group, prefix: String): Unit = { + for (i <- 0 until source.getType.getFieldCount) { + val field = source.getType.getType(i) + val fieldPath = if (prefix.isEmpty) field.getName else s"$prefix.${field.getName}" + for (j <- 0 until source.getFieldRepetitionCount(i)) { + if (!field.isPrimitive) { + copyGroup(source.getGroup(i, j), target.addGroup(i), fieldPath) + } else field.asPrimitiveType().getPrimitiveTypeName match { + case BOOLEAN => target.add(i, source.getBoolean(i, j)) + case INT32 => target.add(i, source.getInteger(i, j)) + case INT64 => target.add(i, source.getLong(i, j)) + case FLOAT => target.add(i, source.getFloat(i, j)) + case DOUBLE => target.add(i, source.getDouble(i, j)) + case BINARY | FIXED_LEN_BYTE_ARRAY => target.add(i, source.getBinary(i, j)) + case INT96 => target.add(i, replacements.getOrElse(fieldPath, source.getInt96(i, j))) + } + } + } + } + + val temporary = checkpoint.resolveSibling("timestamp-int96-rewrite.parquet") + try { + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + try { + val writer = ExampleParquetWriter.builder(new HadoopPath(temporary.toUri)) + .withConf(conf).withType(parquetSchema) + .withExtraMetaData(metadata.getKeyValueMetaData).build() + try { + var source = reader.read() + while (source != null) { + val target = factory.newGroup() + copyGroup(source, target, "") + writer.write(target) + source = reader.read() + } + } finally writer.close() + } finally reader.close() + Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally Files.deleteIfExists(temporary) + } + + private def checkCheckpoint( + checkpoint: Path, encoding: String, lower: String, upper: String): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val parquetSchema = footer.getFooter.getFileMetaData.getSchema + for (bound <- Seq("minValues", "maxValues")) { + val physical = parquetSchema.getType("add", "stats_parsed", bound, "ts") + .asPrimitiveType() + if (encoding == "INT96") { + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT96) + require(physical.getLogicalTypeAnnotation == null) + } else { + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = physical.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + val unit = if (encoding == "TIMESTAMP_MILLIS") { + LogicalTypeAnnotation.TimeUnit.MILLIS + } else LogicalTypeAnnotation.TimeUnit.MICROS + require(annotation.getUnit == unit) + require(annotation.isAdjustedToUTC) + } + println(s"Verified checkpoint add.stats_parsed.$bound.ts: $physical") + } + } finally footer.close() + + // Read the physical cells without Spark's timestamp conversion or Delta's stats parser. + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var adds = 0 + try { + var row = reader.read() + while (row != null) { + if (row.getFieldRepetitionCount("add") > 0) { + adds += 1 + val add = row.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + require(stats.getLong("numRecords", 0) == 3) + for ((bound, value) <- Seq("minValues" -> lower, "maxValues" -> upper)) { + val cell = stats.getGroup(bound, 0) + val instant = Instant.parse(value) + if (encoding == "INT96") { + val bytes = ByteBuffer.wrap(cell.getInt96("ts", 0).getBytes) + .order(ByteOrder.LITTLE_ENDIAN) + val nanosOfDay = bytes.getLong() + val julianDay = bytes.getInt() + require(julianDay == 2440588 + Math.floorDiv(instant.getEpochSecond, 86400L)) + require(nanosOfDay == Math.floorMod(instant.getEpochSecond, 86400L) * + 1000000000L + instant.getNano) + // Check the aligned intermediate separately from the true nanos replacement. + require(nanosOfDay % 1000 == instant.getNano % 1000) + require(nanosOfDay % 1000000 != 0) + println(s"Verified raw INT96 $bound: day=$julianDay nanosOfDay=$nanosOfDay") + } else { + val units = if (encoding == "TIMESTAMP_MILLIS") 1000L else 1000000L + val expected = instant.getEpochSecond * units + + instant.getNano / (1000000000L / units) + require(cell.getLong("ts", 0) == expected) + println(s"Verified raw $encoding $bound: $expected") + } + } + } + row = reader.read() + } + } finally reader.close() + require(adds == 1) + } + + // Each data boundary is crossed with every temporal partition boundary. + // SQL fixture literals and the typed row oracle are deliberately declared separately. + private val epochValues = Seq( + (1, "1969-12-31T23:59:59.998999", "1969-12-31", -1001L), + (2, "1969-12-31T23:59:59.999000", "1969-12-31", -1000L), + (3, "1969-12-31T23:59:59.999001", "1969-12-31", -999L), + (4, "1969-12-31T23:59:59.999999", "1969-12-31", -1L), + (5, "1970-01-01T00:00:00.000000", "1970-01-01", 0L), + (6, "1970-01-01T00:00:00.000001", "1970-01-01", 1L), + (7, "1970-01-01T00:00:00.001000", "1970-01-02", 1000L)) + + private val epochSchema = new StructType().add("id", IntegerType) + .add("ts", TimestampType).add("ntz", org.apache.spark.sql.types.TimestampNTZType) + .add("d", org.apache.spark.sql.types.DateType).add("p_ts", TimestampType) + .add("p_ntz", org.apache.spark.sql.types.TimestampNTZType) + .add("p_d", org.apache.spark.sql.types.DateType) + + private def epochRows: Seq[Row] = { + val data = epochValues ++ epochValues.filter(_._1 == 4) + for ((id, wall, day, _) <- data; (_, pWall, pDay, _) <- epochValues) yield { + Row(id, Timestamp.from(Instant.parse(wall + "Z")), java.time.LocalDateTime.parse(wall), + java.sql.Date.valueOf(day), Timestamp.from(Instant.parse(pWall + "Z")), + java.time.LocalDateTime.parse(pWall), java.sql.Date.valueOf(pDay)) + } + } + + private def epochPartitionStrings: Set[Map[String, String]] = epochValues.map { + case (_, wall, day, _) => + // NTZ uses wall-clock strings; TIMESTAMP uses UTC ISO with six fractional digits. + val partition = wall.replace('T', ' ').replaceAll("0+$", "").stripSuffix(".") + Map("p_ts" -> (wall + "Z"), "p_ntz" -> partition, "p_d" -> day) + }.toSet + + private def createEpochTable(): TableHandle = { + require(spark.conf.get("spark.sql.session.timeZone") == "UTC") + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ, d DATE, + p_ts TIMESTAMP, p_ntz TIMESTAMP_NTZ, p_d DATE) USING delta + PARTITIONED BY (p_ts, p_ntz, p_d) TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', + 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + val fixtures = Seq( + (1, "1969-12-31 23:59:59.998999", "1969-12-31"), + (2, "1969-12-31 23:59:59.999", "1969-12-31"), + (3, "1969-12-31 23:59:59.999001", "1969-12-31"), + (4, "1969-12-31 23:59:59.999999", "1969-12-31"), + (5, "1970-01-01 00:00:00", "1970-01-01"), + (6, "1970-01-01 00:00:00.000001", "1970-01-01"), + (7, "1970-01-01 00:00:00.001", "1970-01-02")) + val partitions = fixtures.map { case (_, ts, day) => + s"(TIMESTAMP '$ts', TIMESTAMP_NTZ '$ts', DATE '$day')" + }.mkString(",") + for ((id, ts, day) <- fixtures) { + val data = s"($id, TIMESTAMP '$ts', TIMESTAMP_NTZ '$ts', DATE '$day')" + val rows = if (id == 4) s"$data,$data" else data + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM + (VALUES $rows) AS data(id, ts, ntz, d) CROSS JOIN + (VALUES $partitions) AS parts(p_ts, p_ntz, p_d)""") + } + val t = registerTable("tbl") + for (v <- 1L to 7L) { + val adds = CommitLog.read(t.sourcePath, v).collect { case a: AddFile => a } + require(adds.size == 7, "Each boundary must have one file per partition") + require(adds.map(_.partitionValues).toSet == epochPartitionStrings) + } + t + } + + for (parsed <- Seq(false, true)) { + val name = if (parsed) "negative_epoch_parsed_stats_partitions" + else "negative_epoch_json_stats_partitions" + test(name) { + val t = createEpochTable() + if (parsed) forceCheckpoint("tbl") + if (!parsed) { + for ((id, wall, day, _) <- epochValues) { + modifyCommitActions(t, version = id.toLong) { actions => + actions.map { + case a: AddFile => + val stats = JsonUtil.mapper.readTree(a.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.get(bound) + .asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + values.put("ts", wall + "Z") + values.put("ntz", wall) + values.put("d", day) + } + a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + } + } + mutateTable(t) { dir => + if (parsed) { + val checkpoint = dir.resolve("_delta_log/00000000000000000007.checkpoint.parquet") + val rewrite = dir.resolve("_delta_log/epoch-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + var ts = col(s"add.stats_parsed.$bound.ts") + var ntz = col(s"add.stats_parsed.$bound.ntz") + for ((id, wall, _, _) <- epochValues) { + val matches = col("add.stats_parsed.minValues.id") === lit(id) + ts = org.apache.spark.sql.functions.when(matches, + lit(Timestamp.from(Instant.parse(wall + "Z")))).otherwise(ts) + ntz = org.apache.spark.sql.functions.when(matches, + lit(java.time.LocalDateTime.parse(wall))).otherwise(ntz) + } + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("ts", ts).withField("ntz", ntz)) + } + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewrite.toString) + val files = Files.list(rewrite) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten checkpoint: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewrite)) FileUtils.deleteDirectory(rewrite.toFile) + } + checkEpochCheckpoint(checkpoint) + } else { + for ((id, wall, day, _) <- epochValues) { + val adds = CommitLog.read(dir, id.toLong).collect { case a: AddFile => a } + require(adds.size == 7) + require(adds.map(_.partitionValues).toSet == epochPartitionStrings) + for (a <- adds) { + val stats = JsonUtil.mapper.readTree(a.stats.get) + require(stats.get("numRecords").asLong() == (if (id == 4) 2 else 1)) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/ts").asText() == wall + "Z") + require(stats.at(s"/$bound/ntz").asText() == wall) + require(stats.at(s"/$bound/d").asText() == day) + require(stats.at(s"/$bound/id").asInt() == id) + } + } + } + println("Verified exact JSON stats and partition strings for all 49 boundary files") + } + val files = Files.list(dir.resolve("_delta_log")) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (parsed && file.endsWith(".json") && + file != "00000000000000000007.json") + }.foreach(Files.delete(_)) finally files.close() + // v7 JSON is retained only for same-version lookup. There is no replay tail after v7. + // v0-v6 are absent, so the checkpoint is necessary for both latest and version 7. + } + checkEpochSpecs(t, parsed) + } + } + + private def checkEpochCheckpoint(checkpoint: Path): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + for ((prefix, fields) <- Seq( + "stats_parsed.minValues" -> Seq("ts", "ntz", "d"), + "stats_parsed.maxValues" -> Seq("ts", "ntz", "d"), + "partitionValues_parsed" -> Seq("p_ts", "p_ntz", "p_d")); field <- fields) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + if (field.endsWith("d")) { + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(cell.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } else { + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == field.endsWith("ts")) + } + println(s"Verified near-epoch checkpoint $prefix.$field: $cell") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var pairs = Vector.empty[(Int, Map[String, String])] + try { + var row = reader.read() + while (row != null) { + if (row.getFieldRepetitionCount("add") > 0) { + val add = row.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + val (_, _, day, micros) = epochValues.find(_._1 == id).get + require(stats.getLong("numRecords", 0) == (if (id == 4) 2 else 1)) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + require(values.getLong("ts", 0) == micros) + require(values.getLong("ntz", 0) == micros) + require(values.getInteger("d", 0) == java.time.LocalDate.parse(day).toEpochDay) + } + val raw = add.getGroup("partitionValues", 0) + val partition = (0 until raw.getFieldRepetitionCount("key_value")).map { i => + val entry = raw.getGroup("key_value", i) + entry.getString("key", 0) -> entry.getString("value", 0) + }.toMap + require(epochPartitionStrings.contains(partition)) + val parsed = add.getGroup("partitionValues_parsed", 0) + val (_, _, pDay, pMicros) = epochValues.find { case (_, wall, _, _) => + java.time.LocalDateTime.parse(wall) == + java.time.LocalDateTime.parse(partition("p_ntz").replace(' ', 'T')) + }.get + require(parsed.getLong("p_ts", 0) == pMicros) + require(parsed.getLong("p_ntz", 0) == pMicros) + require(parsed.getInteger("p_d", 0) == java.time.LocalDate.parse(pDay).toEpochDay) + pairs :+= id -> partition + } + row = reader.read() + } + } finally reader.close() + val expectedPairs = for ((id, _, _, _) <- epochValues; p <- epochPartitionStrings) yield id -> p + require(pairs.size == 49 && pairs.toSet == expectedPairs.toSet) + println("Verified 49 physical stats/partition pairs: micros -1001,-1000,-999,-1,0,1,1000") + } + + private def checkEpochSpecs(t: TableHandle, parsed: Boolean): Unit = { + val workload = WorkloadContext.current.workloadName + val tableDir = outputDir.resolve(workload) + val all = epochRows + def read(name: String, predicate: String, rows: Seq[Row], version: java.lang.Long = null) + : Unit = { + readSpec(t, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, epochSchema) + val captured = tableDir.resolve("expected").resolve(s"${workload}_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + } + } + read("all", null, all) + read("all_v7", null, all, 7L) + for ((field, index, kind) <- Seq(("ts", 1, "TIMESTAMP"), ("ntz", 2, "TIMESTAMP_NTZ"), + ("p_ts", 4, "TIMESTAMP"), ("p_ntz", 5, "TIMESTAMP_NTZ"))) { + for ((id, wall, _, _) <- epochValues) { + val expected = if (kind == "TIMESTAMP") Timestamp.from(Instant.parse(wall + "Z")) + else java.time.LocalDateTime.parse(wall) + read(s"${field}_eq_$id", s"$field = $kind '${wall.replace('T', ' ')}'", + all.filter(_.get(index) == expected)) + } + read(s"${field}_negative", s"$field < $kind '1970-01-01 00:00:00'", + all.filter { r => + if (kind == "TIMESTAMP") r.getTimestamp(index).before(Timestamp.from(Instant.EPOCH)) + else r.getAs[java.time.LocalDateTime](index) + .isBefore(java.time.LocalDateTime.of(1970, 1, 1, 0, 0)) + }) + } + for ((field, index) <- Seq("d" -> 3, "p_d" -> 6); + day <- Seq("1969-12-31", "1970-01-01", "1970-01-02")) { + read(s"${field}_$day", s"$field = DATE '$day'", + all.filter(_.getDate(index) == java.sql.Date.valueOf(day))) + } + read("cross_negative_hits", "ts = TIMESTAMP '1969-12-31 23:59:59.999999' AND " + + "p_ntz = TIMESTAMP_NTZ '1969-12-31 23:59:59.999' AND p_d < DATE '1970-01-01'", + all.filter(r => r.getInt(0) == 4 && r.getTimestamp(4) == + Timestamp.from(Instant.parse("1969-12-31T23:59:59.999Z")))) + read("outside_stats", "ts < TIMESTAMP '1969-12-31 23:59:59.998999' OR " + + "ntz > TIMESTAMP_NTZ '1970-01-01 00:00:00.001'", Seq.empty) + read("outside_partitions", "p_ts < TIMESTAMP '1969-12-31 23:59:59.998999' OR " + + "p_ntz > TIMESTAMP_NTZ '1970-01-01 00:00:00.001'", Seq.empty) + for (version <- Seq[java.lang.Long](null, 7L)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == epochSchema) + require(result.metadata.partitionColumns == Seq("p_ts", "p_ntz", "p_d")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.metadata.configuration("delta.checkpointPolicy") == "classic") + require(result.protocol.minReaderVersion == 3) + require(result.protocol.minWriterVersion == 7) + require(result.protocol.readerFeatures.get.contains("timestampNtz")) + require(result.protocol.writerFeatures.get.contains("timestampNtz")) + case other => fail(s"Near-epoch snapshot must succeed (parsed=$parsed): $other") + } + } + } + } + + // Statistics and partition cases share the absolute-instant fixture and oracle. + private val mismatchSchema = new StructType().add("id", IntegerType) + .add("ts", TimestampType).add("ntz", org.apache.spark.sql.types.TimestampNTZType) + private val mismatchZones = Seq( + ("writer_utc_reader_non_utc", "UTC", "America/Los_Angeles"), + ("writer_non_utc_reader_utc", "America/Los_Angeles", "UTC")) + + // Offsets distinguish both occurrences of 01:30 at the autumn overlap. The unqualified + // controls deliberately produce different instants in the two writer-zone fixtures. + private val mismatchSql = Seq( + (1, "2024-03-10 01:59:59.123456-08:00", "2024-03-10 01:59:59.123456"), + (2, "2024-03-10 03:00:00.123456-07:00", "2024-03-10 03:00:00.123456"), + (3, "2024-11-03 01:30:00.123456-07:00", "2024-11-03 01:30:00.123456"), + (4, "2024-11-03 01:30:00.123456-08:00", "2024-11-03 01:30:00.123456"), + (5, "2024-01-15 12:00:00.654321", "2024-01-15 12:00:00.654321"), + (6, "2024-03-10 02:30:00.123456", "2024-03-10 02:30:00.123456"), + (7, "2024-11-03 01:30:00.123456", "2024-11-03 01:30:00.123456")) + + // This oracle does not parse fixture SQL or read Delta. NTZ retains the written wall clock, + // including the nonexistent LA 02:30. TIMESTAMP instead denotes an absolute UTC instant. + private def mismatchRows(writerZone: String): Seq[Row] = { + val controls = if (writerZone == "UTC") Seq( + "2024-01-15T12:00:00.654321Z", "2024-03-10T02:30:00.123456Z", + "2024-11-03T01:30:00.123456Z") else Seq( + "2024-01-15T20:00:00.654321Z", "2024-03-10T10:30:00.123456Z", + "2024-11-03T08:30:00.123456Z") + val instants = Seq( + "2024-03-10T09:59:59.123456Z", "2024-03-10T10:00:00.123456Z", + "2024-11-03T08:30:00.123456Z", "2024-11-03T09:30:00.123456Z") ++ controls + val walls = Seq( + "2024-03-10T01:59:59.123456", "2024-03-10T03:00:00.123456", + "2024-11-03T01:30:00.123456", "2024-11-03T01:30:00.123456", + "2024-01-15T12:00:00.654321", "2024-03-10T02:30:00.123456", + "2024-11-03T01:30:00.123456") + val rows = instants.zip(walls).zipWithIndex.map { case ((instant, wall), i) => + Row(i + 1, Timestamp.from(Instant.parse(instant)), java.time.LocalDateTime.parse(wall)) + } + rows ++ rows.filter(_.getInt(0) == 3) + } + + private def mismatchMicros(instant: Instant): Long = + instant.getEpochSecond * 1000000L + instant.getNano / 1000 + + // Spark to_json's default timestamp pattern serializes both bounds at millisecond + // precision. Check that physical encoding independently; keep microsecond row expectations. + private def mismatchJsonInstant(instant: Instant): Instant = + instant.truncatedTo(java.time.temporal.ChronoUnit.MILLIS) + + private def createMismatchTable(writerZone: String, partitioned: Boolean): TableHandle = { + spark.conf.set("spark.sql.session.timeZone", writerZone) + val partition = if (partitioned) "PARTITIONED BY (ts)" else "" + sql(s"""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ) USING delta + $partition TBLPROPERTIES ('delta.checkpointPolicy' = 'classic', + 'delta.checkpointInterval' = '100', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + for ((id, instant, wall) <- mismatchSql) { + val row = s"($id, TIMESTAMP '$instant', TIMESTAMP_NTZ '$wall')" + val values = if (id == 3) s"$row, $row" else row + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS rows(id, ts, ntz)") + } + val table = registerTable("tbl") + val rows = mismatchRows(writerZone) + for (version <- 1L to 7L) { + val adds = CommitLog.read(table.sourcePath, version).collect { case a: AddFile => a } + require(adds.size == 1, "Each input batch must produce exactly one file") + val add = adds.head + val row = rows.find(_.getInt(0) == version.toInt).get + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.get("numRecords").asLong() == (if (version == 3) 2 else 1)) + val expectedPartition = if (partitioned) { + Map("ts" -> row.getTimestamp(1).toInstant.toString) + } else Map.empty[String, String] + if (partitioned && writerZone == "America/Los_Angeles" && version == 4 && + add.partitionValues != expectedPartition) { + // The producer reparses an offset-free directory name at the overlap's earlier offset. + // Preserve that limitation, then construct the intended AddFile in the copied fixture. + require(add.partitionValues == Map("ts" -> "2024-11-03T08:30:00.123456Z")) + println(s"Observed producer overlap loss: expected=$expectedPartition " + + s"actual=${add.partitionValues} action=${add.toJson}") + } else require(add.partitionValues == expectedPartition) + for (bound <- Seq("minValues", "maxValues")) { + val fields = stats.get(bound) + require(fields.get("id").asInt() == version.toInt) + if (partitioned) require(!fields.has("ts")) else { + val raw = fields.get("ts").asText() + val parsed = java.time.OffsetDateTime.parse(raw) + val instant = row.getTimestamp(1).toInstant + require(parsed.toInstant == mismatchJsonInstant(instant)) + require(parsed.getOffset == instant.atZone(java.time.ZoneId.of(writerZone)).getOffset) + } + val ntz = row.getAs[java.time.LocalDateTime](2).toInstant(java.time.ZoneOffset.UTC) + val rawNtz = java.time.LocalDateTime.parse(fields.get("ntz").asText()) + require(rawNtz.toInstant(java.time.ZoneOffset.UTC) == mismatchJsonInstant(ntz)) + } + println(s"Verified writer JSON zone=$writerZone partitioned=$partitioned " + + s"id=$version stats=${add.stats.get} partitions=${add.partitionValues}") + } + println(s"Verified writer table zone=${spark.conf.get("spark.sql.session.timeZone")} " + + s"partitioned=$partitioned rows=8 files=7") + table + } + + private def prepareMismatchCapture( + table: TableHandle, writerZone: String, readerZone: String, + partitioned: Boolean, parsed: Boolean): Unit = { + require(writerZone != readerZone) + require(spark.conf.get("spark.sql.session.timeZone") == writerZone) + if (partitioned && writerZone == "America/Los_Angeles") { + val later = mismatchRows(writerZone).find(_.getInt(0) == 4).get.getTimestamp(1).toInstant + modifyCommitActions(table, version = 4) { actions => + actions.map { + case add: AddFile => add.copy(partitionValues = Map("ts" -> later.toString)) + case other => other + } + } + } + if (parsed) { + val encoding = spark.conf.get("spark.sql.parquet.outputTimestampType") + try { + spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MICROS") + forceCheckpoint("tbl") + } finally spark.conf.set("spark.sql.parquet.outputTimestampType", encoding) + } + mutateTable(table) { dir => + require(spark.conf.get("spark.sql.session.timeZone") == readerZone) + val logDir = dir.resolve("_delta_log") + if (parsed) { + val checkpoint = logDir.resolve("00000000000000000007.checkpoint.parquet") + if (partitioned && writerZone == "America/Los_Angeles") { + setMismatchOverlapPartition(checkpoint) + } + checkMismatchCheckpoint(checkpoint, writerZone, partitioned) + } + val files = Files.list(logDir) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (parsed && file.endsWith(".json") && + file != "00000000000000000007.json") + }.foreach(Files.delete(_)) finally files.close() + // Same-version v7 JSON is retained solely for version lookup. No replay tail exists; + // v0-v6 and CRC shortcuts are absent, so the checkpoint is required, not checkpoint-only. + println(s"Verified pre-capture reader=$readerZone writer=$writerZone " + + s"partitioned=$partitioned parsed=$parsed") + } + // WorkloadTestSuite captures after the body. Deliberately leave this reader zone active. + spark.conf.set("spark.sql.session.timeZone", readerZone) + } + + // DAT supports a synthetic, correct overlap partition even when the SQL writer loses the + // later offset while reparsing a partition directory. Preserve its unmodified checkpoint; + // then change only the intended id=4 AddFile's raw and parsed partition cells in the copy. + private def setMismatchOverlapPartition(checkpoint: Path): Unit = { + val workload = WorkloadContext.current.workloadName + val original = outputDir.resolveSibling("producer-checkpoints").resolve(workload) + Files.createDirectories(original) + Files.copy(checkpoint, original.resolve(checkpoint.getFileName), + StandardCopyOption.REPLACE_EXISTING) + val instant = Instant.parse("2024-11-03T09:30:00.123456Z") + val temporary = checkpoint.resolveSibling("overlap-partition-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + val later = col("add.stats_parsed.minValues.id") === lit(4) + val raw = org.apache.spark.sql.functions.when(later, + org.apache.spark.sql.functions.map(lit("ts"), lit(instant.toString))) + .otherwise(col("add.partitionValues")) + val parsed = col("add.partitionValues_parsed").withField("ts", + org.apache.spark.sql.functions.when(later, lit(Timestamp.from(instant))) + .otherwise(col("add.partitionValues_parsed.ts"))) + val add = col("add").withField("partitionValues", raw) + .withField("partitionValues_parsed", parsed) + df.withColumn("add", add).coalesce(1).write.parquet(temporary.toString) + val files = Files.list(temporary) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one overlap checkpoint part: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(temporary)) FileUtils.deleteDirectory(temporary.toFile) + } + println("Constructed later overlap partition fixture: id=4 " + + "instant=2024-11-03T09:30:00.123456Z") + } + + private def checkMismatchCheckpoint( + checkpoint: Path, writerZone: String, partitioned: Boolean): Unit = { + require(Files.isRegularFile(checkpoint)) + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + val timestamps = if (partitioned) Seq("partitionValues_parsed" -> "ts") + else Seq("stats_parsed.minValues" -> "ts", "stats_parsed.maxValues" -> "ts") + for ((prefix, field) <- timestamps ++ Seq( + "stats_parsed.minValues" -> "ntz", "stats_parsed.maxValues" -> "ntz")) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == (field == "ts")) + println(s"Verified timezone checkpoint writer=$writerZone $prefix.$field: $cell") + } + } finally footer.close() + val rows = mismatchRows(writerZone) + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var ids = Vector.empty[Int] + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + val add = record.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + val row = rows.find(_.getInt(0) == id).get + require(stats.getLong("numRecords", 0) == (if (id == 3) 2 else 1)) + val instant = row.getTimestamp(1).toInstant + val ntz = row.getAs[java.time.LocalDateTime](2).toInstant(java.time.ZoneOffset.UTC) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + if (!partitioned) require(values.getLong("ts", 0) == + mismatchMicros(mismatchJsonInstant(instant))) + require(values.getLong("ntz", 0) == mismatchMicros(mismatchJsonInstant(ntz))) + } + if (partitioned) { + val partition = add.getGroup("partitionValues_parsed", 0) + require(partition.getLong("ts", 0) == mismatchMicros(instant)) + val raw = add.getGroup("partitionValues", 0) + require(raw.getFieldRepetitionCount("key_value") == 1) + val entry = raw.getGroup("key_value", 0) + require(entry.getString("key", 0) == "ts") + require(entry.getString("value", 0) == instant.toString) + } + ids :+= id + println(s"Verified timezone raw checkpoint writer=$writerZone partitioned=$partitioned " + + s"id=$id instantMicros=${mismatchMicros(instant)} ntzMicros=${mismatchMicros(ntz)}") + } + record = reader.read() + } + } finally reader.close() + require(ids.sorted == (1 to 7).toVector) + } + + private def checkMismatchSpecs( + table: TableHandle, writerZone: String, readerZone: String, partitioned: Boolean): Unit = { + val workload = WorkloadContext.current.workloadName + val tableDir = outputDir.resolve(workload) + val all = mismatchRows(writerZone) + def checkReaderZone(label: String): Unit = { + require(spark.conf.get("spark.sql.session.timeZone") == readerZone) + val probe = spark.sql("SELECT current_timezone(), " + + "CAST(TIMESTAMP '2024-03-10 10:00:00.123456Z' AS STRING)").head() + require(probe.getString(0) == readerZone) + val wall = if (readerZone == "UTC") "2024-03-10 10:00:00.123456" + else "2024-03-10 03:00:00.123456" + require(probe.getString(1) == wall) + println(s"Verified captured $workload/$label writer=$writerZone reader=$readerZone " + + s"readerWall=$wall") + } + def read(label: String, predicate: String, rows: Seq[Row], version: java.lang.Long): Unit = { + readSpec(table, predicate = predicate, version = version, name = Some(label)).assert { spec => + checkReaderZone(label) + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$label must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, mismatchSchema) + val captured = tableDir.resolve("expected").resolve(s"${workload}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), label) + if (label == "all_latest") { + expected.coalesce(1).write.mode("overwrite") + .parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString) + } + } + } + for (version <- Seq[java.lang.Long](null, 7L)) { + val suffix = if (version == null) "latest" else "v7" + read(s"all_$suffix", null, all, version) + // Explicit offsets keep the emitted predicates portable to consumers with another zone. + for (row <- all.groupBy(_.getInt(0)).values.map(_.head).toSeq.sortBy(_.getInt(0))) { + val instant = row.getTimestamp(1) + read(s"eq_${row.getInt(0)}_$suffix", s"ts = TIMESTAMP '${instant.toInstant}'", + all.filter(_.getTimestamp(1) == instant), version) + } + read(s"spring_range_$suffix", "ts >= TIMESTAMP '2024-03-10T09:59:59.123456Z' AND " + + "ts <= TIMESTAMP '2024-03-10T10:00:00.123456Z'", + all.filter(r => Set(1, 2).contains(r.getInt(0))), version) + read(s"overlap_range_$suffix", "ts >= TIMESTAMP '2024-11-03T08:30:00.123456Z' AND " + + "ts <= TIMESTAMP '2024-11-03T09:30:00.123456Z'", + all.filter(r => Set(3, 4).contains(r.getInt(0)) || + (writerZone != "UTC" && r.getInt(0) == 7)), version) + read(s"outside_$suffix", "ts < TIMESTAMP '2024-01-01T00:00:00Z'", Seq.empty, version) + read(s"ntz_gap_$suffix", "ntz = TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'", + all.filter(_.getInt(0) == 6), version) + snapshotSpec(table, version = version).assert { spec => + checkReaderZone(s"snapshot_$suffix") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == mismatchSchema) + require(result.metadata.partitionColumns == (if (partitioned) Seq("ts") else Seq.empty)) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.metadata.configuration("delta.checkpointPolicy") == "classic") + require(result.metadata.configuration("delta.checkpoint.writeStatsAsStruct") == "true") + require(result.metadata.configuration("delta.checkpoint.writeStatsAsJson") == "false") + require(result.protocol.minReaderVersion == 3 && result.protocol.minWriterVersion == 7) + require(result.protocol.readerFeatures.get.contains("timestampNtz")) + require(result.protocol.writerFeatures.get.contains("timestampNtz")) + case other => fail(s"Timezone snapshot must succeed: $other") + } + } + } + } + + for ((direction, writer, reader) <- mismatchZones; parsed <- Seq(false, true)) { + val representation = if (parsed) "checkpoint" else "json" + test(s"stats_timezone_${direction}_$representation") { + val table = createMismatchTable(writer, partitioned = false) + prepareMismatchCapture(table, writer, reader, partitioned = false, parsed = parsed) + checkMismatchSpecs(table, writer, reader, partitioned = false) + } + } + + // Reuse the timezone fixture and check TIMESTAMP partition reconstruction. + for ((direction, writer, reader) <- mismatchZones; parsed <- Seq(false, true)) { + val representation = if (parsed) "checkpoint" else "json" + test(s"partitions_timezone_${direction}_$representation") { + val table = createMismatchTable(writer, partitioned = true) + prepareMismatchCapture(table, writer, reader, partitioned = true, parsed = parsed) + checkMismatchSpecs(table, writer, reader, partitioned = true) + } + } + + // Later portable cases share only ordinary read/schema assertions; row oracles stay literal. + private def readinessRead( + table: TableHandle, name: String, expectedSchema: StructType, rows: Seq[Row], + predicate: String = null, version: java.lang.Long = null): Unit = { + val workload = WorkloadContext.current.workloadName + readSpec(table, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed against the independent oracle: $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$name").resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + if (name == "all") { + expected.coalesce(1).write.mode("overwrite") + .parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString) + } + println(s"Verified independent literal rows/schema: $workload/$name rows=${rows.size}") + } + } + + private def readinessSnapshot( + table: TableHandle, expectedSchema: StructType, partitions: Seq[String] = Seq.empty, + version: java.lang.Long = null): Unit = { + snapshotSpec(table, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == partitions) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + val ntz = expectedSchema.fields.exists(_.dataType == + org.apache.spark.sql.types.TimestampNTZType) + require(result.protocol.minReaderVersion == (if (ntz) 3 else 1)) + require(result.protocol.minWriterVersion == (if (ntz) 7 else 2)) + case other => fail(s"Snapshot must succeed against independent metadata: $other") + } + } + } + + private def readinessPruneHistory(dir: Path, version: Long): Unit = { + val log = dir.resolve("_delta_log") + val files = Files.list(log) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (file.endsWith(".json") && file != f"$version%020d.json") + }.foreach(Files.delete(_)) finally files.close() + // The same-version JSON supports requested-version lookup only. Earlier history is absent, + // so checkpoint reconstruction is required. No replay tail or checkpoint-only claim. + } + + // Unannotated INT32/INT64 parsed cells are distinct from normal DATE/TIMESTAMP + // Parquet encodings. Keep correct JSON stats too: Spark rejects several parsed-only forms. + // Only INT32 as DATE, the DATE32 physical width, removes every JSON stats copy so that the + // parsed day is the only statistic. The other forms are not claimed as parsed-only routes. + for ((kind, sqlType, valueType, literal, value, integerValue) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType, + "TIMESTAMP '1970-01-01T00:00:01Z'", Timestamp.from(Instant.ofEpochSecond(1)), 1000000L), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1970-01-01 00:00:01'", + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1), 1000000L), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1970-01-02'", java.sql.Date.valueOf("1970-01-02"), 1L)); + integerType <- Seq("int", "long")) { + test(s"checkpoint_integer_${integerType}_as_$kind") { + spark.conf.set("spark.sql.session.timeZone", "UTC") + val expectedSchema = new StructType().add("id", IntegerType).add("v", valueType) + val rows = Seq(Row(1, value), Row(2, value), Row(2, value)) + sql(s"""CREATE TABLE tbl (id INT, v $sqlType) USING delta TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'true')""") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $literal), (2, $literal), (2, $literal) AS rows(id, v)") + val table = registerTable("tbl") + val parsedOnly = kind == "date" && integerType == "int" + forceCheckpoint("tbl") + if (parsedOnly) { + modifyCommitActions(table, 1L) { actions => + actions.map { case add: AddFile => add.copy(stats = None); case other => other } + } + } + mutateTable(table) { dir => + val checkpoint = dir.resolve("_delta_log/00000000000000000001.checkpoint.parquet") + val temporary = dir.resolve("_delta_log/integer-stats-rewrite") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("v", lit(integerValue).cast(integerType))) + } + var add = col("add").withField("stats_parsed", stats) + if (parsedOnly) add = add.withField("stats", lit(null).cast("string")) + df.withColumn("add", add).coalesce(1).write.parquet(temporary.toString) + val parts = Files.list(temporary) + val part = try parts.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten integer stats checkpoint: $other") + } finally parts.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + FileUtils.deleteDirectory(temporary.toFile) + val path = new HadoopPath(checkpoint.toUri) + val conf = spark.sparkContext.hadoopConfiguration + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + for (bound <- Seq("minValues", "maxValues")) { + val physical = footer.getFooter.getFileMetaData.getSchema + .getType("add", "stats_parsed", bound, "v").asPrimitiveType() + require(physical.getLogicalTypeAnnotation == null) + require(physical.getPrimitiveTypeName == (if (integerType == "int") + PrimitiveType.PrimitiveTypeName.INT32 else PrimitiveType.PrimitiveTypeName.INT64)) + println(s"Verified off-type checkpoint $kind/$integerType $bound: " + + s"$physical value=$integerValue jsonFallback=${!parsedOnly}") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + var count = 0 + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + count += 1 + val add = record.getGroup("add", 0) + if (parsedOnly) require(add.getFieldRepetitionCount("stats") == 0) else { + require(add.getFieldRepetitionCount("stats") == 1) + val json = JsonUtil.mapper.readTree(add.getString("stats", 0)) + for (bound <- Seq("minValues", "maxValues")) { + val raw = json.at(s"/$bound/v").asText() + kind match { + case "timestamp" => require(Instant.parse(raw) == Instant.ofEpochSecond(1)) + case "timestamp_ntz" => require(java.time.LocalDateTime.parse(raw) == + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1)) + case "date" => require(java.time.LocalDate.parse(raw) == + java.time.LocalDate.of(1970, 1, 2)) + } + } + } + val fields = add.getGroup("stats_parsed", 0) + require(fields.getLong("numRecords", 0) == 3L) + for (bound <- Seq("minValues", "maxValues")) { + val values = fields.getGroup(bound, 0) + val raw = if (integerType == "int") values.getInteger("v", 0).toLong + else values.getLong("v", 0) + require(raw == integerValue) + } + } + record = reader.read() + } + } finally reader.close() + require(count == 1) + readinessPruneHistory(dir, 1L) + if (parsedOnly) { + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 1 && adds.head.stats.isEmpty) + val log = Files.list(dir.resolve("_delta_log")) + val names = try log.iterator().asScala.map(_.getFileName.toString).toSet + finally log.close() + require(!names.exists(_.endsWith(".crc"))) + require(names.filter(_.endsWith(".json")) == Set("00000000000000000001.json")) + println("Verified parsed-only DATE stats: INT32 day=1 (1970-01-02); JSON stats " + + "absent in checkpoint and v1 commit; CRC removed; v0 pruned") + } + } + readinessRead(table, "all", expectedSchema, rows) + readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) + readinessRead(table, "hit", expectedSchema, rows, predicate = s"v = $literal") + readinessRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"v < $literal OR v > $literal") + readinessSnapshot(table, expectedSchema) + readinessSnapshot(table, expectedSchema, version = 1L) + } + } + + private def readinessRemoveChecksums(dir: Path): Unit = { + val files = Files.list(dir.resolve("_delta_log")) + try files.iterator().asScala.filter(_.toString.endsWith(".crc")) + .foreach(Files.delete(_)) finally files.close() + } + + // JSON numeric stats are separate from physical checkpoint types in the checkpoint fixture. + // Literal rows remain authoritative even when a reference parser discards off-type stats. + for ((kind, sqlType, valueType, literal, value) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType, "TIMESTAMP '1970-01-01T00:00:01Z'", + Timestamp.from(Instant.ofEpochSecond(1))), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1970-01-01 00:00:01'", + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1)), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1970-01-02'", java.sql.Date.valueOf("1970-01-02")), + ("string", "STRING", org.apache.spark.sql.types.StringType, "'1'", "1"))) { + test(s"action_integer_as_$kind") { + spark.conf.set("spark.sql.session.timeZone", "UTC") + val expectedSchema = new StructType().add("id", IntegerType).add("v", valueType) + val rows = Seq(Row(1, value), Row(2, value), Row(2, value)) + sql(s"CREATE TABLE tbl (id INT, v $sqlType) USING delta") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $literal), (2, $literal), (2, $literal) AS rows(id, v)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val stats = JsonUtil.mapper.readTree(add.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + stats.get(bound).asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("v", 1) + } + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + mutateTable(table) { dir => + readinessRemoveChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 1) + val stats = JsonUtil.mapper.readTree(adds.head.stats.get) + require(stats.get("numRecords").asLong() == 3L) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/v").isIntegralNumber) + require(stats.at(s"/$bound/v").intValue() == 1) + } + println(s"Verified numeric JSON stats: $kind min=1 max=1 records=3") + } + // Spark accepts numeric STRING stats. The required-rejection probe is preserved + // separately; this useful correctness fixture does not claim that rejection. + readinessRead(table, "all", expectedSchema, rows) + readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) + readinessRead(table, "hit", expectedSchema, rows, predicate = s"v = $literal") + readinessRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"v < $literal OR v > $literal") + readinessSnapshot(table, expectedSchema) + readinessSnapshot(table, expectedSchema, version = 1L) + } + } + + // Literal empty AddFile partition strings are independent of writer normalization. + // BYTE is numeric TINYINT. Its empty input has no numeric value and is expected to be null. + // STRING and BINARY retain a distinct empty value; writer support was probed separately. + for ((kind, sqlType, valueType, firstValue, secondValue, control, emptyValue) <- Seq( + ("byte", "TINYINT", org.apache.spark.sql.types.ByteType, + "CAST(1 AS TINYINT)", "CAST(2 AS TINYINT)", 2.toByte, null), + ("short", "SMALLINT", org.apache.spark.sql.types.ShortType, + "CAST(1 AS SMALLINT)", "CAST(2 AS SMALLINT)", 2.toShort, null), + ("int", "INT", IntegerType, "1", "2", 2, null), + ("long", "BIGINT", org.apache.spark.sql.types.LongType, "1L", "2L", 2L, null), + ("float", "FLOAT", org.apache.spark.sql.types.FloatType, + "CAST(1.25 AS FLOAT)", "CAST(2.5 AS FLOAT)", 2.5f, null), + ("double", "DOUBLE", org.apache.spark.sql.types.DoubleType, + "CAST(1.25 AS DOUBLE)", "CAST(2.5 AS DOUBLE)", 2.5d, null), + ("decimal", "DECIMAL(10,2)", org.apache.spark.sql.types.DecimalType(10, 2), + "CAST(1.25 AS DECIMAL(10,2))", "CAST(2.50 AS DECIMAL(10,2))", + new java.math.BigDecimal("2.50"), null), + ("boolean", "BOOLEAN", org.apache.spark.sql.types.BooleanType, + "false", "true", true, null), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1969-12-31'", "DATE '2000-02-29'", java.sql.Date.valueOf("2000-02-29"), null), + ("timestamp", "TIMESTAMP", TimestampType, + "TIMESTAMP '1969-12-31T23:59:59Z'", "TIMESTAMP '2024-03-10T10:00:00.123456Z'", + Timestamp.from(Instant.parse("2024-03-10T10:00:00.123456Z")), null), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1969-12-31 23:59:59'", "TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'", + java.time.LocalDateTime.parse("2024-03-10T02:30:00.123456"), null), + ("string", "STRING", org.apache.spark.sql.types.StringType, + "'first'", "'second'", "second", ""), + ("binary", "BINARY", org.apache.spark.sql.types.BinaryType, + "unhex('61')", "unhex('62')", Array[Byte](98), Array.emptyByteArray))) { + test(s"empty_partition_$kind") { + spark.conf.set("spark.sql.session.timeZone", "UTC") + // ANSI predicate casts reject empty numeric input. Leave legacy casts active for capture. + // The existing spec format cannot transport this session setting to an external consumer. + spark.conf.set("spark.sql.ansi.enabled", "false") + val expectedSchema = new StructType().add("id", IntegerType).add("p", valueType) + val rows = Seq(Row(1, emptyValue), Row(2, control), Row(2, control), Row(3, null)) + sql(s"CREATE TABLE tbl (id INT, p $sqlType) USING delta PARTITIONED BY (p)") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $firstValue), (2, $secondValue), (2, $secondValue), " + + s"(3, CAST(NULL AS $sqlType)) AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile if JsonUtil.mapper.readTree(add.stats.get) + .at("/minValues/id").intValue() == 1 => + add.copy(partitionValues = Map("p" -> "")) + case other => other + } + } + mutateTable(table) { dir => + readinessRemoveChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 3) + require(adds.count(_.partitionValues("p") == "") == 1) + require(adds.count(_.partitionValues("p") == null) == 1) + println(s"Verified serialized empty partition: $kind empty=1 null=1 control=1") + } + readinessRead(table, "all", expectedSchema, rows) + readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) + readinessRead(table, "is_null", expectedSchema, rows.filter(_.isNullAt(1)), + predicate = "p IS NULL") + readinessRead(table, "is_not_null", expectedSchema, rows.filterNot(_.isNullAt(1)), + predicate = "p IS NOT NULL") + readinessRead(table, "eq_control", expectedSchema, rows.filter(_.getInt(0) == 2), + predicate = s"p = $secondValue") + if (kind == "string" || kind == "binary") { + val emptyLiteral = if (kind == "string") "''" else "unhex('')" + readinessRead(table, "eq_empty", expectedSchema, rows.filter(_.getInt(0) == 1), + predicate = s"p = $emptyLiteral") + } + snapshotSpec(table).assert { spec => + require(spark.conf.get("spark.sql.ansi.enabled") == "false") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == Seq("p")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.protocol.minReaderVersion == (if (kind == "timestamp_ntz") 3 else 1)) + require(result.protocol.minWriterVersion == (if (kind == "timestamp_ntz") 7 else 2)) + case other => fail(s"Empty partition snapshot must succeed: $other") + } + println(s"Verified capture mode: ${WorkloadContext.current.workloadName} ansi=false") + } + readinessSnapshot(table, expectedSchema, Seq("p"), version = 1L) + } + } + + // Inputs are literal AddFile strings, not SQL timestamp literal spellings. + // The pinned Spark parser permits zone suffixes for NTZ casts and ignores their offsets; + // expected NTZ wall clocks below are separate from the corresponding absolute instants. + private def readinessPartitionFormats: Vector[(String, String, String, String)] = { + val fractions = Seq( + ("0", "", "000000"), ("1", ".1", "100000"), ("2", ".12", "120000"), + ("3", ".123", "123000"), ("4", ".1234", "123400"), + ("5", ".12345", "123450"), ("6", ".123456", "123456"), + ("7", ".1234567", "123456"), ("8", ".12345678", "123456"), + ("9", ".123456789", "123456"), ("dot", ".", "000000"), + ("10", ".1234567891", "123456"), ("18", ".123456789123456789", "123456")) + val zones = Seq( + ("z", "12:34:56.123456Z", "12:34:56.123456"), + ("zero_colon", "12:34:56.123456+00:00", "12:34:56.123456"), + ("zero_basic", "12:34:56.123456+0000", "12:34:56.123456"), + ("zero_hour", "12:34:56.123456+00", "12:34:56.123456"), + ("minus_zero", "12:34:56.123456-00:00", "12:34:56.123456"), + ("utc_name", "12:34:56.123456 UTC", "12:34:56.123456"), + ("gmt_name", "12:34:56.123456 GMT", "12:34:56.123456"), + ("east", "18:04:56.123456+05:30", "18:04:56.123456"), + ("west", "04:34:56.123456-08:00", "04:34:56.123456"), + ("region", "04:34:56.123456 America/Los_Angeles", "04:34:56.123456"), + ("east_basic", "18:04:56.123456+0530", "18:04:56.123456"), + ("east_hour", "17:34:56.123456+05", "17:34:56.123456"), + ("west_basic", "04:34:56.123456-0800", "04:34:56.123456"), + ("west_hour", "04:34:56.123456-08", "04:34:56.123456"), + ("west_half", "09:04:56.123456-03:30", "09:04:56.123456"), + ("single_hour", "18:04:56.123456+5:30", "18:04:56.123456"), + ("single_minute", "17:37:56.123456+05:3", "17:37:56.123456"), + ("offset_seconds", "18:05:26.123456+05:30:30", "18:05:26.123456")) + (for ((separatorName, separator) <- Seq("space" -> " ", "T" -> "T"); + entry <- fractions.map { case (label, fraction, micros) => + (s"${separatorName}_fraction_$label", s"2024-02-29${separator}12:34:56$fraction", + s"2024-02-29T12:34:56.$micros" + "Z", s"2024-02-29T12:34:56.$micros") + } ++ zones.map { case (label, raw, wall) => + (s"${separatorName}_$label", s"2024-02-29$separator$raw", + "2024-02-29T12:34:56.123456Z", s"2024-02-29T$wall") + }) yield entry).toVector + } + + for ((kind, sqlType, valueType) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType))) { + test(s"partition_serializations_$kind") { + spark.conf.set("spark.sql.session.timeZone", "UTC") + spark.conf.set("spark.sql.ansi.enabled", "false") + val formats = readinessPartitionFormats + require(formats.size == 62) + val expectedSchema = new StructType().add("id", IntegerType).add("p", valueType) + val keys = formats.zipWithIndex.map { case ((_, _, instant, wall), index) => + (index + 1) -> (if (kind == "timestamp") instant else wall) + }.toMap + val rows0 = keys.toSeq.sortBy(_._1).map { case (id, key) => + val value = if (kind == "timestamp") Timestamp.from(Instant.parse(key)) + else java.time.LocalDateTime.parse(key) + Row(id, value) + } + val rows = rows0 :+ rows0.head + sql(s"CREATE TABLE tbl (id INT, p $sqlType) USING delta PARTITIONED BY (p)") + // Distinct seed partitions create one data file per raw representation. Only id is + // physically stored; the typed AddFile mutation supplies the independently checked p. + val seeds = formats.indices.map { index => + val seed = Instant.parse("2030-01-01T00:00:00Z").plusSeconds(index.toLong).toString + val literal = if (kind == "timestamp") seed else seed.stripSuffix("Z") + s"(${index + 1}, $sqlType '$literal')" + } + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + (seeds :+ seeds.head).mkString(", ") + " AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val id = JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + add.copy(partitionValues = Map("p" -> formats(id - 1)._2)) + case other => other + } + } + mutateTable(table) { dir => + readinessRemoveChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == formats.size) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + require(stats.at("/maxValues/id").intValue() == id) + require(stats.get("numRecords").asLong() == (if (id == 1) 2L else 1L)) + require(add.partitionValues == Map("p" -> formats(id - 1)._2)) + println(s"Verified raw serialization: $kind ${formats(id - 1)._1} " + + s"value=${formats(id - 1)._2}") + } + } + readinessRead(table, "all", expectedSchema, rows) + readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) + keys.values.toSet.toSeq.sorted.zipWithIndex.foreach { case (key, index) => + val ids = keys.collect { case (id, value) if value == key => id }.toSet + readinessRead(table, s"eq_$index", expectedSchema, + rows.filter(r => ids.contains(r.getInt(0))), predicate = s"p = $sqlType '$key'") + } + def selected(lower: String, upper: String): Seq[Row] = rows.filter { row => + val key = keys(row.getInt(0)) + key >= lower && key < upper + } + readinessRead(table, "fraction_range", expectedSchema, + selected("2024-02-29T12:34:56.100000", "2024-02-29T12:34:56.123456"), + predicate = s"p >= $sqlType '2024-02-29 12:34:56.100000' AND " + + s"p < $sqlType '2024-02-29 12:34:56.123456'") + readinessRead(table, "before_noon", expectedSchema, + selected("2024-02-29T00:00:00", "2024-02-29T12:00:00"), + predicate = s"p < $sqlType '2024-02-29 12:00:00'") + readinessRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"p < $sqlType '2024-02-29 00:00:00'") + readinessSnapshot(table, expectedSchema, Seq("p")) + readinessSnapshot(table, expectedSchema, Seq("p"), version = 1L) + } + } + + // Canonical zero-padded DATE strings use proleptic Gregorian epoch-day oracles. + test("date_partition_canonical_calendar_bounds") { + spark.conf.set("spark.sql.session.timeZone", "UTC") + spark.conf.set("spark.sql.ansi.enabled", "false") + // Expected Parquet must retain proleptic Gregorian days even before the 1582 cutover. + spark.conf.set("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED") + spark.conf.set("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED") + val dates = Seq( + (1, "0001-01-01", -719162), (2, "1582-10-04", -141438), + (3, "1582-10-15", -141427), (4, "1969-12-31", -1), + (5, "1970-01-01", 0), (6, "2000-02-29", 11016), (7, "9999-12-31", 2932896)) + val expectedSchema = new StructType().add("id", IntegerType) + .add("p", org.apache.spark.sql.types.DateType) + val rows = dates.map { case (id, raw, _) => Row(id, java.sql.Date.valueOf(raw)) } ++ + Seq(Row(6, java.sql.Date.valueOf("2000-02-29")), Row(8, null)) + val days: Map[Int, Option[Int]] = dates.map { case (id, _, day) => + id -> Some(day) + }.toMap + (8 -> None) + sql("CREATE TABLE tbl (id INT, p DATE) USING delta PARTITIONED BY (p)") + val inputs = dates.map { case (id, raw, _) => s"($id, DATE '$raw')" } ++ + Seq("(6, DATE '2000-02-29')", "(8, CAST(NULL AS DATE))") + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + inputs.mkString(", ") + " AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val id = JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + val raw = if (id == 8) null else dates.find(_._1 == id).get._2 + add.copy(partitionValues = Map("p" -> raw)) + case other => other + } + } + mutateTable(table) { dir => + readinessRemoveChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 8) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + val raw = if (id == 8) null else dates.find(_._1 == id).get._2 + require(add.partitionValues == Map("p" -> raw)) + require(stats.at("/maxValues/id").intValue() == id) + require(stats.get("numRecords").asLong() == (if (id == 6) 2L else 1L)) + if (raw != null) require(raw.matches("[0-9]{4}-[0-9]{2}-[0-9]{2}")) + println(s"Verified canonical DATE partition: id=$id raw=$raw epochDay=${days(id)}") + } + } + val allName = "all" + readinessRead(table, allName, expectedSchema, rows) + readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) + dates.foreach { case (id, raw, _) => + readinessRead(table, s"eq_$id", expectedSchema, rows.filter(_.getInt(0) == id), + predicate = s"p = DATE '$raw'") + } + readinessRead(table, "is_null", expectedSchema, rows.filter(_.isNullAt(1)), + predicate = "p IS NULL") + readinessRead(table, "pre_epoch", expectedSchema, rows.filter(r => r.getInt(0) <= 4), + predicate = "p < DATE '1970-01-01'") + readinessRead(table, "calendar_transition", expectedSchema, + rows.filter(r => Set(2, 3).contains(r.getInt(0))), + predicate = "p >= DATE '1582-10-04' AND p <= DATE '1582-10-15'") + readinessRead(table, "modern_range", expectedSchema, + rows.filter(r => Set(5, 6).contains(r.getInt(0))), + predicate = "p >= DATE '1970-01-01' AND p <= DATE '2000-02-29'") + val workload = WorkloadContext.current.workloadName + snapshotSpec(table).assert { spec => + require(spark.conf.get("spark.sql.parquet.datetimeRebaseModeInWrite") == "CORRECTED") + require(spark.conf.get("spark.sql.parquet.datetimeRebaseModeInRead") == "CORRECTED") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == Seq("p")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.protocol.minReaderVersion == 1 && result.protocol.minWriterVersion == 2) + case other => fail(s"Canonical DATE snapshot must succeed: $other") + } + + } + readinessSnapshot(table, expectedSchema, Seq("p"), version = 1L) + // Snapshots capture before reads at this pin. Inspect Parquet in the last read callback. + val outsideName = "outside_bounds" + readSpec(table, predicate = "p < DATE '0001-01-01' OR p > DATE '9999-12-31'", + name = Some(outsideName)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == 0L) + case other => fail(s"Outside DATE bounds must succeed with no rows: $other") + } + val expected = spark.createDataFrame(Seq.empty[Row].asJava, expectedSchema) + val outside = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$outsideName").resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(outside.toString), outsideName) + println(s"Verified independent literal rows/schema: $workload/$outsideName rows=0") + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$allName").resolve("expected_data") + val literalOracle = outputDir.resolveSibling("oracle-parquet").resolve(workload) + val expectedUnits = days.toVector :+ (6 -> Some(11016)) + for ((label, directory) <- Seq("captured" -> captured, "literal" -> literalOracle)) { + val files = Files.list(directory) + val parquet = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toVector finally files.close() + require(parquet.nonEmpty) + var units = Vector.empty[(Int, Option[Int])] + parquet.foreach { file => + val path = new HadoopPath(file.toUri) + val conf = spark.sparkContext.hadoopConfiguration + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema.getFields.asScala + .find(_.getName == "p").get.asPrimitiveType() + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(physical.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + try { + var record = reader.read() + while (record != null) { + val day = if (record.getFieldRepetitionCount("p") == 0) None + else Some(record.getInteger("p", 0)) + units :+= record.getInteger("id", 0) -> day + record = reader.read() + } + } finally reader.close() + } + require(units.sorted == expectedUnits.sorted) + println(s"Verified physical DATE oracle: $workload $label units=${units.sorted}") + } + } + } + + // INT32 boundary magnitudes are BIGINT inputs to timestamp_seconds/millis/micros. The unit + // names the input magnitude only; rewritten checkpoint cells are always TIMESTAMP_MICROS. + private val intMaxInputs = Seq(-2147483648L, -2147483647L, -2147483646L, + 2147483646L, 2147483647L, 2147483648L) + // Literal UTC instants per input unit, declared apart from both SQL and java.time. + private val intMaxInstants = Seq( + ("seconds", 1000000L, Seq("1901-12-13T20:45:52Z", "1901-12-13T20:45:53Z", + "1901-12-13T20:45:54Z", "2038-01-19T03:14:06Z", "2038-01-19T03:14:07Z", + "2038-01-19T03:14:08Z")), + ("millis", 1000L, Seq("1969-12-07T03:28:36.352Z", "1969-12-07T03:28:36.353Z", + "1969-12-07T03:28:36.354Z", "1970-01-25T20:31:23.646Z", "1970-01-25T20:31:23.647Z", + "1970-01-25T20:31:23.648Z")), + ("micros", 1L, Seq("1969-12-31T23:24:12.516352Z", "1969-12-31T23:24:12.516353Z", + "1969-12-31T23:24:12.516354Z", "1970-01-01T00:35:47.483646Z", + "1970-01-01T00:35:47.483647Z", "1970-01-01T00:35:47.483648Z"))) + // Data value index -> partition value index; no index maps to itself. Ids 2 and 5 hold the + // -/+2147483647 boundaries twice, and id 7 is NULL in every temporal column twice. + private val intMaxPartition = Map(1 -> 4, 2 -> 6, 3 -> 1, 4 -> 5, 5 -> 2, 6 -> 3) + private val intMaxIds = Seq(1, 2, 2, 3, 4, 5, 5, 6, 7, 7) + private val intMaxSchema = new StructType().add("id", IntegerType).add("ts", TimestampType) + .add("ntz", TimestampNTZType).add("p_ts", TimestampType).add("p_ntz", TimestampNTZType) + private val sixDigits = + java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS") + + for ((unit, scale, literals) <- intMaxInstants; parsed <- Seq(false, true)) { + test(s"int_max_${unit}_${if (parsed) "parsed" else "json"}") { + spark.conf.set("spark.sql.session.timeZone", "UTC") + val instants = literals.map(Instant.parse) + for ((input, instant) <- intMaxInputs.zip(instants)) { + val converted = unit match { + case "seconds" => Instant.ofEpochSecond(input) + case "millis" => Instant.ofEpochMilli(input) + case "micros" => Instant.EPOCH.plus(input, java.time.temporal.ChronoUnit.MICROS) + } + require(converted == instant && mismatchMicros(instant) == input * scale) + } + def wall(index: Int) = + java.time.LocalDateTime.ofInstant(instants(index - 1), java.time.ZoneOffset.UTC) + val rows = intMaxIds.map { + case 7 => Row(7, null, null, null, null) + case id => + val p = intMaxPartition(id) + Row(id, Timestamp.from(instants(id - 1)), wall(id), Timestamp.from(instants(p - 1)), + wall(p)) + } + def partitionStrings(id: Int): Map[String, String] = if (id == 7) { + Map("p_ts" -> null, "p_ntz" -> null) + } else { + val p = wall(intMaxPartition(id)).format(sixDigits) + Map("p_ts" -> (p + "Z"), + "p_ntz" -> p.replace('T', ' ').replaceAll("0+$", "").stripSuffix(".")) + } + def fileId(add: AddFile): Int = + JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ, p_ts TIMESTAMP, + p_ntz TIMESTAMP_NTZ) USING delta PARTITIONED BY (p_ts, p_ntz) TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + val inputs = intMaxIds.map { + case 7 => "(7, CAST(NULL AS BIGINT), CAST(NULL AS BIGINT))" + case id => s"($id, ${intMaxInputs(id - 1)}L, ${intMaxInputs(intMaxPartition(id) - 1)}L)" + } + val f = s"timestamp_$unit" + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ id, $f(v), CAST($f(v) AS TIMESTAMP_NTZ), " + + s"$f(p), CAST($f(p) AS TIMESTAMP_NTZ) FROM VALUES ${inputs.mkString(", ")} " + + "AS rows(id, v, p)") + val table = registerTable("tbl") + val produced = CommitLog.read(table.sourcePath, 1L).collect { case a: AddFile => a } + require(produced.map(a => fileId(a) -> a.partitionValues).toMap == + (1 to 7).map(id => id -> partitionStrings(id)).toMap && produced.size == 7) + + if (parsed) forceCheckpoint("tbl") else { + // The producer writes millisecond JSON bounds. Keep all six fraction digits instead. + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile if fileId(add) != 7 => + val id = fileId(add) + println(s"Producer JSON stats before rewrite: $unit id=$id ${add.stats.get}") + val stats = JsonUtil.mapper.readTree(add.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + stats.get(bound).asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", wall(id).format(sixDigits) + "Z") + .put("ntz", wall(id).format(sixDigits)) + } + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + } + mutateTable(table) { dir => + if (parsed) { + val checkpoint = dir.resolve("_delta_log/00000000000000000001.checkpoint.parquet") + rewriteIntMaxCheckpoint(checkpoint, instants) + checkIntMaxCheckpoint(checkpoint, instants, partitionStrings) + readinessPruneHistory(dir, 1L) + } else { + readinessRemoveChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case a: AddFile => a } + require(adds.size == 7) + for (add <- adds) { + val id = fileId(add) + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(add.partitionValues == partitionStrings(id)) + require(stats.get("numRecords").asLong() == (if (Set(2, 5, 7)(id)) 2L else 1L)) + for (bound <- Seq("minValues", "maxValues")) { + if (id == 7) require(!stats.get(bound).has("ts") && !stats.get(bound).has("ntz")) + else { + require(stats.at(s"/$bound/ts").asText() == wall(id).format(sixDigits) + "Z") + require(stats.at(s"/$bound/ntz").asText() == wall(id).format(sixDigits)) + } + } + if (id == 7) require(stats.at("/nullCount/ts").asLong() == 2L) + } + println(s"Verified six-digit JSON stats and partitions: $unit files=7") + } + } + + readinessRead(table, "all", intMaxSchema, rows) + readinessRead(table, "all_v1", intMaxSchema, rows, version = 1L) + for ((field, kind) <- Seq("ts" -> "TIMESTAMP", "ntz" -> "TIMESTAMP_NTZ", + "p_ts" -> "TIMESTAMP", "p_ntz" -> "TIMESTAMP_NTZ")) { + def v(index: Int): String = if (kind == "TIMESTAMP") { + s"TIMESTAMP '${instants(index - 1)}'" + } else s"TIMESTAMP_NTZ '${wall(index).format(sixDigits).replace('T', ' ')}'" + def index(row: Row): Option[Int] = row.getInt(0) match { + case 7 => None + case id => Some(if (field.startsWith("p_")) intMaxPartition(id) else id) + } + for ((label, predicate, selected) <- Seq( + ("below_negative", s"$field < ${v(2)}", Set(1)), + ("eq_negative", s"$field = ${v(2)}", Set(2)), + ("interior", s"$field > ${v(2)} AND $field < ${v(5)}", Set(3, 4)), + ("eq_positive", s"$field = ${v(5)}", Set(5)), + ("above_positive", s"$field > ${v(5)}", Set(6)), + ("outside", s"$field < ${v(1)} OR $field > ${v(6)}", Set.empty[Int]))) { + readinessRead(table, s"${field}_$label", intMaxSchema, + rows.filter(r => index(r).exists(selected)), predicate) + } + readinessRead(table, s"${field}_is_null", intMaxSchema, + rows.filter(_.getInt(0) == 7), s"$field IS NULL") + } + readinessSnapshot(table, intMaxSchema, Seq("p_ts", "p_ntz")) + readinessSnapshot(table, intMaxSchema, Seq("p_ts", "p_ntz"), version = 1L) + } + } + + // DATE32 epoch-day bounds. 2147483648 is outside DATE32 and is not an input. Rows use + // java.time.LocalDate: java.sql.Date misrenders these years. Raw INT32 days are the oracle. + test("date_epoch_days_int32_bounds") { + spark.conf.set("spark.sql.session.timeZone", "UTC") + spark.conf.set("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED") + spark.conf.set("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED") + // Spark's data-filter translation rebases java.sql.Date literals to Julian days, which + // overflows near -2147483648. LocalDate literals avoid that reference-side conversion. + // Capture runs after this body, so afterEach restores the setting, including on failure. + java8Previous = Some(spark.conf.get(java8Key)) + spark.conf.set(java8Key, "true") + val days = Seq(-2147483648, -2147483647, -2147483646, 2147483646, 2147483647) + val dates = Seq("-5877641-06-23", "-5877641-06-24", "-5877641-06-25", "+5881580-07-10", + "+5881580-07-11").map(LocalDate.parse) + require(dates.map(_.toEpochDay) == days.map(_.toLong)) + val permutation = Map(1 -> 3, 2 -> 5, 3 -> 1, 4 -> 2, 5 -> 4) + val ids = Seq(1, 2, 2, 3, 4, 5, 5, 6, 6) + val expectedSchema = new StructType().add("id", IntegerType) + .add("d", org.apache.spark.sql.types.DateType).add("p", org.apache.spark.sql.types.DateType) + val rows = ids.map { + case 6 => Row(6, null, null) + case id => Row(id, dates(id - 1), dates(permutation(id) - 1)) + } + val units = ids.map { + case 6 => (6, None, None) + case id => (id, Some(days(id - 1)), Some(days(permutation(id) - 1))) + } + sql("CREATE TABLE tbl (id INT, d DATE, p DATE) USING delta PARTITIONED BY (p)") + val inputs = ids.map { + case 6 => "(6, CAST(NULL AS INT), CAST(NULL AS INT))" + case id => s"($id, ${days(id - 1)}, ${days(permutation(id) - 1)})" + } + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ id, date_from_unix_date(d), " + + s"date_from_unix_date(p) FROM VALUES ${inputs.mkString(", ")} AS rows(id, d, p)") + val table = registerTable("tbl") + mutateTable(table) { dir => + readinessRemoveChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 6) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + require(stats.get("numRecords").asLong() == (if (Set(2, 5, 6)(id)) 2L else 1L)) + if (id == 6) { + require(add.partitionValues == Map("p" -> null)) + require(stats.at("/nullCount/d").asLong() == 2L && !stats.at("/minValues").has("d")) + } else { + require(add.partitionValues == Map("p" -> dates(permutation(id) - 1).toString)) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/d").asText() == dates(id - 1).toString) + } + } + println(s"Verified DATE32 bound file: id=$id stats=${add.stats.get} " + + s"partition=${add.partitionValues}") + } + } + def literal(index: Int): String = s"DATE '${dates(index - 1)}'" + readinessRead(table, "all", expectedSchema, rows) + readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) + for (field <- Seq("d", "p")) { + def index(row: Row): Option[Int] = row.getInt(0) match { + case 6 => None + case id => Some(if (field == "p") permutation(id) else id) + } + for ((label, predicate, selected) <- Seq( + ("below_negative", s"$field < ${literal(2)}", Set(1)), + ("eq_negative", s"$field = ${literal(2)}", Set(2)), + ("interior", s"$field > ${literal(2)} AND $field < ${literal(5)}", Set(3, 4)), + ("eq_positive", s"$field = ${literal(5)}", Set(5)))) { + readinessRead(table, s"${field}_$label", expectedSchema, + rows.filter(r => index(r).exists(selected)), predicate) + } + readinessRead(table, s"${field}_is_null", expectedSchema, + rows.filter(_.getInt(0) == 6), s"$field IS NULL") + } + readinessSnapshot(table, expectedSchema, Seq("p")) + readinessSnapshot(table, expectedSchema, Seq("p"), version = 1L) + // Snapshots capture before reads at this pin. Inspect Parquet in the last read callback. + val workload = WorkloadContext.current.workloadName + readSpec(table, predicate = s"d < ${literal(1)} OR d > ${literal(5)}", + name = Some("outside")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == 0L) + case other => fail(s"Outside DATE32 bounds must succeed with no rows: $other") + } + val conf = spark.sparkContext.hadoopConfiguration + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_all").resolve("expected_data") + val literalOracle = outputDir.resolveSibling("oracle-parquet").resolve(workload) + for ((label, directory) <- Seq("captured" -> captured, "literal" -> literalOracle)) { + val files = Files.list(directory) + val parquet = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toVector finally files.close() + require(parquet.nonEmpty) + val found = parquet.flatMap { file => + val path = new HadoopPath(file.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try for (field <- Seq("d", "p")) { + val physical = footer.getFooter.getFileMetaData.getSchema.getFields.asScala + .find(_.getName == field).get.asPrimitiveType() + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(physical.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + var out = Vector.empty[(Int, Option[Int], Option[Int])] + try { + var record = reader.read() + while (record != null) { + def day(field: String) = if (record.getFieldRepetitionCount(field) == 0) None + else Some(record.getInteger(field, 0)) + out :+= ((record.getInteger("id", 0), day("d"), day("p"))) + record = reader.read() + } + } finally reader.close() + out + } + require(found.sorted == units.sorted) + println(s"Verified raw DATE32 epoch days: $workload $label ${found.sorted}") + } + require(spark.conf.get(java8Key) == "true") + } + } + + private val java8Key = "spark.sql.datetime.java8API.enabled" + private var java8Previous: Option[String] = None + + override def afterEach(): Unit = { + try super.afterEach() finally { + java8Previous.foreach(previous => if (_spark != null) _spark.conf.set(java8Key, previous)) + java8Previous = None + } + } + + // Replace producer-derived parsed bounds by the literal instants and drop the JSON fallback. + private def rewriteIntMaxCheckpoint(checkpoint: Path, instants: Seq[Instant]): Unit = { + val rewrite = checkpoint.resolveSibling("int-max-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + var ts = col(s"add.stats_parsed.$bound.ts") + var ntz = col(s"add.stats_parsed.$bound.ntz") + for ((instant, index) <- instants.zipWithIndex) { + val matches = col("add.stats_parsed.minValues.id") === lit(index + 1) + ts = when(matches, lit(Timestamp.from(instant))).otherwise(ts) + ntz = when(matches, + lit(java.time.LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC))) + .otherwise(ntz) + } + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("ts", ts).withField("ntz", ntz)) + } + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewrite.toString) + val files = Files.list(rewrite) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten INT_MAX checkpoint: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewrite)) FileUtils.deleteDirectory(rewrite.toFile) + } + } + + private def checkIntMaxCheckpoint( + checkpoint: Path, instants: Seq[Instant], partitions: Int => Map[String, String]): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + for ((prefix, field) <- Seq("stats_parsed.minValues" -> "ts", + "stats_parsed.minValues" -> "ntz", "stats_parsed.maxValues" -> "ts", + "stats_parsed.maxValues" -> "ntz", "partitionValues_parsed" -> "p_ts", + "partitionValues_parsed" -> "p_ntz")) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == !field.endsWith("ntz")) + println(s"Verified INT_MAX checkpoint $prefix.$field: $cell") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var ids = Vector.empty[Int] + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + val add = record.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + require(stats.getLong("numRecords", 0) == (if (Set(2, 5, 7)(id)) 2L else 1L)) + val parsedPartition = add.getGroup("partitionValues_parsed", 0) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + for (field <- Seq("ts", "ntz")) { + if (id == 7) require(values.getFieldRepetitionCount(field) == 0) + else require(values.getLong(field, 0) == mismatchMicros(instants(id - 1))) + } + } + for (field <- Seq("p_ts", "p_ntz")) { + if (id == 7) require(parsedPartition.getFieldRepetitionCount(field) == 0) + else require(parsedPartition.getLong(field, 0) == + mismatchMicros(instants(intMaxPartition(id) - 1))) + } + if (id == 7) require(stats.getGroup("nullCount", 0).getLong("ts", 0) == 2L) + val raw = add.getGroup("partitionValues", 0) + val partition = (0 until raw.getFieldRepetitionCount("key_value")).map { i => + val entry = raw.getGroup("key_value", i) + val value = if (entry.getFieldRepetitionCount("value") == 0) null + else entry.getString("value", 0) + entry.getString("key", 0) -> value + }.toMap + require(partition == partitions(id)) + ids :+= id + println(s"Verified INT_MAX raw checkpoint id=$id partition=$partition") + } + record = reader.read() + } + } finally reader.close() + require(ids.sorted == (1 to 7).toVector) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTypeWideningSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTypeWideningSuite.scala new file mode 100644 index 00000000..032b448b --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTypeWideningSuite.scala @@ -0,0 +1,278 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.nio.file.Files + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.types.{DataType, StructType} + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.AddFile +import io.delta.workload.model._ + +/** Widened JSON statistics, old parsed statistics, and partition values are separate workloads. */ +class ReadinessTypeWideningSuite extends WorkloadTestSuite("readiness_type_widening") { + + private case class Stage(dataType: String, values: Seq[String]) + private case class Widening(name: String, stages: Seq[Stage], jsonGap: Boolean = true) + + // The numeric chain, decimal physical encodings, and DATE values follow TypeWideningSuite. + // One transition corpus drives all three representations. The chain's SHORT stage also covers + // BYTE -> SHORT; existing full/time-travel JSON reads are not redeclared here. + private val widenings = Seq( + Widening("byte_int", Seq( + Stage("BYTE", Seq("1", "127")), Stage("INT", Seq("128", "2147483647")))), + Widening("byte_long", Seq( + Stage("BYTE", Seq("1", "127")), + Stage("BIGINT", Seq("128", "9223372036854775807")))), + Widening("short_int", Seq( + Stage("SHORT", Seq("10", "100")), Stage("INT", Seq("40000", "50000"))), + jsonGap = false), + Widening("short_long", Seq( + Stage("SHORT", Seq("100", "32767")), + Stage("BIGINT", Seq("40000", "9223372036854775807")))), + Widening("int_long", Seq( + Stage("INT", Seq("100", "200", "300", "400")), + Stage("BIGINT", Seq("3000000000", "9000000000000"))), jsonGap = false), + Widening("byte_double", Seq( + Stage("BYTE", Seq("1", "127")), + Stage("DOUBLE", Seq("127.5", "2147483648.25")))), + Widening("short_double", Seq( + Stage("SHORT", Seq("100", "32767")), + Stage("DOUBLE", Seq("32767.5", "2147483648.25")))), + Widening("int_double", Seq( + Stage("INT", Seq("100", "2147483647")), + Stage("DOUBLE", Seq("2147483647.5", "4294967296.25")))), + Widening("float_double", Seq( + Stage("FLOAT", Seq("1.5", "3.125")), + Stage("DOUBLE", Seq("3.141592653589793", "1.7976931348623157E308"))), + jsonGap = false), + Widening("decimal_precision", Seq( + Stage("DECIMAL(5,2)", Seq("123.45", "999.99")), + Stage("DECIMAL(10,2)", Seq("12345678.90", "99999999.99"))), jsonGap = false), + Widening("decimal_precision_scale", Seq( + Stage("DECIMAL(9,2)", Seq("123.45", "9999999.99")), + Stage("DECIMAL(18,2)", Seq("1234567890123.45", "9999999999999999.99")), + Stage("DECIMAL(28,3)", Seq("1234567890123456789012345.678")))), + Widening("date_timestamp_ntz", Seq( + Stage("DATE", Seq("'2024-01-15'", "'2024-06-30'")), + Stage("TIMESTAMP_NTZ", Seq("'2024-12-31 23:59:59'", "'2025-01-01 12:30:00'")))), + Widening("multiple_steps", Seq( + Stage("BYTE", Seq("1", "100")), Stage("SHORT", Seq("200", "30000")), + Stage("INT", Seq("40000", "2000000000")), + Stage("BIGINT", Seq("3000000000", "9000000000000000000"))))) + + // (stable id, literal, original type). Expected bags are selected explicitly by id, never by + // filtering the table or capturing a fresh query result as its own oracle. + private def rows(w: Widening, stage: Int, partitioned: Boolean): Seq[(Int, String, String)] = + w.stages.take(stage + 1).zipWithIndex.flatMap { case (s, i) => + s.values.zipWithIndex.map { case (v, j) => (i * 10 + j + 1, v, s.dataType) } ++ + (if (partitioned) Seq((i * 10 + 9, "NULL", s.dataType)) else Seq.empty) + } + + private def literal(value: String, from: String, to: String): String = + s"CAST(CAST($value AS $from) AS $to)" + + private def checkedRead( + t: TableHandle, + label: String, + dataType: String, + expected: Seq[(Int, String, String)], + predicate: String, + version: Long): Unit = { + val table = WorkloadContext.current.getTableSpec(t) + readSpec(t, version = version, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(_) => + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val selects = expected.map { case (id, value, from) => + s"SELECT CAST($id AS INT) AS id, ${literal(value, from, dataType)} AS value" + } + val oracle = if (selects.nonEmpty) selects.mkString(" UNION ALL ") else + s"SELECT CAST(NULL AS INT) AS id, CAST(NULL AS $dataType) AS value WHERE false" + val expectedPath = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$label").resolve("expected_data") + RowComparison.assertRowsEqual(spark.sql(oracle), + spark.read.parquet(expectedPath.toString), s"${table.outputName}/$label independent oracle") + } + } + + private def checkedSnapshot(t: TableHandle, w: Widening, stage: Int, + partitioned: Boolean, version: Long): Unit = { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + val expected = StructType.fromDDL(s"id INT, value ${w.stages(stage).dataType}") + require(schema.map(f => f.name -> f.dataType) == + expected.map(f => f.name -> f.dataType), s"Unexpected schema: $schema") + require(result.metadata.partitionColumns == + (if (partitioned) Seq("value") else Seq.empty)) + val fieldMetadata = schema("value").metadata + val changes = if (fieldMetadata.contains("delta.typeChanges")) + fieldMetadata.getMetadataArray("delta.typeChanges").toSeq else Seq.empty + val expectedChanges = w.stages.take(stage + 1).sliding(2).collect { + case Seq(from, to) => + (DataType.fromDDL(from.dataType).json, DataType.fromDDL(to.dataType).json) + }.toSeq + require(changes.map(c => (s"\"${c.getString("fromType")}\"", + s"\"${c.getString("toType")}\"")) == expectedChanges, + s"Unexpected widening history at $version: $changes") + case other => fail(s"Snapshot at $version must succeed: $other") + } + } + } + + private def prepareStats(t: TableHandle, representation: String, original: Stage): Unit = { + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val stream = Files.list(logDir) + val paths = try stream.iterator().asScala.toVector finally stream.close() + // CRC file-state snapshots must not substitute for the statistics under test. + paths.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + if (representation == "parsed") { + val checkpoint = logDir.resolve("00000000000000000001.checkpoint.parquet") + require(Files.exists(checkpoint), "Missing old-schema checkpoint") + val adds = spark.read.parquet(checkpoint.toString).filter("add IS NOT NULL") + val stats = adds.selectExpr("add.stats_parsed.minValues.value AS minValue", + "add.stats_parsed.maxValues.value AS maxValue", "add.stats_parsed.numRecords AS records") + require(stats.schema.head.dataType == DataType.fromDDL(original.dataType), + s"Checkpoint must retain ${original.dataType} statistics: ${stats.schema}") + val addSchema = adds.schema("add").dataType.asInstanceOf[StructType] + if (addSchema.fieldNames.contains("stats")) { + require(adds.filter("add.stats IS NOT NULL").count() == 0, + "Parsed-stat checkpoint must not contain JSON statistics") + } + val oracle = s"""SELECT CAST(${original.values.head} AS ${original.dataType}) AS minValue, + CAST(${original.values.last} AS ${original.dataType}) AS maxValue, + CAST(${original.values.size} AS BIGINT) AS records""" + RowComparison.assertRowsEqual(spark.sql(oracle), stats.selectExpr( + "MIN(minValue) AS minValue", "MAX(maxValue) AS maxValue", "SUM(records) AS records"), + "old parsed checkpoint min/max/count oracle") + // Keep the checkpoint's boundary commit for version discovery; its adds are removed below. + Files.delete(logDir.resolve("00000000000000000000.json")) + } else { + require(!paths.exists(_.getFileName.toString.contains("checkpoint")), + "JSON/partition workloads must not have checkpoints") + val actions = paths.filter(_.getFileName.toString.endsWith(".json")) + .flatMap(p => Files.readAllLines(p).asScala) + .map(JsonUtil.mapper.readTree(_)) + val adds = actions.filter(_.has("add")).map(_.get("add")) + require(adds.nonEmpty) + if (representation == "json") { + require(adds.forall { a => + a.hasNonNull("stats") && Seq("minValues", "maxValues").forall { bound => + JsonUtil.mapper.readTree(a.get("stats").asText()).path(bound).has("value") + } + }) + } else { + require(adds.forall(_.get("partitionValues").has("value"))) + require(adds.exists(_.get("partitionValues").get("value").isNull), + "Retain a null partition written before widening") + } + } + } + if (representation == "parsed") { + modifyCommitActions(t, version = 1)(_.filterNot(_.isInstanceOf[AddFile])) + } + } + + for { + w <- widenings + representation <- Seq("json", "parsed", "partition") + if representation != "json" || w.jsonGap + } { + test(s"${representation}_${w.name}") { + val partitioned = representation == "partition" + val partition = if (partitioned) "PARTITIONED BY (value)" else "" + val timestampFeature = if (w.name == "date_timestamp_ntz") + ", 'delta.feature.timestampNtz' = 'supported'" else "" + val statsProperties = if (representation == "parsed") + ", 'delta.checkpoint.writeStatsAsStruct' = 'true', " + + "'delta.checkpoint.writeStatsAsJson' = 'false'" else "" + sql(s"""CREATE TABLE tbl (id INT, value ${w.stages.head.dataType}) USING delta + $partition TBLPROPERTIES ('delta.enableTypeWidening' = 'true', + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '100', + 'delta.checkpointPolicy' = 'classic' $timestampFeature $statsProperties)""") + + for ((s, i) <- w.stages.zipWithIndex) { + if (i > 0) sql(s"ALTER TABLE tbl ALTER COLUMN value TYPE ${s.dataType}") + val inserted = rows(w, i, partitioned).filter(_._1 / 10 == i) + val values = inserted.map { case (id, value, from) => + s"($id, CAST($value AS $from))" + }.mkString(", ") + sql(s"INSERT INTO tbl VALUES $values") + if (i == 0 && representation == "parsed") forceCheckpoint("tbl") + } + + val t = registerTable("tbl") + prepareStats(t, representation, w.stages.head) + + // Capture the requested latest-schema assertions before historical-stage controls. + for ((s, i) <- w.stages.zipWithIndex.reverse) { + val version = i * 2L + 1 + checkedSnapshot(t, w, i, partitioned, version) + val precisionOnlyStats = representation != "partition" && + w.name == "decimal_precision_scale" && i == 1 + if (i > 0 && !precisionOnlyStats) { + val bag = rows(w, i, partitioned) + val nonNull = bag.filter(_._2 != "NULL") + val original = nonNull.filter(_._1 < 10) + val previous = w.stages(i - 1) + val firstNew = literal(s.values.head, s.dataType, s.dataType) + val lastOld = literal(previous.values.last, previous.dataType, s.dataType) + val lastOriginal = literal(w.stages.head.values.last, + w.stages.head.dataType, s.dataType) + val firstOriginal = literal(w.stages.head.values.head, + w.stages.head.dataType, s.dataType) + val prefix = if (w.name == "multiple_steps" && i == 1) "byte_short" else w.name + val label = s"${prefix}_v$version" + val read = (suffix: String, predicate: String, selected: Seq[(Int, String, String)]) => + checkedRead(t, s"${label}_$suffix", s.dataType, selected, predicate, version) + + // JSON partial cases contribute filters only; TypeWideningSuite owns their full reads. + val partialJson = representation == "json" && + Set("multiple_steps", "decimal_precision_scale", "date_timestamp_ntz").contains(w.name) + if (!partialJson) read("all", null, bag) + read("old_range", s"value <= $lastOriginal", original) + read("new_range", s"value > $lastOld", nonNull.filter(_._1 / 10 == i)) + read("cross_range", s"value >= $lastOld AND value <= $firstNew", + nonNull.filter(r => r._1 == (i - 1) * 10 + previous.values.size || + r._1 == i * 10 + 1)) + read("miss", s"value < $firstOriginal", Seq.empty) + + if (partitioned) { + read("old_equality", s"value = $firstOriginal", original.take(1)) + read("new_equality", s"value = $firstNew", nonNull.filter(_._1 == i * 10 + 1)) + read("null_partitions", "value IS NULL", bag.filter(_._2 == "NULL")) + } + if (w.name == "date_timestamp_ntz") { + read("old_midnight", s"value = $lastOriginal", original.takeRight(1)) + read("new_midnight_miss", "value = TIMESTAMP_NTZ'2024-12-31 00:00:00'", Seq.empty) + read("exclusive_boundary", s"value >= $lastOriginal AND value < $firstNew", + original.takeRight(1)) + } + } + } + } + } +} From edb73ee36439f5cdfc62dd32e87e5329aecd9b4b Mon Sep 17 00:00:00 2001 From: "oussama.saoudi" Date: Wed, 18 Nov 2026 01:12:14 +0000 Subject: [PATCH 2/2] Integrate acceptance cases into existing domain suites --- .../workload/tables/CheckpointsSuite.scala | 522 ++- .../workload/tables/ColumnMappingSuite.scala | 298 +- .../workload/tables/DataSkippingSuite.scala | 3571 ++++++++++++++++- .../tables/DeletionVectorPayloads.scala | 61 + .../tables/DeletionVectorsSuite.scala | 188 +- .../workload/tables/LiteralOracleChecks.scala | 89 + .../workload/tables/LogReplaySuite.scala | 309 +- .../tables/PartitionValuesSuite.scala | 452 ++- .../tables/ReadinessCheckpointSuite.scala | 559 --- .../tables/ReadinessColumnMappingSuite.scala | 403 -- .../tables/ReadinessPredicatesSuite.scala | 779 ---- .../ReadinessSchemaReplacementSuite.scala | 494 --- .../tables/ReadinessStatisticsSuite.scala | 1011 ----- .../ReadinessTableOperationsSuite.scala | 1354 ------- .../tables/ReadinessTimestampSuite.scala | 1942 --------- .../tables/ReadinessTypeWideningSuite.scala | 278 -- .../io/delta/workload/tables/ReadsSuite.scala | 148 + .../tables/SchemaEvolutionSuite.scala | 472 ++- .../workload/tables/ScopedSessionConf.scala | 55 + .../workload/tables/TimeTravelSuite.scala | 410 +- .../workload/tables/TypeWideningSuite.scala | 259 +- 21 files changed, 6818 insertions(+), 6836 deletions(-) create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorPayloads.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/LiteralOracleChecks.scala delete mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessCheckpointSuite.scala delete mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessColumnMappingSuite.scala delete mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessPredicatesSuite.scala delete mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessSchemaReplacementSuite.scala delete mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessStatisticsSuite.scala delete mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTableOperationsSuite.scala delete mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTimestampSuite.scala delete mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTypeWideningSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/ScopedSessionConf.scala diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala index c557f995..7cac9cdb 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala @@ -16,11 +16,28 @@ package io.delta.workload.tables +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} + +import scala.collection.JavaConverters._ + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.spark.sql.Row +import org.apache.spark.sql.functions.{col, lit, map => sparkMap, when} +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + import io.delta.workload.{TableHandle, WorkloadTestSuite} -import io.delta.workload.log.{Action, DomainMetadata, Txn} -import io.delta.workload.model.ErrorCode +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{Action, AddFile, CheckpointMetadata, CommitLog, DomainMetadata, + Metadata, Protocol, SidecarFile, Txn} +import io.delta.workload.model._ -class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { +class CheckpointsSuite extends WorkloadTestSuite("checkpoints") + with ScopedSessionConf { private def checkpoint(name: String): Unit = forceCheckpoint(name) @@ -659,4 +676,503 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { snapshotSpec(t) readSpec(t) } + + // Physical checkpoint forms (JSON/Parquet top-level files, manifests, sidecars, + // multi-part, structured stats, >64 KiB components, allFiles CRC). Session settings + // stay active until the specs are captured. + private def snapshotResult(protocol: Protocol, metadata: Metadata): SnapshotResult = + SnapshotResult( + ProtocolInfo(protocol.minReaderVersion, protocol.minWriterVersion, + protocol.readerFeatures.map(_.toSeq.sorted), + protocol.writerFeatures.map(_.toSeq.sorted)), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat(metadata.format.provider, metadata.format.options), + metadata.schemaString, metadata.partitionColumns, metadata.configuration, + metadata.createdTime)) + + private def assertReadRows( + spec: ReadSpec, + tablePath: Path, + schema: StructType, + rows: Seq[Row], + name: String): Unit = { + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, tablePath, + spec.query.version, spec.query.timestamp), spec.query.predicate, spec.query.columns) + assert(actual.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, schema), actual, name) + } + + Seq( + ("json_inline", "json", "v2", 1000, 0), + ("json_manifest", "json", "v2", 1000, 1), + ("parquet_single_part", "parquet", "classic", 1000, 0), + ("parquet_manifest", "parquet", "v2", 1000, 1), + ("parquet_sidecars", "parquet", "v2", 1, 2) + ).foreach { case (name, format, policy, partSize, sidecarCount) => + test(name) { + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> format, + "spark.databricks.delta.checkpoint.partSize" -> partSize.toString) + settings.foreach { case (key, value) => setForTest(key, value) } + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 'a'), (2, CAST(NULL AS STRING)), (2, CAST(NULL AS STRING)) AS v(id, value)""") + sql("INSERT INTO tbl VALUES (3, 'c')") + forceCheckpoint("tbl") + sql("INSERT INTO tbl VALUES (4, 'd')") + val t = registerTable("tbl") + // Read the original commit actions before mutating the copied fixture. + val actions = (0L to 2L).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 2) + info("Original protocol: " + protocol.toJson) + info("Original metadata: " + metadata.toJson) + adds.foreach(a => info("Original add: " + a.toJson)) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + val expectedSnapshot = snapshotResult(protocol, metadata) + val checkpointRows = Seq(Row(1, "a"), Row(2, null), Row(2, null), Row(3, "c")) + val latestRows = checkpointRows :+ Row(4, "d") + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + assert(checkpoints.size == 1) + val checkpoint = checkpoints.head + if (policy == "classic") { + assert(checkpoint.getFileName.toString == "00000000000000000002.checkpoint.parquet") + val frame = spark.read.parquet(checkpoint.toString) + assert(frame.filter("add IS NOT NULL").count() == adds.size) + assert(frame.filter("metaData IS NOT NULL").count() == 1) + assert(frame.filter("protocol IS NOT NULL").count() == 1) + assert(!Files.exists(log.resolve("_sidecars"))) + } else { + assert(checkpoint.getFileName.toString.matches( + s"00000000000000000002\\.checkpoint\\.[^.]+\\.$format")) + if (name == "json_inline") { + // V2 allows inline file actions; this writer normally emits sidecars. + val checkpointAdds = adds.map(_.copy(dataChange = false)) + val inlineActions: Seq[Action] = + Seq(protocol, metadata, CheckpointMetadata(2L)) ++ checkpointAdds + Files.write(checkpoint, inlineActions.map(_.toJson).asJava, UTF_8) + val sidecars = Files.list(log.resolve("_sidecars")) + try sidecars.iterator().asScala.toVector.foreach(Files.delete) + finally sidecars.close() + Files.delete(log.resolve("_sidecars")) + val content = Files.readAllLines(checkpoint, UTF_8).asScala.map(Action.parse).toVector + assert(content.collect { case a: AddFile => a } == checkpointAdds) + assert(content.collect { case c: CheckpointMetadata => c.version } == Seq(2L)) + assert(!content.exists(_.isInstanceOf[SidecarFile])) + } else { + val manifest = if (format == "json") { + Files.readAllLines(checkpoint, UTF_8).asScala.map(Action.parse).toVector + } else { + spark.read.parquet(checkpoint.toString).toJSON.collect().toVector.map(Action.parse) + } + val references = manifest.collect { case s: SidecarFile => s } + assert(references.size == sidecarCount) + assert(!manifest.exists(_.isInstanceOf[AddFile])) + assert(manifest.collect { case c: CheckpointMetadata => c.version } == Seq(2L)) + assert(manifest.collect { case m: Metadata => m } == Seq(metadata)) + assert(manifest.collect { case p: Protocol => p } == Seq(protocol)) + val sidecarPaths = references.map { ref => + val path = log.resolve("_sidecars").resolve(ref.path) + assert(ref.path.endsWith(".parquet")) + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + path + } + val sidecarListing = Files.list(log.resolve("_sidecars")) + try assert(sidecarListing.iterator().asScala.toSet == sidecarPaths.toSet) + finally sidecarListing.close() + val content = spark.read.parquet(sidecarPaths.map(_.toString): _*) + assert(content.count() == adds.size) + assert(content.filter("add IS NOT NULL").count() == adds.size) + assert(content.filter("remove IS NOT NULL").count() == 0) + assert(content.select("add.path").collect().map(_.getString(0)).toSet == + adds.map(_.path).toSet) + } + } + // Keep same-version JSON commit 2 for the OSS historical snapshot lookup. + // This tests checkpoint plus commit 2, then version 3 tail replay. + // Checkpoint-only historical snapshots remain a separate failing repro. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 2L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + Seq( + ("read_all", ReadQuery(), latestRows, ReadResult(5L, 3, 0L)), + ("read_v2", ReadQuery(version = Some(2L)), checkpointRows, ReadResult(4L, 2, 0L)), + ("read_filtered", ReadQuery(predicate = Some("id >= 3")), + Seq(Row(3, "c"), Row(4, "d")), ReadResult(2L, 2, 1L)) + ).foreach { case (readName, query, rows, expected) => + readSpec(t, name = Some(readName), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, readName) + } + } + Seq(None, Some(2L)).foreach { version => + snapshotSpec(t, version = version.map(Long.box).orNull).assert { spec => + assert(spec.query.version.contains(version.getOrElse(3L))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + } + } + + // stats: OSS exposes structured checkpoint stats, but no product metadata-clustering + // control. These fixtures check shape and filtered correctness, never row groups READ. + Seq( + ("e2e_07_single_part", "classic", 1000, 1, 0), + ("e2e_07_multipart", "classic", 2, 3, 0), + ("e2e_07_sidecars", "v2", 2, 1, 3) + ).foreach { case (name, policy, partSize, checkpointCount, sidecarCount) => + test(name) { + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> "parquet", + "spark.databricks.delta.checkpoint.partSize" -> partSize.toString, + "spark.databricks.delta.statsAsStructInCheckpoint.forcedDisabled" -> "false") + settings.foreach { case (key, value) => setForTest(key, value) } + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpoint.writeStatsAsStruct' = 'true')""") + val rowsByFile = (0 until 6).map { band => + val low = band * 10 + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + ($low, 'band_$band'), (${low + 1}, CAST(NULL AS STRING)), + (${low + 1}, CAST(NULL AS STRING)) AS v(id, value)""") + Seq(Row(low, s"band_$band"), Row(low + 1, null), Row(low + 1, null)) + } + forceCheckpoint("tbl") + val t = registerTable("tbl") + val actions = (0L to 6L).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 6 && adds.map(_.path).distinct.size == 6) + info("stats original protocol: " + protocol.toJson) + info("stats original metadata: " + metadata.toJson) + adds.foreach(a => info("stats original add: " + a.toJson)) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration("delta.checkpointPolicy") == policy) + assert(metadata.configuration("delta.checkpoint.writeStatsAsStruct") == "true") + val expectedSnapshot = snapshotResult(protocol, metadata) + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + assert(checkpoints.size == checkpointCount) + if (name == "e2e_07_single_part") { + assert(checkpoints.head.getFileName.toString == + "00000000000000000006.checkpoint.parquet") + } else if (policy == "classic") { + val expectedNames = (1 to 3).map { part => + f"00000000000000000006.checkpoint.$part%010d.0000000003.parquet" + }.toSet + assert(checkpoints.map(_.getFileName.toString).toSet == expectedNames) + } else { + assert(checkpoints.head.getFileName.toString.matches( + "00000000000000000006\\.checkpoint\\.[^.]+\\.parquet")) + } + val top = spark.read.parquet(checkpoints.map(_.toString): _*) + val topActions = top.toJSON.collect().toVector.map(Action.parse) + assert(topActions.collect { case m: Metadata => m } == Seq(metadata)) + assert(topActions.collect { case p: Protocol => p } == Seq(protocol)) + val sidecarPaths = if (policy == "v2") { + val refs = topActions.collect { case s: SidecarFile => s } + assert(refs.size == sidecarCount && refs.map(_.path).distinct.size == sidecarCount) + assert(topActions.collect { case c: CheckpointMetadata => c.version } == Seq(6L)) + assert(!topActions.exists(_.isInstanceOf[AddFile])) + val paths = refs.map { ref => + val path = log.resolve("_sidecars").resolve(ref.path) + assert(ref.path.endsWith(".parquet")) + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + path + } + val listing = Files.list(log.resolve("_sidecars")) + try assert(listing.iterator().asScala.toSet == paths.toSet) + finally listing.close() + paths + } else { + assert(!Files.exists(log.resolve("_sidecars"))) + Vector.empty[Path] + } + val fileActions = if (sidecarPaths.isEmpty) top else + spark.read.parquet(sidecarPaths.map(_.toString): _*) + assert(fileActions.filter("add IS NOT NULL").count() == 6) + assert(fileActions.filter("remove IS NOT NULL").count() == 0) + if (sidecarPaths.nonEmpty) assert(fileActions.count() == 6) + assert(fileActions.select("add.path").collect().filterNot(_.isNullAt(0)) + .map(_.getString(0)).toSet == adds.map(_.path).toSet) + val stats = fileActions.filter("add IS NOT NULL").select( + "add.stats_parsed.numRecords", "add.stats_parsed.minValues.id", + "add.stats_parsed.maxValues.id", "add.stats_parsed.nullCount.value").collect().toSeq + assert(stats.toSet == (0 until 6).map { band => + Row(3L, band * 10, band * 10 + 1, 2L) + }.toSet) + // Footer row groups are physical shape evidence, not reader IO counters. + val footerRows = (checkpoints ++ sidecarPaths).map { path => + val reader = ParquetFileReader.open(HadoopInputFile.fromPath( + new HadoopPath(path.toUri), spark.sparkContext.hadoopConfiguration)) + try { + val groups = reader.getFooter.getBlocks.asScala + info(s"stats footer: ${path.getFileName} groups=${groups.size} " + + s"rows=${groups.map(_.getRowCount).sum}") + assert(groups.size <= 1) + groups.map(_.getRowCount).sum + } finally reader.close() + } + assert(footerRows.sum == (if (policy == "v2") 12L else 8L)) + // Retain same-version JSON 6 for OSS version lookup, with no post-checkpoint tail. + // Missing commits 0..5 make the checkpoint required even though commit 6 exists. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 6L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + Seq( + ("read_all", None, rowsByFile.flatten, ReadResult(18L, 6, 0L)), + ("read_high", Some("id >= 40"), rowsByFile.drop(4).flatten, ReadResult(6L, 2, 4L)), + ("read_boundary", Some("id = 41"), Seq(Row(41, null), Row(41, null)), + ReadResult(2L, 1, 5L)), + ("read_empty", Some("id >= 100"), Seq.empty[Row], ReadResult(0L, 0, 6L)) + ).foreach { case (readName, predicate, rows, expected) => + readSpec(t, name = Some(readName), predicate = predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, readName) + } + } + snapshotSpec(t).assert { spec => + assert(spec.query.version.contains(6L)) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + } + + // size: 64 KiB is a bounded fixture boundary, not a product dispatch threshold. + // These shapes assert portable read correctness, never distributed execution. + Seq("manifest", "json", "sidecars", "single_part", "multipart", "crc_all_files") + .foreach { shape => + test(s"e2e_43_$shape") { + val isCrc = shape == "crc_all_files" + val policy = if (Set("single_part", "multipart", "crc_all_files")(shape)) + "classic" else "v2" + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> "json", + "spark.databricks.delta.checkpoint.partSize" -> + (if (shape == "multipart") "2" else "1000"), + "spark.databricks.delta.allFilesInCrc.enabled" -> "true", + "spark.databricks.delta.allFilesInCrc.thresholdNumFiles" -> "100") + settings.foreach { case (key, value) => setForTest(key, value) } + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false')""") + val rowsByFile = (0 until 4).map { band => + val low = band * 10 + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + ($low, 'band_$band'), (${low + 1}, CAST(NULL AS STRING)), + (${low + 1}, CAST(NULL AS STRING)) AS v(id, value)""") + Seq(Row(low, s"band_$band"), Row(low + 1, null), Row(low + 1, null)) + } + forceCheckpoint("tbl") + // A complete-file removal distinguishes active files from historical Adds in CRC. + if (isCrc) sql("DELETE FROM tbl WHERE id < 10") + sql("INSERT INTO tbl VALUES (99, 'tail')") + val t = registerTable("tbl") + val baseVersion = if (isCrc) 5L else 4L + val latestVersion = baseVersion + 1L + val actions = (0L to latestVersion).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 5 && adds.map(_.path).distinct.size == 5) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration("delta.checkpointPolicy") == policy) + assert(metadata.configuration("delta.enableDeletionVectors") == "false") + val expectedSnapshot = snapshotResult(protocol, metadata) + val baseRows = (if (isCrc) rowsByFile.drop(1) else rowsByFile).flatten + val latestRows = baseRows :+ Row(99, "tail") + info(s"size $shape protocol: " + protocol.toJson) + info(s"size $shape metadata: " + metadata.toJson) + adds.foreach(a => info(s"size $shape original add: " + a.toJson)) + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + .sortBy(_.getFileName.toString) + assert(checkpoints.size == (if (shape == "multipart") 2 else 1)) + if (policy == "classic") { + val names = if (shape == "multipart") (1 to 2).map { part => + f"00000000000000000004.checkpoint.$part%010d.0000000002.parquet" + }.toSet else Set("00000000000000000004.checkpoint.parquet") + assert(checkpoints.map(_.getFileName.toString).toSet == names) + } else { + assert(checkpoints.head.getFileName.toString.matches( + "00000000000000000004\\.checkpoint\\.[^.]+\\.json")) + } + val manifest = if (policy == "v2") { + Files.readAllLines(checkpoints.head, UTF_8).asScala.map(Action.parse).toVector + } else Vector.empty[Action] + val refs = manifest.collect { case s: SidecarFile => s } + assert(refs.size == (if (policy == "v2") 1 else 0)) + val sidecars = refs.map(ref => log.resolve("_sidecars").resolve(ref.path)) + refs.zip(sidecars).foreach { case (ref, path) => + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + } + if (policy == "v2") { + assert(!manifest.exists(_.isInstanceOf[AddFile])) + assert(manifest.collect { case c: CheckpointMetadata => c.version } == Seq(4L)) + assert(manifest.collect { case m: Metadata => m } == Seq(metadata)) + assert(manifest.collect { case p: Protocol => p } == Seq(protocol)) + } else assert(!Files.exists(log.resolve("_sidecars"))) + val boundary = 64L * 1024L + // Deterministic incompressible padding keeps Parquet fixtures small but >64 KiB. + val bytes = new Array[Byte](96 * 1024) + new scala.util.Random(43L).nextBytes(bytes) + val padding = java.util.Base64.getEncoder.encodeToString(bytes) + val paddedFiles = shape match { + case "sidecars" => sidecars + case "single_part" | "multipart" => checkpoints + case _ => Vector.empty[Path] + } + paddedFiles.foreach { path => + val frame = spark.read.parquet(path.toString) + val padded = frame.withColumn("add", when(col("add").isNotNull, + col("add").withField("tags", sparkMap(lit("e2e43.padding"), lit(padding)))) + .otherwise(col("add"))) + val temp = Files.createTempDirectory("e2e43-checkpoint-") + try { + padded.coalesce(1).write.mode("overwrite").parquet(temp.toString) + val parts = Files.list(temp) + val part = try parts.iterator().asScala + .filter(_.getFileName.toString.endsWith(".parquet")).toVector + finally parts.close() + assert(part.size == 1) + Files.move(part.head, path, java.nio.file.StandardCopyOption.REPLACE_EXISTING) + } finally org.apache.commons.io.FileUtils.deleteDirectory(temp.toFile) + } + if (shape == "manifest" || shape == "sidecars") { + val updated = manifest.map { + case c: CheckpointMetadata if shape == "manifest" => + c.copy(tags = Some(Map("e2e43.padding" -> padding))) + case ref: SidecarFile => + val path = log.resolve("_sidecars").resolve(ref.path) + ref.copy(sizeInBytes = Files.size(path), + modificationTime = Files.getLastModifiedTime(path).toMillis) + case action => action + } + Files.write(checkpoints.head, updated.map(_.toJson).asJava, UTF_8) + } + if (shape == "json") { + val lines = CommitLog.readRaw(tableDir, latestVersion).map { line => + val node = JsonUtil.mapper.readTree(line) + if (node.has("commitInfo")) { + val commitInfo = node.get("commitInfo") + .asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + commitInfo.put("userMetadata", padding) + } + node.toString + } + CommitLog.writeRaw(tableDir, latestVersion, lines) + } + val fileActions = spark.read.parquet( + (if (policy == "v2") sidecars else checkpoints).map(_.toString): _*) + val checkpointAdds = fileActions.toJSON.collect().toVector.map(Action.parse) + .collect { case a: AddFile => a } + val expectedAdds = adds.take(4).map { add => + add.copy(dataChange = false, tags = if (paddedFiles.nonEmpty) + Some(Map("e2e43.padding" -> padding)) else add.tags) + } + assert(checkpointAdds.sortBy(_.path) == expectedAdds.sortBy(_.path)) + if (policy == "classic") { + val top = fileActions.toJSON.collect().toVector.map(Action.parse) + assert(top.collect { case m: Metadata => m } == Seq(metadata)) + assert(top.collect { case p: Protocol => p } == Seq(protocol)) + } + if (isCrc) { + Seq(baseVersion, latestVersion).foreach { version => + val crcPath = log.resolve(f"$version%020d.crc") + assert(Files.exists(crcPath)) + val crc = JsonUtil.mapper.readTree(Files.readString(crcPath, UTF_8)) + assert(crc.has("allFiles") && crc.get("allFiles").isArray) + val active = (if (version == baseVersion) adds.slice(1, 4) else adds.drop(1)) + .map(_.copy(dataChange = false)) + val encoded = crc.get("allFiles").elements().asScala.map { node => + Action.parse("{\"add\":" + node.toString + "}").asInstanceOf[AddFile] + }.toVector + // The optional tags map can round-trip through Parquet as empty, not absent. + val normalized = encoded.map(a => a.copy(tags = a.tags.filter(_.nonEmpty))) + assert(normalized.sortBy(_.path) == active.sortBy(_.path)) + assert(crc.get("numFiles").asLong() == active.size) + assert(crc.get("tableSizeBytes").asLong() == active.map(_.size).sum) + info(s"size CRC v$version bytes=${Files.size(crcPath)} " + + s"allFiles=${encoded.size} activeBytes=${active.map(_.size).sum}") + } + } else { + // JSON 4 supports historical version lookup; absent 0..3 requires checkpoint 4. + // JSON 5 is a real one-commit replay tail with one new AddFile. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 4L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + val jsonPaths = files.filter(p => Files.exists(p) && + p.getFileName.toString.matches("[0-9]{20}\\.json")) + (checkpoints ++ sidecars ++ jsonPaths).foreach { path => + val large = paddedFiles.contains(path) || + (shape == "manifest" && checkpoints.contains(path)) || + (shape == "json" && path == CommitLog.commitFile(tableDir, latestVersion)) + val size = Files.size(path) + assert(if (large) size > boundary else size < boundary) + info(s"size $shape component=${log.relativize(path)} bytes=$size " + + s"large=$large boundary=$boundary") + } + } + Seq( + ("read_base", Some(baseVersion), None, baseRows, + ReadResult(baseRows.size.toLong, if (isCrc) 3 else 4, 0L)), + ("read_latest", None, None, latestRows, + ReadResult(latestRows.size.toLong, if (isCrc) 4 else 5, 0L)), + ("read_duplicates", None, Some("id = 21"), Seq(Row(21, null), Row(21, null)), + ReadResult(2L, 1, if (isCrc) 3L else 4L)) + ).foreach { case (name, version, predicate, rows, expected) => + readSpec(t, name = Some(name), version = version.map(Long.box).orNull, + predicate = predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, s"size $shape $name") + } + } + Seq(Some(baseVersion), None).foreach { version => + snapshotSpec(t, version = version.map(Long.box).orNull).assert { spec => + assert(spec.query.version.contains(version.getOrElse(latestVersion))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + if (isCrc) crcSpec(t, version = baseVersion, name = "crc_complete_active") + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ColumnMappingSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ColumnMappingSuite.scala index ae9780af..83494bf1 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ColumnMappingSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ColumnMappingSuite.scala @@ -16,7 +16,18 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.Files + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog, Metadata => LogMetadata, Protocol, RemoveFile} +import io.delta.workload.model.Succeeded class ColumnMappingSuite extends WorkloadTestSuite("column_mapping") { @@ -419,4 +430,289 @@ class ColumnMappingSuite extends WorkloadTestSuite("column_mapping") { for (v <- 0L to 3L) snapshotSpec(t, version = v) } + // Parsed checkpoint statistics/partition values keyed by physical names, and enabling + // name mapping through a full rewrite. + // One primary scenario covers parsed statistics and parsed partition values together. + // The existing checkpoints_partitioned fixture is the no-mapping parsed-key control. + for (mode <- Seq("name", "id")) { + test(s"parsed_physical_names_$mode") { + val properties = Map( + "delta.columnMapping.mode" -> mode, + "delta.checkpointInterval" -> "1000", + "delta.checkpoint.writeStatsAsStruct" -> "true", + "delta.checkpoint.writeStatsAsJson" -> "false") + val propertySql = properties.toSeq.sortBy(_._1) + .map { case (key, value) => s"'$key' = '$value'" }.mkString(", ") + sql(s"""CREATE TABLE tbl ( + id INT, old_value INT, payload STRUCT, old_part STRING) + USING delta PARTITIONED BY (old_part) TBLPROPERTIES ($propertySql)""") + sql("""INSERT INTO tbl VALUES + (1, 10, named_struct('old_leaf', -10), 'east'), + (1, 10, named_struct('old_leaf', -10), 'east'), + (2, 20, named_struct('old_leaf', -20), 'west'), + (3, NULL, named_struct('old_leaf', CAST(NULL AS INT)), NULL), + (4, 40, NULL, 'east')""") + sql("ALTER TABLE tbl RENAME COLUMN old_value TO value") + sql("ALTER TABLE tbl RENAME COLUMN payload.old_leaf TO leaf") + sql("ALTER TABLE tbl RENAME COLUMN old_part TO region") + sql("""INSERT INTO tbl VALUES + (5, 100, named_struct('leaf', -100), 'north'), + (6, NULL, named_struct('leaf', CAST(NULL AS INT)), NULL), + (7, 200, named_struct('leaf', -200), 'east')""") + forceCheckpoint("tbl") + // A real v6 tail is replayed after the v5 checkpoint; v5 JSON is only version lookup. + sql("INSERT INTO tbl VALUES (8, 300, named_struct('leaf', -300), 'west')") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val actions = (0L to 6L).map { version => + version -> CommitLog.readRaw(t.sourcePath, version).map(JsonUtil.mapper.readTree) + }.toMap + val beforeMetadata = actions(0L).find(_.has("metaData")).get.get("metaData") + val metadata = actions(4L).find(_.has("metaData")).get.get("metaData") + val before = DataType.fromJson(beforeMetadata.get("schemaString").asText) + .asInstanceOf[StructType] + val mapped = DataType.fromJson(metadata.get("schemaString").asText) + .asInstanceOf[StructType] + val physicalKey = "delta.columnMapping.physicalName" + val idKey = "delta.columnMapping.id" + val oldFields = before.fields.toSeq :+ + before("payload").dataType.asInstanceOf[StructType]("old_leaf") + val fields = mapped.fields.toSeq :+ + mapped("payload").dataType.asInstanceOf[StructType]("leaf") + assert(mapped.fieldNames.toSeq == Seq("id", "value", "payload", "region")) + assert(fields.map(_.metadata.getLong(idKey)).toSet == Set(1L, 2L, 3L, 4L, 5L)) + assert(fields.map(_.metadata) == oldFields.map(_.metadata)) + assert(fields.forall(field => field.metadata.getString(physicalKey) != field.name)) + val physical = fields.map(_.metadata.getString(physicalKey)) + assert(physical.distinct.size == 5) + val partitionName = physical(3) + val statsNames = physical.take(3).toSet + val originals = actions.values.flatten.filter(_.has("add")) + .map(_.get("add")).map(add => add.get("path").asText -> add).toMap + + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + for (version <- 0L to 6L; line <- CommitLog.readRaw(tableDir, version)) { + println(s"Physical-name source action: $mode v$version $line") + } + val checkpointPath = logDir.resolve("00000000000000000005.checkpoint.parquet") + assert(Files.exists(checkpointPath)) + val checkpoint = spark.read.parquet(checkpointPath.toString) + val addSchema = checkpoint.schema("add").dataType.asInstanceOf[StructType] + assert(!addSchema.fieldNames.contains("stats")) + val partition = addSchema("partitionValues_parsed").dataType.asInstanceOf[StructType] + assert(partition.fieldNames.toSeq == Seq(partitionName)) + assert(partition.fields.head.dataType == StringType) + val stats = addSchema("stats_parsed").dataType.asInstanceOf[StructType] + for (kind <- Seq("minValues", "maxValues", "nullCount")) { + val leaves = stats(kind).dataType.asInstanceOf[StructType] + assert(leaves.fieldNames.toSet == statsNames) + assert(leaves(physical(2)).dataType.asInstanceOf[StructType] + .fieldNames.toSeq == Seq(physical(4))) + } + val adds = checkpoint.filter("add IS NOT NULL").select("add").toJSON.collect() + assert(adds.nonEmpty) + for (line <- adds) { + val add = JsonUtil.mapper.readTree(line).get("add") + val original = originals(add.get("path").asText) + assert(add.get("partitionValues") == original.get("partitionValues")) + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val parsedPartition = add.get("partitionValues_parsed").get(partitionName) + val rawPartition = original.get("partitionValues").get(partitionName) + assert(Option(parsedPartition).filterNot(_.isNull).map(_.asText) == + Option(rawPartition).filterNot(_.isNull).map(_.asText)) + val rawStats = JsonUtil.mapper.readTree(original.get("stats").asText) + val parsedStats = add.get("stats_parsed") + assert(parsedStats.get("numRecords") == rawStats.get("numRecords")) + for (kind <- Seq("minValues", "maxValues", "nullCount"); + path <- Seq(Seq(physical(0)), Seq(physical(1)), Seq(physical(2), physical(4)))) { + val parsed = path.foldLeft(parsedStats.path(kind))((node, key) => node.path(key)) + val raw = path.foldLeft(rawStats.path(kind))((node, key) => node.path(key)) + assert(Option(parsed).filterNot(n => n.isNull || n.isMissingNode) == + Option(raw).filterNot(n => n.isNull || n.isMissingNode)) + } + } + // Earlier protocol/metadata/file history is absent: v5 checkpoint remains required. + // Keep v5 for version lookup and v6 as the actual post-checkpoint replay tail. + val stream = Files.list(logDir) + val files = try stream.iterator().asScala.toSeq finally stream.close() + files.filter { path => + val name = path.getFileName.toString + name.endsWith(".crc") || + (name.matches("[0-9]{20}\\.json") && name < "00000000000000000005.json") + }.foreach(Files.delete) + println(s"Physical-name checkpoint/layout oracle passed: $mode") + } + + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("payload", new StructType().add("leaf", IntegerType)).add("region", StringType) + val rows = Seq( + Row(1, 10, Row(-10), "east"), Row(1, 10, Row(-10), "east"), + Row(2, 20, Row(-20), "west"), Row(3, null, Row(null), null), + Row(4, 40, null, "east"), Row(5, 100, Row(-100), "north"), + Row(6, null, Row(null), null), Row(7, 200, Row(-200), "east"), + Row(8, 300, Row(-300), "west")) + // Row selections are literal bag oracles, independent of Spark's predicate evaluation. + val reads = Seq( + ("read_all", null, rows.indices, None, None), + ("read_checkpoint", null, 0 until 8, Some(5L), None), + ("read_old_file", "value = 10", Seq(0, 1), None, None), + ("read_new_file", "value = 100", Seq(5), None, None), + ("read_tail", "value = 300", Seq(8), None, None), + ("read_missing", "value = 999", Seq.empty[Int], None, None), + ("read_null", "value IS NULL", Seq(3, 6), None, None), + ("read_nested_old", "payload.leaf = -10", Seq(0, 1), None, None), + ("read_nested_new", "payload.leaf = -100", Seq(5), None, None), + ("read_nested_null", "payload.leaf IS NULL", Seq(3, 4, 6), None, None), + ("read_partition_old", "region = 'east'", Seq(0, 1, 4, 7), None, None), + ("read_partition_new", "region = 'north'", Seq(5), None, None), + ("read_partition_null", "region IS NULL", Seq(3, 6), None, None), + ("read_projection", null, rows.indices, None, Some(Seq("region", "value", "payload")))) + for ((name, predicate, indices, version, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, + version = version.map(Long.box).orNull, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"$mode/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"$mode/$name literal bag") + assert(captured.schema.fields.forall(_.nullable)) + println(s"Physical-name literal row/schema oracle passed: $mode/$name") + } + } + for (version <- Seq(5L, 6L)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.get("schemaString").asText) + assert(result.metadata.partitionColumns == Seq("region")) + assert(result.metadata.configuration == + properties + ("delta.columnMapping.maxColumnId" -> "5")) + case other => fail(s"$mode snapshot v$version must succeed: $other") + } + println(s"Physical-name snapshot mapping oracle passed: $mode/v$version") + } + } + } + } + + test("enable_name_mapping_with_full_rewrite") { + sql("""CREATE TABLE tbl (id INT, value INT, payload STRING) USING delta + TBLPROPERTIES ('delta.columnMapping.mode' = 'none', + 'delta.checkpointInterval' = '1000')""") + sql("INSERT INTO tbl VALUES (1, 10, 'duplicate'), (1, 10, 'duplicate'), (2, NULL, 'nullable')") + sql("INSERT INTO tbl VALUES (3, 30, NULL), (NULL, NULL, NULL), (NULL, NULL, NULL)") + // A separately materialized source prevents replacement from reading its own target. + // Both SQL-created tables are tracked by the normal WorkloadContext cleanup. + sql("CREATE TABLE rewrite_source USING delta AS SELECT id, value, payload FROM tbl") + sql("""CREATE OR REPLACE TABLE tbl USING delta TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', 'delta.minReaderVersion' = '2', + 'delta.minWriterVersion' = '5', 'delta.checkpointInterval' = '1000') + AS SELECT id, value, payload FROM rewrite_source""") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val before = (0L to 2L).flatMap(CommitLog.read(t.sourcePath, _)) + val rewrite = CommitLog.read(t.sourcePath, 3L) + val oldAdds = before.collect { case add: AddFile => add } + val newAdds = rewrite.collect { case add: AddFile => add } + val removed = rewrite.collect { case remove: RemoveFile => remove } + assert(!before.exists(_.isInstanceOf[RemoveFile])) + assert(oldAdds.nonEmpty && newAdds.nonEmpty) + val oldPaths = oldAdds.map(_.path).toSet + val newPaths = newAdds.map(_.path).toSet + assert(oldPaths.size == oldAdds.size && newPaths.size == newAdds.size) + assert(removed.map(_.path).toSet == oldPaths && removed.size == oldPaths.size) + assert(newPaths.intersect(oldPaths).isEmpty) + assert((oldPaths -- removed.map(_.path)) ++ newPaths == newPaths) + assert((oldAdds ++ newAdds).forall(add => + Files.isRegularFile(t.sourcePath.resolve(add.path)) && + Files.size(t.sourcePath.resolve(add.path)) == add.size)) + + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("payload", StringType) + val oldMetadata = before.collect { case metadata: LogMetadata => metadata }.last + val newMetadata = rewrite.collect { case metadata: LogMetadata => metadata }.last + assert(DataType.fromJson(oldMetadata.schemaString) == schema) + assert(oldMetadata.configuration == Map( + "delta.columnMapping.mode" -> "none", "delta.checkpointInterval" -> "1000")) + val mapped = DataType.fromJson(newMetadata.schemaString).asInstanceOf[StructType] + assert(mapped.fields.map(f => (f.name, f.dataType, f.nullable)).toSeq == + schema.fields.map(f => (f.name, f.dataType, f.nullable)).toSeq) + val idKey = "delta.columnMapping.id" + val physicalKey = "delta.columnMapping.physicalName" + assert(mapped.fields.map(_.metadata.getLong(idKey)).toSet == Set(1L, 2L, 3L)) + val physicalNames = mapped.fields.map(_.metadata.getString(physicalKey)).toSeq + assert(physicalNames.distinct.size == 3) + assert(mapped.fields.forall(f => f.metadata.getString(physicalKey) != f.name)) + assert(newMetadata.configuration == Map( + "delta.columnMapping.mode" -> "name", "delta.columnMapping.maxColumnId" -> "3", + "delta.checkpointInterval" -> "1000")) + assert(oldMetadata.partitionColumns.isEmpty && newMetadata.partitionColumns.isEmpty) + assert(before.collect { case p: Protocol => p }.last == Protocol(1, 2)) + assert(rewrite.collect { case p: Protocol => p }.last == Protocol(2, 5)) + for (add <- oldAdds) { + assert(spark.read.parquet(t.sourcePath.resolve(add.path).toString).schema == schema) + } + for (add <- newAdds) { + val physical = spark.read.parquet(t.sourcePath.resolve(add.path).toString).schema + assert(physical.fieldNames.toSeq == physicalNames) + assert(physical.fields.map(_.dataType).toSeq == schema.fields.map(_.dataType).toSeq) + assert(physical.fields.map(_.metadata.getLong("parquet.field.id")).toSeq == + mapped.fields.map(_.metadata.getLong(idKey)).toSeq) + } + println(s"Full-rewrite action/physical-name oracle passed: " + + s"removed=${oldPaths.size}, fresh=${newPaths.size}") + + // Every selection is a literal bag, including both copies of each duplicate/null row. + val rows = Seq(Row(1, 10, "duplicate"), Row(1, 10, "duplicate"), + Row(2, null, "nullable"), Row(3, 30, null), Row(null, null, null), Row(null, null, null)) + val reads = Seq( + ("read_before", null, rows.indices, Some(2L), None), + ("read_all", null, rows.indices, None, None), + ("read_rewritten", null, rows.indices, Some(3L), None), + ("read_projection", null, rows.indices, None, Some(Seq("payload", "id"))), + ("read_hit", "value = 10", Seq(0, 1), None, None), + ("read_miss", "value = 999", Seq.empty[Int], None, None), + ("read_null_value", "value IS NULL", Seq(2, 4, 5), None, None), + ("read_null_payload", "payload IS NULL", Seq(3, 4, 5), None, None), + ("read_null_id_projection", "id IS NULL", Seq(4, 5), None, Some(Seq("id", "value")))) + for ((name, predicate, indices, version, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, + version = version.map(Long.box).orNull, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"full rewrite/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"full rewrite/$name literal bag") + assert(captured.schema == oracle.schema) + println(s"Full-rewrite literal row/schema oracle passed: $name") + } + } + for ((version, metadata, reader, writer) <- Seq( + (2L, oldMetadata, 1, 2), (3L, newMetadata, 2, 5))) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.schemaString) + assert(result.metadata.configuration == metadata.configuration) + assert(result.metadata.partitionColumns.isEmpty) + assert(result.protocol.minReaderVersion == reader) + assert(result.protocol.minWriterVersion == writer) + case other => fail(s"full rewrite snapshot v$version must succeed: $other") + } + println(s"Full-rewrite snapshot transition oracle passed: v$version") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala index 061b7d70..06d0b58b 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala @@ -16,9 +16,38 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.{ByteBuffer, ByteOrder} +import java.nio.file.{Files, Path, StandardCopyOption} +import java.sql.Timestamp +import java.time.{Instant, LocalDate} + +import scala.jdk.CollectionConverters._ + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.commons.io.FileUtils +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.example.data.Group +import org.apache.parquet.example.data.simple.{NanoTime, SimpleGroupFactory} +import org.apache.parquet.hadoop.{ParquetFileReader, ParquetReader} +import org.apache.parquet.hadoop.example.{ExampleParquetWriter, GroupReadSupport} +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.parquet.schema.{LogicalTypeAnnotation, PrimitiveType} +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.functions.{col, lit, when} +import org.apache.spark.sql.types.{ArrayType, BooleanType, DataType, DecimalType, IntegerType, + StringType, StructType, TimestampNTZType, TimestampType} +import org.apache.spark.unsafe.types.UTF8String + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.deltaharness.{DeltaHarness, Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} import io.delta.workload.json.JsonUtil -import io.delta.workload.log.AddFile +import io.delta.workload.log.{Action, AddFile, CommitLog, DomainMetadata, Metadata, Protocol, + RemoveFile, Txn} +import io.delta.workload.model._ +import io.delta.workload.tables.DeletionVectorPayloads.payload /** * Data skipping, statistics, and partitioning workloads. @@ -26,9 +55,12 @@ import io.delta.workload.log.AddFile * nested field predicates, boolean predicates, typed stats, null handling, * multiple files for skipping, missing stats, long strings, column mapping stats, * generated columns, DVs, partitioned skipping, partition pruning, projection, - * stats edge cases (null min/max, numRecords-only, truncated strings). + * stats edge cases (null min/max, numRecords-only, truncated strings), predicate row + * semantics (nondeterminism, signed zero, casts), partial/tight/selected statistics, + * timestamp statistic encodings and time zones, stats refresh, and clustering rewrites. */ -class DataSkippingSuite extends WorkloadTestSuite("data_skipping") { +class DataSkippingSuite extends WorkloadTestSuite("data_skipping") + with ScopedSessionConf with LiteralOracleChecks { // === Data Skipping === @@ -1406,4 +1438,3535 @@ class DataSkippingSuite extends WorkloadTestSuite("data_skipping") { snapshotSpec(t) } + // Row semantics for read predicates, without a pruning assertion: nondeterministic + // values, signed zero, explicit casts at precision boundaries and narrowing wrap-around, + // and complex filters. Signed zero needs Parquet filter pushdown disabled and the three + // narrowing casts need ANSI disabled; both are scoped to those tests. Use `startswith`; + // `starts_with` is unsupported. + + for (partitioned <- Seq(false, true)) { + val layout = if (partitioned) "partitioned" else "unpartitioned" + test(s"nondeterministic_values_$layout") { + val partition = if (partitioned) "PARTITIONED BY (p)" else "" + sql(s"CREATE TABLE tbl (id INT, value INT, p INT) USING delta $partition") + sql("INSERT INTO tbl VALUES (1, 10, 1), (2, 20, 1), (2, 20, 1)") + sql("INSERT INTO tbl VALUES (3, 20, 2), (4, 30, 2), (5, 40, 3)") + sql("INSERT INTO tbl VALUES (6, NULL, 1), (7, 50, NULL), (8, NULL, NULL)") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + + // Literal bags are independent of the filtered Delta reads, including duplicate/NULL rows. + val all = Seq( + "(1, 10, 1)", + "(2, 20, 1)", + "(2, 20, 1)", + "(3, 20, 2)", + "(4, 30, 2)", + "(5, 40, 3)", + "(6, NULL, 1)", + "(7, 50, NULL)", + "(8, NULL, NULL)" + ) + val equality = Seq("(2, 20, 1)", "(2, 20, 1)", "(3, 20, 2)") + val range = equality :+ "(4, 30, 2)" + val partitionBag = Seq("(1, 10, 1)", "(2, 20, 1)", "(2, 20, 1)", "(6, NULL, 1)") + val cases = Seq( + ("read_all", null, all), + ("random_all", "rand() >= 0", all), + ("random_none", "rand() < 0", Seq.empty[String]), + ("equality_control", "value = 20", equality), + ("range_control", "value >= 20 AND value < 40", range), + ("and_equality", "rand() >= 0 AND value = 20", equality), + ("and_range", "rand() >= 0 AND (value >= 20 AND value < 40)", range), + ("and_none", "rand() < 0 AND value = 20", Seq.empty[String]), + ("or_equality", "rand() < 0 OR value = 20", equality), + ("or_range", "rand() < 0 OR (value >= 20 AND value < 40)", range), + ("or_all", "rand() >= 0 OR value = 20", all), + ("partition_control", "p = 1", partitionBag), + ("and_partition", "p = 1 AND rand() >= 0", partitionBag), + ("or_partition", "p = 1 OR rand() < 0", partitionBag), + // Spark can simplify the domain-only controls. Referencing the row's small exact INT id + // keeps a nondeterministic expression at read time: adding [0, 1) cannot decrease id. + ("row_random_all", "rand() + id >= id", all), + ("row_random_none", "rand() + id < id", Seq.empty[String]), + ("row_and_equality", "rand() + id >= id AND value = 20", equality), + ("row_and_range", "rand() + id >= id AND (value >= 20 AND value < 40)", range), + ("row_or_equality", "rand() + id < id OR value = 20", equality), + ("row_or_range", "rand() + id < id OR (value >= 20 AND value < 40)", range), + ("row_and_partition", "p = 1 AND rand() + id >= id", partitionBag), + ("row_or_partition", "p = 1 OR rand() + id < id", partitionBag) + ) + + // rand() stays in the serialized read predicate. Its [0, 1) domain gives a stable bag + // across RNG seeds, partition counts and replays; random values are never stored on INSERT. + for ((label, predicate, bag) <- cases) { + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(_) => + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + s"SELECT CAST(id AS INT) AS id, CAST(value AS INT) AS value, " + + s"CAST(p AS INT) AS p FROM VALUES ${bag.mkString(", ")} AS expected(id, value, p)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS INT) AS value, " + + "CAST(NULL AS INT) AS p WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, value INT, p INT")) + require( + result.metadata.partitionColumns == + (if (partitioned) Seq("p") else Seq.empty) + ) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + } + + test("signed_zero") { + setForTest("spark.sql.parquet.filterPushdown", "false") + require( + spark.conf.get("spark.sql.parquet.filterPushdown") == "false", + "The signed-zero bag is validated with Parquet filter pushdown disabled" + ) + sql("CREATE TABLE tbl (id INT, f FLOAT, d DOUBLE) USING delta") + // Separate writes keep the two zero signs and nonzero ranges in distinct files. + sql("""INSERT INTO tbl VALUES + |(1, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE)), + |(2, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE)), + |(2, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE))""".stripMargin) + sql("""INSERT INTO tbl VALUES + |(3, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE)), + |(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE)), + |(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE))""".stripMargin) + sql("INSERT INTO tbl VALUES (5, -1, -1), (6, -2, -2)") + sql("INSERT INTO tbl VALUES (7, 1, 1), (8, 2, 2)") + sql("INSERT INTO tbl VALUES (9, NULL, NULL)") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + + // Literal bags do not filter the source table. IDs distinguish the original zero signs; + // exact duplicates check multiplicity. Row equality need not retain a zero's sign bit. + val zeros = Seq("(1, 0, 0)", "(2, 0, 0)", "(2, 0, 0)", "(3, 0, 0)", "(4, 0, 0)", "(4, 0, 0)") + val negative = Seq("(5, -1, -1)", "(6, -2, -2)") + val positive = Seq("(7, 1, 1)", "(8, 2, 2)") + val nulls = Seq("(9, NULL, NULL)") + val nonzero = negative ++ positive + val all = zeros ++ nonzero ++ nulls + val cases = Seq(("read_all", null, all)) ++ + Seq(("f", "FLOAT"), ("d", "DOUBLE")).flatMap { + case (column, dataType) => + val plus = s"CAST('0.0' AS $dataType)" + val minus = s"CAST('-0.0' AS $dataType)" + val one = s"CAST(1 AS $dataType)" + val negativeOne = s"CAST(-1 AS $dataType)" + val typedNull = s"CAST(NULL AS $dataType)" + val comparisons = for { + (sign, zero) <- Seq(("positive", plus), ("negative", minus)) + (label, operator, bag) <- Seq( + ("eq", "=", zeros), + ("null_safe_eq", "<=>", zeros), + ("ne", "!=", nonzero), + ("lt", "<", negative), + ("le", "<=", negative ++ zeros), + ("gt", ">", positive), + ("ge", ">=", zeros ++ positive) + ) + } yield (s"${column}_${label}_$sign", s"$column $operator $zero", bag) + val inLists = Seq( + ("in_positive", plus, zeros), + ("in_negative", minus, zeros), + ("in_both", s"$plus, $minus", zeros), + ("in_positive_null", s"$plus, $typedNull", zeros), + ("in_negative_null", s"$minus, $typedNull", zeros), + ("in_both_null", s"$plus, $minus, $typedNull", zeros), + ("in_positive_nonzero", s"$plus, $one", zeros :+ "(7, 1, 1)"), + ("in_negative_nonzero", s"$minus, $negativeOne", zeros :+ "(5, -1, -1)"), + ( + "in_both_null_nonzero", + s"$plus, $minus, $typedNull, $negativeOne, $one", + zeros ++ Seq("(5, -1, -1)", "(7, 1, 1)") + ), + ("in_nonzero_null", s"$negativeOne, $one, $typedNull", Seq("(5, -1, -1)", "(7, 1, 1)")), + ("in_null", typedNull, Seq.empty[String]), + ("in_miss", s"CAST(9 AS $dataType), $typedNull", Seq.empty[String]) + ) + comparisons ++ Seq((s"${column}_null_safe_null", s"$column <=> $typedNull", nulls)) ++ + inLists.map { + case (label, literals, bag) => + (s"${column}_$label", s"$column IN ($literals)", bag) + } + } + + for ((label, predicate, bag) <- cases) { + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + require(spark.conf.get("spark.sql.parquet.filterPushdown") == "false") + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + "SELECT CAST(id AS INT) AS id, CAST(f AS FLOAT) AS f, CAST(d AS DOUBLE) AS d " + + s"FROM VALUES ${bag.mkString(", ")} AS expected(id, f, d)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS FLOAT) AS f, " + + "CAST(NULL AS DOUBLE) AS d WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, f FLOAT, d DOUBLE")) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + // Explicit predicate casts, precision boundaries and legacy narrowing. + // Expected IDs are derived independently of the CAST predicates. Source duplicates stay intact. + for ((subcase, sourceType, sourceRows, checks, legacy) <- Seq( + ( + "float_double", + "FLOAT", + Seq( + (1, "-0.1"), + (2, "0.1"), + (2, "0.1"), + (3, "0.10000001"), + (4, "0.2"), + (5, "1.0"), + (6, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6)), + ("read_hit", "CAST(value AS DOUBLE) = CAST('0.10000000149011612' AS DOUBLE)", Seq(2)), + ( + "decimal_spelling_miss", + "CAST(value AS DOUBLE) = CAST('0.1' AS DOUBLE)", + Seq.empty[Int] + ), + ("above_decimal", "CAST(value AS DOUBLE) > CAST('0.1' AS DOUBLE)", Seq(2, 3, 4, 5)), + ( + "adjacent_range", + "CAST(value AS DOUBLE) >= CAST('0.10000000149011612' AS DOUBLE) AND " + + "CAST(value AS DOUBLE) < CAST('0.10000000894069672' AS DOUBLE)", + Seq(2) + ), + ( + "negative_hit", + "CAST(value AS DOUBLE) = CAST('-0.10000000149011612' AS DOUBLE)", + Seq(1) + ), + ("read_miss", "CAST(value AS DOUBLE) = CAST('2' AS DOUBLE)", Seq.empty[Int]) + ), + false + ), + ( + "double_2p53", + "BIGINT", + Seq( + (1, "9007199254740991"), + (2, "9007199254740992"), + (3, "9007199254740993"), + (3, "9007199254740993"), + (4, "9007199254740994"), + (5, "9007199254740995"), + (6, "-9007199254740993"), + (7, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7)), + ("read_hit", "CAST(value AS DOUBLE) = CAST('9007199254740992' AS DOUBLE)", Seq(2, 3)), + ("adjacent_hit", "CAST(value AS DOUBLE) = CAST('9007199254740994' AS DOUBLE)", Seq(4)), + ( + "adjacent_range", + "CAST(value AS DOUBLE) > CAST('9007199254740992' AS DOUBLE) AND " + + "CAST(value AS DOUBLE) < CAST('9007199254740996' AS DOUBLE)", + Seq(4) + ), + ( + "below_boundary", + "CAST(value AS DOUBLE) < CAST('9007199254740992' AS DOUBLE)", + Seq(1, 6) + ), + ( + "read_miss", + "CAST(value AS DOUBLE) = CAST('9007199254740998' AS DOUBLE)", + Seq.empty[Int] + ) + ), + false + ), + ( + "float_2p24", + "INT", + Seq( + (1, "16777215"), + (2, "16777216"), + (3, "16777217"), + (3, "16777217"), + (4, "16777218"), + (5, "16777219"), + (6, "-16777217"), + (7, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7)), + ("read_hit", "CAST(value AS FLOAT) = CAST('16777216' AS FLOAT)", Seq(2, 3)), + ("adjacent_hit", "CAST(value AS FLOAT) = CAST('16777218' AS FLOAT)", Seq(4)), + ( + "adjacent_range", + "CAST(value AS FLOAT) > CAST('16777216' AS FLOAT) AND " + + "CAST(value AS FLOAT) < CAST('16777220' AS FLOAT)", + Seq(4) + ), + ("below_boundary", "CAST(value AS FLOAT) < CAST('16777216' AS FLOAT)", Seq(1, 6)), + ("read_miss", "CAST(value AS FLOAT) = CAST('16777222' AS FLOAT)", Seq.empty[Int]) + ), + false + ), + ( + "long_int_wrap", + "BIGINT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "4294967301"), + (3, "4294967301"), + (4, "4294967300"), + (5, "4294967302"), + (6, "-4294967291"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS INT) = CAST('5' AS INT)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS INT) = CAST('4' AS INT)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS INT) = CAST('6' AS INT)", Seq(5, 9)), + ("below_five", "CAST(value AS INT) < CAST('5' AS INT)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS INT) > CAST('5' AS INT)", Seq(5, 9)), + ("read_miss", "CAST(value AS INT) = CAST('7' AS INT)", Seq.empty[Int]) + ), + true + ), + ( + "to_short_wrap", + "INT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "65541"), + (3, "65541"), + (4, "65540"), + (5, "65542"), + (6, "-65531"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS SHORT) = CAST('5' AS SHORT)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS SHORT) = CAST('4' AS SHORT)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS SHORT) = CAST('6' AS SHORT)", Seq(5, 9)), + ("below_five", "CAST(value AS SHORT) < CAST('5' AS SHORT)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS SHORT) > CAST('5' AS SHORT)", Seq(5, 9)), + ("read_miss", "CAST(value AS SHORT) = CAST('7' AS SHORT)", Seq.empty[Int]) + ), + true + ), + ( + "int_byte_wrap", + "INT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "261"), + (3, "261"), + (4, "260"), + (5, "262"), + (6, "-251"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS BYTE) = CAST('5' AS BYTE)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS BYTE) = CAST('4' AS BYTE)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS BYTE) = CAST('6' AS BYTE)", Seq(5, 9)), + ("below_five", "CAST(value AS BYTE) < CAST('5' AS BYTE)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS BYTE) > CAST('5' AS BYTE)", Seq(5, 9)), + ("read_miss", "CAST(value AS BYTE) = CAST('7' AS BYTE)", Seq.empty[Int]) + ), + true + ) + )) { + test(s"cast_filters_$subcase") { + // ReadQuery has no session config field. Supply the same legacy session on every replay. + if (legacy) { + setForTest("spark.sql.ansi.enabled", "false") + require( + spark.conf.get("spark.sql.ansi.enabled") == "false", + "Narrowing CAST fixtures require a non-ANSI session for capture and replay" + ) + } + sql(s"CREATE TABLE tbl (id INT, value $sourceType) USING delta") + // Separate writes leave ordinary and boundary values in multiple stats-bearing files. + for (batch <- sourceRows.grouped(2)) { + val values = batch.map { case (id, value) => s"($id, $value)" }.mkString(", ") + sql(s"INSERT INTO tbl VALUES $values") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + for ((label, predicate, expectedIds) <- checks) { + val bag = sourceRows.filter { case (id, _) => expectedIds.contains(id) } + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + if (legacy) require(spark.conf.get("spark.sql.ansi.enabled") == "false") + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + // Only source-type casts construct these literal bags; no predicate CAST is reused. + val oracle = if (bag.nonEmpty) { + val values = bag.map { case (id, value) => s"($id, $value)" }.mkString(", ") + s"SELECT CAST(id AS INT) AS id, CAST(value AS $sourceType) AS value " + + s"FROM VALUES $values AS expected(id, value)" + } else { + s"SELECT CAST(NULL AS INT) AS id, CAST(NULL AS $sourceType) AS value WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL(s"id INT, value $sourceType")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + if (subcase == "float_double") { + val compositionCases = Seq( + ("direct_nonnegative", "CAST(value AS DOUBLE) >= 0", Seq(2, 3, 4, 5)), + ( + "compound_hit", + "(CAST(value AS DOUBLE) > 0.15 AND id < 5) OR " + + "(CAST(value AS DOUBLE) < 0 AND id = 1)", + Seq(1, 4) + ), + ( + "compound_miss", + "(CAST(value AS DOUBLE) > 2 AND id < 5) OR " + + "(CAST(value AS DOUBLE) < -2 AND id = 1)", + Seq.empty[Int] + ) + ) + for ((label, predicate, ids) <- compositionCases) { + val bag = sourceRows.filter { case (id, _) => ids.contains(id) } + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + val values = bag.map { case (id, value) => s"($id, $value)" }.mkString(", ") + s"SELECT CAST(id AS INT) AS id, CAST(value AS FLOAT) AS value " + + s"FROM VALUES $values AS expected(id, value)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS FLOAT) AS value WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + } + + } + } + + // Complex filter row semantics. File counts do not prove filter pushdown. + test("complex_filters_month_timestamp") { + // Read specs have no timezone field; use UTC through queued capture and every replay. + require(spark.conf.get("spark.sql.session.timeZone") == "UTC") + sql("CREATE TABLE tbl (id INT, ts TIMESTAMP) USING delta") + val rows = Seq( + "(1, TIMESTAMP'2023-12-31 23:59:59.999999')", + "(2, TIMESTAMP'2024-01-01 00:00:00')", + "(3, TIMESTAMP'2024-01-31 23:59:59.999999')", + "(4, TIMESTAMP'2024-02-01 00:00:00')", + "(5, TIMESTAMP'2024-02-29 23:59:59.999999')", + "(6, TIMESTAMP'2024-03-01 00:00:00')", + "(7, TIMESTAMP'2024-12-31 23:59:59.999999')", + "(8, TIMESTAMP'2025-01-01 00:00:00')", + "(8, TIMESTAMP'2025-01-01 00:00:00')", + "(9, NULL)" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("january", "month(ts) = 1", Seq(2, 3, 8)), + ("february", "month(ts) = 2", Seq(4, 5)), + ("december", "month(ts) = 12", Seq(1, 7)), + ("miss_june", "month(ts) = 6", Seq.empty[Int]), + ("null_month", "month(ts) IS NULL", Seq(9)) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(ts AS TIMESTAMP) AS ts" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, ts)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, ts TIMESTAMP")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + test("complex_filters_strings") { + sql("CREATE TABLE tbl (id INT, name STRING, s STRUCT) USING delta") + val rows = Seq( + "(1, 'apple', named_struct('name', 'apple'))", + "(2, 'banana', named_struct('name', 'banana'))", + "(3, 'alice', named_struct('name', 'alice'))", + "(4, 'bob', named_struct('name', 'bob'))", + "(5, 'charlie', named_struct('name', 'charlie'))", + "(6, 'diana', named_struct('name', 'diana'))", + "(7, 'Apple', named_struct('name', 'APPLE'))", + "(8, 'ALICE', named_struct('name', 'Alice'))", + "(8, 'ALICE', named_struct('name', 'Alice'))", + "(9, '', named_struct('name', ''))", + "(10, NULL, named_struct('name', NULL))", + "(11, 'apPle', named_struct('name', NULL))" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)), + ("prefix_hit", "startswith(name, 'app')", Seq(1)), + ("prefix_a", "startswith(name, 'a')", Seq(1, 3, 11)), + ("prefix_nested", "startswith(s.name, 'a')", Seq(1, 3)), + ("prefix_miss", "startswith(name, 'z')", Seq.empty[Int]), + ("prefix_empty", "startswith(name, '')", Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 11)), + ("prefix_null", "startswith(name, CAST(NULL AS STRING))", Seq.empty[Int]), + ("prefix_null_result", "startswith(name, 'a') IS NULL", Seq(10)), + ("upper_alice", "upper(name) = 'ALICE'", Seq(3, 8)), + ("upper_apple", "upper(name) = 'APPLE'", Seq(1, 7, 11)), + ("upper_miss", "upper(name) = 'ZEBRA'", Seq.empty[Int]), + ("upper_null", "upper(name) IS NULL", Seq(10)), + ("upper_selective", "upper(name) = 'APPLE' AND id > 1", Seq(7, 11)) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(name AS STRING) AS name, " + + "CAST(s AS STRUCT) AS s" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, name, s)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, name STRING, s STRUCT")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + test("complex_filters_deeply_nested") { + sql( + "CREATE TABLE tbl (id INT, s STRUCT) USING delta" + ) + val rows = Seq( + "(1, named_struct('a', 1, 'b', 1, 'c', NULL, 'd', NULL, 'e', NULL, 'f', NULL, " + + "'g', NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(2, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 1, 'e', NULL, 'f', NULL, 'g', " + + "NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(3, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 1, 'g', NULL, " + + "'h', NULL, 'i', NULL, 'j', NULL))", + "(4, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "1, 'i', NULL, 'j', NULL))", + "(5, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 0))", + "(5, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 0))", + "(6, named_struct('a', 0, 'b', 1, 'c', 0, 'd', 1, 'e', 1, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))", + "(7, named_struct('a', 1, 'b', 0, 'c', 1, 'd', 1, 'e', 1, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))", + "(8, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 1))", + "(9, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', NULL))", + "(10, named_struct('a', NULL, 'b', 1, 'c', 0, 'd', 1, 'e', 1, 'f', 1, 'g', 0, " + + "'h', 1, 'i', 1, 'j', 0))", + "(11, named_struct('a', NULL, 'b', NULL, 'c', NULL, 'd', NULL, 'e', NULL, 'f', " + + "NULL, 'g', NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(12, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 0, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)), + ( + "deep_hit", + "s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))", + Seq(1, 2, 3, 4, 5) + ), + ( + "deep_false", + "NOT (s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND " + + "(s.f = 1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1))))))))))", + Seq(6, 7, 8, 12) + ), + ( + "deep_null", + "(s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))) IS " + + "NULL", + Seq(9, 10, 11) + ), + ( + "deep_miss", + "(s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))) AND " + + "id = 99", + Seq.empty[Int] + ) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(s AS STRUCT) AS s" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, s)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require( + schema == StructType.fromDDL( + "id INT, s STRUCT" + ) + ) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + // Incomplete or selected file statistics. + test("missing_add_stats_predicates") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("flag", BooleanType).add("items", ArrayType(IntegerType, containsNull = true)) + val nullArray = Row(1, 1, true, null) + val emptyArray = Row(2, 2, false, Seq.empty[Integer]) + val nullElement = Row(3, null, null, Seq[Integer](null)) + val duplicate = Row(4, 3, true, Seq[Integer](1, 7)) + val nonmatching = Row(5, 4, false, Seq[Integer](2, 3)) + val matchingWithNull = Row(6, 5, null, Seq[Integer](7, null)) + val nonmatchingWithNull = Row(7, 2, true, Seq[Integer](null, 2)) + val matching = Row(8, null, false, Seq[Integer](7)) + val batches = Seq(Seq(nullArray, emptyArray, nullElement, duplicate, duplicate), + Seq(nonmatching, matchingWithNull, nonmatchingWithNull, matching)) + val rows = batches.flatten + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT, flag BOOLEAN, items ARRAY) USING delta + TBLPROPERTIES ('delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '100')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 1, TRUE, CAST(NULL AS ARRAY)), (2, 2, FALSE, CAST(array() AS ARRAY)), + (3, NULL, NULL, array(CAST(NULL AS INT))), (4, 3, TRUE, array(1, 7)), + (4, 3, TRUE, array(1, 7)) AS v(id, value, flag, items)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (5, 4, FALSE, array(2, 3)), (6, 5, NULL, array(7, CAST(NULL AS INT))), + (7, 2, TRUE, array(CAST(NULL AS INT), 2)), (8, NULL, FALSE, array(7)) + AS v(id, value, flag, items)""") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val metadata = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + batches.zipWithIndex.foreach { case (batch, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.stats.isDefined) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"predicate_input_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + // Arrays have no scalar min/max statistics; their predicates require residual filtering. + require(!stats.path("minValues").has("items") && !stats.path("maxValues").has("items")) + } + val stream = Files.list(dir.resolve("_delta_log")) + val names = try stream.iterator().asScala.map(_.getFileName.toString).toVector + finally stream.close() + require(!names.exists(_.contains("checkpoint"))) + // A cached allFiles set in a version CRC must not hide the missing AddFile statistics. + names.filter(_.matches("[0-9]{20}\\.crc")) + .foreach(name => Files.delete(dir.resolve("_delta_log").resolve(name))) + } + for (version <- 1L to 2L) { + modifyCommitActions(t, version) { actions => + actions.map { + case a: AddFile => a.copy(stats = None) + case other => other + } + } + } + mutateTable(t) { dir => + val actions = (0L to 2L).flatMap(v => CommitLog.read(dir, v)) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 2 && adds.map(_.path).distinct.size == 2) + require(adds.forall(_.stats.isEmpty), "Every persisted AddFile must omit statistics") + require(!actions.exists(_.isInstanceOf[RemoveFile])) + } + + // Expected bags come from literal values and SQL three-valued logic, without a Delta scan. + val casePredicate = "CASE WHEN value IN (1, 3) THEN TRUE " + + "WHEN value = 2 THEN FALSE ELSE CAST(NULL AS BOOLEAN) END" + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("ne", "value <> 2", Seq(nullArray, duplicate, duplicate, nonmatching, matchingWithNull)), + ("in", "value IN (1, 3, 99)", Seq(nullArray, duplicate, duplicate)), + ("in_with_null", "value IN (2, NULL)", Seq(emptyArray, nonmatchingWithNull)), + ("in_miss", "value IN (99, NULL)", Seq.empty), + ("is_not_true", "flag IS NOT TRUE", + Seq(emptyArray, nullElement, nonmatching, matchingWithNull, matching)), + ("array_contains", "array_contains(items, 7)", + Seq(duplicate, duplicate, matchingWithNull, matching)), + ("array_contains_miss", "array_contains(items, 99)", Seq.empty), + ("array_contains_is_not_true", "array_contains(items, 7) IS NOT TRUE", + Seq(nullArray, emptyArray, nullElement, nonmatching, nonmatchingWithNull)), + ("array_contains_not", "NOT array_contains(items, 7)", Seq(emptyArray, nonmatching)), + ("case_when", casePredicate, Seq(nullArray, duplicate, duplicate)), + ("case_when_is_not_true", s"($casePredicate) IS NOT TRUE", + Seq(emptyArray, nullElement, nonmatching, matchingWithNull, nonmatchingWithNull, matching))) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful missing-stat $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_missing_stats_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful missing-stat snapshot, got $other") + } + } + } + + for (variant <- Seq("no_null_count", "no_num_records", "neither_count", "partial_null_count")) { + test(s"partial_counts_$variant") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + val batches = Seq( + Seq(Row(1, 10), Row(2, null), Row(3, 30), Row(3, 30)), + Seq(Row(4, 40), Row(5, 50)), + Seq(Row(6, null), Row(7, null)), + Seq(Row(8, 80), Row(9, null))) + val rows = batches.flatten + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + // One mixed/all-null/non-null file per commit; duplicate rows must survive reads. + val values = batch.map { r => + s"(${r.getInt(0)}, ${if (r.isNullAt(1)) "NULL" else r.getInt(1).toString})" + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS v(id, value)") + } + val t = registerTable("tbl") + var copiedTable: Path = null + + // Check real AddFiles and their row-derived stats, then mutate only the copied table. + mutateTable(t) { dir => + copiedTable = dir + val initial = CommitLog.read(dir, 0) + val originalMetadata = initial.collect { case m: Metadata => m }.head + require(DataType.fromJson(originalMetadata.schemaString) == schema) + require(originalMetadata.configuration == properties) + require(originalMetadata.partitionColumns.isEmpty) + val logFiles = Files.list(dir.resolve("_delta_log")) + try { + val names = logFiles.iterator().asScala.map(_.getFileName.toString).toSeq + require(!names.exists(_.contains("checkpoint")), "Commit replay must be required") + // Version CRCs can contain a cached allFiles set with the original stats. + names.filter(_.matches("[0-9]{20}\\.crc")).foreach { name => + Files.delete(dir.resolve("_delta_log").resolve(name)) + } + } finally logFiles.close() + + batches.zipWithIndex.foreach { case (batch, index) => + val version = index.toLong + 1 + val actions = CommitLog.read(dir, version) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 1, s"Expected one AddFile at version $version") + val add = adds.head + require(add.deletionVector.isEmpty && add.baseRowId.isEmpty && + add.defaultRowCommitVersion.isEmpty) + val oracle = spark.createDataFrame(batch.asJava, schema) + RowComparison.assertRowsEqual(oracle, + spark.read.parquet(dir.resolve(add.path).toString), s"input_v$version") + val stats = JsonUtil.mapper.readTree(add.stats.get).asInstanceOf[ObjectNode] + require(stats.path("numRecords").asLong(-1) == batch.size) + for ((column, ordinal) <- Seq("id" -> 0, "value" -> 1)) { + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + if (values.nonEmpty) { + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + } + val before = stats.deepCopy() + // Keep the last file intact as a control for per-file partial statistics. + if (version < 4) { + variant match { + case "no_null_count" => stats.remove("nullCount") + case "no_num_records" => stats.remove("numRecords") + case "neither_count" => + stats.remove("nullCount") + stats.remove("numRecords") + case "partial_null_count" => + if (version == 3) stats.remove("nullCount") + else stats.get("nullCount").asInstanceOf[ObjectNode] + .remove(if (version == 1) "value" else "id") + } + require(stats != before, s"Mutation must change stats at version $version") + } else require(stats == before) + require(stats.get("minValues") == before.get("minValues")) + require(stats.get("maxValues") == before.get("maxValues")) + val changed = add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + CommitLog.write(dir, version, actions.map { + case _: AddFile => changed + case other => other + }) + val reread = CommitLog.read(dir, version).collect { case a: AddFile => a } + require(reread == Seq(changed), s"Persisted mutation mismatch at version $version") + require(JsonUtil.mapper.readTree(reread.head.stats.get) == stats) + } + } + + // These bags are derived directly from the literal input, without a Delta scan or stats. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("value_is_null", "value IS NULL", rows.filter(_.isNullAt(1))), + ("value_is_not_null", "value IS NOT NULL", rows.filterNot(_.isNullAt(1))), + ("value_eq_30", "value = 30", Seq(Row(3, 30), Row(3, 30))), + ("value_gt_35", "value > 35", Seq(Row(4, 40), Row(5, 50), Row(8, 80))), + ("value_eq_999", "value = 999", Seq.empty), + ("id_is_null", "id IS NULL", Seq.empty), + ("id_is_not_null", "id IS NOT NULL", rows)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful snapshot, got $other") + } + } + } + } + + test("remove_stats_without_active_add_stats") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + val batches = Seq( + Seq(Row(1, 10), Row(5, null)), + Seq(Row(2, 20), Row(3, 30), Row(3, 30), Row(4, null)), + Seq(Row(100, 1000), Row(101, 1010))) + val rewrittenRows = Seq(Row(3, 30), Row(3, 30), Row(4, null)) + val survivors = batches.head ++ rewrittenRows + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { r => + s"(${r.getInt(0)}, ${if (r.isNullAt(1)) "NULL" else r.getInt(1).toString})" + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS v(id, value)") + } + // Rewrite one file, remove another entirely, and leave the first file untouched. + sql("DELETE FROM tbl WHERE id = 2 OR id >= 100") + val t = registerTable("tbl") + var copiedTable: Path = null + + mutateTable(t) { dir => + copiedTable = dir + val commits = (0L to 4L).map(v => v -> CommitLog.read(dir, v)).toMap + val originalMetadata = commits(0).collect { case m: Metadata => m }.head + require(DataType.fromJson(originalMetadata.schemaString) == schema) + require(originalMetadata.configuration == properties) + require(originalMetadata.partitionColumns.isEmpty) + require(commits(0).collect { case a: AddFile => a }.isEmpty) + val logFiles = Files.list(dir.resolve("_delta_log")) + try { + val names = logFiles.iterator().asScala.map(_.getFileName.toString).toSeq + require(!names.exists(_.contains("checkpoint")), "Commit replay must be required") + // Prevent a version CRC's cached allFiles from hiding the stats mutation. + names.filter(_.matches("[0-9]{20}\\.crc")).foreach { name => + Files.delete(dir.resolve("_delta_log").resolve(name)) + } + } finally logFiles.close() + + val inputAdds = batches.zipWithIndex.map { case (batch, index) => + val adds = commits(index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1, s"Expected one input AddFile for batch $index") + require(!commits(index.toLong + 1).exists(_.isInstanceOf[RemoveFile])) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"original_batch_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for ((column, ordinal) <- Seq("id" -> 0, "value" -> 1)) { + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + add + } + val allAdds = commits.values.toSeq.flatten.collect { case a: AddFile => a } + val addsByPath = allAdds.map(a => a.path -> a).toMap + require(addsByPath.size == allAdds.size, "Each path must have one original Add") + require(allAdds.forall(a => a.stats.isDefined && a.partitionValues.isEmpty && + a.deletionVector.isEmpty && a.baseRowId.isEmpty && a.defaultRowCommitVersion.isEmpty)) + val removes = commits(4).collect { case r: RemoveFile => r } + val removedPaths = removes.map(_.path).toSet + require(removes.size == 2 && removedPaths == inputAdds.tail.map(_.path).toSet) + removes.foreach { r => + val original = addsByPath(r.path) + require(r.dataChange && r.deletionTimestamp.isDefined) + require(r.extendedFileMetadata.contains(true)) + require(r.partitionValues.contains(original.partitionValues)) + require(r.size.contains(original.size) && r.deletionVector.isEmpty) + require(r.baseRowId.isEmpty && r.defaultRowCommitVersion.isEmpty) + require(r.stats.forall(s => JsonUtil.mapper.readTree(s) == + JsonUtil.mapper.readTree(original.stats.get))) + } + val rewritten = commits(4).collect { case a: AddFile => a } + require(rewritten.nonEmpty) + RowComparison.assertRowsEqual(spark.createDataFrame(rewrittenRows.asJava, schema), + rewritten.map(a => spark.read.parquet(dir.resolve(a.path).toString)).reduce(_.union(_)), + "rewritten_survivors") + val activePaths = addsByPath.keySet -- removedPaths + require(activePaths == (rewritten.map(_.path).toSet + inputAdds.head.path)) + + for (version <- 1L to 4L) { + val changed = commits(version).map { + case a: AddFile if activePaths.contains(a.path) => a.copy(stats = None) + // Some writers omit Remove stats. Supply the exact, row-checked original Add stats. + case r: RemoveFile => r.copy(stats = addsByPath(r.path).stats) + case other => other + } + CommitLog.write(dir, version, changed) + require(CommitLog.read(dir, version) == changed, + s"Persisted action mutation mismatch at version $version") + } + val persisted = (0L to 4L).flatMap(v => CommitLog.read(dir, v)) + val persistedAdds = persisted.collect { case a: AddFile => a } + val persistedRemoves = persisted.collect { case r: RemoveFile => r } + require(persistedAdds.filter(a => activePaths.contains(a.path)).forall(_.stats.isEmpty)) + require(persistedAdds.filter(a => removedPaths.contains(a.path)).forall { a => + a == addsByPath(a.path) + }) + require(persistedRemoves.size == 2 && persistedRemoves.forall { r => + r.stats.isDefined && r.stats == addsByPath(r.path).stats + }) + } + + // Literal row bags, independent of Delta replay and statistics; duplicates are intentional. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, survivors), + ("hit_untouched", "id = 1", Seq(Row(1, 10))), + ("hit_rewritten", "id = 3", Seq(Row(3, 30), Row(3, 30))), + ("range", "value >= 30", Seq(Row(3, 30), Row(3, 30))), + ("null", "value IS NULL", Seq(Row(5, null), Row(4, null))), + ("miss", "id = 999", Seq.empty), + ("deleted_partial", "id = 2", Seq.empty), + ("deleted_file", "id >= 100", Seq.empty), + ("mixed", "id IN (2, 3, 100)", Seq(Row(3, 30), Row(3, 30)))) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful snapshot, got $other") + } + } + } + + test("unicode_string_stats_predicates") { + val schema = new StructType().add("id", IntegerType).add("s", StringType) + val head = "a" * 31 + // Exact source rows from WriteStatsSuite.utf8_boundary_string_min_max; no NULL rows. + // This independent read fixture avoids depending on another suite's generated output. + val rows = Seq(Row(1, head + "étail"), Row(2, head + "中tail"), Row(3, head + "😀")) + sql("CREATE TABLE tbl (id INT, s STRING) USING delta") + rows.foreach { row => + // Separate files give the three Unicode boundaries distinct min/max statistics. + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (${row.getInt(0)}, '${row.getString(1)}') AS v(id, s)""") + } + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + rows.zipWithIndex.foreach { case (row, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(Seq(row).asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"unicode_input_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == 1) + require(stats.path("nullCount").path("s").asLong(-1) == 0) + val value = row.getString(1) + val minimum = stats.path("minValues").path("s").asText() + val maximum = stats.path("maxValues").path("s").asText() + require(minimum.startsWith(head) && minimum.codePointAt(31) == value.codePointAt(31)) + require(UTF8String.fromString(minimum).compareTo(UTF8String.fromString(value)) <= 0) + require(UTF8String.fromString(value).compareTo(UTF8String.fromString(maximum)) <= 0) + } + } + + // Literal bags use UTF-8 binary ordering: e-acute < CJK < supplementary emoji. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("equal_2byte", s"s = '${head}étail'", Seq(rows(0))), + ("equal_3byte", s"s = '${head}中tail'", Seq(rows(1))), + ("equal_4byte", s"s = '${head}😀'", Seq(rows(2))), + ("range_2_to_3byte", s"s >= '${head}étail' AND s < '${head}中tail'", Seq(rows(0))), + ("range_3_to_4byte", s"s >= '${head}中tail' AND s < '${head}😀'", Seq(rows(1))), + ("range_from_4byte", s"s >= '${head}😀'", Seq(rows(2))), + ("range_through_4byte", s"s > '${head}étail' AND s <= '${head}😀'", rows.tail), + ("miss_same_prefix", s"s = '${head}étail_missing'", Seq.empty), + ("miss_above_4byte", s"s > '${head}😀'", Seq.empty), + ("is_null", "s IS NULL", Seq.empty), + ("is_not_null", "s IS NOT NULL", rows)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful Unicode $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_unicode_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + case other => fail(s"Expected successful Unicode snapshot, got $other") + } + } + } + + for (bounds <- Seq("true", "false", "absent")) { + test(s"tight_bounds_$bounds") { + val schema = new StructType().add("id", IntegerType).add("all_null", IntegerType) + .add("mixed_null", IntegerType).add("non_null", IntegerType) + val physical = Seq(Row(1, null, 10, 10), Row(2, null, null, 20), + Row(3, null, 30, 30), Row(3, null, 30, 30), Row(4, null, null, 40)) + val live = Seq(Row(2, null, null, 20), Row(3, null, 30, 30), Row(3, null, 30, 30)) + val control = physical.map(r => Row(r.getInt(0) + 100, r.get(1), r.get(2), r.get(3))) + // Existing cases cover non-DV partial counts; explicit bounds need controls here. + val batches = Seq.fill(3)(physical) ++ + (if (bounds == "absent") Seq.empty else Seq.fill(2)(control)) + val rows = Seq.fill(3)(live).flatten ++ + (if (bounds == "absent") Seq.empty else Seq.fill(2)(control).flatten) + val properties = Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, all_null INT, mixed_null INT, non_null INT) + USING delta TBLPROPERTIES ('delta.enableDeletionVectors' = 'true', + 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { row => + row.toSeq.map(v => if (v == null) "NULL" else v.toString).mkString("(", ", ", ")") + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values " + + "AS v(id, all_null, mixed_null, non_null)") + } + sql("DELETE FROM tbl WHERE id = 1 OR id = 4") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val lastVersion = batches.size.toLong + 1 + val commits = (0L to lastVersion).map(v => v -> CommitLog.read(dir, v)).toMap + val metadata = commits(0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + val originals = batches.indices.map { index => + val adds = commits(index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batches(index).asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"physical_$bounds/$index") + add + } + val deleted = commits(lastVersion) + val dvAdds = deleted.collect { case a: AddFile => a } + val removes = deleted.collect { case r: RemoveFile => r } + require(dvAdds.size == 3 && removes.size == 3) + require(dvAdds.map(_.path).toSet == originals.take(3).map(_.path).toSet) + require(removes.map(_.path).toSet == dvAdds.map(_.path).toSet) + require(dvAdds.forall(a => a.deletionVector.exists(_.cardinality == 2))) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, schema), + spark.read.format("delta").load(dir.toString), s"live_before_mutation_$bounds") + + val stream = Files.list(dir.resolve("_delta_log")) + val files = try stream.iterator().asScala.toVector finally stream.close() + require(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.matches("[0-9]{20}\\.crc")).foreach(Files.delete) + for (version <- 1L to lastVersion) { + val changed = commits(version).map { + case add: AddFile if add.deletionVector.isDefined || + originals.drop(3).exists(_.path == add.path) => + val index = originals.indexWhere(_.path == add.path) + require(index >= 0 && add.partitionValues.isEmpty) + val valid = if (index < 3) live else control + val basis = if (bounds == "false") batches(index) else valid + val stats = JsonUtil.mapper.createObjectNode() + // DV numRecords is always physical, including when tight null counts are live. + stats.put("numRecords", batches(index).size) + if (bounds != "absent") stats.put("tightBounds", bounds == "true") + val minimum = stats.putObject("minValues") + val maximum = stats.putObject("maxValues") + val nulls = stats.putObject("nullCount") + schema.fieldNames.zipWithIndex.foreach { case (column, ordinal) => + val values = basis.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + if (values.nonEmpty) { + minimum.put(column, values.min) + maximum.put(column, values.max) + } + nulls.put(column, basis.count(_.isNullAt(ordinal))) + } + index match { + case 1 => stats.remove("nullCount") + case 2 => + nulls.remove("all_null") + nulls.remove("mixed_null") + case 3 => stats.remove("numRecords") + case 4 => + stats.remove("numRecords") + stats.remove("nullCount") + case _ => + } + require(stats.has("tightBounds") == (bounds != "absent")) + require(stats.has("numRecords") == (index < 3)) + if (index < 3) require(stats.path("numRecords").asInt() == 5) + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + CommitLog.write(dir, version, changed) + require(CommitLog.read(dir, version) == changed, + s"Persisted bounds mutation mismatch at version $version") + } + } + + // SQL three-valued predicates applied to literal row bags, independent of file stats. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("all_null", "all_null IS NULL", rows), + ("all_null_not_null", "all_null IS NOT NULL", Seq.empty), + ("all_null_equal", "all_null = 10", Seq.empty), + ("mixed_null", "mixed_null IS NULL", rows.filter(_.isNullAt(2))), + ("mixed_not_null", "mixed_null IS NOT NULL", rows.filterNot(_.isNullAt(2))), + ("mixed_equal", "mixed_null = 30", rows.filter(r => !r.isNullAt(2) && + r.getInt(2) == 30)), + ("non_null_is_null", "non_null IS NULL", Seq.empty), + ("non_null_not_null", "non_null IS NOT NULL", rows), + ("surviving_duplicates", "id = 3", Seq.fill(3)(live.tail).flatten), + ("deleted_lower_bound", "id = 1", Seq.empty), + ("deleted_upper_bound", "id = 4", Seq.empty), + ("deleted_mixed_value", "id < 100 AND mixed_null = 10", Seq.empty), + ("surviving_range", "non_null >= 20 AND non_null <= 30", + rows.filter(r => r.getInt(3) >= 20 && r.getInt(3) <= 30)), + ("outside_range", "non_null > 40", Seq.empty)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $bounds/$name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$bounds/$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful $bounds snapshot, got $other") + } + } + } + } + + test("indexed_prefix_beyond_32") { + val columns = (0 until 36).map(i => f"c$i%02d") + val schema = columns.foldLeft(new StructType())((s, c) => s.add(c, IntegerType)) + // The first 32 fields are a fixed prefix; c32/c33 are selected, c34/c35 are not. + val prefix = 0 until 32 + val duplicate = Row.fromSeq(prefix ++ Seq(320, 330, 340, 350)) + val mixed = Row.fromSeq(prefix ++ Seq(321, 331, 341, 351)) + val nullLate = Row.fromSeq(prefix ++ Seq(null, null, null, null)) + val batches = Seq(Seq(duplicate, duplicate), Seq(mixed, nullLate)) + val rows = batches.flatten + val properties = Map("delta.dataSkippingNumIndexedCols" -> "34", + "delta.enableDeletionVectors" -> "false", "delta.checkpointInterval" -> "100") + val ddl = columns.map(c => s"$c INT").mkString(", ") + sql(s"""CREATE TABLE tbl ($ddl) USING delta TBLPROPERTIES ( + 'delta.dataSkippingNumIndexedCols' = '34', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { row => + row.toSeq.map(v => if (v == null) "NULL" else v.toString).mkString("(", ", ", ")") + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values " + + s"AS v(${columns.mkString(", ")})") + } + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val metadata = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + batches.zipWithIndex.foreach { case (batch, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"wide_input_$index") + // Verify the writer's physical AddFile stats, without assuming a reader's stats schema. + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + require(stats.path(field).fieldNames().asScala.toSet == columns.take(34).toSet) + } + columns.take(34).zipWithIndex.foreach { case (column, ordinal) => + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + } + } + + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("selected_c32_equal", "c32 = 320", Seq(duplicate, duplicate)), + ("selected_c33_equal", "c33 = 330", Seq(duplicate, duplicate)), + ("selected_c33_range", "c33 >= 330 AND c33 < 331", Seq(duplicate, duplicate)), + ("selected_c33_null", "c33 IS NULL", Seq(nullLate)), + ("selected_c33_miss", "c33 = 999", Seq.empty), + ("unselected_c34_equal", "c34 = 340", Seq(duplicate, duplicate)), + ("unselected_c34_range", "c34 > 340", Seq(mixed)), + ("unselected_c34_null", "c34 IS NULL", Seq(nullLate)), + ("unselected_c34_miss", "c34 = 999", Seq.empty)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful wide $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_wide_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful wide snapshot, got $other") + } + } + } + + test("alter_selected_stats_columns") { + val schema = new StructType().add("event_ts", TimestampType) + .add("customer", new StructType().add("country", StringType).add("segment", StringType)) + .add("amount", DecimalType(10, 2)).add("note", StringType) + val oldDuplicate = Row(Timestamp.from(Instant.parse("2024-01-01T00:00:00Z")), + Row("US", "retail"), BigDecimal("10.50").bigDecimal, "alpha") + val oldOther = Row(Timestamp.from(Instant.parse("2024-01-02T00:00:00Z")), + Row("CA", "wholesale"), BigDecimal("20.00").bigDecimal, "beta") + val newDuplicate = Row(Timestamp.from(Instant.parse("2024-01-03T00:00:00Z")), + Row("FR", "retail"), BigDecimal("30.25").bigDecimal, "gamma") + val newOther = Row(Timestamp.from(Instant.parse("2024-01-04T00:00:00Z")), + Row("US", "business"), BigDecimal("-5.00").bigDecimal, "alpha") + val nullRow = Row(null, Row(null, null), null, null) + val before = Seq(oldDuplicate, oldDuplicate, oldOther) + val after = Seq(newDuplicate, newDuplicate, newOther, nullRow) + val rows = before ++ after + val property = "delta.dataSkippingStatsColumns" + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100", property -> "note,customer.segment") + val selected = "event_ts,customer.country,amount" + sql("""CREATE TABLE tbl (event_ts TIMESTAMP, customer STRUCT, + amount DECIMAL(10, 2), note STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100', + 'delta.dataSkippingStatsColumns' = 'note,customer.segment')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (TIMESTAMP '2024-01-01T00:00:00Z', named_struct('country','US','segment','retail'), + 10.50, 'alpha'), + (TIMESTAMP '2024-01-01T00:00:00Z', named_struct('country','US','segment','retail'), + 10.50, 'alpha'), + (TIMESTAMP '2024-01-02T00:00:00Z', named_struct('country','CA','segment','wholesale'), + 20.00, 'beta') AS v(event_ts, customer, amount, note)""") + sql(s"ALTER TABLE tbl SET TBLPROPERTIES ('$property' = '$selected')") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (TIMESTAMP '2024-01-03T00:00:00Z', named_struct('country','FR','segment','retail'), + 30.25, 'gamma'), + (TIMESTAMP '2024-01-03T00:00:00Z', named_struct('country','FR','segment','retail'), + 30.25, 'gamma'), + (TIMESTAMP '2024-01-04T00:00:00Z', named_struct('country','US','segment','business'), + -5.00, 'alpha'), + (NULL, named_struct('country',NULL,'segment',NULL), NULL, NULL) + AS v(event_ts, customer, amount, note)""") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val initial = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + val changed = CommitLog.read(dir, 2) + val metadata = changed.collect { case m: Metadata => m }.head + require(initial.configuration == properties) + require(metadata.configuration == properties.updated(property, selected)) + require(initial.id == metadata.id && initial.schemaString == metadata.schemaString) + require(DataType.fromJson(metadata.schemaString) == schema) + require(initial.partitionColumns.isEmpty && metadata.partitionColumns.isEmpty) + require(!changed.exists(a => a.isInstanceOf[AddFile] || a.isInstanceOf[RemoveFile]), + "ALTER must change metadata without rewriting files or backfilling statistics") + for ((version, batch) <- Seq(1L -> before, 3L -> after)) { + val actions = CommitLog.read(dir, version) + require(!actions.exists(_.isInstanceOf[RemoveFile])) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"selected_stats_input_v$version") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + val expectedFields = if (version == 1) Set("customer", "note") + else Set("event_ts", "customer", "amount") + require(stats.path(field).fieldNames().asScala.toSet == expectedFields) + require(stats.path(field).path("customer").fieldNames().asScala.toSet == + (if (version == 1) Set("segment") else Set("country"))) + } + } + } + val queries = Seq[(String, java.lang.Long, String, Seq[Row])]( + ("before_all", 1L, null, before), + ("before_timestamp", 1L, "event_ts = TIMESTAMP '2024-01-01T00:00:00Z'", + Seq(oldDuplicate, oldDuplicate)), + ("all", null, null, rows), + ("selected_timestamp_equal", null, "event_ts = TIMESTAMP '2024-01-01T00:00:00Z'", + Seq(oldDuplicate, oldDuplicate)), + ("selected_timestamp_range", null, "event_ts >= TIMESTAMP '2024-01-03T00:00:00Z'", + Seq(newDuplicate, newDuplicate, newOther)), + ("selected_country_equal", null, "customer.country = 'US'", + Seq(oldDuplicate, oldDuplicate, newOther)), + ("selected_country_null", null, "customer.country IS NULL", Seq(nullRow)), + ("selected_amount_equal", null, "amount = 30.25", Seq(newDuplicate, newDuplicate)), + ("selected_amount_range", null, "amount >= 10.50 AND amount <= 20.00", before), + ("selected_amount_null", null, "amount IS NULL", Seq(nullRow)), + ("selected_country_miss", null, "customer.country = 'ZZ'", Seq.empty), + ("excluded_note_equal", null, "note = 'alpha'", Seq(oldDuplicate, oldDuplicate, newOther)), + ("excluded_segment_equal", null, "customer.segment = 'retail'", + Seq(oldDuplicate, oldDuplicate, newDuplicate, newDuplicate)), + ("excluded_note_null", null, "note IS NULL", Seq(nullRow)), + ("selected_combined", null, + "event_ts >= TIMESTAMP '2024-01-03T00:00:00Z' AND " + + "customer.country = 'FR' AND amount = 30.25", Seq(newDuplicate, newDuplicate))) + queries.foreach { case (name, version, predicate, expectedRows) => + readSpec(t, version = version, predicate = predicate, name = Some(s"read_$name")) + .assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful selected-stats $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_selected_stats_$name") + } + } + for (version <- Seq[java.lang.Long](1L, null)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + // This asserts the table schema and properties, not Kernel's internal stats schema. + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == + (if (version == null) properties.updated(property, selected) else properties)) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful selected-stats snapshot, got $other") + } + } + } + } + + // One checkpoint matrix serves both missing evolved-column stats and nullable evolution. + // Commit/time-travel coverage stays in DataSkippingSuite and SchemaEvolutionSuite. + for (parsed <- Seq(false, true)) { + val variant = if (parsed) "parsed" else "json_only" + test(s"nullable_evolution_checkpoint_$variant") { + val schema = new StructType().add("id", IntegerType).add("part", IntegerType) + .add("value", IntegerType) + val oldRows = Seq(Row(1, 0, null), Row(2, 1, null), Row(2, 1, null)) + val newRows = Seq(Row(3, 0, 30), Row(3, 0, 30), Row(4, 1, null), Row(5, 1, 50)) + val rows = oldRows ++ newRows + sql(s"""CREATE TABLE tbl (id INT, part INT) USING delta PARTITIONED BY (part) + TBLPROPERTIES ('delta.checkpointInterval' = '100', + 'delta.enableDeletionVectors' = 'false', + 'delta.checkpoint.writeStatsAsJson' = '${!parsed}', + 'delta.checkpoint.writeStatsAsStruct' = '$parsed')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 0), (2, 1), (2, 1) AS v(id, part)""") + sql("ALTER TABLE tbl ADD COLUMN value INT") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (3, 0, 30), (3, 0, 30), (4, 1, NULL), (5, 1, 50) AS v(id, part, value)""") + forceCheckpoint("tbl") + val t = registerTable("tbl") + var copiedTable: Path = null + + mutateTable(t) { dir => + copiedTable = dir + val commits = (0L to 3L).map(v => v -> CommitLog.read(dir, v)).toMap + require(!commits.values.flatten.exists(_.isInstanceOf[RemoveFile])) + val evolved = commits(2).collect { case m: Metadata => m }.head + require(DataType.fromJson(evolved.schemaString) == schema) + require(evolved.partitionColumns == Seq("part")) + val oldAdds = commits(1).collect { case a: AddFile => a } + val newAdds = commits(3).collect { case a: AddFile => a } + val adds = oldAdds ++ newAdds + require(oldAdds.size == 2 && newAdds.size == 2) + require(adds.map(_.path).distinct.size == 4, "All four files must remain active") + for ((batch, files) <- Seq(oldRows -> oldAdds, newRows -> newAdds)) { + require(files.map(_.partitionValues).toSet == + Set(Map("part" -> "0"), Map("part" -> "1"))) + files.foreach { add => + val part = add.partitionValues("part").toInt + val raw = spark.read.parquet(dir.resolve(add.path).toString) + require(raw.columns.contains("value") == (batch == newRows)) + val withValue = if (batch == oldRows) raw.withColumn("value", lit(null) + .cast(IntegerType)) else raw + RowComparison.assertRowsEqual( + spark.createDataFrame(batch.filter(_.getInt(1) == part).asJava, schema), + withValue.withColumn("part", lit(part)).select("id", "part", "value"), + s"input_$variant/${add.path}") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.count(_.getInt(1) == part)) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + require(stats.path(field).has("value") == (batch == newRows), + s"$field must omit the evolved column only for old files") + } + } + } + + val logDir = dir.resolve("_delta_log") + val checkpoint = logDir.resolve("00000000000000000003.checkpoint.parquet") + require(Files.isRegularFile(checkpoint), "Expected the final classic checkpoint") + val cp = spark.read.parquet(checkpoint.toString) + val addSchema = cp.schema("add").dataType.asInstanceOf[StructType] + require(addSchema.fieldNames.contains("stats") == !parsed) + require(addSchema.fieldNames.contains("stats_parsed") == parsed) + require(addSchema.fieldNames.contains("partitionValues_parsed") == parsed) + val cpAdds = cp.filter("add IS NOT NULL").select("add.*").collect() + require(cpAdds.length == adds.size, "Checkpoint must contain every active AddFile") + require(cpAdds.map(_.getAs[String]("path")).toSet == adds.map(_.path).toSet) + require(cp.filter("remove IS NOT NULL").count() == 0) + cpAdds.foreach { action => + val original = adds.find(_.path == action.getAs[String]("path")).get + val stats = JsonUtil.mapper.readTree(original.stats.get) + if (parsed) { + val partition = action.getAs[Row]("partitionValues_parsed") + require(partition.schema("part").dataType == IntegerType) + require(partition.getAs[Int]("part") == original.partitionValues("part").toInt) + val actual = action.getAs[Row]("stats_parsed") + require(actual.getAs[Long]("numRecords") == stats.path("numRecords").asLong()) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + val values = actual.getAs[Row](field) + require(values.getAs[Any]("id").toString == stats.path(field).path("id").asText()) + if (oldAdds.exists(_.path == original.path)) { + require(values.isNullAt(values.fieldIndex("value")), + s"Old file $field must have no parsed evolved-column statistic") + } else { + require(values.getAs[Any]("value").toString == + stats.path(field).path("value").asText()) + } + } + } else { + require(JsonUtil.mapper.readTree(action.getAs[String]("stats")) == stats) + } + } + val lastCheckpoint = JsonUtil.mapper.readTree( + Files.readAllBytes(logDir.resolve("_last_checkpoint"))) + require(lastCheckpoint.path("version").asLong(-1) == 3) + require(lastCheckpoint.path("size").asLong(-1) == cp.count()) + + // Keep the checkpoint-version commit for timestamp resolution, but remove its entire + // preceding history and every version CRC. Reconstruction now requires this checkpoint. + for (v <- 0L until 3L) Files.delete(CommitLog.commitFile(dir, v)) + val stream = Files.list(logDir) + val logFiles = try stream.iterator().asScala.toVector finally stream.close() + logFiles.filter(_.getFileName.toString.matches("[0-9]{20}\\.crc")) + .foreach(Files.delete) + require((0L until 3L).forall(v => !Files.exists(CommitLog.commitFile(dir, v)))) + require(Files.exists(CommitLog.commitFile(dir, 3))) + require(Files.exists(checkpoint) && adds.forall(a => Files.exists(dir.resolve(a.path)))) + } + + val queries = Seq[(String, String, Option[Seq[String]], Seq[Row])]( + ("all", null, None, rows), + ("projection", null, Some(Seq("value")), + Seq(Row(null), Row(null), Row(null), Row(30), Row(30), Row(null), Row(50))), + ("equal", "value = 30", None, Seq(Row(3, 0, 30), Row(3, 0, 30))), + ("range", "value >= 30 AND value < 50", None, Seq(Row(3, 0, 30), Row(3, 0, 30))), + ("is_null", "value IS NULL", None, oldRows :+ Row(4, 1, null)), + ("is_not_null", "value IS NOT NULL", None, + Seq(Row(3, 0, 30), Row(3, 0, 30), Row(5, 1, 50))), + ("partition_null", "part = 1 AND value IS NULL", None, + Seq(Row(2, 1, null), Row(2, 1, null), Row(4, 1, null)))) + queries.foreach { case (name, predicate, columns, expectedRows) => + readSpec(t, predicate = predicate, columns = columns, name = Some(s"read_$name")) + .assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $variant/$name read, got $other") + } + val expectedSchema = if (columns.isDefined) { + new StructType().add("value", IntegerType) + } else schema + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual( + spark.createDataFrame(expectedRows.asJava, expectedSchema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$variant/$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.partitionColumns == Seq("part")) + require(result.metadata.configuration("delta.checkpoint.writeStatsAsStruct") == + parsed.toString) + require(result.metadata.configuration("delta.checkpoint.writeStatsAsJson") == + (!parsed).toString) + case other => fail(s"Expected successful $variant snapshot, got $other") + } + } + } + } + + // Timestamp statistics: encodings are checked on disk, independently of the table + // reader. + private val microFirst = "2024-01-01T00:00:00.123456Z" + private val microLast = "2024-01-01T00:00:00.123789Z" + private val nanoLower = "2024-01-01T00:00:00.123455111Z" + private val nanoUpper = "2024-01-01T00:00:00.123789999Z" + private val microSchema = new StructType().add("id", IntegerType).add("ts", TimestampType) + + // Literal instants, not values read back from the table, are the row oracle. + private def microRows: Seq[Row] = Seq( + Row(1, Timestamp.from(Instant.parse(microFirst))), + Row(2, Timestamp.from(Instant.parse(microLast))), + Row(2, Timestamp.from(Instant.parse(microLast)))) + + private def createMicroTable(): TableHandle = { + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP) USING delta TBLPROPERTIES ( + 'delta.minReaderVersion' = '1', 'delta.minWriterVersion' = '2', + 'delta.checkpointPolicy' = 'classic', + 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, TIMESTAMP '2024-01-01 00:00:00.123456'), + (2, TIMESTAMP '2024-01-01 00:00:00.123789'), + (2, TIMESTAMP '2024-01-01 00:00:00.123789') AS rows(id, ts)""") + val t = registerTable("tbl") + val adds = CommitLog.read(t.sourcePath, 1).collect { case a: AddFile => a } + require(adds.size == 1, "The timestamp bounds must describe one file with all three rows") + t + } + + private def checkMicroSpecs(t: TableHandle, checkJsonStats: Boolean = false): Unit = { + val workloadName = WorkloadContext.current.workloadName + val tableOutputDir = outputDir.resolve(workloadName) + val queries = Seq( + ("all", null, microRows), + ("lower_edge", "ts <= TIMESTAMP '2024-01-01 00:00:00.123456'", microRows.take(1)), + ("upper_edge", "ts >= TIMESTAMP '2024-01-01 00:00:00.123789'", microRows.drop(1)), + ("outside", "ts < TIMESTAMP '2024-01-01 00:00:00.123456' OR " + + "ts > TIMESTAMP '2024-01-01 00:00:00.123789'", Seq.empty[Row])) + + for (version <- Seq[java.lang.Long](null, 1L)) { + val suffix = if (version == null) "latest" else "v1" + for ((label, predicate, rows) <- queries) { + val name = s"${label}_$suffix" + readSpec(t, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, microSchema) + val expectedDir = tableOutputDir.resolve("expected") + .resolve(s"${workloadName}_$name") + val captured = expectedDir.resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + if (checkJsonStats && label == "all") { + val scanned = spark.read.parquet( + expectedDir.resolve("expected_metadata").toString).collect() + require(scanned.length == 1) + val scanStats = JsonUtil.mapper.readTree(scanned.head.getString(0)) + .at("/add/stats").asText() + val tableDir = tableOutputDir.resolve("delta") + val rawStats = CommitLog.read(tableDir, 1) + .collectFirst { case a: AddFile => a.stats.get }.get + require(scanStats == rawStats, "Scanned stats must retain the injected precision") + println(s"Verified $name scanned AddFile stats match nanosecond JSON") + } + } + } + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == microSchema) + case other => fail(s"$suffix snapshot must succeed: $other") + } + } + } + } + + test("nanosecond_stats_json") { + val t = createMicroTable() + // The bounds enclose the microsecond rows. All of digits 7-9 are nonzero. + val lower = nanoLower + val upper = nanoUpper + require(Instant.parse(lower).isBefore(Instant.parse(microFirst))) + require(Instant.parse(upper).isAfter(Instant.parse(microLast))) + modifyCommitActions(t, version = 1) { actions => + actions.map { + case a: AddFile => + val stats = JsonUtil.mapper.readTree(a.stats.get) + stats.get("minValues").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", lower) + stats.get("maxValues").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", upper) + a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + mutateTable(t) { tableDir => + val adds = CommitLog.read(tableDir, 1).collect { case a: AddFile => a } + require(adds.size == 1) + val raw = JsonUtil.mapper.readTree(adds.head.stats.get) + require(raw.at("/minValues/ts").asText() == lower) + require(raw.at("/maxValues/ts").asText() == upper) + require(raw.get("numRecords").asLong() == 3) + // A copied checksum can contain the old AddFile stats and bypass JSON parsing. + val files = Files.list(tableDir.resolve("_delta_log")) + try files.iterator().asScala.filter(_.toString.endsWith(".crc")) + .foreach(Files.delete(_)) finally files.close() + println(s"Verified JSON stats: min=$lower max=$upper records=3") + } + checkMicroSpecs(t, checkJsonStats = true) + } + + for ((caseName, encoding, lower, upper) <- Seq( + ("millisecond_stats_checkpoint", "TIMESTAMP_MILLIS", + "2024-01-01T00:00:00.123Z", "2024-01-01T00:00:00.124Z"), + ("microsecond_stats_checkpoint", "TIMESTAMP_MICROS", microFirst, microLast), + ("int96_stats_checkpoint", "INT96", microFirst, microLast))) { + test(caseName) { + val t = createMicroTable() + require(!Instant.parse(lower).isAfter(Instant.parse(microFirst))) + require(!Instant.parse(upper).isBefore(Instant.parse(microLast))) + forceCheckpoint("tbl") + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val checkpoint = logDir.resolve("00000000000000000001.checkpoint.parquet") + require(Files.isRegularFile(checkpoint), "Expected one classic checkpoint") + val rewriteDir = logDir.resolve("timestamp-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, encoding) + val df = spark.read.parquet(checkpoint.toString) + val stats = col("add.stats_parsed") + .withField("minValues", col("add.stats_parsed.minValues") + .withField("ts", lit(Timestamp.from(Instant.parse(lower))))) + .withField("maxValues", col("add.stats_parsed.maxValues") + .withField("ts", lit(Timestamp.from(Instant.parse(upper))))) + // Disable the JSON fallback: these reads must consume parsed checkpoint stats. + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewriteDir.toString) + val files = Files.list(rewriteDir) + val part = try { + val parts = files.iterator().asScala.filter(_.toString.endsWith(".parquet")).toSeq + require(parts.size == 1) + parts.head + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewriteDir)) FileUtils.deleteDirectory(rewriteDir.toFile) + } + checkMicroCheckpoint(checkpoint, encoding, lower, upper) + if (encoding == "INT96") { + // Spark's writer only emits microsecond-aligned INT96. Preserve every checkpoint + // field with the existing Parquet Group API, replacing just the two stats cells. + rewriteInt96Bounds(checkpoint) + checkMicroCheckpoint(checkpoint, encoding, nanoLower, nanoUpper) + } + // Retain v1 JSON as a discovery/metadata tail alongside the checkpoint. Removing v0 + // prevents full JSON replay, so v1 and latest require checkpoint bootstrap. + val files = Files.list(logDir) + try files.iterator().asScala.filter { p => + p.getFileName.toString == "00000000000000000000.json" || + p.toString.endsWith(".crc") + }.foreach(Files.delete(_)) finally files.close() + } + checkMicroSpecs(t) + } + } + + private def rewriteInt96Bounds(checkpoint: Path): Unit = { + import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ + + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + val metadata = try footer.getFooter.getFileMetaData finally footer.close() + val parquetSchema = metadata.getSchema + val factory = new SimpleGroupFactory(parquetSchema) + // 2024-01-01 is Julian day 2460311. These nanoseconds after midnight enclose the rows. + val replacements = Map( + "add.stats_parsed.minValues.ts" -> new NanoTime(2460311, 123455111L).toBinary, + "add.stats_parsed.maxValues.ts" -> new NanoTime(2460311, 123789999L).toBinary) + + def copyGroup(source: Group, target: Group, prefix: String): Unit = { + for (i <- 0 until source.getType.getFieldCount) { + val field = source.getType.getType(i) + val fieldPath = if (prefix.isEmpty) field.getName else s"$prefix.${field.getName}" + for (j <- 0 until source.getFieldRepetitionCount(i)) { + if (!field.isPrimitive) { + copyGroup(source.getGroup(i, j), target.addGroup(i), fieldPath) + } else field.asPrimitiveType().getPrimitiveTypeName match { + case BOOLEAN => target.add(i, source.getBoolean(i, j)) + case INT32 => target.add(i, source.getInteger(i, j)) + case INT64 => target.add(i, source.getLong(i, j)) + case FLOAT => target.add(i, source.getFloat(i, j)) + case DOUBLE => target.add(i, source.getDouble(i, j)) + case BINARY | FIXED_LEN_BYTE_ARRAY => target.add(i, source.getBinary(i, j)) + case INT96 => target.add(i, replacements.getOrElse(fieldPath, source.getInt96(i, j))) + } + } + } + } + + val temporary = checkpoint.resolveSibling("timestamp-int96-rewrite.parquet") + try { + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + try { + val writer = ExampleParquetWriter.builder(new HadoopPath(temporary.toUri)) + .withConf(conf).withType(parquetSchema) + .withExtraMetaData(metadata.getKeyValueMetaData).build() + try { + var source = reader.read() + while (source != null) { + val target = factory.newGroup() + copyGroup(source, target, "") + writer.write(target) + source = reader.read() + } + } finally writer.close() + } finally reader.close() + Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally Files.deleteIfExists(temporary) + } + + private def checkMicroCheckpoint( + checkpoint: Path, encoding: String, lower: String, upper: String): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val parquetSchema = footer.getFooter.getFileMetaData.getSchema + for (bound <- Seq("minValues", "maxValues")) { + val physical = parquetSchema.getType("add", "stats_parsed", bound, "ts") + .asPrimitiveType() + if (encoding == "INT96") { + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT96) + require(physical.getLogicalTypeAnnotation == null) + } else { + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = physical.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + val unit = if (encoding == "TIMESTAMP_MILLIS") { + LogicalTypeAnnotation.TimeUnit.MILLIS + } else LogicalTypeAnnotation.TimeUnit.MICROS + require(annotation.getUnit == unit) + require(annotation.isAdjustedToUTC) + } + println(s"Verified checkpoint add.stats_parsed.$bound.ts: $physical") + } + } finally footer.close() + + // Read the physical cells without Spark's timestamp conversion or Delta's stats parser. + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var adds = 0 + try { + var row = reader.read() + while (row != null) { + if (row.getFieldRepetitionCount("add") > 0) { + adds += 1 + val add = row.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + require(stats.getLong("numRecords", 0) == 3) + for ((bound, value) <- Seq("minValues" -> lower, "maxValues" -> upper)) { + val cell = stats.getGroup(bound, 0) + val instant = Instant.parse(value) + if (encoding == "INT96") { + val bytes = ByteBuffer.wrap(cell.getInt96("ts", 0).getBytes) + .order(ByteOrder.LITTLE_ENDIAN) + val nanosOfDay = bytes.getLong() + val julianDay = bytes.getInt() + require(julianDay == 2440588 + Math.floorDiv(instant.getEpochSecond, 86400L)) + require(nanosOfDay == Math.floorMod(instant.getEpochSecond, 86400L) * + 1000000000L + instant.getNano) + // Check the aligned intermediate separately from the true nanos replacement. + require(nanosOfDay % 1000 == instant.getNano % 1000) + require(nanosOfDay % 1000000 != 0) + println(s"Verified raw INT96 $bound: day=$julianDay nanosOfDay=$nanosOfDay") + } else { + val units = if (encoding == "TIMESTAMP_MILLIS") 1000L else 1000000L + val expected = instant.getEpochSecond * units + + instant.getNano / (1000000000L / units) + require(cell.getLong("ts", 0) == expected) + println(s"Verified raw $encoding $bound: $expected") + } + } + } + row = reader.read() + } + } finally reader.close() + require(adds == 1) + } + + // Each data boundary is crossed with every temporal partition boundary. + // SQL fixture literals and the typed row oracle are deliberately declared separately. + private val epochValues = Seq( + (1, "1969-12-31T23:59:59.998999", "1969-12-31", -1001L), + (2, "1969-12-31T23:59:59.999000", "1969-12-31", -1000L), + (3, "1969-12-31T23:59:59.999001", "1969-12-31", -999L), + (4, "1969-12-31T23:59:59.999999", "1969-12-31", -1L), + (5, "1970-01-01T00:00:00.000000", "1970-01-01", 0L), + (6, "1970-01-01T00:00:00.000001", "1970-01-01", 1L), + (7, "1970-01-01T00:00:00.001000", "1970-01-02", 1000L)) + + private val epochSchema = new StructType().add("id", IntegerType) + .add("ts", TimestampType).add("ntz", org.apache.spark.sql.types.TimestampNTZType) + .add("d", org.apache.spark.sql.types.DateType).add("p_ts", TimestampType) + .add("p_ntz", org.apache.spark.sql.types.TimestampNTZType) + .add("p_d", org.apache.spark.sql.types.DateType) + + private def epochRows: Seq[Row] = { + val data = epochValues ++ epochValues.filter(_._1 == 4) + for ((id, wall, day, _) <- data; (_, pWall, pDay, _) <- epochValues) yield { + Row(id, Timestamp.from(Instant.parse(wall + "Z")), java.time.LocalDateTime.parse(wall), + java.sql.Date.valueOf(day), Timestamp.from(Instant.parse(pWall + "Z")), + java.time.LocalDateTime.parse(pWall), java.sql.Date.valueOf(pDay)) + } + } + + private def epochPartitionStrings: Set[Map[String, String]] = epochValues.map { + case (_, wall, day, _) => + // NTZ uses wall-clock strings; TIMESTAMP uses UTC ISO with six fractional digits. + val partition = wall.replace('T', ' ').replaceAll("0+$", "").stripSuffix(".") + Map("p_ts" -> (wall + "Z"), "p_ntz" -> partition, "p_d" -> day) + }.toSet + + private def createEpochTable(): TableHandle = { + require(spark.conf.get("spark.sql.session.timeZone") == "UTC") + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ, d DATE, + p_ts TIMESTAMP, p_ntz TIMESTAMP_NTZ, p_d DATE) USING delta + PARTITIONED BY (p_ts, p_ntz, p_d) TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', + 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + val fixtures = Seq( + (1, "1969-12-31 23:59:59.998999", "1969-12-31"), + (2, "1969-12-31 23:59:59.999", "1969-12-31"), + (3, "1969-12-31 23:59:59.999001", "1969-12-31"), + (4, "1969-12-31 23:59:59.999999", "1969-12-31"), + (5, "1970-01-01 00:00:00", "1970-01-01"), + (6, "1970-01-01 00:00:00.000001", "1970-01-01"), + (7, "1970-01-01 00:00:00.001", "1970-01-02")) + val partitions = fixtures.map { case (_, ts, day) => + s"(TIMESTAMP '$ts', TIMESTAMP_NTZ '$ts', DATE '$day')" + }.mkString(",") + for ((id, ts, day) <- fixtures) { + val data = s"($id, TIMESTAMP '$ts', TIMESTAMP_NTZ '$ts', DATE '$day')" + val rows = if (id == 4) s"$data,$data" else data + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM + (VALUES $rows) AS data(id, ts, ntz, d) CROSS JOIN + (VALUES $partitions) AS parts(p_ts, p_ntz, p_d)""") + } + val t = registerTable("tbl") + for (v <- 1L to 7L) { + val adds = CommitLog.read(t.sourcePath, v).collect { case a: AddFile => a } + require(adds.size == 7, "Each boundary must have one file per partition") + require(adds.map(_.partitionValues).toSet == epochPartitionStrings) + } + t + } + + for (parsed <- Seq(false, true)) { + val name = if (parsed) "negative_epoch_parsed_stats_partitions" + else "negative_epoch_json_stats_partitions" + test(name) { + val t = createEpochTable() + if (parsed) forceCheckpoint("tbl") + if (!parsed) { + for ((id, wall, day, _) <- epochValues) { + modifyCommitActions(t, version = id.toLong) { actions => + actions.map { + case a: AddFile => + val stats = JsonUtil.mapper.readTree(a.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.get(bound) + .asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + values.put("ts", wall + "Z") + values.put("ntz", wall) + values.put("d", day) + } + a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + } + } + mutateTable(t) { dir => + if (parsed) { + val checkpoint = dir.resolve("_delta_log/00000000000000000007.checkpoint.parquet") + val rewrite = dir.resolve("_delta_log/epoch-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + var ts = col(s"add.stats_parsed.$bound.ts") + var ntz = col(s"add.stats_parsed.$bound.ntz") + for ((id, wall, _, _) <- epochValues) { + val matches = col("add.stats_parsed.minValues.id") === lit(id) + ts = org.apache.spark.sql.functions.when(matches, + lit(Timestamp.from(Instant.parse(wall + "Z")))).otherwise(ts) + ntz = org.apache.spark.sql.functions.when(matches, + lit(java.time.LocalDateTime.parse(wall))).otherwise(ntz) + } + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("ts", ts).withField("ntz", ntz)) + } + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewrite.toString) + val files = Files.list(rewrite) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten checkpoint: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewrite)) FileUtils.deleteDirectory(rewrite.toFile) + } + checkEpochCheckpoint(checkpoint) + } else { + for ((id, wall, day, _) <- epochValues) { + val adds = CommitLog.read(dir, id.toLong).collect { case a: AddFile => a } + require(adds.size == 7) + require(adds.map(_.partitionValues).toSet == epochPartitionStrings) + for (a <- adds) { + val stats = JsonUtil.mapper.readTree(a.stats.get) + require(stats.get("numRecords").asLong() == (if (id == 4) 2 else 1)) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/ts").asText() == wall + "Z") + require(stats.at(s"/$bound/ntz").asText() == wall) + require(stats.at(s"/$bound/d").asText() == day) + require(stats.at(s"/$bound/id").asInt() == id) + } + } + } + println("Verified exact JSON stats and partition strings for all 49 boundary files") + } + val files = Files.list(dir.resolve("_delta_log")) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (parsed && file.endsWith(".json") && + file != "00000000000000000007.json") + }.foreach(Files.delete(_)) finally files.close() + // v7 JSON is retained only for same-version lookup. There is no replay tail after v7. + // v0-v6 are absent, so the checkpoint is necessary for both latest and version 7. + } + checkEpochSpecs(t, parsed) + } + } + + private def checkEpochCheckpoint(checkpoint: Path): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + for ((prefix, fields) <- Seq( + "stats_parsed.minValues" -> Seq("ts", "ntz", "d"), + "stats_parsed.maxValues" -> Seq("ts", "ntz", "d"), + "partitionValues_parsed" -> Seq("p_ts", "p_ntz", "p_d")); field <- fields) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + if (field.endsWith("d")) { + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(cell.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } else { + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == field.endsWith("ts")) + } + println(s"Verified near-epoch checkpoint $prefix.$field: $cell") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var pairs = Vector.empty[(Int, Map[String, String])] + try { + var row = reader.read() + while (row != null) { + if (row.getFieldRepetitionCount("add") > 0) { + val add = row.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + val (_, _, day, micros) = epochValues.find(_._1 == id).get + require(stats.getLong("numRecords", 0) == (if (id == 4) 2 else 1)) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + require(values.getLong("ts", 0) == micros) + require(values.getLong("ntz", 0) == micros) + require(values.getInteger("d", 0) == java.time.LocalDate.parse(day).toEpochDay) + } + val raw = add.getGroup("partitionValues", 0) + val partition = (0 until raw.getFieldRepetitionCount("key_value")).map { i => + val entry = raw.getGroup("key_value", i) + entry.getString("key", 0) -> entry.getString("value", 0) + }.toMap + require(epochPartitionStrings.contains(partition)) + val parsed = add.getGroup("partitionValues_parsed", 0) + val (_, _, pDay, pMicros) = epochValues.find { case (_, wall, _, _) => + java.time.LocalDateTime.parse(wall) == + java.time.LocalDateTime.parse(partition("p_ntz").replace(' ', 'T')) + }.get + require(parsed.getLong("p_ts", 0) == pMicros) + require(parsed.getLong("p_ntz", 0) == pMicros) + require(parsed.getInteger("p_d", 0) == java.time.LocalDate.parse(pDay).toEpochDay) + pairs :+= id -> partition + } + row = reader.read() + } + } finally reader.close() + val expectedPairs = for ((id, _, _, _) <- epochValues; p <- epochPartitionStrings) yield id -> p + require(pairs.size == 49 && pairs.toSet == expectedPairs.toSet) + println("Verified 49 physical stats/partition pairs: micros -1001,-1000,-999,-1,0,1,1000") + } + + private def checkEpochSpecs(t: TableHandle, parsed: Boolean): Unit = { + val workload = WorkloadContext.current.workloadName + val tableDir = outputDir.resolve(workload) + val all = epochRows + def read(name: String, predicate: String, rows: Seq[Row], version: java.lang.Long = null) + : Unit = { + readSpec(t, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, epochSchema) + val captured = tableDir.resolve("expected").resolve(s"${workload}_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + } + } + read("all", null, all) + read("all_v7", null, all, 7L) + for ((field, index, kind) <- Seq(("ts", 1, "TIMESTAMP"), ("ntz", 2, "TIMESTAMP_NTZ"), + ("p_ts", 4, "TIMESTAMP"), ("p_ntz", 5, "TIMESTAMP_NTZ"))) { + for ((id, wall, _, _) <- epochValues) { + val expected = if (kind == "TIMESTAMP") Timestamp.from(Instant.parse(wall + "Z")) + else java.time.LocalDateTime.parse(wall) + read(s"${field}_eq_$id", s"$field = $kind '${wall.replace('T', ' ')}'", + all.filter(_.get(index) == expected)) + } + read(s"${field}_negative", s"$field < $kind '1970-01-01 00:00:00'", + all.filter { r => + if (kind == "TIMESTAMP") r.getTimestamp(index).before(Timestamp.from(Instant.EPOCH)) + else r.getAs[java.time.LocalDateTime](index) + .isBefore(java.time.LocalDateTime.of(1970, 1, 1, 0, 0)) + }) + } + for ((field, index) <- Seq("d" -> 3, "p_d" -> 6); + day <- Seq("1969-12-31", "1970-01-01", "1970-01-02")) { + read(s"${field}_$day", s"$field = DATE '$day'", + all.filter(_.getDate(index) == java.sql.Date.valueOf(day))) + } + read("cross_negative_hits", "ts = TIMESTAMP '1969-12-31 23:59:59.999999' AND " + + "p_ntz = TIMESTAMP_NTZ '1969-12-31 23:59:59.999' AND p_d < DATE '1970-01-01'", + all.filter(r => r.getInt(0) == 4 && r.getTimestamp(4) == + Timestamp.from(Instant.parse("1969-12-31T23:59:59.999Z")))) + read("outside_stats", "ts < TIMESTAMP '1969-12-31 23:59:59.998999' OR " + + "ntz > TIMESTAMP_NTZ '1970-01-01 00:00:00.001'", Seq.empty) + read("outside_partitions", "p_ts < TIMESTAMP '1969-12-31 23:59:59.998999' OR " + + "p_ntz > TIMESTAMP_NTZ '1970-01-01 00:00:00.001'", Seq.empty) + for (version <- Seq[java.lang.Long](null, 7L)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == epochSchema) + require(result.metadata.partitionColumns == Seq("p_ts", "p_ntz", "p_d")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.metadata.configuration("delta.checkpointPolicy") == "classic") + require(result.protocol.minReaderVersion == 3) + require(result.protocol.minWriterVersion == 7) + require(result.protocol.readerFeatures.get.contains("timestampNtz")) + require(result.protocol.writerFeatures.get.contains("timestampNtz")) + case other => fail(s"Near-epoch snapshot must succeed (parsed=$parsed): $other") + } + } + } + } + + // Statistics and partition cases share the absolute-instant fixture and oracle. + private val mismatchSchema = new StructType().add("id", IntegerType) + .add("ts", TimestampType).add("ntz", org.apache.spark.sql.types.TimestampNTZType) + private val mismatchZones = Seq( + ("writer_utc_reader_non_utc", "UTC", "America/Los_Angeles"), + ("writer_non_utc_reader_utc", "America/Los_Angeles", "UTC")) + + // Offsets distinguish both occurrences of 01:30 at the autumn overlap. The unqualified + // controls deliberately produce different instants in the two writer-zone fixtures. + private val mismatchSql = Seq( + (1, "2024-03-10 01:59:59.123456-08:00", "2024-03-10 01:59:59.123456"), + (2, "2024-03-10 03:00:00.123456-07:00", "2024-03-10 03:00:00.123456"), + (3, "2024-11-03 01:30:00.123456-07:00", "2024-11-03 01:30:00.123456"), + (4, "2024-11-03 01:30:00.123456-08:00", "2024-11-03 01:30:00.123456"), + (5, "2024-01-15 12:00:00.654321", "2024-01-15 12:00:00.654321"), + (6, "2024-03-10 02:30:00.123456", "2024-03-10 02:30:00.123456"), + (7, "2024-11-03 01:30:00.123456", "2024-11-03 01:30:00.123456")) + + // This oracle does not parse fixture SQL or read Delta. NTZ retains the written wall clock, + // including the nonexistent LA 02:30. TIMESTAMP instead denotes an absolute UTC instant. + private def mismatchRows(writerZone: String): Seq[Row] = { + val controls = if (writerZone == "UTC") Seq( + "2024-01-15T12:00:00.654321Z", "2024-03-10T02:30:00.123456Z", + "2024-11-03T01:30:00.123456Z") else Seq( + "2024-01-15T20:00:00.654321Z", "2024-03-10T10:30:00.123456Z", + "2024-11-03T08:30:00.123456Z") + val instants = Seq( + "2024-03-10T09:59:59.123456Z", "2024-03-10T10:00:00.123456Z", + "2024-11-03T08:30:00.123456Z", "2024-11-03T09:30:00.123456Z") ++ controls + val walls = Seq( + "2024-03-10T01:59:59.123456", "2024-03-10T03:00:00.123456", + "2024-11-03T01:30:00.123456", "2024-11-03T01:30:00.123456", + "2024-01-15T12:00:00.654321", "2024-03-10T02:30:00.123456", + "2024-11-03T01:30:00.123456") + val rows = instants.zip(walls).zipWithIndex.map { case ((instant, wall), i) => + Row(i + 1, Timestamp.from(Instant.parse(instant)), java.time.LocalDateTime.parse(wall)) + } + rows ++ rows.filter(_.getInt(0) == 3) + } + + private def mismatchMicros(instant: Instant): Long = + instant.getEpochSecond * 1000000L + instant.getNano / 1000 + + // Spark to_json's default timestamp pattern serializes both bounds at millisecond + // precision. Check that physical encoding independently; keep microsecond row expectations. + private def mismatchJsonInstant(instant: Instant): Instant = + instant.truncatedTo(java.time.temporal.ChronoUnit.MILLIS) + + private def createMismatchTable(writerZone: String, partitioned: Boolean): TableHandle = { + setForTest("spark.sql.session.timeZone", writerZone) + val partition = if (partitioned) "PARTITIONED BY (ts)" else "" + sql(s"""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ) USING delta + $partition TBLPROPERTIES ('delta.checkpointPolicy' = 'classic', + 'delta.checkpointInterval' = '100', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + for ((id, instant, wall) <- mismatchSql) { + val row = s"($id, TIMESTAMP '$instant', TIMESTAMP_NTZ '$wall')" + val values = if (id == 3) s"$row, $row" else row + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS rows(id, ts, ntz)") + } + val table = registerTable("tbl") + val rows = mismatchRows(writerZone) + for (version <- 1L to 7L) { + val adds = CommitLog.read(table.sourcePath, version).collect { case a: AddFile => a } + require(adds.size == 1, "Each input batch must produce exactly one file") + val add = adds.head + val row = rows.find(_.getInt(0) == version.toInt).get + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.get("numRecords").asLong() == (if (version == 3) 2 else 1)) + val expectedPartition = if (partitioned) { + Map("ts" -> row.getTimestamp(1).toInstant.toString) + } else Map.empty[String, String] + if (partitioned && writerZone == "America/Los_Angeles" && version == 4 && + add.partitionValues != expectedPartition) { + // The producer reparses an offset-free directory name at the overlap's earlier offset. + // Preserve that limitation, then construct the intended AddFile in the copied fixture. + require(add.partitionValues == Map("ts" -> "2024-11-03T08:30:00.123456Z")) + println(s"Observed producer overlap loss: expected=$expectedPartition " + + s"actual=${add.partitionValues} action=${add.toJson}") + } else require(add.partitionValues == expectedPartition) + for (bound <- Seq("minValues", "maxValues")) { + val fields = stats.get(bound) + require(fields.get("id").asInt() == version.toInt) + if (partitioned) require(!fields.has("ts")) else { + val raw = fields.get("ts").asText() + val parsed = java.time.OffsetDateTime.parse(raw) + val instant = row.getTimestamp(1).toInstant + require(parsed.toInstant == mismatchJsonInstant(instant)) + require(parsed.getOffset == instant.atZone(java.time.ZoneId.of(writerZone)).getOffset) + } + val ntz = row.getAs[java.time.LocalDateTime](2).toInstant(java.time.ZoneOffset.UTC) + val rawNtz = java.time.LocalDateTime.parse(fields.get("ntz").asText()) + require(rawNtz.toInstant(java.time.ZoneOffset.UTC) == mismatchJsonInstant(ntz)) + } + println(s"Verified writer JSON zone=$writerZone partitioned=$partitioned " + + s"id=$version stats=${add.stats.get} partitions=${add.partitionValues}") + } + println(s"Verified writer table zone=${spark.conf.get("spark.sql.session.timeZone")} " + + s"partitioned=$partitioned rows=8 files=7") + table + } + + private def prepareMismatchCapture( + table: TableHandle, writerZone: String, readerZone: String, + partitioned: Boolean, parsed: Boolean): Unit = { + require(writerZone != readerZone) + require(spark.conf.get("spark.sql.session.timeZone") == writerZone) + if (partitioned && writerZone == "America/Los_Angeles") { + val later = mismatchRows(writerZone).find(_.getInt(0) == 4).get.getTimestamp(1).toInstant + modifyCommitActions(table, version = 4) { actions => + actions.map { + case add: AddFile => add.copy(partitionValues = Map("ts" -> later.toString)) + case other => other + } + } + } + if (parsed) { + val encoding = spark.conf.get("spark.sql.parquet.outputTimestampType") + try { + spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MICROS") + forceCheckpoint("tbl") + } finally spark.conf.set("spark.sql.parquet.outputTimestampType", encoding) + } + mutateTable(table) { dir => + require(spark.conf.get("spark.sql.session.timeZone") == readerZone) + val logDir = dir.resolve("_delta_log") + if (parsed) { + val checkpoint = logDir.resolve("00000000000000000007.checkpoint.parquet") + if (partitioned && writerZone == "America/Los_Angeles") { + setMismatchOverlapPartition(checkpoint) + } + checkMismatchCheckpoint(checkpoint, writerZone, partitioned) + } + val files = Files.list(logDir) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (parsed && file.endsWith(".json") && + file != "00000000000000000007.json") + }.foreach(Files.delete(_)) finally files.close() + // Same-version v7 JSON is retained solely for version lookup. No replay tail exists; + // v0-v6 and CRC shortcuts are absent, so the checkpoint is required, not checkpoint-only. + println(s"Verified pre-capture reader=$readerZone writer=$writerZone " + + s"partitioned=$partitioned parsed=$parsed") + } + // WorkloadTestSuite captures after the body. Deliberately leave this reader zone active. + setForTest("spark.sql.session.timeZone", readerZone) + } + + // DAT supports a synthetic, correct overlap partition even when the SQL writer loses the + // later offset while reparsing a partition directory. Preserve its unmodified checkpoint; + // then change only the intended id=4 AddFile's raw and parsed partition cells in the copy. + private def setMismatchOverlapPartition(checkpoint: Path): Unit = { + val workload = WorkloadContext.current.workloadName + val original = outputDir.resolveSibling("producer-checkpoints").resolve(workload) + Files.createDirectories(original) + Files.copy(checkpoint, original.resolve(checkpoint.getFileName), + StandardCopyOption.REPLACE_EXISTING) + val instant = Instant.parse("2024-11-03T09:30:00.123456Z") + val temporary = checkpoint.resolveSibling("overlap-partition-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + val later = col("add.stats_parsed.minValues.id") === lit(4) + val raw = org.apache.spark.sql.functions.when(later, + org.apache.spark.sql.functions.map(lit("ts"), lit(instant.toString))) + .otherwise(col("add.partitionValues")) + val parsed = col("add.partitionValues_parsed").withField("ts", + org.apache.spark.sql.functions.when(later, lit(Timestamp.from(instant))) + .otherwise(col("add.partitionValues_parsed.ts"))) + val add = col("add").withField("partitionValues", raw) + .withField("partitionValues_parsed", parsed) + df.withColumn("add", add).coalesce(1).write.parquet(temporary.toString) + val files = Files.list(temporary) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one overlap checkpoint part: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(temporary)) FileUtils.deleteDirectory(temporary.toFile) + } + println("Constructed later overlap partition fixture: id=4 " + + "instant=2024-11-03T09:30:00.123456Z") + } + + private def checkMismatchCheckpoint( + checkpoint: Path, writerZone: String, partitioned: Boolean): Unit = { + require(Files.isRegularFile(checkpoint)) + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + val timestamps = if (partitioned) Seq("partitionValues_parsed" -> "ts") + else Seq("stats_parsed.minValues" -> "ts", "stats_parsed.maxValues" -> "ts") + for ((prefix, field) <- timestamps ++ Seq( + "stats_parsed.minValues" -> "ntz", "stats_parsed.maxValues" -> "ntz")) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == (field == "ts")) + println(s"Verified timezone checkpoint writer=$writerZone $prefix.$field: $cell") + } + } finally footer.close() + val rows = mismatchRows(writerZone) + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var ids = Vector.empty[Int] + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + val add = record.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + val row = rows.find(_.getInt(0) == id).get + require(stats.getLong("numRecords", 0) == (if (id == 3) 2 else 1)) + val instant = row.getTimestamp(1).toInstant + val ntz = row.getAs[java.time.LocalDateTime](2).toInstant(java.time.ZoneOffset.UTC) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + if (!partitioned) require(values.getLong("ts", 0) == + mismatchMicros(mismatchJsonInstant(instant))) + require(values.getLong("ntz", 0) == mismatchMicros(mismatchJsonInstant(ntz))) + } + if (partitioned) { + val partition = add.getGroup("partitionValues_parsed", 0) + require(partition.getLong("ts", 0) == mismatchMicros(instant)) + val raw = add.getGroup("partitionValues", 0) + require(raw.getFieldRepetitionCount("key_value") == 1) + val entry = raw.getGroup("key_value", 0) + require(entry.getString("key", 0) == "ts") + require(entry.getString("value", 0) == instant.toString) + } + ids :+= id + println(s"Verified timezone raw checkpoint writer=$writerZone partitioned=$partitioned " + + s"id=$id instantMicros=${mismatchMicros(instant)} ntzMicros=${mismatchMicros(ntz)}") + } + record = reader.read() + } + } finally reader.close() + require(ids.sorted == (1 to 7).toVector) + } + + private def checkMismatchSpecs( + table: TableHandle, writerZone: String, readerZone: String, partitioned: Boolean): Unit = { + val workload = WorkloadContext.current.workloadName + val tableDir = outputDir.resolve(workload) + val all = mismatchRows(writerZone) + def checkReaderZone(label: String): Unit = { + require(spark.conf.get("spark.sql.session.timeZone") == readerZone) + val probe = spark.sql("SELECT current_timezone(), " + + "CAST(TIMESTAMP '2024-03-10 10:00:00.123456Z' AS STRING)").head() + require(probe.getString(0) == readerZone) + val wall = if (readerZone == "UTC") "2024-03-10 10:00:00.123456" + else "2024-03-10 03:00:00.123456" + require(probe.getString(1) == wall) + println(s"Verified captured $workload/$label writer=$writerZone reader=$readerZone " + + s"readerWall=$wall") + } + def read(label: String, predicate: String, rows: Seq[Row], version: java.lang.Long): Unit = { + readSpec(table, predicate = predicate, version = version, name = Some(label)).assert { spec => + checkReaderZone(label) + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$label must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, mismatchSchema) + val captured = tableDir.resolve("expected").resolve(s"${workload}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), label) + if (label == "all_latest") { + expected.coalesce(1).write.mode("overwrite") + .parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString) + } + } + } + for (version <- Seq[java.lang.Long](null, 7L)) { + val suffix = if (version == null) "latest" else "v7" + read(s"all_$suffix", null, all, version) + // Explicit offsets keep the emitted predicates portable to consumers with another zone. + for (row <- all.groupBy(_.getInt(0)).values.map(_.head).toSeq.sortBy(_.getInt(0))) { + val instant = row.getTimestamp(1) + read(s"eq_${row.getInt(0)}_$suffix", s"ts = TIMESTAMP '${instant.toInstant}'", + all.filter(_.getTimestamp(1) == instant), version) + } + read(s"spring_range_$suffix", "ts >= TIMESTAMP '2024-03-10T09:59:59.123456Z' AND " + + "ts <= TIMESTAMP '2024-03-10T10:00:00.123456Z'", + all.filter(r => Set(1, 2).contains(r.getInt(0))), version) + read(s"overlap_range_$suffix", "ts >= TIMESTAMP '2024-11-03T08:30:00.123456Z' AND " + + "ts <= TIMESTAMP '2024-11-03T09:30:00.123456Z'", + all.filter(r => Set(3, 4).contains(r.getInt(0)) || + (writerZone != "UTC" && r.getInt(0) == 7)), version) + read(s"outside_$suffix", "ts < TIMESTAMP '2024-01-01T00:00:00Z'", Seq.empty, version) + read(s"ntz_gap_$suffix", "ntz = TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'", + all.filter(_.getInt(0) == 6), version) + snapshotSpec(table, version = version).assert { spec => + checkReaderZone(s"snapshot_$suffix") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == mismatchSchema) + require(result.metadata.partitionColumns == (if (partitioned) Seq("ts") else Seq.empty)) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.metadata.configuration("delta.checkpointPolicy") == "classic") + require(result.metadata.configuration("delta.checkpoint.writeStatsAsStruct") == "true") + require(result.metadata.configuration("delta.checkpoint.writeStatsAsJson") == "false") + require(result.protocol.minReaderVersion == 3 && result.protocol.minWriterVersion == 7) + require(result.protocol.readerFeatures.get.contains("timestampNtz")) + require(result.protocol.writerFeatures.get.contains("timestampNtz")) + case other => fail(s"Timezone snapshot must succeed: $other") + } + } + } + } + + for ((direction, writer, reader) <- mismatchZones; parsed <- Seq(false, true)) { + val representation = if (parsed) "checkpoint" else "json" + test(s"stats_timezone_${direction}_$representation") { + val table = createMismatchTable(writer, partitioned = false) + prepareMismatchCapture(table, writer, reader, partitioned = false, parsed = parsed) + checkMismatchSpecs(table, writer, reader, partitioned = false) + } + } + + // Reuse the timezone fixture and check TIMESTAMP partition reconstruction. + for ((direction, writer, reader) <- mismatchZones; parsed <- Seq(false, true)) { + val representation = if (parsed) "checkpoint" else "json" + test(s"partitions_timezone_${direction}_$representation") { + val table = createMismatchTable(writer, partitioned = true) + prepareMismatchCapture(table, writer, reader, partitioned = true, parsed = parsed) + checkMismatchSpecs(table, writer, reader, partitioned = true) + } + } + + private def pruneHistoryExcept(dir: Path, version: Long): Unit = { + val log = dir.resolve("_delta_log") + val files = Files.list(log) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (file.endsWith(".json") && file != f"$version%020d.json") + }.foreach(Files.delete(_)) finally files.close() + // The same-version JSON supports requested-version lookup only. Earlier history is absent, + // so checkpoint reconstruction is required. No replay tail or checkpoint-only claim. + } + + // Unannotated INT32/INT64 parsed cells are distinct from normal DATE/TIMESTAMP + // Parquet encodings. Keep correct JSON stats too: Spark rejects several parsed-only forms. + // Only INT32 as DATE, the DATE32 physical width, removes every JSON stats copy so that the + // parsed day is the only statistic. The other forms are not claimed as parsed-only routes. + for ((kind, sqlType, valueType, literal, value, integerValue) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType, + "TIMESTAMP '1970-01-01T00:00:01Z'", Timestamp.from(Instant.ofEpochSecond(1)), 1000000L), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1970-01-01 00:00:01'", + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1), 1000000L), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1970-01-02'", java.sql.Date.valueOf("1970-01-02"), 1L)); + integerType <- Seq("int", "long")) { + test(s"checkpoint_integer_${integerType}_as_$kind") { + setForTest("spark.sql.session.timeZone", "UTC") + val expectedSchema = new StructType().add("id", IntegerType).add("v", valueType) + val rows = Seq(Row(1, value), Row(2, value), Row(2, value)) + sql(s"""CREATE TABLE tbl (id INT, v $sqlType) USING delta TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'true')""") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $literal), (2, $literal), (2, $literal) AS rows(id, v)") + val table = registerTable("tbl") + val parsedOnly = kind == "date" && integerType == "int" + forceCheckpoint("tbl") + if (parsedOnly) { + modifyCommitActions(table, 1L) { actions => + actions.map { case add: AddFile => add.copy(stats = None); case other => other } + } + } + mutateTable(table) { dir => + val checkpoint = dir.resolve("_delta_log/00000000000000000001.checkpoint.parquet") + val temporary = dir.resolve("_delta_log/integer-stats-rewrite") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("v", lit(integerValue).cast(integerType))) + } + var add = col("add").withField("stats_parsed", stats) + if (parsedOnly) add = add.withField("stats", lit(null).cast("string")) + df.withColumn("add", add).coalesce(1).write.parquet(temporary.toString) + val parts = Files.list(temporary) + val part = try parts.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten integer stats checkpoint: $other") + } finally parts.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + FileUtils.deleteDirectory(temporary.toFile) + val path = new HadoopPath(checkpoint.toUri) + val conf = spark.sparkContext.hadoopConfiguration + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + for (bound <- Seq("minValues", "maxValues")) { + val physical = footer.getFooter.getFileMetaData.getSchema + .getType("add", "stats_parsed", bound, "v").asPrimitiveType() + require(physical.getLogicalTypeAnnotation == null) + require(physical.getPrimitiveTypeName == (if (integerType == "int") + PrimitiveType.PrimitiveTypeName.INT32 else PrimitiveType.PrimitiveTypeName.INT64)) + println(s"Verified off-type checkpoint $kind/$integerType $bound: " + + s"$physical value=$integerValue jsonFallback=${!parsedOnly}") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + var count = 0 + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + count += 1 + val add = record.getGroup("add", 0) + if (parsedOnly) require(add.getFieldRepetitionCount("stats") == 0) else { + require(add.getFieldRepetitionCount("stats") == 1) + val json = JsonUtil.mapper.readTree(add.getString("stats", 0)) + for (bound <- Seq("minValues", "maxValues")) { + val raw = json.at(s"/$bound/v").asText() + kind match { + case "timestamp" => require(Instant.parse(raw) == Instant.ofEpochSecond(1)) + case "timestamp_ntz" => require(java.time.LocalDateTime.parse(raw) == + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1)) + case "date" => require(java.time.LocalDate.parse(raw) == + java.time.LocalDate.of(1970, 1, 2)) + } + } + } + val fields = add.getGroup("stats_parsed", 0) + require(fields.getLong("numRecords", 0) == 3L) + for (bound <- Seq("minValues", "maxValues")) { + val values = fields.getGroup(bound, 0) + val raw = if (integerType == "int") values.getInteger("v", 0).toLong + else values.getLong("v", 0) + require(raw == integerValue) + } + } + record = reader.read() + } + } finally reader.close() + require(count == 1) + pruneHistoryExcept(dir, 1L) + if (parsedOnly) { + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 1 && adds.head.stats.isEmpty) + val log = Files.list(dir.resolve("_delta_log")) + val names = try log.iterator().asScala.map(_.getFileName.toString).toSet + finally log.close() + require(!names.exists(_.endsWith(".crc"))) + require(names.filter(_.endsWith(".json")) == Set("00000000000000000001.json")) + println("Verified parsed-only DATE stats: INT32 day=1 (1970-01-02); JSON stats " + + "absent in checkpoint and v1 commit; CRC removed; v0 pruned") + } + } + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + checkLiteralRead(table, "hit", expectedSchema, rows, predicate = s"v = $literal") + checkLiteralRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"v < $literal OR v > $literal") + checkLiteralSnapshot(table, expectedSchema) + checkLiteralSnapshot(table, expectedSchema, version = 1L) + } + } + + // JSON numeric stats are separate from physical checkpoint types in the checkpoint fixture. + // Literal rows remain authoritative even when a reference parser discards off-type stats. + for ((kind, sqlType, valueType, literal, value) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType, "TIMESTAMP '1970-01-01T00:00:01Z'", + Timestamp.from(Instant.ofEpochSecond(1))), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1970-01-01 00:00:01'", + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1)), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1970-01-02'", java.sql.Date.valueOf("1970-01-02")), + ("string", "STRING", org.apache.spark.sql.types.StringType, "'1'", "1"))) { + test(s"action_integer_as_$kind") { + setForTest("spark.sql.session.timeZone", "UTC") + val expectedSchema = new StructType().add("id", IntegerType).add("v", valueType) + val rows = Seq(Row(1, value), Row(2, value), Row(2, value)) + sql(s"CREATE TABLE tbl (id INT, v $sqlType) USING delta") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $literal), (2, $literal), (2, $literal) AS rows(id, v)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val stats = JsonUtil.mapper.readTree(add.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + stats.get(bound).asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("v", 1) + } + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 1) + val stats = JsonUtil.mapper.readTree(adds.head.stats.get) + require(stats.get("numRecords").asLong() == 3L) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/v").isIntegralNumber) + require(stats.at(s"/$bound/v").intValue() == 1) + } + println(s"Verified numeric JSON stats: $kind min=1 max=1 records=3") + } + // Spark accepts numeric STRING stats. The required-rejection probe is preserved + // separately; this useful correctness fixture does not claim that rejection. + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + checkLiteralRead(table, "hit", expectedSchema, rows, predicate = s"v = $literal") + checkLiteralRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"v < $literal OR v > $literal") + checkLiteralSnapshot(table, expectedSchema) + checkLiteralSnapshot(table, expectedSchema, version = 1L) + } + } + + // INT32 boundary magnitudes are BIGINT inputs to timestamp_seconds/millis/micros. The unit + // names the input magnitude only; rewritten checkpoint cells are always TIMESTAMP_MICROS. + private val intMaxInputs = Seq(-2147483648L, -2147483647L, -2147483646L, + 2147483646L, 2147483647L, 2147483648L) + // Literal UTC instants per input unit, declared apart from both SQL and java.time. + private val intMaxInstants = Seq( + ("seconds", 1000000L, Seq("1901-12-13T20:45:52Z", "1901-12-13T20:45:53Z", + "1901-12-13T20:45:54Z", "2038-01-19T03:14:06Z", "2038-01-19T03:14:07Z", + "2038-01-19T03:14:08Z")), + ("millis", 1000L, Seq("1969-12-07T03:28:36.352Z", "1969-12-07T03:28:36.353Z", + "1969-12-07T03:28:36.354Z", "1970-01-25T20:31:23.646Z", "1970-01-25T20:31:23.647Z", + "1970-01-25T20:31:23.648Z")), + ("micros", 1L, Seq("1969-12-31T23:24:12.516352Z", "1969-12-31T23:24:12.516353Z", + "1969-12-31T23:24:12.516354Z", "1970-01-01T00:35:47.483646Z", + "1970-01-01T00:35:47.483647Z", "1970-01-01T00:35:47.483648Z"))) + // Data value index -> partition value index; no index maps to itself. Ids 2 and 5 hold the + // -/+2147483647 boundaries twice, and id 7 is NULL in every temporal column twice. + private val intMaxPartition = Map(1 -> 4, 2 -> 6, 3 -> 1, 4 -> 5, 5 -> 2, 6 -> 3) + private val intMaxIds = Seq(1, 2, 2, 3, 4, 5, 5, 6, 7, 7) + private val intMaxSchema = new StructType().add("id", IntegerType).add("ts", TimestampType) + .add("ntz", TimestampNTZType).add("p_ts", TimestampType).add("p_ntz", TimestampNTZType) + private val sixDigits = + java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS") + + for ((unit, scale, literals) <- intMaxInstants; parsed <- Seq(false, true)) { + test(s"int_max_${unit}_${if (parsed) "parsed" else "json"}") { + setForTest("spark.sql.session.timeZone", "UTC") + val instants = literals.map(Instant.parse) + for ((input, instant) <- intMaxInputs.zip(instants)) { + val converted = unit match { + case "seconds" => Instant.ofEpochSecond(input) + case "millis" => Instant.ofEpochMilli(input) + case "micros" => Instant.EPOCH.plus(input, java.time.temporal.ChronoUnit.MICROS) + } + require(converted == instant && mismatchMicros(instant) == input * scale) + } + def wall(index: Int) = + java.time.LocalDateTime.ofInstant(instants(index - 1), java.time.ZoneOffset.UTC) + val rows = intMaxIds.map { + case 7 => Row(7, null, null, null, null) + case id => + val p = intMaxPartition(id) + Row(id, Timestamp.from(instants(id - 1)), wall(id), Timestamp.from(instants(p - 1)), + wall(p)) + } + def partitionStrings(id: Int): Map[String, String] = if (id == 7) { + Map("p_ts" -> null, "p_ntz" -> null) + } else { + val p = wall(intMaxPartition(id)).format(sixDigits) + Map("p_ts" -> (p + "Z"), + "p_ntz" -> p.replace('T', ' ').replaceAll("0+$", "").stripSuffix(".")) + } + def fileId(add: AddFile): Int = + JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ, p_ts TIMESTAMP, + p_ntz TIMESTAMP_NTZ) USING delta PARTITIONED BY (p_ts, p_ntz) TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + val inputs = intMaxIds.map { + case 7 => "(7, CAST(NULL AS BIGINT), CAST(NULL AS BIGINT))" + case id => s"($id, ${intMaxInputs(id - 1)}L, ${intMaxInputs(intMaxPartition(id) - 1)}L)" + } + val f = s"timestamp_$unit" + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ id, $f(v), CAST($f(v) AS TIMESTAMP_NTZ), " + + s"$f(p), CAST($f(p) AS TIMESTAMP_NTZ) FROM VALUES ${inputs.mkString(", ")} " + + "AS rows(id, v, p)") + val table = registerTable("tbl") + val produced = CommitLog.read(table.sourcePath, 1L).collect { case a: AddFile => a } + require(produced.map(a => fileId(a) -> a.partitionValues).toMap == + (1 to 7).map(id => id -> partitionStrings(id)).toMap && produced.size == 7) + + if (parsed) forceCheckpoint("tbl") else { + // The producer writes millisecond JSON bounds. Keep all six fraction digits instead. + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile if fileId(add) != 7 => + val id = fileId(add) + println(s"Producer JSON stats before rewrite: $unit id=$id ${add.stats.get}") + val stats = JsonUtil.mapper.readTree(add.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + stats.get(bound).asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", wall(id).format(sixDigits) + "Z") + .put("ntz", wall(id).format(sixDigits)) + } + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + } + mutateTable(table) { dir => + if (parsed) { + val checkpoint = dir.resolve("_delta_log/00000000000000000001.checkpoint.parquet") + rewriteIntMaxCheckpoint(checkpoint, instants) + checkIntMaxCheckpoint(checkpoint, instants, partitionStrings) + pruneHistoryExcept(dir, 1L) + } else { + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case a: AddFile => a } + require(adds.size == 7) + for (add <- adds) { + val id = fileId(add) + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(add.partitionValues == partitionStrings(id)) + require(stats.get("numRecords").asLong() == (if (Set(2, 5, 7)(id)) 2L else 1L)) + for (bound <- Seq("minValues", "maxValues")) { + if (id == 7) require(!stats.get(bound).has("ts") && !stats.get(bound).has("ntz")) + else { + require(stats.at(s"/$bound/ts").asText() == wall(id).format(sixDigits) + "Z") + require(stats.at(s"/$bound/ntz").asText() == wall(id).format(sixDigits)) + } + } + if (id == 7) require(stats.at("/nullCount/ts").asLong() == 2L) + } + println(s"Verified six-digit JSON stats and partitions: $unit files=7") + } + } + + checkLiteralRead(table, "all", intMaxSchema, rows) + checkLiteralRead(table, "all_v1", intMaxSchema, rows, version = 1L) + for ((field, kind) <- Seq("ts" -> "TIMESTAMP", "ntz" -> "TIMESTAMP_NTZ", + "p_ts" -> "TIMESTAMP", "p_ntz" -> "TIMESTAMP_NTZ")) { + def v(index: Int): String = if (kind == "TIMESTAMP") { + s"TIMESTAMP '${instants(index - 1)}'" + } else s"TIMESTAMP_NTZ '${wall(index).format(sixDigits).replace('T', ' ')}'" + def index(row: Row): Option[Int] = row.getInt(0) match { + case 7 => None + case id => Some(if (field.startsWith("p_")) intMaxPartition(id) else id) + } + for ((label, predicate, selected) <- Seq( + ("below_negative", s"$field < ${v(2)}", Set(1)), + ("eq_negative", s"$field = ${v(2)}", Set(2)), + ("interior", s"$field > ${v(2)} AND $field < ${v(5)}", Set(3, 4)), + ("eq_positive", s"$field = ${v(5)}", Set(5)), + ("above_positive", s"$field > ${v(5)}", Set(6)), + ("outside", s"$field < ${v(1)} OR $field > ${v(6)}", Set.empty[Int]))) { + checkLiteralRead(table, s"${field}_$label", intMaxSchema, + rows.filter(r => index(r).exists(selected)), predicate) + } + checkLiteralRead(table, s"${field}_is_null", intMaxSchema, + rows.filter(_.getInt(0) == 7), s"$field IS NULL") + } + checkLiteralSnapshot(table, intMaxSchema, Seq("p_ts", "p_ntz")) + checkLiteralSnapshot(table, intMaxSchema, Seq("p_ts", "p_ntz"), version = 1L) + } + } + + // DATE32 epoch-day bounds. 2147483648 is outside DATE32 and is not an input. Rows use + // java.time.LocalDate: java.sql.Date misrenders these years. Raw INT32 days are the oracle. + test("date_epoch_days_int32_bounds") { + setForTest("spark.sql.session.timeZone", "UTC") + setForTest("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED") + setForTest("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED") + // Spark's data-filter translation rebases java.sql.Date literals to Julian days, which + // overflows near -2147483648. LocalDate literals avoid that reference-side conversion. + // setForTest keeps it through capture and restores it after the test, including on failure. + setForTest(java8Key, "true") + val days = Seq(-2147483648, -2147483647, -2147483646, 2147483646, 2147483647) + val dates = Seq("-5877641-06-23", "-5877641-06-24", "-5877641-06-25", "+5881580-07-10", + "+5881580-07-11").map(LocalDate.parse) + require(dates.map(_.toEpochDay) == days.map(_.toLong)) + val permutation = Map(1 -> 3, 2 -> 5, 3 -> 1, 4 -> 2, 5 -> 4) + val ids = Seq(1, 2, 2, 3, 4, 5, 5, 6, 6) + val expectedSchema = new StructType().add("id", IntegerType) + .add("d", org.apache.spark.sql.types.DateType).add("p", org.apache.spark.sql.types.DateType) + val rows = ids.map { + case 6 => Row(6, null, null) + case id => Row(id, dates(id - 1), dates(permutation(id) - 1)) + } + val units = ids.map { + case 6 => (6, None, None) + case id => (id, Some(days(id - 1)), Some(days(permutation(id) - 1))) + } + sql("CREATE TABLE tbl (id INT, d DATE, p DATE) USING delta PARTITIONED BY (p)") + val inputs = ids.map { + case 6 => "(6, CAST(NULL AS INT), CAST(NULL AS INT))" + case id => s"($id, ${days(id - 1)}, ${days(permutation(id) - 1)})" + } + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ id, date_from_unix_date(d), " + + s"date_from_unix_date(p) FROM VALUES ${inputs.mkString(", ")} AS rows(id, d, p)") + val table = registerTable("tbl") + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 6) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + require(stats.get("numRecords").asLong() == (if (Set(2, 5, 6)(id)) 2L else 1L)) + if (id == 6) { + require(add.partitionValues == Map("p" -> null)) + require(stats.at("/nullCount/d").asLong() == 2L && !stats.at("/minValues").has("d")) + } else { + require(add.partitionValues == Map("p" -> dates(permutation(id) - 1).toString)) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/d").asText() == dates(id - 1).toString) + } + } + println(s"Verified DATE32 bound file: id=$id stats=${add.stats.get} " + + s"partition=${add.partitionValues}") + } + } + def literal(index: Int): String = s"DATE '${dates(index - 1)}'" + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + for (field <- Seq("d", "p")) { + def index(row: Row): Option[Int] = row.getInt(0) match { + case 6 => None + case id => Some(if (field == "p") permutation(id) else id) + } + for ((label, predicate, selected) <- Seq( + ("below_negative", s"$field < ${literal(2)}", Set(1)), + ("eq_negative", s"$field = ${literal(2)}", Set(2)), + ("interior", s"$field > ${literal(2)} AND $field < ${literal(5)}", Set(3, 4)), + ("eq_positive", s"$field = ${literal(5)}", Set(5)))) { + checkLiteralRead(table, s"${field}_$label", expectedSchema, + rows.filter(r => index(r).exists(selected)), predicate) + } + checkLiteralRead(table, s"${field}_is_null", expectedSchema, + rows.filter(_.getInt(0) == 6), s"$field IS NULL") + } + checkLiteralSnapshot(table, expectedSchema, Seq("p")) + checkLiteralSnapshot(table, expectedSchema, Seq("p"), version = 1L) + // Snapshots capture before reads at this pin. Inspect Parquet in the last read callback. + val workload = WorkloadContext.current.workloadName + readSpec(table, predicate = s"d < ${literal(1)} OR d > ${literal(5)}", + name = Some("outside")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == 0L) + case other => fail(s"Outside DATE32 bounds must succeed with no rows: $other") + } + val conf = spark.sparkContext.hadoopConfiguration + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_all").resolve("expected_data") + val literalOracle = outputDir.resolveSibling("oracle-parquet").resolve(workload) + for ((label, directory) <- Seq("captured" -> captured, "literal" -> literalOracle)) { + val files = Files.list(directory) + val parquet = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toVector finally files.close() + require(parquet.nonEmpty) + val found = parquet.flatMap { file => + val path = new HadoopPath(file.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try for (field <- Seq("d", "p")) { + val physical = footer.getFooter.getFileMetaData.getSchema.getFields.asScala + .find(_.getName == field).get.asPrimitiveType() + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(physical.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + var out = Vector.empty[(Int, Option[Int], Option[Int])] + try { + var record = reader.read() + while (record != null) { + def day(field: String) = if (record.getFieldRepetitionCount(field) == 0) None + else Some(record.getInteger(field, 0)) + out :+= ((record.getInteger("id", 0), day("d"), day("p"))) + record = reader.read() + } + } finally reader.close() + out + } + require(found.sorted == units.sorted) + println(s"Verified raw DATE32 epoch days: $workload $label ${found.sorted}") + } + require(spark.conf.get(java8Key) == "true") + } + } + private val java8Key = "spark.sql.datetime.java8API.enabled" + + // Replace producer-derived parsed bounds by the literal instants and drop the JSON fallback. + private def rewriteIntMaxCheckpoint(checkpoint: Path, instants: Seq[Instant]): Unit = { + val rewrite = checkpoint.resolveSibling("int-max-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + var ts = col(s"add.stats_parsed.$bound.ts") + var ntz = col(s"add.stats_parsed.$bound.ntz") + for ((instant, index) <- instants.zipWithIndex) { + val matches = col("add.stats_parsed.minValues.id") === lit(index + 1) + ts = when(matches, lit(Timestamp.from(instant))).otherwise(ts) + ntz = when(matches, + lit(java.time.LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC))) + .otherwise(ntz) + } + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("ts", ts).withField("ntz", ntz)) + } + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewrite.toString) + val files = Files.list(rewrite) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten INT_MAX checkpoint: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewrite)) FileUtils.deleteDirectory(rewrite.toFile) + } + } + + private def checkIntMaxCheckpoint( + checkpoint: Path, instants: Seq[Instant], partitions: Int => Map[String, String]): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + for ((prefix, field) <- Seq("stats_parsed.minValues" -> "ts", + "stats_parsed.minValues" -> "ntz", "stats_parsed.maxValues" -> "ts", + "stats_parsed.maxValues" -> "ntz", "partitionValues_parsed" -> "p_ts", + "partitionValues_parsed" -> "p_ntz")) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == !field.endsWith("ntz")) + println(s"Verified INT_MAX checkpoint $prefix.$field: $cell") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var ids = Vector.empty[Int] + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + val add = record.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + require(stats.getLong("numRecords", 0) == (if (Set(2, 5, 7)(id)) 2L else 1L)) + val parsedPartition = add.getGroup("partitionValues_parsed", 0) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + for (field <- Seq("ts", "ntz")) { + if (id == 7) require(values.getFieldRepetitionCount(field) == 0) + else require(values.getLong(field, 0) == mismatchMicros(instants(id - 1))) + } + } + for (field <- Seq("p_ts", "p_ntz")) { + if (id == 7) require(parsedPartition.getFieldRepetitionCount(field) == 0) + else require(parsedPartition.getLong(field, 0) == + mismatchMicros(instants(intMaxPartition(id) - 1))) + } + if (id == 7) require(stats.getGroup("nullCount", 0).getLong("ts", 0) == 2L) + val raw = add.getGroup("partitionValues", 0) + val partition = (0 until raw.getFieldRepetitionCount("key_value")).map { i => + val entry = raw.getGroup("key_value", i) + val value = if (entry.getFieldRepetitionCount("value") == 0) null + else entry.getString("value", 0) + entry.getString("key", 0) -> value + }.toMap + require(partition == partitions(id)) + ids :+= id + println(s"Verified INT_MAX raw checkpoint id=$id partition=$partition") + } + record = reader.read() + } + } finally reader.close() + require(ids.sorted == (1 to 7).toVector) + } + + // Table operations that rewrite statistics or file layout. + test("e2e_51_stats_refresh") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'drop'), (0, 'low'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (12, 'middle'), (18, CAST(NULL AS STRING)), (31, 'last'), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("DELETE FROM tbl WHERE id = -7") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val physicalRows = Seq(Row(-7, "drop"), Row(0, "low"), Row(4, null), Row(4, null), + Row(12, "middle"), Row(18, null), Row(31, "last"), Row(null, "missing")) + val rows = Seq(Row(0, "low"), Row(4, null), Row(4, null), Row(12, "middle"), + Row(18, null), Row(31, "last"), Row(null, "missing")) + val original = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = Protocol(3, 7, Some(Set("deletionVectors")), + Some(Set("appendOnly", "deletionVectors", "invariants"))) + assert(original(0L).collect { case p: Protocol => p } == Seq(protocol)) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inserted = original(1L).collect { case a: AddFile => a }.ensuring(_.size == 1).head + val deleted = original(2L).collect { case a: AddFile => a }.ensuring(_.size == 1).head + assert(inserted.path == deleted.path && inserted.deletionVector.isEmpty) + val dv = deleted.deletionVector.get + assert(dv.storageType == "u" && dv.cardinality == 1L) + val physical = spark.read.parquet(table.sourcePath.resolve(deleted.path).toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(physicalRows.asJava, schema), physical, + "replay literal physical rows") + val positions = physical.collect().toSeq.zipWithIndex.collect { + case (row, index) if !row.isNullAt(0) && row.getInt(0) == -7 => index.toLong + } + val dvBytes = payload(table.sourcePath, dv) + assert(RoaringBitmapArray.readFrom(dvBytes).toArray.toSeq == positions) + val fileBytes = Files.readAllBytes(table.sourcePath.resolve(deleted.path)) + // Physical stats include the deleted row. These bounds remain conservative for the DV. + // Model a partial-stats writer followed by a stats-only update, not an autostats service. + val partialStats = """{"numRecords":8,"tightBounds":false}""" + val refreshedStats = """{"numRecords":8,"minValues":{"id":-7,"value":"drop"}, + "maxValues":{"id":31,"value":"missing"},"nullCount":{"id":1,"value":3}, + "tightBounds":false}""" + val partial = deleted.copy(stats = Some(partialStats)) + val refreshed = partial.copy(stats = Some(refreshedStats), dataChange = false) + assert(refreshed.copy(stats = deleted.stats, dataChange = deleted.dataChange) == deleted) + assert(refreshed.path == deleted.path && refreshed.deletionVector == Some(dv)) + assert(refreshed.baseRowId == deleted.baseRowId) + assert(refreshed.defaultRowCommitVersion == deleted.defaultRowCommitVersion) + // Row tracking is not enabled: preserve absent row-ID fields as well as the real DV. + assert(deleted.baseRowId.isEmpty && deleted.defaultRowCommitVersion.isEmpty) + modifyCommitActions(table, 2L) { actions => + actions.map { case _: AddFile => partial; case other => other } + } + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + assert(CommitLog.read(dir, 0L) == original(0L)) + assert(CommitLog.read(dir, 1L) == original(1L)) + assert(CommitLog.read(dir, 2L) == original(2L).map { + case _: AddFile => partial; case other => other + }) + assert(!Files.exists(CommitLog.commitFile(dir, 3L))) + // One AddFile per identity per version; no conflicting same-version remove/add pair. + CommitLog.write(dir, 3L, Seq(refreshed)) + val logs = Files.list(dir.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + DeltaHarness.get.openLog(spark, dir.toString).checkpoint(3L) + val checkpoint = dir.resolve("_delta_log/00000000000000000003.checkpoint.parquet") + assert(Files.isRegularFile(checkpoint)) + val checkpointAdds = spark.read.parquet(checkpoint.toString) + .filter("add IS NOT NULL").select("add.*").toJSON.collect().toSeq + .map(json => Action.parse(s"""{"add":$json}""")) + assert(checkpointAdds == Seq(refreshed)) + val lastCheckpoint = JsonUtil.mapper.readTree( + Files.readAllBytes(dir.resolve("_delta_log/_last_checkpoint"))) + assert(lastCheckpoint.get("version").asLong() == 3L) + // Checkpoint creation can write fresh Hadoop sidecars; retain only protocol artifacts. + val checkpointLogs = Files.list(dir.resolve("_delta_log")) + try checkpointLogs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach { file => + assert(file.getFileName.toString.startsWith(".")) + Files.delete(file) + } + finally checkpointLogs.close() + // A transaction marker provides a genuine JSON tail without changing the row bag. + CommitLog.write(dir, 4L, Seq(Txn("readiness_e2e_51_tail", 1L))) + assert(CommitLog.read(dir, 3L) == Seq(refreshed)) + assert(CommitLog.read(dir, 4L) == Seq(Txn("readiness_e2e_51_tail", 1L))) + assert(Files.readAllBytes(dir.resolve(deleted.path)).sameElements(fileBytes)) + assert(payload(dir, dv).sameElements(dvBytes)) + DeltaLog.clearCache() + val replay = DeltaLog.forTable(spark, dir.toString).update() + assert(replay.version == 4L) + assert(replay.logSegment.checkpointProvider.version == 3L) + assert(replay.logSegment.deltas.map(_.getPath.getName) == Seq(f"${4L}%020d.json")) + (2L to 4L).foreach { version => + val snapshot = DeltaHarness.get.openLog(spark, dir.toString).getSnapshotAt(version) + val active = snapshot.allFiles.select("json").collect().toSeq + .map(r => Action.parse(r.getString(0))) + // Delta's active-file replay normalizes dataChange; commit flags were checked above. + val expectedAdd = (if (version == 2L) partial else refreshed).copy(dataChange = false) + assert(active == Seq(expectedAdd)) + } + val remaining = Files.list(dir.resolve("_delta_log")) + try assert(!remaining.iterator().asScala.exists(_.getFileName.toString.endsWith(".crc"))) + finally remaining.close() + info("replay checkpoint v3 + JSON tail v4 selected; physical file and DV unchanged") + } + val expectedSnapshot = SnapshotResult( + ProtocolInfo(3, 7, Some(Seq("deletionVectors")), + Some(Seq("appendOnly", "deletionVectors", "invariants"))), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](2L, 3L, 4L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + val resolved = Option(version).fold(4L)(_.toLong) + assert(spec.query == SnapshotQuery(version = Some(resolved))) + assert(spec.expectation == Succeeded(if (resolved == 4L) { + expectedSnapshot.copy(setTransactions = Some(Seq(AppTxn("readiness_e2e_51_tail", 1L)))) + } else expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val selective = "id IN (4, 12) OR id IS NULL" + val selected = Seq(Row(null, 4), Row(null, 4), Row("middle", 12), Row("missing", null)) + val versions = Seq("before" -> Some(2L), "refreshed" -> Some(3L), + "tail" -> Some(4L), "latest" -> None) + versions.foreach { case (label, version) => + val cases = Seq( + (s"read_$label", ReadQuery(version = version), schema, rows), + (s"read_${label}_selective", ReadQuery(version = version, + predicate = Some(selective), columns = Some(Seq("value", "id"))), projected, selected), + (s"read_${label}_miss", ReadQuery(version = version, predicate = Some("id > 100")), + schema, Seq.empty[Row])) + cases.foreach { case (name, query, expectedSchema, expectedRows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == expectedRows.size.toLong) + case other => fail(s"replay $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(expectedRows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"replay $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"replay $name expected parquet") + } + } + } + } + + Seq("zorder", "liquid").foreach { mode => + test(s"e2e_49_$mode") { + val liquid = mode == "liquid" + val clusterBy = if (liquid) "CLUSTER BY (id, value)" else "" + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta $clusterBy TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (8, 'late'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-2, 'early'), (5, 'middle'), (8, 'late'), (12, CAST(NULL AS STRING)) AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + // Independent literal bag: includes duplicate whole rows and nulls in both columns. + val firstRows = Seq(Row(8, "late"), Row(3, null), Row(3, null), Row(null, "missing")) + val allRows = firstRows ++ Seq(Row(-2, "early"), Row(5, "middle"), Row(8, "late"), + Row(12, null)) + val before = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = if (liquid) { + Protocol(1, 7, None, Some(Set("appendOnly", "invariants", "clustering", "domainMetadata"))) + } else Protocol(1, 2) + assert(before(0L).collect { case p: Protocol => p } == Seq(protocol)) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val domains = before(0L).collect { case d: DomainMetadata => d } + if (liquid) { + assert(domains.size == 1 && domains.head.domain == "delta.clustering") + assert(!domains.head.removed) + val configuration = JsonUtil.mapper.readTree(domains.head.configuration) + assert(configuration.get("clusteringColumns") == + JsonUtil.mapper.readTree("""[["id"],["value"]]""")) + } else assert(domains.isEmpty) + val literal = spark.createDataFrame(allRows.asJava, schema) + val unoptimized = SnapshotResolver.buildDeltaReader(spark, table.sourcePath, + Some(2L), None) + assert(unoptimized.schema == schema) + RowComparison.assertRowsEqual(literal, unoptimized, s"$mode before OPTIMIZE") + sql(if (liquid) "OPTIMIZE tbl" else "OPTIMIZE tbl ZORDER BY (id, value)") + val optimized = CommitLog.read(table.sourcePath, 3L) + val adds = optimized.collect { case a: AddFile => a } + val removes = optimized.collect { case r: RemoveFile => r } + // Check actual rewrite actions, without promising a file count or layout effectiveness. + assert(adds.nonEmpty && removes.nonEmpty) + assert(adds.forall(!_.dataChange) && removes.forall(!_.dataChange)) + val inputAdds = before.values.flatten.collect { case a: AddFile => a }.toSeq + assert(removes.map(_.path).toSet == inputAdds.map(_.path).toSet) + assert(adds.forall(_.clusteringProvider == (if (liquid) Some("liquid") else None))) + assert(optimized.collect { case m: Metadata => m }.isEmpty) + assert(optimized.collect { case p: Protocol => p }.isEmpty) + assert(optimized.collect { case d: DomainMetadata => d }.isEmpty) + val commitInfo = optimized.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "OPTIMIZE") + val parameters = commitInfo.get("operationParameters") + val columns = JsonUtil.mapper.readTree("""["id","value"]""") + val empty = JsonUtil.mapper.readTree("[]") + assert(JsonUtil.mapper.readTree(parameters.get("zOrderBy").asText()) == + (if (liquid) empty else columns)) + assert(JsonUtil.mapper.readTree(parameters.get("clusterBy").asText()) == + (if (liquid) columns else empty)) + info(s"cluster $mode protocol: " + protocol.toJson) + info(s"cluster $mode metadata: " + metadata.toJson) + info(s"cluster $mode optimize: " + commitInfo.toString) + domains.foreach(d => info(s"cluster $mode domain: " + d.toJson)) + adds.foreach(a => info(s"cluster $mode optimized add: " + a.toJson)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + (0L to 2L).foreach(v => assert(CommitLog.read(dir, v) == before(v))) + assert(CommitLog.read(dir, 3L) == optimized) + } + val expectedSnapshot = SnapshotResult( + ProtocolInfo(protocol.minReaderVersion, protocol.minWriterVersion, + writerFeatures = protocol.writerFeatures.map(_.toSeq.sorted)), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime), + domainMetadata = if (liquid) Some(domains.map(d => + AddDomainMetadata(d.domain, d.configuration))) else None) + Seq[java.lang.Long](0L, 2L, 3L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val predicate = "(id >= 3 AND id <= 8) OR id IS NULL" + val firstFiltered = Seq(Row("late", 8), Row(null, 3), Row(null, 3), Row("missing", null)) + val allFiltered = firstFiltered ++ Seq(Row("middle", 5), Row("late", 8)) + val cases = Seq( + ("read_latest", ReadQuery(), schema, allRows), + ("read_first_insert", ReadQuery(version = Some(1L)), schema, firstRows), + ("read_before_optimize", ReadQuery(version = Some(2L)), schema, allRows), + ("read_optimized_v3", ReadQuery(version = Some(3L)), schema, allRows), + ("read_first_filtered", ReadQuery(version = Some(1L), predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, firstFiltered), + ("read_latest_filtered", ReadQuery(predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, allFiltered), + ("read_optimized_filtered", ReadQuery(version = Some(3L), predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, allFiltered)) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$mode $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"$mode $name copied table") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"$mode $name expected parquet") + } + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorPayloads.scala b/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorPayloads.scala new file mode 100644 index 00000000..8b50714a --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorPayloads.scala @@ -0,0 +1,61 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.io.DataInputStream +import java.nio.file.{Files, Path, Paths} + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore +import org.scalatest.Assertions._ + +import io.delta.workload.log.DeletionVector + +/** Reads the exact serialized bitmap of an inline or UUID-relative deletion vector. */ +object DeletionVectorPayloads { + + def descriptor(dv: DeletionVector): DeletionVectorDescriptor = { + // At this DAT pin USE_LONG_FOR_INTS boxes Option[Int] offsets as Long. Normalize + // only this descriptor input, without changing the shared mapper or DV bytes. + val offset = dv.offset.asInstanceOf[Option[Number]].map(n => Math.toIntExact(n.longValue())) + DeletionVectorDescriptor(dv.storageType, dv.pathOrInlineDv, offset, + dv.sizeInBytes, dv.cardinality) + } + + def sidecar(table: Path, dv: DeletionVector): Path = + Paths.get(descriptor(dv).absolutePath(new HadoopPath(table.toUri)).toUri.getPath) + + def payload(table: Path, dv: DeletionVector): Array[Byte] = { + if (dv.storageType == "i") descriptor(dv).inlineData + else { + assert(dv.storageType == "u") + val file = sidecar(table, dv) + assert(file.normalize().startsWith(table.normalize())) + val offset = descriptor(dv).offset.get + assert(offset > 0) + assert(Files.size(file) >= offset.toLong + 4L + dv.sizeInBytes + 4L) + val input = new DataInputStream(Files.newInputStream(file)) + try { + assert(input.readByte() == DeletionVectorStore.DV_FILE_FORMAT_VERSION_ID_V1) + input.skipNBytes(offset.toLong - 1L) + // This OSS reader checks both the payload length and checksum. + DeletionVectorStore.readRangeFromStream(input, dv.sizeInBytes) + } finally input.close() + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorsSuite.scala index d94a5b7f..7e79dc0e 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorsSuite.scala @@ -16,7 +16,23 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.{Files, Path} + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{Action, AddFile, CommitLog, DeletionVector, Metadata, Protocol, + RemoveFile} +import io.delta.workload.model._ +import io.delta.workload.tables.DeletionVectorPayloads.{payload, sidecar} class DeletionVectorsSuite extends WorkloadTestSuite("deletion_vectors") { @@ -793,4 +809,174 @@ class DeletionVectorsSuite extends WorkloadTestSuite("deletion_vectors") { snapshotSpec(t) } + // Inline and UUID-relative deletion vectors with the same engine-written bitmap. + Seq("inline", "uuid_relative").foreach { form => + test(s"e2e_42_$form") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'remove'), (0, 'keep'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (9, 'remove'), (12, 'keep'), (18, CAST(NULL AS STRING)), (23, 'edge'), + (31, 'keep'), (44, CAST(NULL AS STRING)), (57, 'remove'), (88, 'last') AS v(id, value)""") + sql("DELETE FROM tbl WHERE id IN (-7, 9, 57)") + sql("DELETE FROM tbl WHERE id = 4") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val baseRows = Seq(Row(-7, "remove"), Row(0, "keep"), Row(4, null), Row(4, null), + Row(9, "remove"), Row(12, "keep"), Row(18, null), Row(23, "edge"), + Row(31, "keep"), Row(44, null), Row(57, "remove"), Row(88, "last")) + val firstRows = baseRows.filterNot(r => Set(-7, 9, 57).contains(r.getInt(0))) + val latestRows = firstRows.filterNot(_.getInt(0) == 4) + val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = original(0L).collect { case p: Protocol => p }.ensuring(_.size == 1).head + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + assert(protocol == Protocol(3, 7, Some(Set("deletionVectors")), + Some(Set("appendOnly", "deletionVectors", "invariants")))) + info("DV original protocol: " + protocol.toJson) + info("DV original metadata: " + metadata.toJson) + val adds = (1L to 3L).map { version => + val found = original(version).collect { case a: AddFile => a } + assert(found.size == 1) + info(s"DV original add v$version: " + found.head.toJson) + version -> found.head + }.toMap + assert(adds.values.map(_.path).toSet.size == 1) // Deletes must retain the data file. + assert(adds(1L).deletionVector.isEmpty) + val physical = spark.read.parquet(table.sourcePath.resolve(adds(1L).path).toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(baseRows.asJava, schema), physical, + s"$form physical input") + val physicalRows = physical.collect().toSeq + val sourceDvs = (2L to 3L).map(v => v -> adds(v).deletionVector.get).toMap + sourceDvs.foreach { case (version, dv) => + assert(dv.storageType == "u") + assert(dv.cardinality == (if (version == 2L) 3L else 5L)) + val expectedPositions = physicalRows.zipWithIndex.collect { + case (row, index) if Set(-7, 9, 57).contains(row.getInt(0)) || + (version == 3L && row.getInt(0) == 4) => index.toLong + } + assert(RoaringBitmapArray.readFrom(payload(table.sourcePath, dv)).toArray.toSeq == + expectedPositions) + } + val converted = sourceDvs.values.map { dv => + val bytes = payload(table.sourcePath, dv) + // Use the verified OSS helper with exact engine-generated payload bytes. + val inline = DeletionVectorDescriptor.inlineInLog(bytes, dv.cardinality) + assert(inline.storageType == "i" && inline.offset.isEmpty) + assert(inline.sizeInBytes == bytes.length && inline.cardinality == dv.cardinality) + assert(inline.inlineData.sameElements(bytes)) + dv -> DeletionVector(inline.storageType, inline.pathOrInlineDv, inline.offset, + inline.sizeInBytes, inline.cardinality) + }.toMap + if (form == "inline") { + (2L to 3L).foreach { version => + modifyCommitActions(table, version) { actions => + actions.map { + case add: AddFile => add.copy(deletionVector = add.deletionVector.map(converted)) + // The second delete removes the first DV identity; convert that tombstone too. + case remove: RemoveFile => + remove.copy(deletionVector = remove.deletionVector.map(converted)) + case other => other + } + } + } + } + val expectedAdds = adds.map { case (version, add) => + version -> (if (form == "inline") { + add.copy(deletionVector = add.deletionVector.map(converted)) + } else add) + } + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + val sidecars = sourceDvs.values.map(dv => sidecar(dir, dv)).toSet + if (form == "inline") sidecars.foreach(Files.delete) + else sidecars.foreach(p => assert(Files.isRegularFile(p))) + // Version checksums embed the original descriptors and become stale after conversion. + val logs = Files.list(dir.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + (1L to 3L).foreach { version => + val actions = CommitLog.read(dir, version) + assert(actions.collect { case a: AddFile => a } == Seq(expectedAdds(version))) + expectedAdds(version).deletionVector.foreach { dv => + assert(dv.storageType == (if (form == "inline") "i" else "u")) + assert(dv.cardinality == (if (version == 2L) 3L else 5L)) + assert(payload(dir, dv).sameElements(payload(table.sourcePath, sourceDvs(version)))) + assert(RoaringBitmapArray.readFrom(payload(dir, dv)).cardinality == dv.cardinality) + } + } + val oldIdentity = CommitLog.read(dir, 3L).collect { case r: RemoveFile => r } + assert(oldIdentity.size == 1) + assert(oldIdentity.head.path == adds(1L).path) + assert(oldIdentity.head.deletionVector == expectedAdds(2L).deletionVector) + assert(Files.size(dir.resolve(adds(1L).path)) == adds(1L).size) + if (form == "inline") assert(sidecars.forall(p => !Files.exists(p))) + } + val snapshot = SnapshotResult( + ProtocolInfo(3, 7, Some(Seq("deletionVectors")), + Some(Seq("appendOnly", "deletionVectors", "invariants"))), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), metadata.schemaString, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](1L, 2L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(snapshot)) + } + } + val projected = new StructType().add("value", StringType) + val reordered = projected.add("id", IntegerType) + val filter = "id IN (4, 12, 18, 57)" + val cases = Seq( + ("read_base", ReadQuery(version = Some(1L)), schema, baseRows), + ("read_first_delete", ReadQuery(version = Some(2L)), schema, firstRows), + ("read_latest", ReadQuery(), schema, latestRows), + ("read_v2_filtered_projected", ReadQuery(version = Some(2L), + predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered, + Seq(Row(null, 4), Row(null, 4), Row("keep", 12), Row(null, 18))), + ("read_v3_filtered_projected", ReadQuery(version = Some(3L), + predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered, + Seq(Row("keep", 12), Row(null, 18))), + ("read_projected", ReadQuery(columns = Some(Seq("value"))), projected, + Seq(Row("keep"), Row("keep"), Row(null), Row("edge"), Row("keep"), Row(null), + Row("last"))), + ("read_deleted_only", ReadQuery(predicate = Some("id IN (-7, 4, 9, 57)")), schema, + Seq.empty[Row])) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + // All predicates overlap physical min/max; the single data AddFile remains scanned. + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, 1, 0L))) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, expectedSchema), actual, + s"$form $name") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_metadata").toString) + val scanned = captured.collect().toSeq.map { row => + val node = JsonUtil.mapper.readTree(row.getString(0)) + val wrapped = if (node.has("add")) node.toString else s"""{"add":$node}""" + Action.parse(wrapped) + } + // OSS InMemoryLogReplay stores active AddFiles with dataChange=false; + // ReadCapture obtains these snapshot actions through DeltaSparkHarness.allFiles. + assert(scanned == Seq(expectedAdds(query.version.getOrElse(3L)) + .copy(dataChange = false))) + } + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/LiteralOracleChecks.scala b/workload-generator/src/test/scala/io/delta/workload/tables/LiteralOracleChecks.scala new file mode 100644 index 00000000..7d39bf5f --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/LiteralOracleChecks.scala @@ -0,0 +1,89 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, StructType} + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.model.Succeeded + +/** + * Read and snapshot checks against literal rows and an independent schema, shared by the + * temporal statistics and partition-value workloads. Row oracles are never read from Delta. + */ +trait LiteralOracleChecks { this: WorkloadTestSuite => + + protected def outputDir: Path + + /** + * Declares a read whose captured rows must equal the literal `rows`. The `all` oracle is + * also written to `oracle-parquet/` next to the output directory, for physical + * Parquet checks in later assertions. + */ + protected def checkLiteralRead( + table: TableHandle, name: String, expectedSchema: StructType, rows: Seq[Row], + predicate: String = null, version: java.lang.Long = null): Unit = { + val workload = WorkloadContext.current.workloadName + readSpec(table, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed against the independent oracle: $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$name").resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + if (name == "all") { + expected.coalesce(1).write.mode("overwrite") + .parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString) + } + println(s"Verified independent literal rows/schema: $workload/$name rows=${rows.size}") + } + } + + protected def checkLiteralSnapshot( + table: TableHandle, expectedSchema: StructType, partitions: Seq[String] = Seq.empty, + version: java.lang.Long = null): Unit = { + snapshotSpec(table, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == partitions) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + val ntz = expectedSchema.fields.exists(_.dataType == + org.apache.spark.sql.types.TimestampNTZType) + require(result.protocol.minReaderVersion == (if (ntz) 3 else 1)) + require(result.protocol.minWriterVersion == (if (ntz) 7 else 2)) + case other => fail(s"Snapshot must succeed against independent metadata: $other") + } + } + } + + /** Removes version checksums, which can carry file actions from before a mutation. */ + protected def removeVersionChecksums(dir: Path): Unit = { + val files = Files.list(dir.resolve("_delta_log")) + try files.iterator().asScala.filter(_.toString.endsWith(".crc")) + .foreach(Files.delete(_)) finally files.close() + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/LogReplaySuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/LogReplaySuite.scala index ae228096..d3397f15 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/LogReplaySuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/LogReplaySuite.scala @@ -16,7 +16,20 @@ package io.delta.workload.tables +import java.nio.file.{Files, Path} + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + import io.delta.workload.WorkloadTestSuite +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{Action, AddFile, CommitLog, Metadata, Protocol, RemoveFile} +import io.delta.workload.model._ /** * Log Replay + Production Edge Cases workloads (lr_* + log_* + lc_* + prod_*). @@ -24,7 +37,8 @@ import io.delta.workload.WorkloadTestSuite * checkpoint, metadata latest wins, dataChange=false from compaction, missing * protocol/metadata errors, add-remove-readd, DV key dedup, last checkpoint info, * production edge cases (empty table, many commits, version gaps, truncated logs, - * unknown reader features, varchar metadata, duplicate file refs, external checkpoints). + * unknown reader features, varchar metadata, duplicate file refs, external checkpoints), + * minor log compaction and automatic compaction. * */ class LogReplaySuite extends WorkloadTestSuite("log_replay") { @@ -263,4 +277,297 @@ class LogReplaySuite extends WorkloadTestSuite("log_replay") { readSpec(t) } + // Minor log compaction and automatic compaction. + test("e2e_56_minor_log_compaction") { + import org.apache.spark.sql.delta.actions.{Action => SparkAction, InMemoryLogReplay} + import org.apache.spark.sql.delta.util.FileNames + + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'initial'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing'), (12, 'keep'), (12, 'keep') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (6, 'replacement'), (6, 'replacement'), (CAST(NULL AS INT), 'changed'), + (8, CAST(NULL AS STRING)) AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), (23, 'end') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (31, 'tail'), (31, 'tail'), (CAST(NULL AS INT), 'tail-null'), + (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)) AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (44, 'last'), (44, 'last') AS v(id, value)""") + val table = registerTable("tbl") + val original = (0L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(original(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val adds = (1L to 5L).map { v => + v -> original(v).collect { case a: AddFile => a }.ensuring(_.size == 1).head + }.toMap + assert(adds.values.map(_.path).toSet.size == 5) + assert(original(2L).collect { case r: RemoveFile => r.path } == Seq(adds(1L).path)) + assert(original(4L).collect { case r: RemoveFile => r.path }.toSet == + Set(adds(2L).path, adds(3L).path)) + + // Follow the pinned OSS DeltaLogMinorCompactionSuite: reconcile [0, 3], with + // no retention pruning, then serialize the existing OSS checkpoint iterator. + // Concatenating commits would retain cancelled adds and commitInfo, which is invalid. + val replay = new InMemoryLogReplay(None, None) + (0L to 3L).foreach { v => + val lines = Files.readAllLines(table.sourcePath.resolve(f"_delta_log/$v%020d.json")) + replay.append(v, lines.asScala.iterator.map(SparkAction.fromJson)) + } + val sourceLog = DeltaLog.forTable(spark, table.sourcePath.toString) + val compactedPath = FileNames.compactedDeltaFile(sourceLog.logPath, 0L, 3L) + val compactedName = "00000000000000000000.00000000000000000003.compacted.json" + assert(compactedPath.getName == compactedName) + sourceLog.store.write(compactedPath, replay.checkpoint.map(_.json), + overwrite = false, sourceLog.newDeltaHadoopConf()) + val compactedBytes = Files.readAllBytes(table.sourcePath.resolve(s"_delta_log/$compactedName")) + val compacted = Files.readAllLines(table.sourcePath.resolve(s"_delta_log/$compactedName")) + .asScala.map(Action.parse).toSeq + assert(compacted.size == 5) + assert(compacted.collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(compacted.collect { case m: Metadata => m } == Seq(metadata)) + // OSS Action.fromJson materializes absent optional tags as an empty map. + // FileAction.tagsOrEmpty defines their equivalence; preserve every nonempty tag. + val expectedAdds = Set(adds(2L), adds(3L)).map(a => + a.copy(dataChange = false, tags = a.tags.filter(_.nonEmpty))) + assert(compacted.collect { case a: AddFile => a.copy(tags = a.tags.filter(_.nonEmpty)) } + .toSet == expectedAdds) + assert(compacted.collect { case r: RemoveFile => r.copy(tags = r.tags.filter(_.nonEmpty)) } == + original(2L).collect { case r: RemoveFile => + r.copy(dataChange = false, tags = r.tags.filter(_.nonEmpty)) }) + info("compact compacted [0,3] actions: " + compacted.map(_.toJson).mkString("\n")) + + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + assert(Files.readAllBytes(dir.resolve(s"_delta_log/$compactedName")) + .sameElements(compactedBytes)) + // Compactions are optional accelerators. OSS validates original commit continuity; + // neither compaction nor protocol cleanup authorizes removing this interval. + (0L to 5L).foreach { v => + assert(CommitLog.read(dir, v) == original(v)) + assert(Files.readAllBytes(dir.resolve(f"_delta_log/$v%020d.json")) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(f"_delta_log/$v%020d.json")))) + } + adds.values.foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + val listing = Files.list(dir.resolve("_delta_log")) + try assert(!listing.iterator().asScala.exists(_.getFileName.toString.contains("checkpoint"))) + finally listing.close() + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + Seq[java.lang.Long](0L, 1L, 2L, 3L, 4L, 5L, null).foreach { version => + val v = Option(version).fold(5L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + val selected = snapshot.logSegment.deltas.map(_.getPath.getName) + val expected = if (v < 3L) (0L to v).map(n => f"$n%020d.json") + else Seq(compactedName) ++ (4L to v).map(n => f"$n%020d.json") + assert(selected == expected) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + info(s"compact v$v selected log files: " + selected.mkString(",")) + } + } + val initialRows = Seq(Row(-7, "initial"), Row(4, null), Row(4, null), + Row(null, "missing"), Row(12, "keep"), Row(12, "keep")) + val replacementRows = Seq(Row(6, "replacement"), Row(6, "replacement"), + Row(null, "changed"), Row(8, null)) + val endRows = replacementRows ++ Seq(Row(4, null), Row(4, null), Row(23, "end")) + val tailRows = Seq(Row(31, "tail"), Row(31, "tail"), Row(null, "tail-null"), + Row(4, null), Row(4, null)) + val latestRows = tailRows ++ Seq(Row(44, "last"), Row(44, "last")) + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_start_v0", ReadQuery(version = Some(0L)), schema, Seq.empty[Row]), + ("read_inside_v1", ReadQuery(version = Some(1L)), schema, initialRows), + ("read_inside_v2", ReadQuery(version = Some(2L)), schema, replacementRows), + ("read_end_v3", ReadQuery(version = Some(3L)), schema, endRows), + ("read_tail_v4", ReadQuery(version = Some(4L)), schema, tailRows), + ("read_tail_v5", ReadQuery(version = Some(5L)), schema, latestRows), + ("read_latest", ReadQuery(), schema, latestRows), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("tail-null", null)))) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"compact $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"compact $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"compact $name expected parquet") + } + } + } + test("e2e_57_auto_compaction") { + // These settings affect fixture generation only. Ordinary SQL inserts invoke + // OSS AutoCompact's post-commit hook; no explicit compaction command is called. + val settings = Map( + "spark.databricks.delta.autoCompact.enabled" -> "true", + "spark.databricks.delta.autoCompact.minNumFiles" -> "3", + "spark.databricks.delta.autoCompact.minFileSize" -> "1048576", + "spark.databricks.delta.autoCompact.maxFileSize" -> "2097152") + val saved = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap + try { + settings.foreach { case (key, value) => sql(s"SET $key=$value") } + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.autoOptimize.autoCompact' = 'true', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '1000', 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'initial'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (12, 'keep'), (12, 'keep'), (23, 'end') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (44, 'last'), (44, 'last'), (CAST(NULL AS INT), CAST(NULL AS STRING)), + (88, 'edge') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (99, 'tail'), (99, 'tail') AS v(id, value)""") + } finally { + saved.foreach { + case (key, Some(value)) => spark.conf.set(key, value) + case (key, None) => spark.conf.unset(key) + } + } + val table = registerTable("tbl") + val commits = (0L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + assert(!Files.exists(CommitLog.commitFile(table.sourcePath, 6L))) + val inputAdds = (1L to 3L).flatMap { v => + val adds = commits(v).collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.dataChange) + assert(adds.head.size > 0L && adds.head.size < 1048576L) + assert(commits(v).collect { case r: RemoveFile => r }.isEmpty) + adds + } + val rewritten = commits(4L).collect { case a: AddFile => a } + val removed = commits(4L).collect { case r: RemoveFile => r } + assert(rewritten.size == 1 && !rewritten.head.dataChange) + assert(removed.size == 3 && removed.forall(r => !r.dataChange)) + assert(removed.map(_.path).toSet == inputAdds.map(_.path).toSet) + assert(!inputAdds.exists(_.path == rewritten.head.path)) + val tail = commits(5L).collect { case a: AddFile => a } + assert(tail.size == 1 && tail.head.dataChange) + assert(commits(5L).collect { case r: RemoveFile => r }.isEmpty) + val history = (0L to 5L).map { v => + val info = commits(v).map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + v -> info + }.toMap + assert(history(0L).get("operation").asText() == "CREATE TABLE") + Seq(1L, 2L, 3L, 5L).foreach { v => + assert(history(v).get("operation").asText() == "WRITE") + } + assert(history(4L).get("operation").asText() == "OPTIMIZE") + assert(history(4L).get("operationParameters").get("auto").asText() == "true") + info("auto automatic compaction commit v4: " + history(4L).toString) + + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val metadata = commits(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val configuration = Map("delta.autoOptimize.autoCompact" -> "true", + "delta.enableDeletionVectors" -> "false", "delta.checkpointInterval" -> "1000", + "delta.checkpointPolicy" -> "classic") + assert(commits(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + (0L to 5L).foreach { v => + assert(CommitLog.read(dir, v) == commits(v)) + assert(Files.readAllBytes(CommitLog.commitFile(dir, v)) + .sameElements(Files.readAllBytes(CommitLog.commitFile(table.sourcePath, v)))) + } + (inputAdds ++ rewritten ++ tail).foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + val logs = Files.list(dir.resolve("_delta_log")) + try assert(!logs.iterator().asScala.exists(_.getFileName.toString.contains("checkpoint"))) + finally logs.close() + } + val activePaths = Map(0L -> Set.empty[String], 1L -> Set(inputAdds(0).path), + 2L -> inputAdds.take(2).map(_.path).toSet, 3L -> inputAdds.map(_.path).toSet, + 4L -> Set(rewritten.head.path), 5L -> Set(rewritten.head.path, tail.head.path)) + Seq[java.lang.Long](0L, 1L, 2L, 3L, 4L, 5L, null).foreach { version => + val v = Option(version).fold(5L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + assert(snapshot.allFiles.collect().map(_.path).toSet == activePaths(v)) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + } + } + // Literal, duplicate-sensitive bags are independent of captured expectations. + val firstRows = Seq(Row(-7, "initial"), Row(4, null), Row(4, null), Row(null, "missing")) + val secondRows = firstRows ++ Seq(Row(12, "keep"), Row(12, "keep"), Row(23, "end")) + val beforeRows = secondRows ++ Seq(Row(44, "last"), Row(44, "last"), + Row(null, null), Row(88, "edge")) + val latestRows = beforeRows ++ Seq(Row(99, "tail"), Row(99, "tail")) + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_first_v1", ReadQuery(version = Some(1L)), schema, firstRows, 1, 0L), + ("read_below_threshold_v2", ReadQuery(version = Some(2L)), schema, secondRows, 2, 0L), + ("read_before_v3", ReadQuery(version = Some(3L)), schema, beforeRows, 3, 0L), + ("read_post_v4", ReadQuery(version = Some(4L)), schema, beforeRows, 1, 0L), + ("read_tail_v5", ReadQuery(version = Some(5L)), schema, latestRows, 2, 0L), + ("read_latest", ReadQuery(), schema, latestRows, 2, 0L), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("missing", null), Row(null, null)), 1, 1L)) + cases.foreach { case (name, query, expectedSchema, rows, fileCount, skipped) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, fileCount, skipped))) + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"auto $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"auto $name expected parquet") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/PartitionValuesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/PartitionValuesSuite.scala index 40d3410d..e8adcb57 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/PartitionValuesSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/PartitionValuesSuite.scala @@ -16,9 +16,29 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.Files +import java.sql.Timestamp +import java.time.Instant -class PartitionValuesSuite extends WorkloadTestSuite("partition_values") { +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.example.data.Group +import org.apache.parquet.hadoop.{ParquetFileReader, ParquetReader} +import org.apache.parquet.hadoop.example.GroupReadSupport +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.parquet.schema.{LogicalTypeAnnotation, PrimitiveType} +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType, TimestampType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog} +import io.delta.workload.model.Succeeded + +class PartitionValuesSuite extends WorkloadTestSuite("partition_values") + with ScopedSessionConf with LiteralOracleChecks { test("boolean") { sql("""CREATE TABLE tbl (id INT, flag BOOLEAN) USING delta @@ -169,4 +189,432 @@ class PartitionValuesSuite extends WorkloadTestSuite("partition_values") { snapshotSpec(t) } + // Partition values come from AddFile metadata, not from data file directories. + test("metadata_partition_values_ignore_directories") { + sql("""CREATE TABLE tbl (id INT, payload STRING, part STRING) USING delta + PARTITIONED BY (part) TBLPROPERTIES ('delta.checkpointInterval' = '1000')""") + sql("""INSERT INTO tbl VALUES + (1, 'duplicate', 'east'), (1, 'duplicate', 'east'), + (2, 'conflicting', 'west'), (3, 'null_partition', NULL), + (4, 'no_hive_directory', 'plain')""") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val schema = new StructType().add("id", IntegerType).add("payload", StringType) + .add("part", StringType) + + mutateTable(t) { tableDir => + val originals = CommitLog.read(tableDir, 1L).collect { case add: AddFile => add } + assert(originals.nonEmpty) + assert(originals.map(_.partitionValues("part")).toSet == Set("east", "west", null, "plain")) + CommitLog.mutate(tableDir, 1L) { actions => + actions.map { + case add: AddFile => + val directory = if (add.partitionValues("part") == "plain") "opaque/nested" + else "part=directory_lie" + val original = tableDir.resolve(add.path) + val relative = s"$directory/${original.getFileName}" + val target = tableDir.resolve(relative) + Files.createDirectories(target.getParent) + Files.move(original, target) + assert(!Files.exists(original) && Files.isRegularFile(target)) + assert(Files.size(target) == add.size) + add.copy(path = relative) + case other => other + } + } + val moved = CommitLog.read(tableDir, 1L).collect { case add: AddFile => add } + assert(moved.map(_.partitionValues) == originals.map(_.partitionValues)) + assert(moved.zip(originals).forall { case (after, before) => + after.copy(path = before.path) == before + }) + assert(moved.forall(add => Files.isRegularFile(tableDir.resolve(add.path)))) + assert(moved.forall(add => !originals.exists(_.path == add.path))) + // Require JSON replay: no checkpoint or Delta CRC may retain the original file paths. + val stream = Files.list(tableDir.resolve("_delta_log")) + val files = try stream.iterator().asScala.toSeq finally stream.close() + assert(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + println("Metadata-partition layout oracle passed: conflicting and non-Hive directories") + } + + val rows = Seq(Row(1, "duplicate", "east"), Row(1, "duplicate", "east"), + Row(2, "conflicting", "west"), Row(3, "null_partition", null), + Row(4, "no_hive_directory", "plain")) + val reads = Seq( + ("read_all", null, rows.indices, None), + ("read_metadata_east", "part = 'east'", Seq(0, 1), None), + ("read_metadata_west", "part = 'west'", Seq(2), None), + ("read_directory_lie", "part = 'directory_lie'", Seq.empty[Int], None), + ("read_null", "part IS NULL", Seq(3), None), + ("read_no_hive_directory", "part = 'plain'", Seq(4), None), + ("read_projection", null, rows.indices, Some(Seq("part", "id"))), + ("read_null_projection", "part IS NULL", Seq(3), Some(Seq("part", "payload")))) + for ((name, predicate, indices, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"metadata partition/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"metadata partition/$name literal bag") + assert(captured.schema == oracle.schema) + println(s"Metadata-partition literal row/schema oracle passed: $name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(DataType.fromJson(result.metadata.schemaString) == schema) + assert(result.metadata.partitionColumns == Seq("part")) + assert(result.metadata.configuration == Map("delta.checkpointInterval" -> "1000")) + case other => fail(s"metadata partition snapshot must succeed: $other") + } + println("Metadata-partition snapshot schema/partition oracle passed") + } + } + + // Literal AddFile partition strings: empty values, timestamp serializations, and + // canonical DATE calendar bounds. + // Literal empty AddFile partition strings are independent of writer normalization. + // BYTE is numeric TINYINT. Its empty input has no numeric value and is expected to be null. + // STRING and BINARY retain a distinct empty value; writer support was probed separately. + for ((kind, sqlType, valueType, firstValue, secondValue, control, emptyValue) <- Seq( + ("byte", "TINYINT", org.apache.spark.sql.types.ByteType, + "CAST(1 AS TINYINT)", "CAST(2 AS TINYINT)", 2.toByte, null), + ("short", "SMALLINT", org.apache.spark.sql.types.ShortType, + "CAST(1 AS SMALLINT)", "CAST(2 AS SMALLINT)", 2.toShort, null), + ("int", "INT", IntegerType, "1", "2", 2, null), + ("long", "BIGINT", org.apache.spark.sql.types.LongType, "1L", "2L", 2L, null), + ("float", "FLOAT", org.apache.spark.sql.types.FloatType, + "CAST(1.25 AS FLOAT)", "CAST(2.5 AS FLOAT)", 2.5f, null), + ("double", "DOUBLE", org.apache.spark.sql.types.DoubleType, + "CAST(1.25 AS DOUBLE)", "CAST(2.5 AS DOUBLE)", 2.5d, null), + ("decimal", "DECIMAL(10,2)", org.apache.spark.sql.types.DecimalType(10, 2), + "CAST(1.25 AS DECIMAL(10,2))", "CAST(2.50 AS DECIMAL(10,2))", + new java.math.BigDecimal("2.50"), null), + ("boolean", "BOOLEAN", org.apache.spark.sql.types.BooleanType, + "false", "true", true, null), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1969-12-31'", "DATE '2000-02-29'", java.sql.Date.valueOf("2000-02-29"), null), + ("timestamp", "TIMESTAMP", TimestampType, + "TIMESTAMP '1969-12-31T23:59:59Z'", "TIMESTAMP '2024-03-10T10:00:00.123456Z'", + Timestamp.from(Instant.parse("2024-03-10T10:00:00.123456Z")), null), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1969-12-31 23:59:59'", "TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'", + java.time.LocalDateTime.parse("2024-03-10T02:30:00.123456"), null), + ("string", "STRING", org.apache.spark.sql.types.StringType, + "'first'", "'second'", "second", ""), + ("binary", "BINARY", org.apache.spark.sql.types.BinaryType, + "unhex('61')", "unhex('62')", Array[Byte](98), Array.emptyByteArray))) { + test(s"empty_partition_$kind") { + setForTest("spark.sql.session.timeZone", "UTC") + // ANSI predicate casts reject empty numeric input. Leave legacy casts active for capture. + // The existing spec format cannot transport this session setting to an external consumer. + setForTest("spark.sql.ansi.enabled", "false") + val expectedSchema = new StructType().add("id", IntegerType).add("p", valueType) + val rows = Seq(Row(1, emptyValue), Row(2, control), Row(2, control), Row(3, null)) + sql(s"CREATE TABLE tbl (id INT, p $sqlType) USING delta PARTITIONED BY (p)") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $firstValue), (2, $secondValue), (2, $secondValue), " + + s"(3, CAST(NULL AS $sqlType)) AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile if JsonUtil.mapper.readTree(add.stats.get) + .at("/minValues/id").intValue() == 1 => + add.copy(partitionValues = Map("p" -> "")) + case other => other + } + } + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 3) + require(adds.count(_.partitionValues("p") == "") == 1) + require(adds.count(_.partitionValues("p") == null) == 1) + println(s"Verified serialized empty partition: $kind empty=1 null=1 control=1") + } + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + checkLiteralRead(table, "is_null", expectedSchema, rows.filter(_.isNullAt(1)), + predicate = "p IS NULL") + checkLiteralRead(table, "is_not_null", expectedSchema, rows.filterNot(_.isNullAt(1)), + predicate = "p IS NOT NULL") + checkLiteralRead(table, "eq_control", expectedSchema, rows.filter(_.getInt(0) == 2), + predicate = s"p = $secondValue") + if (kind == "string" || kind == "binary") { + val emptyLiteral = if (kind == "string") "''" else "unhex('')" + checkLiteralRead(table, "eq_empty", expectedSchema, rows.filter(_.getInt(0) == 1), + predicate = s"p = $emptyLiteral") + } + snapshotSpec(table).assert { spec => + require(spark.conf.get("spark.sql.ansi.enabled") == "false") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == Seq("p")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.protocol.minReaderVersion == (if (kind == "timestamp_ntz") 3 else 1)) + require(result.protocol.minWriterVersion == (if (kind == "timestamp_ntz") 7 else 2)) + case other => fail(s"Empty partition snapshot must succeed: $other") + } + println(s"Verified capture mode: ${WorkloadContext.current.workloadName} ansi=false") + } + checkLiteralSnapshot(table, expectedSchema, Seq("p"), version = 1L) + } + } + + // Inputs are literal AddFile strings, not SQL timestamp literal spellings. + // The pinned Spark parser permits zone suffixes for NTZ casts and ignores their offsets; + // expected NTZ wall clocks below are separate from the corresponding absolute instants. + private def timestampPartitionFormats: Vector[(String, String, String, String)] = { + val fractions = Seq( + ("0", "", "000000"), ("1", ".1", "100000"), ("2", ".12", "120000"), + ("3", ".123", "123000"), ("4", ".1234", "123400"), + ("5", ".12345", "123450"), ("6", ".123456", "123456"), + ("7", ".1234567", "123456"), ("8", ".12345678", "123456"), + ("9", ".123456789", "123456"), ("dot", ".", "000000"), + ("10", ".1234567891", "123456"), ("18", ".123456789123456789", "123456")) + val zones = Seq( + ("z", "12:34:56.123456Z", "12:34:56.123456"), + ("zero_colon", "12:34:56.123456+00:00", "12:34:56.123456"), + ("zero_basic", "12:34:56.123456+0000", "12:34:56.123456"), + ("zero_hour", "12:34:56.123456+00", "12:34:56.123456"), + ("minus_zero", "12:34:56.123456-00:00", "12:34:56.123456"), + ("utc_name", "12:34:56.123456 UTC", "12:34:56.123456"), + ("gmt_name", "12:34:56.123456 GMT", "12:34:56.123456"), + ("east", "18:04:56.123456+05:30", "18:04:56.123456"), + ("west", "04:34:56.123456-08:00", "04:34:56.123456"), + ("region", "04:34:56.123456 America/Los_Angeles", "04:34:56.123456"), + ("east_basic", "18:04:56.123456+0530", "18:04:56.123456"), + ("east_hour", "17:34:56.123456+05", "17:34:56.123456"), + ("west_basic", "04:34:56.123456-0800", "04:34:56.123456"), + ("west_hour", "04:34:56.123456-08", "04:34:56.123456"), + ("west_half", "09:04:56.123456-03:30", "09:04:56.123456"), + ("single_hour", "18:04:56.123456+5:30", "18:04:56.123456"), + ("single_minute", "17:37:56.123456+05:3", "17:37:56.123456"), + ("offset_seconds", "18:05:26.123456+05:30:30", "18:05:26.123456")) + (for ((separatorName, separator) <- Seq("space" -> " ", "T" -> "T"); + entry <- fractions.map { case (label, fraction, micros) => + (s"${separatorName}_fraction_$label", s"2024-02-29${separator}12:34:56$fraction", + s"2024-02-29T12:34:56.$micros" + "Z", s"2024-02-29T12:34:56.$micros") + } ++ zones.map { case (label, raw, wall) => + (s"${separatorName}_$label", s"2024-02-29$separator$raw", + "2024-02-29T12:34:56.123456Z", s"2024-02-29T$wall") + }) yield entry).toVector + } + + for ((kind, sqlType, valueType) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType))) { + test(s"partition_serializations_$kind") { + setForTest("spark.sql.session.timeZone", "UTC") + setForTest("spark.sql.ansi.enabled", "false") + val formats = timestampPartitionFormats + require(formats.size == 62) + val expectedSchema = new StructType().add("id", IntegerType).add("p", valueType) + val keys = formats.zipWithIndex.map { case ((_, _, instant, wall), index) => + (index + 1) -> (if (kind == "timestamp") instant else wall) + }.toMap + val rows0 = keys.toSeq.sortBy(_._1).map { case (id, key) => + val value = if (kind == "timestamp") Timestamp.from(Instant.parse(key)) + else java.time.LocalDateTime.parse(key) + Row(id, value) + } + val rows = rows0 :+ rows0.head + sql(s"CREATE TABLE tbl (id INT, p $sqlType) USING delta PARTITIONED BY (p)") + // Distinct seed partitions create one data file per raw representation. Only id is + // physically stored; the typed AddFile mutation supplies the independently checked p. + val seeds = formats.indices.map { index => + val seed = Instant.parse("2030-01-01T00:00:00Z").plusSeconds(index.toLong).toString + val literal = if (kind == "timestamp") seed else seed.stripSuffix("Z") + s"(${index + 1}, $sqlType '$literal')" + } + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + (seeds :+ seeds.head).mkString(", ") + " AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val id = JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + add.copy(partitionValues = Map("p" -> formats(id - 1)._2)) + case other => other + } + } + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == formats.size) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + require(stats.at("/maxValues/id").intValue() == id) + require(stats.get("numRecords").asLong() == (if (id == 1) 2L else 1L)) + require(add.partitionValues == Map("p" -> formats(id - 1)._2)) + println(s"Verified raw serialization: $kind ${formats(id - 1)._1} " + + s"value=${formats(id - 1)._2}") + } + } + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + keys.values.toSet.toSeq.sorted.zipWithIndex.foreach { case (key, index) => + val ids = keys.collect { case (id, value) if value == key => id }.toSet + checkLiteralRead(table, s"eq_$index", expectedSchema, + rows.filter(r => ids.contains(r.getInt(0))), predicate = s"p = $sqlType '$key'") + } + def selected(lower: String, upper: String): Seq[Row] = rows.filter { row => + val key = keys(row.getInt(0)) + key >= lower && key < upper + } + checkLiteralRead(table, "fraction_range", expectedSchema, + selected("2024-02-29T12:34:56.100000", "2024-02-29T12:34:56.123456"), + predicate = s"p >= $sqlType '2024-02-29 12:34:56.100000' AND " + + s"p < $sqlType '2024-02-29 12:34:56.123456'") + checkLiteralRead(table, "before_noon", expectedSchema, + selected("2024-02-29T00:00:00", "2024-02-29T12:00:00"), + predicate = s"p < $sqlType '2024-02-29 12:00:00'") + checkLiteralRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"p < $sqlType '2024-02-29 00:00:00'") + checkLiteralSnapshot(table, expectedSchema, Seq("p")) + checkLiteralSnapshot(table, expectedSchema, Seq("p"), version = 1L) + } + } + + // Canonical zero-padded DATE strings use proleptic Gregorian epoch-day oracles. + test("date_partition_canonical_calendar_bounds") { + setForTest("spark.sql.session.timeZone", "UTC") + setForTest("spark.sql.ansi.enabled", "false") + // Expected Parquet must retain proleptic Gregorian days even before the 1582 cutover. + setForTest("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED") + setForTest("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED") + val dates = Seq( + (1, "0001-01-01", -719162), (2, "1582-10-04", -141438), + (3, "1582-10-15", -141427), (4, "1969-12-31", -1), + (5, "1970-01-01", 0), (6, "2000-02-29", 11016), (7, "9999-12-31", 2932896)) + val expectedSchema = new StructType().add("id", IntegerType) + .add("p", org.apache.spark.sql.types.DateType) + val rows = dates.map { case (id, raw, _) => Row(id, java.sql.Date.valueOf(raw)) } ++ + Seq(Row(6, java.sql.Date.valueOf("2000-02-29")), Row(8, null)) + val days: Map[Int, Option[Int]] = dates.map { case (id, _, day) => + id -> Some(day) + }.toMap + (8 -> None) + sql("CREATE TABLE tbl (id INT, p DATE) USING delta PARTITIONED BY (p)") + val inputs = dates.map { case (id, raw, _) => s"($id, DATE '$raw')" } ++ + Seq("(6, DATE '2000-02-29')", "(8, CAST(NULL AS DATE))") + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + inputs.mkString(", ") + " AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val id = JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + val raw = if (id == 8) null else dates.find(_._1 == id).get._2 + add.copy(partitionValues = Map("p" -> raw)) + case other => other + } + } + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 8) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + val raw = if (id == 8) null else dates.find(_._1 == id).get._2 + require(add.partitionValues == Map("p" -> raw)) + require(stats.at("/maxValues/id").intValue() == id) + require(stats.get("numRecords").asLong() == (if (id == 6) 2L else 1L)) + if (raw != null) require(raw.matches("[0-9]{4}-[0-9]{2}-[0-9]{2}")) + println(s"Verified canonical DATE partition: id=$id raw=$raw epochDay=${days(id)}") + } + } + val allName = "all" + checkLiteralRead(table, allName, expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + dates.foreach { case (id, raw, _) => + checkLiteralRead(table, s"eq_$id", expectedSchema, rows.filter(_.getInt(0) == id), + predicate = s"p = DATE '$raw'") + } + checkLiteralRead(table, "is_null", expectedSchema, rows.filter(_.isNullAt(1)), + predicate = "p IS NULL") + checkLiteralRead(table, "pre_epoch", expectedSchema, rows.filter(r => r.getInt(0) <= 4), + predicate = "p < DATE '1970-01-01'") + checkLiteralRead(table, "calendar_transition", expectedSchema, + rows.filter(r => Set(2, 3).contains(r.getInt(0))), + predicate = "p >= DATE '1582-10-04' AND p <= DATE '1582-10-15'") + checkLiteralRead(table, "modern_range", expectedSchema, + rows.filter(r => Set(5, 6).contains(r.getInt(0))), + predicate = "p >= DATE '1970-01-01' AND p <= DATE '2000-02-29'") + val workload = WorkloadContext.current.workloadName + snapshotSpec(table).assert { spec => + require(spark.conf.get("spark.sql.parquet.datetimeRebaseModeInWrite") == "CORRECTED") + require(spark.conf.get("spark.sql.parquet.datetimeRebaseModeInRead") == "CORRECTED") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == Seq("p")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.protocol.minReaderVersion == 1 && result.protocol.minWriterVersion == 2) + case other => fail(s"Canonical DATE snapshot must succeed: $other") + } + + } + checkLiteralSnapshot(table, expectedSchema, Seq("p"), version = 1L) + // Snapshots capture before reads at this pin. Inspect Parquet in the last read callback. + val outsideName = "outside_bounds" + readSpec(table, predicate = "p < DATE '0001-01-01' OR p > DATE '9999-12-31'", + name = Some(outsideName)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == 0L) + case other => fail(s"Outside DATE bounds must succeed with no rows: $other") + } + val expected = spark.createDataFrame(Seq.empty[Row].asJava, expectedSchema) + val outside = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$outsideName").resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(outside.toString), outsideName) + println(s"Verified independent literal rows/schema: $workload/$outsideName rows=0") + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$allName").resolve("expected_data") + val literalOracle = outputDir.resolveSibling("oracle-parquet").resolve(workload) + val expectedUnits = days.toVector :+ (6 -> Some(11016)) + for ((label, directory) <- Seq("captured" -> captured, "literal" -> literalOracle)) { + val files = Files.list(directory) + val parquet = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toVector finally files.close() + require(parquet.nonEmpty) + var units = Vector.empty[(Int, Option[Int])] + parquet.foreach { file => + val path = new HadoopPath(file.toUri) + val conf = spark.sparkContext.hadoopConfiguration + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema.getFields.asScala + .find(_.getName == "p").get.asPrimitiveType() + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(physical.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + try { + var record = reader.read() + while (record != null) { + val day = if (record.getFieldRepetitionCount("p") == 0) None + else Some(record.getInteger("p", 0)) + units :+= record.getInteger("id", 0) -> day + record = reader.read() + } + } finally reader.close() + } + require(units.sorted == expectedUnits.sorted) + println(s"Verified physical DATE oracle: $workload $label units=${units.sorted}") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessCheckpointSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessCheckpointSuite.scala deleted file mode 100644 index b3724720..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessCheckpointSuite.scala +++ /dev/null @@ -1,559 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.tables - -import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.{Files, Path} - -import scala.jdk.CollectionConverters._ - -import org.apache.hadoop.fs.{Path => HadoopPath} -import org.apache.parquet.hadoop.ParquetFileReader -import org.apache.parquet.hadoop.util.HadoopInputFile -import org.apache.spark.sql.Row -import org.apache.spark.sql.functions.{col, lit, map => sparkMap, when} -import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} - -import io.delta.workload.WorkloadTestSuite -import io.delta.workload.deltaharness.{Format => HarnessFormat} -import io.delta.workload.engine.{RowComparison, SnapshotResolver} -import io.delta.workload.json.JsonUtil -import io.delta.workload.log.{Action, AddFile, CheckpointMetadata, CommitLog, Metadata, Protocol, - SidecarFile} -import io.delta.workload.model._ - -class ReadinessCheckpointSuite extends WorkloadTestSuite("readiness_checkpoint") { - - private def snapshotResult(protocol: Protocol, metadata: Metadata): SnapshotResult = - SnapshotResult( - ProtocolInfo(protocol.minReaderVersion, protocol.minWriterVersion, - protocol.readerFeatures.map(_.toSeq.sorted), - protocol.writerFeatures.map(_.toSeq.sorted)), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat(metadata.format.provider, metadata.format.options), - metadata.schemaString, metadata.partitionColumns, metadata.configuration, - metadata.createdTime)) - - private def assertReadRows( - spec: ReadSpec, - tablePath: Path, - schema: StructType, - rows: Seq[Row], - name: String): Unit = { - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, tablePath, - spec.query.version, spec.query.timestamp), spec.query.predicate, spec.query.columns) - assert(actual.schema == schema) - RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, schema), actual, name) - } - - Seq( - ("json_inline", "json", "v2", 1000, 0), - ("json_manifest", "json", "v2", 1000, 1), - ("parquet_single_part", "parquet", "classic", 1000, 0), - ("parquet_manifest", "parquet", "v2", 1000, 1), - ("parquet_sidecars", "parquet", "v2", 1, 2) - ).foreach { case (name, format, policy, partSize, sidecarCount) => - test(name) { - val settings = Map( - "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> format, - "spark.databricks.delta.checkpoint.partSize" -> partSize.toString) - val previous = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap - settings.foreach { case (key, value) => spark.conf.set(key, value) } - try { - sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta - TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', - 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (1, 'a'), (2, CAST(NULL AS STRING)), (2, CAST(NULL AS STRING)) AS v(id, value)""") - sql("INSERT INTO tbl VALUES (3, 'c')") - forceCheckpoint("tbl") - sql("INSERT INTO tbl VALUES (4, 'd')") - val t = registerTable("tbl") - // Read the original commit actions before mutating the copied fixture. - val actions = (0L to 2L).flatMap(v => CommitLog.read(t.sourcePath, v)) - val metadata = actions.collect { case m: Metadata => m }.last - val protocol = actions.collect { case p: Protocol => p }.last - val adds = actions.collect { case a: AddFile => a } - assert(adds.size == 2) - info("Original protocol: " + protocol.toJson) - info("Original metadata: " + metadata.toJson) - adds.foreach(a => info("Original add: " + a.toJson)) - val schema = new StructType().add("id", IntegerType).add("value", StringType) - assert(DataType.fromJson(metadata.schemaString) == schema) - val expectedSnapshot = snapshotResult(protocol, metadata) - val checkpointRows = Seq(Row(1, "a"), Row(2, null), Row(2, null), Row(3, "c")) - val latestRows = checkpointRows :+ Row(4, "d") - var copiedTablePath: Path = null - mutateTable(t) { tableDir => - copiedTablePath = tableDir - val log = tableDir.resolve("_delta_log") - val listing = Files.list(log) - val files = try listing.iterator().asScala.toVector finally listing.close() - val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) - assert(checkpoints.size == 1) - val checkpoint = checkpoints.head - if (policy == "classic") { - assert(checkpoint.getFileName.toString == "00000000000000000002.checkpoint.parquet") - val frame = spark.read.parquet(checkpoint.toString) - assert(frame.filter("add IS NOT NULL").count() == adds.size) - assert(frame.filter("metaData IS NOT NULL").count() == 1) - assert(frame.filter("protocol IS NOT NULL").count() == 1) - assert(!Files.exists(log.resolve("_sidecars"))) - } else { - assert(checkpoint.getFileName.toString.matches( - s"00000000000000000002\\.checkpoint\\.[^.]+\\.$format")) - if (name == "json_inline") { - // V2 allows inline file actions; this writer normally emits sidecars. - val checkpointAdds = adds.map(_.copy(dataChange = false)) - val inlineActions: Seq[Action] = - Seq(protocol, metadata, CheckpointMetadata(2L)) ++ checkpointAdds - Files.write(checkpoint, inlineActions.map(_.toJson).asJava, UTF_8) - val sidecars = Files.list(log.resolve("_sidecars")) - try sidecars.iterator().asScala.toVector.foreach(Files.delete) - finally sidecars.close() - Files.delete(log.resolve("_sidecars")) - val content = Files.readAllLines(checkpoint, UTF_8).asScala.map(Action.parse).toVector - assert(content.collect { case a: AddFile => a } == checkpointAdds) - assert(content.collect { case c: CheckpointMetadata => c.version } == Seq(2L)) - assert(!content.exists(_.isInstanceOf[SidecarFile])) - } else { - val manifest = if (format == "json") { - Files.readAllLines(checkpoint, UTF_8).asScala.map(Action.parse).toVector - } else { - spark.read.parquet(checkpoint.toString).toJSON.collect().toVector.map(Action.parse) - } - val references = manifest.collect { case s: SidecarFile => s } - assert(references.size == sidecarCount) - assert(!manifest.exists(_.isInstanceOf[AddFile])) - assert(manifest.collect { case c: CheckpointMetadata => c.version } == Seq(2L)) - assert(manifest.collect { case m: Metadata => m } == Seq(metadata)) - assert(manifest.collect { case p: Protocol => p } == Seq(protocol)) - val sidecarPaths = references.map { ref => - val path = log.resolve("_sidecars").resolve(ref.path) - assert(ref.path.endsWith(".parquet")) - assert(Files.size(path) == ref.sizeInBytes) - assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) - path - } - val sidecarListing = Files.list(log.resolve("_sidecars")) - try assert(sidecarListing.iterator().asScala.toSet == sidecarPaths.toSet) - finally sidecarListing.close() - val content = spark.read.parquet(sidecarPaths.map(_.toString): _*) - assert(content.count() == adds.size) - assert(content.filter("add IS NOT NULL").count() == adds.size) - assert(content.filter("remove IS NOT NULL").count() == 0) - assert(content.select("add.path").collect().map(_.getString(0)).toSet == - adds.map(_.path).toSet) - } - } - // Keep same-version JSON commit 2 for the OSS historical snapshot lookup. - // This tests checkpoint plus commit 2, then version 3 tail replay. - // Checkpoint-only historical snapshots remain a separate failing repro. - files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) - .filter(_.getFileName.toString.take(20).toLong < 2L).foreach(Files.delete) - files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) - Files.deleteIfExists(log.resolve("_last_checkpoint")) - } - Seq( - ("read_all", ReadQuery(), latestRows, ReadResult(5L, 3, 0L)), - ("read_v2", ReadQuery(version = Some(2L)), checkpointRows, ReadResult(4L, 2, 0L)), - ("read_filtered", ReadQuery(predicate = Some("id >= 3")), - Seq(Row(3, "c"), Row(4, "d")), ReadResult(2L, 2, 1L)) - ).foreach { case (readName, query, rows, expected) => - readSpec(t, name = Some(readName), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull).assert { spec => - assert(spec.expectation == Succeeded(expected)) - assertReadRows(spec, copiedTablePath, schema, rows, readName) - } - } - Seq(None, Some(2L)).foreach { version => - snapshotSpec(t, version = version.map(Long.box).orNull).assert { spec => - assert(spec.query.version.contains(version.getOrElse(3L))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - } - } - } finally { - previous.foreach { - case (key, Some(value)) => spark.conf.set(key, value) - case (key, None) => spark.conf.unset(key) - } - } - } - } - - // stats: OSS exposes structured checkpoint stats, but no product metadata-clustering - // control. These fixtures check shape and filtered correctness, never row groups READ. - Seq( - ("e2e_07_single_part", "classic", 1000, 1, 0), - ("e2e_07_multipart", "classic", 2, 3, 0), - ("e2e_07_sidecars", "v2", 2, 1, 3) - ).foreach { case (name, policy, partSize, checkpointCount, sidecarCount) => - test(name) { - val settings = Map( - "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> "parquet", - "spark.databricks.delta.checkpoint.partSize" -> partSize.toString, - "spark.databricks.delta.statsAsStructInCheckpoint.forcedDisabled" -> "false") - val previous = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap - settings.foreach { case (key, value) => spark.conf.set(key, value) } - try { - sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta - TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', - 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false', - 'delta.checkpoint.writeStatsAsStruct' = 'true')""") - val rowsByFile = (0 until 6).map { band => - val low = band * 10 - sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - ($low, 'band_$band'), (${low + 1}, CAST(NULL AS STRING)), - (${low + 1}, CAST(NULL AS STRING)) AS v(id, value)""") - Seq(Row(low, s"band_$band"), Row(low + 1, null), Row(low + 1, null)) - } - forceCheckpoint("tbl") - val t = registerTable("tbl") - val actions = (0L to 6L).flatMap(v => CommitLog.read(t.sourcePath, v)) - val metadata = actions.collect { case m: Metadata => m }.last - val protocol = actions.collect { case p: Protocol => p }.last - val adds = actions.collect { case a: AddFile => a } - assert(adds.size == 6 && adds.map(_.path).distinct.size == 6) - info("stats original protocol: " + protocol.toJson) - info("stats original metadata: " + metadata.toJson) - adds.foreach(a => info("stats original add: " + a.toJson)) - val schema = new StructType().add("id", IntegerType).add("value", StringType) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.partitionColumns.isEmpty) - assert(metadata.configuration("delta.checkpointPolicy") == policy) - assert(metadata.configuration("delta.checkpoint.writeStatsAsStruct") == "true") - val expectedSnapshot = snapshotResult(protocol, metadata) - var copiedTablePath: Path = null - mutateTable(t) { tableDir => - copiedTablePath = tableDir - val log = tableDir.resolve("_delta_log") - val listing = Files.list(log) - val files = try listing.iterator().asScala.toVector finally listing.close() - val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) - assert(checkpoints.size == checkpointCount) - if (name == "e2e_07_single_part") { - assert(checkpoints.head.getFileName.toString == - "00000000000000000006.checkpoint.parquet") - } else if (policy == "classic") { - val expectedNames = (1 to 3).map { part => - f"00000000000000000006.checkpoint.$part%010d.0000000003.parquet" - }.toSet - assert(checkpoints.map(_.getFileName.toString).toSet == expectedNames) - } else { - assert(checkpoints.head.getFileName.toString.matches( - "00000000000000000006\\.checkpoint\\.[^.]+\\.parquet")) - } - val top = spark.read.parquet(checkpoints.map(_.toString): _*) - val topActions = top.toJSON.collect().toVector.map(Action.parse) - assert(topActions.collect { case m: Metadata => m } == Seq(metadata)) - assert(topActions.collect { case p: Protocol => p } == Seq(protocol)) - val sidecarPaths = if (policy == "v2") { - val refs = topActions.collect { case s: SidecarFile => s } - assert(refs.size == sidecarCount && refs.map(_.path).distinct.size == sidecarCount) - assert(topActions.collect { case c: CheckpointMetadata => c.version } == Seq(6L)) - assert(!topActions.exists(_.isInstanceOf[AddFile])) - val paths = refs.map { ref => - val path = log.resolve("_sidecars").resolve(ref.path) - assert(ref.path.endsWith(".parquet")) - assert(Files.size(path) == ref.sizeInBytes) - assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) - path - } - val listing = Files.list(log.resolve("_sidecars")) - try assert(listing.iterator().asScala.toSet == paths.toSet) - finally listing.close() - paths - } else { - assert(!Files.exists(log.resolve("_sidecars"))) - Vector.empty[Path] - } - val fileActions = if (sidecarPaths.isEmpty) top else - spark.read.parquet(sidecarPaths.map(_.toString): _*) - assert(fileActions.filter("add IS NOT NULL").count() == 6) - assert(fileActions.filter("remove IS NOT NULL").count() == 0) - if (sidecarPaths.nonEmpty) assert(fileActions.count() == 6) - assert(fileActions.select("add.path").collect().filterNot(_.isNullAt(0)) - .map(_.getString(0)).toSet == adds.map(_.path).toSet) - val stats = fileActions.filter("add IS NOT NULL").select( - "add.stats_parsed.numRecords", "add.stats_parsed.minValues.id", - "add.stats_parsed.maxValues.id", "add.stats_parsed.nullCount.value").collect().toSeq - assert(stats.toSet == (0 until 6).map { band => - Row(3L, band * 10, band * 10 + 1, 2L) - }.toSet) - // Footer row groups are physical shape evidence, not reader IO counters. - val footerRows = (checkpoints ++ sidecarPaths).map { path => - val reader = ParquetFileReader.open(HadoopInputFile.fromPath( - new HadoopPath(path.toUri), spark.sparkContext.hadoopConfiguration)) - try { - val groups = reader.getFooter.getBlocks.asScala - info(s"stats footer: ${path.getFileName} groups=${groups.size} " + - s"rows=${groups.map(_.getRowCount).sum}") - assert(groups.size <= 1) - groups.map(_.getRowCount).sum - } finally reader.close() - } - assert(footerRows.sum == (if (policy == "v2") 12L else 8L)) - // Retain same-version JSON 6 for OSS version lookup, with no post-checkpoint tail. - // Missing commits 0..5 make the checkpoint required even though commit 6 exists. - files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) - .filter(_.getFileName.toString.take(20).toLong < 6L).foreach(Files.delete) - files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) - Files.deleteIfExists(log.resolve("_last_checkpoint")) - } - Seq( - ("read_all", None, rowsByFile.flatten, ReadResult(18L, 6, 0L)), - ("read_high", Some("id >= 40"), rowsByFile.drop(4).flatten, ReadResult(6L, 2, 4L)), - ("read_boundary", Some("id = 41"), Seq(Row(41, null), Row(41, null)), - ReadResult(2L, 1, 5L)), - ("read_empty", Some("id >= 100"), Seq.empty[Row], ReadResult(0L, 0, 6L)) - ).foreach { case (readName, predicate, rows, expected) => - readSpec(t, name = Some(readName), predicate = predicate.orNull).assert { spec => - assert(spec.expectation == Succeeded(expected)) - assertReadRows(spec, copiedTablePath, schema, rows, readName) - } - } - snapshotSpec(t).assert { spec => - assert(spec.query.version.contains(6L)) - assert(spec.expectation == Succeeded(expectedSnapshot)) - } - } finally { - previous.foreach { - case (key, Some(value)) => spark.conf.set(key, value) - case (key, None) => spark.conf.unset(key) - } - } - } - } - - // size: 64 KiB is a bounded fixture boundary, not a product dispatch threshold. - // These shapes assert portable read correctness, never distributed execution. - Seq("manifest", "json", "sidecars", "single_part", "multipart", "crc_all_files") - .foreach { shape => - test(s"e2e_43_$shape") { - val isCrc = shape == "crc_all_files" - val policy = if (Set("single_part", "multipart", "crc_all_files")(shape)) - "classic" else "v2" - val settings = Map( - "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> "json", - "spark.databricks.delta.checkpoint.partSize" -> - (if (shape == "multipart") "2" else "1000"), - "spark.databricks.delta.allFilesInCrc.enabled" -> "true", - "spark.databricks.delta.allFilesInCrc.thresholdNumFiles" -> "100") - val previous = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap - settings.foreach { case (key, value) => spark.conf.set(key, value) } - try { - sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta - TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', - 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false')""") - val rowsByFile = (0 until 4).map { band => - val low = band * 10 - sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - ($low, 'band_$band'), (${low + 1}, CAST(NULL AS STRING)), - (${low + 1}, CAST(NULL AS STRING)) AS v(id, value)""") - Seq(Row(low, s"band_$band"), Row(low + 1, null), Row(low + 1, null)) - } - forceCheckpoint("tbl") - // A complete-file removal distinguishes active files from historical Adds in CRC. - if (isCrc) sql("DELETE FROM tbl WHERE id < 10") - sql("INSERT INTO tbl VALUES (99, 'tail')") - val t = registerTable("tbl") - val baseVersion = if (isCrc) 5L else 4L - val latestVersion = baseVersion + 1L - val actions = (0L to latestVersion).flatMap(v => CommitLog.read(t.sourcePath, v)) - val metadata = actions.collect { case m: Metadata => m }.last - val protocol = actions.collect { case p: Protocol => p }.last - val adds = actions.collect { case a: AddFile => a } - assert(adds.size == 5 && adds.map(_.path).distinct.size == 5) - val schema = new StructType().add("id", IntegerType).add("value", StringType) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.partitionColumns.isEmpty) - assert(metadata.configuration("delta.checkpointPolicy") == policy) - assert(metadata.configuration("delta.enableDeletionVectors") == "false") - val expectedSnapshot = snapshotResult(protocol, metadata) - val baseRows = (if (isCrc) rowsByFile.drop(1) else rowsByFile).flatten - val latestRows = baseRows :+ Row(99, "tail") - info(s"size $shape protocol: " + protocol.toJson) - info(s"size $shape metadata: " + metadata.toJson) - adds.foreach(a => info(s"size $shape original add: " + a.toJson)) - var copiedTablePath: Path = null - mutateTable(t) { tableDir => - copiedTablePath = tableDir - val log = tableDir.resolve("_delta_log") - val listing = Files.list(log) - val files = try listing.iterator().asScala.toVector finally listing.close() - val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) - .sortBy(_.getFileName.toString) - assert(checkpoints.size == (if (shape == "multipart") 2 else 1)) - if (policy == "classic") { - val names = if (shape == "multipart") (1 to 2).map { part => - f"00000000000000000004.checkpoint.$part%010d.0000000002.parquet" - }.toSet else Set("00000000000000000004.checkpoint.parquet") - assert(checkpoints.map(_.getFileName.toString).toSet == names) - } else { - assert(checkpoints.head.getFileName.toString.matches( - "00000000000000000004\\.checkpoint\\.[^.]+\\.json")) - } - val manifest = if (policy == "v2") { - Files.readAllLines(checkpoints.head, UTF_8).asScala.map(Action.parse).toVector - } else Vector.empty[Action] - val refs = manifest.collect { case s: SidecarFile => s } - assert(refs.size == (if (policy == "v2") 1 else 0)) - val sidecars = refs.map(ref => log.resolve("_sidecars").resolve(ref.path)) - refs.zip(sidecars).foreach { case (ref, path) => - assert(Files.size(path) == ref.sizeInBytes) - assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) - } - if (policy == "v2") { - assert(!manifest.exists(_.isInstanceOf[AddFile])) - assert(manifest.collect { case c: CheckpointMetadata => c.version } == Seq(4L)) - assert(manifest.collect { case m: Metadata => m } == Seq(metadata)) - assert(manifest.collect { case p: Protocol => p } == Seq(protocol)) - } else assert(!Files.exists(log.resolve("_sidecars"))) - val boundary = 64L * 1024L - // Deterministic incompressible padding keeps Parquet fixtures small but >64 KiB. - val bytes = new Array[Byte](96 * 1024) - new scala.util.Random(43L).nextBytes(bytes) - val padding = java.util.Base64.getEncoder.encodeToString(bytes) - val paddedFiles = shape match { - case "sidecars" => sidecars - case "single_part" | "multipart" => checkpoints - case _ => Vector.empty[Path] - } - paddedFiles.foreach { path => - val frame = spark.read.parquet(path.toString) - val padded = frame.withColumn("add", when(col("add").isNotNull, - col("add").withField("tags", sparkMap(lit("e2e43.padding"), lit(padding)))) - .otherwise(col("add"))) - val temp = Files.createTempDirectory("e2e43-checkpoint-") - try { - padded.coalesce(1).write.mode("overwrite").parquet(temp.toString) - val parts = Files.list(temp) - val part = try parts.iterator().asScala - .filter(_.getFileName.toString.endsWith(".parquet")).toVector - finally parts.close() - assert(part.size == 1) - Files.move(part.head, path, java.nio.file.StandardCopyOption.REPLACE_EXISTING) - } finally org.apache.commons.io.FileUtils.deleteDirectory(temp.toFile) - } - if (shape == "manifest" || shape == "sidecars") { - val updated = manifest.map { - case c: CheckpointMetadata if shape == "manifest" => - c.copy(tags = Some(Map("e2e43.padding" -> padding))) - case ref: SidecarFile => - val path = log.resolve("_sidecars").resolve(ref.path) - ref.copy(sizeInBytes = Files.size(path), - modificationTime = Files.getLastModifiedTime(path).toMillis) - case action => action - } - Files.write(checkpoints.head, updated.map(_.toJson).asJava, UTF_8) - } - if (shape == "json") { - val lines = CommitLog.readRaw(tableDir, latestVersion).map { line => - val node = JsonUtil.mapper.readTree(line) - if (node.has("commitInfo")) { - val commitInfo = node.get("commitInfo") - .asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] - commitInfo.put("userMetadata", padding) - } - node.toString - } - CommitLog.writeRaw(tableDir, latestVersion, lines) - } - val fileActions = spark.read.parquet( - (if (policy == "v2") sidecars else checkpoints).map(_.toString): _*) - val checkpointAdds = fileActions.toJSON.collect().toVector.map(Action.parse) - .collect { case a: AddFile => a } - val expectedAdds = adds.take(4).map { add => - add.copy(dataChange = false, tags = if (paddedFiles.nonEmpty) - Some(Map("e2e43.padding" -> padding)) else add.tags) - } - assert(checkpointAdds.sortBy(_.path) == expectedAdds.sortBy(_.path)) - if (policy == "classic") { - val top = fileActions.toJSON.collect().toVector.map(Action.parse) - assert(top.collect { case m: Metadata => m } == Seq(metadata)) - assert(top.collect { case p: Protocol => p } == Seq(protocol)) - } - if (isCrc) { - Seq(baseVersion, latestVersion).foreach { version => - val crcPath = log.resolve(f"$version%020d.crc") - assert(Files.exists(crcPath)) - val crc = JsonUtil.mapper.readTree(Files.readString(crcPath, UTF_8)) - assert(crc.has("allFiles") && crc.get("allFiles").isArray) - val active = (if (version == baseVersion) adds.slice(1, 4) else adds.drop(1)) - .map(_.copy(dataChange = false)) - val encoded = crc.get("allFiles").elements().asScala.map { node => - Action.parse("{\"add\":" + node.toString + "}").asInstanceOf[AddFile] - }.toVector - // The optional tags map can round-trip through Parquet as empty, not absent. - val normalized = encoded.map(a => a.copy(tags = a.tags.filter(_.nonEmpty))) - assert(normalized.sortBy(_.path) == active.sortBy(_.path)) - assert(crc.get("numFiles").asLong() == active.size) - assert(crc.get("tableSizeBytes").asLong() == active.map(_.size).sum) - info(s"size CRC v$version bytes=${Files.size(crcPath)} " + - s"allFiles=${encoded.size} activeBytes=${active.map(_.size).sum}") - } - } else { - // JSON 4 supports historical version lookup; absent 0..3 requires checkpoint 4. - // JSON 5 is a real one-commit replay tail with one new AddFile. - files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) - .filter(_.getFileName.toString.take(20).toLong < 4L).foreach(Files.delete) - files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) - Files.deleteIfExists(log.resolve("_last_checkpoint")) - } - val jsonPaths = files.filter(p => Files.exists(p) && - p.getFileName.toString.matches("[0-9]{20}\\.json")) - (checkpoints ++ sidecars ++ jsonPaths).foreach { path => - val large = paddedFiles.contains(path) || - (shape == "manifest" && checkpoints.contains(path)) || - (shape == "json" && path == CommitLog.commitFile(tableDir, latestVersion)) - val size = Files.size(path) - assert(if (large) size > boundary else size < boundary) - info(s"size $shape component=${log.relativize(path)} bytes=$size " + - s"large=$large boundary=$boundary") - } - } - Seq( - ("read_base", Some(baseVersion), None, baseRows, - ReadResult(baseRows.size.toLong, if (isCrc) 3 else 4, 0L)), - ("read_latest", None, None, latestRows, - ReadResult(latestRows.size.toLong, if (isCrc) 4 else 5, 0L)), - ("read_duplicates", None, Some("id = 21"), Seq(Row(21, null), Row(21, null)), - ReadResult(2L, 1, if (isCrc) 3L else 4L)) - ).foreach { case (name, version, predicate, rows, expected) => - readSpec(t, name = Some(name), version = version.map(Long.box).orNull, - predicate = predicate.orNull).assert { spec => - assert(spec.expectation == Succeeded(expected)) - assertReadRows(spec, copiedTablePath, schema, rows, s"size $shape $name") - } - } - Seq(Some(baseVersion), None).foreach { version => - snapshotSpec(t, version = version.map(Long.box).orNull).assert { spec => - assert(spec.query.version.contains(version.getOrElse(latestVersion))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - } - } - if (isCrc) crcSpec(t, version = baseVersion, name = "crc_complete_active") - } finally { - previous.foreach { - case (key, Some(value)) => spark.conf.set(key, value) - case (key, None) => spark.conf.unset(key) - } - } - } - } -} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessColumnMappingSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessColumnMappingSuite.scala deleted file mode 100644 index 1f328c3a..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessColumnMappingSuite.scala +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.tables - -import java.nio.file.Files - -import scala.jdk.CollectionConverters._ - -import org.apache.spark.sql.Row -import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} - -import io.delta.workload.{WorkloadContext, WorkloadTestSuite} -import io.delta.workload.engine.RowComparison -import io.delta.workload.json.JsonUtil -import io.delta.workload.log.{AddFile, CommitLog, Metadata => LogMetadata, Protocol, RemoveFile} -import io.delta.workload.model.Succeeded - -class ReadinessColumnMappingSuite extends WorkloadTestSuite("readiness_column_mapping") { - - // One primary scenario covers parsed statistics and parsed partition values together. - // The existing checkpoints_partitioned fixture is the no-mapping parsed-key control. - for (mode <- Seq("name", "id")) { - test(s"parsed_physical_names_$mode") { - val properties = Map( - "delta.columnMapping.mode" -> mode, - "delta.checkpointInterval" -> "1000", - "delta.checkpoint.writeStatsAsStruct" -> "true", - "delta.checkpoint.writeStatsAsJson" -> "false") - val propertySql = properties.toSeq.sortBy(_._1) - .map { case (key, value) => s"'$key' = '$value'" }.mkString(", ") - sql(s"""CREATE TABLE tbl ( - id INT, old_value INT, payload STRUCT, old_part STRING) - USING delta PARTITIONED BY (old_part) TBLPROPERTIES ($propertySql)""") - sql("""INSERT INTO tbl VALUES - (1, 10, named_struct('old_leaf', -10), 'east'), - (1, 10, named_struct('old_leaf', -10), 'east'), - (2, 20, named_struct('old_leaf', -20), 'west'), - (3, NULL, named_struct('old_leaf', CAST(NULL AS INT)), NULL), - (4, 40, NULL, 'east')""") - sql("ALTER TABLE tbl RENAME COLUMN old_value TO value") - sql("ALTER TABLE tbl RENAME COLUMN payload.old_leaf TO leaf") - sql("ALTER TABLE tbl RENAME COLUMN old_part TO region") - sql("""INSERT INTO tbl VALUES - (5, 100, named_struct('leaf', -100), 'north'), - (6, NULL, named_struct('leaf', CAST(NULL AS INT)), NULL), - (7, 200, named_struct('leaf', -200), 'east')""") - forceCheckpoint("tbl") - // A real v6 tail is replayed after the v5 checkpoint; v5 JSON is only version lookup. - sql("INSERT INTO tbl VALUES (8, 300, named_struct('leaf', -300), 'west')") - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - val actions = (0L to 6L).map { version => - version -> CommitLog.readRaw(t.sourcePath, version).map(JsonUtil.mapper.readTree) - }.toMap - val beforeMetadata = actions(0L).find(_.has("metaData")).get.get("metaData") - val metadata = actions(4L).find(_.has("metaData")).get.get("metaData") - val before = DataType.fromJson(beforeMetadata.get("schemaString").asText) - .asInstanceOf[StructType] - val mapped = DataType.fromJson(metadata.get("schemaString").asText) - .asInstanceOf[StructType] - val physicalKey = "delta.columnMapping.physicalName" - val idKey = "delta.columnMapping.id" - val oldFields = before.fields.toSeq :+ - before("payload").dataType.asInstanceOf[StructType]("old_leaf") - val fields = mapped.fields.toSeq :+ - mapped("payload").dataType.asInstanceOf[StructType]("leaf") - assert(mapped.fieldNames.toSeq == Seq("id", "value", "payload", "region")) - assert(fields.map(_.metadata.getLong(idKey)).toSet == Set(1L, 2L, 3L, 4L, 5L)) - assert(fields.map(_.metadata) == oldFields.map(_.metadata)) - assert(fields.forall(field => field.metadata.getString(physicalKey) != field.name)) - val physical = fields.map(_.metadata.getString(physicalKey)) - assert(physical.distinct.size == 5) - val partitionName = physical(3) - val statsNames = physical.take(3).toSet - val originals = actions.values.flatten.filter(_.has("add")) - .map(_.get("add")).map(add => add.get("path").asText -> add).toMap - - mutateTable(t) { tableDir => - val logDir = tableDir.resolve("_delta_log") - for (version <- 0L to 6L; line <- CommitLog.readRaw(tableDir, version)) { - println(s"Physical-name source action: $mode v$version $line") - } - val checkpointPath = logDir.resolve("00000000000000000005.checkpoint.parquet") - assert(Files.exists(checkpointPath)) - val checkpoint = spark.read.parquet(checkpointPath.toString) - val addSchema = checkpoint.schema("add").dataType.asInstanceOf[StructType] - assert(!addSchema.fieldNames.contains("stats")) - val partition = addSchema("partitionValues_parsed").dataType.asInstanceOf[StructType] - assert(partition.fieldNames.toSeq == Seq(partitionName)) - assert(partition.fields.head.dataType == StringType) - val stats = addSchema("stats_parsed").dataType.asInstanceOf[StructType] - for (kind <- Seq("minValues", "maxValues", "nullCount")) { - val leaves = stats(kind).dataType.asInstanceOf[StructType] - assert(leaves.fieldNames.toSet == statsNames) - assert(leaves(physical(2)).dataType.asInstanceOf[StructType] - .fieldNames.toSeq == Seq(physical(4))) - } - val adds = checkpoint.filter("add IS NOT NULL").select("add").toJSON.collect() - assert(adds.nonEmpty) - for (line <- adds) { - val add = JsonUtil.mapper.readTree(line).get("add") - val original = originals(add.get("path").asText) - assert(add.get("partitionValues") == original.get("partitionValues")) - assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) - val parsedPartition = add.get("partitionValues_parsed").get(partitionName) - val rawPartition = original.get("partitionValues").get(partitionName) - assert(Option(parsedPartition).filterNot(_.isNull).map(_.asText) == - Option(rawPartition).filterNot(_.isNull).map(_.asText)) - val rawStats = JsonUtil.mapper.readTree(original.get("stats").asText) - val parsedStats = add.get("stats_parsed") - assert(parsedStats.get("numRecords") == rawStats.get("numRecords")) - for (kind <- Seq("minValues", "maxValues", "nullCount"); - path <- Seq(Seq(physical(0)), Seq(physical(1)), Seq(physical(2), physical(4)))) { - val parsed = path.foldLeft(parsedStats.path(kind))((node, key) => node.path(key)) - val raw = path.foldLeft(rawStats.path(kind))((node, key) => node.path(key)) - assert(Option(parsed).filterNot(n => n.isNull || n.isMissingNode) == - Option(raw).filterNot(n => n.isNull || n.isMissingNode)) - } - } - // Earlier protocol/metadata/file history is absent: v5 checkpoint remains required. - // Keep v5 for version lookup and v6 as the actual post-checkpoint replay tail. - val stream = Files.list(logDir) - val files = try stream.iterator().asScala.toSeq finally stream.close() - files.filter { path => - val name = path.getFileName.toString - name.endsWith(".crc") || - (name.matches("[0-9]{20}\\.json") && name < "00000000000000000005.json") - }.foreach(Files.delete) - println(s"Physical-name checkpoint/layout oracle passed: $mode") - } - - val schema = new StructType().add("id", IntegerType).add("value", IntegerType) - .add("payload", new StructType().add("leaf", IntegerType)).add("region", StringType) - val rows = Seq( - Row(1, 10, Row(-10), "east"), Row(1, 10, Row(-10), "east"), - Row(2, 20, Row(-20), "west"), Row(3, null, Row(null), null), - Row(4, 40, null, "east"), Row(5, 100, Row(-100), "north"), - Row(6, null, Row(null), null), Row(7, 200, Row(-200), "east"), - Row(8, 300, Row(-300), "west")) - // Row selections are literal bag oracles, independent of Spark's predicate evaluation. - val reads = Seq( - ("read_all", null, rows.indices, None, None), - ("read_checkpoint", null, 0 until 8, Some(5L), None), - ("read_old_file", "value = 10", Seq(0, 1), None, None), - ("read_new_file", "value = 100", Seq(5), None, None), - ("read_tail", "value = 300", Seq(8), None, None), - ("read_missing", "value = 999", Seq.empty[Int], None, None), - ("read_null", "value IS NULL", Seq(3, 6), None, None), - ("read_nested_old", "payload.leaf = -10", Seq(0, 1), None, None), - ("read_nested_new", "payload.leaf = -100", Seq(5), None, None), - ("read_nested_null", "payload.leaf IS NULL", Seq(3, 4, 6), None, None), - ("read_partition_old", "region = 'east'", Seq(0, 1, 4, 7), None, None), - ("read_partition_new", "region = 'north'", Seq(5), None, None), - ("read_partition_null", "region IS NULL", Seq(3, 6), None, None), - ("read_projection", null, rows.indices, None, Some(Seq("region", "value", "payload")))) - for ((name, predicate, indices, version, columns) <- reads) { - readSpec(t, name = Some(name), predicate = predicate, - version = version.map(Long.box).orNull, columns = columns).assert { spec => - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == indices.size.toLong) - case other => fail(s"$mode/$name must succeed: $other") - } - val literal = spark.createDataFrame(indices.map(rows).asJava, schema) - val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) - val expected = outputDir.resolve(table.outputName).resolve("expected") - .resolve(s"${table.outputName}_$name").resolve("expected_data") - val captured = spark.read.parquet(expected.toString) - RowComparison.assertRowsEqual(oracle, captured, s"$mode/$name literal bag") - assert(captured.schema.fields.forall(_.nullable)) - println(s"Physical-name literal row/schema oracle passed: $mode/$name") - } - } - for (version <- Seq(5L, 6L)) { - snapshotSpec(t, version = version).assert { spec => - spec.expectation match { - case Succeeded(result) => - assert(result.metadata.schemaString == metadata.get("schemaString").asText) - assert(result.metadata.partitionColumns == Seq("region")) - assert(result.metadata.configuration == - properties + ("delta.columnMapping.maxColumnId" -> "5")) - case other => fail(s"$mode snapshot v$version must succeed: $other") - } - println(s"Physical-name snapshot mapping oracle passed: $mode/v$version") - } - } - } - } - - test("metadata_partition_values_ignore_directories") { - sql("""CREATE TABLE tbl (id INT, payload STRING, part STRING) USING delta - PARTITIONED BY (part) TBLPROPERTIES ('delta.checkpointInterval' = '1000')""") - sql("""INSERT INTO tbl VALUES - (1, 'duplicate', 'east'), (1, 'duplicate', 'east'), - (2, 'conflicting', 'west'), (3, 'null_partition', NULL), - (4, 'no_hive_directory', 'plain')""") - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - val schema = new StructType().add("id", IntegerType).add("payload", StringType) - .add("part", StringType) - - mutateTable(t) { tableDir => - val originals = CommitLog.read(tableDir, 1L).collect { case add: AddFile => add } - assert(originals.nonEmpty) - assert(originals.map(_.partitionValues("part")).toSet == Set("east", "west", null, "plain")) - CommitLog.mutate(tableDir, 1L) { actions => - actions.map { - case add: AddFile => - val directory = if (add.partitionValues("part") == "plain") "opaque/nested" - else "part=directory_lie" - val original = tableDir.resolve(add.path) - val relative = s"$directory/${original.getFileName}" - val target = tableDir.resolve(relative) - Files.createDirectories(target.getParent) - Files.move(original, target) - assert(!Files.exists(original) && Files.isRegularFile(target)) - assert(Files.size(target) == add.size) - add.copy(path = relative) - case other => other - } - } - val moved = CommitLog.read(tableDir, 1L).collect { case add: AddFile => add } - assert(moved.map(_.partitionValues) == originals.map(_.partitionValues)) - assert(moved.zip(originals).forall { case (after, before) => - after.copy(path = before.path) == before - }) - assert(moved.forall(add => Files.isRegularFile(tableDir.resolve(add.path)))) - assert(moved.forall(add => !originals.exists(_.path == add.path))) - // Require JSON replay: no checkpoint or Delta CRC may retain the original file paths. - val stream = Files.list(tableDir.resolve("_delta_log")) - val files = try stream.iterator().asScala.toSeq finally stream.close() - assert(!files.exists(_.getFileName.toString.contains("checkpoint"))) - files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) - println("Metadata-partition layout oracle passed: conflicting and non-Hive directories") - } - - val rows = Seq(Row(1, "duplicate", "east"), Row(1, "duplicate", "east"), - Row(2, "conflicting", "west"), Row(3, "null_partition", null), - Row(4, "no_hive_directory", "plain")) - val reads = Seq( - ("read_all", null, rows.indices, None), - ("read_metadata_east", "part = 'east'", Seq(0, 1), None), - ("read_metadata_west", "part = 'west'", Seq(2), None), - ("read_directory_lie", "part = 'directory_lie'", Seq.empty[Int], None), - ("read_null", "part IS NULL", Seq(3), None), - ("read_no_hive_directory", "part = 'plain'", Seq(4), None), - ("read_projection", null, rows.indices, Some(Seq("part", "id"))), - ("read_null_projection", "part IS NULL", Seq(3), Some(Seq("part", "payload")))) - for ((name, predicate, indices, columns) <- reads) { - readSpec(t, name = Some(name), predicate = predicate, columns = columns).assert { spec => - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == indices.size.toLong) - case other => fail(s"metadata partition/$name must succeed: $other") - } - val literal = spark.createDataFrame(indices.map(rows).asJava, schema) - val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) - val expected = outputDir.resolve(table.outputName).resolve("expected") - .resolve(s"${table.outputName}_$name").resolve("expected_data") - val captured = spark.read.parquet(expected.toString) - RowComparison.assertRowsEqual(oracle, captured, s"metadata partition/$name literal bag") - assert(captured.schema == oracle.schema) - println(s"Metadata-partition literal row/schema oracle passed: $name") - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - assert(DataType.fromJson(result.metadata.schemaString) == schema) - assert(result.metadata.partitionColumns == Seq("part")) - assert(result.metadata.configuration == Map("delta.checkpointInterval" -> "1000")) - case other => fail(s"metadata partition snapshot must succeed: $other") - } - println("Metadata-partition snapshot schema/partition oracle passed") - } - } - - test("enable_name_mapping_with_full_rewrite") { - sql("""CREATE TABLE tbl (id INT, value INT, payload STRING) USING delta - TBLPROPERTIES ('delta.columnMapping.mode' = 'none', - 'delta.checkpointInterval' = '1000')""") - sql("INSERT INTO tbl VALUES (1, 10, 'duplicate'), (1, 10, 'duplicate'), (2, NULL, 'nullable')") - sql("INSERT INTO tbl VALUES (3, 30, NULL), (NULL, NULL, NULL), (NULL, NULL, NULL)") - // A separately materialized source prevents replacement from reading its own target. - // Both SQL-created tables are tracked by the normal WorkloadContext cleanup. - sql("CREATE TABLE rewrite_source USING delta AS SELECT id, value, payload FROM tbl") - sql("""CREATE OR REPLACE TABLE tbl USING delta TBLPROPERTIES ( - 'delta.columnMapping.mode' = 'name', 'delta.minReaderVersion' = '2', - 'delta.minWriterVersion' = '5', 'delta.checkpointInterval' = '1000') - AS SELECT id, value, payload FROM rewrite_source""") - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - val before = (0L to 2L).flatMap(CommitLog.read(t.sourcePath, _)) - val rewrite = CommitLog.read(t.sourcePath, 3L) - val oldAdds = before.collect { case add: AddFile => add } - val newAdds = rewrite.collect { case add: AddFile => add } - val removed = rewrite.collect { case remove: RemoveFile => remove } - assert(!before.exists(_.isInstanceOf[RemoveFile])) - assert(oldAdds.nonEmpty && newAdds.nonEmpty) - val oldPaths = oldAdds.map(_.path).toSet - val newPaths = newAdds.map(_.path).toSet - assert(oldPaths.size == oldAdds.size && newPaths.size == newAdds.size) - assert(removed.map(_.path).toSet == oldPaths && removed.size == oldPaths.size) - assert(newPaths.intersect(oldPaths).isEmpty) - assert((oldPaths -- removed.map(_.path)) ++ newPaths == newPaths) - assert((oldAdds ++ newAdds).forall(add => - Files.isRegularFile(t.sourcePath.resolve(add.path)) && - Files.size(t.sourcePath.resolve(add.path)) == add.size)) - - val schema = new StructType().add("id", IntegerType).add("value", IntegerType) - .add("payload", StringType) - val oldMetadata = before.collect { case metadata: LogMetadata => metadata }.last - val newMetadata = rewrite.collect { case metadata: LogMetadata => metadata }.last - assert(DataType.fromJson(oldMetadata.schemaString) == schema) - assert(oldMetadata.configuration == Map( - "delta.columnMapping.mode" -> "none", "delta.checkpointInterval" -> "1000")) - val mapped = DataType.fromJson(newMetadata.schemaString).asInstanceOf[StructType] - assert(mapped.fields.map(f => (f.name, f.dataType, f.nullable)).toSeq == - schema.fields.map(f => (f.name, f.dataType, f.nullable)).toSeq) - val idKey = "delta.columnMapping.id" - val physicalKey = "delta.columnMapping.physicalName" - assert(mapped.fields.map(_.metadata.getLong(idKey)).toSet == Set(1L, 2L, 3L)) - val physicalNames = mapped.fields.map(_.metadata.getString(physicalKey)).toSeq - assert(physicalNames.distinct.size == 3) - assert(mapped.fields.forall(f => f.metadata.getString(physicalKey) != f.name)) - assert(newMetadata.configuration == Map( - "delta.columnMapping.mode" -> "name", "delta.columnMapping.maxColumnId" -> "3", - "delta.checkpointInterval" -> "1000")) - assert(oldMetadata.partitionColumns.isEmpty && newMetadata.partitionColumns.isEmpty) - assert(before.collect { case p: Protocol => p }.last == Protocol(1, 2)) - assert(rewrite.collect { case p: Protocol => p }.last == Protocol(2, 5)) - for (add <- oldAdds) { - assert(spark.read.parquet(t.sourcePath.resolve(add.path).toString).schema == schema) - } - for (add <- newAdds) { - val physical = spark.read.parquet(t.sourcePath.resolve(add.path).toString).schema - assert(physical.fieldNames.toSeq == physicalNames) - assert(physical.fields.map(_.dataType).toSeq == schema.fields.map(_.dataType).toSeq) - assert(physical.fields.map(_.metadata.getLong("parquet.field.id")).toSeq == - mapped.fields.map(_.metadata.getLong(idKey)).toSeq) - } - println(s"Full-rewrite action/physical-name oracle passed: " + - s"removed=${oldPaths.size}, fresh=${newPaths.size}") - - // Every selection is a literal bag, including both copies of each duplicate/null row. - val rows = Seq(Row(1, 10, "duplicate"), Row(1, 10, "duplicate"), - Row(2, null, "nullable"), Row(3, 30, null), Row(null, null, null), Row(null, null, null)) - val reads = Seq( - ("read_before", null, rows.indices, Some(2L), None), - ("read_all", null, rows.indices, None, None), - ("read_rewritten", null, rows.indices, Some(3L), None), - ("read_projection", null, rows.indices, None, Some(Seq("payload", "id"))), - ("read_hit", "value = 10", Seq(0, 1), None, None), - ("read_miss", "value = 999", Seq.empty[Int], None, None), - ("read_null_value", "value IS NULL", Seq(2, 4, 5), None, None), - ("read_null_payload", "payload IS NULL", Seq(3, 4, 5), None, None), - ("read_null_id_projection", "id IS NULL", Seq(4, 5), None, Some(Seq("id", "value")))) - for ((name, predicate, indices, version, columns) <- reads) { - readSpec(t, name = Some(name), predicate = predicate, - version = version.map(Long.box).orNull, columns = columns).assert { spec => - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == indices.size.toLong) - case other => fail(s"full rewrite/$name must succeed: $other") - } - val literal = spark.createDataFrame(indices.map(rows).asJava, schema) - val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) - val expected = outputDir.resolve(table.outputName).resolve("expected") - .resolve(s"${table.outputName}_$name").resolve("expected_data") - val captured = spark.read.parquet(expected.toString) - RowComparison.assertRowsEqual(oracle, captured, s"full rewrite/$name literal bag") - assert(captured.schema == oracle.schema) - println(s"Full-rewrite literal row/schema oracle passed: $name") - } - } - for ((version, metadata, reader, writer) <- Seq( - (2L, oldMetadata, 1, 2), (3L, newMetadata, 2, 5))) { - snapshotSpec(t, version = version).assert { spec => - spec.expectation match { - case Succeeded(result) => - assert(result.metadata.schemaString == metadata.schemaString) - assert(result.metadata.configuration == metadata.configuration) - assert(result.metadata.partitionColumns.isEmpty) - assert(result.protocol.minReaderVersion == reader) - assert(result.protocol.minWriterVersion == writer) - case other => fail(s"full rewrite snapshot v$version must succeed: $other") - } - println(s"Full-rewrite snapshot transition oracle passed: v$version") - } - } - } -} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessPredicatesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessPredicatesSuite.scala deleted file mode 100644 index 208d081f..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessPredicatesSuite.scala +++ /dev/null @@ -1,779 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.tables - -import org.apache.spark.sql.types.{DataType, StructType} - -import io.delta.workload.{WorkloadContext, WorkloadTestSuite} -import io.delta.workload.engine.RowComparison -import io.delta.workload.model._ - -/** - * Portable row semantics for read predicates, without a pruning assertion. - * Capture and replay use UTC. Signed-zero requires Parquet filter pushdown disabled; three - * narrowing CAST cases require ANSI disabled. Use `startswith`; `starts_with` is unsupported. - */ -class ReadinessPredicatesSuite extends WorkloadTestSuite("readiness_predicates") { - - for (partitioned <- Seq(false, true)) { - val layout = if (partitioned) "partitioned" else "unpartitioned" - test(s"nondeterministic_values_$layout") { - val partition = if (partitioned) "PARTITIONED BY (p)" else "" - sql(s"CREATE TABLE tbl (id INT, value INT, p INT) USING delta $partition") - sql("INSERT INTO tbl VALUES (1, 10, 1), (2, 20, 1), (2, 20, 1)") - sql("INSERT INTO tbl VALUES (3, 20, 2), (4, 30, 2), (5, 40, 3)") - sql("INSERT INTO tbl VALUES (6, NULL, 1), (7, 50, NULL), (8, NULL, NULL)") - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - - // Literal bags are independent of the filtered Delta reads, including duplicate/NULL rows. - val all = Seq( - "(1, 10, 1)", - "(2, 20, 1)", - "(2, 20, 1)", - "(3, 20, 2)", - "(4, 30, 2)", - "(5, 40, 3)", - "(6, NULL, 1)", - "(7, 50, NULL)", - "(8, NULL, NULL)" - ) - val equality = Seq("(2, 20, 1)", "(2, 20, 1)", "(3, 20, 2)") - val range = equality :+ "(4, 30, 2)" - val partitionBag = Seq("(1, 10, 1)", "(2, 20, 1)", "(2, 20, 1)", "(6, NULL, 1)") - val cases = Seq( - ("read_all", null, all), - ("random_all", "rand() >= 0", all), - ("random_none", "rand() < 0", Seq.empty[String]), - ("equality_control", "value = 20", equality), - ("range_control", "value >= 20 AND value < 40", range), - ("and_equality", "rand() >= 0 AND value = 20", equality), - ("and_range", "rand() >= 0 AND (value >= 20 AND value < 40)", range), - ("and_none", "rand() < 0 AND value = 20", Seq.empty[String]), - ("or_equality", "rand() < 0 OR value = 20", equality), - ("or_range", "rand() < 0 OR (value >= 20 AND value < 40)", range), - ("or_all", "rand() >= 0 OR value = 20", all), - ("partition_control", "p = 1", partitionBag), - ("and_partition", "p = 1 AND rand() >= 0", partitionBag), - ("or_partition", "p = 1 OR rand() < 0", partitionBag), - // Spark can simplify the domain-only controls. Referencing the row's small exact INT id - // keeps a nondeterministic expression at read time: adding [0, 1) cannot decrease id. - ("row_random_all", "rand() + id >= id", all), - ("row_random_none", "rand() + id < id", Seq.empty[String]), - ("row_and_equality", "rand() + id >= id AND value = 20", equality), - ("row_and_range", "rand() + id >= id AND (value >= 20 AND value < 40)", range), - ("row_or_equality", "rand() + id < id OR value = 20", equality), - ("row_or_range", "rand() + id < id OR (value >= 20 AND value < 40)", range), - ("row_and_partition", "p = 1 AND rand() + id >= id", partitionBag), - ("row_or_partition", "p = 1 OR rand() + id < id", partitionBag) - ) - - // rand() stays in the serialized read predicate. Its [0, 1) domain gives a stable bag - // across RNG seeds, partition counts and replays; random values are never stored on INSERT. - for ((label, predicate, bag) <- cases) { - readSpec(t, predicate = predicate, name = Some(label)).assert { spec => - spec.expectation match { - case Succeeded(_) => - case other => fail(s"${table.outputName}/$label must succeed: $other") - } - val oracle = if (bag.nonEmpty) { - s"SELECT CAST(id AS INT) AS id, CAST(value AS INT) AS value, " + - s"CAST(p AS INT) AS p FROM VALUES ${bag.mkString(", ")} AS expected(id, value, p)" - } else { - "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS INT) AS value, " + - "CAST(NULL AS INT) AS p WHERE false" - } - val expectedPath = outputDir - .resolve(table.outputName) - .resolve("expected") - .resolve(s"${table.outputName}_$label") - .resolve("expected_data") - RowComparison.assertRowsEqual( - spark.sql(oracle), - spark.read.parquet(expectedPath.toString), - s"${table.outputName}/$label literal oracle" - ) - } - } - - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] - require(schema == StructType.fromDDL("id INT, value INT, p INT")) - require( - result.metadata.partitionColumns == - (if (partitioned) Seq("p") else Seq.empty) - ) - case other => fail(s"${table.outputName} snapshot must succeed: $other") - } - } - } - } - - test("signed_zero") { - require( - spark.conf.get("spark.sql.parquet.filterPushdown") == "false", - "The signed-zero bag is validated with Parquet filter pushdown disabled" - ) - sql("CREATE TABLE tbl (id INT, f FLOAT, d DOUBLE) USING delta") - // Separate writes keep the two zero signs and nonzero ranges in distinct files. - sql("""INSERT INTO tbl VALUES - |(1, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE)), - |(2, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE)), - |(2, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE))""".stripMargin) - sql("""INSERT INTO tbl VALUES - |(3, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE)), - |(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE)), - |(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE))""".stripMargin) - sql("INSERT INTO tbl VALUES (5, -1, -1), (6, -2, -2)") - sql("INSERT INTO tbl VALUES (7, 1, 1), (8, 2, 2)") - sql("INSERT INTO tbl VALUES (9, NULL, NULL)") - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - - // Literal bags do not filter the source table. IDs distinguish the original zero signs; - // exact duplicates check multiplicity. Row equality need not retain a zero's sign bit. - val zeros = Seq("(1, 0, 0)", "(2, 0, 0)", "(2, 0, 0)", "(3, 0, 0)", "(4, 0, 0)", "(4, 0, 0)") - val negative = Seq("(5, -1, -1)", "(6, -2, -2)") - val positive = Seq("(7, 1, 1)", "(8, 2, 2)") - val nulls = Seq("(9, NULL, NULL)") - val nonzero = negative ++ positive - val all = zeros ++ nonzero ++ nulls - val cases = Seq(("read_all", null, all)) ++ - Seq(("f", "FLOAT"), ("d", "DOUBLE")).flatMap { - case (column, dataType) => - val plus = s"CAST('0.0' AS $dataType)" - val minus = s"CAST('-0.0' AS $dataType)" - val one = s"CAST(1 AS $dataType)" - val negativeOne = s"CAST(-1 AS $dataType)" - val typedNull = s"CAST(NULL AS $dataType)" - val comparisons = for { - (sign, zero) <- Seq(("positive", plus), ("negative", minus)) - (label, operator, bag) <- Seq( - ("eq", "=", zeros), - ("null_safe_eq", "<=>", zeros), - ("ne", "!=", nonzero), - ("lt", "<", negative), - ("le", "<=", negative ++ zeros), - ("gt", ">", positive), - ("ge", ">=", zeros ++ positive) - ) - } yield (s"${column}_${label}_$sign", s"$column $operator $zero", bag) - val inLists = Seq( - ("in_positive", plus, zeros), - ("in_negative", minus, zeros), - ("in_both", s"$plus, $minus", zeros), - ("in_positive_null", s"$plus, $typedNull", zeros), - ("in_negative_null", s"$minus, $typedNull", zeros), - ("in_both_null", s"$plus, $minus, $typedNull", zeros), - ("in_positive_nonzero", s"$plus, $one", zeros :+ "(7, 1, 1)"), - ("in_negative_nonzero", s"$minus, $negativeOne", zeros :+ "(5, -1, -1)"), - ( - "in_both_null_nonzero", - s"$plus, $minus, $typedNull, $negativeOne, $one", - zeros ++ Seq("(5, -1, -1)", "(7, 1, 1)") - ), - ("in_nonzero_null", s"$negativeOne, $one, $typedNull", Seq("(5, -1, -1)", "(7, 1, 1)")), - ("in_null", typedNull, Seq.empty[String]), - ("in_miss", s"CAST(9 AS $dataType), $typedNull", Seq.empty[String]) - ) - comparisons ++ Seq((s"${column}_null_safe_null", s"$column <=> $typedNull", nulls)) ++ - inLists.map { - case (label, literals, bag) => - (s"${column}_$label", s"$column IN ($literals)", bag) - } - } - - for ((label, predicate, bag) <- cases) { - readSpec(t, predicate = predicate, name = Some(label)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == bag.size) - case other => fail(s"${table.outputName}/$label must succeed: $other") - } - val oracle = if (bag.nonEmpty) { - "SELECT CAST(id AS INT) AS id, CAST(f AS FLOAT) AS f, CAST(d AS DOUBLE) AS d " + - s"FROM VALUES ${bag.mkString(", ")} AS expected(id, f, d)" - } else { - "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS FLOAT) AS f, " + - "CAST(NULL AS DOUBLE) AS d WHERE false" - } - val expectedPath = outputDir - .resolve(table.outputName) - .resolve("expected") - .resolve(s"${table.outputName}_$label") - .resolve("expected_data") - RowComparison.assertRowsEqual( - spark.sql(oracle), - spark.read.parquet(expectedPath.toString), - s"${table.outputName}/$label literal oracle" - ) - } - } - - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] - require(schema == StructType.fromDDL("id INT, f FLOAT, d DOUBLE")) - require(result.metadata.partitionColumns.isEmpty) - case other => fail(s"${table.outputName} snapshot must succeed: $other") - } - } - } - - // Explicit predicate casts, precision boundaries and legacy narrowing. - // Expected IDs are derived independently of the CAST predicates. Source duplicates stay intact. - for ((subcase, sourceType, sourceRows, checks, legacy) <- Seq( - ( - "float_double", - "FLOAT", - Seq( - (1, "-0.1"), - (2, "0.1"), - (2, "0.1"), - (3, "0.10000001"), - (4, "0.2"), - (5, "1.0"), - (6, "NULL") - ), - Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6)), - ("read_hit", "CAST(value AS DOUBLE) = CAST('0.10000000149011612' AS DOUBLE)", Seq(2)), - ( - "decimal_spelling_miss", - "CAST(value AS DOUBLE) = CAST('0.1' AS DOUBLE)", - Seq.empty[Int] - ), - ("above_decimal", "CAST(value AS DOUBLE) > CAST('0.1' AS DOUBLE)", Seq(2, 3, 4, 5)), - ( - "adjacent_range", - "CAST(value AS DOUBLE) >= CAST('0.10000000149011612' AS DOUBLE) AND " + - "CAST(value AS DOUBLE) < CAST('0.10000000894069672' AS DOUBLE)", - Seq(2) - ), - ( - "negative_hit", - "CAST(value AS DOUBLE) = CAST('-0.10000000149011612' AS DOUBLE)", - Seq(1) - ), - ("read_miss", "CAST(value AS DOUBLE) = CAST('2' AS DOUBLE)", Seq.empty[Int]) - ), - false - ), - ( - "double_2p53", - "BIGINT", - Seq( - (1, "9007199254740991"), - (2, "9007199254740992"), - (3, "9007199254740993"), - (3, "9007199254740993"), - (4, "9007199254740994"), - (5, "9007199254740995"), - (6, "-9007199254740993"), - (7, "NULL") - ), - Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7)), - ("read_hit", "CAST(value AS DOUBLE) = CAST('9007199254740992' AS DOUBLE)", Seq(2, 3)), - ("adjacent_hit", "CAST(value AS DOUBLE) = CAST('9007199254740994' AS DOUBLE)", Seq(4)), - ( - "adjacent_range", - "CAST(value AS DOUBLE) > CAST('9007199254740992' AS DOUBLE) AND " + - "CAST(value AS DOUBLE) < CAST('9007199254740996' AS DOUBLE)", - Seq(4) - ), - ( - "below_boundary", - "CAST(value AS DOUBLE) < CAST('9007199254740992' AS DOUBLE)", - Seq(1, 6) - ), - ( - "read_miss", - "CAST(value AS DOUBLE) = CAST('9007199254740998' AS DOUBLE)", - Seq.empty[Int] - ) - ), - false - ), - ( - "float_2p24", - "INT", - Seq( - (1, "16777215"), - (2, "16777216"), - (3, "16777217"), - (3, "16777217"), - (4, "16777218"), - (5, "16777219"), - (6, "-16777217"), - (7, "NULL") - ), - Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7)), - ("read_hit", "CAST(value AS FLOAT) = CAST('16777216' AS FLOAT)", Seq(2, 3)), - ("adjacent_hit", "CAST(value AS FLOAT) = CAST('16777218' AS FLOAT)", Seq(4)), - ( - "adjacent_range", - "CAST(value AS FLOAT) > CAST('16777216' AS FLOAT) AND " + - "CAST(value AS FLOAT) < CAST('16777220' AS FLOAT)", - Seq(4) - ), - ("below_boundary", "CAST(value AS FLOAT) < CAST('16777216' AS FLOAT)", Seq(1, 6)), - ("read_miss", "CAST(value AS FLOAT) = CAST('16777222' AS FLOAT)", Seq.empty[Int]) - ), - false - ), - ( - "long_int_wrap", - "BIGINT", - Seq( - (1, "4"), - (2, "5"), - (2, "5"), - (3, "4294967301"), - (3, "4294967301"), - (4, "4294967300"), - (5, "4294967302"), - (6, "-4294967291"), - (7, "NULL"), - (8, "-1"), - (9, "6") - ), - Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), - ("read_hit", "CAST(value AS INT) = CAST('5' AS INT)", Seq(2, 3, 6)), - ("lower_neighbor", "CAST(value AS INT) = CAST('4' AS INT)", Seq(1, 4)), - ("upper_neighbor", "CAST(value AS INT) = CAST('6' AS INT)", Seq(5, 9)), - ("below_five", "CAST(value AS INT) < CAST('5' AS INT)", Seq(1, 4, 8)), - ("above_five", "CAST(value AS INT) > CAST('5' AS INT)", Seq(5, 9)), - ("read_miss", "CAST(value AS INT) = CAST('7' AS INT)", Seq.empty[Int]) - ), - true - ), - ( - "to_short_wrap", - "INT", - Seq( - (1, "4"), - (2, "5"), - (2, "5"), - (3, "65541"), - (3, "65541"), - (4, "65540"), - (5, "65542"), - (6, "-65531"), - (7, "NULL"), - (8, "-1"), - (9, "6") - ), - Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), - ("read_hit", "CAST(value AS SHORT) = CAST('5' AS SHORT)", Seq(2, 3, 6)), - ("lower_neighbor", "CAST(value AS SHORT) = CAST('4' AS SHORT)", Seq(1, 4)), - ("upper_neighbor", "CAST(value AS SHORT) = CAST('6' AS SHORT)", Seq(5, 9)), - ("below_five", "CAST(value AS SHORT) < CAST('5' AS SHORT)", Seq(1, 4, 8)), - ("above_five", "CAST(value AS SHORT) > CAST('5' AS SHORT)", Seq(5, 9)), - ("read_miss", "CAST(value AS SHORT) = CAST('7' AS SHORT)", Seq.empty[Int]) - ), - true - ), - ( - "int_byte_wrap", - "INT", - Seq( - (1, "4"), - (2, "5"), - (2, "5"), - (3, "261"), - (3, "261"), - (4, "260"), - (5, "262"), - (6, "-251"), - (7, "NULL"), - (8, "-1"), - (9, "6") - ), - Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), - ("read_hit", "CAST(value AS BYTE) = CAST('5' AS BYTE)", Seq(2, 3, 6)), - ("lower_neighbor", "CAST(value AS BYTE) = CAST('4' AS BYTE)", Seq(1, 4)), - ("upper_neighbor", "CAST(value AS BYTE) = CAST('6' AS BYTE)", Seq(5, 9)), - ("below_five", "CAST(value AS BYTE) < CAST('5' AS BYTE)", Seq(1, 4, 8)), - ("above_five", "CAST(value AS BYTE) > CAST('5' AS BYTE)", Seq(5, 9)), - ("read_miss", "CAST(value AS BYTE) = CAST('7' AS BYTE)", Seq.empty[Int]) - ), - true - ) - )) { - test(s"cast_filters_$subcase") { - // ReadQuery has no session config field. Supply the same legacy session on every replay. - if (legacy) { - require( - spark.conf.get("spark.sql.ansi.enabled") == "false", - "Narrowing CAST fixtures require an existing non-ANSI session for capture and replay" - ) - } - sql(s"CREATE TABLE tbl (id INT, value $sourceType) USING delta") - // Separate writes leave ordinary and boundary values in multiple stats-bearing files. - for (batch <- sourceRows.grouped(2)) { - val values = batch.map { case (id, value) => s"($id, $value)" }.mkString(", ") - sql(s"INSERT INTO tbl VALUES $values") - } - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - for ((label, predicate, expectedIds) <- checks) { - val bag = sourceRows.filter { case (id, _) => expectedIds.contains(id) } - readSpec(t, predicate = predicate, name = Some(label)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == bag.size) - case other => fail(s"${table.outputName}/$label must succeed: $other") - } - // Only source-type casts construct these literal bags; no predicate CAST is reused. - val oracle = if (bag.nonEmpty) { - val values = bag.map { case (id, value) => s"($id, $value)" }.mkString(", ") - s"SELECT CAST(id AS INT) AS id, CAST(value AS $sourceType) AS value " + - s"FROM VALUES $values AS expected(id, value)" - } else { - s"SELECT CAST(NULL AS INT) AS id, CAST(NULL AS $sourceType) AS value WHERE false" - } - val expectedPath = outputDir - .resolve(table.outputName) - .resolve("expected") - .resolve(s"${table.outputName}_$label") - .resolve("expected_data") - RowComparison.assertRowsEqual( - spark.sql(oracle), - spark.read.parquet(expectedPath.toString), - s"${table.outputName}/$label literal oracle" - ) - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] - require(schema == StructType.fromDDL(s"id INT, value $sourceType")) - require(result.metadata.partitionColumns.isEmpty) - require(result.metadata.configuration.isEmpty) - require(result.protocol.minReaderVersion == 1) - require(result.protocol.minWriterVersion == 2) - case other => fail(s"${table.outputName} snapshot must succeed: $other") - } - } - if (subcase == "float_double") { - val compositionCases = Seq( - ("direct_nonnegative", "CAST(value AS DOUBLE) >= 0", Seq(2, 3, 4, 5)), - ( - "compound_hit", - "(CAST(value AS DOUBLE) > 0.15 AND id < 5) OR " + - "(CAST(value AS DOUBLE) < 0 AND id = 1)", - Seq(1, 4) - ), - ( - "compound_miss", - "(CAST(value AS DOUBLE) > 2 AND id < 5) OR " + - "(CAST(value AS DOUBLE) < -2 AND id = 1)", - Seq.empty[Int] - ) - ) - for ((label, predicate, ids) <- compositionCases) { - val bag = sourceRows.filter { case (id, _) => ids.contains(id) } - readSpec(t, predicate = predicate, name = Some(label)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == bag.size) - case other => fail(s"${table.outputName}/$label must succeed: $other") - } - val oracle = if (bag.nonEmpty) { - val values = bag.map { case (id, value) => s"($id, $value)" }.mkString(", ") - s"SELECT CAST(id AS INT) AS id, CAST(value AS FLOAT) AS value " + - s"FROM VALUES $values AS expected(id, value)" - } else { - "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS FLOAT) AS value WHERE false" - } - val expectedPath = outputDir - .resolve(table.outputName) - .resolve("expected") - .resolve(s"${table.outputName}_$label") - .resolve("expected_data") - RowComparison.assertRowsEqual( - spark.sql(oracle), - spark.read.parquet(expectedPath.toString), - s"${table.outputName}/$label literal oracle" - ) - } - } - } - - } - } - - // Complex filter row semantics. File counts do not prove filter pushdown. - test("complex_filters_month_timestamp") { - // Read specs have no timezone field; use UTC through queued capture and every replay. - require(spark.conf.get("spark.sql.session.timeZone") == "UTC") - sql("CREATE TABLE tbl (id INT, ts TIMESTAMP) USING delta") - val rows = Seq( - "(1, TIMESTAMP'2023-12-31 23:59:59.999999')", - "(2, TIMESTAMP'2024-01-01 00:00:00')", - "(3, TIMESTAMP'2024-01-31 23:59:59.999999')", - "(4, TIMESTAMP'2024-02-01 00:00:00')", - "(5, TIMESTAMP'2024-02-29 23:59:59.999999')", - "(6, TIMESTAMP'2024-03-01 00:00:00')", - "(7, TIMESTAMP'2024-12-31 23:59:59.999999')", - "(8, TIMESTAMP'2025-01-01 00:00:00')", - "(8, TIMESTAMP'2025-01-01 00:00:00')", - "(9, NULL)" - ) - for (batch <- rows.grouped(2)) { - sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") - } - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. - val cases = Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), - ("january", "month(ts) = 1", Seq(2, 3, 8)), - ("february", "month(ts) = 2", Seq(4, 5)), - ("december", "month(ts) = 12", Seq(1, 7)), - ("miss_june", "month(ts) = 6", Seq.empty[Int]), - ("null_month", "month(ts) IS NULL", Seq(9)) - ) - for ((label, predicate, ids) <- cases) { - val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) - readSpec(t, predicate = predicate, name = Some(label)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == bag.size) - case other => fail(s"${table.outputName}/$label must succeed: $other") - } - val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL)" - val projection = "CAST(id AS INT) AS id, CAST(ts AS TIMESTAMP) AS ts" - val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, ts)" + - (if (bag.nonEmpty) "" else " WHERE false") - val expectedPath = outputDir - .resolve(table.outputName) - .resolve("expected") - .resolve(s"${table.outputName}_$label") - .resolve("expected_data") - RowComparison.assertRowsEqual( - spark.sql(oracle), - spark.read.parquet(expectedPath.toString), - s"${table.outputName}/$label literal oracle" - ) - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] - require(schema == StructType.fromDDL("id INT, ts TIMESTAMP")) - require(result.metadata.partitionColumns.isEmpty) - require(result.metadata.configuration.isEmpty) - require(result.protocol.minReaderVersion == 1) - require(result.protocol.minWriterVersion == 2) - case other => fail(s"${table.outputName} snapshot must succeed: $other") - } - } - } - - test("complex_filters_strings") { - sql("CREATE TABLE tbl (id INT, name STRING, s STRUCT) USING delta") - val rows = Seq( - "(1, 'apple', named_struct('name', 'apple'))", - "(2, 'banana', named_struct('name', 'banana'))", - "(3, 'alice', named_struct('name', 'alice'))", - "(4, 'bob', named_struct('name', 'bob'))", - "(5, 'charlie', named_struct('name', 'charlie'))", - "(6, 'diana', named_struct('name', 'diana'))", - "(7, 'Apple', named_struct('name', 'APPLE'))", - "(8, 'ALICE', named_struct('name', 'Alice'))", - "(8, 'ALICE', named_struct('name', 'Alice'))", - "(9, '', named_struct('name', ''))", - "(10, NULL, named_struct('name', NULL))", - "(11, 'apPle', named_struct('name', NULL))" - ) - for (batch <- rows.grouped(2)) { - sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") - } - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. - val cases = Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)), - ("prefix_hit", "startswith(name, 'app')", Seq(1)), - ("prefix_a", "startswith(name, 'a')", Seq(1, 3, 11)), - ("prefix_nested", "startswith(s.name, 'a')", Seq(1, 3)), - ("prefix_miss", "startswith(name, 'z')", Seq.empty[Int]), - ("prefix_empty", "startswith(name, '')", Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 11)), - ("prefix_null", "startswith(name, CAST(NULL AS STRING))", Seq.empty[Int]), - ("prefix_null_result", "startswith(name, 'a') IS NULL", Seq(10)), - ("upper_alice", "upper(name) = 'ALICE'", Seq(3, 8)), - ("upper_apple", "upper(name) = 'APPLE'", Seq(1, 7, 11)), - ("upper_miss", "upper(name) = 'ZEBRA'", Seq.empty[Int]), - ("upper_null", "upper(name) IS NULL", Seq(10)), - ("upper_selective", "upper(name) = 'APPLE' AND id > 1", Seq(7, 11)) - ) - for ((label, predicate, ids) <- cases) { - val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) - readSpec(t, predicate = predicate, name = Some(label)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == bag.size) - case other => fail(s"${table.outputName}/$label must succeed: $other") - } - val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL, NULL)" - val projection = "CAST(id AS INT) AS id, CAST(name AS STRING) AS name, " + - "CAST(s AS STRUCT) AS s" - val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, name, s)" + - (if (bag.nonEmpty) "" else " WHERE false") - val expectedPath = outputDir - .resolve(table.outputName) - .resolve("expected") - .resolve(s"${table.outputName}_$label") - .resolve("expected_data") - RowComparison.assertRowsEqual( - spark.sql(oracle), - spark.read.parquet(expectedPath.toString), - s"${table.outputName}/$label literal oracle" - ) - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] - require(schema == StructType.fromDDL("id INT, name STRING, s STRUCT")) - require(result.metadata.partitionColumns.isEmpty) - require(result.metadata.configuration.isEmpty) - require(result.protocol.minReaderVersion == 1) - require(result.protocol.minWriterVersion == 2) - case other => fail(s"${table.outputName} snapshot must succeed: $other") - } - } - } - - test("complex_filters_deeply_nested") { - sql( - "CREATE TABLE tbl (id INT, s STRUCT) USING delta" - ) - val rows = Seq( - "(1, named_struct('a', 1, 'b', 1, 'c', NULL, 'd', NULL, 'e', NULL, 'f', NULL, " + - "'g', NULL, 'h', NULL, 'i', NULL, 'j', NULL))", - "(2, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 1, 'e', NULL, 'f', NULL, 'g', " + - "NULL, 'h', NULL, 'i', NULL, 'j', NULL))", - "(3, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 1, 'g', NULL, " + - "'h', NULL, 'i', NULL, 'j', NULL))", - "(4, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + - "1, 'i', NULL, 'j', NULL))", - "(5, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + - "0, 'i', 1, 'j', 0))", - "(5, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + - "0, 'i', 1, 'j', 0))", - "(6, named_struct('a', 0, 'b', 1, 'c', 0, 'd', 1, 'e', 1, 'f', 1, 'g', 0, 'h', " + - "1, 'i', 1, 'j', 0))", - "(7, named_struct('a', 1, 'b', 0, 'c', 1, 'd', 1, 'e', 1, 'f', 1, 'g', 0, 'h', " + - "1, 'i', 1, 'j', 0))", - "(8, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + - "0, 'i', 1, 'j', 1))", - "(9, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + - "0, 'i', 1, 'j', NULL))", - "(10, named_struct('a', NULL, 'b', 1, 'c', 0, 'd', 1, 'e', 1, 'f', 1, 'g', 0, " + - "'h', 1, 'i', 1, 'j', 0))", - "(11, named_struct('a', NULL, 'b', NULL, 'c', NULL, 'd', NULL, 'e', NULL, 'f', " + - "NULL, 'g', NULL, 'h', NULL, 'i', NULL, 'j', NULL))", - "(12, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 0, 'f', 1, 'g', 0, 'h', " + - "1, 'i', 1, 'j', 0))" - ) - for (batch <- rows.grouped(2)) { - sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") - } - val t = registerTable("tbl") - val table = WorkloadContext.current.getTableSpec(t) - // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. - val cases = Seq( - ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)), - ( - "deep_hit", - "s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + - "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))", - Seq(1, 2, 3, 4, 5) - ), - ( - "deep_false", - "NOT (s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND " + - "(s.f = 1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1))))))))))", - Seq(6, 7, 8, 12) - ), - ( - "deep_null", - "(s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + - "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))) IS " + - "NULL", - Seq(9, 10, 11) - ), - ( - "deep_miss", - "(s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + - "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))) AND " + - "id = 99", - Seq.empty[Int] - ) - ) - for ((label, predicate, ids) <- cases) { - val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) - readSpec(t, predicate = predicate, name = Some(label)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == bag.size) - case other => fail(s"${table.outputName}/$label must succeed: $other") - } - val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL)" - val projection = "CAST(id AS INT) AS id, CAST(s AS STRUCT) AS s" - val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, s)" + - (if (bag.nonEmpty) "" else " WHERE false") - val expectedPath = outputDir - .resolve(table.outputName) - .resolve("expected") - .resolve(s"${table.outputName}_$label") - .resolve("expected_data") - RowComparison.assertRowsEqual( - spark.sql(oracle), - spark.read.parquet(expectedPath.toString), - s"${table.outputName}/$label literal oracle" - ) - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] - require( - schema == StructType.fromDDL( - "id INT, s STRUCT" - ) - ) - require(result.metadata.partitionColumns.isEmpty) - require(result.metadata.configuration.isEmpty) - require(result.protocol.minReaderVersion == 1) - require(result.protocol.minWriterVersion == 2) - case other => fail(s"${table.outputName} snapshot must succeed: $other") - } - } - } -} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessSchemaReplacementSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessSchemaReplacementSuite.scala deleted file mode 100644 index fdfe4326..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessSchemaReplacementSuite.scala +++ /dev/null @@ -1,494 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.tables - -import scala.jdk.CollectionConverters._ - -import org.apache.spark.sql.{DataFrame, Row} -import org.apache.spark.sql.types.{ByteType, DataType, IntegerType, StringType, StructType} - -import io.delta.workload.WorkloadTestSuite -import io.delta.workload.engine.RowComparison -import io.delta.workload.model.Succeeded - -class ReadinessSchemaReplacementSuite extends WorkloadTestSuite("readiness_schema_replacement") { - - // Shared by the stats-column and partition-column variants of the same replacement scenario. - private def replaceSameName( - caseName: String, - partitioned: Boolean, - fromType: DataType, - toType: DataType, - seedValues: String, - replacementValues: String, - beforeRows: Seq[Row], - afterRows: Seq[Row], - matchingPredicate: String, - missingPredicate: String): Unit = { - val partitionClause = if (partitioned) "PARTITIONED BY (c)" else "" - val beforeSchema = new StructType().add("id", IntegerType) - .add("c", fromType).add("payload", StringType) - val afterSchema = new StructType().add("id", IntegerType) - .add("c", toType).add("payload", StringType) - - sql(s"CREATE TABLE tbl (${beforeSchema.toDDL}) USING delta $partitionClause") - sql(s"INSERT INTO tbl VALUES $seedValues") - // This is a REPLACE TABLE commit, rather than DROP/ADD COLUMN or an append with evolution. - sql(s"CREATE OR REPLACE TABLE tbl (${afterSchema.toDDL}) USING delta $partitionClause") - sql(s"INSERT INTO tbl VALUES $replacementValues") - val t = registerTable("tbl") - val table = io.delta.workload.WorkloadContext.current.getTableSpec(t) - - // Literal rows form an oracle independent of reading the generated Delta table. - def frame(rows: Seq[Row], schema: StructType): DataFrame = - spark.createDataFrame(rows.asJava, schema) - - def checkRead( - name: String, - rows: Seq[Row], - schema: StructType, - version: java.lang.Long = null, - predicate: String = null, - columns: Option[Seq[String]] = None): Unit = { - val oracle = frame(rows, schema) - val projectedOracle = columns.map(cs => oracle.selectExpr(cs: _*)).getOrElse(oracle) - readSpec(t, name = Some(name), version = version, predicate = predicate, - columns = columns).assert { spec => - val workloadName = table.outputName - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == rows.size.toLong) - case other => fail(s"$workloadName/$name must succeed: $other") - } - val expectedPath = outputDir.resolve(workloadName).resolve("expected") - .resolve(s"${workloadName}_$name").resolve("expected_data") - val captured = spark.read.parquet(expectedPath.toString) - assert(captured.schema == projectedOracle.schema) - RowComparison.assertRowsEqual( - projectedOracle, captured, s"$workloadName/$name literal oracle") - println(s"Independent row oracle passed: $workloadName/$name (${rows.size} rows)") - } - } - - checkRead("read_all", afterRows, afterSchema) - checkRead("read_before", beforeRows, beforeSchema, version = 1L) - checkRead("read_replaced_empty", Seq.empty, afterSchema, version = 2L) - checkRead("read_match", afterRows.take(2), afterSchema, predicate = matchingPredicate) - checkRead("read_missing", Seq.empty, afterSchema, predicate = missingPredicate) - checkRead("read_null", afterRows.slice(3, 4), afterSchema, predicate = "c IS NULL") - checkRead("read_non_null", afterRows.take(3) ++ afterRows.drop(4), afterSchema, - predicate = "c IS NOT NULL") - checkRead("read_projection", afterRows, afterSchema, columns = Some(Seq("c", "payload"))) - checkRead("read_before_projection", beforeRows, beforeSchema, version = 1L, - columns = Some(Seq("c", "payload"))) - checkRead("read_before_null", beforeRows.takeRight(1), beforeSchema, version = 1L, - predicate = "c IS NULL") - - for (v <- 0L to 3L) { - val expectedSchema = if (v < 2L) beforeSchema else afterSchema - snapshotSpec(t, version = v).assert { spec => - spec.expectation match { - case Succeeded(result) => - assert(DataType.fromJson(result.metadata.schemaString) == expectedSchema) - assert(result.metadata.partitionColumns == (if (partitioned) Seq("c") else Seq.empty)) - case other => fail(s"${table.outputName} snapshot v$v must succeed: $other") - } - } - } - } - - test("string_to_int_stats") { - replaceSameName( - "string_to_int_stats", partitioned = false, StringType, IntegerType, - seedValues = """(1, 'not-an-int', 'old'), (1, 'not-an-int', 'old'), """ + - """(2, '007', 'old-padded'), (3, NULL, 'old-null')""", - replacementValues = """(101, 7, 'new-seven'), (101, 7, 'new-seven'), """ + - """(102, -5, 'new-negative'), (103, NULL, 'new-null'), (104, 0, 'new-zero')""", - beforeRows = Seq( - Row(1, "not-an-int", "old"), - Row(1, "not-an-int", "old"), - Row(2, "007", "old-padded"), - Row(3, null, "old-null")), - afterRows = Seq( - Row(101, 7, "new-seven"), - Row(101, 7, "new-seven"), - Row(102, -5, "new-negative"), - Row(103, null, "new-null"), - Row(104, 0, "new-zero")), - matchingPredicate = "c = 7", missingPredicate = "c = 999") - } - - test("int_to_string_stats") { - replaceSameName( - "int_to_string_stats", partitioned = false, IntegerType, StringType, - seedValues = """(1, 7, 'old'), (1, 7, 'old'), (2, -5, 'old-negative'), """ + - """(3, NULL, 'old-null')""", - replacementValues = """(101, 'seven', 'new-word'), (101, 'seven', 'new-word'), """ + - """(102, '007', 'new-padded'), (103, NULL, 'new-null'), (104, '', 'new-empty')""", - beforeRows = Seq( - Row(1, 7, "old"), - Row(1, 7, "old"), - Row(2, -5, "old-negative"), - Row(3, null, "old-null")), - afterRows = Seq( - Row(101, "seven", "new-word"), - Row(101, "seven", "new-word"), - Row(102, "007", "new-padded"), - Row(103, null, "new-null"), - Row(104, "", "new-empty")), - matchingPredicate = "c = 'seven'", missingPredicate = "c = 'missing'") - } - - test("byte_to_string_stats") { - replaceSameName( - "byte_to_string_stats", partitioned = false, ByteType, StringType, - seedValues = """(1, CAST(-128 AS BYTE), 'old-min'), (1, CAST(-128 AS BYTE), 'old-min'), """ + - """(2, CAST(127 AS BYTE), 'old-max'), (3, CAST(NULL AS BYTE), 'old-null')""", - replacementValues = """(101, 'byte-min', 'new-word'), (101, 'byte-min', 'new-word'), """ + - """(102, '127', 'new-numeric'), (103, NULL, 'new-null'), (104, '', 'new-empty')""", - beforeRows = Seq( - Row(1, -128.toByte, "old-min"), - Row(1, -128.toByte, "old-min"), - Row(2, 127.toByte, "old-max"), - Row(3, null, "old-null")), - afterRows = Seq( - Row(101, "byte-min", "new-word"), - Row(101, "byte-min", "new-word"), - Row(102, "127", "new-numeric"), - Row(103, null, "new-null"), - Row(104, "", "new-empty")), - matchingPredicate = "c = 'byte-min'", missingPredicate = "c = 'missing'") - } - - test("string_to_int_partition") { - replaceSameName( - "string_to_int_partition", partitioned = true, StringType, IntegerType, - seedValues = """(1, 'not-an-int', 'old'), (1, 'not-an-int', 'old'), """ + - """(2, '007', 'old-padded'), (3, NULL, 'old-null')""", - replacementValues = """(101, 7, 'new-seven'), (101, 7, 'new-seven'), """ + - """(102, -5, 'new-negative'), (103, NULL, 'new-null'), (104, 0, 'new-zero')""", - beforeRows = Seq( - Row(1, "not-an-int", "old"), - Row(1, "not-an-int", "old"), - Row(2, "007", "old-padded"), - Row(3, null, "old-null")), - afterRows = Seq( - Row(101, 7, "new-seven"), - Row(101, 7, "new-seven"), - Row(102, -5, "new-negative"), - Row(103, null, "new-null"), - Row(104, 0, "new-zero")), - matchingPredicate = "c = 7", missingPredicate = "c = 999") - } - - test("int_to_string_partition") { - replaceSameName( - "int_to_string_partition", partitioned = true, IntegerType, StringType, - seedValues = """(1, 7, 'old'), (1, 7, 'old'), (2, -5, 'old-negative'), """ + - """(3, NULL, 'old-null')""", - replacementValues = """(101, 'seven', 'new-word'), (101, 'seven', 'new-word'), """ + - """(102, '007', 'new-padded'), (103, NULL, 'new-null'), (104, 'zero', 'new-zero')""", - beforeRows = Seq( - Row(1, 7, "old"), - Row(1, 7, "old"), - Row(2, -5, "old-negative"), - Row(3, null, "old-null")), - afterRows = Seq( - Row(101, "seven", "new-word"), - Row(101, "seven", "new-word"), - Row(102, "007", "new-padded"), - Row(103, null, "new-null"), - Row(104, "zero", "new-zero")), - matchingPredicate = "c = 'seven'", missingPredicate = "c = 'missing'") - } - - test("byte_to_string_partition") { - replaceSameName( - "byte_to_string_partition", partitioned = true, ByteType, StringType, - seedValues = """(1, CAST(-128 AS BYTE), 'old-min'), (1, CAST(-128 AS BYTE), 'old-min'), """ + - """(2, CAST(127 AS BYTE), 'old-max'), (3, CAST(NULL AS BYTE), 'old-null')""", - replacementValues = """(101, 'byte-min', 'new-word'), (101, 'byte-min', 'new-word'), """ + - """(102, '127', 'new-numeric'), (103, NULL, 'new-null'), (104, 'zero', 'new-zero')""", - beforeRows = Seq( - Row(1, -128.toByte, "old-min"), - Row(1, -128.toByte, "old-min"), - Row(2, 127.toByte, "old-max"), - Row(3, null, "old-null")), - afterRows = Seq( - Row(101, "byte-min", "new-word"), - Row(101, "byte-min", "new-word"), - Row(102, "127", "new-numeric"), - Row(103, null, "new-null"), - Row(104, "zero", "new-zero")), - matchingPredicate = "c = 'byte-min'", missingPredicate = "c = 'missing'") - } - - // Changed partition/file sets and fresh mapped identities share the same replacement oracle. - import java.nio.file.Files - - import org.apache.spark.sql.types.{DoubleType, Metadata} - - import io.delta.workload.TableHandle - import io.delta.workload.json.JsonUtil - import io.delta.workload.log.CommitLog - - private def replacementRead( - t: TableHandle, - name: String, - rows: Seq[Row], - schema: StructType, - version: java.lang.Long = null, - predicate: String = null, - columns: Option[Seq[String]] = None): Unit = { - val table = io.delta.workload.WorkloadContext.current.getTableSpec(t) - val ref = readSpec(t, name = Some(name), version = version, predicate = predicate, - columns = columns) - ref.assert { spec => - val workloadName = table.outputName - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == rows.size.toLong) - case other => fail(s"$workloadName/$name must succeed: $other") - } - val literal = spark.createDataFrame(rows.asJava, schema) - val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) - val path = outputDir.resolve(workloadName).resolve("expected") - .resolve(s"${workloadName}_${name}").resolve("expected_data") - val captured = spark.read.parquet(path.toString) - // RowComparison checks exact names/types and duplicate-sensitive row bags. - RowComparison.assertRowsEqual(oracle, captured, s"$workloadName/$name literal oracle") - assert(captured.schema.fields.forall(_.nullable)) - println(s"Independent row oracle passed: $workloadName/$name (${rows.size} rows)") - } - } - - private def replacementLayout( - caseName: String, - layout: String, - mappingMode: Option[String] = None): Unit = { - val mapped = mappingMode.isDefined - val parsed = layout == "checkpoint_parsed" - val checkpointed = layout != "commits" - val beforeNames = if (mapped) Seq("id", "label", "score", "region") - else Seq("id", "old_label", "old_score", "old_region") - val afterNames = Seq("id", "label", "score", "region") - def logicalSchema(names: Seq[String]): StructType = new StructType() - .add(names(0), IntegerType).add(names(1), StringType) - .add(names(2), DoubleType).add(names(3), StringType) - val beforeSchema = logicalSchema(beforeNames) - val afterSchema = logicalSchema(afterNames) - val beforeRows = Seq( - Row(1, "old", 1000.0, "old-east"), Row(1, "old", 1000.0, "old-east"), - Row(2, "old-negative", -1000.0, "old-west"), Row(3, null, null, null)) - val afterRows = Seq( - Row(101, "new-a", 7.5, "east"), Row(101, "new-a", 7.5, "east"), - Row(102, "new-b", -2.5, "west"), Row(103, null, null, null), - Row(104, "new-zero", 0.0, "east")) - val properties = Map( - "delta.checkpointInterval" -> "1000", - "delta.checkpoint.writeStatsAsJson" -> (!parsed).toString, - "delta.checkpoint.writeStatsAsStruct" -> parsed.toString) ++ - mappingMode.map("delta.columnMapping.mode" -> _).toMap - val propertySql = properties.toSeq.sortBy(_._1) - .map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") - sql(s"""CREATE TABLE tbl (${beforeSchema.toDDL}) USING delta - PARTITIONED BY (${beforeNames.last}) TBLPROPERTIES ($propertySql)""") - sql("""INSERT INTO tbl VALUES - (1, 'old', 1000D, 'old-east'), (1, 'old', 1000D, 'old-east'), - (2, 'old-negative', -1000D, 'old-west'), (3, NULL, NULL, NULL)""") - - // The setting exists in OSS Delta 4.1.0 as well as Runtime. Restore after the DDL; - // it controls authoring only, and is not needed while queued read specs are captured. - val reuseKey = "spark.databricks.delta.columnMapping.reuseColumnMetadataDuringReplace" - val oldReuse = spark.conf.getOption(reuseKey) - try { - if (mapped) spark.conf.set(reuseKey, "false") - sql(s"""CREATE OR REPLACE TABLE tbl (${afterSchema.toDDL}) USING delta - PARTITIONED BY (region) TBLPROPERTIES ($propertySql)""") - } finally { - if (mapped) oldReuse match { - case Some(value) => spark.conf.set(reuseKey, value) - case None => spark.conf.unset(reuseKey) - } - } - sql("""INSERT INTO tbl VALUES - (101, 'new-a', 7.5D, 'east'), (101, 'new-a', 7.5D, 'east'), - (102, 'new-b', -2.5D, 'west'), (103, NULL, NULL, NULL), - (104, 'new-zero', 0D, 'east')""") - if (checkpointed) forceCheckpoint("tbl") - val t = registerTable("tbl") - val rawBefore = CommitLog.readRaw(t.sourcePath, 0L).map(JsonUtil.mapper.readTree) - .find(_.has("metaData")).get.get("metaData") - val rawAfter = CommitLog.readRaw(t.sourcePath, 2L).map(JsonUtil.mapper.readTree) - .find(_.has("metaData")).get.get("metaData") - val mappedBefore = DataType.fromJson(rawBefore.get("schemaString").asText) - .asInstanceOf[StructType] - val mappedAfter = DataType.fromJson(rawAfter.get("schemaString").asText) - .asInstanceOf[StructType] - def withoutMapping(schema: StructType): StructType = - StructType(schema.fields.map(_.copy(metadata = Metadata.empty))) - assert(withoutMapping(mappedBefore) == beforeSchema) - assert(withoutMapping(mappedAfter) == afterSchema) - if (mapped) { - val idKey = "delta.columnMapping.id" - val physicalKey = "delta.columnMapping.physicalName" - assert(mappedBefore.fields.map(_.metadata.getLong(idKey)).toSeq == Seq(1L, 2L, 3L, 4L)) - assert(mappedAfter.fields.map(_.metadata.getLong(idKey)).toSeq == Seq(5L, 6L, 7L, 8L)) - val oldPhysical = mappedBefore.fields.map(_.metadata.getString(physicalKey)).toSet - val newPhysical = mappedAfter.fields.map(_.metadata.getString(physicalKey)).toSet - assert(oldPhysical.size == 4 && newPhysical.size == 4) - assert(oldPhysical.intersect(newPhysical).isEmpty) - assert(newPhysical.intersect(afterNames.toSet).isEmpty) - } - val physicalNames = mappedAfter.fields.map { field => - if (mapped) field.metadata.getString("delta.columnMapping.physicalName") else field.name - }.toSeq - val statsNames = physicalNames.init.toSet - val partitionName = physicalNames.last - // Inspect new commit actions directly too, including the commit-only mapping variants. - val newAdds = CommitLog.readRaw(t.sourcePath, 3L).map(JsonUtil.mapper.readTree) - .filter(_.has("add")).map(_.get("add")) - assert(newAdds.nonEmpty) - for (add <- newAdds) { - assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) - val stats = JsonUtil.mapper.readTree(add.get("stats").asText) - assert(stats.get("nullCount").fieldNames().asScala.toSet == statsNames) - for (kind <- Seq("minValues", "maxValues")) { - assert(stats.get(kind).fieldNames().asScala.toSet.subsetOf(statsNames)) - } - } - - mutateTable(t) { tableDir => - val logDir = tableDir.resolve("_delta_log") - // Retain exact pre-truncation actions in command evidence, including operation/removes. - for (v <- 0L to 3L; line <- CommitLog.readRaw(tableDir, v)) { - println(s"Replacement source action: $caseName v$v $line") - } - val stream = Files.list(logDir) - val files = try stream.iterator().asScala.toSeq finally stream.close() - if (checkpointed) { - val checkpoints = files.filter(_.getFileName.toString.endsWith(".checkpoint.parquet")) - assert(checkpoints.size == 1) - assert(checkpoints.head.getFileName.toString == "00000000000000000003.checkpoint.parquet") - val checkpoint = spark.read.parquet(checkpoints.head.toString) - val addSchema = checkpoint.schema("add").dataType.asInstanceOf[StructType] - assert(addSchema.fieldNames.contains("stats") == !parsed) - assert(addSchema.fieldNames.contains("stats_parsed") == parsed) - assert(addSchema.fieldNames.contains("partitionValues_parsed") == parsed) - if (parsed) { - val partitions = addSchema("partitionValues_parsed").dataType.asInstanceOf[StructType] - assert(partitions.fieldNames.toSeq == Seq(partitionName)) - assert(partitions.fields.head.dataType == StringType) - val stats = addSchema("stats_parsed").dataType.asInstanceOf[StructType] - for (kind <- Seq("minValues", "maxValues", "nullCount")) { - assert(stats(kind).dataType.asInstanceOf[StructType].fieldNames.toSet == statsNames) - } - } - val adds = checkpoint.filter("add IS NOT NULL").select("add").toJSON.collect() - assert(adds.nonEmpty) - for (json <- adds) { - val add = JsonUtil.mapper.readTree(json).get("add") - assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) - val stats = if (parsed) add.get("stats_parsed") - else JsonUtil.mapper.readTree(add.get("stats").asText) - assert(stats.get("nullCount").fieldNames().asScala.toSet == statsNames) - for (kind <- Seq("minValues", "maxValues")) { - assert(stats.get(kind).fieldNames().asScala.toSet.subsetOf(statsNames)) - } - } - // Keep only the checkpoint-era JSON for version discovery. Earlier metadata and file - // state must come from the checkpoint; remove protocol CRC shortcuts as well. - files.filter { path => - val name = path.getFileName.toString - name.endsWith(".crc") || - (name.matches("[0-9]{20}\\.json") && name != "00000000000000000003.json") - }.foreach(Files.delete) - } else { - assert(!files.exists(_.getFileName.toString.contains("checkpoint"))) - files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) - } - println(s"Replacement layout oracle passed: $caseName ($layout)") - } - - replacementRead(t, "read_all", afterRows, afterSchema) - replacementRead(t, "read_projection", afterRows, afterSchema, - columns = Some(Seq("region", "score", "label"))) - replacementRead(t, "read_partition", afterRows.take(2) ++ afterRows.drop(4), - afterSchema, predicate = "region = 'east'") - replacementRead(t, "read_stats", afterRows.take(2), afterSchema, - predicate = "score > 7D") - replacementRead(t, "read_missing", Seq.empty, afterSchema, - predicate = "region = 'missing' OR score > 999D") - replacementRead(t, "read_null", afterRows.slice(3, 4), afterSchema, - predicate = "region IS NULL AND score IS NULL AND label IS NULL") - replacementRead(t, "read_old_stats", Seq.empty, afterSchema, - predicate = "score = 1000D OR label = 'old'") - if (!checkpointed) { - replacementRead(t, "read_before", beforeRows, beforeSchema, version = 1L) - replacementRead(t, "read_replaced_empty", Seq.empty, afterSchema, version = 2L) - } - for (v <- (if (checkpointed) Seq(3L) else Seq(1L, 2L, 3L))) { - val metadata = if (v < 2L) rawBefore else rawAfter - val expectedProperties = properties ++ mappingMode.map { mode => - Map("delta.columnMapping.mode" -> mode, - "delta.columnMapping.maxColumnId" -> (if (v < 2L) "4" else "8")) - }.getOrElse(Map.empty[String, String]) - snapshotSpec(t, version = v).assert { spec => - spec.expectation match { - case Succeeded(result) => - assert(result.metadata.schemaString == metadata.get("schemaString").asText) - assert(result.metadata.partitionColumns == - Seq(if (v < 2L) beforeNames.last else "region")) - assert(result.metadata.configuration == expectedProperties) - case other => fail(s"$caseName snapshot v$v must succeed: $other") - } - println(s"Independent metadata oracle passed: $caseName v$v") - } - } - } - - test("changed_sets_checkpoint_json") { - replacementLayout("changed_sets_checkpoint_json", "checkpoint_json") - } - - test("changed_sets_checkpoint_parsed") { - replacementLayout("changed_sets_checkpoint_parsed", "checkpoint_parsed") - } - - test("mapped_name_commits") { - replacementLayout("mapped_name_commits", "commits", Some("name")) - } - - test("mapped_name_checkpoint_json") { - replacementLayout("mapped_name_checkpoint_json", "checkpoint_json", Some("name")) - } - - test("mapped_name_checkpoint_parsed") { - replacementLayout("mapped_name_checkpoint_parsed", "checkpoint_parsed", Some("name")) - } - - test("mapped_id_commits") { - replacementLayout("mapped_id_commits", "commits", Some("id")) - } - - test("mapped_id_checkpoint_json") { - replacementLayout("mapped_id_checkpoint_json", "checkpoint_json", Some("id")) - } - - test("mapped_id_checkpoint_parsed") { - replacementLayout("mapped_id_checkpoint_parsed", "checkpoint_parsed", Some("id")) - } - -} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessStatisticsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessStatisticsSuite.scala deleted file mode 100644 index cd7200d1..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessStatisticsSuite.scala +++ /dev/null @@ -1,1011 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.tables - -import java.nio.file.{Files, Path} -import java.sql.Timestamp -import java.time.Instant - -import scala.collection.JavaConverters._ - -import com.fasterxml.jackson.databind.node.ObjectNode -import org.apache.spark.sql.Row -import org.apache.spark.sql.functions.lit -import org.apache.spark.sql.types.{ArrayType, BooleanType, DataType, DecimalType, IntegerType} -import org.apache.spark.sql.types.{StringType, StructType, TimestampType} -import org.apache.spark.unsafe.types.UTF8String - -import io.delta.workload.WorkloadTestSuite -import io.delta.workload.engine.RowComparison -import io.delta.workload.json.JsonUtil -import io.delta.workload.log.{AddFile, CommitLog, Metadata, RemoveFile} -import io.delta.workload.model.Succeeded - -/** Portable read specifications for incomplete or selected file statistics. */ -class ReadinessStatisticsSuite extends WorkloadTestSuite("readiness_statistics") { - test("missing_add_stats_predicates") { - val schema = new StructType().add("id", IntegerType).add("value", IntegerType) - .add("flag", BooleanType).add("items", ArrayType(IntegerType, containsNull = true)) - val nullArray = Row(1, 1, true, null) - val emptyArray = Row(2, 2, false, Seq.empty[Integer]) - val nullElement = Row(3, null, null, Seq[Integer](null)) - val duplicate = Row(4, 3, true, Seq[Integer](1, 7)) - val nonmatching = Row(5, 4, false, Seq[Integer](2, 3)) - val matchingWithNull = Row(6, 5, null, Seq[Integer](7, null)) - val nonmatchingWithNull = Row(7, 2, true, Seq[Integer](null, 2)) - val matching = Row(8, null, false, Seq[Integer](7)) - val batches = Seq(Seq(nullArray, emptyArray, nullElement, duplicate, duplicate), - Seq(nonmatching, matchingWithNull, nonmatchingWithNull, matching)) - val rows = batches.flatten - val properties = Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "100") - sql("""CREATE TABLE tbl (id INT, value INT, flag BOOLEAN, items ARRAY) USING delta - TBLPROPERTIES ('delta.enableDeletionVectors' = 'false', - 'delta.checkpointInterval' = '100')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (1, 1, TRUE, CAST(NULL AS ARRAY)), (2, 2, FALSE, CAST(array() AS ARRAY)), - (3, NULL, NULL, array(CAST(NULL AS INT))), (4, 3, TRUE, array(1, 7)), - (4, 3, TRUE, array(1, 7)) AS v(id, value, flag, items)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (5, 4, FALSE, array(2, 3)), (6, 5, NULL, array(7, CAST(NULL AS INT))), - (7, 2, TRUE, array(CAST(NULL AS INT), 2)), (8, NULL, FALSE, array(7)) - AS v(id, value, flag, items)""") - val t = registerTable("tbl") - var copiedTable: Path = null - mutateTable(t) { dir => - copiedTable = dir - val metadata = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head - require(DataType.fromJson(metadata.schemaString) == schema) - require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) - batches.zipWithIndex.foreach { case (batch, index) => - val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } - require(adds.size == 1 && adds.head.stats.isDefined) - val add = adds.head - RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), - spark.read.parquet(dir.resolve(add.path).toString), s"predicate_input_$index") - val stats = JsonUtil.mapper.readTree(add.stats.get) - require(stats.path("numRecords").asLong(-1) == batch.size) - // Arrays have no scalar min/max statistics; their predicates require residual filtering. - require(!stats.path("minValues").has("items") && !stats.path("maxValues").has("items")) - } - val stream = Files.list(dir.resolve("_delta_log")) - val names = try stream.iterator().asScala.map(_.getFileName.toString).toVector - finally stream.close() - require(!names.exists(_.contains("checkpoint"))) - // A cached allFiles set in a version CRC must not hide the missing AddFile statistics. - names.filter(_.matches("[0-9]{20}\\.crc")) - .foreach(name => Files.delete(dir.resolve("_delta_log").resolve(name))) - } - for (version <- 1L to 2L) { - modifyCommitActions(t, version) { actions => - actions.map { - case a: AddFile => a.copy(stats = None) - case other => other - } - } - } - mutateTable(t) { dir => - val actions = (0L to 2L).flatMap(v => CommitLog.read(dir, v)) - val adds = actions.collect { case a: AddFile => a } - require(adds.size == 2 && adds.map(_.path).distinct.size == 2) - require(adds.forall(_.stats.isEmpty), "Every persisted AddFile must omit statistics") - require(!actions.exists(_.isInstanceOf[RemoveFile])) - } - - // Expected bags come from literal values and SQL three-valued logic, without a Delta scan. - val casePredicate = "CASE WHEN value IN (1, 3) THEN TRUE " + - "WHEN value = 2 THEN FALSE ELSE CAST(NULL AS BOOLEAN) END" - val queries = Seq[(String, String, Seq[Row])]( - ("all", null, rows), - ("ne", "value <> 2", Seq(nullArray, duplicate, duplicate, nonmatching, matchingWithNull)), - ("in", "value IN (1, 3, 99)", Seq(nullArray, duplicate, duplicate)), - ("in_with_null", "value IN (2, NULL)", Seq(emptyArray, nonmatchingWithNull)), - ("in_miss", "value IN (99, NULL)", Seq.empty), - ("is_not_true", "flag IS NOT TRUE", - Seq(emptyArray, nullElement, nonmatching, matchingWithNull, matching)), - ("array_contains", "array_contains(items, 7)", - Seq(duplicate, duplicate, matchingWithNull, matching)), - ("array_contains_miss", "array_contains(items, 99)", Seq.empty), - ("array_contains_is_not_true", "array_contains(items, 7) IS NOT TRUE", - Seq(nullArray, emptyArray, nullElement, nonmatching, nonmatchingWithNull)), - ("array_contains_not", "NOT array_contains(items, 7)", Seq(emptyArray, nonmatching)), - ("case_when", casePredicate, Seq(nullArray, duplicate, duplicate)), - ("case_when_is_not_true", s"($casePredicate) IS NOT TRUE", - Seq(emptyArray, nullElement, nonmatching, matchingWithNull, nonmatchingWithNull, matching))) - queries.foreach { case (name, predicate, expectedRows) => - readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == expectedRows.size) - case other => fail(s"Expected successful missing-stat $name read, got $other") - } - val outputName = copiedTable.getParent.getFileName.toString - val expectedPath = outputDir.resolve(outputName).resolve("expected") - .resolve(s"${outputName}_read_$name").resolve("expected_data") - RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), - spark.read.parquet(expectedPath.toString), s"independent_missing_stats_oracle_$name") - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == schema) - require(result.metadata.configuration == properties) - require(result.metadata.partitionColumns.isEmpty) - case other => fail(s"Expected successful missing-stat snapshot, got $other") - } - } - } - - for (variant <- Seq("no_null_count", "no_num_records", "neither_count", "partial_null_count")) { - test(s"partial_counts_$variant") { - val schema = new StructType().add("id", IntegerType).add("value", IntegerType) - val batches = Seq( - Seq(Row(1, 10), Row(2, null), Row(3, 30), Row(3, 30)), - Seq(Row(4, 40), Row(5, 50)), - Seq(Row(6, null), Row(7, null)), - Seq(Row(8, 80), Row(9, null))) - val rows = batches.flatten - val properties = Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "100") - sql("""CREATE TABLE tbl (id INT, value INT) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100')""") - batches.foreach { batch => - // One mixed/all-null/non-null file per commit; duplicate rows must survive reads. - val values = batch.map { r => - s"(${r.getInt(0)}, ${if (r.isNullAt(1)) "NULL" else r.getInt(1).toString})" - }.mkString(", ") - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS v(id, value)") - } - val t = registerTable("tbl") - var copiedTable: Path = null - - // Check real AddFiles and their row-derived stats, then mutate only the copied table. - mutateTable(t) { dir => - copiedTable = dir - val initial = CommitLog.read(dir, 0) - val originalMetadata = initial.collect { case m: Metadata => m }.head - require(DataType.fromJson(originalMetadata.schemaString) == schema) - require(originalMetadata.configuration == properties) - require(originalMetadata.partitionColumns.isEmpty) - val logFiles = Files.list(dir.resolve("_delta_log")) - try { - val names = logFiles.iterator().asScala.map(_.getFileName.toString).toSeq - require(!names.exists(_.contains("checkpoint")), "Commit replay must be required") - // Version CRCs can contain a cached allFiles set with the original stats. - names.filter(_.matches("[0-9]{20}\\.crc")).foreach { name => - Files.delete(dir.resolve("_delta_log").resolve(name)) - } - } finally logFiles.close() - - batches.zipWithIndex.foreach { case (batch, index) => - val version = index.toLong + 1 - val actions = CommitLog.read(dir, version) - val adds = actions.collect { case a: AddFile => a } - require(adds.size == 1, s"Expected one AddFile at version $version") - val add = adds.head - require(add.deletionVector.isEmpty && add.baseRowId.isEmpty && - add.defaultRowCommitVersion.isEmpty) - val oracle = spark.createDataFrame(batch.asJava, schema) - RowComparison.assertRowsEqual(oracle, - spark.read.parquet(dir.resolve(add.path).toString), s"input_v$version") - val stats = JsonUtil.mapper.readTree(add.stats.get).asInstanceOf[ObjectNode] - require(stats.path("numRecords").asLong(-1) == batch.size) - for ((column, ordinal) <- Seq("id" -> 0, "value" -> 1)) { - val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) - require(stats.path("nullCount").path(column).asLong(-1) == - batch.count(_.isNullAt(ordinal))) - if (values.nonEmpty) { - require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) - require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) - } - } - val before = stats.deepCopy() - // Keep the last file intact as a control for per-file partial statistics. - if (version < 4) { - variant match { - case "no_null_count" => stats.remove("nullCount") - case "no_num_records" => stats.remove("numRecords") - case "neither_count" => - stats.remove("nullCount") - stats.remove("numRecords") - case "partial_null_count" => - if (version == 3) stats.remove("nullCount") - else stats.get("nullCount").asInstanceOf[ObjectNode] - .remove(if (version == 1) "value" else "id") - } - require(stats != before, s"Mutation must change stats at version $version") - } else require(stats == before) - require(stats.get("minValues") == before.get("minValues")) - require(stats.get("maxValues") == before.get("maxValues")) - val changed = add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) - CommitLog.write(dir, version, actions.map { - case _: AddFile => changed - case other => other - }) - val reread = CommitLog.read(dir, version).collect { case a: AddFile => a } - require(reread == Seq(changed), s"Persisted mutation mismatch at version $version") - require(JsonUtil.mapper.readTree(reread.head.stats.get) == stats) - } - } - - // These bags are derived directly from the literal input, without a Delta scan or stats. - val queries = Seq[(String, String, Seq[Row])]( - ("all", null, rows), - ("value_is_null", "value IS NULL", rows.filter(_.isNullAt(1))), - ("value_is_not_null", "value IS NOT NULL", rows.filterNot(_.isNullAt(1))), - ("value_eq_30", "value = 30", Seq(Row(3, 30), Row(3, 30))), - ("value_gt_35", "value > 35", Seq(Row(4, 40), Row(5, 50), Row(8, 80))), - ("value_eq_999", "value = 999", Seq.empty), - ("id_is_null", "id IS NULL", Seq.empty), - ("id_is_not_null", "id IS NOT NULL", rows)) - queries.foreach { case (name, predicate, expectedRows) => - readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == expectedRows.size) - case other => fail(s"Expected successful $name read, got $other") - } - val outputName = copiedTable.getParent.getFileName.toString - val expectedPath = outputDir.resolve(outputName).resolve("expected") - .resolve(s"${outputName}_read_$name") - .resolve("expected_data") - RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), - spark.read.parquet(expectedPath.toString), s"independent_oracle_$name") - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == schema) - require(result.metadata.configuration == properties) - require(result.metadata.partitionColumns.isEmpty) - case other => fail(s"Expected successful snapshot, got $other") - } - } - } - } - - test("remove_stats_without_active_add_stats") { - val schema = new StructType().add("id", IntegerType).add("value", IntegerType) - val batches = Seq( - Seq(Row(1, 10), Row(5, null)), - Seq(Row(2, 20), Row(3, 30), Row(3, 30), Row(4, null)), - Seq(Row(100, 1000), Row(101, 1010))) - val rewrittenRows = Seq(Row(3, 30), Row(3, 30), Row(4, null)) - val survivors = batches.head ++ rewrittenRows - val properties = Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "100") - sql("""CREATE TABLE tbl (id INT, value INT) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100')""") - batches.foreach { batch => - val values = batch.map { r => - s"(${r.getInt(0)}, ${if (r.isNullAt(1)) "NULL" else r.getInt(1).toString})" - }.mkString(", ") - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS v(id, value)") - } - // Rewrite one file, remove another entirely, and leave the first file untouched. - sql("DELETE FROM tbl WHERE id = 2 OR id >= 100") - val t = registerTable("tbl") - var copiedTable: Path = null - - mutateTable(t) { dir => - copiedTable = dir - val commits = (0L to 4L).map(v => v -> CommitLog.read(dir, v)).toMap - val originalMetadata = commits(0).collect { case m: Metadata => m }.head - require(DataType.fromJson(originalMetadata.schemaString) == schema) - require(originalMetadata.configuration == properties) - require(originalMetadata.partitionColumns.isEmpty) - require(commits(0).collect { case a: AddFile => a }.isEmpty) - val logFiles = Files.list(dir.resolve("_delta_log")) - try { - val names = logFiles.iterator().asScala.map(_.getFileName.toString).toSeq - require(!names.exists(_.contains("checkpoint")), "Commit replay must be required") - // Prevent a version CRC's cached allFiles from hiding the stats mutation. - names.filter(_.matches("[0-9]{20}\\.crc")).foreach { name => - Files.delete(dir.resolve("_delta_log").resolve(name)) - } - } finally logFiles.close() - - val inputAdds = batches.zipWithIndex.map { case (batch, index) => - val adds = commits(index.toLong + 1).collect { case a: AddFile => a } - require(adds.size == 1, s"Expected one input AddFile for batch $index") - require(!commits(index.toLong + 1).exists(_.isInstanceOf[RemoveFile])) - val add = adds.head - RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), - spark.read.parquet(dir.resolve(add.path).toString), s"original_batch_$index") - val stats = JsonUtil.mapper.readTree(add.stats.get) - require(stats.path("numRecords").asLong(-1) == batch.size) - for ((column, ordinal) <- Seq("id" -> 0, "value" -> 1)) { - val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) - require(stats.path("nullCount").path(column).asLong(-1) == - batch.count(_.isNullAt(ordinal))) - require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) - require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) - } - add - } - val allAdds = commits.values.toSeq.flatten.collect { case a: AddFile => a } - val addsByPath = allAdds.map(a => a.path -> a).toMap - require(addsByPath.size == allAdds.size, "Each path must have one original Add") - require(allAdds.forall(a => a.stats.isDefined && a.partitionValues.isEmpty && - a.deletionVector.isEmpty && a.baseRowId.isEmpty && a.defaultRowCommitVersion.isEmpty)) - val removes = commits(4).collect { case r: RemoveFile => r } - val removedPaths = removes.map(_.path).toSet - require(removes.size == 2 && removedPaths == inputAdds.tail.map(_.path).toSet) - removes.foreach { r => - val original = addsByPath(r.path) - require(r.dataChange && r.deletionTimestamp.isDefined) - require(r.extendedFileMetadata.contains(true)) - require(r.partitionValues.contains(original.partitionValues)) - require(r.size.contains(original.size) && r.deletionVector.isEmpty) - require(r.baseRowId.isEmpty && r.defaultRowCommitVersion.isEmpty) - require(r.stats.forall(s => JsonUtil.mapper.readTree(s) == - JsonUtil.mapper.readTree(original.stats.get))) - } - val rewritten = commits(4).collect { case a: AddFile => a } - require(rewritten.nonEmpty) - RowComparison.assertRowsEqual(spark.createDataFrame(rewrittenRows.asJava, schema), - rewritten.map(a => spark.read.parquet(dir.resolve(a.path).toString)).reduce(_.union(_)), - "rewritten_survivors") - val activePaths = addsByPath.keySet -- removedPaths - require(activePaths == (rewritten.map(_.path).toSet + inputAdds.head.path)) - - for (version <- 1L to 4L) { - val changed = commits(version).map { - case a: AddFile if activePaths.contains(a.path) => a.copy(stats = None) - // Some writers omit Remove stats. Supply the exact, row-checked original Add stats. - case r: RemoveFile => r.copy(stats = addsByPath(r.path).stats) - case other => other - } - CommitLog.write(dir, version, changed) - require(CommitLog.read(dir, version) == changed, - s"Persisted action mutation mismatch at version $version") - } - val persisted = (0L to 4L).flatMap(v => CommitLog.read(dir, v)) - val persistedAdds = persisted.collect { case a: AddFile => a } - val persistedRemoves = persisted.collect { case r: RemoveFile => r } - require(persistedAdds.filter(a => activePaths.contains(a.path)).forall(_.stats.isEmpty)) - require(persistedAdds.filter(a => removedPaths.contains(a.path)).forall { a => - a == addsByPath(a.path) - }) - require(persistedRemoves.size == 2 && persistedRemoves.forall { r => - r.stats.isDefined && r.stats == addsByPath(r.path).stats - }) - } - - // Literal row bags, independent of Delta replay and statistics; duplicates are intentional. - val queries = Seq[(String, String, Seq[Row])]( - ("all", null, survivors), - ("hit_untouched", "id = 1", Seq(Row(1, 10))), - ("hit_rewritten", "id = 3", Seq(Row(3, 30), Row(3, 30))), - ("range", "value >= 30", Seq(Row(3, 30), Row(3, 30))), - ("null", "value IS NULL", Seq(Row(5, null), Row(4, null))), - ("miss", "id = 999", Seq.empty), - ("deleted_partial", "id = 2", Seq.empty), - ("deleted_file", "id >= 100", Seq.empty), - ("mixed", "id IN (2, 3, 100)", Seq(Row(3, 30), Row(3, 30)))) - queries.foreach { case (name, predicate, expectedRows) => - readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == expectedRows.size) - case other => fail(s"Expected successful $name read, got $other") - } - val outputName = copiedTable.getParent.getFileName.toString - val expectedPath = outputDir.resolve(outputName).resolve("expected") - .resolve(s"${outputName}_read_$name") - .resolve("expected_data") - RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), - spark.read.parquet(expectedPath.toString), s"independent_oracle_$name") - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == schema) - require(result.metadata.configuration == properties) - require(result.metadata.partitionColumns.isEmpty) - case other => fail(s"Expected successful snapshot, got $other") - } - } - } - - test("unicode_string_stats_predicates") { - val schema = new StructType().add("id", IntegerType).add("s", StringType) - val head = "a" * 31 - // Exact source rows from WriteStatsSuite.utf8_boundary_string_min_max; no NULL rows. - // This independent read fixture avoids depending on another suite's generated output. - val rows = Seq(Row(1, head + "étail"), Row(2, head + "中tail"), Row(3, head + "😀")) - sql("CREATE TABLE tbl (id INT, s STRING) USING delta") - rows.foreach { row => - // Separate files give the three Unicode boundaries distinct min/max statistics. - sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (${row.getInt(0)}, '${row.getString(1)}') AS v(id, s)""") - } - val t = registerTable("tbl") - var copiedTable: Path = null - mutateTable(t) { dir => - copiedTable = dir - rows.zipWithIndex.foreach { case (row, index) => - val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } - require(adds.size == 1) - val add = adds.head - RowComparison.assertRowsEqual(spark.createDataFrame(Seq(row).asJava, schema), - spark.read.parquet(dir.resolve(add.path).toString), s"unicode_input_$index") - val stats = JsonUtil.mapper.readTree(add.stats.get) - require(stats.path("numRecords").asLong(-1) == 1) - require(stats.path("nullCount").path("s").asLong(-1) == 0) - val value = row.getString(1) - val minimum = stats.path("minValues").path("s").asText() - val maximum = stats.path("maxValues").path("s").asText() - require(minimum.startsWith(head) && minimum.codePointAt(31) == value.codePointAt(31)) - require(UTF8String.fromString(minimum).compareTo(UTF8String.fromString(value)) <= 0) - require(UTF8String.fromString(value).compareTo(UTF8String.fromString(maximum)) <= 0) - } - } - - // Literal bags use UTF-8 binary ordering: e-acute < CJK < supplementary emoji. - val queries = Seq[(String, String, Seq[Row])]( - ("all", null, rows), - ("equal_2byte", s"s = '${head}étail'", Seq(rows(0))), - ("equal_3byte", s"s = '${head}中tail'", Seq(rows(1))), - ("equal_4byte", s"s = '${head}😀'", Seq(rows(2))), - ("range_2_to_3byte", s"s >= '${head}étail' AND s < '${head}中tail'", Seq(rows(0))), - ("range_3_to_4byte", s"s >= '${head}中tail' AND s < '${head}😀'", Seq(rows(1))), - ("range_from_4byte", s"s >= '${head}😀'", Seq(rows(2))), - ("range_through_4byte", s"s > '${head}étail' AND s <= '${head}😀'", rows.tail), - ("miss_same_prefix", s"s = '${head}étail_missing'", Seq.empty), - ("miss_above_4byte", s"s > '${head}😀'", Seq.empty), - ("is_null", "s IS NULL", Seq.empty), - ("is_not_null", "s IS NOT NULL", rows)) - queries.foreach { case (name, predicate, expectedRows) => - readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == expectedRows.size) - case other => fail(s"Expected successful Unicode $name read, got $other") - } - val outputName = copiedTable.getParent.getFileName.toString - val expectedPath = outputDir.resolve(outputName).resolve("expected") - .resolve(s"${outputName}_read_$name").resolve("expected_data") - RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), - spark.read.parquet(expectedPath.toString), s"independent_unicode_oracle_$name") - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == schema) - require(result.metadata.partitionColumns.isEmpty) - require(result.metadata.configuration.isEmpty) - case other => fail(s"Expected successful Unicode snapshot, got $other") - } - } - } - - for (bounds <- Seq("true", "false", "absent")) { - test(s"tight_bounds_$bounds") { - val schema = new StructType().add("id", IntegerType).add("all_null", IntegerType) - .add("mixed_null", IntegerType).add("non_null", IntegerType) - val physical = Seq(Row(1, null, 10, 10), Row(2, null, null, 20), - Row(3, null, 30, 30), Row(3, null, 30, 30), Row(4, null, null, 40)) - val live = Seq(Row(2, null, null, 20), Row(3, null, 30, 30), Row(3, null, 30, 30)) - val control = physical.map(r => Row(r.getInt(0) + 100, r.get(1), r.get(2), r.get(3))) - // Existing cases cover non-DV partial counts; explicit bounds need controls here. - val batches = Seq.fill(3)(physical) ++ - (if (bounds == "absent") Seq.empty else Seq.fill(2)(control)) - val rows = Seq.fill(3)(live).flatten ++ - (if (bounds == "absent") Seq.empty else Seq.fill(2)(control).flatten) - val properties = Map("delta.enableDeletionVectors" -> "true", - "delta.checkpointInterval" -> "100") - sql("""CREATE TABLE tbl (id INT, all_null INT, mixed_null INT, non_null INT) - USING delta TBLPROPERTIES ('delta.enableDeletionVectors' = 'true', - 'delta.checkpointInterval' = '100')""") - batches.foreach { batch => - val values = batch.map { row => - row.toSeq.map(v => if (v == null) "NULL" else v.toString).mkString("(", ", ", ")") - }.mkString(", ") - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values " + - "AS v(id, all_null, mixed_null, non_null)") - } - sql("DELETE FROM tbl WHERE id = 1 OR id = 4") - val t = registerTable("tbl") - var copiedTable: Path = null - mutateTable(t) { dir => - copiedTable = dir - val lastVersion = batches.size.toLong + 1 - val commits = (0L to lastVersion).map(v => v -> CommitLog.read(dir, v)).toMap - val metadata = commits(0).collect { case m: Metadata => m }.head - require(DataType.fromJson(metadata.schemaString) == schema) - require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) - val originals = batches.indices.map { index => - val adds = commits(index.toLong + 1).collect { case a: AddFile => a } - require(adds.size == 1 && adds.head.deletionVector.isEmpty) - val add = adds.head - RowComparison.assertRowsEqual(spark.createDataFrame(batches(index).asJava, schema), - spark.read.parquet(dir.resolve(add.path).toString), s"physical_$bounds/$index") - add - } - val deleted = commits(lastVersion) - val dvAdds = deleted.collect { case a: AddFile => a } - val removes = deleted.collect { case r: RemoveFile => r } - require(dvAdds.size == 3 && removes.size == 3) - require(dvAdds.map(_.path).toSet == originals.take(3).map(_.path).toSet) - require(removes.map(_.path).toSet == dvAdds.map(_.path).toSet) - require(dvAdds.forall(a => a.deletionVector.exists(_.cardinality == 2))) - RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, schema), - spark.read.format("delta").load(dir.toString), s"live_before_mutation_$bounds") - - val stream = Files.list(dir.resolve("_delta_log")) - val files = try stream.iterator().asScala.toVector finally stream.close() - require(!files.exists(_.getFileName.toString.contains("checkpoint"))) - files.filter(_.getFileName.toString.matches("[0-9]{20}\\.crc")).foreach(Files.delete) - for (version <- 1L to lastVersion) { - val changed = commits(version).map { - case add: AddFile if add.deletionVector.isDefined || - originals.drop(3).exists(_.path == add.path) => - val index = originals.indexWhere(_.path == add.path) - require(index >= 0 && add.partitionValues.isEmpty) - val valid = if (index < 3) live else control - val basis = if (bounds == "false") batches(index) else valid - val stats = JsonUtil.mapper.createObjectNode() - // DV numRecords is always physical, including when tight null counts are live. - stats.put("numRecords", batches(index).size) - if (bounds != "absent") stats.put("tightBounds", bounds == "true") - val minimum = stats.putObject("minValues") - val maximum = stats.putObject("maxValues") - val nulls = stats.putObject("nullCount") - schema.fieldNames.zipWithIndex.foreach { case (column, ordinal) => - val values = basis.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) - if (values.nonEmpty) { - minimum.put(column, values.min) - maximum.put(column, values.max) - } - nulls.put(column, basis.count(_.isNullAt(ordinal))) - } - index match { - case 1 => stats.remove("nullCount") - case 2 => - nulls.remove("all_null") - nulls.remove("mixed_null") - case 3 => stats.remove("numRecords") - case 4 => - stats.remove("numRecords") - stats.remove("nullCount") - case _ => - } - require(stats.has("tightBounds") == (bounds != "absent")) - require(stats.has("numRecords") == (index < 3)) - if (index < 3) require(stats.path("numRecords").asInt() == 5) - add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) - case other => other - } - CommitLog.write(dir, version, changed) - require(CommitLog.read(dir, version) == changed, - s"Persisted bounds mutation mismatch at version $version") - } - } - - // SQL three-valued predicates applied to literal row bags, independent of file stats. - val queries = Seq[(String, String, Seq[Row])]( - ("all", null, rows), - ("all_null", "all_null IS NULL", rows), - ("all_null_not_null", "all_null IS NOT NULL", Seq.empty), - ("all_null_equal", "all_null = 10", Seq.empty), - ("mixed_null", "mixed_null IS NULL", rows.filter(_.isNullAt(2))), - ("mixed_not_null", "mixed_null IS NOT NULL", rows.filterNot(_.isNullAt(2))), - ("mixed_equal", "mixed_null = 30", rows.filter(r => !r.isNullAt(2) && - r.getInt(2) == 30)), - ("non_null_is_null", "non_null IS NULL", Seq.empty), - ("non_null_not_null", "non_null IS NOT NULL", rows), - ("surviving_duplicates", "id = 3", Seq.fill(3)(live.tail).flatten), - ("deleted_lower_bound", "id = 1", Seq.empty), - ("deleted_upper_bound", "id = 4", Seq.empty), - ("deleted_mixed_value", "id < 100 AND mixed_null = 10", Seq.empty), - ("surviving_range", "non_null >= 20 AND non_null <= 30", - rows.filter(r => r.getInt(3) >= 20 && r.getInt(3) <= 30)), - ("outside_range", "non_null > 40", Seq.empty)) - queries.foreach { case (name, predicate, expectedRows) => - readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == expectedRows.size) - case other => fail(s"Expected successful $bounds/$name read, got $other") - } - val outputName = copiedTable.getParent.getFileName.toString - val expectedPath = outputDir.resolve(outputName).resolve("expected") - .resolve(s"${outputName}_read_$name").resolve("expected_data") - RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), - spark.read.parquet(expectedPath.toString), s"independent_oracle_$bounds/$name") - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == schema) - require(result.metadata.configuration == properties) - require(result.metadata.partitionColumns.isEmpty) - case other => fail(s"Expected successful $bounds snapshot, got $other") - } - } - } - } - - test("indexed_prefix_beyond_32") { - val columns = (0 until 36).map(i => f"c$i%02d") - val schema = columns.foldLeft(new StructType())((s, c) => s.add(c, IntegerType)) - // The first 32 fields are a fixed prefix; c32/c33 are selected, c34/c35 are not. - val prefix = 0 until 32 - val duplicate = Row.fromSeq(prefix ++ Seq(320, 330, 340, 350)) - val mixed = Row.fromSeq(prefix ++ Seq(321, 331, 341, 351)) - val nullLate = Row.fromSeq(prefix ++ Seq(null, null, null, null)) - val batches = Seq(Seq(duplicate, duplicate), Seq(mixed, nullLate)) - val rows = batches.flatten - val properties = Map("delta.dataSkippingNumIndexedCols" -> "34", - "delta.enableDeletionVectors" -> "false", "delta.checkpointInterval" -> "100") - val ddl = columns.map(c => s"$c INT").mkString(", ") - sql(s"""CREATE TABLE tbl ($ddl) USING delta TBLPROPERTIES ( - 'delta.dataSkippingNumIndexedCols' = '34', 'delta.enableDeletionVectors' = 'false', - 'delta.checkpointInterval' = '100')""") - batches.foreach { batch => - val values = batch.map { row => - row.toSeq.map(v => if (v == null) "NULL" else v.toString).mkString("(", ", ", ")") - }.mkString(", ") - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values " + - s"AS v(${columns.mkString(", ")})") - } - val t = registerTable("tbl") - var copiedTable: Path = null - mutateTable(t) { dir => - copiedTable = dir - val metadata = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head - require(DataType.fromJson(metadata.schemaString) == schema) - require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) - batches.zipWithIndex.foreach { case (batch, index) => - val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } - require(adds.size == 1 && adds.head.deletionVector.isEmpty) - val add = adds.head - RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), - spark.read.parquet(dir.resolve(add.path).toString), s"wide_input_$index") - // Verify the writer's physical AddFile stats, without assuming a reader's stats schema. - val stats = JsonUtil.mapper.readTree(add.stats.get) - require(stats.path("numRecords").asLong(-1) == batch.size) - for (field <- Seq("minValues", "maxValues", "nullCount")) { - require(stats.path(field).fieldNames().asScala.toSet == columns.take(34).toSet) - } - columns.take(34).zipWithIndex.foreach { case (column, ordinal) => - val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) - require(stats.path("nullCount").path(column).asLong(-1) == - batch.count(_.isNullAt(ordinal))) - require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) - require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) - } - } - } - - val queries = Seq[(String, String, Seq[Row])]( - ("all", null, rows), - ("selected_c32_equal", "c32 = 320", Seq(duplicate, duplicate)), - ("selected_c33_equal", "c33 = 330", Seq(duplicate, duplicate)), - ("selected_c33_range", "c33 >= 330 AND c33 < 331", Seq(duplicate, duplicate)), - ("selected_c33_null", "c33 IS NULL", Seq(nullLate)), - ("selected_c33_miss", "c33 = 999", Seq.empty), - ("unselected_c34_equal", "c34 = 340", Seq(duplicate, duplicate)), - ("unselected_c34_range", "c34 > 340", Seq(mixed)), - ("unselected_c34_null", "c34 IS NULL", Seq(nullLate)), - ("unselected_c34_miss", "c34 = 999", Seq.empty)) - queries.foreach { case (name, predicate, expectedRows) => - readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == expectedRows.size) - case other => fail(s"Expected successful wide $name read, got $other") - } - val outputName = copiedTable.getParent.getFileName.toString - val expectedPath = outputDir.resolve(outputName).resolve("expected") - .resolve(s"${outputName}_read_$name").resolve("expected_data") - RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), - spark.read.parquet(expectedPath.toString), s"independent_oracle_wide_$name") - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == schema) - require(result.metadata.configuration == properties) - require(result.metadata.partitionColumns.isEmpty) - case other => fail(s"Expected successful wide snapshot, got $other") - } - } - } - - test("alter_selected_stats_columns") { - val schema = new StructType().add("event_ts", TimestampType) - .add("customer", new StructType().add("country", StringType).add("segment", StringType)) - .add("amount", DecimalType(10, 2)).add("note", StringType) - val oldDuplicate = Row(Timestamp.from(Instant.parse("2024-01-01T00:00:00Z")), - Row("US", "retail"), BigDecimal("10.50").bigDecimal, "alpha") - val oldOther = Row(Timestamp.from(Instant.parse("2024-01-02T00:00:00Z")), - Row("CA", "wholesale"), BigDecimal("20.00").bigDecimal, "beta") - val newDuplicate = Row(Timestamp.from(Instant.parse("2024-01-03T00:00:00Z")), - Row("FR", "retail"), BigDecimal("30.25").bigDecimal, "gamma") - val newOther = Row(Timestamp.from(Instant.parse("2024-01-04T00:00:00Z")), - Row("US", "business"), BigDecimal("-5.00").bigDecimal, "alpha") - val nullRow = Row(null, Row(null, null), null, null) - val before = Seq(oldDuplicate, oldDuplicate, oldOther) - val after = Seq(newDuplicate, newDuplicate, newOther, nullRow) - val rows = before ++ after - val property = "delta.dataSkippingStatsColumns" - val properties = Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "100", property -> "note,customer.segment") - val selected = "event_ts,customer.country,amount" - sql("""CREATE TABLE tbl (event_ts TIMESTAMP, customer STRUCT, - amount DECIMAL(10, 2), note STRING) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100', - 'delta.dataSkippingStatsColumns' = 'note,customer.segment')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (TIMESTAMP '2024-01-01T00:00:00Z', named_struct('country','US','segment','retail'), - 10.50, 'alpha'), - (TIMESTAMP '2024-01-01T00:00:00Z', named_struct('country','US','segment','retail'), - 10.50, 'alpha'), - (TIMESTAMP '2024-01-02T00:00:00Z', named_struct('country','CA','segment','wholesale'), - 20.00, 'beta') AS v(event_ts, customer, amount, note)""") - sql(s"ALTER TABLE tbl SET TBLPROPERTIES ('$property' = '$selected')") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (TIMESTAMP '2024-01-03T00:00:00Z', named_struct('country','FR','segment','retail'), - 30.25, 'gamma'), - (TIMESTAMP '2024-01-03T00:00:00Z', named_struct('country','FR','segment','retail'), - 30.25, 'gamma'), - (TIMESTAMP '2024-01-04T00:00:00Z', named_struct('country','US','segment','business'), - -5.00, 'alpha'), - (NULL, named_struct('country',NULL,'segment',NULL), NULL, NULL) - AS v(event_ts, customer, amount, note)""") - val t = registerTable("tbl") - var copiedTable: Path = null - mutateTable(t) { dir => - copiedTable = dir - val initial = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head - val changed = CommitLog.read(dir, 2) - val metadata = changed.collect { case m: Metadata => m }.head - require(initial.configuration == properties) - require(metadata.configuration == properties.updated(property, selected)) - require(initial.id == metadata.id && initial.schemaString == metadata.schemaString) - require(DataType.fromJson(metadata.schemaString) == schema) - require(initial.partitionColumns.isEmpty && metadata.partitionColumns.isEmpty) - require(!changed.exists(a => a.isInstanceOf[AddFile] || a.isInstanceOf[RemoveFile]), - "ALTER must change metadata without rewriting files or backfilling statistics") - for ((version, batch) <- Seq(1L -> before, 3L -> after)) { - val actions = CommitLog.read(dir, version) - require(!actions.exists(_.isInstanceOf[RemoveFile])) - val adds = actions.collect { case a: AddFile => a } - require(adds.size == 1 && adds.head.deletionVector.isEmpty) - val add = adds.head - RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), - spark.read.parquet(dir.resolve(add.path).toString), s"selected_stats_input_v$version") - val stats = JsonUtil.mapper.readTree(add.stats.get) - require(stats.path("numRecords").asLong(-1) == batch.size) - for (field <- Seq("minValues", "maxValues", "nullCount")) { - val expectedFields = if (version == 1) Set("customer", "note") - else Set("event_ts", "customer", "amount") - require(stats.path(field).fieldNames().asScala.toSet == expectedFields) - require(stats.path(field).path("customer").fieldNames().asScala.toSet == - (if (version == 1) Set("segment") else Set("country"))) - } - } - } - val queries = Seq[(String, java.lang.Long, String, Seq[Row])]( - ("before_all", 1L, null, before), - ("before_timestamp", 1L, "event_ts = TIMESTAMP '2024-01-01T00:00:00Z'", - Seq(oldDuplicate, oldDuplicate)), - ("all", null, null, rows), - ("selected_timestamp_equal", null, "event_ts = TIMESTAMP '2024-01-01T00:00:00Z'", - Seq(oldDuplicate, oldDuplicate)), - ("selected_timestamp_range", null, "event_ts >= TIMESTAMP '2024-01-03T00:00:00Z'", - Seq(newDuplicate, newDuplicate, newOther)), - ("selected_country_equal", null, "customer.country = 'US'", - Seq(oldDuplicate, oldDuplicate, newOther)), - ("selected_country_null", null, "customer.country IS NULL", Seq(nullRow)), - ("selected_amount_equal", null, "amount = 30.25", Seq(newDuplicate, newDuplicate)), - ("selected_amount_range", null, "amount >= 10.50 AND amount <= 20.00", before), - ("selected_amount_null", null, "amount IS NULL", Seq(nullRow)), - ("selected_country_miss", null, "customer.country = 'ZZ'", Seq.empty), - ("excluded_note_equal", null, "note = 'alpha'", Seq(oldDuplicate, oldDuplicate, newOther)), - ("excluded_segment_equal", null, "customer.segment = 'retail'", - Seq(oldDuplicate, oldDuplicate, newDuplicate, newDuplicate)), - ("excluded_note_null", null, "note IS NULL", Seq(nullRow)), - ("selected_combined", null, - "event_ts >= TIMESTAMP '2024-01-03T00:00:00Z' AND " + - "customer.country = 'FR' AND amount = 30.25", Seq(newDuplicate, newDuplicate))) - queries.foreach { case (name, version, predicate, expectedRows) => - readSpec(t, version = version, predicate = predicate, name = Some(s"read_$name")) - .assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == expectedRows.size) - case other => fail(s"Expected successful selected-stats $name read, got $other") - } - val outputName = copiedTable.getParent.getFileName.toString - val expectedPath = outputDir.resolve(outputName).resolve("expected") - .resolve(s"${outputName}_read_$name").resolve("expected_data") - RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), - spark.read.parquet(expectedPath.toString), s"independent_oracle_selected_stats_$name") - } - } - for (version <- Seq[java.lang.Long](1L, null)) { - snapshotSpec(t, version = version).assert { spec => - spec.expectation match { - case Succeeded(result) => - // This asserts the table schema and properties, not Kernel's internal stats schema. - require(DataType.fromJson(result.metadata.schemaString) == schema) - require(result.metadata.configuration == - (if (version == null) properties.updated(property, selected) else properties)) - require(result.metadata.partitionColumns.isEmpty) - case other => fail(s"Expected successful selected-stats snapshot, got $other") - } - } - } - } - - // One checkpoint matrix serves both missing evolved-column stats and nullable evolution. - // Commit/time-travel coverage stays in DataSkippingSuite and SchemaEvolutionSuite. - for (parsed <- Seq(false, true)) { - val variant = if (parsed) "parsed" else "json_only" - test(s"nullable_evolution_checkpoint_$variant") { - val schema = new StructType().add("id", IntegerType).add("part", IntegerType) - .add("value", IntegerType) - val oldRows = Seq(Row(1, 0, null), Row(2, 1, null), Row(2, 1, null)) - val newRows = Seq(Row(3, 0, 30), Row(3, 0, 30), Row(4, 1, null), Row(5, 1, 50)) - val rows = oldRows ++ newRows - sql(s"""CREATE TABLE tbl (id INT, part INT) USING delta PARTITIONED BY (part) - TBLPROPERTIES ('delta.checkpointInterval' = '100', - 'delta.enableDeletionVectors' = 'false', - 'delta.checkpoint.writeStatsAsJson' = '${!parsed}', - 'delta.checkpoint.writeStatsAsStruct' = '$parsed')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (1, 0), (2, 1), (2, 1) AS v(id, part)""") - sql("ALTER TABLE tbl ADD COLUMN value INT") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (3, 0, 30), (3, 0, 30), (4, 1, NULL), (5, 1, 50) AS v(id, part, value)""") - forceCheckpoint("tbl") - val t = registerTable("tbl") - var copiedTable: Path = null - - mutateTable(t) { dir => - copiedTable = dir - val commits = (0L to 3L).map(v => v -> CommitLog.read(dir, v)).toMap - require(!commits.values.flatten.exists(_.isInstanceOf[RemoveFile])) - val evolved = commits(2).collect { case m: Metadata => m }.head - require(DataType.fromJson(evolved.schemaString) == schema) - require(evolved.partitionColumns == Seq("part")) - val oldAdds = commits(1).collect { case a: AddFile => a } - val newAdds = commits(3).collect { case a: AddFile => a } - val adds = oldAdds ++ newAdds - require(oldAdds.size == 2 && newAdds.size == 2) - require(adds.map(_.path).distinct.size == 4, "All four files must remain active") - for ((batch, files) <- Seq(oldRows -> oldAdds, newRows -> newAdds)) { - require(files.map(_.partitionValues).toSet == - Set(Map("part" -> "0"), Map("part" -> "1"))) - files.foreach { add => - val part = add.partitionValues("part").toInt - val raw = spark.read.parquet(dir.resolve(add.path).toString) - require(raw.columns.contains("value") == (batch == newRows)) - val withValue = if (batch == oldRows) raw.withColumn("value", lit(null) - .cast(IntegerType)) else raw - RowComparison.assertRowsEqual( - spark.createDataFrame(batch.filter(_.getInt(1) == part).asJava, schema), - withValue.withColumn("part", lit(part)).select("id", "part", "value"), - s"input_$variant/${add.path}") - val stats = JsonUtil.mapper.readTree(add.stats.get) - require(stats.path("numRecords").asLong(-1) == batch.count(_.getInt(1) == part)) - for (field <- Seq("minValues", "maxValues", "nullCount")) { - require(stats.path(field).has("value") == (batch == newRows), - s"$field must omit the evolved column only for old files") - } - } - } - - val logDir = dir.resolve("_delta_log") - val checkpoint = logDir.resolve("00000000000000000003.checkpoint.parquet") - require(Files.isRegularFile(checkpoint), "Expected the final classic checkpoint") - val cp = spark.read.parquet(checkpoint.toString) - val addSchema = cp.schema("add").dataType.asInstanceOf[StructType] - require(addSchema.fieldNames.contains("stats") == !parsed) - require(addSchema.fieldNames.contains("stats_parsed") == parsed) - require(addSchema.fieldNames.contains("partitionValues_parsed") == parsed) - val cpAdds = cp.filter("add IS NOT NULL").select("add.*").collect() - require(cpAdds.length == adds.size, "Checkpoint must contain every active AddFile") - require(cpAdds.map(_.getAs[String]("path")).toSet == adds.map(_.path).toSet) - require(cp.filter("remove IS NOT NULL").count() == 0) - cpAdds.foreach { action => - val original = adds.find(_.path == action.getAs[String]("path")).get - val stats = JsonUtil.mapper.readTree(original.stats.get) - if (parsed) { - val partition = action.getAs[Row]("partitionValues_parsed") - require(partition.schema("part").dataType == IntegerType) - require(partition.getAs[Int]("part") == original.partitionValues("part").toInt) - val actual = action.getAs[Row]("stats_parsed") - require(actual.getAs[Long]("numRecords") == stats.path("numRecords").asLong()) - for (field <- Seq("minValues", "maxValues", "nullCount")) { - val values = actual.getAs[Row](field) - require(values.getAs[Any]("id").toString == stats.path(field).path("id").asText()) - if (oldAdds.exists(_.path == original.path)) { - require(values.isNullAt(values.fieldIndex("value")), - s"Old file $field must have no parsed evolved-column statistic") - } else { - require(values.getAs[Any]("value").toString == - stats.path(field).path("value").asText()) - } - } - } else { - require(JsonUtil.mapper.readTree(action.getAs[String]("stats")) == stats) - } - } - val lastCheckpoint = JsonUtil.mapper.readTree( - Files.readAllBytes(logDir.resolve("_last_checkpoint"))) - require(lastCheckpoint.path("version").asLong(-1) == 3) - require(lastCheckpoint.path("size").asLong(-1) == cp.count()) - - // Keep the checkpoint-version commit for timestamp resolution, but remove its entire - // preceding history and every version CRC. Reconstruction now requires this checkpoint. - for (v <- 0L until 3L) Files.delete(CommitLog.commitFile(dir, v)) - val stream = Files.list(logDir) - val logFiles = try stream.iterator().asScala.toVector finally stream.close() - logFiles.filter(_.getFileName.toString.matches("[0-9]{20}\\.crc")) - .foreach(Files.delete) - require((0L until 3L).forall(v => !Files.exists(CommitLog.commitFile(dir, v)))) - require(Files.exists(CommitLog.commitFile(dir, 3))) - require(Files.exists(checkpoint) && adds.forall(a => Files.exists(dir.resolve(a.path)))) - } - - val queries = Seq[(String, String, Option[Seq[String]], Seq[Row])]( - ("all", null, None, rows), - ("projection", null, Some(Seq("value")), - Seq(Row(null), Row(null), Row(null), Row(30), Row(30), Row(null), Row(50))), - ("equal", "value = 30", None, Seq(Row(3, 0, 30), Row(3, 0, 30))), - ("range", "value >= 30 AND value < 50", None, Seq(Row(3, 0, 30), Row(3, 0, 30))), - ("is_null", "value IS NULL", None, oldRows :+ Row(4, 1, null)), - ("is_not_null", "value IS NOT NULL", None, - Seq(Row(3, 0, 30), Row(3, 0, 30), Row(5, 1, 50))), - ("partition_null", "part = 1 AND value IS NULL", None, - Seq(Row(2, 1, null), Row(2, 1, null), Row(4, 1, null)))) - queries.foreach { case (name, predicate, columns, expectedRows) => - readSpec(t, predicate = predicate, columns = columns, name = Some(s"read_$name")) - .assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == expectedRows.size) - case other => fail(s"Expected successful $variant/$name read, got $other") - } - val expectedSchema = if (columns.isDefined) { - new StructType().add("value", IntegerType) - } else schema - val outputName = copiedTable.getParent.getFileName.toString - val expectedPath = outputDir.resolve(outputName).resolve("expected") - .resolve(s"${outputName}_read_$name").resolve("expected_data") - RowComparison.assertRowsEqual( - spark.createDataFrame(expectedRows.asJava, expectedSchema), - spark.read.parquet(expectedPath.toString), s"independent_oracle_$variant/$name") - } - } - snapshotSpec(t).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == schema) - require(result.metadata.partitionColumns == Seq("part")) - require(result.metadata.configuration("delta.checkpoint.writeStatsAsStruct") == - parsed.toString) - require(result.metadata.configuration("delta.checkpoint.writeStatsAsJson") == - (!parsed).toString) - case other => fail(s"Expected successful $variant snapshot, got $other") - } - } - } - } -} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTableOperationsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTableOperationsSuite.scala deleted file mode 100644 index a64a4672..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTableOperationsSuite.scala +++ /dev/null @@ -1,1354 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.tables - -import java.io.DataInputStream -import java.nio.file.{Files, Path, Paths} - -import scala.jdk.CollectionConverters._ - -import org.apache.hadoop.fs.{Path => HadoopPath} -import org.apache.spark.sql.Row -import org.apache.spark.sql.delta.DeltaLog -import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor -import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray -import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore -import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} - -import io.delta.workload.{WorkloadContext, WorkloadTestSuite} -import io.delta.workload.deltaharness.{DeltaHarness, Format => HarnessFormat} -import io.delta.workload.engine.{RowComparison, SnapshotResolver} -import io.delta.workload.json.JsonUtil -import io.delta.workload.log.{Action, AddFile, CommitLog, DeletionVector, DomainMetadata, Metadata, - Protocol, RemoveFile, Txn} -import io.delta.workload.model._ - -class ReadinessTableOperationsSuite extends WorkloadTestSuite("readiness_table_operations") { - - private def descriptor(dv: DeletionVector): DeletionVectorDescriptor = { - // At this DAT pin USE_LONG_FOR_INTS boxes Option[Int] offsets as Long. Normalize - // only this suite's descriptor input, without changing the shared mapper or DV bytes. - val offset = dv.offset.asInstanceOf[Option[Number]].map(n => Math.toIntExact(n.longValue())) - DeletionVectorDescriptor(dv.storageType, dv.pathOrInlineDv, offset, - dv.sizeInBytes, dv.cardinality) - } - - private def sidecar(table: Path, dv: DeletionVector): Path = - Paths.get(descriptor(dv).absolutePath(new HadoopPath(table.toUri)).toUri.getPath) - - private def payload(table: Path, dv: DeletionVector): Array[Byte] = { - if (dv.storageType == "i") descriptor(dv).inlineData - else { - assert(dv.storageType == "u") - val file = sidecar(table, dv) - assert(file.normalize().startsWith(table.normalize())) - val offset = descriptor(dv).offset.get - assert(offset > 0) - assert(Files.size(file) >= offset.toLong + 4L + dv.sizeInBytes + 4L) - val input = new DataInputStream(Files.newInputStream(file)) - try { - assert(input.readByte() == DeletionVectorStore.DV_FILE_FORMAT_VERSION_ID_V1) - input.skipNBytes(offset.toLong - 1L) - // This OSS reader checks both the payload length and checksum. - DeletionVectorStore.readRangeFromStream(input, dv.sizeInBytes) - } finally input.close() - } - } - - Seq("inline", "uuid_relative").foreach { form => - test(s"e2e_42_$form") { - sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000', - 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-7, 'remove'), (0, 'keep'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), - (9, 'remove'), (12, 'keep'), (18, CAST(NULL AS STRING)), (23, 'edge'), - (31, 'keep'), (44, CAST(NULL AS STRING)), (57, 'remove'), (88, 'last') AS v(id, value)""") - sql("DELETE FROM tbl WHERE id IN (-7, 9, 57)") - sql("DELETE FROM tbl WHERE id = 4") - val table = registerTable("tbl") - val schema = new StructType().add("id", IntegerType).add("value", StringType) - val baseRows = Seq(Row(-7, "remove"), Row(0, "keep"), Row(4, null), Row(4, null), - Row(9, "remove"), Row(12, "keep"), Row(18, null), Row(23, "edge"), - Row(31, "keep"), Row(44, null), Row(57, "remove"), Row(88, "last")) - val firstRows = baseRows.filterNot(r => Set(-7, 9, 57).contains(r.getInt(0))) - val latestRows = firstRows.filterNot(_.getInt(0) == 4) - val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head - val protocol = original(0L).collect { case p: Protocol => p }.ensuring(_.size == 1).head - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.partitionColumns.isEmpty) - assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true", - "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) - assert(protocol == Protocol(3, 7, Some(Set("deletionVectors")), - Some(Set("appendOnly", "deletionVectors", "invariants")))) - info("DV original protocol: " + protocol.toJson) - info("DV original metadata: " + metadata.toJson) - val adds = (1L to 3L).map { version => - val found = original(version).collect { case a: AddFile => a } - assert(found.size == 1) - info(s"DV original add v$version: " + found.head.toJson) - version -> found.head - }.toMap - assert(adds.values.map(_.path).toSet.size == 1) // Deletes must retain the data file. - assert(adds(1L).deletionVector.isEmpty) - val physical = spark.read.parquet(table.sourcePath.resolve(adds(1L).path).toString) - assert(physical.schema == schema) - RowComparison.assertRowsEqual(spark.createDataFrame(baseRows.asJava, schema), physical, - s"$form physical input") - val physicalRows = physical.collect().toSeq - val sourceDvs = (2L to 3L).map(v => v -> adds(v).deletionVector.get).toMap - sourceDvs.foreach { case (version, dv) => - assert(dv.storageType == "u") - assert(dv.cardinality == (if (version == 2L) 3L else 5L)) - val expectedPositions = physicalRows.zipWithIndex.collect { - case (row, index) if Set(-7, 9, 57).contains(row.getInt(0)) || - (version == 3L && row.getInt(0) == 4) => index.toLong - } - assert(RoaringBitmapArray.readFrom(payload(table.sourcePath, dv)).toArray.toSeq == - expectedPositions) - } - val converted = sourceDvs.values.map { dv => - val bytes = payload(table.sourcePath, dv) - // Use the verified OSS helper with exact engine-generated payload bytes. - val inline = DeletionVectorDescriptor.inlineInLog(bytes, dv.cardinality) - assert(inline.storageType == "i" && inline.offset.isEmpty) - assert(inline.sizeInBytes == bytes.length && inline.cardinality == dv.cardinality) - assert(inline.inlineData.sameElements(bytes)) - dv -> DeletionVector(inline.storageType, inline.pathOrInlineDv, inline.offset, - inline.sizeInBytes, inline.cardinality) - }.toMap - if (form == "inline") { - (2L to 3L).foreach { version => - modifyCommitActions(table, version) { actions => - actions.map { - case add: AddFile => add.copy(deletionVector = add.deletionVector.map(converted)) - // The second delete removes the first DV identity; convert that tombstone too. - case remove: RemoveFile => - remove.copy(deletionVector = remove.deletionVector.map(converted)) - case other => other - } - } - } - } - val expectedAdds = adds.map { case (version, add) => - version -> (if (form == "inline") { - add.copy(deletionVector = add.deletionVector.map(converted)) - } else add) - } - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - val sidecars = sourceDvs.values.map(dv => sidecar(dir, dv)).toSet - if (form == "inline") sidecars.foreach(Files.delete) - else sidecars.foreach(p => assert(Files.isRegularFile(p))) - // Version checksums embed the original descriptors and become stale after conversion. - val logs = Files.list(dir.resolve("_delta_log")) - try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) - .foreach(Files.delete) - finally logs.close() - (1L to 3L).foreach { version => - val actions = CommitLog.read(dir, version) - assert(actions.collect { case a: AddFile => a } == Seq(expectedAdds(version))) - expectedAdds(version).deletionVector.foreach { dv => - assert(dv.storageType == (if (form == "inline") "i" else "u")) - assert(dv.cardinality == (if (version == 2L) 3L else 5L)) - assert(payload(dir, dv).sameElements(payload(table.sourcePath, sourceDvs(version)))) - assert(RoaringBitmapArray.readFrom(payload(dir, dv)).cardinality == dv.cardinality) - } - } - val oldIdentity = CommitLog.read(dir, 3L).collect { case r: RemoveFile => r } - assert(oldIdentity.size == 1) - assert(oldIdentity.head.path == adds(1L).path) - assert(oldIdentity.head.deletionVector == expectedAdds(2L).deletionVector) - assert(Files.size(dir.resolve(adds(1L).path)) == adds(1L).size) - if (form == "inline") assert(sidecars.forall(p => !Files.exists(p))) - } - val snapshot = SnapshotResult( - ProtocolInfo(3, 7, Some(Seq("deletionVectors")), - Some(Seq("appendOnly", "deletionVectors", "invariants"))), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), metadata.schemaString, Seq.empty, - metadata.configuration, metadata.createdTime)) - Seq[java.lang.Long](1L, 2L, null).foreach { version => - snapshotSpec(table, version = version).assert { spec => - assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) - assert(spec.expectation == Succeeded(snapshot)) - } - } - val projected = new StructType().add("value", StringType) - val reordered = projected.add("id", IntegerType) - val filter = "id IN (4, 12, 18, 57)" - val cases = Seq( - ("read_base", ReadQuery(version = Some(1L)), schema, baseRows), - ("read_first_delete", ReadQuery(version = Some(2L)), schema, firstRows), - ("read_latest", ReadQuery(), schema, latestRows), - ("read_v2_filtered_projected", ReadQuery(version = Some(2L), - predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered, - Seq(Row(null, 4), Row(null, 4), Row("keep", 12), Row(null, 18))), - ("read_v3_filtered_projected", ReadQuery(version = Some(3L), - predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered, - Seq(Row("keep", 12), Row(null, 18))), - ("read_projected", ReadQuery(columns = Some(Seq("value"))), projected, - Seq(Row("keep"), Row("keep"), Row(null), Row("edge"), Row("keep"), Row(null), - Row("last"))), - ("read_deleted_only", ReadQuery(predicate = Some("id IN (-7, 4, 9, 57)")), schema, - Seq.empty[Row])) - cases.foreach { case (name, query, expectedSchema, rows) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull, columns = query.columns).assert { spec => - assert(spec.query == query) - // All predicates overlap physical min/max; the single data AddFile remains scanned. - assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, 1, 0L))) - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), - query.predicate, query.columns) - assert(actual.schema == expectedSchema) - RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, expectedSchema), actual, - s"$form $name") - val fixture = outputDir.resolve(WorkloadContext.current.workloadName) - val specName = s"${fixture.getFileName}_$name" - val captured = spark.read.parquet( - fixture.resolve(s"expected/$specName/expected_metadata").toString) - val scanned = captured.collect().toSeq.map { row => - val node = JsonUtil.mapper.readTree(row.getString(0)) - val wrapped = if (node.has("add")) node.toString else s"""{"add":$node}""" - Action.parse(wrapped) - } - // OSS InMemoryLogReplay stores active AddFiles with dataChange=false; - // ReadCapture obtains these snapshot actions through DeltaSparkHarness.allFiles. - assert(scanned == Seq(expectedAdds(query.version.getOrElse(3L)) - .copy(dataChange = false))) - } - } - } - } - - test("e2e_54_truncated_history") { - sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', - 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-1, 'old'), (2, 'old'), (2, 'old'), (CAST(NULL AS INT), 'old') AS v(id, value)""") - sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (7, 'keep'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), - (CAST(NULL AS INT), 'missing') AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (9, 'boundary'), (7, 'keep') AS v(id, value)""") - val table = registerTable("tbl") - val schema = new StructType().add("id", IntegerType).add("value", StringType) - val boundaryRows = Seq(Row(7, "keep"), Row(3, null), Row(3, null), Row(null, "missing"), - Row(9, "boundary"), Row(7, "keep")) - val latestRows = boundaryRows ++ Seq(Row(3, null), Row(11, "after"), Row(null, "after")) - val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head - assert(original(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.format == io.delta.workload.log.Format()) - assert(metadata.partitionColumns.isEmpty) - assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) - val inputs = (1L to 3L).map { version => - val adds = original(version).collect { case a: AddFile => a } - assert(adds.size == 1 && adds.head.deletionVector.isEmpty) - version -> adds.head - }.toMap - assert(inputs.values.map(_.path).toSet.size == 3) - assert(original(2L).collect { case r: RemoveFile => r.path } == Seq(inputs(1L).path)) - DeltaHarness.get.openLog(spark, table.sourcePath.toString).checkpoint() - val checkpointName = "00000000000000000003.checkpoint.parquet" - val checkpointBytes = Files.readAllBytes(table.sourcePath.resolve("_delta_log") - .resolve(checkpointName)) - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - val logDir = dir.resolve("_delta_log") - (0L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) - assert(Files.readAllBytes(logDir.resolve(checkpointName)).sameElements(checkpointBytes)) - // Retain v2 and v3 JSON for OSS historical lookup, plus the oldest checkpoint at v3. - // The missing prefix makes v1 unavailable; no table data is deleted. - (0L to 1L).foreach { version => - val commit = logDir.resolve(f"$version%020d.json") - assert(Files.isRegularFile(commit)) - Files.delete(commit) - assert(!Files.exists(commit)) - } - (2L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) - inputs.values.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) - val boundary = SnapshotResolver.buildDeltaReader(spark, dir, Some(3L), None) - assert(boundary.schema == schema) - RowComparison.assertRowsEqual(spark.createDataFrame(boundaryRows.asJava, schema), boundary, - "history boundary after JSON removal, before append") - // Append to the already truncated copy: v4 must be replayed after the v3 checkpoint. - spark.sql(s"""INSERT INTO delta.`$dir` SELECT /*+ COALESCE(1) */ * FROM VALUES - (3, CAST(NULL AS STRING)), (11, 'after'), (CAST(NULL AS INT), 'after') AS v(id, value)""") - .collect() - // The generator strips Hadoop checksum sidecars while copying. The append's LogStore - // may recreate this sidecar; keep the same portable fixture form after the mutation. - Files.deleteIfExists(logDir.resolve(".00000000000000000004.json.crc")) - val appended = CommitLog.read(dir, 4L) - val adds = appended.collect { case a: AddFile => a } - assert(adds.size == 1 && adds.head.deletionVector.isEmpty) - assert(!inputs.values.exists(_.path == adds.head.path)) - assert(appended.forall(a => !a.isInstanceOf[RemoveFile] && !a.isInstanceOf[Metadata] && - !a.isInstanceOf[Protocol])) - val commitInfo = appended.map(a => JsonUtil.mapper.readTree(a.toJson)) - .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") - assert(commitInfo.get("operation").asText() == "WRITE") - assert(commitInfo.get("operationParameters").get("mode").asText() == "Append") - assert(Files.size(dir.resolve(adds.head.path)) == adds.head.size) - assert(Files.readAllBytes(logDir.resolve(checkpointName)).sameElements(checkpointBytes)) - val files = Files.list(logDir) - try { - val names = files.iterator().asScala.map(_.getFileName.toString).toSeq - assert(names.filter(_.endsWith(".checkpoint.parquet")) == Seq(checkpointName)) - assert(names.filter(_.endsWith(".json")).sorted == (2L to 4L).map(v => f"$v%020d.json")) - } finally files.close() - assert(JsonUtil.mapper.readTree(Files.readAllBytes(logDir.resolve("_last_checkpoint"))) - .get("version").asLong() == 3L) - (2L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) - inputs.values.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) - info("history removed JSON v0/v1; retained v2/v3 and checkpoint v3; real append v4: " + - commitInfo.toString) - } - val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, - metadata.configuration, metadata.createdTime)) - Seq[java.lang.Long](3L, null).foreach { version => - snapshotSpec(table, version = version).assert { spec => - assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(4L)(_.toLong)))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - } - } - val cases = Seq( - ("read_boundary_v3", ReadQuery(version = Some(3L)), boundaryRows, 2), - ("read_latest_after_append", ReadQuery(), latestRows, 3)) - cases.foreach { case (name, query, rows, fileCount) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull) - .assert { spec => - assert(spec.query == query) - assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, fileCount, 0L))) - val expected = spark.createDataFrame(rows.asJava, schema) - val actual = SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, None) - assert(actual.schema == schema) - RowComparison.assertRowsEqual(expected, actual, s"history $name copied table") - val fixture = copiedTable.getParent - val specName = s"${fixture.getFileName}_$name" - val captured = spark.read.parquet( - fixture.resolve(s"expected/$specName/expected_data").toString) - assert(captured.schema == schema) - RowComparison.assertRowsEqual(expected, captured, s"history $name expected parquet") - } - } - // No checkpoint or JSON exists at or before v1. DAT resolves getSnapshotAt first; - // OSS raises DELTA_LOG_FILE_NOT_FOUND for that empty log segment, before scanning data. - readSpec(table, name = Some("read_unavailable_v1"), version = 1L, - expectError = ErrorCode("DELTA_LOG_FILE_NOT_FOUND")).assert { spec => - assert(spec.query == ReadQuery(version = Some(1L))) - spec.expectation match { - case Failed(error) => - assert(error.errorCode == "DELTA_LOG_FILE_NOT_FOUND") - assert(error.errorMessage.contains("version 1")) - info("history typed unavailable-history read: " + error.toString) - case other => fail(s"history unavailable-history read expected Failed, got $other") - } - } - snapshotSpec(table, version = 1L, expectError = ErrorCode("DELTA_LOG_FILE_NOT_FOUND")) - .assert { spec => - assert(spec.query == SnapshotQuery(version = Some(1L))) - spec.expectation match { - case Failed(error) => - assert(error.errorCode == "DELTA_LOG_FILE_NOT_FOUND") - assert(error.errorMessage.contains("version 1")) - case other => fail(s"history unavailable-history snapshot expected Failed, got $other") - } - } - } - - test("e2e_50_actual_vacuum") { - sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', - 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (8, 'keep'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), - (CAST(NULL AS INT), 'missing') AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-2, 'early'), (5, 'middle'), (8, 'keep'), (12, CAST(NULL AS STRING)) AS v(id, value)""") - val table = registerTable("tbl") - val schema = new StructType().add("id", IntegerType).add("value", StringType) - val rows = Seq(Row(8, "keep"), Row(3, null), Row(3, null), Row(null, "missing"), - Row(-2, "early"), Row(5, "middle"), Row(8, "keep"), Row(12, null)) - val literal = spark.createDataFrame(rows.asJava, schema) - val before = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head - assert(before(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.format == io.delta.workload.log.Format()) - assert(metadata.partitionColumns.isEmpty) - assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) - val inputs = (1L to 2L).flatMap(v => before(v).collect { case a: AddFile => a }) - assert(inputs.size == 2 && inputs.map(_.path).distinct.size == 2) - assert(inputs.forall(_.deletionVector.isEmpty)) - RowComparison.assertRowsEqual(literal, - SnapshotResolver.buildDeltaReader(spark, table.sourcePath, Some(2L), None), - "vacuum historical rows before pruning") - sql("OPTIMIZE tbl") - val rewrite = CommitLog.read(table.sourcePath, 3L) - val active = rewrite.collect { case a: AddFile => a } - val obsolete = rewrite.collect { case r: RemoveFile => r } - assert(active.nonEmpty && obsolete.size == 2) - assert(obsolete.map(_.path).toSet == inputs.map(_.path).toSet) - assert((active.map(_.path).toSet intersect obsolete.map(_.path).toSet).isEmpty) - assert(active.forall(a => !a.dataChange && a.deletionVector.isEmpty)) - assert(obsolete.forall(r => !r.dataChange && r.deletionVector.isEmpty)) - assert(rewrite.collect { case m: Metadata => m }.isEmpty) - assert(rewrite.collect { case p: Protocol => p }.isEmpty) - val optimizeInfo = rewrite.map(a => JsonUtil.mapper.readTree(a.toJson)) - .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") - assert(optimizeInfo.get("operation").asText() == "OPTIMIZE") - (inputs ++ active).foreach { add => - assert(Files.isRegularFile(table.sourcePath.resolve(add.path))) - assert(Files.size(table.sourcePath.resolve(add.path)) == add.size) - } - info("vacuum pre-VACUUM obsolete paths physically present: " + - obsolete.map(_.path).mkString(",")) - val retentionCheck = "spark.databricks.delta.retentionDurationCheck.enabled" - val originalRetentionCheck = spark.conf.getOption(retentionCheck) - try { - spark.conf.set(retentionCheck, "false") - sql("VACUUM tbl RETAIN 0 HOURS") - } finally { - originalRetentionCheck match { - case Some(value) => spark.conf.set(retentionCheck, value) - case None => spark.conf.unset(retentionCheck) - } - } - assert(spark.conf.getOption(retentionCheck) == originalRetentionCheck) - obsolete.foreach(r => assert(!Files.exists(table.sourcePath.resolve(r.path)))) - active.foreach(a => assert(Files.size(table.sourcePath.resolve(a.path)) == a.size)) - // VACUUM emits audit commits; they must not change the active file set or metadata. - val vacuum = (4L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - val vacuumInfo = vacuum.toSeq.sortBy(_._1).flatMap { case (_, actions) => - assert(actions.forall(a => !a.isInstanceOf[AddFile] && !a.isInstanceOf[RemoveFile] && - !a.isInstanceOf[Metadata] && !a.isInstanceOf[Protocol])) - actions.map(a => JsonUtil.mapper.readTree(a.toJson)).filter(_.has("commitInfo")) - .map(_.get("commitInfo")) - } - assert(vacuumInfo.map(_.get("operation").asText()) == Seq("VACUUM START", "VACUUM END")) - vacuumInfo.foreach(n => info("vacuum actual vacuum commit: " + n.toString)) - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - (before ++ Map(3L -> rewrite) ++ vacuum).foreach { case (version, actions) => - assert(CommitLog.read(dir, version) == actions) - } - obsolete.foreach(r => assert(!Files.exists(dir.resolve(r.path)))) - active.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) - // Force row materialization, independently of DAT's typed error capture. - val thrown = intercept[Exception] { - SnapshotResolver.buildDeltaReader(spark, dir, Some(2L), None).collect() - } - val causes = Iterator.iterate(thrown: Throwable)(_.getCause).takeWhile(_ != null).toSeq - assert(causes.exists(_.isInstanceOf[java.io.FileNotFoundException])) - assert(causes.exists(e => obsolete.exists(r => - Option(e.getMessage).exists(_.contains(r.path))))) - info("vacuum independent historical missing-file cause: " + - causes.map(e => e.getClass.getName + ": " + e.getMessage).mkString(" <- ")) - } - val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, - metadata.configuration, metadata.createdTime)) - Seq[java.lang.Long](3L, 5L, null).foreach { version => - snapshotSpec(table, version = version).assert { spec => - assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(5L)(_.toLong)))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - } - } - val projected = new StructType().add("value", StringType).add("id", IntegerType) - val cases = Seq( - ("read_latest", ReadQuery(), schema, rows), - ("read_rewritten_v3", ReadQuery(version = Some(3L)), schema, rows), - ("read_vacuumed_v5", ReadQuery(version = Some(5L)), schema, rows), - ("read_latest_filtered", ReadQuery(predicate = Some("id = 3 OR id IS NULL"), - columns = Some(Seq("value", "id"))), projected, - Seq(Row(null, 3), Row(null, 3), Row("missing", null)))) - cases.foreach { case (name, query, expectedSchema, expectedRows) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull, columns = query.columns).assert { spec => - assert(spec.query == query) - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == expectedRows.size.toLong) - case other => fail(s"vacuum $name expected Succeeded, got $other") - } - val expected = spark.createDataFrame(expectedRows.asJava, expectedSchema) - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), - query.predicate, query.columns) - assert(actual.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, actual, s"vacuum $name copied table") - val fixture = outputDir.resolve(WorkloadContext.current.workloadName) - val specName = s"${fixture.getFileName}_$name" - val captured = spark.read.parquet( - fixture.resolve(s"expected/$specName/expected_data").toString) - assert(captured.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, captured, s"vacuum $name expected parquet") - } - } - readSpec(table, name = Some("read_pruned_v2"), version = 2L, - expectError = AnyError).assert { spec => - assert(spec.query == ReadQuery(version = Some(2L))) - spec.expectation match { - case Failed(error) => - assert(error.errorCode.nonEmpty) - assert(obsolete.exists(r => error.errorMessage.contains(r.path))) - val message = error.errorMessage.toLowerCase(java.util.Locale.ROOT) - assert(message.contains("not exist") || message.contains("not found")) - info("vacuum typed historical error: " + error.toString) - case other => fail(s"vacuum pruned historical read expected Failed, got $other") - } - // An error at the historical version must leave the current table readable. - RowComparison.assertRowsEqual(literal, - SnapshotResolver.buildDeltaReader(spark, copiedTable, None, None), - "vacuum current rows after historical failure") - } - } - - test("e2e_51_stats_refresh") { - sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000', - 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-7, 'drop'), (0, 'low'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), - (12, 'middle'), (18, CAST(NULL AS STRING)), (31, 'last'), - (CAST(NULL AS INT), 'missing') AS v(id, value)""") - sql("DELETE FROM tbl WHERE id = -7") - val table = registerTable("tbl") - val schema = new StructType().add("id", IntegerType).add("value", StringType) - val physicalRows = Seq(Row(-7, "drop"), Row(0, "low"), Row(4, null), Row(4, null), - Row(12, "middle"), Row(18, null), Row(31, "last"), Row(null, "missing")) - val rows = Seq(Row(0, "low"), Row(4, null), Row(4, null), Row(12, "middle"), - Row(18, null), Row(31, "last"), Row(null, "missing")) - val original = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head - val protocol = Protocol(3, 7, Some(Set("deletionVectors")), - Some(Set("appendOnly", "deletionVectors", "invariants"))) - assert(original(0L).collect { case p: Protocol => p } == Seq(protocol)) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.partitionColumns.isEmpty) - assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true", - "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) - val inserted = original(1L).collect { case a: AddFile => a }.ensuring(_.size == 1).head - val deleted = original(2L).collect { case a: AddFile => a }.ensuring(_.size == 1).head - assert(inserted.path == deleted.path && inserted.deletionVector.isEmpty) - val dv = deleted.deletionVector.get - assert(dv.storageType == "u" && dv.cardinality == 1L) - val physical = spark.read.parquet(table.sourcePath.resolve(deleted.path).toString) - assert(physical.schema == schema) - RowComparison.assertRowsEqual(spark.createDataFrame(physicalRows.asJava, schema), physical, - "replay literal physical rows") - val positions = physical.collect().toSeq.zipWithIndex.collect { - case (row, index) if !row.isNullAt(0) && row.getInt(0) == -7 => index.toLong - } - val dvBytes = payload(table.sourcePath, dv) - assert(RoaringBitmapArray.readFrom(dvBytes).toArray.toSeq == positions) - val fileBytes = Files.readAllBytes(table.sourcePath.resolve(deleted.path)) - // Physical stats include the deleted row. These bounds remain conservative for the DV. - // Model a partial-stats writer followed by a stats-only update, not an autostats service. - val partialStats = """{"numRecords":8,"tightBounds":false}""" - val refreshedStats = """{"numRecords":8,"minValues":{"id":-7,"value":"drop"}, - "maxValues":{"id":31,"value":"missing"},"nullCount":{"id":1,"value":3}, - "tightBounds":false}""" - val partial = deleted.copy(stats = Some(partialStats)) - val refreshed = partial.copy(stats = Some(refreshedStats), dataChange = false) - assert(refreshed.copy(stats = deleted.stats, dataChange = deleted.dataChange) == deleted) - assert(refreshed.path == deleted.path && refreshed.deletionVector == Some(dv)) - assert(refreshed.baseRowId == deleted.baseRowId) - assert(refreshed.defaultRowCommitVersion == deleted.defaultRowCommitVersion) - // Row tracking is not enabled: preserve absent row-ID fields as well as the real DV. - assert(deleted.baseRowId.isEmpty && deleted.defaultRowCommitVersion.isEmpty) - modifyCommitActions(table, 2L) { actions => - actions.map { case _: AddFile => partial; case other => other } - } - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - assert(dir.getParent.getParent == outputDir) - assert(CommitLog.read(dir, 0L) == original(0L)) - assert(CommitLog.read(dir, 1L) == original(1L)) - assert(CommitLog.read(dir, 2L) == original(2L).map { - case _: AddFile => partial; case other => other - }) - assert(!Files.exists(CommitLog.commitFile(dir, 3L))) - // One AddFile per identity per version; no conflicting same-version remove/add pair. - CommitLog.write(dir, 3L, Seq(refreshed)) - val logs = Files.list(dir.resolve("_delta_log")) - try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) - .foreach(Files.delete) - finally logs.close() - DeltaHarness.get.openLog(spark, dir.toString).checkpoint(3L) - val checkpoint = dir.resolve("_delta_log/00000000000000000003.checkpoint.parquet") - assert(Files.isRegularFile(checkpoint)) - val checkpointAdds = spark.read.parquet(checkpoint.toString) - .filter("add IS NOT NULL").select("add.*").toJSON.collect().toSeq - .map(json => Action.parse(s"""{"add":$json}""")) - assert(checkpointAdds == Seq(refreshed)) - val lastCheckpoint = JsonUtil.mapper.readTree( - Files.readAllBytes(dir.resolve("_delta_log/_last_checkpoint"))) - assert(lastCheckpoint.get("version").asLong() == 3L) - // Checkpoint creation can write fresh Hadoop sidecars; retain only protocol artifacts. - val checkpointLogs = Files.list(dir.resolve("_delta_log")) - try checkpointLogs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) - .foreach { file => - assert(file.getFileName.toString.startsWith(".")) - Files.delete(file) - } - finally checkpointLogs.close() - // A transaction marker provides a genuine JSON tail without changing the row bag. - CommitLog.write(dir, 4L, Seq(Txn("readiness_e2e_51_tail", 1L))) - assert(CommitLog.read(dir, 3L) == Seq(refreshed)) - assert(CommitLog.read(dir, 4L) == Seq(Txn("readiness_e2e_51_tail", 1L))) - assert(Files.readAllBytes(dir.resolve(deleted.path)).sameElements(fileBytes)) - assert(payload(dir, dv).sameElements(dvBytes)) - DeltaLog.clearCache() - val replay = DeltaLog.forTable(spark, dir.toString).update() - assert(replay.version == 4L) - assert(replay.logSegment.checkpointProvider.version == 3L) - assert(replay.logSegment.deltas.map(_.getPath.getName) == Seq(f"${4L}%020d.json")) - (2L to 4L).foreach { version => - val snapshot = DeltaHarness.get.openLog(spark, dir.toString).getSnapshotAt(version) - val active = snapshot.allFiles.select("json").collect().toSeq - .map(r => Action.parse(r.getString(0))) - // Delta's active-file replay normalizes dataChange; commit flags were checked above. - val expectedAdd = (if (version == 2L) partial else refreshed).copy(dataChange = false) - assert(active == Seq(expectedAdd)) - } - val remaining = Files.list(dir.resolve("_delta_log")) - try assert(!remaining.iterator().asScala.exists(_.getFileName.toString.endsWith(".crc"))) - finally remaining.close() - info("replay checkpoint v3 + JSON tail v4 selected; physical file and DV unchanged") - } - val expectedSnapshot = SnapshotResult( - ProtocolInfo(3, 7, Some(Seq("deletionVectors")), - Some(Seq("appendOnly", "deletionVectors", "invariants"))), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, - metadata.configuration, metadata.createdTime)) - Seq[java.lang.Long](2L, 3L, 4L, null).foreach { version => - snapshotSpec(table, version = version).assert { spec => - val resolved = Option(version).fold(4L)(_.toLong) - assert(spec.query == SnapshotQuery(version = Some(resolved))) - assert(spec.expectation == Succeeded(if (resolved == 4L) { - expectedSnapshot.copy(setTransactions = Some(Seq(AppTxn("readiness_e2e_51_tail", 1L)))) - } else expectedSnapshot)) - } - } - val projected = new StructType().add("value", StringType).add("id", IntegerType) - val selective = "id IN (4, 12) OR id IS NULL" - val selected = Seq(Row(null, 4), Row(null, 4), Row("middle", 12), Row("missing", null)) - val versions = Seq("before" -> Some(2L), "refreshed" -> Some(3L), - "tail" -> Some(4L), "latest" -> None) - versions.foreach { case (label, version) => - val cases = Seq( - (s"read_$label", ReadQuery(version = version), schema, rows), - (s"read_${label}_selective", ReadQuery(version = version, - predicate = Some(selective), columns = Some(Seq("value", "id"))), projected, selected), - (s"read_${label}_miss", ReadQuery(version = version, predicate = Some("id > 100")), - schema, Seq.empty[Row])) - cases.foreach { case (name, query, expectedSchema, expectedRows) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull, columns = query.columns).assert { spec => - assert(spec.query == query) - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == expectedRows.size.toLong) - case other => fail(s"replay $name expected Succeeded, got $other") - } - val expected = spark.createDataFrame(expectedRows.asJava, expectedSchema) - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), - query.predicate, query.columns) - assert(actual.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, actual, s"replay $name copied table") - val fixture = copiedTable.getParent - val specName = s"${fixture.getFileName}_$name" - val captured = spark.read.parquet( - fixture.resolve(s"expected/$specName/expected_data").toString) - assert(captured.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, captured, s"replay $name expected parquet") - } - } - } - } - - Seq("zorder", "liquid").foreach { mode => - test(s"e2e_49_$mode") { - val liquid = mode == "liquid" - val clusterBy = if (liquid) "CLUSTER BY (id, value)" else "" - sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta $clusterBy TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', - 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (8, 'late'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), - (CAST(NULL AS INT), 'missing') AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-2, 'early'), (5, 'middle'), (8, 'late'), (12, CAST(NULL AS STRING)) AS v(id, value)""") - val table = registerTable("tbl") - val schema = new StructType().add("id", IntegerType).add("value", StringType) - // Independent literal bag: includes duplicate whole rows and nulls in both columns. - val firstRows = Seq(Row(8, "late"), Row(3, null), Row(3, null), Row(null, "missing")) - val allRows = firstRows ++ Seq(Row(-2, "early"), Row(5, "middle"), Row(8, "late"), - Row(12, null)) - val before = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head - val protocol = if (liquid) { - Protocol(1, 7, None, Some(Set("appendOnly", "invariants", "clustering", "domainMetadata"))) - } else Protocol(1, 2) - assert(before(0L).collect { case p: Protocol => p } == Seq(protocol)) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.format == io.delta.workload.log.Format()) - assert(metadata.partitionColumns.isEmpty) - assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) - val domains = before(0L).collect { case d: DomainMetadata => d } - if (liquid) { - assert(domains.size == 1 && domains.head.domain == "delta.clustering") - assert(!domains.head.removed) - val configuration = JsonUtil.mapper.readTree(domains.head.configuration) - assert(configuration.get("clusteringColumns") == - JsonUtil.mapper.readTree("""[["id"],["value"]]""")) - } else assert(domains.isEmpty) - val literal = spark.createDataFrame(allRows.asJava, schema) - val unoptimized = SnapshotResolver.buildDeltaReader(spark, table.sourcePath, - Some(2L), None) - assert(unoptimized.schema == schema) - RowComparison.assertRowsEqual(literal, unoptimized, s"$mode before OPTIMIZE") - sql(if (liquid) "OPTIMIZE tbl" else "OPTIMIZE tbl ZORDER BY (id, value)") - val optimized = CommitLog.read(table.sourcePath, 3L) - val adds = optimized.collect { case a: AddFile => a } - val removes = optimized.collect { case r: RemoveFile => r } - // Check actual rewrite actions, without promising a file count or layout effectiveness. - assert(adds.nonEmpty && removes.nonEmpty) - assert(adds.forall(!_.dataChange) && removes.forall(!_.dataChange)) - val inputAdds = before.values.flatten.collect { case a: AddFile => a }.toSeq - assert(removes.map(_.path).toSet == inputAdds.map(_.path).toSet) - assert(adds.forall(_.clusteringProvider == (if (liquid) Some("liquid") else None))) - assert(optimized.collect { case m: Metadata => m }.isEmpty) - assert(optimized.collect { case p: Protocol => p }.isEmpty) - assert(optimized.collect { case d: DomainMetadata => d }.isEmpty) - val commitInfo = optimized.map(a => JsonUtil.mapper.readTree(a.toJson)) - .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") - assert(commitInfo.get("operation").asText() == "OPTIMIZE") - val parameters = commitInfo.get("operationParameters") - val columns = JsonUtil.mapper.readTree("""["id","value"]""") - val empty = JsonUtil.mapper.readTree("[]") - assert(JsonUtil.mapper.readTree(parameters.get("zOrderBy").asText()) == - (if (liquid) empty else columns)) - assert(JsonUtil.mapper.readTree(parameters.get("clusterBy").asText()) == - (if (liquid) columns else empty)) - info(s"cluster $mode protocol: " + protocol.toJson) - info(s"cluster $mode metadata: " + metadata.toJson) - info(s"cluster $mode optimize: " + commitInfo.toString) - domains.foreach(d => info(s"cluster $mode domain: " + d.toJson)) - adds.foreach(a => info(s"cluster $mode optimized add: " + a.toJson)) - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - (0L to 2L).foreach(v => assert(CommitLog.read(dir, v) == before(v))) - assert(CommitLog.read(dir, 3L) == optimized) - } - val expectedSnapshot = SnapshotResult( - ProtocolInfo(protocol.minReaderVersion, protocol.minWriterVersion, - writerFeatures = protocol.writerFeatures.map(_.toSeq.sorted)), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, - metadata.configuration, metadata.createdTime), - domainMetadata = if (liquid) Some(domains.map(d => - AddDomainMetadata(d.domain, d.configuration))) else None) - Seq[java.lang.Long](0L, 2L, 3L, null).foreach { version => - snapshotSpec(table, version = version).assert { spec => - assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - } - } - val projected = new StructType().add("value", StringType).add("id", IntegerType) - val predicate = "(id >= 3 AND id <= 8) OR id IS NULL" - val firstFiltered = Seq(Row("late", 8), Row(null, 3), Row(null, 3), Row("missing", null)) - val allFiltered = firstFiltered ++ Seq(Row("middle", 5), Row("late", 8)) - val cases = Seq( - ("read_latest", ReadQuery(), schema, allRows), - ("read_first_insert", ReadQuery(version = Some(1L)), schema, firstRows), - ("read_before_optimize", ReadQuery(version = Some(2L)), schema, allRows), - ("read_optimized_v3", ReadQuery(version = Some(3L)), schema, allRows), - ("read_first_filtered", ReadQuery(version = Some(1L), predicate = Some(predicate), - columns = Some(Seq("value", "id"))), projected, firstFiltered), - ("read_latest_filtered", ReadQuery(predicate = Some(predicate), - columns = Some(Seq("value", "id"))), projected, allFiltered), - ("read_optimized_filtered", ReadQuery(version = Some(3L), predicate = Some(predicate), - columns = Some(Seq("value", "id"))), projected, allFiltered)) - cases.foreach { case (name, query, expectedSchema, rows) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull, columns = query.columns).assert { spec => - assert(spec.query == query) - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == rows.size.toLong) - case other => fail(s"$mode $name expected Succeeded, got $other") - } - val expected = spark.createDataFrame(rows.asJava, expectedSchema) - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), - query.predicate, query.columns) - assert(actual.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, actual, s"$mode $name copied table") - val fixture = outputDir.resolve(WorkloadContext.current.workloadName) - val specName = s"${fixture.getFileName}_$name" - val captured = spark.read.parquet( - fixture.resolve(s"expected/$specName/expected_data").toString) - assert(captured.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, captured, s"$mode $name expected parquet") - } - } - } - } - - test("e2e_55_restore") { - sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', - 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-7, 'before'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), - (CAST(NULL AS INT), 'missing'), (12, 'keep'), (12, 'keep') AS v(id, value)""") - val table = registerTable("tbl") - val before = (0L to 1L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (99, 'mutated'), (99, 'mutated'), (CAST(NULL AS INT), 'changed'), - (6, CAST(NULL AS STRING)) AS v(id, value)""") - val mutated = CommitLog.read(table.sourcePath, 2L) - sql("RESTORE TABLE tbl TO VERSION AS OF 1") - val restored = CommitLog.read(table.sourcePath, 3L) - assert(!Files.exists(CommitLog.commitFile(table.sourcePath, 4L))) - (0L to 1L).foreach(v => assert(CommitLog.read(table.sourcePath, v) == before(v))) - assert(CommitLog.read(table.sourcePath, 2L) == mutated) - val originalAdds = before(1L).collect { case a: AddFile => a } - val mutatedAdds = mutated.collect { case a: AddFile => a } - val restoredAdds = restored.collect { case a: AddFile => a } - assert(originalAdds.nonEmpty && mutatedAdds.nonEmpty) - assert(originalAdds.map(_.path).toSet.intersect(mutatedAdds.map(_.path).toSet).isEmpty) - assert(mutated.collect { case r: RemoveFile => r.path }.toSet == - originalAdds.map(_.path).toSet) - assert(restoredAdds.toSet == originalAdds.map(_.copy(dataChange = true)).toSet) - val restoredRemoves = restored.collect { case r: RemoveFile => r } - assert(restoredRemoves.map(_.path).toSet == mutatedAdds.map(_.path).toSet) - assert(restoredAdds.forall(_.dataChange) && restoredRemoves.forall(_.dataChange)) - val commitInfo = restored.map(a => JsonUtil.mapper.readTree(a.toJson)) - .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") - assert(commitInfo.get("operation").asText() == "RESTORE") - assert(commitInfo.get("operationParameters").get("version").asText() == "1") - info("restore OSS Delta 4.1.0 RESTORE commit: " + commitInfo.toString) - - val schema = new StructType().add("id", IntegerType).add("value", StringType) - // Literal bags are independent of both the captured result and the Delta reader. - val initialRows = Seq(Row(-7, "before"), Row(4, null), Row(4, null), - Row(null, "missing"), Row(12, "keep"), Row(12, "keep")) - val mutatedRows = Seq(Row(99, "mutated"), Row(99, "mutated"), - Row(null, "changed"), Row(6, null)) - val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head - val configuration = Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") - assert(before(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) - val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, - configuration, metadata.createdTime)) - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - assert(dir.getParent.getParent == outputDir) - (0L to 1L).foreach(v => assert(CommitLog.read(dir, v) == before(v))) - assert(CommitLog.read(dir, 2L) == mutated) - assert(CommitLog.read(dir, 3L) == restored) - (originalAdds ++ mutatedAdds).foreach { add => - assert(Files.readAllBytes(dir.resolve(add.path)) - .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) - } - } - Seq[java.lang.Long](1L, 2L, null).foreach { version => - snapshotSpec(table, version = version).assert { spec => - assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - } - } - val projected = new StructType().add("value", StringType).add("id", IntegerType) - val cases = Seq( - ("read_before_v1", ReadQuery(version = Some(1L)), schema, initialRows), - ("read_mutated_v2", ReadQuery(version = Some(2L)), schema, mutatedRows), - ("read_restored_v3", ReadQuery(version = Some(3L)), schema, initialRows), - ("read_latest", ReadQuery(), schema, initialRows), - ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), - columns = Some(Seq("value", "id"))), projected, - Seq(Row(null, 4), Row(null, 4), Row("missing", null)))) - cases.foreach { case (name, query, expectedSchema, rows) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull, columns = query.columns).assert { spec => - assert(spec.query == query) - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == rows.size.toLong) - case other => fail(s"restore $name expected Succeeded, got $other") - } - val expected = spark.createDataFrame(rows.asJava, expectedSchema) - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), - query.predicate, query.columns) - assert(actual.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, actual, s"restore $name copied table") - val fixture = copiedTable.getParent - val specName = s"${fixture.getFileName}_$name" - val captured = spark.read.parquet( - fixture.resolve(s"expected/$specName/expected_data").toString) - assert(captured.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, captured, s"restore $name expected parquet") - } - } - } - test("e2e_56_minor_log_compaction") { - import org.apache.spark.sql.delta.actions.{Action => SparkAction, InMemoryLogReplay} - import org.apache.spark.sql.delta.util.FileNames - - sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( - 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', - 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-7, 'initial'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), - (CAST(NULL AS INT), 'missing'), (12, 'keep'), (12, 'keep') AS v(id, value)""") - sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (6, 'replacement'), (6, 'replacement'), (CAST(NULL AS INT), 'changed'), - (8, CAST(NULL AS STRING)) AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), (23, 'end') AS v(id, value)""") - sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (31, 'tail'), (31, 'tail'), (CAST(NULL AS INT), 'tail-null'), - (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)) AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (44, 'last'), (44, 'last') AS v(id, value)""") - val table = registerTable("tbl") - val original = (0L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head - val schema = new StructType().add("id", IntegerType).add("value", StringType) - val configuration = Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") - assert(original(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) - val adds = (1L to 5L).map { v => - v -> original(v).collect { case a: AddFile => a }.ensuring(_.size == 1).head - }.toMap - assert(adds.values.map(_.path).toSet.size == 5) - assert(original(2L).collect { case r: RemoveFile => r.path } == Seq(adds(1L).path)) - assert(original(4L).collect { case r: RemoveFile => r.path }.toSet == - Set(adds(2L).path, adds(3L).path)) - - // Follow the pinned OSS DeltaLogMinorCompactionSuite: reconcile [0, 3], with - // no retention pruning, then serialize the existing OSS checkpoint iterator. - // Concatenating commits would retain cancelled adds and commitInfo, which is invalid. - val replay = new InMemoryLogReplay(None, None) - (0L to 3L).foreach { v => - val lines = Files.readAllLines(table.sourcePath.resolve(f"_delta_log/$v%020d.json")) - replay.append(v, lines.asScala.iterator.map(SparkAction.fromJson)) - } - val sourceLog = DeltaLog.forTable(spark, table.sourcePath.toString) - val compactedPath = FileNames.compactedDeltaFile(sourceLog.logPath, 0L, 3L) - val compactedName = "00000000000000000000.00000000000000000003.compacted.json" - assert(compactedPath.getName == compactedName) - sourceLog.store.write(compactedPath, replay.checkpoint.map(_.json), - overwrite = false, sourceLog.newDeltaHadoopConf()) - val compactedBytes = Files.readAllBytes(table.sourcePath.resolve(s"_delta_log/$compactedName")) - val compacted = Files.readAllLines(table.sourcePath.resolve(s"_delta_log/$compactedName")) - .asScala.map(Action.parse).toSeq - assert(compacted.size == 5) - assert(compacted.collect { case p: Protocol => p } == Seq(Protocol(1, 2))) - assert(compacted.collect { case m: Metadata => m } == Seq(metadata)) - // OSS Action.fromJson materializes absent optional tags as an empty map. - // FileAction.tagsOrEmpty defines their equivalence; preserve every nonempty tag. - val expectedAdds = Set(adds(2L), adds(3L)).map(a => - a.copy(dataChange = false, tags = a.tags.filter(_.nonEmpty))) - assert(compacted.collect { case a: AddFile => a.copy(tags = a.tags.filter(_.nonEmpty)) } - .toSet == expectedAdds) - assert(compacted.collect { case r: RemoveFile => r.copy(tags = r.tags.filter(_.nonEmpty)) } == - original(2L).collect { case r: RemoveFile => - r.copy(dataChange = false, tags = r.tags.filter(_.nonEmpty)) }) - info("compact compacted [0,3] actions: " + compacted.map(_.toJson).mkString("\n")) - - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - assert(dir.getParent.getParent == outputDir) - assert(Files.readAllBytes(dir.resolve(s"_delta_log/$compactedName")) - .sameElements(compactedBytes)) - // Compactions are optional accelerators. OSS validates original commit continuity; - // neither compaction nor protocol cleanup authorizes removing this interval. - (0L to 5L).foreach { v => - assert(CommitLog.read(dir, v) == original(v)) - assert(Files.readAllBytes(dir.resolve(f"_delta_log/$v%020d.json")) - .sameElements(Files.readAllBytes(table.sourcePath.resolve(f"_delta_log/$v%020d.json")))) - } - adds.values.foreach { add => - assert(Files.readAllBytes(dir.resolve(add.path)) - .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) - } - val listing = Files.list(dir.resolve("_delta_log")) - try assert(!listing.iterator().asScala.exists(_.getFileName.toString.contains("checkpoint"))) - finally listing.close() - } - val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, - configuration, metadata.createdTime)) - Seq[java.lang.Long](0L, 1L, 2L, 3L, 4L, 5L, null).foreach { version => - val v = Option(version).fold(5L)(_.toLong) - snapshotSpec(table, version = version).assert { spec => - assert(spec.query == SnapshotQuery(version = Some(v))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) - val selected = snapshot.logSegment.deltas.map(_.getPath.getName) - val expected = if (v < 3L) (0L to v).map(n => f"$n%020d.json") - else Seq(compactedName) ++ (4L to v).map(n => f"$n%020d.json") - assert(selected == expected) - assert(snapshot.logSegment.checkpointProvider.version == -1L) - info(s"compact v$v selected log files: " + selected.mkString(",")) - } - } - val initialRows = Seq(Row(-7, "initial"), Row(4, null), Row(4, null), - Row(null, "missing"), Row(12, "keep"), Row(12, "keep")) - val replacementRows = Seq(Row(6, "replacement"), Row(6, "replacement"), - Row(null, "changed"), Row(8, null)) - val endRows = replacementRows ++ Seq(Row(4, null), Row(4, null), Row(23, "end")) - val tailRows = Seq(Row(31, "tail"), Row(31, "tail"), Row(null, "tail-null"), - Row(4, null), Row(4, null)) - val latestRows = tailRows ++ Seq(Row(44, "last"), Row(44, "last")) - val projected = new StructType().add("value", StringType).add("id", IntegerType) - val cases = Seq( - ("read_start_v0", ReadQuery(version = Some(0L)), schema, Seq.empty[Row]), - ("read_inside_v1", ReadQuery(version = Some(1L)), schema, initialRows), - ("read_inside_v2", ReadQuery(version = Some(2L)), schema, replacementRows), - ("read_end_v3", ReadQuery(version = Some(3L)), schema, endRows), - ("read_tail_v4", ReadQuery(version = Some(4L)), schema, tailRows), - ("read_tail_v5", ReadQuery(version = Some(5L)), schema, latestRows), - ("read_latest", ReadQuery(), schema, latestRows), - ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), - columns = Some(Seq("value", "id"))), projected, - Seq(Row(null, 4), Row(null, 4), Row("tail-null", null)))) - cases.foreach { case (name, query, expectedSchema, rows) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull, columns = query.columns).assert { spec => - assert(spec.query == query) - spec.expectation match { - case Succeeded(result) => assert(result.rowCount == rows.size.toLong) - case other => fail(s"compact $name expected Succeeded, got $other") - } - val expected = spark.createDataFrame(rows.asJava, expectedSchema) - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), - query.predicate, query.columns) - assert(actual.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, actual, s"compact $name copied table") - val fixture = copiedTable.getParent - val specName = s"${fixture.getFileName}_$name" - val captured = spark.read.parquet( - fixture.resolve(s"expected/$specName/expected_data").toString) - assert(captured.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, captured, s"compact $name expected parquet") - } - } - } - test("e2e_57_auto_compaction") { - // These settings affect fixture generation only. Ordinary SQL inserts invoke - // OSS AutoCompact's post-commit hook; no explicit compaction command is called. - val settings = Map( - "spark.databricks.delta.autoCompact.enabled" -> "true", - "spark.databricks.delta.autoCompact.minNumFiles" -> "3", - "spark.databricks.delta.autoCompact.minFileSize" -> "1048576", - "spark.databricks.delta.autoCompact.maxFileSize" -> "2097152") - val saved = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap - try { - settings.foreach { case (key, value) => sql(s"SET $key=$value") } - sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( - 'delta.autoOptimize.autoCompact' = 'true', 'delta.enableDeletionVectors' = 'false', - 'delta.checkpointInterval' = '1000', 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-7, 'initial'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), - (CAST(NULL AS INT), 'missing') AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (12, 'keep'), (12, 'keep'), (23, 'end') AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (44, 'last'), (44, 'last'), (CAST(NULL AS INT), CAST(NULL AS STRING)), - (88, 'edge') AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (99, 'tail'), (99, 'tail') AS v(id, value)""") - } finally { - saved.foreach { - case (key, Some(value)) => spark.conf.set(key, value) - case (key, None) => spark.conf.unset(key) - } - } - val table = registerTable("tbl") - val commits = (0L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap - assert(!Files.exists(CommitLog.commitFile(table.sourcePath, 6L))) - val inputAdds = (1L to 3L).flatMap { v => - val adds = commits(v).collect { case a: AddFile => a } - assert(adds.size == 1 && adds.head.dataChange) - assert(adds.head.size > 0L && adds.head.size < 1048576L) - assert(commits(v).collect { case r: RemoveFile => r }.isEmpty) - adds - } - val rewritten = commits(4L).collect { case a: AddFile => a } - val removed = commits(4L).collect { case r: RemoveFile => r } - assert(rewritten.size == 1 && !rewritten.head.dataChange) - assert(removed.size == 3 && removed.forall(r => !r.dataChange)) - assert(removed.map(_.path).toSet == inputAdds.map(_.path).toSet) - assert(!inputAdds.exists(_.path == rewritten.head.path)) - val tail = commits(5L).collect { case a: AddFile => a } - assert(tail.size == 1 && tail.head.dataChange) - assert(commits(5L).collect { case r: RemoveFile => r }.isEmpty) - val history = (0L to 5L).map { v => - val info = commits(v).map(a => JsonUtil.mapper.readTree(a.toJson)) - .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") - v -> info - }.toMap - assert(history(0L).get("operation").asText() == "CREATE TABLE") - Seq(1L, 2L, 3L, 5L).foreach { v => - assert(history(v).get("operation").asText() == "WRITE") - } - assert(history(4L).get("operation").asText() == "OPTIMIZE") - assert(history(4L).get("operationParameters").get("auto").asText() == "true") - info("auto automatic compaction commit v4: " + history(4L).toString) - - val schema = new StructType().add("id", IntegerType).add("value", StringType) - val metadata = commits(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head - val configuration = Map("delta.autoOptimize.autoCompact" -> "true", - "delta.enableDeletionVectors" -> "false", "delta.checkpointInterval" -> "1000", - "delta.checkpointPolicy" -> "classic") - assert(commits(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) - val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, - configuration, metadata.createdTime)) - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - assert(dir.getParent.getParent == outputDir) - (0L to 5L).foreach { v => - assert(CommitLog.read(dir, v) == commits(v)) - assert(Files.readAllBytes(CommitLog.commitFile(dir, v)) - .sameElements(Files.readAllBytes(CommitLog.commitFile(table.sourcePath, v)))) - } - (inputAdds ++ rewritten ++ tail).foreach { add => - assert(Files.readAllBytes(dir.resolve(add.path)) - .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) - } - val logs = Files.list(dir.resolve("_delta_log")) - try assert(!logs.iterator().asScala.exists(_.getFileName.toString.contains("checkpoint"))) - finally logs.close() - } - val activePaths = Map(0L -> Set.empty[String], 1L -> Set(inputAdds(0).path), - 2L -> inputAdds.take(2).map(_.path).toSet, 3L -> inputAdds.map(_.path).toSet, - 4L -> Set(rewritten.head.path), 5L -> Set(rewritten.head.path, tail.head.path)) - Seq[java.lang.Long](0L, 1L, 2L, 3L, 4L, 5L, null).foreach { version => - val v = Option(version).fold(5L)(_.toLong) - snapshotSpec(table, version = version).assert { spec => - assert(spec.query == SnapshotQuery(version = Some(v))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) - assert(snapshot.allFiles.collect().map(_.path).toSet == activePaths(v)) - assert(snapshot.logSegment.checkpointProvider.version == -1L) - } - } - // Literal, duplicate-sensitive bags are independent of captured expectations. - val firstRows = Seq(Row(-7, "initial"), Row(4, null), Row(4, null), Row(null, "missing")) - val secondRows = firstRows ++ Seq(Row(12, "keep"), Row(12, "keep"), Row(23, "end")) - val beforeRows = secondRows ++ Seq(Row(44, "last"), Row(44, "last"), - Row(null, null), Row(88, "edge")) - val latestRows = beforeRows ++ Seq(Row(99, "tail"), Row(99, "tail")) - val projected = new StructType().add("value", StringType).add("id", IntegerType) - val cases = Seq( - ("read_first_v1", ReadQuery(version = Some(1L)), schema, firstRows, 1, 0L), - ("read_below_threshold_v2", ReadQuery(version = Some(2L)), schema, secondRows, 2, 0L), - ("read_before_v3", ReadQuery(version = Some(3L)), schema, beforeRows, 3, 0L), - ("read_post_v4", ReadQuery(version = Some(4L)), schema, beforeRows, 1, 0L), - ("read_tail_v5", ReadQuery(version = Some(5L)), schema, latestRows, 2, 0L), - ("read_latest", ReadQuery(), schema, latestRows, 2, 0L), - ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), - columns = Some(Seq("value", "id"))), projected, - Seq(Row(null, 4), Row(null, 4), Row("missing", null), Row(null, null)), 1, 1L)) - cases.foreach { case (name, query, expectedSchema, rows, fileCount, skipped) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull, columns = query.columns).assert { spec => - assert(spec.query == query) - assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, fileCount, skipped))) - val expected = spark.createDataFrame(rows.asJava, expectedSchema) - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), - query.predicate, query.columns) - assert(actual.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, actual, s"auto $name copied table") - val fixture = copiedTable.getParent - val specName = s"${fixture.getFileName}_$name" - val captured = spark.read.parquet( - fixture.resolve(s"expected/$specName/expected_data").toString) - assert(captured.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, captured, s"auto $name expected parquet") - } - } - } - - test("e2e_58_unicode_路径") { - // The generator copies contents into /delta. Relative log paths must - // survive both the Unicode source root and that different destination root. - val parent = Files.createTempDirectory("paths-source-") - val source = parent.resolve("源_café_表_😀") - sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta LOCATION '$source' - TBLPROPERTIES ('delta.enableDeletionVectors' = 'false', - 'delta.checkpointInterval' = '1000', 'delta.checkpointPolicy' = 'classic')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (-7, 'base'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), - (CAST(NULL AS INT), 'missing') AS v(id, value)""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (4, CAST(NULL AS STRING)), (23, 'end'), - (CAST(NULL AS INT), CAST(NULL AS STRING)) AS v(id, value)""") - // Canonically equivalent NFC/NFD basenames must remain distinct byte names. - val basenames = Seq("données_café_表_😀.parquet", "données_cafe\u0301_表_😀.parquet") - assert(basenames.distinct.size == 2) - val schema = new StructType().add("id", IntegerType).add("value", StringType) - val first = Seq(Row(-7, "base"), Row(4, null), Row(4, null), Row(null, "missing")) - val second = Seq(Row(4, null), Row(23, "end"), Row(null, null)) - val inputRows = Seq(first, second) - val renamedAdds = (1L to 2L).map { v => - val add = CommitLog.read(source, v).collect { case a: AddFile => a } - .ensuring(_.size == 1).head - assert(!add.path.contains("/") && add.deletionVector.isEmpty) - val basename = basenames((v - 1).toInt) - val file = source.resolve(basename) - Files.move(source.resolve(add.path), file) - // Change only the file reference in raw JSON; preserve every other action field. - val commit = CommitLog.commitFile(source, v) - val lines = Files.readAllLines(commit).asScala.map { line => - val node = JsonUtil.mapper.readTree(line) - if (node.has("add")) { - node.get("add").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] - .put("path", new HadoopPath(basename).toUri.toString) - } - node.toString - } - Files.write(commit, lines.asJava) - val physical = spark.read.parquet(file.toString) - assert(physical.schema == schema) - RowComparison.assertRowsEqual(spark.createDataFrame(inputRows((v - 1).toInt).asJava, - schema), physical, s"paths source physical v$v") - val renamed = CommitLog.read(source, v).collect { case a: AddFile => a }.head - assert(new HadoopPath(renamed.path).toUri.getPath == basename) - assert(Files.size(file) == renamed.size) - renamed - } - // Version checksums can embed the old paths; do not transport stale checksums. - val logs = Files.list(source.resolve("_delta_log")) - try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) - .foreach(Files.delete) - finally logs.close() - val commits = (0L to 2L).map(v => v -> Files.readAllBytes( - CommitLog.commitFile(source, v))).toMap - val metadata = CommitLog.read(source, 0L).collect { case m: Metadata => m } - .ensuring(_.size == 1).head - val configuration = Map("delta.enableDeletionVectors" -> "false", - "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") - assert(CommitLog.read(source, 0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) - assert(DataType.fromJson(metadata.schemaString) == schema) - assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) - val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), - MetadataInfo(metadata.id, metadata.name, metadata.description, - HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, - configuration, metadata.createdTime)) - val table = registerTableFromPath(source.toString) - var copiedTable: Path = null - mutateTable(table) { dir => - copiedTable = dir - assert(dir != source && dir.getParent.getFileName.toString.endsWith("路径")) - commits.foreach { case (v, bytes) => - assert(Files.readAllBytes(CommitLog.commitFile(dir, v)).sameElements(bytes)) - assert(!new String(bytes, java.nio.charset.StandardCharsets.UTF_8) - .contains(source.toString)) - } - basenames.foreach { basename => - val file = dir.resolve(basename) - assert(file.getFileName.toString.getBytes(java.nio.charset.StandardCharsets.UTF_8) - .sameElements(basename.getBytes(java.nio.charset.StandardCharsets.UTF_8))) - assert(Files.readAllBytes(file).sameElements(Files.readAllBytes(source.resolve(basename)))) - } - // Remove the entire source before any snapshot/read capture. An absolute or - // stale reference to it cannot make the relocated fixture pass by accident. - org.apache.commons.io.FileUtils.deleteDirectory(parent.toFile) - assert(!Files.exists(parent) && !Files.exists(source)) - DeltaLog.clearCache() - info(s"paths source removed: $source; relocated root: $dir") - } - Seq[java.lang.Long](0L, 1L, 2L, null).foreach { version => - val v = Option(version).fold(2L)(_.toLong) - snapshotSpec(table, version = version).assert { spec => - assert(!Files.exists(source)) - assert(spec.query == SnapshotQuery(version = Some(v))) - assert(spec.expectation == Succeeded(expectedSnapshot)) - val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) - assert(snapshot.allFiles.collect().map(_.path).toSet == - renamedAdds.take(v.toInt).map(_.path).toSet) - assert(snapshot.logSegment.checkpointProvider.version == -1L) - } - } - val projected = new StructType().add("value", StringType).add("id", IntegerType) - val cases = Seq( - ("read_v1", ReadQuery(version = Some(1L)), schema, first, 1), - ("read_v2", ReadQuery(version = Some(2L)), schema, first ++ second, 2), - ("read_latest", ReadQuery(), schema, first ++ second, 2), - ("read_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), - columns = Some(Seq("value", "id"))), projected, - Seq(Row(null, 4), Row(null, 4), Row(null, 4), Row("missing", null), Row(null, null)), 2)) - cases.foreach { case (name, query, expectedSchema, rows, files) => - readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, - predicate = query.predicate.orNull, columns = query.columns).assert { spec => - assert(!Files.exists(source)) - assert(spec.query == query) - assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, files, 0L))) - val actual = SnapshotResolver.applyFilters( - SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), - query.predicate, query.columns) - val paths = actual.inputFiles.map(p => Paths.get(java.net.URI.create(p))).toSet - val expectedFiles = basenames.take(files).map(copiedTable.resolve).toSet - assert(paths == expectedFiles && paths.forall(_.startsWith(copiedTable))) - assert(actual.schema == expectedSchema) - val expected = spark.createDataFrame(rows.asJava, expectedSchema) - RowComparison.assertRowsEqual(expected, actual, s"paths $name relocated table") - val fixture = copiedTable.getParent - val captured = spark.read.parquet(fixture.resolve( - s"expected/${fixture.getFileName}_$name/expected_data").toString) - assert(captured.schema == expectedSchema) - RowComparison.assertRowsEqual(expected, captured, s"paths $name expected parquet") - } - } - } -} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTimestampSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTimestampSuite.scala deleted file mode 100644 index 768c30d9..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTimestampSuite.scala +++ /dev/null @@ -1,1942 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.tables - -import java.nio.{ByteBuffer, ByteOrder} -import java.nio.file.{Files, Path, StandardCopyOption} -import java.sql.Timestamp -import java.time.{Instant, LocalDate} - -import scala.jdk.CollectionConverters._ - -import org.apache.commons.io.FileUtils -import org.apache.hadoop.fs.{Path => HadoopPath} -import org.apache.parquet.example.data.Group -import org.apache.parquet.example.data.simple.{NanoTime, SimpleGroupFactory} -import org.apache.parquet.hadoop.{ParquetFileReader, ParquetReader} -import org.apache.parquet.hadoop.example.{ExampleParquetWriter, GroupReadSupport} -import org.apache.parquet.hadoop.util.HadoopInputFile -import org.apache.parquet.schema.{LogicalTypeAnnotation, PrimitiveType} -import org.apache.spark.sql.Row -import org.apache.spark.sql.functions.{col, lit, when} -import org.apache.spark.sql.types.{DataType, IntegerType, StructType, TimestampNTZType, - TimestampType} - -import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} -import io.delta.workload.engine.RowComparison -import io.delta.workload.json.JsonUtil -import io.delta.workload.log.{AddFile, CommitLog} -import io.delta.workload.model.Succeeded - -/** Timestamp stats encodings are checked on disk, independently of the table reader. */ -class ReadinessTimestampSuite extends WorkloadTestSuite("readiness_timestamp") { - private val first = "2024-01-01T00:00:00.123456Z" - private val last = "2024-01-01T00:00:00.123789Z" - private val nanoLower = "2024-01-01T00:00:00.123455111Z" - private val nanoUpper = "2024-01-01T00:00:00.123789999Z" - private val schema = new StructType().add("id", IntegerType).add("ts", TimestampType) - - // Literal instants, not values read back from the table, are the row oracle. - private def expectedRows: Seq[Row] = Seq( - Row(1, Timestamp.from(Instant.parse(first))), - Row(2, Timestamp.from(Instant.parse(last))), - Row(2, Timestamp.from(Instant.parse(last)))) - - private def createTable(): TableHandle = { - sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP) USING delta TBLPROPERTIES ( - 'delta.minReaderVersion' = '1', 'delta.minWriterVersion' = '2', - 'delta.checkpointPolicy' = 'classic', - 'delta.checkpoint.writeStatsAsStruct' = 'true', - 'delta.checkpoint.writeStatsAsJson' = 'false')""") - sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES - (1, TIMESTAMP '2024-01-01 00:00:00.123456'), - (2, TIMESTAMP '2024-01-01 00:00:00.123789'), - (2, TIMESTAMP '2024-01-01 00:00:00.123789') AS rows(id, ts)""") - val t = registerTable("tbl") - val adds = CommitLog.read(t.sourcePath, 1).collect { case a: AddFile => a } - require(adds.size == 1, "The timestamp bounds must describe one file with all three rows") - t - } - - private def checkSpecs(t: TableHandle, checkJsonStats: Boolean = false): Unit = { - val workloadName = WorkloadContext.current.workloadName - val tableOutputDir = outputDir.resolve(workloadName) - val queries = Seq( - ("all", null, expectedRows), - ("lower_edge", "ts <= TIMESTAMP '2024-01-01 00:00:00.123456'", expectedRows.take(1)), - ("upper_edge", "ts >= TIMESTAMP '2024-01-01 00:00:00.123789'", expectedRows.drop(1)), - ("outside", "ts < TIMESTAMP '2024-01-01 00:00:00.123456' OR " + - "ts > TIMESTAMP '2024-01-01 00:00:00.123789'", Seq.empty[Row])) - - for (version <- Seq[java.lang.Long](null, 1L)) { - val suffix = if (version == null) "latest" else "v1" - for ((label, predicate, rows) <- queries) { - val name = s"${label}_$suffix" - readSpec(t, predicate = predicate, version = version, name = Some(name)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == rows.size) - case other => fail(s"$name must succeed: $other") - } - val expected = spark.createDataFrame(rows.asJava, schema) - val expectedDir = tableOutputDir.resolve("expected") - .resolve(s"${workloadName}_$name") - val captured = expectedDir.resolve("expected_data") - RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) - if (checkJsonStats && label == "all") { - val scanned = spark.read.parquet( - expectedDir.resolve("expected_metadata").toString).collect() - require(scanned.length == 1) - val scanStats = JsonUtil.mapper.readTree(scanned.head.getString(0)) - .at("/add/stats").asText() - val tableDir = tableOutputDir.resolve("delta") - val rawStats = CommitLog.read(tableDir, 1) - .collectFirst { case a: AddFile => a.stats.get }.get - require(scanStats == rawStats, "Scanned stats must retain the injected precision") - println(s"Verified $name scanned AddFile stats match nanosecond JSON") - } - } - } - snapshotSpec(t, version = version).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == schema) - case other => fail(s"$suffix snapshot must succeed: $other") - } - } - } - } - - test("nanosecond_stats_json") { - val t = createTable() - // The bounds enclose the microsecond rows. All of digits 7-9 are nonzero. - val lower = nanoLower - val upper = nanoUpper - require(Instant.parse(lower).isBefore(Instant.parse(first))) - require(Instant.parse(upper).isAfter(Instant.parse(last))) - modifyCommitActions(t, version = 1) { actions => - actions.map { - case a: AddFile => - val stats = JsonUtil.mapper.readTree(a.stats.get) - stats.get("minValues").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] - .put("ts", lower) - stats.get("maxValues").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] - .put("ts", upper) - a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) - case other => other - } - } - mutateTable(t) { tableDir => - val adds = CommitLog.read(tableDir, 1).collect { case a: AddFile => a } - require(adds.size == 1) - val raw = JsonUtil.mapper.readTree(adds.head.stats.get) - require(raw.at("/minValues/ts").asText() == lower) - require(raw.at("/maxValues/ts").asText() == upper) - require(raw.get("numRecords").asLong() == 3) - // A copied checksum can contain the old AddFile stats and bypass JSON parsing. - val files = Files.list(tableDir.resolve("_delta_log")) - try files.iterator().asScala.filter(_.toString.endsWith(".crc")) - .foreach(Files.delete(_)) finally files.close() - println(s"Verified JSON stats: min=$lower max=$upper records=3") - } - checkSpecs(t, checkJsonStats = true) - } - - for ((caseName, encoding, lower, upper) <- Seq( - ("millisecond_stats_checkpoint", "TIMESTAMP_MILLIS", - "2024-01-01T00:00:00.123Z", "2024-01-01T00:00:00.124Z"), - ("microsecond_stats_checkpoint", "TIMESTAMP_MICROS", first, last), - ("int96_stats_checkpoint", "INT96", first, last))) { - test(caseName) { - val t = createTable() - require(!Instant.parse(lower).isAfter(Instant.parse(first))) - require(!Instant.parse(upper).isBefore(Instant.parse(last))) - forceCheckpoint("tbl") - mutateTable(t) { tableDir => - val logDir = tableDir.resolve("_delta_log") - val checkpoint = logDir.resolve("00000000000000000001.checkpoint.parquet") - require(Files.isRegularFile(checkpoint), "Expected one classic checkpoint") - val rewriteDir = logDir.resolve("timestamp-checkpoint-rewrite") - val confKey = "spark.sql.parquet.outputTimestampType" - val previous = spark.conf.get(confKey) - try { - spark.conf.set(confKey, encoding) - val df = spark.read.parquet(checkpoint.toString) - val stats = col("add.stats_parsed") - .withField("minValues", col("add.stats_parsed.minValues") - .withField("ts", lit(Timestamp.from(Instant.parse(lower))))) - .withField("maxValues", col("add.stats_parsed.maxValues") - .withField("ts", lit(Timestamp.from(Instant.parse(upper))))) - // Disable the JSON fallback: these reads must consume parsed checkpoint stats. - var add = col("add").withField("stats_parsed", stats) - if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { - add = add.withField("stats", lit(null).cast("string")) - } - df.withColumn("add", add).coalesce(1).write.parquet(rewriteDir.toString) - val files = Files.list(rewriteDir) - val part = try { - val parts = files.iterator().asScala.filter(_.toString.endsWith(".parquet")).toSeq - require(parts.size == 1) - parts.head - } finally files.close() - Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) - } finally { - spark.conf.set(confKey, previous) - if (Files.exists(rewriteDir)) FileUtils.deleteDirectory(rewriteDir.toFile) - } - checkCheckpoint(checkpoint, encoding, lower, upper) - if (encoding == "INT96") { - // Spark's writer only emits microsecond-aligned INT96. Preserve every checkpoint - // field with the existing Parquet Group API, replacing just the two stats cells. - rewriteInt96Bounds(checkpoint) - checkCheckpoint(checkpoint, encoding, nanoLower, nanoUpper) - } - // Retain v1 JSON as a discovery/metadata tail alongside the checkpoint. Removing v0 - // prevents full JSON replay, so v1 and latest require checkpoint bootstrap. - val files = Files.list(logDir) - try files.iterator().asScala.filter { p => - p.getFileName.toString == "00000000000000000000.json" || - p.toString.endsWith(".crc") - }.foreach(Files.delete(_)) finally files.close() - } - checkSpecs(t) - } - } - - private def rewriteInt96Bounds(checkpoint: Path): Unit = { - import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ - - val conf = spark.sparkContext.hadoopConfiguration - val path = new HadoopPath(checkpoint.toUri) - val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) - val metadata = try footer.getFooter.getFileMetaData finally footer.close() - val parquetSchema = metadata.getSchema - val factory = new SimpleGroupFactory(parquetSchema) - // 2024-01-01 is Julian day 2460311. These nanoseconds after midnight enclose the rows. - val replacements = Map( - "add.stats_parsed.minValues.ts" -> new NanoTime(2460311, 123455111L).toBinary, - "add.stats_parsed.maxValues.ts" -> new NanoTime(2460311, 123789999L).toBinary) - - def copyGroup(source: Group, target: Group, prefix: String): Unit = { - for (i <- 0 until source.getType.getFieldCount) { - val field = source.getType.getType(i) - val fieldPath = if (prefix.isEmpty) field.getName else s"$prefix.${field.getName}" - for (j <- 0 until source.getFieldRepetitionCount(i)) { - if (!field.isPrimitive) { - copyGroup(source.getGroup(i, j), target.addGroup(i), fieldPath) - } else field.asPrimitiveType().getPrimitiveTypeName match { - case BOOLEAN => target.add(i, source.getBoolean(i, j)) - case INT32 => target.add(i, source.getInteger(i, j)) - case INT64 => target.add(i, source.getLong(i, j)) - case FLOAT => target.add(i, source.getFloat(i, j)) - case DOUBLE => target.add(i, source.getDouble(i, j)) - case BINARY | FIXED_LEN_BYTE_ARRAY => target.add(i, source.getBinary(i, j)) - case INT96 => target.add(i, replacements.getOrElse(fieldPath, source.getInt96(i, j))) - } - } - } - } - - val temporary = checkpoint.resolveSibling("timestamp-int96-rewrite.parquet") - try { - val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() - try { - val writer = ExampleParquetWriter.builder(new HadoopPath(temporary.toUri)) - .withConf(conf).withType(parquetSchema) - .withExtraMetaData(metadata.getKeyValueMetaData).build() - try { - var source = reader.read() - while (source != null) { - val target = factory.newGroup() - copyGroup(source, target, "") - writer.write(target) - source = reader.read() - } - } finally writer.close() - } finally reader.close() - Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING) - } finally Files.deleteIfExists(temporary) - } - - private def checkCheckpoint( - checkpoint: Path, encoding: String, lower: String, upper: String): Unit = { - val conf = spark.sparkContext.hadoopConfiguration - val path = new HadoopPath(checkpoint.toUri) - val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) - try { - val parquetSchema = footer.getFooter.getFileMetaData.getSchema - for (bound <- Seq("minValues", "maxValues")) { - val physical = parquetSchema.getType("add", "stats_parsed", bound, "ts") - .asPrimitiveType() - if (encoding == "INT96") { - require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT96) - require(physical.getLogicalTypeAnnotation == null) - } else { - require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) - val annotation = physical.getLogicalTypeAnnotation - .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] - val unit = if (encoding == "TIMESTAMP_MILLIS") { - LogicalTypeAnnotation.TimeUnit.MILLIS - } else LogicalTypeAnnotation.TimeUnit.MICROS - require(annotation.getUnit == unit) - require(annotation.isAdjustedToUTC) - } - println(s"Verified checkpoint add.stats_parsed.$bound.ts: $physical") - } - } finally footer.close() - - // Read the physical cells without Spark's timestamp conversion or Delta's stats parser. - val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() - var adds = 0 - try { - var row = reader.read() - while (row != null) { - if (row.getFieldRepetitionCount("add") > 0) { - adds += 1 - val add = row.getGroup("add", 0) - if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) - val stats = add.getGroup("stats_parsed", 0) - require(stats.getLong("numRecords", 0) == 3) - for ((bound, value) <- Seq("minValues" -> lower, "maxValues" -> upper)) { - val cell = stats.getGroup(bound, 0) - val instant = Instant.parse(value) - if (encoding == "INT96") { - val bytes = ByteBuffer.wrap(cell.getInt96("ts", 0).getBytes) - .order(ByteOrder.LITTLE_ENDIAN) - val nanosOfDay = bytes.getLong() - val julianDay = bytes.getInt() - require(julianDay == 2440588 + Math.floorDiv(instant.getEpochSecond, 86400L)) - require(nanosOfDay == Math.floorMod(instant.getEpochSecond, 86400L) * - 1000000000L + instant.getNano) - // Check the aligned intermediate separately from the true nanos replacement. - require(nanosOfDay % 1000 == instant.getNano % 1000) - require(nanosOfDay % 1000000 != 0) - println(s"Verified raw INT96 $bound: day=$julianDay nanosOfDay=$nanosOfDay") - } else { - val units = if (encoding == "TIMESTAMP_MILLIS") 1000L else 1000000L - val expected = instant.getEpochSecond * units + - instant.getNano / (1000000000L / units) - require(cell.getLong("ts", 0) == expected) - println(s"Verified raw $encoding $bound: $expected") - } - } - } - row = reader.read() - } - } finally reader.close() - require(adds == 1) - } - - // Each data boundary is crossed with every temporal partition boundary. - // SQL fixture literals and the typed row oracle are deliberately declared separately. - private val epochValues = Seq( - (1, "1969-12-31T23:59:59.998999", "1969-12-31", -1001L), - (2, "1969-12-31T23:59:59.999000", "1969-12-31", -1000L), - (3, "1969-12-31T23:59:59.999001", "1969-12-31", -999L), - (4, "1969-12-31T23:59:59.999999", "1969-12-31", -1L), - (5, "1970-01-01T00:00:00.000000", "1970-01-01", 0L), - (6, "1970-01-01T00:00:00.000001", "1970-01-01", 1L), - (7, "1970-01-01T00:00:00.001000", "1970-01-02", 1000L)) - - private val epochSchema = new StructType().add("id", IntegerType) - .add("ts", TimestampType).add("ntz", org.apache.spark.sql.types.TimestampNTZType) - .add("d", org.apache.spark.sql.types.DateType).add("p_ts", TimestampType) - .add("p_ntz", org.apache.spark.sql.types.TimestampNTZType) - .add("p_d", org.apache.spark.sql.types.DateType) - - private def epochRows: Seq[Row] = { - val data = epochValues ++ epochValues.filter(_._1 == 4) - for ((id, wall, day, _) <- data; (_, pWall, pDay, _) <- epochValues) yield { - Row(id, Timestamp.from(Instant.parse(wall + "Z")), java.time.LocalDateTime.parse(wall), - java.sql.Date.valueOf(day), Timestamp.from(Instant.parse(pWall + "Z")), - java.time.LocalDateTime.parse(pWall), java.sql.Date.valueOf(pDay)) - } - } - - private def epochPartitionStrings: Set[Map[String, String]] = epochValues.map { - case (_, wall, day, _) => - // NTZ uses wall-clock strings; TIMESTAMP uses UTC ISO with six fractional digits. - val partition = wall.replace('T', ' ').replaceAll("0+$", "").stripSuffix(".") - Map("p_ts" -> (wall + "Z"), "p_ntz" -> partition, "p_d" -> day) - }.toSet - - private def createEpochTable(): TableHandle = { - require(spark.conf.get("spark.sql.session.timeZone") == "UTC") - sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ, d DATE, - p_ts TIMESTAMP, p_ntz TIMESTAMP_NTZ, p_d DATE) USING delta - PARTITIONED BY (p_ts, p_ntz, p_d) TBLPROPERTIES ( - 'delta.checkpointPolicy' = 'classic', - 'delta.checkpoint.writeStatsAsStruct' = 'true', - 'delta.checkpoint.writeStatsAsJson' = 'false')""") - val fixtures = Seq( - (1, "1969-12-31 23:59:59.998999", "1969-12-31"), - (2, "1969-12-31 23:59:59.999", "1969-12-31"), - (3, "1969-12-31 23:59:59.999001", "1969-12-31"), - (4, "1969-12-31 23:59:59.999999", "1969-12-31"), - (5, "1970-01-01 00:00:00", "1970-01-01"), - (6, "1970-01-01 00:00:00.000001", "1970-01-01"), - (7, "1970-01-01 00:00:00.001", "1970-01-02")) - val partitions = fixtures.map { case (_, ts, day) => - s"(TIMESTAMP '$ts', TIMESTAMP_NTZ '$ts', DATE '$day')" - }.mkString(",") - for ((id, ts, day) <- fixtures) { - val data = s"($id, TIMESTAMP '$ts', TIMESTAMP_NTZ '$ts', DATE '$day')" - val rows = if (id == 4) s"$data,$data" else data - sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM - (VALUES $rows) AS data(id, ts, ntz, d) CROSS JOIN - (VALUES $partitions) AS parts(p_ts, p_ntz, p_d)""") - } - val t = registerTable("tbl") - for (v <- 1L to 7L) { - val adds = CommitLog.read(t.sourcePath, v).collect { case a: AddFile => a } - require(adds.size == 7, "Each boundary must have one file per partition") - require(adds.map(_.partitionValues).toSet == epochPartitionStrings) - } - t - } - - for (parsed <- Seq(false, true)) { - val name = if (parsed) "negative_epoch_parsed_stats_partitions" - else "negative_epoch_json_stats_partitions" - test(name) { - val t = createEpochTable() - if (parsed) forceCheckpoint("tbl") - if (!parsed) { - for ((id, wall, day, _) <- epochValues) { - modifyCommitActions(t, version = id.toLong) { actions => - actions.map { - case a: AddFile => - val stats = JsonUtil.mapper.readTree(a.stats.get) - for (bound <- Seq("minValues", "maxValues")) { - val values = stats.get(bound) - .asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] - values.put("ts", wall + "Z") - values.put("ntz", wall) - values.put("d", day) - } - a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) - case other => other - } - } - } - } - mutateTable(t) { dir => - if (parsed) { - val checkpoint = dir.resolve("_delta_log/00000000000000000007.checkpoint.parquet") - val rewrite = dir.resolve("_delta_log/epoch-checkpoint-rewrite") - val confKey = "spark.sql.parquet.outputTimestampType" - val previous = spark.conf.get(confKey) - try { - spark.conf.set(confKey, "TIMESTAMP_MICROS") - val df = spark.read.parquet(checkpoint.toString) - var stats = col("add.stats_parsed") - for (bound <- Seq("minValues", "maxValues")) { - var ts = col(s"add.stats_parsed.$bound.ts") - var ntz = col(s"add.stats_parsed.$bound.ntz") - for ((id, wall, _, _) <- epochValues) { - val matches = col("add.stats_parsed.minValues.id") === lit(id) - ts = org.apache.spark.sql.functions.when(matches, - lit(Timestamp.from(Instant.parse(wall + "Z")))).otherwise(ts) - ntz = org.apache.spark.sql.functions.when(matches, - lit(java.time.LocalDateTime.parse(wall))).otherwise(ntz) - } - stats = stats.withField(bound, col(s"add.stats_parsed.$bound") - .withField("ts", ts).withField("ntz", ntz)) - } - var add = col("add").withField("stats_parsed", stats) - if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { - add = add.withField("stats", lit(null).cast("string")) - } - df.withColumn("add", add).coalesce(1).write.parquet(rewrite.toString) - val files = Files.list(rewrite) - val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) - .toSeq match { - case Seq(single) => single - case other => fail(s"Expected one rewritten checkpoint: $other") - } finally files.close() - Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) - } finally { - spark.conf.set(confKey, previous) - if (Files.exists(rewrite)) FileUtils.deleteDirectory(rewrite.toFile) - } - checkEpochCheckpoint(checkpoint) - } else { - for ((id, wall, day, _) <- epochValues) { - val adds = CommitLog.read(dir, id.toLong).collect { case a: AddFile => a } - require(adds.size == 7) - require(adds.map(_.partitionValues).toSet == epochPartitionStrings) - for (a <- adds) { - val stats = JsonUtil.mapper.readTree(a.stats.get) - require(stats.get("numRecords").asLong() == (if (id == 4) 2 else 1)) - for (bound <- Seq("minValues", "maxValues")) { - require(stats.at(s"/$bound/ts").asText() == wall + "Z") - require(stats.at(s"/$bound/ntz").asText() == wall) - require(stats.at(s"/$bound/d").asText() == day) - require(stats.at(s"/$bound/id").asInt() == id) - } - } - } - println("Verified exact JSON stats and partition strings for all 49 boundary files") - } - val files = Files.list(dir.resolve("_delta_log")) - try files.iterator().asScala.filter { p => - val file = p.getFileName.toString - file.endsWith(".crc") || (parsed && file.endsWith(".json") && - file != "00000000000000000007.json") - }.foreach(Files.delete(_)) finally files.close() - // v7 JSON is retained only for same-version lookup. There is no replay tail after v7. - // v0-v6 are absent, so the checkpoint is necessary for both latest and version 7. - } - checkEpochSpecs(t, parsed) - } - } - - private def checkEpochCheckpoint(checkpoint: Path): Unit = { - val conf = spark.sparkContext.hadoopConfiguration - val path = new HadoopPath(checkpoint.toUri) - val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) - try { - val physical = footer.getFooter.getFileMetaData.getSchema - for ((prefix, fields) <- Seq( - "stats_parsed.minValues" -> Seq("ts", "ntz", "d"), - "stats_parsed.maxValues" -> Seq("ts", "ntz", "d"), - "partitionValues_parsed" -> Seq("p_ts", "p_ntz", "p_d")); field <- fields) { - val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) - .asPrimitiveType() - if (field.endsWith("d")) { - require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) - require(cell.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) - } else { - require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) - val annotation = cell.getLogicalTypeAnnotation - .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] - require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) - require(annotation.isAdjustedToUTC == field.endsWith("ts")) - } - println(s"Verified near-epoch checkpoint $prefix.$field: $cell") - } - } finally footer.close() - val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() - var pairs = Vector.empty[(Int, Map[String, String])] - try { - var row = reader.read() - while (row != null) { - if (row.getFieldRepetitionCount("add") > 0) { - val add = row.getGroup("add", 0) - if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) - val stats = add.getGroup("stats_parsed", 0) - val id = stats.getGroup("minValues", 0).getInteger("id", 0) - val (_, _, day, micros) = epochValues.find(_._1 == id).get - require(stats.getLong("numRecords", 0) == (if (id == 4) 2 else 1)) - for (bound <- Seq("minValues", "maxValues")) { - val values = stats.getGroup(bound, 0) - require(values.getInteger("id", 0) == id) - require(values.getLong("ts", 0) == micros) - require(values.getLong("ntz", 0) == micros) - require(values.getInteger("d", 0) == java.time.LocalDate.parse(day).toEpochDay) - } - val raw = add.getGroup("partitionValues", 0) - val partition = (0 until raw.getFieldRepetitionCount("key_value")).map { i => - val entry = raw.getGroup("key_value", i) - entry.getString("key", 0) -> entry.getString("value", 0) - }.toMap - require(epochPartitionStrings.contains(partition)) - val parsed = add.getGroup("partitionValues_parsed", 0) - val (_, _, pDay, pMicros) = epochValues.find { case (_, wall, _, _) => - java.time.LocalDateTime.parse(wall) == - java.time.LocalDateTime.parse(partition("p_ntz").replace(' ', 'T')) - }.get - require(parsed.getLong("p_ts", 0) == pMicros) - require(parsed.getLong("p_ntz", 0) == pMicros) - require(parsed.getInteger("p_d", 0) == java.time.LocalDate.parse(pDay).toEpochDay) - pairs :+= id -> partition - } - row = reader.read() - } - } finally reader.close() - val expectedPairs = for ((id, _, _, _) <- epochValues; p <- epochPartitionStrings) yield id -> p - require(pairs.size == 49 && pairs.toSet == expectedPairs.toSet) - println("Verified 49 physical stats/partition pairs: micros -1001,-1000,-999,-1,0,1,1000") - } - - private def checkEpochSpecs(t: TableHandle, parsed: Boolean): Unit = { - val workload = WorkloadContext.current.workloadName - val tableDir = outputDir.resolve(workload) - val all = epochRows - def read(name: String, predicate: String, rows: Seq[Row], version: java.lang.Long = null) - : Unit = { - readSpec(t, predicate = predicate, version = version, name = Some(name)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == rows.size) - case other => fail(s"$name must succeed: $other") - } - val expected = spark.createDataFrame(rows.asJava, epochSchema) - val captured = tableDir.resolve("expected").resolve(s"${workload}_$name") - .resolve("expected_data") - RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) - } - } - read("all", null, all) - read("all_v7", null, all, 7L) - for ((field, index, kind) <- Seq(("ts", 1, "TIMESTAMP"), ("ntz", 2, "TIMESTAMP_NTZ"), - ("p_ts", 4, "TIMESTAMP"), ("p_ntz", 5, "TIMESTAMP_NTZ"))) { - for ((id, wall, _, _) <- epochValues) { - val expected = if (kind == "TIMESTAMP") Timestamp.from(Instant.parse(wall + "Z")) - else java.time.LocalDateTime.parse(wall) - read(s"${field}_eq_$id", s"$field = $kind '${wall.replace('T', ' ')}'", - all.filter(_.get(index) == expected)) - } - read(s"${field}_negative", s"$field < $kind '1970-01-01 00:00:00'", - all.filter { r => - if (kind == "TIMESTAMP") r.getTimestamp(index).before(Timestamp.from(Instant.EPOCH)) - else r.getAs[java.time.LocalDateTime](index) - .isBefore(java.time.LocalDateTime.of(1970, 1, 1, 0, 0)) - }) - } - for ((field, index) <- Seq("d" -> 3, "p_d" -> 6); - day <- Seq("1969-12-31", "1970-01-01", "1970-01-02")) { - read(s"${field}_$day", s"$field = DATE '$day'", - all.filter(_.getDate(index) == java.sql.Date.valueOf(day))) - } - read("cross_negative_hits", "ts = TIMESTAMP '1969-12-31 23:59:59.999999' AND " + - "p_ntz = TIMESTAMP_NTZ '1969-12-31 23:59:59.999' AND p_d < DATE '1970-01-01'", - all.filter(r => r.getInt(0) == 4 && r.getTimestamp(4) == - Timestamp.from(Instant.parse("1969-12-31T23:59:59.999Z")))) - read("outside_stats", "ts < TIMESTAMP '1969-12-31 23:59:59.998999' OR " + - "ntz > TIMESTAMP_NTZ '1970-01-01 00:00:00.001'", Seq.empty) - read("outside_partitions", "p_ts < TIMESTAMP '1969-12-31 23:59:59.998999' OR " + - "p_ntz > TIMESTAMP_NTZ '1970-01-01 00:00:00.001'", Seq.empty) - for (version <- Seq[java.lang.Long](null, 7L)) { - snapshotSpec(t, version = version).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == epochSchema) - require(result.metadata.partitionColumns == Seq("p_ts", "p_ntz", "p_d")) - require(result.metadata.format.provider == "parquet") - require(result.metadata.format.options.isEmpty) - require(result.metadata.configuration("delta.checkpointPolicy") == "classic") - require(result.protocol.minReaderVersion == 3) - require(result.protocol.minWriterVersion == 7) - require(result.protocol.readerFeatures.get.contains("timestampNtz")) - require(result.protocol.writerFeatures.get.contains("timestampNtz")) - case other => fail(s"Near-epoch snapshot must succeed (parsed=$parsed): $other") - } - } - } - } - - // Statistics and partition cases share the absolute-instant fixture and oracle. - private val mismatchSchema = new StructType().add("id", IntegerType) - .add("ts", TimestampType).add("ntz", org.apache.spark.sql.types.TimestampNTZType) - private val mismatchZones = Seq( - ("writer_utc_reader_non_utc", "UTC", "America/Los_Angeles"), - ("writer_non_utc_reader_utc", "America/Los_Angeles", "UTC")) - - // Offsets distinguish both occurrences of 01:30 at the autumn overlap. The unqualified - // controls deliberately produce different instants in the two writer-zone fixtures. - private val mismatchSql = Seq( - (1, "2024-03-10 01:59:59.123456-08:00", "2024-03-10 01:59:59.123456"), - (2, "2024-03-10 03:00:00.123456-07:00", "2024-03-10 03:00:00.123456"), - (3, "2024-11-03 01:30:00.123456-07:00", "2024-11-03 01:30:00.123456"), - (4, "2024-11-03 01:30:00.123456-08:00", "2024-11-03 01:30:00.123456"), - (5, "2024-01-15 12:00:00.654321", "2024-01-15 12:00:00.654321"), - (6, "2024-03-10 02:30:00.123456", "2024-03-10 02:30:00.123456"), - (7, "2024-11-03 01:30:00.123456", "2024-11-03 01:30:00.123456")) - - // This oracle does not parse fixture SQL or read Delta. NTZ retains the written wall clock, - // including the nonexistent LA 02:30. TIMESTAMP instead denotes an absolute UTC instant. - private def mismatchRows(writerZone: String): Seq[Row] = { - val controls = if (writerZone == "UTC") Seq( - "2024-01-15T12:00:00.654321Z", "2024-03-10T02:30:00.123456Z", - "2024-11-03T01:30:00.123456Z") else Seq( - "2024-01-15T20:00:00.654321Z", "2024-03-10T10:30:00.123456Z", - "2024-11-03T08:30:00.123456Z") - val instants = Seq( - "2024-03-10T09:59:59.123456Z", "2024-03-10T10:00:00.123456Z", - "2024-11-03T08:30:00.123456Z", "2024-11-03T09:30:00.123456Z") ++ controls - val walls = Seq( - "2024-03-10T01:59:59.123456", "2024-03-10T03:00:00.123456", - "2024-11-03T01:30:00.123456", "2024-11-03T01:30:00.123456", - "2024-01-15T12:00:00.654321", "2024-03-10T02:30:00.123456", - "2024-11-03T01:30:00.123456") - val rows = instants.zip(walls).zipWithIndex.map { case ((instant, wall), i) => - Row(i + 1, Timestamp.from(Instant.parse(instant)), java.time.LocalDateTime.parse(wall)) - } - rows ++ rows.filter(_.getInt(0) == 3) - } - - private def mismatchMicros(instant: Instant): Long = - instant.getEpochSecond * 1000000L + instant.getNano / 1000 - - // Spark to_json's default timestamp pattern serializes both bounds at millisecond - // precision. Check that physical encoding independently; keep microsecond row expectations. - private def mismatchJsonInstant(instant: Instant): Instant = - instant.truncatedTo(java.time.temporal.ChronoUnit.MILLIS) - - private def createMismatchTable(writerZone: String, partitioned: Boolean): TableHandle = { - spark.conf.set("spark.sql.session.timeZone", writerZone) - val partition = if (partitioned) "PARTITIONED BY (ts)" else "" - sql(s"""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ) USING delta - $partition TBLPROPERTIES ('delta.checkpointPolicy' = 'classic', - 'delta.checkpointInterval' = '100', 'delta.checkpoint.writeStatsAsStruct' = 'true', - 'delta.checkpoint.writeStatsAsJson' = 'false')""") - for ((id, instant, wall) <- mismatchSql) { - val row = s"($id, TIMESTAMP '$instant', TIMESTAMP_NTZ '$wall')" - val values = if (id == 3) s"$row, $row" else row - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS rows(id, ts, ntz)") - } - val table = registerTable("tbl") - val rows = mismatchRows(writerZone) - for (version <- 1L to 7L) { - val adds = CommitLog.read(table.sourcePath, version).collect { case a: AddFile => a } - require(adds.size == 1, "Each input batch must produce exactly one file") - val add = adds.head - val row = rows.find(_.getInt(0) == version.toInt).get - val stats = JsonUtil.mapper.readTree(add.stats.get) - require(stats.get("numRecords").asLong() == (if (version == 3) 2 else 1)) - val expectedPartition = if (partitioned) { - Map("ts" -> row.getTimestamp(1).toInstant.toString) - } else Map.empty[String, String] - if (partitioned && writerZone == "America/Los_Angeles" && version == 4 && - add.partitionValues != expectedPartition) { - // The producer reparses an offset-free directory name at the overlap's earlier offset. - // Preserve that limitation, then construct the intended AddFile in the copied fixture. - require(add.partitionValues == Map("ts" -> "2024-11-03T08:30:00.123456Z")) - println(s"Observed producer overlap loss: expected=$expectedPartition " + - s"actual=${add.partitionValues} action=${add.toJson}") - } else require(add.partitionValues == expectedPartition) - for (bound <- Seq("minValues", "maxValues")) { - val fields = stats.get(bound) - require(fields.get("id").asInt() == version.toInt) - if (partitioned) require(!fields.has("ts")) else { - val raw = fields.get("ts").asText() - val parsed = java.time.OffsetDateTime.parse(raw) - val instant = row.getTimestamp(1).toInstant - require(parsed.toInstant == mismatchJsonInstant(instant)) - require(parsed.getOffset == instant.atZone(java.time.ZoneId.of(writerZone)).getOffset) - } - val ntz = row.getAs[java.time.LocalDateTime](2).toInstant(java.time.ZoneOffset.UTC) - val rawNtz = java.time.LocalDateTime.parse(fields.get("ntz").asText()) - require(rawNtz.toInstant(java.time.ZoneOffset.UTC) == mismatchJsonInstant(ntz)) - } - println(s"Verified writer JSON zone=$writerZone partitioned=$partitioned " + - s"id=$version stats=${add.stats.get} partitions=${add.partitionValues}") - } - println(s"Verified writer table zone=${spark.conf.get("spark.sql.session.timeZone")} " + - s"partitioned=$partitioned rows=8 files=7") - table - } - - private def prepareMismatchCapture( - table: TableHandle, writerZone: String, readerZone: String, - partitioned: Boolean, parsed: Boolean): Unit = { - require(writerZone != readerZone) - require(spark.conf.get("spark.sql.session.timeZone") == writerZone) - if (partitioned && writerZone == "America/Los_Angeles") { - val later = mismatchRows(writerZone).find(_.getInt(0) == 4).get.getTimestamp(1).toInstant - modifyCommitActions(table, version = 4) { actions => - actions.map { - case add: AddFile => add.copy(partitionValues = Map("ts" -> later.toString)) - case other => other - } - } - } - if (parsed) { - val encoding = spark.conf.get("spark.sql.parquet.outputTimestampType") - try { - spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MICROS") - forceCheckpoint("tbl") - } finally spark.conf.set("spark.sql.parquet.outputTimestampType", encoding) - } - mutateTable(table) { dir => - require(spark.conf.get("spark.sql.session.timeZone") == readerZone) - val logDir = dir.resolve("_delta_log") - if (parsed) { - val checkpoint = logDir.resolve("00000000000000000007.checkpoint.parquet") - if (partitioned && writerZone == "America/Los_Angeles") { - setMismatchOverlapPartition(checkpoint) - } - checkMismatchCheckpoint(checkpoint, writerZone, partitioned) - } - val files = Files.list(logDir) - try files.iterator().asScala.filter { p => - val file = p.getFileName.toString - file.endsWith(".crc") || (parsed && file.endsWith(".json") && - file != "00000000000000000007.json") - }.foreach(Files.delete(_)) finally files.close() - // Same-version v7 JSON is retained solely for version lookup. No replay tail exists; - // v0-v6 and CRC shortcuts are absent, so the checkpoint is required, not checkpoint-only. - println(s"Verified pre-capture reader=$readerZone writer=$writerZone " + - s"partitioned=$partitioned parsed=$parsed") - } - // WorkloadTestSuite captures after the body. Deliberately leave this reader zone active. - spark.conf.set("spark.sql.session.timeZone", readerZone) - } - - // DAT supports a synthetic, correct overlap partition even when the SQL writer loses the - // later offset while reparsing a partition directory. Preserve its unmodified checkpoint; - // then change only the intended id=4 AddFile's raw and parsed partition cells in the copy. - private def setMismatchOverlapPartition(checkpoint: Path): Unit = { - val workload = WorkloadContext.current.workloadName - val original = outputDir.resolveSibling("producer-checkpoints").resolve(workload) - Files.createDirectories(original) - Files.copy(checkpoint, original.resolve(checkpoint.getFileName), - StandardCopyOption.REPLACE_EXISTING) - val instant = Instant.parse("2024-11-03T09:30:00.123456Z") - val temporary = checkpoint.resolveSibling("overlap-partition-rewrite") - val confKey = "spark.sql.parquet.outputTimestampType" - val previous = spark.conf.get(confKey) - try { - spark.conf.set(confKey, "TIMESTAMP_MICROS") - val df = spark.read.parquet(checkpoint.toString) - val later = col("add.stats_parsed.minValues.id") === lit(4) - val raw = org.apache.spark.sql.functions.when(later, - org.apache.spark.sql.functions.map(lit("ts"), lit(instant.toString))) - .otherwise(col("add.partitionValues")) - val parsed = col("add.partitionValues_parsed").withField("ts", - org.apache.spark.sql.functions.when(later, lit(Timestamp.from(instant))) - .otherwise(col("add.partitionValues_parsed.ts"))) - val add = col("add").withField("partitionValues", raw) - .withField("partitionValues_parsed", parsed) - df.withColumn("add", add).coalesce(1).write.parquet(temporary.toString) - val files = Files.list(temporary) - val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) - .toSeq match { - case Seq(single) => single - case other => fail(s"Expected one overlap checkpoint part: $other") - } finally files.close() - Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) - } finally { - spark.conf.set(confKey, previous) - if (Files.exists(temporary)) FileUtils.deleteDirectory(temporary.toFile) - } - println("Constructed later overlap partition fixture: id=4 " + - "instant=2024-11-03T09:30:00.123456Z") - } - - private def checkMismatchCheckpoint( - checkpoint: Path, writerZone: String, partitioned: Boolean): Unit = { - require(Files.isRegularFile(checkpoint)) - val conf = spark.sparkContext.hadoopConfiguration - val path = new HadoopPath(checkpoint.toUri) - val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) - try { - val physical = footer.getFooter.getFileMetaData.getSchema - val timestamps = if (partitioned) Seq("partitionValues_parsed" -> "ts") - else Seq("stats_parsed.minValues" -> "ts", "stats_parsed.maxValues" -> "ts") - for ((prefix, field) <- timestamps ++ Seq( - "stats_parsed.minValues" -> "ntz", "stats_parsed.maxValues" -> "ntz")) { - val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) - .asPrimitiveType() - require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) - val annotation = cell.getLogicalTypeAnnotation - .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] - require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) - require(annotation.isAdjustedToUTC == (field == "ts")) - println(s"Verified timezone checkpoint writer=$writerZone $prefix.$field: $cell") - } - } finally footer.close() - val rows = mismatchRows(writerZone) - val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() - var ids = Vector.empty[Int] - try { - var record = reader.read() - while (record != null) { - if (record.getFieldRepetitionCount("add") > 0) { - val add = record.getGroup("add", 0) - if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) - val stats = add.getGroup("stats_parsed", 0) - val id = stats.getGroup("minValues", 0).getInteger("id", 0) - val row = rows.find(_.getInt(0) == id).get - require(stats.getLong("numRecords", 0) == (if (id == 3) 2 else 1)) - val instant = row.getTimestamp(1).toInstant - val ntz = row.getAs[java.time.LocalDateTime](2).toInstant(java.time.ZoneOffset.UTC) - for (bound <- Seq("minValues", "maxValues")) { - val values = stats.getGroup(bound, 0) - require(values.getInteger("id", 0) == id) - if (!partitioned) require(values.getLong("ts", 0) == - mismatchMicros(mismatchJsonInstant(instant))) - require(values.getLong("ntz", 0) == mismatchMicros(mismatchJsonInstant(ntz))) - } - if (partitioned) { - val partition = add.getGroup("partitionValues_parsed", 0) - require(partition.getLong("ts", 0) == mismatchMicros(instant)) - val raw = add.getGroup("partitionValues", 0) - require(raw.getFieldRepetitionCount("key_value") == 1) - val entry = raw.getGroup("key_value", 0) - require(entry.getString("key", 0) == "ts") - require(entry.getString("value", 0) == instant.toString) - } - ids :+= id - println(s"Verified timezone raw checkpoint writer=$writerZone partitioned=$partitioned " + - s"id=$id instantMicros=${mismatchMicros(instant)} ntzMicros=${mismatchMicros(ntz)}") - } - record = reader.read() - } - } finally reader.close() - require(ids.sorted == (1 to 7).toVector) - } - - private def checkMismatchSpecs( - table: TableHandle, writerZone: String, readerZone: String, partitioned: Boolean): Unit = { - val workload = WorkloadContext.current.workloadName - val tableDir = outputDir.resolve(workload) - val all = mismatchRows(writerZone) - def checkReaderZone(label: String): Unit = { - require(spark.conf.get("spark.sql.session.timeZone") == readerZone) - val probe = spark.sql("SELECT current_timezone(), " + - "CAST(TIMESTAMP '2024-03-10 10:00:00.123456Z' AS STRING)").head() - require(probe.getString(0) == readerZone) - val wall = if (readerZone == "UTC") "2024-03-10 10:00:00.123456" - else "2024-03-10 03:00:00.123456" - require(probe.getString(1) == wall) - println(s"Verified captured $workload/$label writer=$writerZone reader=$readerZone " + - s"readerWall=$wall") - } - def read(label: String, predicate: String, rows: Seq[Row], version: java.lang.Long): Unit = { - readSpec(table, predicate = predicate, version = version, name = Some(label)).assert { spec => - checkReaderZone(label) - spec.expectation match { - case Succeeded(result) => require(result.rowCount == rows.size) - case other => fail(s"$label must succeed: $other") - } - val expected = spark.createDataFrame(rows.asJava, mismatchSchema) - val captured = tableDir.resolve("expected").resolve(s"${workload}_$label") - .resolve("expected_data") - RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), label) - if (label == "all_latest") { - expected.coalesce(1).write.mode("overwrite") - .parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString) - } - } - } - for (version <- Seq[java.lang.Long](null, 7L)) { - val suffix = if (version == null) "latest" else "v7" - read(s"all_$suffix", null, all, version) - // Explicit offsets keep the emitted predicates portable to consumers with another zone. - for (row <- all.groupBy(_.getInt(0)).values.map(_.head).toSeq.sortBy(_.getInt(0))) { - val instant = row.getTimestamp(1) - read(s"eq_${row.getInt(0)}_$suffix", s"ts = TIMESTAMP '${instant.toInstant}'", - all.filter(_.getTimestamp(1) == instant), version) - } - read(s"spring_range_$suffix", "ts >= TIMESTAMP '2024-03-10T09:59:59.123456Z' AND " + - "ts <= TIMESTAMP '2024-03-10T10:00:00.123456Z'", - all.filter(r => Set(1, 2).contains(r.getInt(0))), version) - read(s"overlap_range_$suffix", "ts >= TIMESTAMP '2024-11-03T08:30:00.123456Z' AND " + - "ts <= TIMESTAMP '2024-11-03T09:30:00.123456Z'", - all.filter(r => Set(3, 4).contains(r.getInt(0)) || - (writerZone != "UTC" && r.getInt(0) == 7)), version) - read(s"outside_$suffix", "ts < TIMESTAMP '2024-01-01T00:00:00Z'", Seq.empty, version) - read(s"ntz_gap_$suffix", "ntz = TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'", - all.filter(_.getInt(0) == 6), version) - snapshotSpec(table, version = version).assert { spec => - checkReaderZone(s"snapshot_$suffix") - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == mismatchSchema) - require(result.metadata.partitionColumns == (if (partitioned) Seq("ts") else Seq.empty)) - require(result.metadata.format.provider == "parquet") - require(result.metadata.format.options.isEmpty) - require(result.metadata.configuration("delta.checkpointPolicy") == "classic") - require(result.metadata.configuration("delta.checkpoint.writeStatsAsStruct") == "true") - require(result.metadata.configuration("delta.checkpoint.writeStatsAsJson") == "false") - require(result.protocol.minReaderVersion == 3 && result.protocol.minWriterVersion == 7) - require(result.protocol.readerFeatures.get.contains("timestampNtz")) - require(result.protocol.writerFeatures.get.contains("timestampNtz")) - case other => fail(s"Timezone snapshot must succeed: $other") - } - } - } - } - - for ((direction, writer, reader) <- mismatchZones; parsed <- Seq(false, true)) { - val representation = if (parsed) "checkpoint" else "json" - test(s"stats_timezone_${direction}_$representation") { - val table = createMismatchTable(writer, partitioned = false) - prepareMismatchCapture(table, writer, reader, partitioned = false, parsed = parsed) - checkMismatchSpecs(table, writer, reader, partitioned = false) - } - } - - // Reuse the timezone fixture and check TIMESTAMP partition reconstruction. - for ((direction, writer, reader) <- mismatchZones; parsed <- Seq(false, true)) { - val representation = if (parsed) "checkpoint" else "json" - test(s"partitions_timezone_${direction}_$representation") { - val table = createMismatchTable(writer, partitioned = true) - prepareMismatchCapture(table, writer, reader, partitioned = true, parsed = parsed) - checkMismatchSpecs(table, writer, reader, partitioned = true) - } - } - - // Later portable cases share only ordinary read/schema assertions; row oracles stay literal. - private def readinessRead( - table: TableHandle, name: String, expectedSchema: StructType, rows: Seq[Row], - predicate: String = null, version: java.lang.Long = null): Unit = { - val workload = WorkloadContext.current.workloadName - readSpec(table, predicate = predicate, version = version, name = Some(name)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == rows.size) - case other => fail(s"$name must succeed against the independent oracle: $other") - } - val expected = spark.createDataFrame(rows.asJava, expectedSchema) - val captured = outputDir.resolve(workload).resolve("expected") - .resolve(s"${workload}_$name").resolve("expected_data") - RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) - if (name == "all") { - expected.coalesce(1).write.mode("overwrite") - .parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString) - } - println(s"Verified independent literal rows/schema: $workload/$name rows=${rows.size}") - } - } - - private def readinessSnapshot( - table: TableHandle, expectedSchema: StructType, partitions: Seq[String] = Seq.empty, - version: java.lang.Long = null): Unit = { - snapshotSpec(table, version = version).assert { spec => - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) - require(result.metadata.partitionColumns == partitions) - require(result.metadata.format.provider == "parquet") - require(result.metadata.format.options.isEmpty) - val ntz = expectedSchema.fields.exists(_.dataType == - org.apache.spark.sql.types.TimestampNTZType) - require(result.protocol.minReaderVersion == (if (ntz) 3 else 1)) - require(result.protocol.minWriterVersion == (if (ntz) 7 else 2)) - case other => fail(s"Snapshot must succeed against independent metadata: $other") - } - } - } - - private def readinessPruneHistory(dir: Path, version: Long): Unit = { - val log = dir.resolve("_delta_log") - val files = Files.list(log) - try files.iterator().asScala.filter { p => - val file = p.getFileName.toString - file.endsWith(".crc") || (file.endsWith(".json") && file != f"$version%020d.json") - }.foreach(Files.delete(_)) finally files.close() - // The same-version JSON supports requested-version lookup only. Earlier history is absent, - // so checkpoint reconstruction is required. No replay tail or checkpoint-only claim. - } - - // Unannotated INT32/INT64 parsed cells are distinct from normal DATE/TIMESTAMP - // Parquet encodings. Keep correct JSON stats too: Spark rejects several parsed-only forms. - // Only INT32 as DATE, the DATE32 physical width, removes every JSON stats copy so that the - // parsed day is the only statistic. The other forms are not claimed as parsed-only routes. - for ((kind, sqlType, valueType, literal, value, integerValue) <- Seq( - ("timestamp", "TIMESTAMP", TimestampType, - "TIMESTAMP '1970-01-01T00:00:01Z'", Timestamp.from(Instant.ofEpochSecond(1)), 1000000L), - ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, - "TIMESTAMP_NTZ '1970-01-01 00:00:01'", - java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1), 1000000L), - ("date", "DATE", org.apache.spark.sql.types.DateType, - "DATE '1970-01-02'", java.sql.Date.valueOf("1970-01-02"), 1L)); - integerType <- Seq("int", "long")) { - test(s"checkpoint_integer_${integerType}_as_$kind") { - spark.conf.set("spark.sql.session.timeZone", "UTC") - val expectedSchema = new StructType().add("id", IntegerType).add("v", valueType) - val rows = Seq(Row(1, value), Row(2, value), Row(2, value)) - sql(s"""CREATE TABLE tbl (id INT, v $sqlType) USING delta TBLPROPERTIES ( - 'delta.checkpointPolicy' = 'classic', 'delta.checkpoint.writeStatsAsStruct' = 'true', - 'delta.checkpoint.writeStatsAsJson' = 'true')""") - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + - s"(1, $literal), (2, $literal), (2, $literal) AS rows(id, v)") - val table = registerTable("tbl") - val parsedOnly = kind == "date" && integerType == "int" - forceCheckpoint("tbl") - if (parsedOnly) { - modifyCommitActions(table, 1L) { actions => - actions.map { case add: AddFile => add.copy(stats = None); case other => other } - } - } - mutateTable(table) { dir => - val checkpoint = dir.resolve("_delta_log/00000000000000000001.checkpoint.parquet") - val temporary = dir.resolve("_delta_log/integer-stats-rewrite") - val df = spark.read.parquet(checkpoint.toString) - var stats = col("add.stats_parsed") - for (bound <- Seq("minValues", "maxValues")) { - stats = stats.withField(bound, col(s"add.stats_parsed.$bound") - .withField("v", lit(integerValue).cast(integerType))) - } - var add = col("add").withField("stats_parsed", stats) - if (parsedOnly) add = add.withField("stats", lit(null).cast("string")) - df.withColumn("add", add).coalesce(1).write.parquet(temporary.toString) - val parts = Files.list(temporary) - val part = try parts.iterator().asScala.filter(_.toString.endsWith(".parquet")) - .toSeq match { - case Seq(single) => single - case other => fail(s"Expected one rewritten integer stats checkpoint: $other") - } finally parts.close() - Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) - FileUtils.deleteDirectory(temporary.toFile) - val path = new HadoopPath(checkpoint.toUri) - val conf = spark.sparkContext.hadoopConfiguration - val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) - try { - for (bound <- Seq("minValues", "maxValues")) { - val physical = footer.getFooter.getFileMetaData.getSchema - .getType("add", "stats_parsed", bound, "v").asPrimitiveType() - require(physical.getLogicalTypeAnnotation == null) - require(physical.getPrimitiveTypeName == (if (integerType == "int") - PrimitiveType.PrimitiveTypeName.INT32 else PrimitiveType.PrimitiveTypeName.INT64)) - println(s"Verified off-type checkpoint $kind/$integerType $bound: " + - s"$physical value=$integerValue jsonFallback=${!parsedOnly}") - } - } finally footer.close() - val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) - .withConf(conf).build() - var count = 0 - try { - var record = reader.read() - while (record != null) { - if (record.getFieldRepetitionCount("add") > 0) { - count += 1 - val add = record.getGroup("add", 0) - if (parsedOnly) require(add.getFieldRepetitionCount("stats") == 0) else { - require(add.getFieldRepetitionCount("stats") == 1) - val json = JsonUtil.mapper.readTree(add.getString("stats", 0)) - for (bound <- Seq("minValues", "maxValues")) { - val raw = json.at(s"/$bound/v").asText() - kind match { - case "timestamp" => require(Instant.parse(raw) == Instant.ofEpochSecond(1)) - case "timestamp_ntz" => require(java.time.LocalDateTime.parse(raw) == - java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1)) - case "date" => require(java.time.LocalDate.parse(raw) == - java.time.LocalDate.of(1970, 1, 2)) - } - } - } - val fields = add.getGroup("stats_parsed", 0) - require(fields.getLong("numRecords", 0) == 3L) - for (bound <- Seq("minValues", "maxValues")) { - val values = fields.getGroup(bound, 0) - val raw = if (integerType == "int") values.getInteger("v", 0).toLong - else values.getLong("v", 0) - require(raw == integerValue) - } - } - record = reader.read() - } - } finally reader.close() - require(count == 1) - readinessPruneHistory(dir, 1L) - if (parsedOnly) { - val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } - require(adds.size == 1 && adds.head.stats.isEmpty) - val log = Files.list(dir.resolve("_delta_log")) - val names = try log.iterator().asScala.map(_.getFileName.toString).toSet - finally log.close() - require(!names.exists(_.endsWith(".crc"))) - require(names.filter(_.endsWith(".json")) == Set("00000000000000000001.json")) - println("Verified parsed-only DATE stats: INT32 day=1 (1970-01-02); JSON stats " + - "absent in checkpoint and v1 commit; CRC removed; v0 pruned") - } - } - readinessRead(table, "all", expectedSchema, rows) - readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) - readinessRead(table, "hit", expectedSchema, rows, predicate = s"v = $literal") - readinessRead(table, "miss", expectedSchema, Seq.empty, - predicate = s"v < $literal OR v > $literal") - readinessSnapshot(table, expectedSchema) - readinessSnapshot(table, expectedSchema, version = 1L) - } - } - - private def readinessRemoveChecksums(dir: Path): Unit = { - val files = Files.list(dir.resolve("_delta_log")) - try files.iterator().asScala.filter(_.toString.endsWith(".crc")) - .foreach(Files.delete(_)) finally files.close() - } - - // JSON numeric stats are separate from physical checkpoint types in the checkpoint fixture. - // Literal rows remain authoritative even when a reference parser discards off-type stats. - for ((kind, sqlType, valueType, literal, value) <- Seq( - ("timestamp", "TIMESTAMP", TimestampType, "TIMESTAMP '1970-01-01T00:00:01Z'", - Timestamp.from(Instant.ofEpochSecond(1))), - ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, - "TIMESTAMP_NTZ '1970-01-01 00:00:01'", - java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1)), - ("date", "DATE", org.apache.spark.sql.types.DateType, - "DATE '1970-01-02'", java.sql.Date.valueOf("1970-01-02")), - ("string", "STRING", org.apache.spark.sql.types.StringType, "'1'", "1"))) { - test(s"action_integer_as_$kind") { - spark.conf.set("spark.sql.session.timeZone", "UTC") - val expectedSchema = new StructType().add("id", IntegerType).add("v", valueType) - val rows = Seq(Row(1, value), Row(2, value), Row(2, value)) - sql(s"CREATE TABLE tbl (id INT, v $sqlType) USING delta") - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + - s"(1, $literal), (2, $literal), (2, $literal) AS rows(id, v)") - val table = registerTable("tbl") - modifyCommitActions(table, 1L) { actions => - actions.map { - case add: AddFile => - val stats = JsonUtil.mapper.readTree(add.stats.get) - for (bound <- Seq("minValues", "maxValues")) { - stats.get(bound).asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] - .put("v", 1) - } - add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) - case other => other - } - } - mutateTable(table) { dir => - readinessRemoveChecksums(dir) - val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } - require(adds.size == 1) - val stats = JsonUtil.mapper.readTree(adds.head.stats.get) - require(stats.get("numRecords").asLong() == 3L) - for (bound <- Seq("minValues", "maxValues")) { - require(stats.at(s"/$bound/v").isIntegralNumber) - require(stats.at(s"/$bound/v").intValue() == 1) - } - println(s"Verified numeric JSON stats: $kind min=1 max=1 records=3") - } - // Spark accepts numeric STRING stats. The required-rejection probe is preserved - // separately; this useful correctness fixture does not claim that rejection. - readinessRead(table, "all", expectedSchema, rows) - readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) - readinessRead(table, "hit", expectedSchema, rows, predicate = s"v = $literal") - readinessRead(table, "miss", expectedSchema, Seq.empty, - predicate = s"v < $literal OR v > $literal") - readinessSnapshot(table, expectedSchema) - readinessSnapshot(table, expectedSchema, version = 1L) - } - } - - // Literal empty AddFile partition strings are independent of writer normalization. - // BYTE is numeric TINYINT. Its empty input has no numeric value and is expected to be null. - // STRING and BINARY retain a distinct empty value; writer support was probed separately. - for ((kind, sqlType, valueType, firstValue, secondValue, control, emptyValue) <- Seq( - ("byte", "TINYINT", org.apache.spark.sql.types.ByteType, - "CAST(1 AS TINYINT)", "CAST(2 AS TINYINT)", 2.toByte, null), - ("short", "SMALLINT", org.apache.spark.sql.types.ShortType, - "CAST(1 AS SMALLINT)", "CAST(2 AS SMALLINT)", 2.toShort, null), - ("int", "INT", IntegerType, "1", "2", 2, null), - ("long", "BIGINT", org.apache.spark.sql.types.LongType, "1L", "2L", 2L, null), - ("float", "FLOAT", org.apache.spark.sql.types.FloatType, - "CAST(1.25 AS FLOAT)", "CAST(2.5 AS FLOAT)", 2.5f, null), - ("double", "DOUBLE", org.apache.spark.sql.types.DoubleType, - "CAST(1.25 AS DOUBLE)", "CAST(2.5 AS DOUBLE)", 2.5d, null), - ("decimal", "DECIMAL(10,2)", org.apache.spark.sql.types.DecimalType(10, 2), - "CAST(1.25 AS DECIMAL(10,2))", "CAST(2.50 AS DECIMAL(10,2))", - new java.math.BigDecimal("2.50"), null), - ("boolean", "BOOLEAN", org.apache.spark.sql.types.BooleanType, - "false", "true", true, null), - ("date", "DATE", org.apache.spark.sql.types.DateType, - "DATE '1969-12-31'", "DATE '2000-02-29'", java.sql.Date.valueOf("2000-02-29"), null), - ("timestamp", "TIMESTAMP", TimestampType, - "TIMESTAMP '1969-12-31T23:59:59Z'", "TIMESTAMP '2024-03-10T10:00:00.123456Z'", - Timestamp.from(Instant.parse("2024-03-10T10:00:00.123456Z")), null), - ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, - "TIMESTAMP_NTZ '1969-12-31 23:59:59'", "TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'", - java.time.LocalDateTime.parse("2024-03-10T02:30:00.123456"), null), - ("string", "STRING", org.apache.spark.sql.types.StringType, - "'first'", "'second'", "second", ""), - ("binary", "BINARY", org.apache.spark.sql.types.BinaryType, - "unhex('61')", "unhex('62')", Array[Byte](98), Array.emptyByteArray))) { - test(s"empty_partition_$kind") { - spark.conf.set("spark.sql.session.timeZone", "UTC") - // ANSI predicate casts reject empty numeric input. Leave legacy casts active for capture. - // The existing spec format cannot transport this session setting to an external consumer. - spark.conf.set("spark.sql.ansi.enabled", "false") - val expectedSchema = new StructType().add("id", IntegerType).add("p", valueType) - val rows = Seq(Row(1, emptyValue), Row(2, control), Row(2, control), Row(3, null)) - sql(s"CREATE TABLE tbl (id INT, p $sqlType) USING delta PARTITIONED BY (p)") - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + - s"(1, $firstValue), (2, $secondValue), (2, $secondValue), " + - s"(3, CAST(NULL AS $sqlType)) AS rows(id, p)") - val table = registerTable("tbl") - modifyCommitActions(table, 1L) { actions => - actions.map { - case add: AddFile if JsonUtil.mapper.readTree(add.stats.get) - .at("/minValues/id").intValue() == 1 => - add.copy(partitionValues = Map("p" -> "")) - case other => other - } - } - mutateTable(table) { dir => - readinessRemoveChecksums(dir) - val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } - require(adds.size == 3) - require(adds.count(_.partitionValues("p") == "") == 1) - require(adds.count(_.partitionValues("p") == null) == 1) - println(s"Verified serialized empty partition: $kind empty=1 null=1 control=1") - } - readinessRead(table, "all", expectedSchema, rows) - readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) - readinessRead(table, "is_null", expectedSchema, rows.filter(_.isNullAt(1)), - predicate = "p IS NULL") - readinessRead(table, "is_not_null", expectedSchema, rows.filterNot(_.isNullAt(1)), - predicate = "p IS NOT NULL") - readinessRead(table, "eq_control", expectedSchema, rows.filter(_.getInt(0) == 2), - predicate = s"p = $secondValue") - if (kind == "string" || kind == "binary") { - val emptyLiteral = if (kind == "string") "''" else "unhex('')" - readinessRead(table, "eq_empty", expectedSchema, rows.filter(_.getInt(0) == 1), - predicate = s"p = $emptyLiteral") - } - snapshotSpec(table).assert { spec => - require(spark.conf.get("spark.sql.ansi.enabled") == "false") - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) - require(result.metadata.partitionColumns == Seq("p")) - require(result.metadata.format.provider == "parquet") - require(result.metadata.format.options.isEmpty) - require(result.protocol.minReaderVersion == (if (kind == "timestamp_ntz") 3 else 1)) - require(result.protocol.minWriterVersion == (if (kind == "timestamp_ntz") 7 else 2)) - case other => fail(s"Empty partition snapshot must succeed: $other") - } - println(s"Verified capture mode: ${WorkloadContext.current.workloadName} ansi=false") - } - readinessSnapshot(table, expectedSchema, Seq("p"), version = 1L) - } - } - - // Inputs are literal AddFile strings, not SQL timestamp literal spellings. - // The pinned Spark parser permits zone suffixes for NTZ casts and ignores their offsets; - // expected NTZ wall clocks below are separate from the corresponding absolute instants. - private def readinessPartitionFormats: Vector[(String, String, String, String)] = { - val fractions = Seq( - ("0", "", "000000"), ("1", ".1", "100000"), ("2", ".12", "120000"), - ("3", ".123", "123000"), ("4", ".1234", "123400"), - ("5", ".12345", "123450"), ("6", ".123456", "123456"), - ("7", ".1234567", "123456"), ("8", ".12345678", "123456"), - ("9", ".123456789", "123456"), ("dot", ".", "000000"), - ("10", ".1234567891", "123456"), ("18", ".123456789123456789", "123456")) - val zones = Seq( - ("z", "12:34:56.123456Z", "12:34:56.123456"), - ("zero_colon", "12:34:56.123456+00:00", "12:34:56.123456"), - ("zero_basic", "12:34:56.123456+0000", "12:34:56.123456"), - ("zero_hour", "12:34:56.123456+00", "12:34:56.123456"), - ("minus_zero", "12:34:56.123456-00:00", "12:34:56.123456"), - ("utc_name", "12:34:56.123456 UTC", "12:34:56.123456"), - ("gmt_name", "12:34:56.123456 GMT", "12:34:56.123456"), - ("east", "18:04:56.123456+05:30", "18:04:56.123456"), - ("west", "04:34:56.123456-08:00", "04:34:56.123456"), - ("region", "04:34:56.123456 America/Los_Angeles", "04:34:56.123456"), - ("east_basic", "18:04:56.123456+0530", "18:04:56.123456"), - ("east_hour", "17:34:56.123456+05", "17:34:56.123456"), - ("west_basic", "04:34:56.123456-0800", "04:34:56.123456"), - ("west_hour", "04:34:56.123456-08", "04:34:56.123456"), - ("west_half", "09:04:56.123456-03:30", "09:04:56.123456"), - ("single_hour", "18:04:56.123456+5:30", "18:04:56.123456"), - ("single_minute", "17:37:56.123456+05:3", "17:37:56.123456"), - ("offset_seconds", "18:05:26.123456+05:30:30", "18:05:26.123456")) - (for ((separatorName, separator) <- Seq("space" -> " ", "T" -> "T"); - entry <- fractions.map { case (label, fraction, micros) => - (s"${separatorName}_fraction_$label", s"2024-02-29${separator}12:34:56$fraction", - s"2024-02-29T12:34:56.$micros" + "Z", s"2024-02-29T12:34:56.$micros") - } ++ zones.map { case (label, raw, wall) => - (s"${separatorName}_$label", s"2024-02-29$separator$raw", - "2024-02-29T12:34:56.123456Z", s"2024-02-29T$wall") - }) yield entry).toVector - } - - for ((kind, sqlType, valueType) <- Seq( - ("timestamp", "TIMESTAMP", TimestampType), - ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType))) { - test(s"partition_serializations_$kind") { - spark.conf.set("spark.sql.session.timeZone", "UTC") - spark.conf.set("spark.sql.ansi.enabled", "false") - val formats = readinessPartitionFormats - require(formats.size == 62) - val expectedSchema = new StructType().add("id", IntegerType).add("p", valueType) - val keys = formats.zipWithIndex.map { case ((_, _, instant, wall), index) => - (index + 1) -> (if (kind == "timestamp") instant else wall) - }.toMap - val rows0 = keys.toSeq.sortBy(_._1).map { case (id, key) => - val value = if (kind == "timestamp") Timestamp.from(Instant.parse(key)) - else java.time.LocalDateTime.parse(key) - Row(id, value) - } - val rows = rows0 :+ rows0.head - sql(s"CREATE TABLE tbl (id INT, p $sqlType) USING delta PARTITIONED BY (p)") - // Distinct seed partitions create one data file per raw representation. Only id is - // physically stored; the typed AddFile mutation supplies the independently checked p. - val seeds = formats.indices.map { index => - val seed = Instant.parse("2030-01-01T00:00:00Z").plusSeconds(index.toLong).toString - val literal = if (kind == "timestamp") seed else seed.stripSuffix("Z") - s"(${index + 1}, $sqlType '$literal')" - } - sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + - (seeds :+ seeds.head).mkString(", ") + " AS rows(id, p)") - val table = registerTable("tbl") - modifyCommitActions(table, 1L) { actions => - actions.map { - case add: AddFile => - val id = JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() - add.copy(partitionValues = Map("p" -> formats(id - 1)._2)) - case other => other - } - } - mutateTable(table) { dir => - readinessRemoveChecksums(dir) - val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } - require(adds.size == formats.size) - adds.foreach { add => - val stats = JsonUtil.mapper.readTree(add.stats.get) - val id = stats.at("/minValues/id").intValue() - require(stats.at("/maxValues/id").intValue() == id) - require(stats.get("numRecords").asLong() == (if (id == 1) 2L else 1L)) - require(add.partitionValues == Map("p" -> formats(id - 1)._2)) - println(s"Verified raw serialization: $kind ${formats(id - 1)._1} " + - s"value=${formats(id - 1)._2}") - } - } - readinessRead(table, "all", expectedSchema, rows) - readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) - keys.values.toSet.toSeq.sorted.zipWithIndex.foreach { case (key, index) => - val ids = keys.collect { case (id, value) if value == key => id }.toSet - readinessRead(table, s"eq_$index", expectedSchema, - rows.filter(r => ids.contains(r.getInt(0))), predicate = s"p = $sqlType '$key'") - } - def selected(lower: String, upper: String): Seq[Row] = rows.filter { row => - val key = keys(row.getInt(0)) - key >= lower && key < upper - } - readinessRead(table, "fraction_range", expectedSchema, - selected("2024-02-29T12:34:56.100000", "2024-02-29T12:34:56.123456"), - predicate = s"p >= $sqlType '2024-02-29 12:34:56.100000' AND " + - s"p < $sqlType '2024-02-29 12:34:56.123456'") - readinessRead(table, "before_noon", expectedSchema, - selected("2024-02-29T00:00:00", "2024-02-29T12:00:00"), - predicate = s"p < $sqlType '2024-02-29 12:00:00'") - readinessRead(table, "miss", expectedSchema, Seq.empty, - predicate = s"p < $sqlType '2024-02-29 00:00:00'") - readinessSnapshot(table, expectedSchema, Seq("p")) - readinessSnapshot(table, expectedSchema, Seq("p"), version = 1L) - } - } - - // Canonical zero-padded DATE strings use proleptic Gregorian epoch-day oracles. - test("date_partition_canonical_calendar_bounds") { - spark.conf.set("spark.sql.session.timeZone", "UTC") - spark.conf.set("spark.sql.ansi.enabled", "false") - // Expected Parquet must retain proleptic Gregorian days even before the 1582 cutover. - spark.conf.set("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED") - spark.conf.set("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED") - val dates = Seq( - (1, "0001-01-01", -719162), (2, "1582-10-04", -141438), - (3, "1582-10-15", -141427), (4, "1969-12-31", -1), - (5, "1970-01-01", 0), (6, "2000-02-29", 11016), (7, "9999-12-31", 2932896)) - val expectedSchema = new StructType().add("id", IntegerType) - .add("p", org.apache.spark.sql.types.DateType) - val rows = dates.map { case (id, raw, _) => Row(id, java.sql.Date.valueOf(raw)) } ++ - Seq(Row(6, java.sql.Date.valueOf("2000-02-29")), Row(8, null)) - val days: Map[Int, Option[Int]] = dates.map { case (id, _, day) => - id -> Some(day) - }.toMap + (8 -> None) - sql("CREATE TABLE tbl (id INT, p DATE) USING delta PARTITIONED BY (p)") - val inputs = dates.map { case (id, raw, _) => s"($id, DATE '$raw')" } ++ - Seq("(6, DATE '2000-02-29')", "(8, CAST(NULL AS DATE))") - sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + - inputs.mkString(", ") + " AS rows(id, p)") - val table = registerTable("tbl") - modifyCommitActions(table, 1L) { actions => - actions.map { - case add: AddFile => - val id = JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() - val raw = if (id == 8) null else dates.find(_._1 == id).get._2 - add.copy(partitionValues = Map("p" -> raw)) - case other => other - } - } - mutateTable(table) { dir => - readinessRemoveChecksums(dir) - val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } - require(adds.size == 8) - adds.foreach { add => - val stats = JsonUtil.mapper.readTree(add.stats.get) - val id = stats.at("/minValues/id").intValue() - val raw = if (id == 8) null else dates.find(_._1 == id).get._2 - require(add.partitionValues == Map("p" -> raw)) - require(stats.at("/maxValues/id").intValue() == id) - require(stats.get("numRecords").asLong() == (if (id == 6) 2L else 1L)) - if (raw != null) require(raw.matches("[0-9]{4}-[0-9]{2}-[0-9]{2}")) - println(s"Verified canonical DATE partition: id=$id raw=$raw epochDay=${days(id)}") - } - } - val allName = "all" - readinessRead(table, allName, expectedSchema, rows) - readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) - dates.foreach { case (id, raw, _) => - readinessRead(table, s"eq_$id", expectedSchema, rows.filter(_.getInt(0) == id), - predicate = s"p = DATE '$raw'") - } - readinessRead(table, "is_null", expectedSchema, rows.filter(_.isNullAt(1)), - predicate = "p IS NULL") - readinessRead(table, "pre_epoch", expectedSchema, rows.filter(r => r.getInt(0) <= 4), - predicate = "p < DATE '1970-01-01'") - readinessRead(table, "calendar_transition", expectedSchema, - rows.filter(r => Set(2, 3).contains(r.getInt(0))), - predicate = "p >= DATE '1582-10-04' AND p <= DATE '1582-10-15'") - readinessRead(table, "modern_range", expectedSchema, - rows.filter(r => Set(5, 6).contains(r.getInt(0))), - predicate = "p >= DATE '1970-01-01' AND p <= DATE '2000-02-29'") - val workload = WorkloadContext.current.workloadName - snapshotSpec(table).assert { spec => - require(spark.conf.get("spark.sql.parquet.datetimeRebaseModeInWrite") == "CORRECTED") - require(spark.conf.get("spark.sql.parquet.datetimeRebaseModeInRead") == "CORRECTED") - spec.expectation match { - case Succeeded(result) => - require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) - require(result.metadata.partitionColumns == Seq("p")) - require(result.metadata.format.provider == "parquet") - require(result.metadata.format.options.isEmpty) - require(result.protocol.minReaderVersion == 1 && result.protocol.minWriterVersion == 2) - case other => fail(s"Canonical DATE snapshot must succeed: $other") - } - - } - readinessSnapshot(table, expectedSchema, Seq("p"), version = 1L) - // Snapshots capture before reads at this pin. Inspect Parquet in the last read callback. - val outsideName = "outside_bounds" - readSpec(table, predicate = "p < DATE '0001-01-01' OR p > DATE '9999-12-31'", - name = Some(outsideName)).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == 0L) - case other => fail(s"Outside DATE bounds must succeed with no rows: $other") - } - val expected = spark.createDataFrame(Seq.empty[Row].asJava, expectedSchema) - val outside = outputDir.resolve(workload).resolve("expected") - .resolve(s"${workload}_$outsideName").resolve("expected_data") - RowComparison.assertRowsEqual(expected, spark.read.parquet(outside.toString), outsideName) - println(s"Verified independent literal rows/schema: $workload/$outsideName rows=0") - val captured = outputDir.resolve(workload).resolve("expected") - .resolve(s"${workload}_$allName").resolve("expected_data") - val literalOracle = outputDir.resolveSibling("oracle-parquet").resolve(workload) - val expectedUnits = days.toVector :+ (6 -> Some(11016)) - for ((label, directory) <- Seq("captured" -> captured, "literal" -> literalOracle)) { - val files = Files.list(directory) - val parquet = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) - .toVector finally files.close() - require(parquet.nonEmpty) - var units = Vector.empty[(Int, Option[Int])] - parquet.foreach { file => - val path = new HadoopPath(file.toUri) - val conf = spark.sparkContext.hadoopConfiguration - val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) - try { - val physical = footer.getFooter.getFileMetaData.getSchema.getFields.asScala - .find(_.getName == "p").get.asPrimitiveType() - require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) - require(physical.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) - } finally footer.close() - val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) - .withConf(conf).build() - try { - var record = reader.read() - while (record != null) { - val day = if (record.getFieldRepetitionCount("p") == 0) None - else Some(record.getInteger("p", 0)) - units :+= record.getInteger("id", 0) -> day - record = reader.read() - } - } finally reader.close() - } - require(units.sorted == expectedUnits.sorted) - println(s"Verified physical DATE oracle: $workload $label units=${units.sorted}") - } - } - } - - // INT32 boundary magnitudes are BIGINT inputs to timestamp_seconds/millis/micros. The unit - // names the input magnitude only; rewritten checkpoint cells are always TIMESTAMP_MICROS. - private val intMaxInputs = Seq(-2147483648L, -2147483647L, -2147483646L, - 2147483646L, 2147483647L, 2147483648L) - // Literal UTC instants per input unit, declared apart from both SQL and java.time. - private val intMaxInstants = Seq( - ("seconds", 1000000L, Seq("1901-12-13T20:45:52Z", "1901-12-13T20:45:53Z", - "1901-12-13T20:45:54Z", "2038-01-19T03:14:06Z", "2038-01-19T03:14:07Z", - "2038-01-19T03:14:08Z")), - ("millis", 1000L, Seq("1969-12-07T03:28:36.352Z", "1969-12-07T03:28:36.353Z", - "1969-12-07T03:28:36.354Z", "1970-01-25T20:31:23.646Z", "1970-01-25T20:31:23.647Z", - "1970-01-25T20:31:23.648Z")), - ("micros", 1L, Seq("1969-12-31T23:24:12.516352Z", "1969-12-31T23:24:12.516353Z", - "1969-12-31T23:24:12.516354Z", "1970-01-01T00:35:47.483646Z", - "1970-01-01T00:35:47.483647Z", "1970-01-01T00:35:47.483648Z"))) - // Data value index -> partition value index; no index maps to itself. Ids 2 and 5 hold the - // -/+2147483647 boundaries twice, and id 7 is NULL in every temporal column twice. - private val intMaxPartition = Map(1 -> 4, 2 -> 6, 3 -> 1, 4 -> 5, 5 -> 2, 6 -> 3) - private val intMaxIds = Seq(1, 2, 2, 3, 4, 5, 5, 6, 7, 7) - private val intMaxSchema = new StructType().add("id", IntegerType).add("ts", TimestampType) - .add("ntz", TimestampNTZType).add("p_ts", TimestampType).add("p_ntz", TimestampNTZType) - private val sixDigits = - java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS") - - for ((unit, scale, literals) <- intMaxInstants; parsed <- Seq(false, true)) { - test(s"int_max_${unit}_${if (parsed) "parsed" else "json"}") { - spark.conf.set("spark.sql.session.timeZone", "UTC") - val instants = literals.map(Instant.parse) - for ((input, instant) <- intMaxInputs.zip(instants)) { - val converted = unit match { - case "seconds" => Instant.ofEpochSecond(input) - case "millis" => Instant.ofEpochMilli(input) - case "micros" => Instant.EPOCH.plus(input, java.time.temporal.ChronoUnit.MICROS) - } - require(converted == instant && mismatchMicros(instant) == input * scale) - } - def wall(index: Int) = - java.time.LocalDateTime.ofInstant(instants(index - 1), java.time.ZoneOffset.UTC) - val rows = intMaxIds.map { - case 7 => Row(7, null, null, null, null) - case id => - val p = intMaxPartition(id) - Row(id, Timestamp.from(instants(id - 1)), wall(id), Timestamp.from(instants(p - 1)), - wall(p)) - } - def partitionStrings(id: Int): Map[String, String] = if (id == 7) { - Map("p_ts" -> null, "p_ntz" -> null) - } else { - val p = wall(intMaxPartition(id)).format(sixDigits) - Map("p_ts" -> (p + "Z"), - "p_ntz" -> p.replace('T', ' ').replaceAll("0+$", "").stripSuffix(".")) - } - def fileId(add: AddFile): Int = - JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() - - sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ, p_ts TIMESTAMP, - p_ntz TIMESTAMP_NTZ) USING delta PARTITIONED BY (p_ts, p_ntz) TBLPROPERTIES ( - 'delta.checkpointPolicy' = 'classic', 'delta.checkpoint.writeStatsAsStruct' = 'true', - 'delta.checkpoint.writeStatsAsJson' = 'false')""") - val inputs = intMaxIds.map { - case 7 => "(7, CAST(NULL AS BIGINT), CAST(NULL AS BIGINT))" - case id => s"($id, ${intMaxInputs(id - 1)}L, ${intMaxInputs(intMaxPartition(id) - 1)}L)" - } - val f = s"timestamp_$unit" - sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ id, $f(v), CAST($f(v) AS TIMESTAMP_NTZ), " + - s"$f(p), CAST($f(p) AS TIMESTAMP_NTZ) FROM VALUES ${inputs.mkString(", ")} " + - "AS rows(id, v, p)") - val table = registerTable("tbl") - val produced = CommitLog.read(table.sourcePath, 1L).collect { case a: AddFile => a } - require(produced.map(a => fileId(a) -> a.partitionValues).toMap == - (1 to 7).map(id => id -> partitionStrings(id)).toMap && produced.size == 7) - - if (parsed) forceCheckpoint("tbl") else { - // The producer writes millisecond JSON bounds. Keep all six fraction digits instead. - modifyCommitActions(table, 1L) { actions => - actions.map { - case add: AddFile if fileId(add) != 7 => - val id = fileId(add) - println(s"Producer JSON stats before rewrite: $unit id=$id ${add.stats.get}") - val stats = JsonUtil.mapper.readTree(add.stats.get) - for (bound <- Seq("minValues", "maxValues")) { - stats.get(bound).asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] - .put("ts", wall(id).format(sixDigits) + "Z") - .put("ntz", wall(id).format(sixDigits)) - } - add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) - case other => other - } - } - } - mutateTable(table) { dir => - if (parsed) { - val checkpoint = dir.resolve("_delta_log/00000000000000000001.checkpoint.parquet") - rewriteIntMaxCheckpoint(checkpoint, instants) - checkIntMaxCheckpoint(checkpoint, instants, partitionStrings) - readinessPruneHistory(dir, 1L) - } else { - readinessRemoveChecksums(dir) - val adds = CommitLog.read(dir, 1L).collect { case a: AddFile => a } - require(adds.size == 7) - for (add <- adds) { - val id = fileId(add) - val stats = JsonUtil.mapper.readTree(add.stats.get) - require(add.partitionValues == partitionStrings(id)) - require(stats.get("numRecords").asLong() == (if (Set(2, 5, 7)(id)) 2L else 1L)) - for (bound <- Seq("minValues", "maxValues")) { - if (id == 7) require(!stats.get(bound).has("ts") && !stats.get(bound).has("ntz")) - else { - require(stats.at(s"/$bound/ts").asText() == wall(id).format(sixDigits) + "Z") - require(stats.at(s"/$bound/ntz").asText() == wall(id).format(sixDigits)) - } - } - if (id == 7) require(stats.at("/nullCount/ts").asLong() == 2L) - } - println(s"Verified six-digit JSON stats and partitions: $unit files=7") - } - } - - readinessRead(table, "all", intMaxSchema, rows) - readinessRead(table, "all_v1", intMaxSchema, rows, version = 1L) - for ((field, kind) <- Seq("ts" -> "TIMESTAMP", "ntz" -> "TIMESTAMP_NTZ", - "p_ts" -> "TIMESTAMP", "p_ntz" -> "TIMESTAMP_NTZ")) { - def v(index: Int): String = if (kind == "TIMESTAMP") { - s"TIMESTAMP '${instants(index - 1)}'" - } else s"TIMESTAMP_NTZ '${wall(index).format(sixDigits).replace('T', ' ')}'" - def index(row: Row): Option[Int] = row.getInt(0) match { - case 7 => None - case id => Some(if (field.startsWith("p_")) intMaxPartition(id) else id) - } - for ((label, predicate, selected) <- Seq( - ("below_negative", s"$field < ${v(2)}", Set(1)), - ("eq_negative", s"$field = ${v(2)}", Set(2)), - ("interior", s"$field > ${v(2)} AND $field < ${v(5)}", Set(3, 4)), - ("eq_positive", s"$field = ${v(5)}", Set(5)), - ("above_positive", s"$field > ${v(5)}", Set(6)), - ("outside", s"$field < ${v(1)} OR $field > ${v(6)}", Set.empty[Int]))) { - readinessRead(table, s"${field}_$label", intMaxSchema, - rows.filter(r => index(r).exists(selected)), predicate) - } - readinessRead(table, s"${field}_is_null", intMaxSchema, - rows.filter(_.getInt(0) == 7), s"$field IS NULL") - } - readinessSnapshot(table, intMaxSchema, Seq("p_ts", "p_ntz")) - readinessSnapshot(table, intMaxSchema, Seq("p_ts", "p_ntz"), version = 1L) - } - } - - // DATE32 epoch-day bounds. 2147483648 is outside DATE32 and is not an input. Rows use - // java.time.LocalDate: java.sql.Date misrenders these years. Raw INT32 days are the oracle. - test("date_epoch_days_int32_bounds") { - spark.conf.set("spark.sql.session.timeZone", "UTC") - spark.conf.set("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED") - spark.conf.set("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED") - // Spark's data-filter translation rebases java.sql.Date literals to Julian days, which - // overflows near -2147483648. LocalDate literals avoid that reference-side conversion. - // Capture runs after this body, so afterEach restores the setting, including on failure. - java8Previous = Some(spark.conf.get(java8Key)) - spark.conf.set(java8Key, "true") - val days = Seq(-2147483648, -2147483647, -2147483646, 2147483646, 2147483647) - val dates = Seq("-5877641-06-23", "-5877641-06-24", "-5877641-06-25", "+5881580-07-10", - "+5881580-07-11").map(LocalDate.parse) - require(dates.map(_.toEpochDay) == days.map(_.toLong)) - val permutation = Map(1 -> 3, 2 -> 5, 3 -> 1, 4 -> 2, 5 -> 4) - val ids = Seq(1, 2, 2, 3, 4, 5, 5, 6, 6) - val expectedSchema = new StructType().add("id", IntegerType) - .add("d", org.apache.spark.sql.types.DateType).add("p", org.apache.spark.sql.types.DateType) - val rows = ids.map { - case 6 => Row(6, null, null) - case id => Row(id, dates(id - 1), dates(permutation(id) - 1)) - } - val units = ids.map { - case 6 => (6, None, None) - case id => (id, Some(days(id - 1)), Some(days(permutation(id) - 1))) - } - sql("CREATE TABLE tbl (id INT, d DATE, p DATE) USING delta PARTITIONED BY (p)") - val inputs = ids.map { - case 6 => "(6, CAST(NULL AS INT), CAST(NULL AS INT))" - case id => s"($id, ${days(id - 1)}, ${days(permutation(id) - 1)})" - } - sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ id, date_from_unix_date(d), " + - s"date_from_unix_date(p) FROM VALUES ${inputs.mkString(", ")} AS rows(id, d, p)") - val table = registerTable("tbl") - mutateTable(table) { dir => - readinessRemoveChecksums(dir) - val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } - require(adds.size == 6) - adds.foreach { add => - val stats = JsonUtil.mapper.readTree(add.stats.get) - val id = stats.at("/minValues/id").intValue() - require(stats.get("numRecords").asLong() == (if (Set(2, 5, 6)(id)) 2L else 1L)) - if (id == 6) { - require(add.partitionValues == Map("p" -> null)) - require(stats.at("/nullCount/d").asLong() == 2L && !stats.at("/minValues").has("d")) - } else { - require(add.partitionValues == Map("p" -> dates(permutation(id) - 1).toString)) - for (bound <- Seq("minValues", "maxValues")) { - require(stats.at(s"/$bound/d").asText() == dates(id - 1).toString) - } - } - println(s"Verified DATE32 bound file: id=$id stats=${add.stats.get} " + - s"partition=${add.partitionValues}") - } - } - def literal(index: Int): String = s"DATE '${dates(index - 1)}'" - readinessRead(table, "all", expectedSchema, rows) - readinessRead(table, "all_v1", expectedSchema, rows, version = 1L) - for (field <- Seq("d", "p")) { - def index(row: Row): Option[Int] = row.getInt(0) match { - case 6 => None - case id => Some(if (field == "p") permutation(id) else id) - } - for ((label, predicate, selected) <- Seq( - ("below_negative", s"$field < ${literal(2)}", Set(1)), - ("eq_negative", s"$field = ${literal(2)}", Set(2)), - ("interior", s"$field > ${literal(2)} AND $field < ${literal(5)}", Set(3, 4)), - ("eq_positive", s"$field = ${literal(5)}", Set(5)))) { - readinessRead(table, s"${field}_$label", expectedSchema, - rows.filter(r => index(r).exists(selected)), predicate) - } - readinessRead(table, s"${field}_is_null", expectedSchema, - rows.filter(_.getInt(0) == 6), s"$field IS NULL") - } - readinessSnapshot(table, expectedSchema, Seq("p")) - readinessSnapshot(table, expectedSchema, Seq("p"), version = 1L) - // Snapshots capture before reads at this pin. Inspect Parquet in the last read callback. - val workload = WorkloadContext.current.workloadName - readSpec(table, predicate = s"d < ${literal(1)} OR d > ${literal(5)}", - name = Some("outside")).assert { spec => - spec.expectation match { - case Succeeded(result) => require(result.rowCount == 0L) - case other => fail(s"Outside DATE32 bounds must succeed with no rows: $other") - } - val conf = spark.sparkContext.hadoopConfiguration - val captured = outputDir.resolve(workload).resolve("expected") - .resolve(s"${workload}_all").resolve("expected_data") - val literalOracle = outputDir.resolveSibling("oracle-parquet").resolve(workload) - for ((label, directory) <- Seq("captured" -> captured, "literal" -> literalOracle)) { - val files = Files.list(directory) - val parquet = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) - .toVector finally files.close() - require(parquet.nonEmpty) - val found = parquet.flatMap { file => - val path = new HadoopPath(file.toUri) - val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) - try for (field <- Seq("d", "p")) { - val physical = footer.getFooter.getFileMetaData.getSchema.getFields.asScala - .find(_.getName == field).get.asPrimitiveType() - require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) - require(physical.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) - } finally footer.close() - val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) - .withConf(conf).build() - var out = Vector.empty[(Int, Option[Int], Option[Int])] - try { - var record = reader.read() - while (record != null) { - def day(field: String) = if (record.getFieldRepetitionCount(field) == 0) None - else Some(record.getInteger(field, 0)) - out :+= ((record.getInteger("id", 0), day("d"), day("p"))) - record = reader.read() - } - } finally reader.close() - out - } - require(found.sorted == units.sorted) - println(s"Verified raw DATE32 epoch days: $workload $label ${found.sorted}") - } - require(spark.conf.get(java8Key) == "true") - } - } - - private val java8Key = "spark.sql.datetime.java8API.enabled" - private var java8Previous: Option[String] = None - - override def afterEach(): Unit = { - try super.afterEach() finally { - java8Previous.foreach(previous => if (_spark != null) _spark.conf.set(java8Key, previous)) - java8Previous = None - } - } - - // Replace producer-derived parsed bounds by the literal instants and drop the JSON fallback. - private def rewriteIntMaxCheckpoint(checkpoint: Path, instants: Seq[Instant]): Unit = { - val rewrite = checkpoint.resolveSibling("int-max-checkpoint-rewrite") - val confKey = "spark.sql.parquet.outputTimestampType" - val previous = spark.conf.get(confKey) - try { - spark.conf.set(confKey, "TIMESTAMP_MICROS") - val df = spark.read.parquet(checkpoint.toString) - var stats = col("add.stats_parsed") - for (bound <- Seq("minValues", "maxValues")) { - var ts = col(s"add.stats_parsed.$bound.ts") - var ntz = col(s"add.stats_parsed.$bound.ntz") - for ((instant, index) <- instants.zipWithIndex) { - val matches = col("add.stats_parsed.minValues.id") === lit(index + 1) - ts = when(matches, lit(Timestamp.from(instant))).otherwise(ts) - ntz = when(matches, - lit(java.time.LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC))) - .otherwise(ntz) - } - stats = stats.withField(bound, col(s"add.stats_parsed.$bound") - .withField("ts", ts).withField("ntz", ntz)) - } - var add = col("add").withField("stats_parsed", stats) - if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { - add = add.withField("stats", lit(null).cast("string")) - } - df.withColumn("add", add).coalesce(1).write.parquet(rewrite.toString) - val files = Files.list(rewrite) - val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) - .toSeq match { - case Seq(single) => single - case other => fail(s"Expected one rewritten INT_MAX checkpoint: $other") - } finally files.close() - Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) - } finally { - spark.conf.set(confKey, previous) - if (Files.exists(rewrite)) FileUtils.deleteDirectory(rewrite.toFile) - } - } - - private def checkIntMaxCheckpoint( - checkpoint: Path, instants: Seq[Instant], partitions: Int => Map[String, String]): Unit = { - val conf = spark.sparkContext.hadoopConfiguration - val path = new HadoopPath(checkpoint.toUri) - val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) - try { - val physical = footer.getFooter.getFileMetaData.getSchema - for ((prefix, field) <- Seq("stats_parsed.minValues" -> "ts", - "stats_parsed.minValues" -> "ntz", "stats_parsed.maxValues" -> "ts", - "stats_parsed.maxValues" -> "ntz", "partitionValues_parsed" -> "p_ts", - "partitionValues_parsed" -> "p_ntz")) { - val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) - .asPrimitiveType() - require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) - val annotation = cell.getLogicalTypeAnnotation - .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] - require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) - require(annotation.isAdjustedToUTC == !field.endsWith("ntz")) - println(s"Verified INT_MAX checkpoint $prefix.$field: $cell") - } - } finally footer.close() - val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() - var ids = Vector.empty[Int] - try { - var record = reader.read() - while (record != null) { - if (record.getFieldRepetitionCount("add") > 0) { - val add = record.getGroup("add", 0) - if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) - val stats = add.getGroup("stats_parsed", 0) - val id = stats.getGroup("minValues", 0).getInteger("id", 0) - require(stats.getLong("numRecords", 0) == (if (Set(2, 5, 7)(id)) 2L else 1L)) - val parsedPartition = add.getGroup("partitionValues_parsed", 0) - for (bound <- Seq("minValues", "maxValues")) { - val values = stats.getGroup(bound, 0) - require(values.getInteger("id", 0) == id) - for (field <- Seq("ts", "ntz")) { - if (id == 7) require(values.getFieldRepetitionCount(field) == 0) - else require(values.getLong(field, 0) == mismatchMicros(instants(id - 1))) - } - } - for (field <- Seq("p_ts", "p_ntz")) { - if (id == 7) require(parsedPartition.getFieldRepetitionCount(field) == 0) - else require(parsedPartition.getLong(field, 0) == - mismatchMicros(instants(intMaxPartition(id) - 1))) - } - if (id == 7) require(stats.getGroup("nullCount", 0).getLong("ts", 0) == 2L) - val raw = add.getGroup("partitionValues", 0) - val partition = (0 until raw.getFieldRepetitionCount("key_value")).map { i => - val entry = raw.getGroup("key_value", i) - val value = if (entry.getFieldRepetitionCount("value") == 0) null - else entry.getString("value", 0) - entry.getString("key", 0) -> value - }.toMap - require(partition == partitions(id)) - ids :+= id - println(s"Verified INT_MAX raw checkpoint id=$id partition=$partition") - } - record = reader.read() - } - } finally reader.close() - require(ids.sorted == (1 to 7).toVector) - } -} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTypeWideningSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTypeWideningSuite.scala deleted file mode 100644 index 032b448b..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadinessTypeWideningSuite.scala +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (2026) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.tables - -import java.nio.file.Files - -import scala.jdk.CollectionConverters._ - -import org.apache.spark.sql.types.{DataType, StructType} - -import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} -import io.delta.workload.engine.RowComparison -import io.delta.workload.json.JsonUtil -import io.delta.workload.log.AddFile -import io.delta.workload.model._ - -/** Widened JSON statistics, old parsed statistics, and partition values are separate workloads. */ -class ReadinessTypeWideningSuite extends WorkloadTestSuite("readiness_type_widening") { - - private case class Stage(dataType: String, values: Seq[String]) - private case class Widening(name: String, stages: Seq[Stage], jsonGap: Boolean = true) - - // The numeric chain, decimal physical encodings, and DATE values follow TypeWideningSuite. - // One transition corpus drives all three representations. The chain's SHORT stage also covers - // BYTE -> SHORT; existing full/time-travel JSON reads are not redeclared here. - private val widenings = Seq( - Widening("byte_int", Seq( - Stage("BYTE", Seq("1", "127")), Stage("INT", Seq("128", "2147483647")))), - Widening("byte_long", Seq( - Stage("BYTE", Seq("1", "127")), - Stage("BIGINT", Seq("128", "9223372036854775807")))), - Widening("short_int", Seq( - Stage("SHORT", Seq("10", "100")), Stage("INT", Seq("40000", "50000"))), - jsonGap = false), - Widening("short_long", Seq( - Stage("SHORT", Seq("100", "32767")), - Stage("BIGINT", Seq("40000", "9223372036854775807")))), - Widening("int_long", Seq( - Stage("INT", Seq("100", "200", "300", "400")), - Stage("BIGINT", Seq("3000000000", "9000000000000"))), jsonGap = false), - Widening("byte_double", Seq( - Stage("BYTE", Seq("1", "127")), - Stage("DOUBLE", Seq("127.5", "2147483648.25")))), - Widening("short_double", Seq( - Stage("SHORT", Seq("100", "32767")), - Stage("DOUBLE", Seq("32767.5", "2147483648.25")))), - Widening("int_double", Seq( - Stage("INT", Seq("100", "2147483647")), - Stage("DOUBLE", Seq("2147483647.5", "4294967296.25")))), - Widening("float_double", Seq( - Stage("FLOAT", Seq("1.5", "3.125")), - Stage("DOUBLE", Seq("3.141592653589793", "1.7976931348623157E308"))), - jsonGap = false), - Widening("decimal_precision", Seq( - Stage("DECIMAL(5,2)", Seq("123.45", "999.99")), - Stage("DECIMAL(10,2)", Seq("12345678.90", "99999999.99"))), jsonGap = false), - Widening("decimal_precision_scale", Seq( - Stage("DECIMAL(9,2)", Seq("123.45", "9999999.99")), - Stage("DECIMAL(18,2)", Seq("1234567890123.45", "9999999999999999.99")), - Stage("DECIMAL(28,3)", Seq("1234567890123456789012345.678")))), - Widening("date_timestamp_ntz", Seq( - Stage("DATE", Seq("'2024-01-15'", "'2024-06-30'")), - Stage("TIMESTAMP_NTZ", Seq("'2024-12-31 23:59:59'", "'2025-01-01 12:30:00'")))), - Widening("multiple_steps", Seq( - Stage("BYTE", Seq("1", "100")), Stage("SHORT", Seq("200", "30000")), - Stage("INT", Seq("40000", "2000000000")), - Stage("BIGINT", Seq("3000000000", "9000000000000000000"))))) - - // (stable id, literal, original type). Expected bags are selected explicitly by id, never by - // filtering the table or capturing a fresh query result as its own oracle. - private def rows(w: Widening, stage: Int, partitioned: Boolean): Seq[(Int, String, String)] = - w.stages.take(stage + 1).zipWithIndex.flatMap { case (s, i) => - s.values.zipWithIndex.map { case (v, j) => (i * 10 + j + 1, v, s.dataType) } ++ - (if (partitioned) Seq((i * 10 + 9, "NULL", s.dataType)) else Seq.empty) - } - - private def literal(value: String, from: String, to: String): String = - s"CAST(CAST($value AS $from) AS $to)" - - private def checkedRead( - t: TableHandle, - label: String, - dataType: String, - expected: Seq[(Int, String, String)], - predicate: String, - version: Long): Unit = { - val table = WorkloadContext.current.getTableSpec(t) - readSpec(t, version = version, predicate = predicate, name = Some(label)).assert { spec => - spec.expectation match { - case Succeeded(_) => - case other => fail(s"${table.outputName}/$label must succeed: $other") - } - val selects = expected.map { case (id, value, from) => - s"SELECT CAST($id AS INT) AS id, ${literal(value, from, dataType)} AS value" - } - val oracle = if (selects.nonEmpty) selects.mkString(" UNION ALL ") else - s"SELECT CAST(NULL AS INT) AS id, CAST(NULL AS $dataType) AS value WHERE false" - val expectedPath = outputDir.resolve(table.outputName).resolve("expected") - .resolve(s"${table.outputName}_$label").resolve("expected_data") - RowComparison.assertRowsEqual(spark.sql(oracle), - spark.read.parquet(expectedPath.toString), s"${table.outputName}/$label independent oracle") - } - } - - private def checkedSnapshot(t: TableHandle, w: Widening, stage: Int, - partitioned: Boolean, version: Long): Unit = { - snapshotSpec(t, version = version).assert { spec => - spec.expectation match { - case Succeeded(result) => - val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] - val expected = StructType.fromDDL(s"id INT, value ${w.stages(stage).dataType}") - require(schema.map(f => f.name -> f.dataType) == - expected.map(f => f.name -> f.dataType), s"Unexpected schema: $schema") - require(result.metadata.partitionColumns == - (if (partitioned) Seq("value") else Seq.empty)) - val fieldMetadata = schema("value").metadata - val changes = if (fieldMetadata.contains("delta.typeChanges")) - fieldMetadata.getMetadataArray("delta.typeChanges").toSeq else Seq.empty - val expectedChanges = w.stages.take(stage + 1).sliding(2).collect { - case Seq(from, to) => - (DataType.fromDDL(from.dataType).json, DataType.fromDDL(to.dataType).json) - }.toSeq - require(changes.map(c => (s"\"${c.getString("fromType")}\"", - s"\"${c.getString("toType")}\"")) == expectedChanges, - s"Unexpected widening history at $version: $changes") - case other => fail(s"Snapshot at $version must succeed: $other") - } - } - } - - private def prepareStats(t: TableHandle, representation: String, original: Stage): Unit = { - mutateTable(t) { tableDir => - val logDir = tableDir.resolve("_delta_log") - val stream = Files.list(logDir) - val paths = try stream.iterator().asScala.toVector finally stream.close() - // CRC file-state snapshots must not substitute for the statistics under test. - paths.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) - if (representation == "parsed") { - val checkpoint = logDir.resolve("00000000000000000001.checkpoint.parquet") - require(Files.exists(checkpoint), "Missing old-schema checkpoint") - val adds = spark.read.parquet(checkpoint.toString).filter("add IS NOT NULL") - val stats = adds.selectExpr("add.stats_parsed.minValues.value AS minValue", - "add.stats_parsed.maxValues.value AS maxValue", "add.stats_parsed.numRecords AS records") - require(stats.schema.head.dataType == DataType.fromDDL(original.dataType), - s"Checkpoint must retain ${original.dataType} statistics: ${stats.schema}") - val addSchema = adds.schema("add").dataType.asInstanceOf[StructType] - if (addSchema.fieldNames.contains("stats")) { - require(adds.filter("add.stats IS NOT NULL").count() == 0, - "Parsed-stat checkpoint must not contain JSON statistics") - } - val oracle = s"""SELECT CAST(${original.values.head} AS ${original.dataType}) AS minValue, - CAST(${original.values.last} AS ${original.dataType}) AS maxValue, - CAST(${original.values.size} AS BIGINT) AS records""" - RowComparison.assertRowsEqual(spark.sql(oracle), stats.selectExpr( - "MIN(minValue) AS minValue", "MAX(maxValue) AS maxValue", "SUM(records) AS records"), - "old parsed checkpoint min/max/count oracle") - // Keep the checkpoint's boundary commit for version discovery; its adds are removed below. - Files.delete(logDir.resolve("00000000000000000000.json")) - } else { - require(!paths.exists(_.getFileName.toString.contains("checkpoint")), - "JSON/partition workloads must not have checkpoints") - val actions = paths.filter(_.getFileName.toString.endsWith(".json")) - .flatMap(p => Files.readAllLines(p).asScala) - .map(JsonUtil.mapper.readTree(_)) - val adds = actions.filter(_.has("add")).map(_.get("add")) - require(adds.nonEmpty) - if (representation == "json") { - require(adds.forall { a => - a.hasNonNull("stats") && Seq("minValues", "maxValues").forall { bound => - JsonUtil.mapper.readTree(a.get("stats").asText()).path(bound).has("value") - } - }) - } else { - require(adds.forall(_.get("partitionValues").has("value"))) - require(adds.exists(_.get("partitionValues").get("value").isNull), - "Retain a null partition written before widening") - } - } - } - if (representation == "parsed") { - modifyCommitActions(t, version = 1)(_.filterNot(_.isInstanceOf[AddFile])) - } - } - - for { - w <- widenings - representation <- Seq("json", "parsed", "partition") - if representation != "json" || w.jsonGap - } { - test(s"${representation}_${w.name}") { - val partitioned = representation == "partition" - val partition = if (partitioned) "PARTITIONED BY (value)" else "" - val timestampFeature = if (w.name == "date_timestamp_ntz") - ", 'delta.feature.timestampNtz' = 'supported'" else "" - val statsProperties = if (representation == "parsed") - ", 'delta.checkpoint.writeStatsAsStruct' = 'true', " + - "'delta.checkpoint.writeStatsAsJson' = 'false'" else "" - sql(s"""CREATE TABLE tbl (id INT, value ${w.stages.head.dataType}) USING delta - $partition TBLPROPERTIES ('delta.enableTypeWidening' = 'true', - 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '100', - 'delta.checkpointPolicy' = 'classic' $timestampFeature $statsProperties)""") - - for ((s, i) <- w.stages.zipWithIndex) { - if (i > 0) sql(s"ALTER TABLE tbl ALTER COLUMN value TYPE ${s.dataType}") - val inserted = rows(w, i, partitioned).filter(_._1 / 10 == i) - val values = inserted.map { case (id, value, from) => - s"($id, CAST($value AS $from))" - }.mkString(", ") - sql(s"INSERT INTO tbl VALUES $values") - if (i == 0 && representation == "parsed") forceCheckpoint("tbl") - } - - val t = registerTable("tbl") - prepareStats(t, representation, w.stages.head) - - // Capture the requested latest-schema assertions before historical-stage controls. - for ((s, i) <- w.stages.zipWithIndex.reverse) { - val version = i * 2L + 1 - checkedSnapshot(t, w, i, partitioned, version) - val precisionOnlyStats = representation != "partition" && - w.name == "decimal_precision_scale" && i == 1 - if (i > 0 && !precisionOnlyStats) { - val bag = rows(w, i, partitioned) - val nonNull = bag.filter(_._2 != "NULL") - val original = nonNull.filter(_._1 < 10) - val previous = w.stages(i - 1) - val firstNew = literal(s.values.head, s.dataType, s.dataType) - val lastOld = literal(previous.values.last, previous.dataType, s.dataType) - val lastOriginal = literal(w.stages.head.values.last, - w.stages.head.dataType, s.dataType) - val firstOriginal = literal(w.stages.head.values.head, - w.stages.head.dataType, s.dataType) - val prefix = if (w.name == "multiple_steps" && i == 1) "byte_short" else w.name - val label = s"${prefix}_v$version" - val read = (suffix: String, predicate: String, selected: Seq[(Int, String, String)]) => - checkedRead(t, s"${label}_$suffix", s.dataType, selected, predicate, version) - - // JSON partial cases contribute filters only; TypeWideningSuite owns their full reads. - val partialJson = representation == "json" && - Set("multiple_steps", "decimal_precision_scale", "date_timestamp_ntz").contains(w.name) - if (!partialJson) read("all", null, bag) - read("old_range", s"value <= $lastOriginal", original) - read("new_range", s"value > $lastOld", nonNull.filter(_._1 / 10 == i)) - read("cross_range", s"value >= $lastOld AND value <= $firstNew", - nonNull.filter(r => r._1 == (i - 1) * 10 + previous.values.size || - r._1 == i * 10 + 1)) - read("miss", s"value < $firstOriginal", Seq.empty) - - if (partitioned) { - read("old_equality", s"value = $firstOriginal", original.take(1)) - read("new_equality", s"value = $firstNew", nonNull.filter(_._1 == i * 10 + 1)) - read("null_partitions", "value IS NULL", bag.filter(_._2 == "NULL")) - } - if (w.name == "date_timestamp_ntz") { - read("old_midnight", s"value = $lastOriginal", original.takeRight(1)) - read("new_midnight_miss", "value = TIMESTAMP_NTZ'2024-12-31 00:00:00'", Seq.empty) - read("exclusive_boundary", s"value >= $lastOriginal AND value < $firstNew", - original.takeRight(1)) - } - } - } - } - } -} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadsSuite.scala index 61e701f2..c19ed18e 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadsSuite.scala @@ -16,7 +16,21 @@ package io.delta.workload.tables +import java.nio.file.{Files, Path, Paths} + +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + import io.delta.workload.WorkloadTestSuite +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog, Metadata, Protocol} +import io.delta.workload.model._ /** * Consolidated read workloads. @@ -540,4 +554,138 @@ class ReadsSuite extends WorkloadTestSuite("reads") { snapshotSpec(t) } + // Relative Unicode file paths in a relocated table. + test("e2e_58_unicode_路径") { + // The generator copies contents into /delta. Relative log paths must + // survive both the Unicode source root and that different destination root. + val parent = Files.createTempDirectory("paths-source-") + val source = parent.resolve("源_café_表_😀") + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta LOCATION '$source' + TBLPROPERTIES ('delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '1000', 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'base'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (4, CAST(NULL AS STRING)), (23, 'end'), + (CAST(NULL AS INT), CAST(NULL AS STRING)) AS v(id, value)""") + // Canonically equivalent NFC/NFD basenames must remain distinct byte names. + val basenames = Seq("données_café_表_😀.parquet", "données_cafe\u0301_表_😀.parquet") + assert(basenames.distinct.size == 2) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val first = Seq(Row(-7, "base"), Row(4, null), Row(4, null), Row(null, "missing")) + val second = Seq(Row(4, null), Row(23, "end"), Row(null, null)) + val inputRows = Seq(first, second) + val renamedAdds = (1L to 2L).map { v => + val add = CommitLog.read(source, v).collect { case a: AddFile => a } + .ensuring(_.size == 1).head + assert(!add.path.contains("/") && add.deletionVector.isEmpty) + val basename = basenames((v - 1).toInt) + val file = source.resolve(basename) + Files.move(source.resolve(add.path), file) + // Change only the file reference in raw JSON; preserve every other action field. + val commit = CommitLog.commitFile(source, v) + val lines = Files.readAllLines(commit).asScala.map { line => + val node = JsonUtil.mapper.readTree(line) + if (node.has("add")) { + node.get("add").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("path", new HadoopPath(basename).toUri.toString) + } + node.toString + } + Files.write(commit, lines.asJava) + val physical = spark.read.parquet(file.toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(inputRows((v - 1).toInt).asJava, + schema), physical, s"paths source physical v$v") + val renamed = CommitLog.read(source, v).collect { case a: AddFile => a }.head + assert(new HadoopPath(renamed.path).toUri.getPath == basename) + assert(Files.size(file) == renamed.size) + renamed + } + // Version checksums can embed the old paths; do not transport stale checksums. + val logs = Files.list(source.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + val commits = (0L to 2L).map(v => v -> Files.readAllBytes( + CommitLog.commitFile(source, v))).toMap + val metadata = CommitLog.read(source, 0L).collect { case m: Metadata => m } + .ensuring(_.size == 1).head + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(CommitLog.read(source, 0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + val table = registerTableFromPath(source.toString) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir != source && dir.getParent.getFileName.toString.endsWith("路径")) + commits.foreach { case (v, bytes) => + assert(Files.readAllBytes(CommitLog.commitFile(dir, v)).sameElements(bytes)) + assert(!new String(bytes, java.nio.charset.StandardCharsets.UTF_8) + .contains(source.toString)) + } + basenames.foreach { basename => + val file = dir.resolve(basename) + assert(file.getFileName.toString.getBytes(java.nio.charset.StandardCharsets.UTF_8) + .sameElements(basename.getBytes(java.nio.charset.StandardCharsets.UTF_8))) + assert(Files.readAllBytes(file).sameElements(Files.readAllBytes(source.resolve(basename)))) + } + // Remove the entire source before any snapshot/read capture. An absolute or + // stale reference to it cannot make the relocated fixture pass by accident. + org.apache.commons.io.FileUtils.deleteDirectory(parent.toFile) + assert(!Files.exists(parent) && !Files.exists(source)) + DeltaLog.clearCache() + info(s"paths source removed: $source; relocated root: $dir") + } + Seq[java.lang.Long](0L, 1L, 2L, null).foreach { version => + val v = Option(version).fold(2L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(!Files.exists(source)) + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + assert(snapshot.allFiles.collect().map(_.path).toSet == + renamedAdds.take(v.toInt).map(_.path).toSet) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_v1", ReadQuery(version = Some(1L)), schema, first, 1), + ("read_v2", ReadQuery(version = Some(2L)), schema, first ++ second, 2), + ("read_latest", ReadQuery(), schema, first ++ second, 2), + ("read_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row(null, 4), Row("missing", null), Row(null, null)), 2)) + cases.foreach { case (name, query, expectedSchema, rows, files) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(!Files.exists(source)) + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, files, 0L))) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + val paths = actual.inputFiles.map(p => Paths.get(java.net.URI.create(p))).toSet + val expectedFiles = basenames.take(files).map(copiedTable.resolve).toSet + assert(paths == expectedFiles && paths.forall(_.startsWith(copiedTable))) + assert(actual.schema == expectedSchema) + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"paths $name relocated table") + val fixture = copiedTable.getParent + val captured = spark.read.parquet(fixture.resolve( + s"expected/${fixture.getFileName}_$name/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"paths $name expected parquet") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/SchemaEvolutionSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/SchemaEvolutionSuite.scala index 9feef2e6..d27d015d 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/SchemaEvolutionSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/SchemaEvolutionSuite.scala @@ -16,7 +16,19 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.Files + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.types.{ByteType, DataType, DoubleType, IntegerType, Metadata, + StringType, StructType} + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.CommitLog +import io.delta.workload.model.Succeeded class SchemaEvolutionSuite extends WorkloadTestSuite("schema_evolution") { @@ -454,4 +466,462 @@ class SchemaEvolutionSuite extends WorkloadTestSuite("schema_evolution") { snapshotSpec(t) } + // CREATE OR REPLACE with same-name columns of a different type, changed partition and + // file sets, and fresh column-mapping identities. + // Shared by the stats-column and partition-column variants of the same replacement scenario. + private def replaceSameName( + caseName: String, + partitioned: Boolean, + fromType: DataType, + toType: DataType, + seedValues: String, + replacementValues: String, + beforeRows: Seq[Row], + afterRows: Seq[Row], + matchingPredicate: String, + missingPredicate: String): Unit = { + val partitionClause = if (partitioned) "PARTITIONED BY (c)" else "" + val beforeSchema = new StructType().add("id", IntegerType) + .add("c", fromType).add("payload", StringType) + val afterSchema = new StructType().add("id", IntegerType) + .add("c", toType).add("payload", StringType) + + sql(s"CREATE TABLE tbl (${beforeSchema.toDDL}) USING delta $partitionClause") + sql(s"INSERT INTO tbl VALUES $seedValues") + // This is a REPLACE TABLE commit, rather than DROP/ADD COLUMN or an append with evolution. + sql(s"CREATE OR REPLACE TABLE tbl (${afterSchema.toDDL}) USING delta $partitionClause") + sql(s"INSERT INTO tbl VALUES $replacementValues") + val t = registerTable("tbl") + val table = io.delta.workload.WorkloadContext.current.getTableSpec(t) + + // Literal rows form an oracle independent of reading the generated Delta table. + def frame(rows: Seq[Row], schema: StructType): DataFrame = + spark.createDataFrame(rows.asJava, schema) + + def checkRead( + name: String, + rows: Seq[Row], + schema: StructType, + version: java.lang.Long = null, + predicate: String = null, + columns: Option[Seq[String]] = None): Unit = { + val oracle = frame(rows, schema) + val projectedOracle = columns.map(cs => oracle.selectExpr(cs: _*)).getOrElse(oracle) + readSpec(t, name = Some(name), version = version, predicate = predicate, + columns = columns).assert { spec => + val workloadName = table.outputName + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$workloadName/$name must succeed: $other") + } + val expectedPath = outputDir.resolve(workloadName).resolve("expected") + .resolve(s"${workloadName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expectedPath.toString) + assert(captured.schema == projectedOracle.schema) + RowComparison.assertRowsEqual( + projectedOracle, captured, s"$workloadName/$name literal oracle") + println(s"Independent row oracle passed: $workloadName/$name (${rows.size} rows)") + } + } + + checkRead("read_all", afterRows, afterSchema) + checkRead("read_before", beforeRows, beforeSchema, version = 1L) + checkRead("read_replaced_empty", Seq.empty, afterSchema, version = 2L) + checkRead("read_match", afterRows.take(2), afterSchema, predicate = matchingPredicate) + checkRead("read_missing", Seq.empty, afterSchema, predicate = missingPredicate) + checkRead("read_null", afterRows.slice(3, 4), afterSchema, predicate = "c IS NULL") + checkRead("read_non_null", afterRows.take(3) ++ afterRows.drop(4), afterSchema, + predicate = "c IS NOT NULL") + checkRead("read_projection", afterRows, afterSchema, columns = Some(Seq("c", "payload"))) + checkRead("read_before_projection", beforeRows, beforeSchema, version = 1L, + columns = Some(Seq("c", "payload"))) + checkRead("read_before_null", beforeRows.takeRight(1), beforeSchema, version = 1L, + predicate = "c IS NULL") + + for (v <- 0L to 3L) { + val expectedSchema = if (v < 2L) beforeSchema else afterSchema + snapshotSpec(t, version = v).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + assert(result.metadata.partitionColumns == (if (partitioned) Seq("c") else Seq.empty)) + case other => fail(s"${table.outputName} snapshot v$v must succeed: $other") + } + } + } + } + + test("string_to_int_stats") { + replaceSameName( + "string_to_int_stats", partitioned = false, StringType, IntegerType, + seedValues = """(1, 'not-an-int', 'old'), (1, 'not-an-int', 'old'), """ + + """(2, '007', 'old-padded'), (3, NULL, 'old-null')""", + replacementValues = """(101, 7, 'new-seven'), (101, 7, 'new-seven'), """ + + """(102, -5, 'new-negative'), (103, NULL, 'new-null'), (104, 0, 'new-zero')""", + beforeRows = Seq( + Row(1, "not-an-int", "old"), + Row(1, "not-an-int", "old"), + Row(2, "007", "old-padded"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, 7, "new-seven"), + Row(101, 7, "new-seven"), + Row(102, -5, "new-negative"), + Row(103, null, "new-null"), + Row(104, 0, "new-zero")), + matchingPredicate = "c = 7", missingPredicate = "c = 999") + } + + test("int_to_string_stats") { + replaceSameName( + "int_to_string_stats", partitioned = false, IntegerType, StringType, + seedValues = """(1, 7, 'old'), (1, 7, 'old'), (2, -5, 'old-negative'), """ + + """(3, NULL, 'old-null')""", + replacementValues = """(101, 'seven', 'new-word'), (101, 'seven', 'new-word'), """ + + """(102, '007', 'new-padded'), (103, NULL, 'new-null'), (104, '', 'new-empty')""", + beforeRows = Seq( + Row(1, 7, "old"), + Row(1, 7, "old"), + Row(2, -5, "old-negative"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "seven", "new-word"), + Row(101, "seven", "new-word"), + Row(102, "007", "new-padded"), + Row(103, null, "new-null"), + Row(104, "", "new-empty")), + matchingPredicate = "c = 'seven'", missingPredicate = "c = 'missing'") + } + + test("byte_to_string_stats") { + replaceSameName( + "byte_to_string_stats", partitioned = false, ByteType, StringType, + seedValues = """(1, CAST(-128 AS BYTE), 'old-min'), (1, CAST(-128 AS BYTE), 'old-min'), """ + + """(2, CAST(127 AS BYTE), 'old-max'), (3, CAST(NULL AS BYTE), 'old-null')""", + replacementValues = """(101, 'byte-min', 'new-word'), (101, 'byte-min', 'new-word'), """ + + """(102, '127', 'new-numeric'), (103, NULL, 'new-null'), (104, '', 'new-empty')""", + beforeRows = Seq( + Row(1, -128.toByte, "old-min"), + Row(1, -128.toByte, "old-min"), + Row(2, 127.toByte, "old-max"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "byte-min", "new-word"), + Row(101, "byte-min", "new-word"), + Row(102, "127", "new-numeric"), + Row(103, null, "new-null"), + Row(104, "", "new-empty")), + matchingPredicate = "c = 'byte-min'", missingPredicate = "c = 'missing'") + } + + test("string_to_int_partition") { + replaceSameName( + "string_to_int_partition", partitioned = true, StringType, IntegerType, + seedValues = """(1, 'not-an-int', 'old'), (1, 'not-an-int', 'old'), """ + + """(2, '007', 'old-padded'), (3, NULL, 'old-null')""", + replacementValues = """(101, 7, 'new-seven'), (101, 7, 'new-seven'), """ + + """(102, -5, 'new-negative'), (103, NULL, 'new-null'), (104, 0, 'new-zero')""", + beforeRows = Seq( + Row(1, "not-an-int", "old"), + Row(1, "not-an-int", "old"), + Row(2, "007", "old-padded"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, 7, "new-seven"), + Row(101, 7, "new-seven"), + Row(102, -5, "new-negative"), + Row(103, null, "new-null"), + Row(104, 0, "new-zero")), + matchingPredicate = "c = 7", missingPredicate = "c = 999") + } + + test("int_to_string_partition") { + replaceSameName( + "int_to_string_partition", partitioned = true, IntegerType, StringType, + seedValues = """(1, 7, 'old'), (1, 7, 'old'), (2, -5, 'old-negative'), """ + + """(3, NULL, 'old-null')""", + replacementValues = """(101, 'seven', 'new-word'), (101, 'seven', 'new-word'), """ + + """(102, '007', 'new-padded'), (103, NULL, 'new-null'), (104, 'zero', 'new-zero')""", + beforeRows = Seq( + Row(1, 7, "old"), + Row(1, 7, "old"), + Row(2, -5, "old-negative"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "seven", "new-word"), + Row(101, "seven", "new-word"), + Row(102, "007", "new-padded"), + Row(103, null, "new-null"), + Row(104, "zero", "new-zero")), + matchingPredicate = "c = 'seven'", missingPredicate = "c = 'missing'") + } + + test("byte_to_string_partition") { + replaceSameName( + "byte_to_string_partition", partitioned = true, ByteType, StringType, + seedValues = """(1, CAST(-128 AS BYTE), 'old-min'), (1, CAST(-128 AS BYTE), 'old-min'), """ + + """(2, CAST(127 AS BYTE), 'old-max'), (3, CAST(NULL AS BYTE), 'old-null')""", + replacementValues = """(101, 'byte-min', 'new-word'), (101, 'byte-min', 'new-word'), """ + + """(102, '127', 'new-numeric'), (103, NULL, 'new-null'), (104, 'zero', 'new-zero')""", + beforeRows = Seq( + Row(1, -128.toByte, "old-min"), + Row(1, -128.toByte, "old-min"), + Row(2, 127.toByte, "old-max"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "byte-min", "new-word"), + Row(101, "byte-min", "new-word"), + Row(102, "127", "new-numeric"), + Row(103, null, "new-null"), + Row(104, "zero", "new-zero")), + matchingPredicate = "c = 'byte-min'", missingPredicate = "c = 'missing'") + } + + // Changed partition/file sets and fresh mapped identities share the same replacement oracle. + private def replacementRead( + t: TableHandle, + name: String, + rows: Seq[Row], + schema: StructType, + version: java.lang.Long = null, + predicate: String = null, + columns: Option[Seq[String]] = None): Unit = { + val table = io.delta.workload.WorkloadContext.current.getTableSpec(t) + val ref = readSpec(t, name = Some(name), version = version, predicate = predicate, + columns = columns) + ref.assert { spec => + val workloadName = table.outputName + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$workloadName/$name must succeed: $other") + } + val literal = spark.createDataFrame(rows.asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val path = outputDir.resolve(workloadName).resolve("expected") + .resolve(s"${workloadName}_${name}").resolve("expected_data") + val captured = spark.read.parquet(path.toString) + // RowComparison checks exact names/types and duplicate-sensitive row bags. + RowComparison.assertRowsEqual(oracle, captured, s"$workloadName/$name literal oracle") + assert(captured.schema.fields.forall(_.nullable)) + println(s"Independent row oracle passed: $workloadName/$name (${rows.size} rows)") + } + } + + private def replacementLayout( + caseName: String, + layout: String, + mappingMode: Option[String] = None): Unit = { + val mapped = mappingMode.isDefined + val parsed = layout == "checkpoint_parsed" + val checkpointed = layout != "commits" + val beforeNames = if (mapped) Seq("id", "label", "score", "region") + else Seq("id", "old_label", "old_score", "old_region") + val afterNames = Seq("id", "label", "score", "region") + def logicalSchema(names: Seq[String]): StructType = new StructType() + .add(names(0), IntegerType).add(names(1), StringType) + .add(names(2), DoubleType).add(names(3), StringType) + val beforeSchema = logicalSchema(beforeNames) + val afterSchema = logicalSchema(afterNames) + val beforeRows = Seq( + Row(1, "old", 1000.0, "old-east"), Row(1, "old", 1000.0, "old-east"), + Row(2, "old-negative", -1000.0, "old-west"), Row(3, null, null, null)) + val afterRows = Seq( + Row(101, "new-a", 7.5, "east"), Row(101, "new-a", 7.5, "east"), + Row(102, "new-b", -2.5, "west"), Row(103, null, null, null), + Row(104, "new-zero", 0.0, "east")) + val properties = Map( + "delta.checkpointInterval" -> "1000", + "delta.checkpoint.writeStatsAsJson" -> (!parsed).toString, + "delta.checkpoint.writeStatsAsStruct" -> parsed.toString) ++ + mappingMode.map("delta.columnMapping.mode" -> _).toMap + val propertySql = properties.toSeq.sortBy(_._1) + .map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + sql(s"""CREATE TABLE tbl (${beforeSchema.toDDL}) USING delta + PARTITIONED BY (${beforeNames.last}) TBLPROPERTIES ($propertySql)""") + sql("""INSERT INTO tbl VALUES + (1, 'old', 1000D, 'old-east'), (1, 'old', 1000D, 'old-east'), + (2, 'old-negative', -1000D, 'old-west'), (3, NULL, NULL, NULL)""") + + // The setting exists in OSS Delta 4.1.0 as well as Runtime. Restore after the DDL; + // it controls authoring only, and is not needed while queued read specs are captured. + val reuseKey = "spark.databricks.delta.columnMapping.reuseColumnMetadataDuringReplace" + val oldReuse = spark.conf.getOption(reuseKey) + try { + if (mapped) spark.conf.set(reuseKey, "false") + sql(s"""CREATE OR REPLACE TABLE tbl (${afterSchema.toDDL}) USING delta + PARTITIONED BY (region) TBLPROPERTIES ($propertySql)""") + } finally { + if (mapped) oldReuse match { + case Some(value) => spark.conf.set(reuseKey, value) + case None => spark.conf.unset(reuseKey) + } + } + sql("""INSERT INTO tbl VALUES + (101, 'new-a', 7.5D, 'east'), (101, 'new-a', 7.5D, 'east'), + (102, 'new-b', -2.5D, 'west'), (103, NULL, NULL, NULL), + (104, 'new-zero', 0D, 'east')""") + if (checkpointed) forceCheckpoint("tbl") + val t = registerTable("tbl") + val rawBefore = CommitLog.readRaw(t.sourcePath, 0L).map(JsonUtil.mapper.readTree) + .find(_.has("metaData")).get.get("metaData") + val rawAfter = CommitLog.readRaw(t.sourcePath, 2L).map(JsonUtil.mapper.readTree) + .find(_.has("metaData")).get.get("metaData") + val mappedBefore = DataType.fromJson(rawBefore.get("schemaString").asText) + .asInstanceOf[StructType] + val mappedAfter = DataType.fromJson(rawAfter.get("schemaString").asText) + .asInstanceOf[StructType] + def withoutMapping(schema: StructType): StructType = + StructType(schema.fields.map(_.copy(metadata = Metadata.empty))) + assert(withoutMapping(mappedBefore) == beforeSchema) + assert(withoutMapping(mappedAfter) == afterSchema) + if (mapped) { + val idKey = "delta.columnMapping.id" + val physicalKey = "delta.columnMapping.physicalName" + assert(mappedBefore.fields.map(_.metadata.getLong(idKey)).toSeq == Seq(1L, 2L, 3L, 4L)) + assert(mappedAfter.fields.map(_.metadata.getLong(idKey)).toSeq == Seq(5L, 6L, 7L, 8L)) + val oldPhysical = mappedBefore.fields.map(_.metadata.getString(physicalKey)).toSet + val newPhysical = mappedAfter.fields.map(_.metadata.getString(physicalKey)).toSet + assert(oldPhysical.size == 4 && newPhysical.size == 4) + assert(oldPhysical.intersect(newPhysical).isEmpty) + assert(newPhysical.intersect(afterNames.toSet).isEmpty) + } + val physicalNames = mappedAfter.fields.map { field => + if (mapped) field.metadata.getString("delta.columnMapping.physicalName") else field.name + }.toSeq + val statsNames = physicalNames.init.toSet + val partitionName = physicalNames.last + // Inspect new commit actions directly too, including the commit-only mapping variants. + val newAdds = CommitLog.readRaw(t.sourcePath, 3L).map(JsonUtil.mapper.readTree) + .filter(_.has("add")).map(_.get("add")) + assert(newAdds.nonEmpty) + for (add <- newAdds) { + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val stats = JsonUtil.mapper.readTree(add.get("stats").asText) + assert(stats.get("nullCount").fieldNames().asScala.toSet == statsNames) + for (kind <- Seq("minValues", "maxValues")) { + assert(stats.get(kind).fieldNames().asScala.toSet.subsetOf(statsNames)) + } + } + + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + // Retain exact pre-truncation actions in command evidence, including operation/removes. + for (v <- 0L to 3L; line <- CommitLog.readRaw(tableDir, v)) { + println(s"Replacement source action: $caseName v$v $line") + } + val stream = Files.list(logDir) + val files = try stream.iterator().asScala.toSeq finally stream.close() + if (checkpointed) { + val checkpoints = files.filter(_.getFileName.toString.endsWith(".checkpoint.parquet")) + assert(checkpoints.size == 1) + assert(checkpoints.head.getFileName.toString == "00000000000000000003.checkpoint.parquet") + val checkpoint = spark.read.parquet(checkpoints.head.toString) + val addSchema = checkpoint.schema("add").dataType.asInstanceOf[StructType] + assert(addSchema.fieldNames.contains("stats") == !parsed) + assert(addSchema.fieldNames.contains("stats_parsed") == parsed) + assert(addSchema.fieldNames.contains("partitionValues_parsed") == parsed) + if (parsed) { + val partitions = addSchema("partitionValues_parsed").dataType.asInstanceOf[StructType] + assert(partitions.fieldNames.toSeq == Seq(partitionName)) + assert(partitions.fields.head.dataType == StringType) + val stats = addSchema("stats_parsed").dataType.asInstanceOf[StructType] + for (kind <- Seq("minValues", "maxValues", "nullCount")) { + assert(stats(kind).dataType.asInstanceOf[StructType].fieldNames.toSet == statsNames) + } + } + val adds = checkpoint.filter("add IS NOT NULL").select("add").toJSON.collect() + assert(adds.nonEmpty) + for (json <- adds) { + val add = JsonUtil.mapper.readTree(json).get("add") + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val stats = if (parsed) add.get("stats_parsed") + else JsonUtil.mapper.readTree(add.get("stats").asText) + assert(stats.get("nullCount").fieldNames().asScala.toSet == statsNames) + for (kind <- Seq("minValues", "maxValues")) { + assert(stats.get(kind).fieldNames().asScala.toSet.subsetOf(statsNames)) + } + } + // Keep only the checkpoint-era JSON for version discovery. Earlier metadata and file + // state must come from the checkpoint; remove protocol CRC shortcuts as well. + files.filter { path => + val name = path.getFileName.toString + name.endsWith(".crc") || + (name.matches("[0-9]{20}\\.json") && name != "00000000000000000003.json") + }.foreach(Files.delete) + } else { + assert(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + } + println(s"Replacement layout oracle passed: $caseName ($layout)") + } + + replacementRead(t, "read_all", afterRows, afterSchema) + replacementRead(t, "read_projection", afterRows, afterSchema, + columns = Some(Seq("region", "score", "label"))) + replacementRead(t, "read_partition", afterRows.take(2) ++ afterRows.drop(4), + afterSchema, predicate = "region = 'east'") + replacementRead(t, "read_stats", afterRows.take(2), afterSchema, + predicate = "score > 7D") + replacementRead(t, "read_missing", Seq.empty, afterSchema, + predicate = "region = 'missing' OR score > 999D") + replacementRead(t, "read_null", afterRows.slice(3, 4), afterSchema, + predicate = "region IS NULL AND score IS NULL AND label IS NULL") + replacementRead(t, "read_old_stats", Seq.empty, afterSchema, + predicate = "score = 1000D OR label = 'old'") + if (!checkpointed) { + replacementRead(t, "read_before", beforeRows, beforeSchema, version = 1L) + replacementRead(t, "read_replaced_empty", Seq.empty, afterSchema, version = 2L) + } + for (v <- (if (checkpointed) Seq(3L) else Seq(1L, 2L, 3L))) { + val metadata = if (v < 2L) rawBefore else rawAfter + val expectedProperties = properties ++ mappingMode.map { mode => + Map("delta.columnMapping.mode" -> mode, + "delta.columnMapping.maxColumnId" -> (if (v < 2L) "4" else "8")) + }.getOrElse(Map.empty[String, String]) + snapshotSpec(t, version = v).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.get("schemaString").asText) + assert(result.metadata.partitionColumns == + Seq(if (v < 2L) beforeNames.last else "region")) + assert(result.metadata.configuration == expectedProperties) + case other => fail(s"$caseName snapshot v$v must succeed: $other") + } + println(s"Independent metadata oracle passed: $caseName v$v") + } + } + } + + test("changed_sets_checkpoint_json") { + replacementLayout("changed_sets_checkpoint_json", "checkpoint_json") + } + + test("changed_sets_checkpoint_parsed") { + replacementLayout("changed_sets_checkpoint_parsed", "checkpoint_parsed") + } + + test("mapped_name_commits") { + replacementLayout("mapped_name_commits", "commits", Some("name")) + } + + test("mapped_name_checkpoint_json") { + replacementLayout("mapped_name_checkpoint_json", "checkpoint_json", Some("name")) + } + + test("mapped_name_checkpoint_parsed") { + replacementLayout("mapped_name_checkpoint_parsed", "checkpoint_parsed", Some("name")) + } + + test("mapped_id_commits") { + replacementLayout("mapped_id_commits", "commits", Some("id")) + } + + test("mapped_id_checkpoint_json") { + replacementLayout("mapped_id_checkpoint_json", "checkpoint_json", Some("id")) + } + + test("mapped_id_checkpoint_parsed") { + replacementLayout("mapped_id_checkpoint_parsed", "checkpoint_parsed", Some("id")) + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ScopedSessionConf.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ScopedSessionConf.scala new file mode 100644 index 00000000..587567ff --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ScopedSessionConf.scala @@ -0,0 +1,55 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import scala.collection.mutable + +import org.apache.spark.sql.SparkSession +import org.scalatest.BeforeAndAfterEach + +import io.delta.workload.WorkloadTestSuite + +/** + * Session settings that a single workload test needs while its specs are captured. + * + * `WorkloadTestSuite` generates and asserts specs after the test body returns, so a + * try/finally inside the body would restore a setting before capture. A setting applied + * with [[setForTest]] stays active through generation and spec assertions, and is restored + * after the suite's per-test cleanup, including when the body, generation or an assertion + * fails. Other tests in the suite keep the session's original settings. + */ +trait ScopedSessionConf extends BeforeAndAfterEach { this: WorkloadTestSuite => + + // Key -> (session, value before this test first set it). + private val savedConf = mutable.LinkedHashMap.empty[String, (SparkSession, Option[String])] + + protected def setForTest(key: String, value: String): Unit = { + val session = spark + if (!savedConf.contains(key)) savedConf(key) = session -> session.conf.getOption(key) + session.conf.set(key, value) + } + + override def afterEach(): Unit = { + try super.afterEach() finally { + savedConf.toSeq.reverse.foreach { + case (key, (session, Some(previous))) => session.conf.set(key, previous) + case (key, (session, None)) => session.conf.unset(key) + } + savedConf.clear() + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/TimeTravelSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/TimeTravelSuite.scala index 9197d8c6..e8c3000c 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/TimeTravelSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/TimeTravelSuite.scala @@ -16,8 +16,19 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite -import io.delta.workload.engine.SnapshotResolver +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.deltaharness.{DeltaHarness, Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog, Metadata, Protocol, RemoveFile} +import io.delta.workload.model._ class TimeTravelSuite extends WorkloadTestSuite("time_travel") { @@ -536,4 +547,399 @@ class TimeTravelSuite extends WorkloadTestSuite("time_travel") { snapshotSpec(t) } + // Truncated history, VACUUM, and RESTORE. + test("e2e_54_truncated_history") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-1, 'old'), (2, 'old'), (2, 'old'), (CAST(NULL AS INT), 'old') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (7, 'keep'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (9, 'boundary'), (7, 'keep') AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val boundaryRows = Seq(Row(7, "keep"), Row(3, null), Row(3, null), Row(null, "missing"), + Row(9, "boundary"), Row(7, "keep")) + val latestRows = boundaryRows ++ Seq(Row(3, null), Row(11, "after"), Row(null, "after")) + val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + assert(original(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inputs = (1L to 3L).map { version => + val adds = original(version).collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.deletionVector.isEmpty) + version -> adds.head + }.toMap + assert(inputs.values.map(_.path).toSet.size == 3) + assert(original(2L).collect { case r: RemoveFile => r.path } == Seq(inputs(1L).path)) + DeltaHarness.get.openLog(spark, table.sourcePath.toString).checkpoint() + val checkpointName = "00000000000000000003.checkpoint.parquet" + val checkpointBytes = Files.readAllBytes(table.sourcePath.resolve("_delta_log") + .resolve(checkpointName)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + val logDir = dir.resolve("_delta_log") + (0L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + assert(Files.readAllBytes(logDir.resolve(checkpointName)).sameElements(checkpointBytes)) + // Retain v2 and v3 JSON for OSS historical lookup, plus the oldest checkpoint at v3. + // The missing prefix makes v1 unavailable; no table data is deleted. + (0L to 1L).foreach { version => + val commit = logDir.resolve(f"$version%020d.json") + assert(Files.isRegularFile(commit)) + Files.delete(commit) + assert(!Files.exists(commit)) + } + (2L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + inputs.values.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + val boundary = SnapshotResolver.buildDeltaReader(spark, dir, Some(3L), None) + assert(boundary.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(boundaryRows.asJava, schema), boundary, + "history boundary after JSON removal, before append") + // Append to the already truncated copy: v4 must be replayed after the v3 checkpoint. + spark.sql(s"""INSERT INTO delta.`$dir` SELECT /*+ COALESCE(1) */ * FROM VALUES + (3, CAST(NULL AS STRING)), (11, 'after'), (CAST(NULL AS INT), 'after') AS v(id, value)""") + .collect() + // The generator strips Hadoop checksum sidecars while copying. The append's LogStore + // may recreate this sidecar; keep the same portable fixture form after the mutation. + Files.deleteIfExists(logDir.resolve(".00000000000000000004.json.crc")) + val appended = CommitLog.read(dir, 4L) + val adds = appended.collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.deletionVector.isEmpty) + assert(!inputs.values.exists(_.path == adds.head.path)) + assert(appended.forall(a => !a.isInstanceOf[RemoveFile] && !a.isInstanceOf[Metadata] && + !a.isInstanceOf[Protocol])) + val commitInfo = appended.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "WRITE") + assert(commitInfo.get("operationParameters").get("mode").asText() == "Append") + assert(Files.size(dir.resolve(adds.head.path)) == adds.head.size) + assert(Files.readAllBytes(logDir.resolve(checkpointName)).sameElements(checkpointBytes)) + val files = Files.list(logDir) + try { + val names = files.iterator().asScala.map(_.getFileName.toString).toSeq + assert(names.filter(_.endsWith(".checkpoint.parquet")) == Seq(checkpointName)) + assert(names.filter(_.endsWith(".json")).sorted == (2L to 4L).map(v => f"$v%020d.json")) + } finally files.close() + assert(JsonUtil.mapper.readTree(Files.readAllBytes(logDir.resolve("_last_checkpoint"))) + .get("version").asLong() == 3L) + (2L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + inputs.values.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + info("history removed JSON v0/v1; retained v2/v3 and checkpoint v3; real append v4: " + + commitInfo.toString) + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](3L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(4L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val cases = Seq( + ("read_boundary_v3", ReadQuery(version = Some(3L)), boundaryRows, 2), + ("read_latest_after_append", ReadQuery(), latestRows, 3)) + cases.foreach { case (name, query, rows, fileCount) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull) + .assert { spec => + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, fileCount, 0L))) + val expected = spark.createDataFrame(rows.asJava, schema) + val actual = SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, None) + assert(actual.schema == schema) + RowComparison.assertRowsEqual(expected, actual, s"history $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == schema) + RowComparison.assertRowsEqual(expected, captured, s"history $name expected parquet") + } + } + // No checkpoint or JSON exists at or before v1. DAT resolves getSnapshotAt first; + // OSS raises DELTA_LOG_FILE_NOT_FOUND for that empty log segment, before scanning data. + readSpec(table, name = Some("read_unavailable_v1"), version = 1L, + expectError = ErrorCode("DELTA_LOG_FILE_NOT_FOUND")).assert { spec => + assert(spec.query == ReadQuery(version = Some(1L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode == "DELTA_LOG_FILE_NOT_FOUND") + assert(error.errorMessage.contains("version 1")) + info("history typed unavailable-history read: " + error.toString) + case other => fail(s"history unavailable-history read expected Failed, got $other") + } + } + snapshotSpec(table, version = 1L, expectError = ErrorCode("DELTA_LOG_FILE_NOT_FOUND")) + .assert { spec => + assert(spec.query == SnapshotQuery(version = Some(1L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode == "DELTA_LOG_FILE_NOT_FOUND") + assert(error.errorMessage.contains("version 1")) + case other => fail(s"history unavailable-history snapshot expected Failed, got $other") + } + } + } + + test("e2e_50_actual_vacuum") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (8, 'keep'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-2, 'early'), (5, 'middle'), (8, 'keep'), (12, CAST(NULL AS STRING)) AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val rows = Seq(Row(8, "keep"), Row(3, null), Row(3, null), Row(null, "missing"), + Row(-2, "early"), Row(5, "middle"), Row(8, "keep"), Row(12, null)) + val literal = spark.createDataFrame(rows.asJava, schema) + val before = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + assert(before(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inputs = (1L to 2L).flatMap(v => before(v).collect { case a: AddFile => a }) + assert(inputs.size == 2 && inputs.map(_.path).distinct.size == 2) + assert(inputs.forall(_.deletionVector.isEmpty)) + RowComparison.assertRowsEqual(literal, + SnapshotResolver.buildDeltaReader(spark, table.sourcePath, Some(2L), None), + "vacuum historical rows before pruning") + sql("OPTIMIZE tbl") + val rewrite = CommitLog.read(table.sourcePath, 3L) + val active = rewrite.collect { case a: AddFile => a } + val obsolete = rewrite.collect { case r: RemoveFile => r } + assert(active.nonEmpty && obsolete.size == 2) + assert(obsolete.map(_.path).toSet == inputs.map(_.path).toSet) + assert((active.map(_.path).toSet intersect obsolete.map(_.path).toSet).isEmpty) + assert(active.forall(a => !a.dataChange && a.deletionVector.isEmpty)) + assert(obsolete.forall(r => !r.dataChange && r.deletionVector.isEmpty)) + assert(rewrite.collect { case m: Metadata => m }.isEmpty) + assert(rewrite.collect { case p: Protocol => p }.isEmpty) + val optimizeInfo = rewrite.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(optimizeInfo.get("operation").asText() == "OPTIMIZE") + (inputs ++ active).foreach { add => + assert(Files.isRegularFile(table.sourcePath.resolve(add.path))) + assert(Files.size(table.sourcePath.resolve(add.path)) == add.size) + } + info("vacuum pre-VACUUM obsolete paths physically present: " + + obsolete.map(_.path).mkString(",")) + val retentionCheck = "spark.databricks.delta.retentionDurationCheck.enabled" + val originalRetentionCheck = spark.conf.getOption(retentionCheck) + try { + spark.conf.set(retentionCheck, "false") + sql("VACUUM tbl RETAIN 0 HOURS") + } finally { + originalRetentionCheck match { + case Some(value) => spark.conf.set(retentionCheck, value) + case None => spark.conf.unset(retentionCheck) + } + } + assert(spark.conf.getOption(retentionCheck) == originalRetentionCheck) + obsolete.foreach(r => assert(!Files.exists(table.sourcePath.resolve(r.path)))) + active.foreach(a => assert(Files.size(table.sourcePath.resolve(a.path)) == a.size)) + // VACUUM emits audit commits; they must not change the active file set or metadata. + val vacuum = (4L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val vacuumInfo = vacuum.toSeq.sortBy(_._1).flatMap { case (_, actions) => + assert(actions.forall(a => !a.isInstanceOf[AddFile] && !a.isInstanceOf[RemoveFile] && + !a.isInstanceOf[Metadata] && !a.isInstanceOf[Protocol])) + actions.map(a => JsonUtil.mapper.readTree(a.toJson)).filter(_.has("commitInfo")) + .map(_.get("commitInfo")) + } + assert(vacuumInfo.map(_.get("operation").asText()) == Seq("VACUUM START", "VACUUM END")) + vacuumInfo.foreach(n => info("vacuum actual vacuum commit: " + n.toString)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + (before ++ Map(3L -> rewrite) ++ vacuum).foreach { case (version, actions) => + assert(CommitLog.read(dir, version) == actions) + } + obsolete.foreach(r => assert(!Files.exists(dir.resolve(r.path)))) + active.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + // Force row materialization, independently of DAT's typed error capture. + val thrown = intercept[Exception] { + SnapshotResolver.buildDeltaReader(spark, dir, Some(2L), None).collect() + } + val causes = Iterator.iterate(thrown: Throwable)(_.getCause).takeWhile(_ != null).toSeq + assert(causes.exists(_.isInstanceOf[java.io.FileNotFoundException])) + assert(causes.exists(e => obsolete.exists(r => + Option(e.getMessage).exists(_.contains(r.path))))) + info("vacuum independent historical missing-file cause: " + + causes.map(e => e.getClass.getName + ": " + e.getMessage).mkString(" <- ")) + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](3L, 5L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(5L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_latest", ReadQuery(), schema, rows), + ("read_rewritten_v3", ReadQuery(version = Some(3L)), schema, rows), + ("read_vacuumed_v5", ReadQuery(version = Some(5L)), schema, rows), + ("read_latest_filtered", ReadQuery(predicate = Some("id = 3 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 3), Row(null, 3), Row("missing", null)))) + cases.foreach { case (name, query, expectedSchema, expectedRows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == expectedRows.size.toLong) + case other => fail(s"vacuum $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(expectedRows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"vacuum $name copied table") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"vacuum $name expected parquet") + } + } + readSpec(table, name = Some("read_pruned_v2"), version = 2L, + expectError = AnyError).assert { spec => + assert(spec.query == ReadQuery(version = Some(2L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode.nonEmpty) + assert(obsolete.exists(r => error.errorMessage.contains(r.path))) + val message = error.errorMessage.toLowerCase(java.util.Locale.ROOT) + assert(message.contains("not exist") || message.contains("not found")) + info("vacuum typed historical error: " + error.toString) + case other => fail(s"vacuum pruned historical read expected Failed, got $other") + } + // An error at the historical version must leave the current table readable. + RowComparison.assertRowsEqual(literal, + SnapshotResolver.buildDeltaReader(spark, copiedTable, None, None), + "vacuum current rows after historical failure") + } + } + + test("e2e_55_restore") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'before'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing'), (12, 'keep'), (12, 'keep') AS v(id, value)""") + val table = registerTable("tbl") + val before = (0L to 1L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (99, 'mutated'), (99, 'mutated'), (CAST(NULL AS INT), 'changed'), + (6, CAST(NULL AS STRING)) AS v(id, value)""") + val mutated = CommitLog.read(table.sourcePath, 2L) + sql("RESTORE TABLE tbl TO VERSION AS OF 1") + val restored = CommitLog.read(table.sourcePath, 3L) + assert(!Files.exists(CommitLog.commitFile(table.sourcePath, 4L))) + (0L to 1L).foreach(v => assert(CommitLog.read(table.sourcePath, v) == before(v))) + assert(CommitLog.read(table.sourcePath, 2L) == mutated) + val originalAdds = before(1L).collect { case a: AddFile => a } + val mutatedAdds = mutated.collect { case a: AddFile => a } + val restoredAdds = restored.collect { case a: AddFile => a } + assert(originalAdds.nonEmpty && mutatedAdds.nonEmpty) + assert(originalAdds.map(_.path).toSet.intersect(mutatedAdds.map(_.path).toSet).isEmpty) + assert(mutated.collect { case r: RemoveFile => r.path }.toSet == + originalAdds.map(_.path).toSet) + assert(restoredAdds.toSet == originalAdds.map(_.copy(dataChange = true)).toSet) + val restoredRemoves = restored.collect { case r: RemoveFile => r } + assert(restoredRemoves.map(_.path).toSet == mutatedAdds.map(_.path).toSet) + assert(restoredAdds.forall(_.dataChange) && restoredRemoves.forall(_.dataChange)) + val commitInfo = restored.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "RESTORE") + assert(commitInfo.get("operationParameters").get("version").asText() == "1") + info("restore OSS Delta 4.1.0 RESTORE commit: " + commitInfo.toString) + + val schema = new StructType().add("id", IntegerType).add("value", StringType) + // Literal bags are independent of both the captured result and the Delta reader. + val initialRows = Seq(Row(-7, "before"), Row(4, null), Row(4, null), + Row(null, "missing"), Row(12, "keep"), Row(12, "keep")) + val mutatedRows = Seq(Row(99, "mutated"), Row(99, "mutated"), + Row(null, "changed"), Row(6, null)) + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(before(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + (0L to 1L).foreach(v => assert(CommitLog.read(dir, v) == before(v))) + assert(CommitLog.read(dir, 2L) == mutated) + assert(CommitLog.read(dir, 3L) == restored) + (originalAdds ++ mutatedAdds).foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + } + Seq[java.lang.Long](1L, 2L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_before_v1", ReadQuery(version = Some(1L)), schema, initialRows), + ("read_mutated_v2", ReadQuery(version = Some(2L)), schema, mutatedRows), + ("read_restored_v3", ReadQuery(version = Some(3L)), schema, initialRows), + ("read_latest", ReadQuery(), schema, initialRows), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("missing", null)))) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"restore $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"restore $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"restore $name expected parquet") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/TypeWideningSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/TypeWideningSuite.scala index 4b8bef5e..62865cb3 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/TypeWideningSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/TypeWideningSuite.scala @@ -16,7 +16,17 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.Files + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.types.{DataType, StructType} + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.AddFile +import io.delta.workload.model._ /** * Type widening workloads: numeric chains, float->double, decimal precision, @@ -357,4 +367,251 @@ class TypeWideningSuite extends WorkloadTestSuite("type_widening") { snapshotSpec(t) } + // Widened JSON statistics, old-type parsed checkpoint statistics, and partition values + // are separate workloads per transition. + private case class Stage(dataType: String, values: Seq[String]) + private case class Widening(name: String, stages: Seq[Stage], jsonGap: Boolean = true) + + // The numeric chain, decimal physical encodings, and DATE values follow TypeWideningSuite. + // One transition corpus drives all three representations. The chain's SHORT stage also covers + // BYTE -> SHORT; existing full/time-travel JSON reads are not redeclared here. + private val widenings = Seq( + Widening("byte_int", Seq( + Stage("BYTE", Seq("1", "127")), Stage("INT", Seq("128", "2147483647")))), + Widening("byte_long", Seq( + Stage("BYTE", Seq("1", "127")), + Stage("BIGINT", Seq("128", "9223372036854775807")))), + Widening("short_int", Seq( + Stage("SHORT", Seq("10", "100")), Stage("INT", Seq("40000", "50000"))), + jsonGap = false), + Widening("short_long", Seq( + Stage("SHORT", Seq("100", "32767")), + Stage("BIGINT", Seq("40000", "9223372036854775807")))), + Widening("int_long", Seq( + Stage("INT", Seq("100", "200", "300", "400")), + Stage("BIGINT", Seq("3000000000", "9000000000000"))), jsonGap = false), + Widening("byte_double", Seq( + Stage("BYTE", Seq("1", "127")), + Stage("DOUBLE", Seq("127.5", "2147483648.25")))), + Widening("short_double", Seq( + Stage("SHORT", Seq("100", "32767")), + Stage("DOUBLE", Seq("32767.5", "2147483648.25")))), + Widening("int_double", Seq( + Stage("INT", Seq("100", "2147483647")), + Stage("DOUBLE", Seq("2147483647.5", "4294967296.25")))), + Widening("float_double", Seq( + Stage("FLOAT", Seq("1.5", "3.125")), + Stage("DOUBLE", Seq("3.141592653589793", "1.7976931348623157E308"))), + jsonGap = false), + Widening("decimal_precision", Seq( + Stage("DECIMAL(5,2)", Seq("123.45", "999.99")), + Stage("DECIMAL(10,2)", Seq("12345678.90", "99999999.99"))), jsonGap = false), + Widening("decimal_precision_scale", Seq( + Stage("DECIMAL(9,2)", Seq("123.45", "9999999.99")), + Stage("DECIMAL(18,2)", Seq("1234567890123.45", "9999999999999999.99")), + Stage("DECIMAL(28,3)", Seq("1234567890123456789012345.678")))), + Widening("date_timestamp_ntz", Seq( + Stage("DATE", Seq("'2024-01-15'", "'2024-06-30'")), + Stage("TIMESTAMP_NTZ", Seq("'2024-12-31 23:59:59'", "'2025-01-01 12:30:00'")))), + Widening("multiple_steps", Seq( + Stage("BYTE", Seq("1", "100")), Stage("SHORT", Seq("200", "30000")), + Stage("INT", Seq("40000", "2000000000")), + Stage("BIGINT", Seq("3000000000", "9000000000000000000"))))) + + // (stable id, literal, original type). Expected bags are selected explicitly by id, never by + // filtering the table or capturing a fresh query result as its own oracle. + private def rows(w: Widening, stage: Int, partitioned: Boolean): Seq[(Int, String, String)] = + w.stages.take(stage + 1).zipWithIndex.flatMap { case (s, i) => + s.values.zipWithIndex.map { case (v, j) => (i * 10 + j + 1, v, s.dataType) } ++ + (if (partitioned) Seq((i * 10 + 9, "NULL", s.dataType)) else Seq.empty) + } + + private def literal(value: String, from: String, to: String): String = + s"CAST(CAST($value AS $from) AS $to)" + + private def checkedRead( + t: TableHandle, + label: String, + dataType: String, + expected: Seq[(Int, String, String)], + predicate: String, + version: Long): Unit = { + val table = WorkloadContext.current.getTableSpec(t) + readSpec(t, version = version, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(_) => + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val selects = expected.map { case (id, value, from) => + s"SELECT CAST($id AS INT) AS id, ${literal(value, from, dataType)} AS value" + } + val oracle = if (selects.nonEmpty) selects.mkString(" UNION ALL ") else + s"SELECT CAST(NULL AS INT) AS id, CAST(NULL AS $dataType) AS value WHERE false" + val expectedPath = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$label").resolve("expected_data") + RowComparison.assertRowsEqual(spark.sql(oracle), + spark.read.parquet(expectedPath.toString), s"${table.outputName}/$label independent oracle") + } + } + + private def checkedSnapshot(t: TableHandle, w: Widening, stage: Int, + partitioned: Boolean, version: Long): Unit = { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + val expected = StructType.fromDDL(s"id INT, value ${w.stages(stage).dataType}") + require(schema.map(f => f.name -> f.dataType) == + expected.map(f => f.name -> f.dataType), s"Unexpected schema: $schema") + require(result.metadata.partitionColumns == + (if (partitioned) Seq("value") else Seq.empty)) + val fieldMetadata = schema("value").metadata + val changes = if (fieldMetadata.contains("delta.typeChanges")) + fieldMetadata.getMetadataArray("delta.typeChanges").toSeq else Seq.empty + val expectedChanges = w.stages.take(stage + 1).sliding(2).collect { + case Seq(from, to) => + (DataType.fromDDL(from.dataType).json, DataType.fromDDL(to.dataType).json) + }.toSeq + require(changes.map(c => (s"\"${c.getString("fromType")}\"", + s"\"${c.getString("toType")}\"")) == expectedChanges, + s"Unexpected widening history at $version: $changes") + case other => fail(s"Snapshot at $version must succeed: $other") + } + } + } + + private def prepareStats(t: TableHandle, representation: String, original: Stage): Unit = { + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val stream = Files.list(logDir) + val paths = try stream.iterator().asScala.toVector finally stream.close() + // CRC file-state snapshots must not substitute for the statistics under test. + paths.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + if (representation == "parsed") { + val checkpoint = logDir.resolve("00000000000000000001.checkpoint.parquet") + require(Files.exists(checkpoint), "Missing old-schema checkpoint") + val adds = spark.read.parquet(checkpoint.toString).filter("add IS NOT NULL") + val stats = adds.selectExpr("add.stats_parsed.minValues.value AS minValue", + "add.stats_parsed.maxValues.value AS maxValue", "add.stats_parsed.numRecords AS records") + require(stats.schema.head.dataType == DataType.fromDDL(original.dataType), + s"Checkpoint must retain ${original.dataType} statistics: ${stats.schema}") + val addSchema = adds.schema("add").dataType.asInstanceOf[StructType] + if (addSchema.fieldNames.contains("stats")) { + require(adds.filter("add.stats IS NOT NULL").count() == 0, + "Parsed-stat checkpoint must not contain JSON statistics") + } + val oracle = s"""SELECT CAST(${original.values.head} AS ${original.dataType}) AS minValue, + CAST(${original.values.last} AS ${original.dataType}) AS maxValue, + CAST(${original.values.size} AS BIGINT) AS records""" + RowComparison.assertRowsEqual(spark.sql(oracle), stats.selectExpr( + "MIN(minValue) AS minValue", "MAX(maxValue) AS maxValue", "SUM(records) AS records"), + "old parsed checkpoint min/max/count oracle") + // Keep the checkpoint's boundary commit for version discovery; its adds are removed below. + Files.delete(logDir.resolve("00000000000000000000.json")) + } else { + require(!paths.exists(_.getFileName.toString.contains("checkpoint")), + "JSON/partition workloads must not have checkpoints") + val actions = paths.filter(_.getFileName.toString.endsWith(".json")) + .flatMap(p => Files.readAllLines(p).asScala) + .map(JsonUtil.mapper.readTree(_)) + val adds = actions.filter(_.has("add")).map(_.get("add")) + require(adds.nonEmpty) + if (representation == "json") { + require(adds.forall { a => + a.hasNonNull("stats") && Seq("minValues", "maxValues").forall { bound => + JsonUtil.mapper.readTree(a.get("stats").asText()).path(bound).has("value") + } + }) + } else { + require(adds.forall(_.get("partitionValues").has("value"))) + require(adds.exists(_.get("partitionValues").get("value").isNull), + "Retain a null partition written before widening") + } + } + } + if (representation == "parsed") { + modifyCommitActions(t, version = 1)(_.filterNot(_.isInstanceOf[AddFile])) + } + } + + for { + w <- widenings + representation <- Seq("json", "parsed", "partition") + if representation != "json" || w.jsonGap + } { + test(s"${representation}_${w.name}") { + val partitioned = representation == "partition" + val partition = if (partitioned) "PARTITIONED BY (value)" else "" + val timestampFeature = if (w.name == "date_timestamp_ntz") + ", 'delta.feature.timestampNtz' = 'supported'" else "" + val statsProperties = if (representation == "parsed") + ", 'delta.checkpoint.writeStatsAsStruct' = 'true', " + + "'delta.checkpoint.writeStatsAsJson' = 'false'" else "" + sql(s"""CREATE TABLE tbl (id INT, value ${w.stages.head.dataType}) USING delta + $partition TBLPROPERTIES ('delta.enableTypeWidening' = 'true', + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '100', + 'delta.checkpointPolicy' = 'classic' $timestampFeature $statsProperties)""") + + for ((s, i) <- w.stages.zipWithIndex) { + if (i > 0) sql(s"ALTER TABLE tbl ALTER COLUMN value TYPE ${s.dataType}") + val inserted = rows(w, i, partitioned).filter(_._1 / 10 == i) + val values = inserted.map { case (id, value, from) => + s"($id, CAST($value AS $from))" + }.mkString(", ") + sql(s"INSERT INTO tbl VALUES $values") + if (i == 0 && representation == "parsed") forceCheckpoint("tbl") + } + + val t = registerTable("tbl") + prepareStats(t, representation, w.stages.head) + + // Capture the requested latest-schema assertions before historical-stage controls. + for ((s, i) <- w.stages.zipWithIndex.reverse) { + val version = i * 2L + 1 + checkedSnapshot(t, w, i, partitioned, version) + val precisionOnlyStats = representation != "partition" && + w.name == "decimal_precision_scale" && i == 1 + if (i > 0 && !precisionOnlyStats) { + val bag = rows(w, i, partitioned) + val nonNull = bag.filter(_._2 != "NULL") + val original = nonNull.filter(_._1 < 10) + val previous = w.stages(i - 1) + val firstNew = literal(s.values.head, s.dataType, s.dataType) + val lastOld = literal(previous.values.last, previous.dataType, s.dataType) + val lastOriginal = literal(w.stages.head.values.last, + w.stages.head.dataType, s.dataType) + val firstOriginal = literal(w.stages.head.values.head, + w.stages.head.dataType, s.dataType) + val prefix = if (w.name == "multiple_steps" && i == 1) "byte_short" else w.name + val label = s"${prefix}_v$version" + val read = (suffix: String, predicate: String, selected: Seq[(Int, String, String)]) => + checkedRead(t, s"${label}_$suffix", s.dataType, selected, predicate, version) + + // JSON partial cases contribute filters only; TypeWideningSuite owns their full reads. + val partialJson = representation == "json" && + Set("multiple_steps", "decimal_precision_scale", "date_timestamp_ntz").contains(w.name) + if (!partialJson) read("all", null, bag) + read("old_range", s"value <= $lastOriginal", original) + read("new_range", s"value > $lastOld", nonNull.filter(_._1 / 10 == i)) + read("cross_range", s"value >= $lastOld AND value <= $firstNew", + nonNull.filter(r => r._1 == (i - 1) * 10 + previous.values.size || + r._1 == i * 10 + 1)) + read("miss", s"value < $firstOriginal", Seq.empty) + + if (partitioned) { + read("old_equality", s"value = $firstOriginal", original.take(1)) + read("new_equality", s"value = $firstNew", nonNull.filter(_._1 == i * 10 + 1)) + read("null_partitions", "value IS NULL", bag.filter(_._2 == "NULL")) + } + if (w.name == "date_timestamp_ntz") { + read("old_midnight", s"value = $lastOriginal", original.takeRight(1)) + read("new_midnight_miss", "value = TIMESTAMP_NTZ'2024-12-31 00:00:00'", Seq.empty) + read("exclusive_boundary", s"value >= $lastOriginal AND value < $firstNew", + original.takeRight(1)) + } + } + } + } + } + }