diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala index c557f995..7cac9cdb 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala @@ -16,11 +16,28 @@ package io.delta.workload.tables +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Path} + +import scala.collection.JavaConverters._ + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.spark.sql.Row +import org.apache.spark.sql.functions.{col, lit, map => sparkMap, when} +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + import io.delta.workload.{TableHandle, WorkloadTestSuite} -import io.delta.workload.log.{Action, DomainMetadata, Txn} -import io.delta.workload.model.ErrorCode +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{Action, AddFile, CheckpointMetadata, CommitLog, DomainMetadata, + Metadata, Protocol, SidecarFile, Txn} +import io.delta.workload.model._ -class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { +class CheckpointsSuite extends WorkloadTestSuite("checkpoints") + with ScopedSessionConf { private def checkpoint(name: String): Unit = forceCheckpoint(name) @@ -659,4 +676,503 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { snapshotSpec(t) readSpec(t) } + + // Physical checkpoint forms (JSON/Parquet top-level files, manifests, sidecars, + // multi-part, structured stats, >64 KiB components, allFiles CRC). Session settings + // stay active until the specs are captured. + private def snapshotResult(protocol: Protocol, metadata: Metadata): SnapshotResult = + SnapshotResult( + ProtocolInfo(protocol.minReaderVersion, protocol.minWriterVersion, + protocol.readerFeatures.map(_.toSeq.sorted), + protocol.writerFeatures.map(_.toSeq.sorted)), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat(metadata.format.provider, metadata.format.options), + metadata.schemaString, metadata.partitionColumns, metadata.configuration, + metadata.createdTime)) + + private def assertReadRows( + spec: ReadSpec, + tablePath: Path, + schema: StructType, + rows: Seq[Row], + name: String): Unit = { + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, tablePath, + spec.query.version, spec.query.timestamp), spec.query.predicate, spec.query.columns) + assert(actual.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, schema), actual, name) + } + + Seq( + ("json_inline", "json", "v2", 1000, 0), + ("json_manifest", "json", "v2", 1000, 1), + ("parquet_single_part", "parquet", "classic", 1000, 0), + ("parquet_manifest", "parquet", "v2", 1000, 1), + ("parquet_sidecars", "parquet", "v2", 1, 2) + ).foreach { case (name, format, policy, partSize, sidecarCount) => + test(name) { + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> format, + "spark.databricks.delta.checkpoint.partSize" -> partSize.toString) + settings.foreach { case (key, value) => setForTest(key, value) } + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 'a'), (2, CAST(NULL AS STRING)), (2, CAST(NULL AS STRING)) AS v(id, value)""") + sql("INSERT INTO tbl VALUES (3, 'c')") + forceCheckpoint("tbl") + sql("INSERT INTO tbl VALUES (4, 'd')") + val t = registerTable("tbl") + // Read the original commit actions before mutating the copied fixture. + val actions = (0L to 2L).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 2) + info("Original protocol: " + protocol.toJson) + info("Original metadata: " + metadata.toJson) + adds.foreach(a => info("Original add: " + a.toJson)) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + val expectedSnapshot = snapshotResult(protocol, metadata) + val checkpointRows = Seq(Row(1, "a"), Row(2, null), Row(2, null), Row(3, "c")) + val latestRows = checkpointRows :+ Row(4, "d") + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + assert(checkpoints.size == 1) + val checkpoint = checkpoints.head + if (policy == "classic") { + assert(checkpoint.getFileName.toString == "00000000000000000002.checkpoint.parquet") + val frame = spark.read.parquet(checkpoint.toString) + assert(frame.filter("add IS NOT NULL").count() == adds.size) + assert(frame.filter("metaData IS NOT NULL").count() == 1) + assert(frame.filter("protocol IS NOT NULL").count() == 1) + assert(!Files.exists(log.resolve("_sidecars"))) + } else { + assert(checkpoint.getFileName.toString.matches( + s"00000000000000000002\\.checkpoint\\.[^.]+\\.$format")) + if (name == "json_inline") { + // V2 allows inline file actions; this writer normally emits sidecars. + val checkpointAdds = adds.map(_.copy(dataChange = false)) + val inlineActions: Seq[Action] = + Seq(protocol, metadata, CheckpointMetadata(2L)) ++ checkpointAdds + Files.write(checkpoint, inlineActions.map(_.toJson).asJava, UTF_8) + val sidecars = Files.list(log.resolve("_sidecars")) + try sidecars.iterator().asScala.toVector.foreach(Files.delete) + finally sidecars.close() + Files.delete(log.resolve("_sidecars")) + val content = Files.readAllLines(checkpoint, UTF_8).asScala.map(Action.parse).toVector + assert(content.collect { case a: AddFile => a } == checkpointAdds) + assert(content.collect { case c: CheckpointMetadata => c.version } == Seq(2L)) + assert(!content.exists(_.isInstanceOf[SidecarFile])) + } else { + val manifest = if (format == "json") { + Files.readAllLines(checkpoint, UTF_8).asScala.map(Action.parse).toVector + } else { + spark.read.parquet(checkpoint.toString).toJSON.collect().toVector.map(Action.parse) + } + val references = manifest.collect { case s: SidecarFile => s } + assert(references.size == sidecarCount) + assert(!manifest.exists(_.isInstanceOf[AddFile])) + assert(manifest.collect { case c: CheckpointMetadata => c.version } == Seq(2L)) + assert(manifest.collect { case m: Metadata => m } == Seq(metadata)) + assert(manifest.collect { case p: Protocol => p } == Seq(protocol)) + val sidecarPaths = references.map { ref => + val path = log.resolve("_sidecars").resolve(ref.path) + assert(ref.path.endsWith(".parquet")) + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + path + } + val sidecarListing = Files.list(log.resolve("_sidecars")) + try assert(sidecarListing.iterator().asScala.toSet == sidecarPaths.toSet) + finally sidecarListing.close() + val content = spark.read.parquet(sidecarPaths.map(_.toString): _*) + assert(content.count() == adds.size) + assert(content.filter("add IS NOT NULL").count() == adds.size) + assert(content.filter("remove IS NOT NULL").count() == 0) + assert(content.select("add.path").collect().map(_.getString(0)).toSet == + adds.map(_.path).toSet) + } + } + // Keep same-version JSON commit 2 for the OSS historical snapshot lookup. + // This tests checkpoint plus commit 2, then version 3 tail replay. + // Checkpoint-only historical snapshots remain a separate failing repro. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 2L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + Seq( + ("read_all", ReadQuery(), latestRows, ReadResult(5L, 3, 0L)), + ("read_v2", ReadQuery(version = Some(2L)), checkpointRows, ReadResult(4L, 2, 0L)), + ("read_filtered", ReadQuery(predicate = Some("id >= 3")), + Seq(Row(3, "c"), Row(4, "d")), ReadResult(2L, 2, 1L)) + ).foreach { case (readName, query, rows, expected) => + readSpec(t, name = Some(readName), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, readName) + } + } + Seq(None, Some(2L)).foreach { version => + snapshotSpec(t, version = version.map(Long.box).orNull).assert { spec => + assert(spec.query.version.contains(version.getOrElse(3L))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + } + } + + // stats: OSS exposes structured checkpoint stats, but no product metadata-clustering + // control. These fixtures check shape and filtered correctness, never row groups READ. + Seq( + ("e2e_07_single_part", "classic", 1000, 1, 0), + ("e2e_07_multipart", "classic", 2, 3, 0), + ("e2e_07_sidecars", "v2", 2, 1, 3) + ).foreach { case (name, policy, partSize, checkpointCount, sidecarCount) => + test(name) { + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> "parquet", + "spark.databricks.delta.checkpoint.partSize" -> partSize.toString, + "spark.databricks.delta.statsAsStructInCheckpoint.forcedDisabled" -> "false") + settings.foreach { case (key, value) => setForTest(key, value) } + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpoint.writeStatsAsStruct' = 'true')""") + val rowsByFile = (0 until 6).map { band => + val low = band * 10 + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + ($low, 'band_$band'), (${low + 1}, CAST(NULL AS STRING)), + (${low + 1}, CAST(NULL AS STRING)) AS v(id, value)""") + Seq(Row(low, s"band_$band"), Row(low + 1, null), Row(low + 1, null)) + } + forceCheckpoint("tbl") + val t = registerTable("tbl") + val actions = (0L to 6L).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 6 && adds.map(_.path).distinct.size == 6) + info("stats original protocol: " + protocol.toJson) + info("stats original metadata: " + metadata.toJson) + adds.foreach(a => info("stats original add: " + a.toJson)) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration("delta.checkpointPolicy") == policy) + assert(metadata.configuration("delta.checkpoint.writeStatsAsStruct") == "true") + val expectedSnapshot = snapshotResult(protocol, metadata) + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + assert(checkpoints.size == checkpointCount) + if (name == "e2e_07_single_part") { + assert(checkpoints.head.getFileName.toString == + "00000000000000000006.checkpoint.parquet") + } else if (policy == "classic") { + val expectedNames = (1 to 3).map { part => + f"00000000000000000006.checkpoint.$part%010d.0000000003.parquet" + }.toSet + assert(checkpoints.map(_.getFileName.toString).toSet == expectedNames) + } else { + assert(checkpoints.head.getFileName.toString.matches( + "00000000000000000006\\.checkpoint\\.[^.]+\\.parquet")) + } + val top = spark.read.parquet(checkpoints.map(_.toString): _*) + val topActions = top.toJSON.collect().toVector.map(Action.parse) + assert(topActions.collect { case m: Metadata => m } == Seq(metadata)) + assert(topActions.collect { case p: Protocol => p } == Seq(protocol)) + val sidecarPaths = if (policy == "v2") { + val refs = topActions.collect { case s: SidecarFile => s } + assert(refs.size == sidecarCount && refs.map(_.path).distinct.size == sidecarCount) + assert(topActions.collect { case c: CheckpointMetadata => c.version } == Seq(6L)) + assert(!topActions.exists(_.isInstanceOf[AddFile])) + val paths = refs.map { ref => + val path = log.resolve("_sidecars").resolve(ref.path) + assert(ref.path.endsWith(".parquet")) + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + path + } + val listing = Files.list(log.resolve("_sidecars")) + try assert(listing.iterator().asScala.toSet == paths.toSet) + finally listing.close() + paths + } else { + assert(!Files.exists(log.resolve("_sidecars"))) + Vector.empty[Path] + } + val fileActions = if (sidecarPaths.isEmpty) top else + spark.read.parquet(sidecarPaths.map(_.toString): _*) + assert(fileActions.filter("add IS NOT NULL").count() == 6) + assert(fileActions.filter("remove IS NOT NULL").count() == 0) + if (sidecarPaths.nonEmpty) assert(fileActions.count() == 6) + assert(fileActions.select("add.path").collect().filterNot(_.isNullAt(0)) + .map(_.getString(0)).toSet == adds.map(_.path).toSet) + val stats = fileActions.filter("add IS NOT NULL").select( + "add.stats_parsed.numRecords", "add.stats_parsed.minValues.id", + "add.stats_parsed.maxValues.id", "add.stats_parsed.nullCount.value").collect().toSeq + assert(stats.toSet == (0 until 6).map { band => + Row(3L, band * 10, band * 10 + 1, 2L) + }.toSet) + // Footer row groups are physical shape evidence, not reader IO counters. + val footerRows = (checkpoints ++ sidecarPaths).map { path => + val reader = ParquetFileReader.open(HadoopInputFile.fromPath( + new HadoopPath(path.toUri), spark.sparkContext.hadoopConfiguration)) + try { + val groups = reader.getFooter.getBlocks.asScala + info(s"stats footer: ${path.getFileName} groups=${groups.size} " + + s"rows=${groups.map(_.getRowCount).sum}") + assert(groups.size <= 1) + groups.map(_.getRowCount).sum + } finally reader.close() + } + assert(footerRows.sum == (if (policy == "v2") 12L else 8L)) + // Retain same-version JSON 6 for OSS version lookup, with no post-checkpoint tail. + // Missing commits 0..5 make the checkpoint required even though commit 6 exists. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 6L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + Seq( + ("read_all", None, rowsByFile.flatten, ReadResult(18L, 6, 0L)), + ("read_high", Some("id >= 40"), rowsByFile.drop(4).flatten, ReadResult(6L, 2, 4L)), + ("read_boundary", Some("id = 41"), Seq(Row(41, null), Row(41, null)), + ReadResult(2L, 1, 5L)), + ("read_empty", Some("id >= 100"), Seq.empty[Row], ReadResult(0L, 0, 6L)) + ).foreach { case (readName, predicate, rows, expected) => + readSpec(t, name = Some(readName), predicate = predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, readName) + } + } + snapshotSpec(t).assert { spec => + assert(spec.query.version.contains(6L)) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + } + + // size: 64 KiB is a bounded fixture boundary, not a product dispatch threshold. + // These shapes assert portable read correctness, never distributed execution. + Seq("manifest", "json", "sidecars", "single_part", "multipart", "crc_all_files") + .foreach { shape => + test(s"e2e_43_$shape") { + val isCrc = shape == "crc_all_files" + val policy = if (Set("single_part", "multipart", "crc_all_files")(shape)) + "classic" else "v2" + val settings = Map( + "spark.databricks.delta.checkpointV2.topLevelFileFormat" -> "json", + "spark.databricks.delta.checkpoint.partSize" -> + (if (shape == "multipart") "2" else "1000"), + "spark.databricks.delta.allFilesInCrc.enabled" -> "true", + "spark.databricks.delta.allFilesInCrc.thresholdNumFiles" -> "100") + settings.foreach { case (key, value) => setForTest(key, value) } + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = '$policy', + 'delta.checkpointInterval' = '1000', 'delta.enableDeletionVectors' = 'false')""") + val rowsByFile = (0 until 4).map { band => + val low = band * 10 + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + ($low, 'band_$band'), (${low + 1}, CAST(NULL AS STRING)), + (${low + 1}, CAST(NULL AS STRING)) AS v(id, value)""") + Seq(Row(low, s"band_$band"), Row(low + 1, null), Row(low + 1, null)) + } + forceCheckpoint("tbl") + // A complete-file removal distinguishes active files from historical Adds in CRC. + if (isCrc) sql("DELETE FROM tbl WHERE id < 10") + sql("INSERT INTO tbl VALUES (99, 'tail')") + val t = registerTable("tbl") + val baseVersion = if (isCrc) 5L else 4L + val latestVersion = baseVersion + 1L + val actions = (0L to latestVersion).flatMap(v => CommitLog.read(t.sourcePath, v)) + val metadata = actions.collect { case m: Metadata => m }.last + val protocol = actions.collect { case p: Protocol => p }.last + val adds = actions.collect { case a: AddFile => a } + assert(adds.size == 5 && adds.map(_.path).distinct.size == 5) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration("delta.checkpointPolicy") == policy) + assert(metadata.configuration("delta.enableDeletionVectors") == "false") + val expectedSnapshot = snapshotResult(protocol, metadata) + val baseRows = (if (isCrc) rowsByFile.drop(1) else rowsByFile).flatten + val latestRows = baseRows :+ Row(99, "tail") + info(s"size $shape protocol: " + protocol.toJson) + info(s"size $shape metadata: " + metadata.toJson) + adds.foreach(a => info(s"size $shape original add: " + a.toJson)) + var copiedTablePath: Path = null + mutateTable(t) { tableDir => + copiedTablePath = tableDir + val log = tableDir.resolve("_delta_log") + val listing = Files.list(log) + val files = try listing.iterator().asScala.toVector finally listing.close() + val checkpoints = files.filter(_.getFileName.toString.contains(".checkpoint.")) + .sortBy(_.getFileName.toString) + assert(checkpoints.size == (if (shape == "multipart") 2 else 1)) + if (policy == "classic") { + val names = if (shape == "multipart") (1 to 2).map { part => + f"00000000000000000004.checkpoint.$part%010d.0000000002.parquet" + }.toSet else Set("00000000000000000004.checkpoint.parquet") + assert(checkpoints.map(_.getFileName.toString).toSet == names) + } else { + assert(checkpoints.head.getFileName.toString.matches( + "00000000000000000004\\.checkpoint\\.[^.]+\\.json")) + } + val manifest = if (policy == "v2") { + Files.readAllLines(checkpoints.head, UTF_8).asScala.map(Action.parse).toVector + } else Vector.empty[Action] + val refs = manifest.collect { case s: SidecarFile => s } + assert(refs.size == (if (policy == "v2") 1 else 0)) + val sidecars = refs.map(ref => log.resolve("_sidecars").resolve(ref.path)) + refs.zip(sidecars).foreach { case (ref, path) => + assert(Files.size(path) == ref.sizeInBytes) + assert(Files.getLastModifiedTime(path).toMillis == ref.modificationTime) + } + if (policy == "v2") { + assert(!manifest.exists(_.isInstanceOf[AddFile])) + assert(manifest.collect { case c: CheckpointMetadata => c.version } == Seq(4L)) + assert(manifest.collect { case m: Metadata => m } == Seq(metadata)) + assert(manifest.collect { case p: Protocol => p } == Seq(protocol)) + } else assert(!Files.exists(log.resolve("_sidecars"))) + val boundary = 64L * 1024L + // Deterministic incompressible padding keeps Parquet fixtures small but >64 KiB. + val bytes = new Array[Byte](96 * 1024) + new scala.util.Random(43L).nextBytes(bytes) + val padding = java.util.Base64.getEncoder.encodeToString(bytes) + val paddedFiles = shape match { + case "sidecars" => sidecars + case "single_part" | "multipart" => checkpoints + case _ => Vector.empty[Path] + } + paddedFiles.foreach { path => + val frame = spark.read.parquet(path.toString) + val padded = frame.withColumn("add", when(col("add").isNotNull, + col("add").withField("tags", sparkMap(lit("e2e43.padding"), lit(padding)))) + .otherwise(col("add"))) + val temp = Files.createTempDirectory("e2e43-checkpoint-") + try { + padded.coalesce(1).write.mode("overwrite").parquet(temp.toString) + val parts = Files.list(temp) + val part = try parts.iterator().asScala + .filter(_.getFileName.toString.endsWith(".parquet")).toVector + finally parts.close() + assert(part.size == 1) + Files.move(part.head, path, java.nio.file.StandardCopyOption.REPLACE_EXISTING) + } finally org.apache.commons.io.FileUtils.deleteDirectory(temp.toFile) + } + if (shape == "manifest" || shape == "sidecars") { + val updated = manifest.map { + case c: CheckpointMetadata if shape == "manifest" => + c.copy(tags = Some(Map("e2e43.padding" -> padding))) + case ref: SidecarFile => + val path = log.resolve("_sidecars").resolve(ref.path) + ref.copy(sizeInBytes = Files.size(path), + modificationTime = Files.getLastModifiedTime(path).toMillis) + case action => action + } + Files.write(checkpoints.head, updated.map(_.toJson).asJava, UTF_8) + } + if (shape == "json") { + val lines = CommitLog.readRaw(tableDir, latestVersion).map { line => + val node = JsonUtil.mapper.readTree(line) + if (node.has("commitInfo")) { + val commitInfo = node.get("commitInfo") + .asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + commitInfo.put("userMetadata", padding) + } + node.toString + } + CommitLog.writeRaw(tableDir, latestVersion, lines) + } + val fileActions = spark.read.parquet( + (if (policy == "v2") sidecars else checkpoints).map(_.toString): _*) + val checkpointAdds = fileActions.toJSON.collect().toVector.map(Action.parse) + .collect { case a: AddFile => a } + val expectedAdds = adds.take(4).map { add => + add.copy(dataChange = false, tags = if (paddedFiles.nonEmpty) + Some(Map("e2e43.padding" -> padding)) else add.tags) + } + assert(checkpointAdds.sortBy(_.path) == expectedAdds.sortBy(_.path)) + if (policy == "classic") { + val top = fileActions.toJSON.collect().toVector.map(Action.parse) + assert(top.collect { case m: Metadata => m } == Seq(metadata)) + assert(top.collect { case p: Protocol => p } == Seq(protocol)) + } + if (isCrc) { + Seq(baseVersion, latestVersion).foreach { version => + val crcPath = log.resolve(f"$version%020d.crc") + assert(Files.exists(crcPath)) + val crc = JsonUtil.mapper.readTree(Files.readString(crcPath, UTF_8)) + assert(crc.has("allFiles") && crc.get("allFiles").isArray) + val active = (if (version == baseVersion) adds.slice(1, 4) else adds.drop(1)) + .map(_.copy(dataChange = false)) + val encoded = crc.get("allFiles").elements().asScala.map { node => + Action.parse("{\"add\":" + node.toString + "}").asInstanceOf[AddFile] + }.toVector + // The optional tags map can round-trip through Parquet as empty, not absent. + val normalized = encoded.map(a => a.copy(tags = a.tags.filter(_.nonEmpty))) + assert(normalized.sortBy(_.path) == active.sortBy(_.path)) + assert(crc.get("numFiles").asLong() == active.size) + assert(crc.get("tableSizeBytes").asLong() == active.map(_.size).sum) + info(s"size CRC v$version bytes=${Files.size(crcPath)} " + + s"allFiles=${encoded.size} activeBytes=${active.map(_.size).sum}") + } + } else { + // JSON 4 supports historical version lookup; absent 0..3 requires checkpoint 4. + // JSON 5 is a real one-commit replay tail with one new AddFile. + files.filter(p => p.getFileName.toString.matches("[0-9]{20}\\.json")) + .filter(_.getFileName.toString.take(20).toLong < 4L).foreach(Files.delete) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + Files.deleteIfExists(log.resolve("_last_checkpoint")) + } + val jsonPaths = files.filter(p => Files.exists(p) && + p.getFileName.toString.matches("[0-9]{20}\\.json")) + (checkpoints ++ sidecars ++ jsonPaths).foreach { path => + val large = paddedFiles.contains(path) || + (shape == "manifest" && checkpoints.contains(path)) || + (shape == "json" && path == CommitLog.commitFile(tableDir, latestVersion)) + val size = Files.size(path) + assert(if (large) size > boundary else size < boundary) + info(s"size $shape component=${log.relativize(path)} bytes=$size " + + s"large=$large boundary=$boundary") + } + } + Seq( + ("read_base", Some(baseVersion), None, baseRows, + ReadResult(baseRows.size.toLong, if (isCrc) 3 else 4, 0L)), + ("read_latest", None, None, latestRows, + ReadResult(latestRows.size.toLong, if (isCrc) 4 else 5, 0L)), + ("read_duplicates", None, Some("id = 21"), Seq(Row(21, null), Row(21, null)), + ReadResult(2L, 1, if (isCrc) 3L else 4L)) + ).foreach { case (name, version, predicate, rows, expected) => + readSpec(t, name = Some(name), version = version.map(Long.box).orNull, + predicate = predicate.orNull).assert { spec => + assert(spec.expectation == Succeeded(expected)) + assertReadRows(spec, copiedTablePath, schema, rows, s"size $shape $name") + } + } + Seq(Some(baseVersion), None).foreach { version => + snapshotSpec(t, version = version.map(Long.box).orNull).assert { spec => + assert(spec.query.version.contains(version.getOrElse(latestVersion))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + if (isCrc) crcSpec(t, version = baseVersion, name = "crc_complete_active") + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ColumnMappingSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ColumnMappingSuite.scala index ae9780af..83494bf1 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ColumnMappingSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ColumnMappingSuite.scala @@ -16,7 +16,18 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.Files + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog, Metadata => LogMetadata, Protocol, RemoveFile} +import io.delta.workload.model.Succeeded class ColumnMappingSuite extends WorkloadTestSuite("column_mapping") { @@ -419,4 +430,289 @@ class ColumnMappingSuite extends WorkloadTestSuite("column_mapping") { for (v <- 0L to 3L) snapshotSpec(t, version = v) } + // Parsed checkpoint statistics/partition values keyed by physical names, and enabling + // name mapping through a full rewrite. + // One primary scenario covers parsed statistics and parsed partition values together. + // The existing checkpoints_partitioned fixture is the no-mapping parsed-key control. + for (mode <- Seq("name", "id")) { + test(s"parsed_physical_names_$mode") { + val properties = Map( + "delta.columnMapping.mode" -> mode, + "delta.checkpointInterval" -> "1000", + "delta.checkpoint.writeStatsAsStruct" -> "true", + "delta.checkpoint.writeStatsAsJson" -> "false") + val propertySql = properties.toSeq.sortBy(_._1) + .map { case (key, value) => s"'$key' = '$value'" }.mkString(", ") + sql(s"""CREATE TABLE tbl ( + id INT, old_value INT, payload STRUCT, old_part STRING) + USING delta PARTITIONED BY (old_part) TBLPROPERTIES ($propertySql)""") + sql("""INSERT INTO tbl VALUES + (1, 10, named_struct('old_leaf', -10), 'east'), + (1, 10, named_struct('old_leaf', -10), 'east'), + (2, 20, named_struct('old_leaf', -20), 'west'), + (3, NULL, named_struct('old_leaf', CAST(NULL AS INT)), NULL), + (4, 40, NULL, 'east')""") + sql("ALTER TABLE tbl RENAME COLUMN old_value TO value") + sql("ALTER TABLE tbl RENAME COLUMN payload.old_leaf TO leaf") + sql("ALTER TABLE tbl RENAME COLUMN old_part TO region") + sql("""INSERT INTO tbl VALUES + (5, 100, named_struct('leaf', -100), 'north'), + (6, NULL, named_struct('leaf', CAST(NULL AS INT)), NULL), + (7, 200, named_struct('leaf', -200), 'east')""") + forceCheckpoint("tbl") + // A real v6 tail is replayed after the v5 checkpoint; v5 JSON is only version lookup. + sql("INSERT INTO tbl VALUES (8, 300, named_struct('leaf', -300), 'west')") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val actions = (0L to 6L).map { version => + version -> CommitLog.readRaw(t.sourcePath, version).map(JsonUtil.mapper.readTree) + }.toMap + val beforeMetadata = actions(0L).find(_.has("metaData")).get.get("metaData") + val metadata = actions(4L).find(_.has("metaData")).get.get("metaData") + val before = DataType.fromJson(beforeMetadata.get("schemaString").asText) + .asInstanceOf[StructType] + val mapped = DataType.fromJson(metadata.get("schemaString").asText) + .asInstanceOf[StructType] + val physicalKey = "delta.columnMapping.physicalName" + val idKey = "delta.columnMapping.id" + val oldFields = before.fields.toSeq :+ + before("payload").dataType.asInstanceOf[StructType]("old_leaf") + val fields = mapped.fields.toSeq :+ + mapped("payload").dataType.asInstanceOf[StructType]("leaf") + assert(mapped.fieldNames.toSeq == Seq("id", "value", "payload", "region")) + assert(fields.map(_.metadata.getLong(idKey)).toSet == Set(1L, 2L, 3L, 4L, 5L)) + assert(fields.map(_.metadata) == oldFields.map(_.metadata)) + assert(fields.forall(field => field.metadata.getString(physicalKey) != field.name)) + val physical = fields.map(_.metadata.getString(physicalKey)) + assert(physical.distinct.size == 5) + val partitionName = physical(3) + val statsNames = physical.take(3).toSet + val originals = actions.values.flatten.filter(_.has("add")) + .map(_.get("add")).map(add => add.get("path").asText -> add).toMap + + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + for (version <- 0L to 6L; line <- CommitLog.readRaw(tableDir, version)) { + println(s"Physical-name source action: $mode v$version $line") + } + val checkpointPath = logDir.resolve("00000000000000000005.checkpoint.parquet") + assert(Files.exists(checkpointPath)) + val checkpoint = spark.read.parquet(checkpointPath.toString) + val addSchema = checkpoint.schema("add").dataType.asInstanceOf[StructType] + assert(!addSchema.fieldNames.contains("stats")) + val partition = addSchema("partitionValues_parsed").dataType.asInstanceOf[StructType] + assert(partition.fieldNames.toSeq == Seq(partitionName)) + assert(partition.fields.head.dataType == StringType) + val stats = addSchema("stats_parsed").dataType.asInstanceOf[StructType] + for (kind <- Seq("minValues", "maxValues", "nullCount")) { + val leaves = stats(kind).dataType.asInstanceOf[StructType] + assert(leaves.fieldNames.toSet == statsNames) + assert(leaves(physical(2)).dataType.asInstanceOf[StructType] + .fieldNames.toSeq == Seq(physical(4))) + } + val adds = checkpoint.filter("add IS NOT NULL").select("add").toJSON.collect() + assert(adds.nonEmpty) + for (line <- adds) { + val add = JsonUtil.mapper.readTree(line).get("add") + val original = originals(add.get("path").asText) + assert(add.get("partitionValues") == original.get("partitionValues")) + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val parsedPartition = add.get("partitionValues_parsed").get(partitionName) + val rawPartition = original.get("partitionValues").get(partitionName) + assert(Option(parsedPartition).filterNot(_.isNull).map(_.asText) == + Option(rawPartition).filterNot(_.isNull).map(_.asText)) + val rawStats = JsonUtil.mapper.readTree(original.get("stats").asText) + val parsedStats = add.get("stats_parsed") + assert(parsedStats.get("numRecords") == rawStats.get("numRecords")) + for (kind <- Seq("minValues", "maxValues", "nullCount"); + path <- Seq(Seq(physical(0)), Seq(physical(1)), Seq(physical(2), physical(4)))) { + val parsed = path.foldLeft(parsedStats.path(kind))((node, key) => node.path(key)) + val raw = path.foldLeft(rawStats.path(kind))((node, key) => node.path(key)) + assert(Option(parsed).filterNot(n => n.isNull || n.isMissingNode) == + Option(raw).filterNot(n => n.isNull || n.isMissingNode)) + } + } + // Earlier protocol/metadata/file history is absent: v5 checkpoint remains required. + // Keep v5 for version lookup and v6 as the actual post-checkpoint replay tail. + val stream = Files.list(logDir) + val files = try stream.iterator().asScala.toSeq finally stream.close() + files.filter { path => + val name = path.getFileName.toString + name.endsWith(".crc") || + (name.matches("[0-9]{20}\\.json") && name < "00000000000000000005.json") + }.foreach(Files.delete) + println(s"Physical-name checkpoint/layout oracle passed: $mode") + } + + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("payload", new StructType().add("leaf", IntegerType)).add("region", StringType) + val rows = Seq( + Row(1, 10, Row(-10), "east"), Row(1, 10, Row(-10), "east"), + Row(2, 20, Row(-20), "west"), Row(3, null, Row(null), null), + Row(4, 40, null, "east"), Row(5, 100, Row(-100), "north"), + Row(6, null, Row(null), null), Row(7, 200, Row(-200), "east"), + Row(8, 300, Row(-300), "west")) + // Row selections are literal bag oracles, independent of Spark's predicate evaluation. + val reads = Seq( + ("read_all", null, rows.indices, None, None), + ("read_checkpoint", null, 0 until 8, Some(5L), None), + ("read_old_file", "value = 10", Seq(0, 1), None, None), + ("read_new_file", "value = 100", Seq(5), None, None), + ("read_tail", "value = 300", Seq(8), None, None), + ("read_missing", "value = 999", Seq.empty[Int], None, None), + ("read_null", "value IS NULL", Seq(3, 6), None, None), + ("read_nested_old", "payload.leaf = -10", Seq(0, 1), None, None), + ("read_nested_new", "payload.leaf = -100", Seq(5), None, None), + ("read_nested_null", "payload.leaf IS NULL", Seq(3, 4, 6), None, None), + ("read_partition_old", "region = 'east'", Seq(0, 1, 4, 7), None, None), + ("read_partition_new", "region = 'north'", Seq(5), None, None), + ("read_partition_null", "region IS NULL", Seq(3, 6), None, None), + ("read_projection", null, rows.indices, None, Some(Seq("region", "value", "payload")))) + for ((name, predicate, indices, version, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, + version = version.map(Long.box).orNull, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"$mode/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"$mode/$name literal bag") + assert(captured.schema.fields.forall(_.nullable)) + println(s"Physical-name literal row/schema oracle passed: $mode/$name") + } + } + for (version <- Seq(5L, 6L)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.get("schemaString").asText) + assert(result.metadata.partitionColumns == Seq("region")) + assert(result.metadata.configuration == + properties + ("delta.columnMapping.maxColumnId" -> "5")) + case other => fail(s"$mode snapshot v$version must succeed: $other") + } + println(s"Physical-name snapshot mapping oracle passed: $mode/v$version") + } + } + } + } + + test("enable_name_mapping_with_full_rewrite") { + sql("""CREATE TABLE tbl (id INT, value INT, payload STRING) USING delta + TBLPROPERTIES ('delta.columnMapping.mode' = 'none', + 'delta.checkpointInterval' = '1000')""") + sql("INSERT INTO tbl VALUES (1, 10, 'duplicate'), (1, 10, 'duplicate'), (2, NULL, 'nullable')") + sql("INSERT INTO tbl VALUES (3, 30, NULL), (NULL, NULL, NULL), (NULL, NULL, NULL)") + // A separately materialized source prevents replacement from reading its own target. + // Both SQL-created tables are tracked by the normal WorkloadContext cleanup. + sql("CREATE TABLE rewrite_source USING delta AS SELECT id, value, payload FROM tbl") + sql("""CREATE OR REPLACE TABLE tbl USING delta TBLPROPERTIES ( + 'delta.columnMapping.mode' = 'name', 'delta.minReaderVersion' = '2', + 'delta.minWriterVersion' = '5', 'delta.checkpointInterval' = '1000') + AS SELECT id, value, payload FROM rewrite_source""") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val before = (0L to 2L).flatMap(CommitLog.read(t.sourcePath, _)) + val rewrite = CommitLog.read(t.sourcePath, 3L) + val oldAdds = before.collect { case add: AddFile => add } + val newAdds = rewrite.collect { case add: AddFile => add } + val removed = rewrite.collect { case remove: RemoveFile => remove } + assert(!before.exists(_.isInstanceOf[RemoveFile])) + assert(oldAdds.nonEmpty && newAdds.nonEmpty) + val oldPaths = oldAdds.map(_.path).toSet + val newPaths = newAdds.map(_.path).toSet + assert(oldPaths.size == oldAdds.size && newPaths.size == newAdds.size) + assert(removed.map(_.path).toSet == oldPaths && removed.size == oldPaths.size) + assert(newPaths.intersect(oldPaths).isEmpty) + assert((oldPaths -- removed.map(_.path)) ++ newPaths == newPaths) + assert((oldAdds ++ newAdds).forall(add => + Files.isRegularFile(t.sourcePath.resolve(add.path)) && + Files.size(t.sourcePath.resolve(add.path)) == add.size)) + + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("payload", StringType) + val oldMetadata = before.collect { case metadata: LogMetadata => metadata }.last + val newMetadata = rewrite.collect { case metadata: LogMetadata => metadata }.last + assert(DataType.fromJson(oldMetadata.schemaString) == schema) + assert(oldMetadata.configuration == Map( + "delta.columnMapping.mode" -> "none", "delta.checkpointInterval" -> "1000")) + val mapped = DataType.fromJson(newMetadata.schemaString).asInstanceOf[StructType] + assert(mapped.fields.map(f => (f.name, f.dataType, f.nullable)).toSeq == + schema.fields.map(f => (f.name, f.dataType, f.nullable)).toSeq) + val idKey = "delta.columnMapping.id" + val physicalKey = "delta.columnMapping.physicalName" + assert(mapped.fields.map(_.metadata.getLong(idKey)).toSet == Set(1L, 2L, 3L)) + val physicalNames = mapped.fields.map(_.metadata.getString(physicalKey)).toSeq + assert(physicalNames.distinct.size == 3) + assert(mapped.fields.forall(f => f.metadata.getString(physicalKey) != f.name)) + assert(newMetadata.configuration == Map( + "delta.columnMapping.mode" -> "name", "delta.columnMapping.maxColumnId" -> "3", + "delta.checkpointInterval" -> "1000")) + assert(oldMetadata.partitionColumns.isEmpty && newMetadata.partitionColumns.isEmpty) + assert(before.collect { case p: Protocol => p }.last == Protocol(1, 2)) + assert(rewrite.collect { case p: Protocol => p }.last == Protocol(2, 5)) + for (add <- oldAdds) { + assert(spark.read.parquet(t.sourcePath.resolve(add.path).toString).schema == schema) + } + for (add <- newAdds) { + val physical = spark.read.parquet(t.sourcePath.resolve(add.path).toString).schema + assert(physical.fieldNames.toSeq == physicalNames) + assert(physical.fields.map(_.dataType).toSeq == schema.fields.map(_.dataType).toSeq) + assert(physical.fields.map(_.metadata.getLong("parquet.field.id")).toSeq == + mapped.fields.map(_.metadata.getLong(idKey)).toSeq) + } + println(s"Full-rewrite action/physical-name oracle passed: " + + s"removed=${oldPaths.size}, fresh=${newPaths.size}") + + // Every selection is a literal bag, including both copies of each duplicate/null row. + val rows = Seq(Row(1, 10, "duplicate"), Row(1, 10, "duplicate"), + Row(2, null, "nullable"), Row(3, 30, null), Row(null, null, null), Row(null, null, null)) + val reads = Seq( + ("read_before", null, rows.indices, Some(2L), None), + ("read_all", null, rows.indices, None, None), + ("read_rewritten", null, rows.indices, Some(3L), None), + ("read_projection", null, rows.indices, None, Some(Seq("payload", "id"))), + ("read_hit", "value = 10", Seq(0, 1), None, None), + ("read_miss", "value = 999", Seq.empty[Int], None, None), + ("read_null_value", "value IS NULL", Seq(2, 4, 5), None, None), + ("read_null_payload", "payload IS NULL", Seq(3, 4, 5), None, None), + ("read_null_id_projection", "id IS NULL", Seq(4, 5), None, Some(Seq("id", "value")))) + for ((name, predicate, indices, version, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, + version = version.map(Long.box).orNull, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"full rewrite/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"full rewrite/$name literal bag") + assert(captured.schema == oracle.schema) + println(s"Full-rewrite literal row/schema oracle passed: $name") + } + } + for ((version, metadata, reader, writer) <- Seq( + (2L, oldMetadata, 1, 2), (3L, newMetadata, 2, 5))) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.schemaString) + assert(result.metadata.configuration == metadata.configuration) + assert(result.metadata.partitionColumns.isEmpty) + assert(result.protocol.minReaderVersion == reader) + assert(result.protocol.minWriterVersion == writer) + case other => fail(s"full rewrite snapshot v$version must succeed: $other") + } + println(s"Full-rewrite snapshot transition oracle passed: v$version") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala index 061b7d70..06d0b58b 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala @@ -16,9 +16,38 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.{ByteBuffer, ByteOrder} +import java.nio.file.{Files, Path, StandardCopyOption} +import java.sql.Timestamp +import java.time.{Instant, LocalDate} + +import scala.jdk.CollectionConverters._ + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.commons.io.FileUtils +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.example.data.Group +import org.apache.parquet.example.data.simple.{NanoTime, SimpleGroupFactory} +import org.apache.parquet.hadoop.{ParquetFileReader, ParquetReader} +import org.apache.parquet.hadoop.example.{ExampleParquetWriter, GroupReadSupport} +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.parquet.schema.{LogicalTypeAnnotation, PrimitiveType} +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.functions.{col, lit, when} +import org.apache.spark.sql.types.{ArrayType, BooleanType, DataType, DecimalType, IntegerType, + StringType, StructType, TimestampNTZType, TimestampType} +import org.apache.spark.unsafe.types.UTF8String + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.deltaharness.{DeltaHarness, Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} import io.delta.workload.json.JsonUtil -import io.delta.workload.log.AddFile +import io.delta.workload.log.{Action, AddFile, CommitLog, DomainMetadata, Metadata, Protocol, + RemoveFile, Txn} +import io.delta.workload.model._ +import io.delta.workload.tables.DeletionVectorPayloads.payload /** * Data skipping, statistics, and partitioning workloads. @@ -26,9 +55,12 @@ import io.delta.workload.log.AddFile * nested field predicates, boolean predicates, typed stats, null handling, * multiple files for skipping, missing stats, long strings, column mapping stats, * generated columns, DVs, partitioned skipping, partition pruning, projection, - * stats edge cases (null min/max, numRecords-only, truncated strings). + * stats edge cases (null min/max, numRecords-only, truncated strings), predicate row + * semantics (nondeterminism, signed zero, casts), partial/tight/selected statistics, + * timestamp statistic encodings and time zones, stats refresh, and clustering rewrites. */ -class DataSkippingSuite extends WorkloadTestSuite("data_skipping") { +class DataSkippingSuite extends WorkloadTestSuite("data_skipping") + with ScopedSessionConf with LiteralOracleChecks { // === Data Skipping === @@ -1406,4 +1438,3535 @@ class DataSkippingSuite extends WorkloadTestSuite("data_skipping") { snapshotSpec(t) } + // Row semantics for read predicates, without a pruning assertion: nondeterministic + // values, signed zero, explicit casts at precision boundaries and narrowing wrap-around, + // and complex filters. Signed zero needs Parquet filter pushdown disabled and the three + // narrowing casts need ANSI disabled; both are scoped to those tests. Use `startswith`; + // `starts_with` is unsupported. + + for (partitioned <- Seq(false, true)) { + val layout = if (partitioned) "partitioned" else "unpartitioned" + test(s"nondeterministic_values_$layout") { + val partition = if (partitioned) "PARTITIONED BY (p)" else "" + sql(s"CREATE TABLE tbl (id INT, value INT, p INT) USING delta $partition") + sql("INSERT INTO tbl VALUES (1, 10, 1), (2, 20, 1), (2, 20, 1)") + sql("INSERT INTO tbl VALUES (3, 20, 2), (4, 30, 2), (5, 40, 3)") + sql("INSERT INTO tbl VALUES (6, NULL, 1), (7, 50, NULL), (8, NULL, NULL)") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + + // Literal bags are independent of the filtered Delta reads, including duplicate/NULL rows. + val all = Seq( + "(1, 10, 1)", + "(2, 20, 1)", + "(2, 20, 1)", + "(3, 20, 2)", + "(4, 30, 2)", + "(5, 40, 3)", + "(6, NULL, 1)", + "(7, 50, NULL)", + "(8, NULL, NULL)" + ) + val equality = Seq("(2, 20, 1)", "(2, 20, 1)", "(3, 20, 2)") + val range = equality :+ "(4, 30, 2)" + val partitionBag = Seq("(1, 10, 1)", "(2, 20, 1)", "(2, 20, 1)", "(6, NULL, 1)") + val cases = Seq( + ("read_all", null, all), + ("random_all", "rand() >= 0", all), + ("random_none", "rand() < 0", Seq.empty[String]), + ("equality_control", "value = 20", equality), + ("range_control", "value >= 20 AND value < 40", range), + ("and_equality", "rand() >= 0 AND value = 20", equality), + ("and_range", "rand() >= 0 AND (value >= 20 AND value < 40)", range), + ("and_none", "rand() < 0 AND value = 20", Seq.empty[String]), + ("or_equality", "rand() < 0 OR value = 20", equality), + ("or_range", "rand() < 0 OR (value >= 20 AND value < 40)", range), + ("or_all", "rand() >= 0 OR value = 20", all), + ("partition_control", "p = 1", partitionBag), + ("and_partition", "p = 1 AND rand() >= 0", partitionBag), + ("or_partition", "p = 1 OR rand() < 0", partitionBag), + // Spark can simplify the domain-only controls. Referencing the row's small exact INT id + // keeps a nondeterministic expression at read time: adding [0, 1) cannot decrease id. + ("row_random_all", "rand() + id >= id", all), + ("row_random_none", "rand() + id < id", Seq.empty[String]), + ("row_and_equality", "rand() + id >= id AND value = 20", equality), + ("row_and_range", "rand() + id >= id AND (value >= 20 AND value < 40)", range), + ("row_or_equality", "rand() + id < id OR value = 20", equality), + ("row_or_range", "rand() + id < id OR (value >= 20 AND value < 40)", range), + ("row_and_partition", "p = 1 AND rand() + id >= id", partitionBag), + ("row_or_partition", "p = 1 OR rand() + id < id", partitionBag) + ) + + // rand() stays in the serialized read predicate. Its [0, 1) domain gives a stable bag + // across RNG seeds, partition counts and replays; random values are never stored on INSERT. + for ((label, predicate, bag) <- cases) { + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(_) => + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + s"SELECT CAST(id AS INT) AS id, CAST(value AS INT) AS value, " + + s"CAST(p AS INT) AS p FROM VALUES ${bag.mkString(", ")} AS expected(id, value, p)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS INT) AS value, " + + "CAST(NULL AS INT) AS p WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, value INT, p INT")) + require( + result.metadata.partitionColumns == + (if (partitioned) Seq("p") else Seq.empty) + ) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + } + + test("signed_zero") { + setForTest("spark.sql.parquet.filterPushdown", "false") + require( + spark.conf.get("spark.sql.parquet.filterPushdown") == "false", + "The signed-zero bag is validated with Parquet filter pushdown disabled" + ) + sql("CREATE TABLE tbl (id INT, f FLOAT, d DOUBLE) USING delta") + // Separate writes keep the two zero signs and nonzero ranges in distinct files. + sql("""INSERT INTO tbl VALUES + |(1, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE)), + |(2, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE)), + |(2, CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE))""".stripMargin) + sql("""INSERT INTO tbl VALUES + |(3, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE)), + |(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE)), + |(4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE))""".stripMargin) + sql("INSERT INTO tbl VALUES (5, -1, -1), (6, -2, -2)") + sql("INSERT INTO tbl VALUES (7, 1, 1), (8, 2, 2)") + sql("INSERT INTO tbl VALUES (9, NULL, NULL)") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + + // Literal bags do not filter the source table. IDs distinguish the original zero signs; + // exact duplicates check multiplicity. Row equality need not retain a zero's sign bit. + val zeros = Seq("(1, 0, 0)", "(2, 0, 0)", "(2, 0, 0)", "(3, 0, 0)", "(4, 0, 0)", "(4, 0, 0)") + val negative = Seq("(5, -1, -1)", "(6, -2, -2)") + val positive = Seq("(7, 1, 1)", "(8, 2, 2)") + val nulls = Seq("(9, NULL, NULL)") + val nonzero = negative ++ positive + val all = zeros ++ nonzero ++ nulls + val cases = Seq(("read_all", null, all)) ++ + Seq(("f", "FLOAT"), ("d", "DOUBLE")).flatMap { + case (column, dataType) => + val plus = s"CAST('0.0' AS $dataType)" + val minus = s"CAST('-0.0' AS $dataType)" + val one = s"CAST(1 AS $dataType)" + val negativeOne = s"CAST(-1 AS $dataType)" + val typedNull = s"CAST(NULL AS $dataType)" + val comparisons = for { + (sign, zero) <- Seq(("positive", plus), ("negative", minus)) + (label, operator, bag) <- Seq( + ("eq", "=", zeros), + ("null_safe_eq", "<=>", zeros), + ("ne", "!=", nonzero), + ("lt", "<", negative), + ("le", "<=", negative ++ zeros), + ("gt", ">", positive), + ("ge", ">=", zeros ++ positive) + ) + } yield (s"${column}_${label}_$sign", s"$column $operator $zero", bag) + val inLists = Seq( + ("in_positive", plus, zeros), + ("in_negative", minus, zeros), + ("in_both", s"$plus, $minus", zeros), + ("in_positive_null", s"$plus, $typedNull", zeros), + ("in_negative_null", s"$minus, $typedNull", zeros), + ("in_both_null", s"$plus, $minus, $typedNull", zeros), + ("in_positive_nonzero", s"$plus, $one", zeros :+ "(7, 1, 1)"), + ("in_negative_nonzero", s"$minus, $negativeOne", zeros :+ "(5, -1, -1)"), + ( + "in_both_null_nonzero", + s"$plus, $minus, $typedNull, $negativeOne, $one", + zeros ++ Seq("(5, -1, -1)", "(7, 1, 1)") + ), + ("in_nonzero_null", s"$negativeOne, $one, $typedNull", Seq("(5, -1, -1)", "(7, 1, 1)")), + ("in_null", typedNull, Seq.empty[String]), + ("in_miss", s"CAST(9 AS $dataType), $typedNull", Seq.empty[String]) + ) + comparisons ++ Seq((s"${column}_null_safe_null", s"$column <=> $typedNull", nulls)) ++ + inLists.map { + case (label, literals, bag) => + (s"${column}_$label", s"$column IN ($literals)", bag) + } + } + + for ((label, predicate, bag) <- cases) { + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + require(spark.conf.get("spark.sql.parquet.filterPushdown") == "false") + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + "SELECT CAST(id AS INT) AS id, CAST(f AS FLOAT) AS f, CAST(d AS DOUBLE) AS d " + + s"FROM VALUES ${bag.mkString(", ")} AS expected(id, f, d)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS FLOAT) AS f, " + + "CAST(NULL AS DOUBLE) AS d WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, f FLOAT, d DOUBLE")) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + // Explicit predicate casts, precision boundaries and legacy narrowing. + // Expected IDs are derived independently of the CAST predicates. Source duplicates stay intact. + for ((subcase, sourceType, sourceRows, checks, legacy) <- Seq( + ( + "float_double", + "FLOAT", + Seq( + (1, "-0.1"), + (2, "0.1"), + (2, "0.1"), + (3, "0.10000001"), + (4, "0.2"), + (5, "1.0"), + (6, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6)), + ("read_hit", "CAST(value AS DOUBLE) = CAST('0.10000000149011612' AS DOUBLE)", Seq(2)), + ( + "decimal_spelling_miss", + "CAST(value AS DOUBLE) = CAST('0.1' AS DOUBLE)", + Seq.empty[Int] + ), + ("above_decimal", "CAST(value AS DOUBLE) > CAST('0.1' AS DOUBLE)", Seq(2, 3, 4, 5)), + ( + "adjacent_range", + "CAST(value AS DOUBLE) >= CAST('0.10000000149011612' AS DOUBLE) AND " + + "CAST(value AS DOUBLE) < CAST('0.10000000894069672' AS DOUBLE)", + Seq(2) + ), + ( + "negative_hit", + "CAST(value AS DOUBLE) = CAST('-0.10000000149011612' AS DOUBLE)", + Seq(1) + ), + ("read_miss", "CAST(value AS DOUBLE) = CAST('2' AS DOUBLE)", Seq.empty[Int]) + ), + false + ), + ( + "double_2p53", + "BIGINT", + Seq( + (1, "9007199254740991"), + (2, "9007199254740992"), + (3, "9007199254740993"), + (3, "9007199254740993"), + (4, "9007199254740994"), + (5, "9007199254740995"), + (6, "-9007199254740993"), + (7, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7)), + ("read_hit", "CAST(value AS DOUBLE) = CAST('9007199254740992' AS DOUBLE)", Seq(2, 3)), + ("adjacent_hit", "CAST(value AS DOUBLE) = CAST('9007199254740994' AS DOUBLE)", Seq(4)), + ( + "adjacent_range", + "CAST(value AS DOUBLE) > CAST('9007199254740992' AS DOUBLE) AND " + + "CAST(value AS DOUBLE) < CAST('9007199254740996' AS DOUBLE)", + Seq(4) + ), + ( + "below_boundary", + "CAST(value AS DOUBLE) < CAST('9007199254740992' AS DOUBLE)", + Seq(1, 6) + ), + ( + "read_miss", + "CAST(value AS DOUBLE) = CAST('9007199254740998' AS DOUBLE)", + Seq.empty[Int] + ) + ), + false + ), + ( + "float_2p24", + "INT", + Seq( + (1, "16777215"), + (2, "16777216"), + (3, "16777217"), + (3, "16777217"), + (4, "16777218"), + (5, "16777219"), + (6, "-16777217"), + (7, "NULL") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7)), + ("read_hit", "CAST(value AS FLOAT) = CAST('16777216' AS FLOAT)", Seq(2, 3)), + ("adjacent_hit", "CAST(value AS FLOAT) = CAST('16777218' AS FLOAT)", Seq(4)), + ( + "adjacent_range", + "CAST(value AS FLOAT) > CAST('16777216' AS FLOAT) AND " + + "CAST(value AS FLOAT) < CAST('16777220' AS FLOAT)", + Seq(4) + ), + ("below_boundary", "CAST(value AS FLOAT) < CAST('16777216' AS FLOAT)", Seq(1, 6)), + ("read_miss", "CAST(value AS FLOAT) = CAST('16777222' AS FLOAT)", Seq.empty[Int]) + ), + false + ), + ( + "long_int_wrap", + "BIGINT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "4294967301"), + (3, "4294967301"), + (4, "4294967300"), + (5, "4294967302"), + (6, "-4294967291"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS INT) = CAST('5' AS INT)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS INT) = CAST('4' AS INT)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS INT) = CAST('6' AS INT)", Seq(5, 9)), + ("below_five", "CAST(value AS INT) < CAST('5' AS INT)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS INT) > CAST('5' AS INT)", Seq(5, 9)), + ("read_miss", "CAST(value AS INT) = CAST('7' AS INT)", Seq.empty[Int]) + ), + true + ), + ( + "to_short_wrap", + "INT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "65541"), + (3, "65541"), + (4, "65540"), + (5, "65542"), + (6, "-65531"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS SHORT) = CAST('5' AS SHORT)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS SHORT) = CAST('4' AS SHORT)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS SHORT) = CAST('6' AS SHORT)", Seq(5, 9)), + ("below_five", "CAST(value AS SHORT) < CAST('5' AS SHORT)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS SHORT) > CAST('5' AS SHORT)", Seq(5, 9)), + ("read_miss", "CAST(value AS SHORT) = CAST('7' AS SHORT)", Seq.empty[Int]) + ), + true + ), + ( + "int_byte_wrap", + "INT", + Seq( + (1, "4"), + (2, "5"), + (2, "5"), + (3, "261"), + (3, "261"), + (4, "260"), + (5, "262"), + (6, "-251"), + (7, "NULL"), + (8, "-1"), + (9, "6") + ), + Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("read_hit", "CAST(value AS BYTE) = CAST('5' AS BYTE)", Seq(2, 3, 6)), + ("lower_neighbor", "CAST(value AS BYTE) = CAST('4' AS BYTE)", Seq(1, 4)), + ("upper_neighbor", "CAST(value AS BYTE) = CAST('6' AS BYTE)", Seq(5, 9)), + ("below_five", "CAST(value AS BYTE) < CAST('5' AS BYTE)", Seq(1, 4, 8)), + ("above_five", "CAST(value AS BYTE) > CAST('5' AS BYTE)", Seq(5, 9)), + ("read_miss", "CAST(value AS BYTE) = CAST('7' AS BYTE)", Seq.empty[Int]) + ), + true + ) + )) { + test(s"cast_filters_$subcase") { + // ReadQuery has no session config field. Supply the same legacy session on every replay. + if (legacy) { + setForTest("spark.sql.ansi.enabled", "false") + require( + spark.conf.get("spark.sql.ansi.enabled") == "false", + "Narrowing CAST fixtures require a non-ANSI session for capture and replay" + ) + } + sql(s"CREATE TABLE tbl (id INT, value $sourceType) USING delta") + // Separate writes leave ordinary and boundary values in multiple stats-bearing files. + for (batch <- sourceRows.grouped(2)) { + val values = batch.map { case (id, value) => s"($id, $value)" }.mkString(", ") + sql(s"INSERT INTO tbl VALUES $values") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + for ((label, predicate, expectedIds) <- checks) { + val bag = sourceRows.filter { case (id, _) => expectedIds.contains(id) } + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + if (legacy) require(spark.conf.get("spark.sql.ansi.enabled") == "false") + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + // Only source-type casts construct these literal bags; no predicate CAST is reused. + val oracle = if (bag.nonEmpty) { + val values = bag.map { case (id, value) => s"($id, $value)" }.mkString(", ") + s"SELECT CAST(id AS INT) AS id, CAST(value AS $sourceType) AS value " + + s"FROM VALUES $values AS expected(id, value)" + } else { + s"SELECT CAST(NULL AS INT) AS id, CAST(NULL AS $sourceType) AS value WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL(s"id INT, value $sourceType")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + if (subcase == "float_double") { + val compositionCases = Seq( + ("direct_nonnegative", "CAST(value AS DOUBLE) >= 0", Seq(2, 3, 4, 5)), + ( + "compound_hit", + "(CAST(value AS DOUBLE) > 0.15 AND id < 5) OR " + + "(CAST(value AS DOUBLE) < 0 AND id = 1)", + Seq(1, 4) + ), + ( + "compound_miss", + "(CAST(value AS DOUBLE) > 2 AND id < 5) OR " + + "(CAST(value AS DOUBLE) < -2 AND id = 1)", + Seq.empty[Int] + ) + ) + for ((label, predicate, ids) <- compositionCases) { + val bag = sourceRows.filter { case (id, _) => ids.contains(id) } + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val oracle = if (bag.nonEmpty) { + val values = bag.map { case (id, value) => s"($id, $value)" }.mkString(", ") + s"SELECT CAST(id AS INT) AS id, CAST(value AS FLOAT) AS value " + + s"FROM VALUES $values AS expected(id, value)" + } else { + "SELECT CAST(NULL AS INT) AS id, CAST(NULL AS FLOAT) AS value WHERE false" + } + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + } + + } + } + + // Complex filter row semantics. File counts do not prove filter pushdown. + test("complex_filters_month_timestamp") { + // Read specs have no timezone field; use UTC through queued capture and every replay. + require(spark.conf.get("spark.sql.session.timeZone") == "UTC") + sql("CREATE TABLE tbl (id INT, ts TIMESTAMP) USING delta") + val rows = Seq( + "(1, TIMESTAMP'2023-12-31 23:59:59.999999')", + "(2, TIMESTAMP'2024-01-01 00:00:00')", + "(3, TIMESTAMP'2024-01-31 23:59:59.999999')", + "(4, TIMESTAMP'2024-02-01 00:00:00')", + "(5, TIMESTAMP'2024-02-29 23:59:59.999999')", + "(6, TIMESTAMP'2024-03-01 00:00:00')", + "(7, TIMESTAMP'2024-12-31 23:59:59.999999')", + "(8, TIMESTAMP'2025-01-01 00:00:00')", + "(8, TIMESTAMP'2025-01-01 00:00:00')", + "(9, NULL)" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("january", "month(ts) = 1", Seq(2, 3, 8)), + ("february", "month(ts) = 2", Seq(4, 5)), + ("december", "month(ts) = 12", Seq(1, 7)), + ("miss_june", "month(ts) = 6", Seq.empty[Int]), + ("null_month", "month(ts) IS NULL", Seq(9)) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(ts AS TIMESTAMP) AS ts" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, ts)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, ts TIMESTAMP")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + test("complex_filters_strings") { + sql("CREATE TABLE tbl (id INT, name STRING, s STRUCT) USING delta") + val rows = Seq( + "(1, 'apple', named_struct('name', 'apple'))", + "(2, 'banana', named_struct('name', 'banana'))", + "(3, 'alice', named_struct('name', 'alice'))", + "(4, 'bob', named_struct('name', 'bob'))", + "(5, 'charlie', named_struct('name', 'charlie'))", + "(6, 'diana', named_struct('name', 'diana'))", + "(7, 'Apple', named_struct('name', 'APPLE'))", + "(8, 'ALICE', named_struct('name', 'Alice'))", + "(8, 'ALICE', named_struct('name', 'Alice'))", + "(9, '', named_struct('name', ''))", + "(10, NULL, named_struct('name', NULL))", + "(11, 'apPle', named_struct('name', NULL))" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)), + ("prefix_hit", "startswith(name, 'app')", Seq(1)), + ("prefix_a", "startswith(name, 'a')", Seq(1, 3, 11)), + ("prefix_nested", "startswith(s.name, 'a')", Seq(1, 3)), + ("prefix_miss", "startswith(name, 'z')", Seq.empty[Int]), + ("prefix_empty", "startswith(name, '')", Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 11)), + ("prefix_null", "startswith(name, CAST(NULL AS STRING))", Seq.empty[Int]), + ("prefix_null_result", "startswith(name, 'a') IS NULL", Seq(10)), + ("upper_alice", "upper(name) = 'ALICE'", Seq(3, 8)), + ("upper_apple", "upper(name) = 'APPLE'", Seq(1, 7, 11)), + ("upper_miss", "upper(name) = 'ZEBRA'", Seq.empty[Int]), + ("upper_null", "upper(name) IS NULL", Seq(10)), + ("upper_selective", "upper(name) = 'APPLE' AND id > 1", Seq(7, 11)) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(name AS STRING) AS name, " + + "CAST(s AS STRUCT) AS s" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, name, s)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require(schema == StructType.fromDDL("id INT, name STRING, s STRUCT")) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + test("complex_filters_deeply_nested") { + sql( + "CREATE TABLE tbl (id INT, s STRUCT) USING delta" + ) + val rows = Seq( + "(1, named_struct('a', 1, 'b', 1, 'c', NULL, 'd', NULL, 'e', NULL, 'f', NULL, " + + "'g', NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(2, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 1, 'e', NULL, 'f', NULL, 'g', " + + "NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(3, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 1, 'g', NULL, " + + "'h', NULL, 'i', NULL, 'j', NULL))", + "(4, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "1, 'i', NULL, 'j', NULL))", + "(5, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 0))", + "(5, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 0))", + "(6, named_struct('a', 0, 'b', 1, 'c', 0, 'd', 1, 'e', 1, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))", + "(7, named_struct('a', 1, 'b', 0, 'c', 1, 'd', 1, 'e', 1, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))", + "(8, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', 1))", + "(9, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 1, 'f', 0, 'g', 0, 'h', " + + "0, 'i', 1, 'j', NULL))", + "(10, named_struct('a', NULL, 'b', 1, 'c', 0, 'd', 1, 'e', 1, 'f', 1, 'g', 0, " + + "'h', 1, 'i', 1, 'j', 0))", + "(11, named_struct('a', NULL, 'b', NULL, 'c', NULL, 'd', NULL, 'e', NULL, 'f', " + + "NULL, 'g', NULL, 'h', NULL, 'i', NULL, 'j', NULL))", + "(12, named_struct('a', 1, 'b', 0, 'c', 0, 'd', 0, 'e', 0, 'f', 1, 'g', 0, 'h', " + + "1, 'i', 1, 'j', 0))" + ) + for (batch <- rows.grouped(2)) { + sql(s"INSERT INTO tbl VALUES ${batch.mkString(", ")}") + } + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + // Enumerated source IDs preserve duplicates without evaluating the predicate in the oracle. + val cases = Seq( + ("read_all", null, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)), + ( + "deep_hit", + "s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))", + Seq(1, 2, 3, 4, 5) + ), + ( + "deep_false", + "NOT (s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND " + + "(s.f = 1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1))))))))))", + Seq(6, 7, 8, 12) + ), + ( + "deep_null", + "(s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))) IS " + + "NULL", + Seq(9, 10, 11) + ), + ( + "deep_miss", + "(s.a = 1 AND (s.b = 1 OR (NOT (s.c = 1) AND (s.d = 1 OR (s.e = 1 AND (s.f = " + + "1 OR (NOT (s.g = 1) AND (s.h = 1 OR (s.i = 1 AND NOT (s.j = 1)))))))))) AND " + + "id = 99", + Seq.empty[Int] + ) + ) + for ((label, predicate, ids) <- cases) { + val bag = rows.filter(row => ids.contains(row.drop(1).takeWhile(_ != ',').toInt)) + readSpec(t, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == bag.size) + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val values = if (bag.nonEmpty) bag.mkString(", ") else "(NULL, NULL)" + val projection = "CAST(id AS INT) AS id, CAST(s AS STRUCT) AS s" + val oracle = s"SELECT $projection FROM VALUES $values AS expected(id, s)" + + (if (bag.nonEmpty) "" else " WHERE false") + val expectedPath = outputDir + .resolve(table.outputName) + .resolve("expected") + .resolve(s"${table.outputName}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual( + spark.sql(oracle), + spark.read.parquet(expectedPath.toString), + s"${table.outputName}/$label literal oracle" + ) + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + require( + schema == StructType.fromDDL( + "id INT, s STRUCT" + ) + ) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + require(result.protocol.minReaderVersion == 1) + require(result.protocol.minWriterVersion == 2) + case other => fail(s"${table.outputName} snapshot must succeed: $other") + } + } + } + + // Incomplete or selected file statistics. + test("missing_add_stats_predicates") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + .add("flag", BooleanType).add("items", ArrayType(IntegerType, containsNull = true)) + val nullArray = Row(1, 1, true, null) + val emptyArray = Row(2, 2, false, Seq.empty[Integer]) + val nullElement = Row(3, null, null, Seq[Integer](null)) + val duplicate = Row(4, 3, true, Seq[Integer](1, 7)) + val nonmatching = Row(5, 4, false, Seq[Integer](2, 3)) + val matchingWithNull = Row(6, 5, null, Seq[Integer](7, null)) + val nonmatchingWithNull = Row(7, 2, true, Seq[Integer](null, 2)) + val matching = Row(8, null, false, Seq[Integer](7)) + val batches = Seq(Seq(nullArray, emptyArray, nullElement, duplicate, duplicate), + Seq(nonmatching, matchingWithNull, nonmatchingWithNull, matching)) + val rows = batches.flatten + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT, flag BOOLEAN, items ARRAY) USING delta + TBLPROPERTIES ('delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '100')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 1, TRUE, CAST(NULL AS ARRAY)), (2, 2, FALSE, CAST(array() AS ARRAY)), + (3, NULL, NULL, array(CAST(NULL AS INT))), (4, 3, TRUE, array(1, 7)), + (4, 3, TRUE, array(1, 7)) AS v(id, value, flag, items)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (5, 4, FALSE, array(2, 3)), (6, 5, NULL, array(7, CAST(NULL AS INT))), + (7, 2, TRUE, array(CAST(NULL AS INT), 2)), (8, NULL, FALSE, array(7)) + AS v(id, value, flag, items)""") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val metadata = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + batches.zipWithIndex.foreach { case (batch, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.stats.isDefined) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"predicate_input_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + // Arrays have no scalar min/max statistics; their predicates require residual filtering. + require(!stats.path("minValues").has("items") && !stats.path("maxValues").has("items")) + } + val stream = Files.list(dir.resolve("_delta_log")) + val names = try stream.iterator().asScala.map(_.getFileName.toString).toVector + finally stream.close() + require(!names.exists(_.contains("checkpoint"))) + // A cached allFiles set in a version CRC must not hide the missing AddFile statistics. + names.filter(_.matches("[0-9]{20}\\.crc")) + .foreach(name => Files.delete(dir.resolve("_delta_log").resolve(name))) + } + for (version <- 1L to 2L) { + modifyCommitActions(t, version) { actions => + actions.map { + case a: AddFile => a.copy(stats = None) + case other => other + } + } + } + mutateTable(t) { dir => + val actions = (0L to 2L).flatMap(v => CommitLog.read(dir, v)) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 2 && adds.map(_.path).distinct.size == 2) + require(adds.forall(_.stats.isEmpty), "Every persisted AddFile must omit statistics") + require(!actions.exists(_.isInstanceOf[RemoveFile])) + } + + // Expected bags come from literal values and SQL three-valued logic, without a Delta scan. + val casePredicate = "CASE WHEN value IN (1, 3) THEN TRUE " + + "WHEN value = 2 THEN FALSE ELSE CAST(NULL AS BOOLEAN) END" + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("ne", "value <> 2", Seq(nullArray, duplicate, duplicate, nonmatching, matchingWithNull)), + ("in", "value IN (1, 3, 99)", Seq(nullArray, duplicate, duplicate)), + ("in_with_null", "value IN (2, NULL)", Seq(emptyArray, nonmatchingWithNull)), + ("in_miss", "value IN (99, NULL)", Seq.empty), + ("is_not_true", "flag IS NOT TRUE", + Seq(emptyArray, nullElement, nonmatching, matchingWithNull, matching)), + ("array_contains", "array_contains(items, 7)", + Seq(duplicate, duplicate, matchingWithNull, matching)), + ("array_contains_miss", "array_contains(items, 99)", Seq.empty), + ("array_contains_is_not_true", "array_contains(items, 7) IS NOT TRUE", + Seq(nullArray, emptyArray, nullElement, nonmatching, nonmatchingWithNull)), + ("array_contains_not", "NOT array_contains(items, 7)", Seq(emptyArray, nonmatching)), + ("case_when", casePredicate, Seq(nullArray, duplicate, duplicate)), + ("case_when_is_not_true", s"($casePredicate) IS NOT TRUE", + Seq(emptyArray, nullElement, nonmatching, matchingWithNull, nonmatchingWithNull, matching))) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful missing-stat $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_missing_stats_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful missing-stat snapshot, got $other") + } + } + } + + for (variant <- Seq("no_null_count", "no_num_records", "neither_count", "partial_null_count")) { + test(s"partial_counts_$variant") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + val batches = Seq( + Seq(Row(1, 10), Row(2, null), Row(3, 30), Row(3, 30)), + Seq(Row(4, 40), Row(5, 50)), + Seq(Row(6, null), Row(7, null)), + Seq(Row(8, 80), Row(9, null))) + val rows = batches.flatten + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + // One mixed/all-null/non-null file per commit; duplicate rows must survive reads. + val values = batch.map { r => + s"(${r.getInt(0)}, ${if (r.isNullAt(1)) "NULL" else r.getInt(1).toString})" + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS v(id, value)") + } + val t = registerTable("tbl") + var copiedTable: Path = null + + // Check real AddFiles and their row-derived stats, then mutate only the copied table. + mutateTable(t) { dir => + copiedTable = dir + val initial = CommitLog.read(dir, 0) + val originalMetadata = initial.collect { case m: Metadata => m }.head + require(DataType.fromJson(originalMetadata.schemaString) == schema) + require(originalMetadata.configuration == properties) + require(originalMetadata.partitionColumns.isEmpty) + val logFiles = Files.list(dir.resolve("_delta_log")) + try { + val names = logFiles.iterator().asScala.map(_.getFileName.toString).toSeq + require(!names.exists(_.contains("checkpoint")), "Commit replay must be required") + // Version CRCs can contain a cached allFiles set with the original stats. + names.filter(_.matches("[0-9]{20}\\.crc")).foreach { name => + Files.delete(dir.resolve("_delta_log").resolve(name)) + } + } finally logFiles.close() + + batches.zipWithIndex.foreach { case (batch, index) => + val version = index.toLong + 1 + val actions = CommitLog.read(dir, version) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 1, s"Expected one AddFile at version $version") + val add = adds.head + require(add.deletionVector.isEmpty && add.baseRowId.isEmpty && + add.defaultRowCommitVersion.isEmpty) + val oracle = spark.createDataFrame(batch.asJava, schema) + RowComparison.assertRowsEqual(oracle, + spark.read.parquet(dir.resolve(add.path).toString), s"input_v$version") + val stats = JsonUtil.mapper.readTree(add.stats.get).asInstanceOf[ObjectNode] + require(stats.path("numRecords").asLong(-1) == batch.size) + for ((column, ordinal) <- Seq("id" -> 0, "value" -> 1)) { + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + if (values.nonEmpty) { + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + } + val before = stats.deepCopy() + // Keep the last file intact as a control for per-file partial statistics. + if (version < 4) { + variant match { + case "no_null_count" => stats.remove("nullCount") + case "no_num_records" => stats.remove("numRecords") + case "neither_count" => + stats.remove("nullCount") + stats.remove("numRecords") + case "partial_null_count" => + if (version == 3) stats.remove("nullCount") + else stats.get("nullCount").asInstanceOf[ObjectNode] + .remove(if (version == 1) "value" else "id") + } + require(stats != before, s"Mutation must change stats at version $version") + } else require(stats == before) + require(stats.get("minValues") == before.get("minValues")) + require(stats.get("maxValues") == before.get("maxValues")) + val changed = add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + CommitLog.write(dir, version, actions.map { + case _: AddFile => changed + case other => other + }) + val reread = CommitLog.read(dir, version).collect { case a: AddFile => a } + require(reread == Seq(changed), s"Persisted mutation mismatch at version $version") + require(JsonUtil.mapper.readTree(reread.head.stats.get) == stats) + } + } + + // These bags are derived directly from the literal input, without a Delta scan or stats. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("value_is_null", "value IS NULL", rows.filter(_.isNullAt(1))), + ("value_is_not_null", "value IS NOT NULL", rows.filterNot(_.isNullAt(1))), + ("value_eq_30", "value = 30", Seq(Row(3, 30), Row(3, 30))), + ("value_gt_35", "value > 35", Seq(Row(4, 40), Row(5, 50), Row(8, 80))), + ("value_eq_999", "value = 999", Seq.empty), + ("id_is_null", "id IS NULL", Seq.empty), + ("id_is_not_null", "id IS NOT NULL", rows)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful snapshot, got $other") + } + } + } + } + + test("remove_stats_without_active_add_stats") { + val schema = new StructType().add("id", IntegerType).add("value", IntegerType) + val batches = Seq( + Seq(Row(1, 10), Row(5, null)), + Seq(Row(2, 20), Row(3, 30), Row(3, 30), Row(4, null)), + Seq(Row(100, 1000), Row(101, 1010))) + val rewrittenRows = Seq(Row(3, 30), Row(3, 30), Row(4, null)) + val survivors = batches.head ++ rewrittenRows + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, value INT) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { r => + s"(${r.getInt(0)}, ${if (r.isNullAt(1)) "NULL" else r.getInt(1).toString})" + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS v(id, value)") + } + // Rewrite one file, remove another entirely, and leave the first file untouched. + sql("DELETE FROM tbl WHERE id = 2 OR id >= 100") + val t = registerTable("tbl") + var copiedTable: Path = null + + mutateTable(t) { dir => + copiedTable = dir + val commits = (0L to 4L).map(v => v -> CommitLog.read(dir, v)).toMap + val originalMetadata = commits(0).collect { case m: Metadata => m }.head + require(DataType.fromJson(originalMetadata.schemaString) == schema) + require(originalMetadata.configuration == properties) + require(originalMetadata.partitionColumns.isEmpty) + require(commits(0).collect { case a: AddFile => a }.isEmpty) + val logFiles = Files.list(dir.resolve("_delta_log")) + try { + val names = logFiles.iterator().asScala.map(_.getFileName.toString).toSeq + require(!names.exists(_.contains("checkpoint")), "Commit replay must be required") + // Prevent a version CRC's cached allFiles from hiding the stats mutation. + names.filter(_.matches("[0-9]{20}\\.crc")).foreach { name => + Files.delete(dir.resolve("_delta_log").resolve(name)) + } + } finally logFiles.close() + + val inputAdds = batches.zipWithIndex.map { case (batch, index) => + val adds = commits(index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1, s"Expected one input AddFile for batch $index") + require(!commits(index.toLong + 1).exists(_.isInstanceOf[RemoveFile])) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"original_batch_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for ((column, ordinal) <- Seq("id" -> 0, "value" -> 1)) { + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + add + } + val allAdds = commits.values.toSeq.flatten.collect { case a: AddFile => a } + val addsByPath = allAdds.map(a => a.path -> a).toMap + require(addsByPath.size == allAdds.size, "Each path must have one original Add") + require(allAdds.forall(a => a.stats.isDefined && a.partitionValues.isEmpty && + a.deletionVector.isEmpty && a.baseRowId.isEmpty && a.defaultRowCommitVersion.isEmpty)) + val removes = commits(4).collect { case r: RemoveFile => r } + val removedPaths = removes.map(_.path).toSet + require(removes.size == 2 && removedPaths == inputAdds.tail.map(_.path).toSet) + removes.foreach { r => + val original = addsByPath(r.path) + require(r.dataChange && r.deletionTimestamp.isDefined) + require(r.extendedFileMetadata.contains(true)) + require(r.partitionValues.contains(original.partitionValues)) + require(r.size.contains(original.size) && r.deletionVector.isEmpty) + require(r.baseRowId.isEmpty && r.defaultRowCommitVersion.isEmpty) + require(r.stats.forall(s => JsonUtil.mapper.readTree(s) == + JsonUtil.mapper.readTree(original.stats.get))) + } + val rewritten = commits(4).collect { case a: AddFile => a } + require(rewritten.nonEmpty) + RowComparison.assertRowsEqual(spark.createDataFrame(rewrittenRows.asJava, schema), + rewritten.map(a => spark.read.parquet(dir.resolve(a.path).toString)).reduce(_.union(_)), + "rewritten_survivors") + val activePaths = addsByPath.keySet -- removedPaths + require(activePaths == (rewritten.map(_.path).toSet + inputAdds.head.path)) + + for (version <- 1L to 4L) { + val changed = commits(version).map { + case a: AddFile if activePaths.contains(a.path) => a.copy(stats = None) + // Some writers omit Remove stats. Supply the exact, row-checked original Add stats. + case r: RemoveFile => r.copy(stats = addsByPath(r.path).stats) + case other => other + } + CommitLog.write(dir, version, changed) + require(CommitLog.read(dir, version) == changed, + s"Persisted action mutation mismatch at version $version") + } + val persisted = (0L to 4L).flatMap(v => CommitLog.read(dir, v)) + val persistedAdds = persisted.collect { case a: AddFile => a } + val persistedRemoves = persisted.collect { case r: RemoveFile => r } + require(persistedAdds.filter(a => activePaths.contains(a.path)).forall(_.stats.isEmpty)) + require(persistedAdds.filter(a => removedPaths.contains(a.path)).forall { a => + a == addsByPath(a.path) + }) + require(persistedRemoves.size == 2 && persistedRemoves.forall { r => + r.stats.isDefined && r.stats == addsByPath(r.path).stats + }) + } + + // Literal row bags, independent of Delta replay and statistics; duplicates are intentional. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, survivors), + ("hit_untouched", "id = 1", Seq(Row(1, 10))), + ("hit_rewritten", "id = 3", Seq(Row(3, 30), Row(3, 30))), + ("range", "value >= 30", Seq(Row(3, 30), Row(3, 30))), + ("null", "value IS NULL", Seq(Row(5, null), Row(4, null))), + ("miss", "id = 999", Seq.empty), + ("deleted_partial", "id = 2", Seq.empty), + ("deleted_file", "id >= 100", Seq.empty), + ("mixed", "id IN (2, 3, 100)", Seq(Row(3, 30), Row(3, 30)))) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful snapshot, got $other") + } + } + } + + test("unicode_string_stats_predicates") { + val schema = new StructType().add("id", IntegerType).add("s", StringType) + val head = "a" * 31 + // Exact source rows from WriteStatsSuite.utf8_boundary_string_min_max; no NULL rows. + // This independent read fixture avoids depending on another suite's generated output. + val rows = Seq(Row(1, head + "étail"), Row(2, head + "中tail"), Row(3, head + "😀")) + sql("CREATE TABLE tbl (id INT, s STRING) USING delta") + rows.foreach { row => + // Separate files give the three Unicode boundaries distinct min/max statistics. + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (${row.getInt(0)}, '${row.getString(1)}') AS v(id, s)""") + } + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + rows.zipWithIndex.foreach { case (row, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(Seq(row).asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"unicode_input_$index") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == 1) + require(stats.path("nullCount").path("s").asLong(-1) == 0) + val value = row.getString(1) + val minimum = stats.path("minValues").path("s").asText() + val maximum = stats.path("maxValues").path("s").asText() + require(minimum.startsWith(head) && minimum.codePointAt(31) == value.codePointAt(31)) + require(UTF8String.fromString(minimum).compareTo(UTF8String.fromString(value)) <= 0) + require(UTF8String.fromString(value).compareTo(UTF8String.fromString(maximum)) <= 0) + } + } + + // Literal bags use UTF-8 binary ordering: e-acute < CJK < supplementary emoji. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("equal_2byte", s"s = '${head}étail'", Seq(rows(0))), + ("equal_3byte", s"s = '${head}中tail'", Seq(rows(1))), + ("equal_4byte", s"s = '${head}😀'", Seq(rows(2))), + ("range_2_to_3byte", s"s >= '${head}étail' AND s < '${head}中tail'", Seq(rows(0))), + ("range_3_to_4byte", s"s >= '${head}中tail' AND s < '${head}😀'", Seq(rows(1))), + ("range_from_4byte", s"s >= '${head}😀'", Seq(rows(2))), + ("range_through_4byte", s"s > '${head}étail' AND s <= '${head}😀'", rows.tail), + ("miss_same_prefix", s"s = '${head}étail_missing'", Seq.empty), + ("miss_above_4byte", s"s > '${head}😀'", Seq.empty), + ("is_null", "s IS NULL", Seq.empty), + ("is_not_null", "s IS NOT NULL", rows)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful Unicode $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_unicode_oracle_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.partitionColumns.isEmpty) + require(result.metadata.configuration.isEmpty) + case other => fail(s"Expected successful Unicode snapshot, got $other") + } + } + } + + for (bounds <- Seq("true", "false", "absent")) { + test(s"tight_bounds_$bounds") { + val schema = new StructType().add("id", IntegerType).add("all_null", IntegerType) + .add("mixed_null", IntegerType).add("non_null", IntegerType) + val physical = Seq(Row(1, null, 10, 10), Row(2, null, null, 20), + Row(3, null, 30, 30), Row(3, null, 30, 30), Row(4, null, null, 40)) + val live = Seq(Row(2, null, null, 20), Row(3, null, 30, 30), Row(3, null, 30, 30)) + val control = physical.map(r => Row(r.getInt(0) + 100, r.get(1), r.get(2), r.get(3))) + // Existing cases cover non-DV partial counts; explicit bounds need controls here. + val batches = Seq.fill(3)(physical) ++ + (if (bounds == "absent") Seq.empty else Seq.fill(2)(control)) + val rows = Seq.fill(3)(live).flatten ++ + (if (bounds == "absent") Seq.empty else Seq.fill(2)(control).flatten) + val properties = Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "100") + sql("""CREATE TABLE tbl (id INT, all_null INT, mixed_null INT, non_null INT) + USING delta TBLPROPERTIES ('delta.enableDeletionVectors' = 'true', + 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { row => + row.toSeq.map(v => if (v == null) "NULL" else v.toString).mkString("(", ", ", ")") + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values " + + "AS v(id, all_null, mixed_null, non_null)") + } + sql("DELETE FROM tbl WHERE id = 1 OR id = 4") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val lastVersion = batches.size.toLong + 1 + val commits = (0L to lastVersion).map(v => v -> CommitLog.read(dir, v)).toMap + val metadata = commits(0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + val originals = batches.indices.map { index => + val adds = commits(index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batches(index).asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"physical_$bounds/$index") + add + } + val deleted = commits(lastVersion) + val dvAdds = deleted.collect { case a: AddFile => a } + val removes = deleted.collect { case r: RemoveFile => r } + require(dvAdds.size == 3 && removes.size == 3) + require(dvAdds.map(_.path).toSet == originals.take(3).map(_.path).toSet) + require(removes.map(_.path).toSet == dvAdds.map(_.path).toSet) + require(dvAdds.forall(a => a.deletionVector.exists(_.cardinality == 2))) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, schema), + spark.read.format("delta").load(dir.toString), s"live_before_mutation_$bounds") + + val stream = Files.list(dir.resolve("_delta_log")) + val files = try stream.iterator().asScala.toVector finally stream.close() + require(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.matches("[0-9]{20}\\.crc")).foreach(Files.delete) + for (version <- 1L to lastVersion) { + val changed = commits(version).map { + case add: AddFile if add.deletionVector.isDefined || + originals.drop(3).exists(_.path == add.path) => + val index = originals.indexWhere(_.path == add.path) + require(index >= 0 && add.partitionValues.isEmpty) + val valid = if (index < 3) live else control + val basis = if (bounds == "false") batches(index) else valid + val stats = JsonUtil.mapper.createObjectNode() + // DV numRecords is always physical, including when tight null counts are live. + stats.put("numRecords", batches(index).size) + if (bounds != "absent") stats.put("tightBounds", bounds == "true") + val minimum = stats.putObject("minValues") + val maximum = stats.putObject("maxValues") + val nulls = stats.putObject("nullCount") + schema.fieldNames.zipWithIndex.foreach { case (column, ordinal) => + val values = basis.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + if (values.nonEmpty) { + minimum.put(column, values.min) + maximum.put(column, values.max) + } + nulls.put(column, basis.count(_.isNullAt(ordinal))) + } + index match { + case 1 => stats.remove("nullCount") + case 2 => + nulls.remove("all_null") + nulls.remove("mixed_null") + case 3 => stats.remove("numRecords") + case 4 => + stats.remove("numRecords") + stats.remove("nullCount") + case _ => + } + require(stats.has("tightBounds") == (bounds != "absent")) + require(stats.has("numRecords") == (index < 3)) + if (index < 3) require(stats.path("numRecords").asInt() == 5) + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + CommitLog.write(dir, version, changed) + require(CommitLog.read(dir, version) == changed, + s"Persisted bounds mutation mismatch at version $version") + } + } + + // SQL three-valued predicates applied to literal row bags, independent of file stats. + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("all_null", "all_null IS NULL", rows), + ("all_null_not_null", "all_null IS NOT NULL", Seq.empty), + ("all_null_equal", "all_null = 10", Seq.empty), + ("mixed_null", "mixed_null IS NULL", rows.filter(_.isNullAt(2))), + ("mixed_not_null", "mixed_null IS NOT NULL", rows.filterNot(_.isNullAt(2))), + ("mixed_equal", "mixed_null = 30", rows.filter(r => !r.isNullAt(2) && + r.getInt(2) == 30)), + ("non_null_is_null", "non_null IS NULL", Seq.empty), + ("non_null_not_null", "non_null IS NOT NULL", rows), + ("surviving_duplicates", "id = 3", Seq.fill(3)(live.tail).flatten), + ("deleted_lower_bound", "id = 1", Seq.empty), + ("deleted_upper_bound", "id = 4", Seq.empty), + ("deleted_mixed_value", "id < 100 AND mixed_null = 10", Seq.empty), + ("surviving_range", "non_null >= 20 AND non_null <= 30", + rows.filter(r => r.getInt(3) >= 20 && r.getInt(3) <= 30)), + ("outside_range", "non_null > 40", Seq.empty)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $bounds/$name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$bounds/$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful $bounds snapshot, got $other") + } + } + } + } + + test("indexed_prefix_beyond_32") { + val columns = (0 until 36).map(i => f"c$i%02d") + val schema = columns.foldLeft(new StructType())((s, c) => s.add(c, IntegerType)) + // The first 32 fields are a fixed prefix; c32/c33 are selected, c34/c35 are not. + val prefix = 0 until 32 + val duplicate = Row.fromSeq(prefix ++ Seq(320, 330, 340, 350)) + val mixed = Row.fromSeq(prefix ++ Seq(321, 331, 341, 351)) + val nullLate = Row.fromSeq(prefix ++ Seq(null, null, null, null)) + val batches = Seq(Seq(duplicate, duplicate), Seq(mixed, nullLate)) + val rows = batches.flatten + val properties = Map("delta.dataSkippingNumIndexedCols" -> "34", + "delta.enableDeletionVectors" -> "false", "delta.checkpointInterval" -> "100") + val ddl = columns.map(c => s"$c INT").mkString(", ") + sql(s"""CREATE TABLE tbl ($ddl) USING delta TBLPROPERTIES ( + 'delta.dataSkippingNumIndexedCols' = '34', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '100')""") + batches.foreach { batch => + val values = batch.map { row => + row.toSeq.map(v => if (v == null) "NULL" else v.toString).mkString("(", ", ", ")") + }.mkString(", ") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values " + + s"AS v(${columns.mkString(", ")})") + } + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val metadata = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + require(DataType.fromJson(metadata.schemaString) == schema) + require(metadata.configuration == properties && metadata.partitionColumns.isEmpty) + batches.zipWithIndex.foreach { case (batch, index) => + val adds = CommitLog.read(dir, index.toLong + 1).collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"wide_input_$index") + // Verify the writer's physical AddFile stats, without assuming a reader's stats schema. + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + require(stats.path(field).fieldNames().asScala.toSet == columns.take(34).toSet) + } + columns.take(34).zipWithIndex.foreach { case (column, ordinal) => + val values = batch.filterNot(_.isNullAt(ordinal)).map(_.getInt(ordinal)) + require(stats.path("nullCount").path(column).asLong(-1) == + batch.count(_.isNullAt(ordinal))) + require(stats.path("minValues").path(column).asInt(Int.MinValue) == values.min) + require(stats.path("maxValues").path(column).asInt(Int.MinValue) == values.max) + } + } + } + + val queries = Seq[(String, String, Seq[Row])]( + ("all", null, rows), + ("selected_c32_equal", "c32 = 320", Seq(duplicate, duplicate)), + ("selected_c33_equal", "c33 = 330", Seq(duplicate, duplicate)), + ("selected_c33_range", "c33 >= 330 AND c33 < 331", Seq(duplicate, duplicate)), + ("selected_c33_null", "c33 IS NULL", Seq(nullLate)), + ("selected_c33_miss", "c33 = 999", Seq.empty), + ("unselected_c34_equal", "c34 = 340", Seq(duplicate, duplicate)), + ("unselected_c34_range", "c34 > 340", Seq(mixed)), + ("unselected_c34_null", "c34 IS NULL", Seq(nullLate)), + ("unselected_c34_miss", "c34 = 999", Seq.empty)) + queries.foreach { case (name, predicate, expectedRows) => + readSpec(t, predicate = predicate, name = Some(s"read_$name")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful wide $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_wide_$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == properties) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful wide snapshot, got $other") + } + } + } + + test("alter_selected_stats_columns") { + val schema = new StructType().add("event_ts", TimestampType) + .add("customer", new StructType().add("country", StringType).add("segment", StringType)) + .add("amount", DecimalType(10, 2)).add("note", StringType) + val oldDuplicate = Row(Timestamp.from(Instant.parse("2024-01-01T00:00:00Z")), + Row("US", "retail"), BigDecimal("10.50").bigDecimal, "alpha") + val oldOther = Row(Timestamp.from(Instant.parse("2024-01-02T00:00:00Z")), + Row("CA", "wholesale"), BigDecimal("20.00").bigDecimal, "beta") + val newDuplicate = Row(Timestamp.from(Instant.parse("2024-01-03T00:00:00Z")), + Row("FR", "retail"), BigDecimal("30.25").bigDecimal, "gamma") + val newOther = Row(Timestamp.from(Instant.parse("2024-01-04T00:00:00Z")), + Row("US", "business"), BigDecimal("-5.00").bigDecimal, "alpha") + val nullRow = Row(null, Row(null, null), null, null) + val before = Seq(oldDuplicate, oldDuplicate, oldOther) + val after = Seq(newDuplicate, newDuplicate, newOther, nullRow) + val rows = before ++ after + val property = "delta.dataSkippingStatsColumns" + val properties = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "100", property -> "note,customer.segment") + val selected = "event_ts,customer.country,amount" + sql("""CREATE TABLE tbl (event_ts TIMESTAMP, customer STRUCT, + amount DECIMAL(10, 2), note STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '100', + 'delta.dataSkippingStatsColumns' = 'note,customer.segment')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (TIMESTAMP '2024-01-01T00:00:00Z', named_struct('country','US','segment','retail'), + 10.50, 'alpha'), + (TIMESTAMP '2024-01-01T00:00:00Z', named_struct('country','US','segment','retail'), + 10.50, 'alpha'), + (TIMESTAMP '2024-01-02T00:00:00Z', named_struct('country','CA','segment','wholesale'), + 20.00, 'beta') AS v(event_ts, customer, amount, note)""") + sql(s"ALTER TABLE tbl SET TBLPROPERTIES ('$property' = '$selected')") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (TIMESTAMP '2024-01-03T00:00:00Z', named_struct('country','FR','segment','retail'), + 30.25, 'gamma'), + (TIMESTAMP '2024-01-03T00:00:00Z', named_struct('country','FR','segment','retail'), + 30.25, 'gamma'), + (TIMESTAMP '2024-01-04T00:00:00Z', named_struct('country','US','segment','business'), + -5.00, 'alpha'), + (NULL, named_struct('country',NULL,'segment',NULL), NULL, NULL) + AS v(event_ts, customer, amount, note)""") + val t = registerTable("tbl") + var copiedTable: Path = null + mutateTable(t) { dir => + copiedTable = dir + val initial = CommitLog.read(dir, 0).collect { case m: Metadata => m }.head + val changed = CommitLog.read(dir, 2) + val metadata = changed.collect { case m: Metadata => m }.head + require(initial.configuration == properties) + require(metadata.configuration == properties.updated(property, selected)) + require(initial.id == metadata.id && initial.schemaString == metadata.schemaString) + require(DataType.fromJson(metadata.schemaString) == schema) + require(initial.partitionColumns.isEmpty && metadata.partitionColumns.isEmpty) + require(!changed.exists(a => a.isInstanceOf[AddFile] || a.isInstanceOf[RemoveFile]), + "ALTER must change metadata without rewriting files or backfilling statistics") + for ((version, batch) <- Seq(1L -> before, 3L -> after)) { + val actions = CommitLog.read(dir, version) + require(!actions.exists(_.isInstanceOf[RemoveFile])) + val adds = actions.collect { case a: AddFile => a } + require(adds.size == 1 && adds.head.deletionVector.isEmpty) + val add = adds.head + RowComparison.assertRowsEqual(spark.createDataFrame(batch.asJava, schema), + spark.read.parquet(dir.resolve(add.path).toString), s"selected_stats_input_v$version") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.size) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + val expectedFields = if (version == 1) Set("customer", "note") + else Set("event_ts", "customer", "amount") + require(stats.path(field).fieldNames().asScala.toSet == expectedFields) + require(stats.path(field).path("customer").fieldNames().asScala.toSet == + (if (version == 1) Set("segment") else Set("country"))) + } + } + } + val queries = Seq[(String, java.lang.Long, String, Seq[Row])]( + ("before_all", 1L, null, before), + ("before_timestamp", 1L, "event_ts = TIMESTAMP '2024-01-01T00:00:00Z'", + Seq(oldDuplicate, oldDuplicate)), + ("all", null, null, rows), + ("selected_timestamp_equal", null, "event_ts = TIMESTAMP '2024-01-01T00:00:00Z'", + Seq(oldDuplicate, oldDuplicate)), + ("selected_timestamp_range", null, "event_ts >= TIMESTAMP '2024-01-03T00:00:00Z'", + Seq(newDuplicate, newDuplicate, newOther)), + ("selected_country_equal", null, "customer.country = 'US'", + Seq(oldDuplicate, oldDuplicate, newOther)), + ("selected_country_null", null, "customer.country IS NULL", Seq(nullRow)), + ("selected_amount_equal", null, "amount = 30.25", Seq(newDuplicate, newDuplicate)), + ("selected_amount_range", null, "amount >= 10.50 AND amount <= 20.00", before), + ("selected_amount_null", null, "amount IS NULL", Seq(nullRow)), + ("selected_country_miss", null, "customer.country = 'ZZ'", Seq.empty), + ("excluded_note_equal", null, "note = 'alpha'", Seq(oldDuplicate, oldDuplicate, newOther)), + ("excluded_segment_equal", null, "customer.segment = 'retail'", + Seq(oldDuplicate, oldDuplicate, newDuplicate, newDuplicate)), + ("excluded_note_null", null, "note IS NULL", Seq(nullRow)), + ("selected_combined", null, + "event_ts >= TIMESTAMP '2024-01-03T00:00:00Z' AND " + + "customer.country = 'FR' AND amount = 30.25", Seq(newDuplicate, newDuplicate))) + queries.foreach { case (name, version, predicate, expectedRows) => + readSpec(t, version = version, predicate = predicate, name = Some(s"read_$name")) + .assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful selected-stats $name read, got $other") + } + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual(spark.createDataFrame(expectedRows.asJava, schema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_selected_stats_$name") + } + } + for (version <- Seq[java.lang.Long](1L, null)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + // This asserts the table schema and properties, not Kernel's internal stats schema. + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.configuration == + (if (version == null) properties.updated(property, selected) else properties)) + require(result.metadata.partitionColumns.isEmpty) + case other => fail(s"Expected successful selected-stats snapshot, got $other") + } + } + } + } + + // One checkpoint matrix serves both missing evolved-column stats and nullable evolution. + // Commit/time-travel coverage stays in DataSkippingSuite and SchemaEvolutionSuite. + for (parsed <- Seq(false, true)) { + val variant = if (parsed) "parsed" else "json_only" + test(s"nullable_evolution_checkpoint_$variant") { + val schema = new StructType().add("id", IntegerType).add("part", IntegerType) + .add("value", IntegerType) + val oldRows = Seq(Row(1, 0, null), Row(2, 1, null), Row(2, 1, null)) + val newRows = Seq(Row(3, 0, 30), Row(3, 0, 30), Row(4, 1, null), Row(5, 1, 50)) + val rows = oldRows ++ newRows + sql(s"""CREATE TABLE tbl (id INT, part INT) USING delta PARTITIONED BY (part) + TBLPROPERTIES ('delta.checkpointInterval' = '100', + 'delta.enableDeletionVectors' = 'false', + 'delta.checkpoint.writeStatsAsJson' = '${!parsed}', + 'delta.checkpoint.writeStatsAsStruct' = '$parsed')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, 0), (2, 1), (2, 1) AS v(id, part)""") + sql("ALTER TABLE tbl ADD COLUMN value INT") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (3, 0, 30), (3, 0, 30), (4, 1, NULL), (5, 1, 50) AS v(id, part, value)""") + forceCheckpoint("tbl") + val t = registerTable("tbl") + var copiedTable: Path = null + + mutateTable(t) { dir => + copiedTable = dir + val commits = (0L to 3L).map(v => v -> CommitLog.read(dir, v)).toMap + require(!commits.values.flatten.exists(_.isInstanceOf[RemoveFile])) + val evolved = commits(2).collect { case m: Metadata => m }.head + require(DataType.fromJson(evolved.schemaString) == schema) + require(evolved.partitionColumns == Seq("part")) + val oldAdds = commits(1).collect { case a: AddFile => a } + val newAdds = commits(3).collect { case a: AddFile => a } + val adds = oldAdds ++ newAdds + require(oldAdds.size == 2 && newAdds.size == 2) + require(adds.map(_.path).distinct.size == 4, "All four files must remain active") + for ((batch, files) <- Seq(oldRows -> oldAdds, newRows -> newAdds)) { + require(files.map(_.partitionValues).toSet == + Set(Map("part" -> "0"), Map("part" -> "1"))) + files.foreach { add => + val part = add.partitionValues("part").toInt + val raw = spark.read.parquet(dir.resolve(add.path).toString) + require(raw.columns.contains("value") == (batch == newRows)) + val withValue = if (batch == oldRows) raw.withColumn("value", lit(null) + .cast(IntegerType)) else raw + RowComparison.assertRowsEqual( + spark.createDataFrame(batch.filter(_.getInt(1) == part).asJava, schema), + withValue.withColumn("part", lit(part)).select("id", "part", "value"), + s"input_$variant/${add.path}") + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.path("numRecords").asLong(-1) == batch.count(_.getInt(1) == part)) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + require(stats.path(field).has("value") == (batch == newRows), + s"$field must omit the evolved column only for old files") + } + } + } + + val logDir = dir.resolve("_delta_log") + val checkpoint = logDir.resolve("00000000000000000003.checkpoint.parquet") + require(Files.isRegularFile(checkpoint), "Expected the final classic checkpoint") + val cp = spark.read.parquet(checkpoint.toString) + val addSchema = cp.schema("add").dataType.asInstanceOf[StructType] + require(addSchema.fieldNames.contains("stats") == !parsed) + require(addSchema.fieldNames.contains("stats_parsed") == parsed) + require(addSchema.fieldNames.contains("partitionValues_parsed") == parsed) + val cpAdds = cp.filter("add IS NOT NULL").select("add.*").collect() + require(cpAdds.length == adds.size, "Checkpoint must contain every active AddFile") + require(cpAdds.map(_.getAs[String]("path")).toSet == adds.map(_.path).toSet) + require(cp.filter("remove IS NOT NULL").count() == 0) + cpAdds.foreach { action => + val original = adds.find(_.path == action.getAs[String]("path")).get + val stats = JsonUtil.mapper.readTree(original.stats.get) + if (parsed) { + val partition = action.getAs[Row]("partitionValues_parsed") + require(partition.schema("part").dataType == IntegerType) + require(partition.getAs[Int]("part") == original.partitionValues("part").toInt) + val actual = action.getAs[Row]("stats_parsed") + require(actual.getAs[Long]("numRecords") == stats.path("numRecords").asLong()) + for (field <- Seq("minValues", "maxValues", "nullCount")) { + val values = actual.getAs[Row](field) + require(values.getAs[Any]("id").toString == stats.path(field).path("id").asText()) + if (oldAdds.exists(_.path == original.path)) { + require(values.isNullAt(values.fieldIndex("value")), + s"Old file $field must have no parsed evolved-column statistic") + } else { + require(values.getAs[Any]("value").toString == + stats.path(field).path("value").asText()) + } + } + } else { + require(JsonUtil.mapper.readTree(action.getAs[String]("stats")) == stats) + } + } + val lastCheckpoint = JsonUtil.mapper.readTree( + Files.readAllBytes(logDir.resolve("_last_checkpoint"))) + require(lastCheckpoint.path("version").asLong(-1) == 3) + require(lastCheckpoint.path("size").asLong(-1) == cp.count()) + + // Keep the checkpoint-version commit for timestamp resolution, but remove its entire + // preceding history and every version CRC. Reconstruction now requires this checkpoint. + for (v <- 0L until 3L) Files.delete(CommitLog.commitFile(dir, v)) + val stream = Files.list(logDir) + val logFiles = try stream.iterator().asScala.toVector finally stream.close() + logFiles.filter(_.getFileName.toString.matches("[0-9]{20}\\.crc")) + .foreach(Files.delete) + require((0L until 3L).forall(v => !Files.exists(CommitLog.commitFile(dir, v)))) + require(Files.exists(CommitLog.commitFile(dir, 3))) + require(Files.exists(checkpoint) && adds.forall(a => Files.exists(dir.resolve(a.path)))) + } + + val queries = Seq[(String, String, Option[Seq[String]], Seq[Row])]( + ("all", null, None, rows), + ("projection", null, Some(Seq("value")), + Seq(Row(null), Row(null), Row(null), Row(30), Row(30), Row(null), Row(50))), + ("equal", "value = 30", None, Seq(Row(3, 0, 30), Row(3, 0, 30))), + ("range", "value >= 30 AND value < 50", None, Seq(Row(3, 0, 30), Row(3, 0, 30))), + ("is_null", "value IS NULL", None, oldRows :+ Row(4, 1, null)), + ("is_not_null", "value IS NOT NULL", None, + Seq(Row(3, 0, 30), Row(3, 0, 30), Row(5, 1, 50))), + ("partition_null", "part = 1 AND value IS NULL", None, + Seq(Row(2, 1, null), Row(2, 1, null), Row(4, 1, null)))) + queries.foreach { case (name, predicate, columns, expectedRows) => + readSpec(t, predicate = predicate, columns = columns, name = Some(s"read_$name")) + .assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == expectedRows.size) + case other => fail(s"Expected successful $variant/$name read, got $other") + } + val expectedSchema = if (columns.isDefined) { + new StructType().add("value", IntegerType) + } else schema + val outputName = copiedTable.getParent.getFileName.toString + val expectedPath = outputDir.resolve(outputName).resolve("expected") + .resolve(s"${outputName}_read_$name").resolve("expected_data") + RowComparison.assertRowsEqual( + spark.createDataFrame(expectedRows.asJava, expectedSchema), + spark.read.parquet(expectedPath.toString), s"independent_oracle_$variant/$name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == schema) + require(result.metadata.partitionColumns == Seq("part")) + require(result.metadata.configuration("delta.checkpoint.writeStatsAsStruct") == + parsed.toString) + require(result.metadata.configuration("delta.checkpoint.writeStatsAsJson") == + (!parsed).toString) + case other => fail(s"Expected successful $variant snapshot, got $other") + } + } + } + } + + // Timestamp statistics: encodings are checked on disk, independently of the table + // reader. + private val microFirst = "2024-01-01T00:00:00.123456Z" + private val microLast = "2024-01-01T00:00:00.123789Z" + private val nanoLower = "2024-01-01T00:00:00.123455111Z" + private val nanoUpper = "2024-01-01T00:00:00.123789999Z" + private val microSchema = new StructType().add("id", IntegerType).add("ts", TimestampType) + + // Literal instants, not values read back from the table, are the row oracle. + private def microRows: Seq[Row] = Seq( + Row(1, Timestamp.from(Instant.parse(microFirst))), + Row(2, Timestamp.from(Instant.parse(microLast))), + Row(2, Timestamp.from(Instant.parse(microLast)))) + + private def createMicroTable(): TableHandle = { + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP) USING delta TBLPROPERTIES ( + 'delta.minReaderVersion' = '1', 'delta.minWriterVersion' = '2', + 'delta.checkpointPolicy' = 'classic', + 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (1, TIMESTAMP '2024-01-01 00:00:00.123456'), + (2, TIMESTAMP '2024-01-01 00:00:00.123789'), + (2, TIMESTAMP '2024-01-01 00:00:00.123789') AS rows(id, ts)""") + val t = registerTable("tbl") + val adds = CommitLog.read(t.sourcePath, 1).collect { case a: AddFile => a } + require(adds.size == 1, "The timestamp bounds must describe one file with all three rows") + t + } + + private def checkMicroSpecs(t: TableHandle, checkJsonStats: Boolean = false): Unit = { + val workloadName = WorkloadContext.current.workloadName + val tableOutputDir = outputDir.resolve(workloadName) + val queries = Seq( + ("all", null, microRows), + ("lower_edge", "ts <= TIMESTAMP '2024-01-01 00:00:00.123456'", microRows.take(1)), + ("upper_edge", "ts >= TIMESTAMP '2024-01-01 00:00:00.123789'", microRows.drop(1)), + ("outside", "ts < TIMESTAMP '2024-01-01 00:00:00.123456' OR " + + "ts > TIMESTAMP '2024-01-01 00:00:00.123789'", Seq.empty[Row])) + + for (version <- Seq[java.lang.Long](null, 1L)) { + val suffix = if (version == null) "latest" else "v1" + for ((label, predicate, rows) <- queries) { + val name = s"${label}_$suffix" + readSpec(t, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, microSchema) + val expectedDir = tableOutputDir.resolve("expected") + .resolve(s"${workloadName}_$name") + val captured = expectedDir.resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + if (checkJsonStats && label == "all") { + val scanned = spark.read.parquet( + expectedDir.resolve("expected_metadata").toString).collect() + require(scanned.length == 1) + val scanStats = JsonUtil.mapper.readTree(scanned.head.getString(0)) + .at("/add/stats").asText() + val tableDir = tableOutputDir.resolve("delta") + val rawStats = CommitLog.read(tableDir, 1) + .collectFirst { case a: AddFile => a.stats.get }.get + require(scanStats == rawStats, "Scanned stats must retain the injected precision") + println(s"Verified $name scanned AddFile stats match nanosecond JSON") + } + } + } + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == microSchema) + case other => fail(s"$suffix snapshot must succeed: $other") + } + } + } + } + + test("nanosecond_stats_json") { + val t = createMicroTable() + // The bounds enclose the microsecond rows. All of digits 7-9 are nonzero. + val lower = nanoLower + val upper = nanoUpper + require(Instant.parse(lower).isBefore(Instant.parse(microFirst))) + require(Instant.parse(upper).isAfter(Instant.parse(microLast))) + modifyCommitActions(t, version = 1) { actions => + actions.map { + case a: AddFile => + val stats = JsonUtil.mapper.readTree(a.stats.get) + stats.get("minValues").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", lower) + stats.get("maxValues").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", upper) + a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + mutateTable(t) { tableDir => + val adds = CommitLog.read(tableDir, 1).collect { case a: AddFile => a } + require(adds.size == 1) + val raw = JsonUtil.mapper.readTree(adds.head.stats.get) + require(raw.at("/minValues/ts").asText() == lower) + require(raw.at("/maxValues/ts").asText() == upper) + require(raw.get("numRecords").asLong() == 3) + // A copied checksum can contain the old AddFile stats and bypass JSON parsing. + val files = Files.list(tableDir.resolve("_delta_log")) + try files.iterator().asScala.filter(_.toString.endsWith(".crc")) + .foreach(Files.delete(_)) finally files.close() + println(s"Verified JSON stats: min=$lower max=$upper records=3") + } + checkMicroSpecs(t, checkJsonStats = true) + } + + for ((caseName, encoding, lower, upper) <- Seq( + ("millisecond_stats_checkpoint", "TIMESTAMP_MILLIS", + "2024-01-01T00:00:00.123Z", "2024-01-01T00:00:00.124Z"), + ("microsecond_stats_checkpoint", "TIMESTAMP_MICROS", microFirst, microLast), + ("int96_stats_checkpoint", "INT96", microFirst, microLast))) { + test(caseName) { + val t = createMicroTable() + require(!Instant.parse(lower).isAfter(Instant.parse(microFirst))) + require(!Instant.parse(upper).isBefore(Instant.parse(microLast))) + forceCheckpoint("tbl") + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val checkpoint = logDir.resolve("00000000000000000001.checkpoint.parquet") + require(Files.isRegularFile(checkpoint), "Expected one classic checkpoint") + val rewriteDir = logDir.resolve("timestamp-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, encoding) + val df = spark.read.parquet(checkpoint.toString) + val stats = col("add.stats_parsed") + .withField("minValues", col("add.stats_parsed.minValues") + .withField("ts", lit(Timestamp.from(Instant.parse(lower))))) + .withField("maxValues", col("add.stats_parsed.maxValues") + .withField("ts", lit(Timestamp.from(Instant.parse(upper))))) + // Disable the JSON fallback: these reads must consume parsed checkpoint stats. + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewriteDir.toString) + val files = Files.list(rewriteDir) + val part = try { + val parts = files.iterator().asScala.filter(_.toString.endsWith(".parquet")).toSeq + require(parts.size == 1) + parts.head + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewriteDir)) FileUtils.deleteDirectory(rewriteDir.toFile) + } + checkMicroCheckpoint(checkpoint, encoding, lower, upper) + if (encoding == "INT96") { + // Spark's writer only emits microsecond-aligned INT96. Preserve every checkpoint + // field with the existing Parquet Group API, replacing just the two stats cells. + rewriteInt96Bounds(checkpoint) + checkMicroCheckpoint(checkpoint, encoding, nanoLower, nanoUpper) + } + // Retain v1 JSON as a discovery/metadata tail alongside the checkpoint. Removing v0 + // prevents full JSON replay, so v1 and latest require checkpoint bootstrap. + val files = Files.list(logDir) + try files.iterator().asScala.filter { p => + p.getFileName.toString == "00000000000000000000.json" || + p.toString.endsWith(".crc") + }.foreach(Files.delete(_)) finally files.close() + } + checkMicroSpecs(t) + } + } + + private def rewriteInt96Bounds(checkpoint: Path): Unit = { + import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ + + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + val metadata = try footer.getFooter.getFileMetaData finally footer.close() + val parquetSchema = metadata.getSchema + val factory = new SimpleGroupFactory(parquetSchema) + // 2024-01-01 is Julian day 2460311. These nanoseconds after midnight enclose the rows. + val replacements = Map( + "add.stats_parsed.minValues.ts" -> new NanoTime(2460311, 123455111L).toBinary, + "add.stats_parsed.maxValues.ts" -> new NanoTime(2460311, 123789999L).toBinary) + + def copyGroup(source: Group, target: Group, prefix: String): Unit = { + for (i <- 0 until source.getType.getFieldCount) { + val field = source.getType.getType(i) + val fieldPath = if (prefix.isEmpty) field.getName else s"$prefix.${field.getName}" + for (j <- 0 until source.getFieldRepetitionCount(i)) { + if (!field.isPrimitive) { + copyGroup(source.getGroup(i, j), target.addGroup(i), fieldPath) + } else field.asPrimitiveType().getPrimitiveTypeName match { + case BOOLEAN => target.add(i, source.getBoolean(i, j)) + case INT32 => target.add(i, source.getInteger(i, j)) + case INT64 => target.add(i, source.getLong(i, j)) + case FLOAT => target.add(i, source.getFloat(i, j)) + case DOUBLE => target.add(i, source.getDouble(i, j)) + case BINARY | FIXED_LEN_BYTE_ARRAY => target.add(i, source.getBinary(i, j)) + case INT96 => target.add(i, replacements.getOrElse(fieldPath, source.getInt96(i, j))) + } + } + } + } + + val temporary = checkpoint.resolveSibling("timestamp-int96-rewrite.parquet") + try { + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + try { + val writer = ExampleParquetWriter.builder(new HadoopPath(temporary.toUri)) + .withConf(conf).withType(parquetSchema) + .withExtraMetaData(metadata.getKeyValueMetaData).build() + try { + var source = reader.read() + while (source != null) { + val target = factory.newGroup() + copyGroup(source, target, "") + writer.write(target) + source = reader.read() + } + } finally writer.close() + } finally reader.close() + Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally Files.deleteIfExists(temporary) + } + + private def checkMicroCheckpoint( + checkpoint: Path, encoding: String, lower: String, upper: String): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val parquetSchema = footer.getFooter.getFileMetaData.getSchema + for (bound <- Seq("minValues", "maxValues")) { + val physical = parquetSchema.getType("add", "stats_parsed", bound, "ts") + .asPrimitiveType() + if (encoding == "INT96") { + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT96) + require(physical.getLogicalTypeAnnotation == null) + } else { + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = physical.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + val unit = if (encoding == "TIMESTAMP_MILLIS") { + LogicalTypeAnnotation.TimeUnit.MILLIS + } else LogicalTypeAnnotation.TimeUnit.MICROS + require(annotation.getUnit == unit) + require(annotation.isAdjustedToUTC) + } + println(s"Verified checkpoint add.stats_parsed.$bound.ts: $physical") + } + } finally footer.close() + + // Read the physical cells without Spark's timestamp conversion or Delta's stats parser. + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var adds = 0 + try { + var row = reader.read() + while (row != null) { + if (row.getFieldRepetitionCount("add") > 0) { + adds += 1 + val add = row.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + require(stats.getLong("numRecords", 0) == 3) + for ((bound, value) <- Seq("minValues" -> lower, "maxValues" -> upper)) { + val cell = stats.getGroup(bound, 0) + val instant = Instant.parse(value) + if (encoding == "INT96") { + val bytes = ByteBuffer.wrap(cell.getInt96("ts", 0).getBytes) + .order(ByteOrder.LITTLE_ENDIAN) + val nanosOfDay = bytes.getLong() + val julianDay = bytes.getInt() + require(julianDay == 2440588 + Math.floorDiv(instant.getEpochSecond, 86400L)) + require(nanosOfDay == Math.floorMod(instant.getEpochSecond, 86400L) * + 1000000000L + instant.getNano) + // Check the aligned intermediate separately from the true nanos replacement. + require(nanosOfDay % 1000 == instant.getNano % 1000) + require(nanosOfDay % 1000000 != 0) + println(s"Verified raw INT96 $bound: day=$julianDay nanosOfDay=$nanosOfDay") + } else { + val units = if (encoding == "TIMESTAMP_MILLIS") 1000L else 1000000L + val expected = instant.getEpochSecond * units + + instant.getNano / (1000000000L / units) + require(cell.getLong("ts", 0) == expected) + println(s"Verified raw $encoding $bound: $expected") + } + } + } + row = reader.read() + } + } finally reader.close() + require(adds == 1) + } + + // Each data boundary is crossed with every temporal partition boundary. + // SQL fixture literals and the typed row oracle are deliberately declared separately. + private val epochValues = Seq( + (1, "1969-12-31T23:59:59.998999", "1969-12-31", -1001L), + (2, "1969-12-31T23:59:59.999000", "1969-12-31", -1000L), + (3, "1969-12-31T23:59:59.999001", "1969-12-31", -999L), + (4, "1969-12-31T23:59:59.999999", "1969-12-31", -1L), + (5, "1970-01-01T00:00:00.000000", "1970-01-01", 0L), + (6, "1970-01-01T00:00:00.000001", "1970-01-01", 1L), + (7, "1970-01-01T00:00:00.001000", "1970-01-02", 1000L)) + + private val epochSchema = new StructType().add("id", IntegerType) + .add("ts", TimestampType).add("ntz", org.apache.spark.sql.types.TimestampNTZType) + .add("d", org.apache.spark.sql.types.DateType).add("p_ts", TimestampType) + .add("p_ntz", org.apache.spark.sql.types.TimestampNTZType) + .add("p_d", org.apache.spark.sql.types.DateType) + + private def epochRows: Seq[Row] = { + val data = epochValues ++ epochValues.filter(_._1 == 4) + for ((id, wall, day, _) <- data; (_, pWall, pDay, _) <- epochValues) yield { + Row(id, Timestamp.from(Instant.parse(wall + "Z")), java.time.LocalDateTime.parse(wall), + java.sql.Date.valueOf(day), Timestamp.from(Instant.parse(pWall + "Z")), + java.time.LocalDateTime.parse(pWall), java.sql.Date.valueOf(pDay)) + } + } + + private def epochPartitionStrings: Set[Map[String, String]] = epochValues.map { + case (_, wall, day, _) => + // NTZ uses wall-clock strings; TIMESTAMP uses UTC ISO with six fractional digits. + val partition = wall.replace('T', ' ').replaceAll("0+$", "").stripSuffix(".") + Map("p_ts" -> (wall + "Z"), "p_ntz" -> partition, "p_d" -> day) + }.toSet + + private def createEpochTable(): TableHandle = { + require(spark.conf.get("spark.sql.session.timeZone") == "UTC") + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ, d DATE, + p_ts TIMESTAMP, p_ntz TIMESTAMP_NTZ, p_d DATE) USING delta + PARTITIONED BY (p_ts, p_ntz, p_d) TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', + 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + val fixtures = Seq( + (1, "1969-12-31 23:59:59.998999", "1969-12-31"), + (2, "1969-12-31 23:59:59.999", "1969-12-31"), + (3, "1969-12-31 23:59:59.999001", "1969-12-31"), + (4, "1969-12-31 23:59:59.999999", "1969-12-31"), + (5, "1970-01-01 00:00:00", "1970-01-01"), + (6, "1970-01-01 00:00:00.000001", "1970-01-01"), + (7, "1970-01-01 00:00:00.001", "1970-01-02")) + val partitions = fixtures.map { case (_, ts, day) => + s"(TIMESTAMP '$ts', TIMESTAMP_NTZ '$ts', DATE '$day')" + }.mkString(",") + for ((id, ts, day) <- fixtures) { + val data = s"($id, TIMESTAMP '$ts', TIMESTAMP_NTZ '$ts', DATE '$day')" + val rows = if (id == 4) s"$data,$data" else data + sql(s"""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM + (VALUES $rows) AS data(id, ts, ntz, d) CROSS JOIN + (VALUES $partitions) AS parts(p_ts, p_ntz, p_d)""") + } + val t = registerTable("tbl") + for (v <- 1L to 7L) { + val adds = CommitLog.read(t.sourcePath, v).collect { case a: AddFile => a } + require(adds.size == 7, "Each boundary must have one file per partition") + require(adds.map(_.partitionValues).toSet == epochPartitionStrings) + } + t + } + + for (parsed <- Seq(false, true)) { + val name = if (parsed) "negative_epoch_parsed_stats_partitions" + else "negative_epoch_json_stats_partitions" + test(name) { + val t = createEpochTable() + if (parsed) forceCheckpoint("tbl") + if (!parsed) { + for ((id, wall, day, _) <- epochValues) { + modifyCommitActions(t, version = id.toLong) { actions => + actions.map { + case a: AddFile => + val stats = JsonUtil.mapper.readTree(a.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.get(bound) + .asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + values.put("ts", wall + "Z") + values.put("ntz", wall) + values.put("d", day) + } + a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + } + } + mutateTable(t) { dir => + if (parsed) { + val checkpoint = dir.resolve("_delta_log/00000000000000000007.checkpoint.parquet") + val rewrite = dir.resolve("_delta_log/epoch-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + var ts = col(s"add.stats_parsed.$bound.ts") + var ntz = col(s"add.stats_parsed.$bound.ntz") + for ((id, wall, _, _) <- epochValues) { + val matches = col("add.stats_parsed.minValues.id") === lit(id) + ts = org.apache.spark.sql.functions.when(matches, + lit(Timestamp.from(Instant.parse(wall + "Z")))).otherwise(ts) + ntz = org.apache.spark.sql.functions.when(matches, + lit(java.time.LocalDateTime.parse(wall))).otherwise(ntz) + } + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("ts", ts).withField("ntz", ntz)) + } + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewrite.toString) + val files = Files.list(rewrite) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten checkpoint: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewrite)) FileUtils.deleteDirectory(rewrite.toFile) + } + checkEpochCheckpoint(checkpoint) + } else { + for ((id, wall, day, _) <- epochValues) { + val adds = CommitLog.read(dir, id.toLong).collect { case a: AddFile => a } + require(adds.size == 7) + require(adds.map(_.partitionValues).toSet == epochPartitionStrings) + for (a <- adds) { + val stats = JsonUtil.mapper.readTree(a.stats.get) + require(stats.get("numRecords").asLong() == (if (id == 4) 2 else 1)) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/ts").asText() == wall + "Z") + require(stats.at(s"/$bound/ntz").asText() == wall) + require(stats.at(s"/$bound/d").asText() == day) + require(stats.at(s"/$bound/id").asInt() == id) + } + } + } + println("Verified exact JSON stats and partition strings for all 49 boundary files") + } + val files = Files.list(dir.resolve("_delta_log")) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (parsed && file.endsWith(".json") && + file != "00000000000000000007.json") + }.foreach(Files.delete(_)) finally files.close() + // v7 JSON is retained only for same-version lookup. There is no replay tail after v7. + // v0-v6 are absent, so the checkpoint is necessary for both latest and version 7. + } + checkEpochSpecs(t, parsed) + } + } + + private def checkEpochCheckpoint(checkpoint: Path): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + for ((prefix, fields) <- Seq( + "stats_parsed.minValues" -> Seq("ts", "ntz", "d"), + "stats_parsed.maxValues" -> Seq("ts", "ntz", "d"), + "partitionValues_parsed" -> Seq("p_ts", "p_ntz", "p_d")); field <- fields) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + if (field.endsWith("d")) { + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(cell.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } else { + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == field.endsWith("ts")) + } + println(s"Verified near-epoch checkpoint $prefix.$field: $cell") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var pairs = Vector.empty[(Int, Map[String, String])] + try { + var row = reader.read() + while (row != null) { + if (row.getFieldRepetitionCount("add") > 0) { + val add = row.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + val (_, _, day, micros) = epochValues.find(_._1 == id).get + require(stats.getLong("numRecords", 0) == (if (id == 4) 2 else 1)) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + require(values.getLong("ts", 0) == micros) + require(values.getLong("ntz", 0) == micros) + require(values.getInteger("d", 0) == java.time.LocalDate.parse(day).toEpochDay) + } + val raw = add.getGroup("partitionValues", 0) + val partition = (0 until raw.getFieldRepetitionCount("key_value")).map { i => + val entry = raw.getGroup("key_value", i) + entry.getString("key", 0) -> entry.getString("value", 0) + }.toMap + require(epochPartitionStrings.contains(partition)) + val parsed = add.getGroup("partitionValues_parsed", 0) + val (_, _, pDay, pMicros) = epochValues.find { case (_, wall, _, _) => + java.time.LocalDateTime.parse(wall) == + java.time.LocalDateTime.parse(partition("p_ntz").replace(' ', 'T')) + }.get + require(parsed.getLong("p_ts", 0) == pMicros) + require(parsed.getLong("p_ntz", 0) == pMicros) + require(parsed.getInteger("p_d", 0) == java.time.LocalDate.parse(pDay).toEpochDay) + pairs :+= id -> partition + } + row = reader.read() + } + } finally reader.close() + val expectedPairs = for ((id, _, _, _) <- epochValues; p <- epochPartitionStrings) yield id -> p + require(pairs.size == 49 && pairs.toSet == expectedPairs.toSet) + println("Verified 49 physical stats/partition pairs: micros -1001,-1000,-999,-1,0,1,1000") + } + + private def checkEpochSpecs(t: TableHandle, parsed: Boolean): Unit = { + val workload = WorkloadContext.current.workloadName + val tableDir = outputDir.resolve(workload) + val all = epochRows + def read(name: String, predicate: String, rows: Seq[Row], version: java.lang.Long = null) + : Unit = { + readSpec(t, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, epochSchema) + val captured = tableDir.resolve("expected").resolve(s"${workload}_$name") + .resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + } + } + read("all", null, all) + read("all_v7", null, all, 7L) + for ((field, index, kind) <- Seq(("ts", 1, "TIMESTAMP"), ("ntz", 2, "TIMESTAMP_NTZ"), + ("p_ts", 4, "TIMESTAMP"), ("p_ntz", 5, "TIMESTAMP_NTZ"))) { + for ((id, wall, _, _) <- epochValues) { + val expected = if (kind == "TIMESTAMP") Timestamp.from(Instant.parse(wall + "Z")) + else java.time.LocalDateTime.parse(wall) + read(s"${field}_eq_$id", s"$field = $kind '${wall.replace('T', ' ')}'", + all.filter(_.get(index) == expected)) + } + read(s"${field}_negative", s"$field < $kind '1970-01-01 00:00:00'", + all.filter { r => + if (kind == "TIMESTAMP") r.getTimestamp(index).before(Timestamp.from(Instant.EPOCH)) + else r.getAs[java.time.LocalDateTime](index) + .isBefore(java.time.LocalDateTime.of(1970, 1, 1, 0, 0)) + }) + } + for ((field, index) <- Seq("d" -> 3, "p_d" -> 6); + day <- Seq("1969-12-31", "1970-01-01", "1970-01-02")) { + read(s"${field}_$day", s"$field = DATE '$day'", + all.filter(_.getDate(index) == java.sql.Date.valueOf(day))) + } + read("cross_negative_hits", "ts = TIMESTAMP '1969-12-31 23:59:59.999999' AND " + + "p_ntz = TIMESTAMP_NTZ '1969-12-31 23:59:59.999' AND p_d < DATE '1970-01-01'", + all.filter(r => r.getInt(0) == 4 && r.getTimestamp(4) == + Timestamp.from(Instant.parse("1969-12-31T23:59:59.999Z")))) + read("outside_stats", "ts < TIMESTAMP '1969-12-31 23:59:59.998999' OR " + + "ntz > TIMESTAMP_NTZ '1970-01-01 00:00:00.001'", Seq.empty) + read("outside_partitions", "p_ts < TIMESTAMP '1969-12-31 23:59:59.998999' OR " + + "p_ntz > TIMESTAMP_NTZ '1970-01-01 00:00:00.001'", Seq.empty) + for (version <- Seq[java.lang.Long](null, 7L)) { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == epochSchema) + require(result.metadata.partitionColumns == Seq("p_ts", "p_ntz", "p_d")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.metadata.configuration("delta.checkpointPolicy") == "classic") + require(result.protocol.minReaderVersion == 3) + require(result.protocol.minWriterVersion == 7) + require(result.protocol.readerFeatures.get.contains("timestampNtz")) + require(result.protocol.writerFeatures.get.contains("timestampNtz")) + case other => fail(s"Near-epoch snapshot must succeed (parsed=$parsed): $other") + } + } + } + } + + // Statistics and partition cases share the absolute-instant fixture and oracle. + private val mismatchSchema = new StructType().add("id", IntegerType) + .add("ts", TimestampType).add("ntz", org.apache.spark.sql.types.TimestampNTZType) + private val mismatchZones = Seq( + ("writer_utc_reader_non_utc", "UTC", "America/Los_Angeles"), + ("writer_non_utc_reader_utc", "America/Los_Angeles", "UTC")) + + // Offsets distinguish both occurrences of 01:30 at the autumn overlap. The unqualified + // controls deliberately produce different instants in the two writer-zone fixtures. + private val mismatchSql = Seq( + (1, "2024-03-10 01:59:59.123456-08:00", "2024-03-10 01:59:59.123456"), + (2, "2024-03-10 03:00:00.123456-07:00", "2024-03-10 03:00:00.123456"), + (3, "2024-11-03 01:30:00.123456-07:00", "2024-11-03 01:30:00.123456"), + (4, "2024-11-03 01:30:00.123456-08:00", "2024-11-03 01:30:00.123456"), + (5, "2024-01-15 12:00:00.654321", "2024-01-15 12:00:00.654321"), + (6, "2024-03-10 02:30:00.123456", "2024-03-10 02:30:00.123456"), + (7, "2024-11-03 01:30:00.123456", "2024-11-03 01:30:00.123456")) + + // This oracle does not parse fixture SQL or read Delta. NTZ retains the written wall clock, + // including the nonexistent LA 02:30. TIMESTAMP instead denotes an absolute UTC instant. + private def mismatchRows(writerZone: String): Seq[Row] = { + val controls = if (writerZone == "UTC") Seq( + "2024-01-15T12:00:00.654321Z", "2024-03-10T02:30:00.123456Z", + "2024-11-03T01:30:00.123456Z") else Seq( + "2024-01-15T20:00:00.654321Z", "2024-03-10T10:30:00.123456Z", + "2024-11-03T08:30:00.123456Z") + val instants = Seq( + "2024-03-10T09:59:59.123456Z", "2024-03-10T10:00:00.123456Z", + "2024-11-03T08:30:00.123456Z", "2024-11-03T09:30:00.123456Z") ++ controls + val walls = Seq( + "2024-03-10T01:59:59.123456", "2024-03-10T03:00:00.123456", + "2024-11-03T01:30:00.123456", "2024-11-03T01:30:00.123456", + "2024-01-15T12:00:00.654321", "2024-03-10T02:30:00.123456", + "2024-11-03T01:30:00.123456") + val rows = instants.zip(walls).zipWithIndex.map { case ((instant, wall), i) => + Row(i + 1, Timestamp.from(Instant.parse(instant)), java.time.LocalDateTime.parse(wall)) + } + rows ++ rows.filter(_.getInt(0) == 3) + } + + private def mismatchMicros(instant: Instant): Long = + instant.getEpochSecond * 1000000L + instant.getNano / 1000 + + // Spark to_json's default timestamp pattern serializes both bounds at millisecond + // precision. Check that physical encoding independently; keep microsecond row expectations. + private def mismatchJsonInstant(instant: Instant): Instant = + instant.truncatedTo(java.time.temporal.ChronoUnit.MILLIS) + + private def createMismatchTable(writerZone: String, partitioned: Boolean): TableHandle = { + setForTest("spark.sql.session.timeZone", writerZone) + val partition = if (partitioned) "PARTITIONED BY (ts)" else "" + sql(s"""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ) USING delta + $partition TBLPROPERTIES ('delta.checkpointPolicy' = 'classic', + 'delta.checkpointInterval' = '100', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + for ((id, instant, wall) <- mismatchSql) { + val row = s"($id, TIMESTAMP '$instant', TIMESTAMP_NTZ '$wall')" + val values = if (id == 3) s"$row, $row" else row + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES $values AS rows(id, ts, ntz)") + } + val table = registerTable("tbl") + val rows = mismatchRows(writerZone) + for (version <- 1L to 7L) { + val adds = CommitLog.read(table.sourcePath, version).collect { case a: AddFile => a } + require(adds.size == 1, "Each input batch must produce exactly one file") + val add = adds.head + val row = rows.find(_.getInt(0) == version.toInt).get + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(stats.get("numRecords").asLong() == (if (version == 3) 2 else 1)) + val expectedPartition = if (partitioned) { + Map("ts" -> row.getTimestamp(1).toInstant.toString) + } else Map.empty[String, String] + if (partitioned && writerZone == "America/Los_Angeles" && version == 4 && + add.partitionValues != expectedPartition) { + // The producer reparses an offset-free directory name at the overlap's earlier offset. + // Preserve that limitation, then construct the intended AddFile in the copied fixture. + require(add.partitionValues == Map("ts" -> "2024-11-03T08:30:00.123456Z")) + println(s"Observed producer overlap loss: expected=$expectedPartition " + + s"actual=${add.partitionValues} action=${add.toJson}") + } else require(add.partitionValues == expectedPartition) + for (bound <- Seq("minValues", "maxValues")) { + val fields = stats.get(bound) + require(fields.get("id").asInt() == version.toInt) + if (partitioned) require(!fields.has("ts")) else { + val raw = fields.get("ts").asText() + val parsed = java.time.OffsetDateTime.parse(raw) + val instant = row.getTimestamp(1).toInstant + require(parsed.toInstant == mismatchJsonInstant(instant)) + require(parsed.getOffset == instant.atZone(java.time.ZoneId.of(writerZone)).getOffset) + } + val ntz = row.getAs[java.time.LocalDateTime](2).toInstant(java.time.ZoneOffset.UTC) + val rawNtz = java.time.LocalDateTime.parse(fields.get("ntz").asText()) + require(rawNtz.toInstant(java.time.ZoneOffset.UTC) == mismatchJsonInstant(ntz)) + } + println(s"Verified writer JSON zone=$writerZone partitioned=$partitioned " + + s"id=$version stats=${add.stats.get} partitions=${add.partitionValues}") + } + println(s"Verified writer table zone=${spark.conf.get("spark.sql.session.timeZone")} " + + s"partitioned=$partitioned rows=8 files=7") + table + } + + private def prepareMismatchCapture( + table: TableHandle, writerZone: String, readerZone: String, + partitioned: Boolean, parsed: Boolean): Unit = { + require(writerZone != readerZone) + require(spark.conf.get("spark.sql.session.timeZone") == writerZone) + if (partitioned && writerZone == "America/Los_Angeles") { + val later = mismatchRows(writerZone).find(_.getInt(0) == 4).get.getTimestamp(1).toInstant + modifyCommitActions(table, version = 4) { actions => + actions.map { + case add: AddFile => add.copy(partitionValues = Map("ts" -> later.toString)) + case other => other + } + } + } + if (parsed) { + val encoding = spark.conf.get("spark.sql.parquet.outputTimestampType") + try { + spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MICROS") + forceCheckpoint("tbl") + } finally spark.conf.set("spark.sql.parquet.outputTimestampType", encoding) + } + mutateTable(table) { dir => + require(spark.conf.get("spark.sql.session.timeZone") == readerZone) + val logDir = dir.resolve("_delta_log") + if (parsed) { + val checkpoint = logDir.resolve("00000000000000000007.checkpoint.parquet") + if (partitioned && writerZone == "America/Los_Angeles") { + setMismatchOverlapPartition(checkpoint) + } + checkMismatchCheckpoint(checkpoint, writerZone, partitioned) + } + val files = Files.list(logDir) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (parsed && file.endsWith(".json") && + file != "00000000000000000007.json") + }.foreach(Files.delete(_)) finally files.close() + // Same-version v7 JSON is retained solely for version lookup. No replay tail exists; + // v0-v6 and CRC shortcuts are absent, so the checkpoint is required, not checkpoint-only. + println(s"Verified pre-capture reader=$readerZone writer=$writerZone " + + s"partitioned=$partitioned parsed=$parsed") + } + // WorkloadTestSuite captures after the body. Deliberately leave this reader zone active. + setForTest("spark.sql.session.timeZone", readerZone) + } + + // DAT supports a synthetic, correct overlap partition even when the SQL writer loses the + // later offset while reparsing a partition directory. Preserve its unmodified checkpoint; + // then change only the intended id=4 AddFile's raw and parsed partition cells in the copy. + private def setMismatchOverlapPartition(checkpoint: Path): Unit = { + val workload = WorkloadContext.current.workloadName + val original = outputDir.resolveSibling("producer-checkpoints").resolve(workload) + Files.createDirectories(original) + Files.copy(checkpoint, original.resolve(checkpoint.getFileName), + StandardCopyOption.REPLACE_EXISTING) + val instant = Instant.parse("2024-11-03T09:30:00.123456Z") + val temporary = checkpoint.resolveSibling("overlap-partition-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + val later = col("add.stats_parsed.minValues.id") === lit(4) + val raw = org.apache.spark.sql.functions.when(later, + org.apache.spark.sql.functions.map(lit("ts"), lit(instant.toString))) + .otherwise(col("add.partitionValues")) + val parsed = col("add.partitionValues_parsed").withField("ts", + org.apache.spark.sql.functions.when(later, lit(Timestamp.from(instant))) + .otherwise(col("add.partitionValues_parsed.ts"))) + val add = col("add").withField("partitionValues", raw) + .withField("partitionValues_parsed", parsed) + df.withColumn("add", add).coalesce(1).write.parquet(temporary.toString) + val files = Files.list(temporary) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one overlap checkpoint part: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(temporary)) FileUtils.deleteDirectory(temporary.toFile) + } + println("Constructed later overlap partition fixture: id=4 " + + "instant=2024-11-03T09:30:00.123456Z") + } + + private def checkMismatchCheckpoint( + checkpoint: Path, writerZone: String, partitioned: Boolean): Unit = { + require(Files.isRegularFile(checkpoint)) + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + val timestamps = if (partitioned) Seq("partitionValues_parsed" -> "ts") + else Seq("stats_parsed.minValues" -> "ts", "stats_parsed.maxValues" -> "ts") + for ((prefix, field) <- timestamps ++ Seq( + "stats_parsed.minValues" -> "ntz", "stats_parsed.maxValues" -> "ntz")) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == (field == "ts")) + println(s"Verified timezone checkpoint writer=$writerZone $prefix.$field: $cell") + } + } finally footer.close() + val rows = mismatchRows(writerZone) + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var ids = Vector.empty[Int] + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + val add = record.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + val row = rows.find(_.getInt(0) == id).get + require(stats.getLong("numRecords", 0) == (if (id == 3) 2 else 1)) + val instant = row.getTimestamp(1).toInstant + val ntz = row.getAs[java.time.LocalDateTime](2).toInstant(java.time.ZoneOffset.UTC) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + if (!partitioned) require(values.getLong("ts", 0) == + mismatchMicros(mismatchJsonInstant(instant))) + require(values.getLong("ntz", 0) == mismatchMicros(mismatchJsonInstant(ntz))) + } + if (partitioned) { + val partition = add.getGroup("partitionValues_parsed", 0) + require(partition.getLong("ts", 0) == mismatchMicros(instant)) + val raw = add.getGroup("partitionValues", 0) + require(raw.getFieldRepetitionCount("key_value") == 1) + val entry = raw.getGroup("key_value", 0) + require(entry.getString("key", 0) == "ts") + require(entry.getString("value", 0) == instant.toString) + } + ids :+= id + println(s"Verified timezone raw checkpoint writer=$writerZone partitioned=$partitioned " + + s"id=$id instantMicros=${mismatchMicros(instant)} ntzMicros=${mismatchMicros(ntz)}") + } + record = reader.read() + } + } finally reader.close() + require(ids.sorted == (1 to 7).toVector) + } + + private def checkMismatchSpecs( + table: TableHandle, writerZone: String, readerZone: String, partitioned: Boolean): Unit = { + val workload = WorkloadContext.current.workloadName + val tableDir = outputDir.resolve(workload) + val all = mismatchRows(writerZone) + def checkReaderZone(label: String): Unit = { + require(spark.conf.get("spark.sql.session.timeZone") == readerZone) + val probe = spark.sql("SELECT current_timezone(), " + + "CAST(TIMESTAMP '2024-03-10 10:00:00.123456Z' AS STRING)").head() + require(probe.getString(0) == readerZone) + val wall = if (readerZone == "UTC") "2024-03-10 10:00:00.123456" + else "2024-03-10 03:00:00.123456" + require(probe.getString(1) == wall) + println(s"Verified captured $workload/$label writer=$writerZone reader=$readerZone " + + s"readerWall=$wall") + } + def read(label: String, predicate: String, rows: Seq[Row], version: java.lang.Long): Unit = { + readSpec(table, predicate = predicate, version = version, name = Some(label)).assert { spec => + checkReaderZone(label) + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$label must succeed: $other") + } + val expected = spark.createDataFrame(rows.asJava, mismatchSchema) + val captured = tableDir.resolve("expected").resolve(s"${workload}_$label") + .resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), label) + if (label == "all_latest") { + expected.coalesce(1).write.mode("overwrite") + .parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString) + } + } + } + for (version <- Seq[java.lang.Long](null, 7L)) { + val suffix = if (version == null) "latest" else "v7" + read(s"all_$suffix", null, all, version) + // Explicit offsets keep the emitted predicates portable to consumers with another zone. + for (row <- all.groupBy(_.getInt(0)).values.map(_.head).toSeq.sortBy(_.getInt(0))) { + val instant = row.getTimestamp(1) + read(s"eq_${row.getInt(0)}_$suffix", s"ts = TIMESTAMP '${instant.toInstant}'", + all.filter(_.getTimestamp(1) == instant), version) + } + read(s"spring_range_$suffix", "ts >= TIMESTAMP '2024-03-10T09:59:59.123456Z' AND " + + "ts <= TIMESTAMP '2024-03-10T10:00:00.123456Z'", + all.filter(r => Set(1, 2).contains(r.getInt(0))), version) + read(s"overlap_range_$suffix", "ts >= TIMESTAMP '2024-11-03T08:30:00.123456Z' AND " + + "ts <= TIMESTAMP '2024-11-03T09:30:00.123456Z'", + all.filter(r => Set(3, 4).contains(r.getInt(0)) || + (writerZone != "UTC" && r.getInt(0) == 7)), version) + read(s"outside_$suffix", "ts < TIMESTAMP '2024-01-01T00:00:00Z'", Seq.empty, version) + read(s"ntz_gap_$suffix", "ntz = TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'", + all.filter(_.getInt(0) == 6), version) + snapshotSpec(table, version = version).assert { spec => + checkReaderZone(s"snapshot_$suffix") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == mismatchSchema) + require(result.metadata.partitionColumns == (if (partitioned) Seq("ts") else Seq.empty)) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.metadata.configuration("delta.checkpointPolicy") == "classic") + require(result.metadata.configuration("delta.checkpoint.writeStatsAsStruct") == "true") + require(result.metadata.configuration("delta.checkpoint.writeStatsAsJson") == "false") + require(result.protocol.minReaderVersion == 3 && result.protocol.minWriterVersion == 7) + require(result.protocol.readerFeatures.get.contains("timestampNtz")) + require(result.protocol.writerFeatures.get.contains("timestampNtz")) + case other => fail(s"Timezone snapshot must succeed: $other") + } + } + } + } + + for ((direction, writer, reader) <- mismatchZones; parsed <- Seq(false, true)) { + val representation = if (parsed) "checkpoint" else "json" + test(s"stats_timezone_${direction}_$representation") { + val table = createMismatchTable(writer, partitioned = false) + prepareMismatchCapture(table, writer, reader, partitioned = false, parsed = parsed) + checkMismatchSpecs(table, writer, reader, partitioned = false) + } + } + + // Reuse the timezone fixture and check TIMESTAMP partition reconstruction. + for ((direction, writer, reader) <- mismatchZones; parsed <- Seq(false, true)) { + val representation = if (parsed) "checkpoint" else "json" + test(s"partitions_timezone_${direction}_$representation") { + val table = createMismatchTable(writer, partitioned = true) + prepareMismatchCapture(table, writer, reader, partitioned = true, parsed = parsed) + checkMismatchSpecs(table, writer, reader, partitioned = true) + } + } + + private def pruneHistoryExcept(dir: Path, version: Long): Unit = { + val log = dir.resolve("_delta_log") + val files = Files.list(log) + try files.iterator().asScala.filter { p => + val file = p.getFileName.toString + file.endsWith(".crc") || (file.endsWith(".json") && file != f"$version%020d.json") + }.foreach(Files.delete(_)) finally files.close() + // The same-version JSON supports requested-version lookup only. Earlier history is absent, + // so checkpoint reconstruction is required. No replay tail or checkpoint-only claim. + } + + // Unannotated INT32/INT64 parsed cells are distinct from normal DATE/TIMESTAMP + // Parquet encodings. Keep correct JSON stats too: Spark rejects several parsed-only forms. + // Only INT32 as DATE, the DATE32 physical width, removes every JSON stats copy so that the + // parsed day is the only statistic. The other forms are not claimed as parsed-only routes. + for ((kind, sqlType, valueType, literal, value, integerValue) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType, + "TIMESTAMP '1970-01-01T00:00:01Z'", Timestamp.from(Instant.ofEpochSecond(1)), 1000000L), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1970-01-01 00:00:01'", + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1), 1000000L), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1970-01-02'", java.sql.Date.valueOf("1970-01-02"), 1L)); + integerType <- Seq("int", "long")) { + test(s"checkpoint_integer_${integerType}_as_$kind") { + setForTest("spark.sql.session.timeZone", "UTC") + val expectedSchema = new StructType().add("id", IntegerType).add("v", valueType) + val rows = Seq(Row(1, value), Row(2, value), Row(2, value)) + sql(s"""CREATE TABLE tbl (id INT, v $sqlType) USING delta TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'true')""") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $literal), (2, $literal), (2, $literal) AS rows(id, v)") + val table = registerTable("tbl") + val parsedOnly = kind == "date" && integerType == "int" + forceCheckpoint("tbl") + if (parsedOnly) { + modifyCommitActions(table, 1L) { actions => + actions.map { case add: AddFile => add.copy(stats = None); case other => other } + } + } + mutateTable(table) { dir => + val checkpoint = dir.resolve("_delta_log/00000000000000000001.checkpoint.parquet") + val temporary = dir.resolve("_delta_log/integer-stats-rewrite") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("v", lit(integerValue).cast(integerType))) + } + var add = col("add").withField("stats_parsed", stats) + if (parsedOnly) add = add.withField("stats", lit(null).cast("string")) + df.withColumn("add", add).coalesce(1).write.parquet(temporary.toString) + val parts = Files.list(temporary) + val part = try parts.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten integer stats checkpoint: $other") + } finally parts.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + FileUtils.deleteDirectory(temporary.toFile) + val path = new HadoopPath(checkpoint.toUri) + val conf = spark.sparkContext.hadoopConfiguration + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + for (bound <- Seq("minValues", "maxValues")) { + val physical = footer.getFooter.getFileMetaData.getSchema + .getType("add", "stats_parsed", bound, "v").asPrimitiveType() + require(physical.getLogicalTypeAnnotation == null) + require(physical.getPrimitiveTypeName == (if (integerType == "int") + PrimitiveType.PrimitiveTypeName.INT32 else PrimitiveType.PrimitiveTypeName.INT64)) + println(s"Verified off-type checkpoint $kind/$integerType $bound: " + + s"$physical value=$integerValue jsonFallback=${!parsedOnly}") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + var count = 0 + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + count += 1 + val add = record.getGroup("add", 0) + if (parsedOnly) require(add.getFieldRepetitionCount("stats") == 0) else { + require(add.getFieldRepetitionCount("stats") == 1) + val json = JsonUtil.mapper.readTree(add.getString("stats", 0)) + for (bound <- Seq("minValues", "maxValues")) { + val raw = json.at(s"/$bound/v").asText() + kind match { + case "timestamp" => require(Instant.parse(raw) == Instant.ofEpochSecond(1)) + case "timestamp_ntz" => require(java.time.LocalDateTime.parse(raw) == + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1)) + case "date" => require(java.time.LocalDate.parse(raw) == + java.time.LocalDate.of(1970, 1, 2)) + } + } + } + val fields = add.getGroup("stats_parsed", 0) + require(fields.getLong("numRecords", 0) == 3L) + for (bound <- Seq("minValues", "maxValues")) { + val values = fields.getGroup(bound, 0) + val raw = if (integerType == "int") values.getInteger("v", 0).toLong + else values.getLong("v", 0) + require(raw == integerValue) + } + } + record = reader.read() + } + } finally reader.close() + require(count == 1) + pruneHistoryExcept(dir, 1L) + if (parsedOnly) { + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 1 && adds.head.stats.isEmpty) + val log = Files.list(dir.resolve("_delta_log")) + val names = try log.iterator().asScala.map(_.getFileName.toString).toSet + finally log.close() + require(!names.exists(_.endsWith(".crc"))) + require(names.filter(_.endsWith(".json")) == Set("00000000000000000001.json")) + println("Verified parsed-only DATE stats: INT32 day=1 (1970-01-02); JSON stats " + + "absent in checkpoint and v1 commit; CRC removed; v0 pruned") + } + } + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + checkLiteralRead(table, "hit", expectedSchema, rows, predicate = s"v = $literal") + checkLiteralRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"v < $literal OR v > $literal") + checkLiteralSnapshot(table, expectedSchema) + checkLiteralSnapshot(table, expectedSchema, version = 1L) + } + } + + // JSON numeric stats are separate from physical checkpoint types in the checkpoint fixture. + // Literal rows remain authoritative even when a reference parser discards off-type stats. + for ((kind, sqlType, valueType, literal, value) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType, "TIMESTAMP '1970-01-01T00:00:01Z'", + Timestamp.from(Instant.ofEpochSecond(1))), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1970-01-01 00:00:01'", + java.time.LocalDateTime.of(1970, 1, 1, 0, 0, 1)), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1970-01-02'", java.sql.Date.valueOf("1970-01-02")), + ("string", "STRING", org.apache.spark.sql.types.StringType, "'1'", "1"))) { + test(s"action_integer_as_$kind") { + setForTest("spark.sql.session.timeZone", "UTC") + val expectedSchema = new StructType().add("id", IntegerType).add("v", valueType) + val rows = Seq(Row(1, value), Row(2, value), Row(2, value)) + sql(s"CREATE TABLE tbl (id INT, v $sqlType) USING delta") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $literal), (2, $literal), (2, $literal) AS rows(id, v)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val stats = JsonUtil.mapper.readTree(add.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + stats.get(bound).asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("v", 1) + } + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 1) + val stats = JsonUtil.mapper.readTree(adds.head.stats.get) + require(stats.get("numRecords").asLong() == 3L) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/v").isIntegralNumber) + require(stats.at(s"/$bound/v").intValue() == 1) + } + println(s"Verified numeric JSON stats: $kind min=1 max=1 records=3") + } + // Spark accepts numeric STRING stats. The required-rejection probe is preserved + // separately; this useful correctness fixture does not claim that rejection. + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + checkLiteralRead(table, "hit", expectedSchema, rows, predicate = s"v = $literal") + checkLiteralRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"v < $literal OR v > $literal") + checkLiteralSnapshot(table, expectedSchema) + checkLiteralSnapshot(table, expectedSchema, version = 1L) + } + } + + // INT32 boundary magnitudes are BIGINT inputs to timestamp_seconds/millis/micros. The unit + // names the input magnitude only; rewritten checkpoint cells are always TIMESTAMP_MICROS. + private val intMaxInputs = Seq(-2147483648L, -2147483647L, -2147483646L, + 2147483646L, 2147483647L, 2147483648L) + // Literal UTC instants per input unit, declared apart from both SQL and java.time. + private val intMaxInstants = Seq( + ("seconds", 1000000L, Seq("1901-12-13T20:45:52Z", "1901-12-13T20:45:53Z", + "1901-12-13T20:45:54Z", "2038-01-19T03:14:06Z", "2038-01-19T03:14:07Z", + "2038-01-19T03:14:08Z")), + ("millis", 1000L, Seq("1969-12-07T03:28:36.352Z", "1969-12-07T03:28:36.353Z", + "1969-12-07T03:28:36.354Z", "1970-01-25T20:31:23.646Z", "1970-01-25T20:31:23.647Z", + "1970-01-25T20:31:23.648Z")), + ("micros", 1L, Seq("1969-12-31T23:24:12.516352Z", "1969-12-31T23:24:12.516353Z", + "1969-12-31T23:24:12.516354Z", "1970-01-01T00:35:47.483646Z", + "1970-01-01T00:35:47.483647Z", "1970-01-01T00:35:47.483648Z"))) + // Data value index -> partition value index; no index maps to itself. Ids 2 and 5 hold the + // -/+2147483647 boundaries twice, and id 7 is NULL in every temporal column twice. + private val intMaxPartition = Map(1 -> 4, 2 -> 6, 3 -> 1, 4 -> 5, 5 -> 2, 6 -> 3) + private val intMaxIds = Seq(1, 2, 2, 3, 4, 5, 5, 6, 7, 7) + private val intMaxSchema = new StructType().add("id", IntegerType).add("ts", TimestampType) + .add("ntz", TimestampNTZType).add("p_ts", TimestampType).add("p_ntz", TimestampNTZType) + private val sixDigits = + java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS") + + for ((unit, scale, literals) <- intMaxInstants; parsed <- Seq(false, true)) { + test(s"int_max_${unit}_${if (parsed) "parsed" else "json"}") { + setForTest("spark.sql.session.timeZone", "UTC") + val instants = literals.map(Instant.parse) + for ((input, instant) <- intMaxInputs.zip(instants)) { + val converted = unit match { + case "seconds" => Instant.ofEpochSecond(input) + case "millis" => Instant.ofEpochMilli(input) + case "micros" => Instant.EPOCH.plus(input, java.time.temporal.ChronoUnit.MICROS) + } + require(converted == instant && mismatchMicros(instant) == input * scale) + } + def wall(index: Int) = + java.time.LocalDateTime.ofInstant(instants(index - 1), java.time.ZoneOffset.UTC) + val rows = intMaxIds.map { + case 7 => Row(7, null, null, null, null) + case id => + val p = intMaxPartition(id) + Row(id, Timestamp.from(instants(id - 1)), wall(id), Timestamp.from(instants(p - 1)), + wall(p)) + } + def partitionStrings(id: Int): Map[String, String] = if (id == 7) { + Map("p_ts" -> null, "p_ntz" -> null) + } else { + val p = wall(intMaxPartition(id)).format(sixDigits) + Map("p_ts" -> (p + "Z"), + "p_ntz" -> p.replace('T', ' ').replaceAll("0+$", "").stripSuffix(".")) + } + def fileId(add: AddFile): Int = + JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + + sql("""CREATE TABLE tbl (id INT, ts TIMESTAMP, ntz TIMESTAMP_NTZ, p_ts TIMESTAMP, + p_ntz TIMESTAMP_NTZ) USING delta PARTITIONED BY (p_ts, p_ntz) TBLPROPERTIES ( + 'delta.checkpointPolicy' = 'classic', 'delta.checkpoint.writeStatsAsStruct' = 'true', + 'delta.checkpoint.writeStatsAsJson' = 'false')""") + val inputs = intMaxIds.map { + case 7 => "(7, CAST(NULL AS BIGINT), CAST(NULL AS BIGINT))" + case id => s"($id, ${intMaxInputs(id - 1)}L, ${intMaxInputs(intMaxPartition(id) - 1)}L)" + } + val f = s"timestamp_$unit" + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ id, $f(v), CAST($f(v) AS TIMESTAMP_NTZ), " + + s"$f(p), CAST($f(p) AS TIMESTAMP_NTZ) FROM VALUES ${inputs.mkString(", ")} " + + "AS rows(id, v, p)") + val table = registerTable("tbl") + val produced = CommitLog.read(table.sourcePath, 1L).collect { case a: AddFile => a } + require(produced.map(a => fileId(a) -> a.partitionValues).toMap == + (1 to 7).map(id => id -> partitionStrings(id)).toMap && produced.size == 7) + + if (parsed) forceCheckpoint("tbl") else { + // The producer writes millisecond JSON bounds. Keep all six fraction digits instead. + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile if fileId(add) != 7 => + val id = fileId(add) + println(s"Producer JSON stats before rewrite: $unit id=$id ${add.stats.get}") + val stats = JsonUtil.mapper.readTree(add.stats.get) + for (bound <- Seq("minValues", "maxValues")) { + stats.get(bound).asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("ts", wall(id).format(sixDigits) + "Z") + .put("ntz", wall(id).format(sixDigits)) + } + add.copy(stats = Some(JsonUtil.mapper.writeValueAsString(stats))) + case other => other + } + } + } + mutateTable(table) { dir => + if (parsed) { + val checkpoint = dir.resolve("_delta_log/00000000000000000001.checkpoint.parquet") + rewriteIntMaxCheckpoint(checkpoint, instants) + checkIntMaxCheckpoint(checkpoint, instants, partitionStrings) + pruneHistoryExcept(dir, 1L) + } else { + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case a: AddFile => a } + require(adds.size == 7) + for (add <- adds) { + val id = fileId(add) + val stats = JsonUtil.mapper.readTree(add.stats.get) + require(add.partitionValues == partitionStrings(id)) + require(stats.get("numRecords").asLong() == (if (Set(2, 5, 7)(id)) 2L else 1L)) + for (bound <- Seq("minValues", "maxValues")) { + if (id == 7) require(!stats.get(bound).has("ts") && !stats.get(bound).has("ntz")) + else { + require(stats.at(s"/$bound/ts").asText() == wall(id).format(sixDigits) + "Z") + require(stats.at(s"/$bound/ntz").asText() == wall(id).format(sixDigits)) + } + } + if (id == 7) require(stats.at("/nullCount/ts").asLong() == 2L) + } + println(s"Verified six-digit JSON stats and partitions: $unit files=7") + } + } + + checkLiteralRead(table, "all", intMaxSchema, rows) + checkLiteralRead(table, "all_v1", intMaxSchema, rows, version = 1L) + for ((field, kind) <- Seq("ts" -> "TIMESTAMP", "ntz" -> "TIMESTAMP_NTZ", + "p_ts" -> "TIMESTAMP", "p_ntz" -> "TIMESTAMP_NTZ")) { + def v(index: Int): String = if (kind == "TIMESTAMP") { + s"TIMESTAMP '${instants(index - 1)}'" + } else s"TIMESTAMP_NTZ '${wall(index).format(sixDigits).replace('T', ' ')}'" + def index(row: Row): Option[Int] = row.getInt(0) match { + case 7 => None + case id => Some(if (field.startsWith("p_")) intMaxPartition(id) else id) + } + for ((label, predicate, selected) <- Seq( + ("below_negative", s"$field < ${v(2)}", Set(1)), + ("eq_negative", s"$field = ${v(2)}", Set(2)), + ("interior", s"$field > ${v(2)} AND $field < ${v(5)}", Set(3, 4)), + ("eq_positive", s"$field = ${v(5)}", Set(5)), + ("above_positive", s"$field > ${v(5)}", Set(6)), + ("outside", s"$field < ${v(1)} OR $field > ${v(6)}", Set.empty[Int]))) { + checkLiteralRead(table, s"${field}_$label", intMaxSchema, + rows.filter(r => index(r).exists(selected)), predicate) + } + checkLiteralRead(table, s"${field}_is_null", intMaxSchema, + rows.filter(_.getInt(0) == 7), s"$field IS NULL") + } + checkLiteralSnapshot(table, intMaxSchema, Seq("p_ts", "p_ntz")) + checkLiteralSnapshot(table, intMaxSchema, Seq("p_ts", "p_ntz"), version = 1L) + } + } + + // DATE32 epoch-day bounds. 2147483648 is outside DATE32 and is not an input. Rows use + // java.time.LocalDate: java.sql.Date misrenders these years. Raw INT32 days are the oracle. + test("date_epoch_days_int32_bounds") { + setForTest("spark.sql.session.timeZone", "UTC") + setForTest("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED") + setForTest("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED") + // Spark's data-filter translation rebases java.sql.Date literals to Julian days, which + // overflows near -2147483648. LocalDate literals avoid that reference-side conversion. + // setForTest keeps it through capture and restores it after the test, including on failure. + setForTest(java8Key, "true") + val days = Seq(-2147483648, -2147483647, -2147483646, 2147483646, 2147483647) + val dates = Seq("-5877641-06-23", "-5877641-06-24", "-5877641-06-25", "+5881580-07-10", + "+5881580-07-11").map(LocalDate.parse) + require(dates.map(_.toEpochDay) == days.map(_.toLong)) + val permutation = Map(1 -> 3, 2 -> 5, 3 -> 1, 4 -> 2, 5 -> 4) + val ids = Seq(1, 2, 2, 3, 4, 5, 5, 6, 6) + val expectedSchema = new StructType().add("id", IntegerType) + .add("d", org.apache.spark.sql.types.DateType).add("p", org.apache.spark.sql.types.DateType) + val rows = ids.map { + case 6 => Row(6, null, null) + case id => Row(id, dates(id - 1), dates(permutation(id) - 1)) + } + val units = ids.map { + case 6 => (6, None, None) + case id => (id, Some(days(id - 1)), Some(days(permutation(id) - 1))) + } + sql("CREATE TABLE tbl (id INT, d DATE, p DATE) USING delta PARTITIONED BY (p)") + val inputs = ids.map { + case 6 => "(6, CAST(NULL AS INT), CAST(NULL AS INT))" + case id => s"($id, ${days(id - 1)}, ${days(permutation(id) - 1)})" + } + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ id, date_from_unix_date(d), " + + s"date_from_unix_date(p) FROM VALUES ${inputs.mkString(", ")} AS rows(id, d, p)") + val table = registerTable("tbl") + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 6) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + require(stats.get("numRecords").asLong() == (if (Set(2, 5, 6)(id)) 2L else 1L)) + if (id == 6) { + require(add.partitionValues == Map("p" -> null)) + require(stats.at("/nullCount/d").asLong() == 2L && !stats.at("/minValues").has("d")) + } else { + require(add.partitionValues == Map("p" -> dates(permutation(id) - 1).toString)) + for (bound <- Seq("minValues", "maxValues")) { + require(stats.at(s"/$bound/d").asText() == dates(id - 1).toString) + } + } + println(s"Verified DATE32 bound file: id=$id stats=${add.stats.get} " + + s"partition=${add.partitionValues}") + } + } + def literal(index: Int): String = s"DATE '${dates(index - 1)}'" + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + for (field <- Seq("d", "p")) { + def index(row: Row): Option[Int] = row.getInt(0) match { + case 6 => None + case id => Some(if (field == "p") permutation(id) else id) + } + for ((label, predicate, selected) <- Seq( + ("below_negative", s"$field < ${literal(2)}", Set(1)), + ("eq_negative", s"$field = ${literal(2)}", Set(2)), + ("interior", s"$field > ${literal(2)} AND $field < ${literal(5)}", Set(3, 4)), + ("eq_positive", s"$field = ${literal(5)}", Set(5)))) { + checkLiteralRead(table, s"${field}_$label", expectedSchema, + rows.filter(r => index(r).exists(selected)), predicate) + } + checkLiteralRead(table, s"${field}_is_null", expectedSchema, + rows.filter(_.getInt(0) == 6), s"$field IS NULL") + } + checkLiteralSnapshot(table, expectedSchema, Seq("p")) + checkLiteralSnapshot(table, expectedSchema, Seq("p"), version = 1L) + // Snapshots capture before reads at this pin. Inspect Parquet in the last read callback. + val workload = WorkloadContext.current.workloadName + readSpec(table, predicate = s"d < ${literal(1)} OR d > ${literal(5)}", + name = Some("outside")).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == 0L) + case other => fail(s"Outside DATE32 bounds must succeed with no rows: $other") + } + val conf = spark.sparkContext.hadoopConfiguration + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_all").resolve("expected_data") + val literalOracle = outputDir.resolveSibling("oracle-parquet").resolve(workload) + for ((label, directory) <- Seq("captured" -> captured, "literal" -> literalOracle)) { + val files = Files.list(directory) + val parquet = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toVector finally files.close() + require(parquet.nonEmpty) + val found = parquet.flatMap { file => + val path = new HadoopPath(file.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try for (field <- Seq("d", "p")) { + val physical = footer.getFooter.getFileMetaData.getSchema.getFields.asScala + .find(_.getName == field).get.asPrimitiveType() + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(physical.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + var out = Vector.empty[(Int, Option[Int], Option[Int])] + try { + var record = reader.read() + while (record != null) { + def day(field: String) = if (record.getFieldRepetitionCount(field) == 0) None + else Some(record.getInteger(field, 0)) + out :+= ((record.getInteger("id", 0), day("d"), day("p"))) + record = reader.read() + } + } finally reader.close() + out + } + require(found.sorted == units.sorted) + println(s"Verified raw DATE32 epoch days: $workload $label ${found.sorted}") + } + require(spark.conf.get(java8Key) == "true") + } + } + private val java8Key = "spark.sql.datetime.java8API.enabled" + + // Replace producer-derived parsed bounds by the literal instants and drop the JSON fallback. + private def rewriteIntMaxCheckpoint(checkpoint: Path, instants: Seq[Instant]): Unit = { + val rewrite = checkpoint.resolveSibling("int-max-checkpoint-rewrite") + val confKey = "spark.sql.parquet.outputTimestampType" + val previous = spark.conf.get(confKey) + try { + spark.conf.set(confKey, "TIMESTAMP_MICROS") + val df = spark.read.parquet(checkpoint.toString) + var stats = col("add.stats_parsed") + for (bound <- Seq("minValues", "maxValues")) { + var ts = col(s"add.stats_parsed.$bound.ts") + var ntz = col(s"add.stats_parsed.$bound.ntz") + for ((instant, index) <- instants.zipWithIndex) { + val matches = col("add.stats_parsed.minValues.id") === lit(index + 1) + ts = when(matches, lit(Timestamp.from(instant))).otherwise(ts) + ntz = when(matches, + lit(java.time.LocalDateTime.ofInstant(instant, java.time.ZoneOffset.UTC))) + .otherwise(ntz) + } + stats = stats.withField(bound, col(s"add.stats_parsed.$bound") + .withField("ts", ts).withField("ntz", ntz)) + } + var add = col("add").withField("stats_parsed", stats) + if (df.schema("add").dataType.asInstanceOf[StructType].fieldNames.contains("stats")) { + add = add.withField("stats", lit(null).cast("string")) + } + df.withColumn("add", add).coalesce(1).write.parquet(rewrite.toString) + val files = Files.list(rewrite) + val part = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toSeq match { + case Seq(single) => single + case other => fail(s"Expected one rewritten INT_MAX checkpoint: $other") + } finally files.close() + Files.move(part, checkpoint, StandardCopyOption.REPLACE_EXISTING) + } finally { + spark.conf.set(confKey, previous) + if (Files.exists(rewrite)) FileUtils.deleteDirectory(rewrite.toFile) + } + } + + private def checkIntMaxCheckpoint( + checkpoint: Path, instants: Seq[Instant], partitions: Int => Map[String, String]): Unit = { + val conf = spark.sparkContext.hadoopConfiguration + val path = new HadoopPath(checkpoint.toUri) + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema + for ((prefix, field) <- Seq("stats_parsed.minValues" -> "ts", + "stats_parsed.minValues" -> "ntz", "stats_parsed.maxValues" -> "ts", + "stats_parsed.maxValues" -> "ntz", "partitionValues_parsed" -> "p_ts", + "partitionValues_parsed" -> "p_ntz")) { + val cell = physical.getType((Seq("add") ++ prefix.split("\\.") :+ field): _*) + .asPrimitiveType() + require(cell.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) + val annotation = cell.getLogicalTypeAnnotation + .asInstanceOf[LogicalTypeAnnotation.TimestampLogicalTypeAnnotation] + require(annotation.getUnit == LogicalTypeAnnotation.TimeUnit.MICROS) + require(annotation.isAdjustedToUTC == !field.endsWith("ntz")) + println(s"Verified INT_MAX checkpoint $prefix.$field: $cell") + } + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path).withConf(conf).build() + var ids = Vector.empty[Int] + try { + var record = reader.read() + while (record != null) { + if (record.getFieldRepetitionCount("add") > 0) { + val add = record.getGroup("add", 0) + if (add.getType.containsField("stats")) require(add.getFieldRepetitionCount("stats") == 0) + val stats = add.getGroup("stats_parsed", 0) + val id = stats.getGroup("minValues", 0).getInteger("id", 0) + require(stats.getLong("numRecords", 0) == (if (Set(2, 5, 7)(id)) 2L else 1L)) + val parsedPartition = add.getGroup("partitionValues_parsed", 0) + for (bound <- Seq("minValues", "maxValues")) { + val values = stats.getGroup(bound, 0) + require(values.getInteger("id", 0) == id) + for (field <- Seq("ts", "ntz")) { + if (id == 7) require(values.getFieldRepetitionCount(field) == 0) + else require(values.getLong(field, 0) == mismatchMicros(instants(id - 1))) + } + } + for (field <- Seq("p_ts", "p_ntz")) { + if (id == 7) require(parsedPartition.getFieldRepetitionCount(field) == 0) + else require(parsedPartition.getLong(field, 0) == + mismatchMicros(instants(intMaxPartition(id) - 1))) + } + if (id == 7) require(stats.getGroup("nullCount", 0).getLong("ts", 0) == 2L) + val raw = add.getGroup("partitionValues", 0) + val partition = (0 until raw.getFieldRepetitionCount("key_value")).map { i => + val entry = raw.getGroup("key_value", i) + val value = if (entry.getFieldRepetitionCount("value") == 0) null + else entry.getString("value", 0) + entry.getString("key", 0) -> value + }.toMap + require(partition == partitions(id)) + ids :+= id + println(s"Verified INT_MAX raw checkpoint id=$id partition=$partition") + } + record = reader.read() + } + } finally reader.close() + require(ids.sorted == (1 to 7).toVector) + } + + // Table operations that rewrite statistics or file layout. + test("e2e_51_stats_refresh") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'drop'), (0, 'low'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (12, 'middle'), (18, CAST(NULL AS STRING)), (31, 'last'), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("DELETE FROM tbl WHERE id = -7") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val physicalRows = Seq(Row(-7, "drop"), Row(0, "low"), Row(4, null), Row(4, null), + Row(12, "middle"), Row(18, null), Row(31, "last"), Row(null, "missing")) + val rows = Seq(Row(0, "low"), Row(4, null), Row(4, null), Row(12, "middle"), + Row(18, null), Row(31, "last"), Row(null, "missing")) + val original = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = Protocol(3, 7, Some(Set("deletionVectors")), + Some(Set("appendOnly", "deletionVectors", "invariants"))) + assert(original(0L).collect { case p: Protocol => p } == Seq(protocol)) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inserted = original(1L).collect { case a: AddFile => a }.ensuring(_.size == 1).head + val deleted = original(2L).collect { case a: AddFile => a }.ensuring(_.size == 1).head + assert(inserted.path == deleted.path && inserted.deletionVector.isEmpty) + val dv = deleted.deletionVector.get + assert(dv.storageType == "u" && dv.cardinality == 1L) + val physical = spark.read.parquet(table.sourcePath.resolve(deleted.path).toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(physicalRows.asJava, schema), physical, + "replay literal physical rows") + val positions = physical.collect().toSeq.zipWithIndex.collect { + case (row, index) if !row.isNullAt(0) && row.getInt(0) == -7 => index.toLong + } + val dvBytes = payload(table.sourcePath, dv) + assert(RoaringBitmapArray.readFrom(dvBytes).toArray.toSeq == positions) + val fileBytes = Files.readAllBytes(table.sourcePath.resolve(deleted.path)) + // Physical stats include the deleted row. These bounds remain conservative for the DV. + // Model a partial-stats writer followed by a stats-only update, not an autostats service. + val partialStats = """{"numRecords":8,"tightBounds":false}""" + val refreshedStats = """{"numRecords":8,"minValues":{"id":-7,"value":"drop"}, + "maxValues":{"id":31,"value":"missing"},"nullCount":{"id":1,"value":3}, + "tightBounds":false}""" + val partial = deleted.copy(stats = Some(partialStats)) + val refreshed = partial.copy(stats = Some(refreshedStats), dataChange = false) + assert(refreshed.copy(stats = deleted.stats, dataChange = deleted.dataChange) == deleted) + assert(refreshed.path == deleted.path && refreshed.deletionVector == Some(dv)) + assert(refreshed.baseRowId == deleted.baseRowId) + assert(refreshed.defaultRowCommitVersion == deleted.defaultRowCommitVersion) + // Row tracking is not enabled: preserve absent row-ID fields as well as the real DV. + assert(deleted.baseRowId.isEmpty && deleted.defaultRowCommitVersion.isEmpty) + modifyCommitActions(table, 2L) { actions => + actions.map { case _: AddFile => partial; case other => other } + } + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + assert(CommitLog.read(dir, 0L) == original(0L)) + assert(CommitLog.read(dir, 1L) == original(1L)) + assert(CommitLog.read(dir, 2L) == original(2L).map { + case _: AddFile => partial; case other => other + }) + assert(!Files.exists(CommitLog.commitFile(dir, 3L))) + // One AddFile per identity per version; no conflicting same-version remove/add pair. + CommitLog.write(dir, 3L, Seq(refreshed)) + val logs = Files.list(dir.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + DeltaHarness.get.openLog(spark, dir.toString).checkpoint(3L) + val checkpoint = dir.resolve("_delta_log/00000000000000000003.checkpoint.parquet") + assert(Files.isRegularFile(checkpoint)) + val checkpointAdds = spark.read.parquet(checkpoint.toString) + .filter("add IS NOT NULL").select("add.*").toJSON.collect().toSeq + .map(json => Action.parse(s"""{"add":$json}""")) + assert(checkpointAdds == Seq(refreshed)) + val lastCheckpoint = JsonUtil.mapper.readTree( + Files.readAllBytes(dir.resolve("_delta_log/_last_checkpoint"))) + assert(lastCheckpoint.get("version").asLong() == 3L) + // Checkpoint creation can write fresh Hadoop sidecars; retain only protocol artifacts. + val checkpointLogs = Files.list(dir.resolve("_delta_log")) + try checkpointLogs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach { file => + assert(file.getFileName.toString.startsWith(".")) + Files.delete(file) + } + finally checkpointLogs.close() + // A transaction marker provides a genuine JSON tail without changing the row bag. + CommitLog.write(dir, 4L, Seq(Txn("readiness_e2e_51_tail", 1L))) + assert(CommitLog.read(dir, 3L) == Seq(refreshed)) + assert(CommitLog.read(dir, 4L) == Seq(Txn("readiness_e2e_51_tail", 1L))) + assert(Files.readAllBytes(dir.resolve(deleted.path)).sameElements(fileBytes)) + assert(payload(dir, dv).sameElements(dvBytes)) + DeltaLog.clearCache() + val replay = DeltaLog.forTable(spark, dir.toString).update() + assert(replay.version == 4L) + assert(replay.logSegment.checkpointProvider.version == 3L) + assert(replay.logSegment.deltas.map(_.getPath.getName) == Seq(f"${4L}%020d.json")) + (2L to 4L).foreach { version => + val snapshot = DeltaHarness.get.openLog(spark, dir.toString).getSnapshotAt(version) + val active = snapshot.allFiles.select("json").collect().toSeq + .map(r => Action.parse(r.getString(0))) + // Delta's active-file replay normalizes dataChange; commit flags were checked above. + val expectedAdd = (if (version == 2L) partial else refreshed).copy(dataChange = false) + assert(active == Seq(expectedAdd)) + } + val remaining = Files.list(dir.resolve("_delta_log")) + try assert(!remaining.iterator().asScala.exists(_.getFileName.toString.endsWith(".crc"))) + finally remaining.close() + info("replay checkpoint v3 + JSON tail v4 selected; physical file and DV unchanged") + } + val expectedSnapshot = SnapshotResult( + ProtocolInfo(3, 7, Some(Seq("deletionVectors")), + Some(Seq("appendOnly", "deletionVectors", "invariants"))), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](2L, 3L, 4L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + val resolved = Option(version).fold(4L)(_.toLong) + assert(spec.query == SnapshotQuery(version = Some(resolved))) + assert(spec.expectation == Succeeded(if (resolved == 4L) { + expectedSnapshot.copy(setTransactions = Some(Seq(AppTxn("readiness_e2e_51_tail", 1L)))) + } else expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val selective = "id IN (4, 12) OR id IS NULL" + val selected = Seq(Row(null, 4), Row(null, 4), Row("middle", 12), Row("missing", null)) + val versions = Seq("before" -> Some(2L), "refreshed" -> Some(3L), + "tail" -> Some(4L), "latest" -> None) + versions.foreach { case (label, version) => + val cases = Seq( + (s"read_$label", ReadQuery(version = version), schema, rows), + (s"read_${label}_selective", ReadQuery(version = version, + predicate = Some(selective), columns = Some(Seq("value", "id"))), projected, selected), + (s"read_${label}_miss", ReadQuery(version = version, predicate = Some("id > 100")), + schema, Seq.empty[Row])) + cases.foreach { case (name, query, expectedSchema, expectedRows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == expectedRows.size.toLong) + case other => fail(s"replay $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(expectedRows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"replay $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"replay $name expected parquet") + } + } + } + } + + Seq("zorder", "liquid").foreach { mode => + test(s"e2e_49_$mode") { + val liquid = mode == "liquid" + val clusterBy = if (liquid) "CLUSTER BY (id, value)" else "" + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta $clusterBy TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (8, 'late'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-2, 'early'), (5, 'middle'), (8, 'late'), (12, CAST(NULL AS STRING)) AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + // Independent literal bag: includes duplicate whole rows and nulls in both columns. + val firstRows = Seq(Row(8, "late"), Row(3, null), Row(3, null), Row(null, "missing")) + val allRows = firstRows ++ Seq(Row(-2, "early"), Row(5, "middle"), Row(8, "late"), + Row(12, null)) + val before = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = if (liquid) { + Protocol(1, 7, None, Some(Set("appendOnly", "invariants", "clustering", "domainMetadata"))) + } else Protocol(1, 2) + assert(before(0L).collect { case p: Protocol => p } == Seq(protocol)) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val domains = before(0L).collect { case d: DomainMetadata => d } + if (liquid) { + assert(domains.size == 1 && domains.head.domain == "delta.clustering") + assert(!domains.head.removed) + val configuration = JsonUtil.mapper.readTree(domains.head.configuration) + assert(configuration.get("clusteringColumns") == + JsonUtil.mapper.readTree("""[["id"],["value"]]""")) + } else assert(domains.isEmpty) + val literal = spark.createDataFrame(allRows.asJava, schema) + val unoptimized = SnapshotResolver.buildDeltaReader(spark, table.sourcePath, + Some(2L), None) + assert(unoptimized.schema == schema) + RowComparison.assertRowsEqual(literal, unoptimized, s"$mode before OPTIMIZE") + sql(if (liquid) "OPTIMIZE tbl" else "OPTIMIZE tbl ZORDER BY (id, value)") + val optimized = CommitLog.read(table.sourcePath, 3L) + val adds = optimized.collect { case a: AddFile => a } + val removes = optimized.collect { case r: RemoveFile => r } + // Check actual rewrite actions, without promising a file count or layout effectiveness. + assert(adds.nonEmpty && removes.nonEmpty) + assert(adds.forall(!_.dataChange) && removes.forall(!_.dataChange)) + val inputAdds = before.values.flatten.collect { case a: AddFile => a }.toSeq + assert(removes.map(_.path).toSet == inputAdds.map(_.path).toSet) + assert(adds.forall(_.clusteringProvider == (if (liquid) Some("liquid") else None))) + assert(optimized.collect { case m: Metadata => m }.isEmpty) + assert(optimized.collect { case p: Protocol => p }.isEmpty) + assert(optimized.collect { case d: DomainMetadata => d }.isEmpty) + val commitInfo = optimized.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "OPTIMIZE") + val parameters = commitInfo.get("operationParameters") + val columns = JsonUtil.mapper.readTree("""["id","value"]""") + val empty = JsonUtil.mapper.readTree("[]") + assert(JsonUtil.mapper.readTree(parameters.get("zOrderBy").asText()) == + (if (liquid) empty else columns)) + assert(JsonUtil.mapper.readTree(parameters.get("clusterBy").asText()) == + (if (liquid) columns else empty)) + info(s"cluster $mode protocol: " + protocol.toJson) + info(s"cluster $mode metadata: " + metadata.toJson) + info(s"cluster $mode optimize: " + commitInfo.toString) + domains.foreach(d => info(s"cluster $mode domain: " + d.toJson)) + adds.foreach(a => info(s"cluster $mode optimized add: " + a.toJson)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + (0L to 2L).foreach(v => assert(CommitLog.read(dir, v) == before(v))) + assert(CommitLog.read(dir, 3L) == optimized) + } + val expectedSnapshot = SnapshotResult( + ProtocolInfo(protocol.minReaderVersion, protocol.minWriterVersion, + writerFeatures = protocol.writerFeatures.map(_.toSeq.sorted)), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime), + domainMetadata = if (liquid) Some(domains.map(d => + AddDomainMetadata(d.domain, d.configuration))) else None) + Seq[java.lang.Long](0L, 2L, 3L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val predicate = "(id >= 3 AND id <= 8) OR id IS NULL" + val firstFiltered = Seq(Row("late", 8), Row(null, 3), Row(null, 3), Row("missing", null)) + val allFiltered = firstFiltered ++ Seq(Row("middle", 5), Row("late", 8)) + val cases = Seq( + ("read_latest", ReadQuery(), schema, allRows), + ("read_first_insert", ReadQuery(version = Some(1L)), schema, firstRows), + ("read_before_optimize", ReadQuery(version = Some(2L)), schema, allRows), + ("read_optimized_v3", ReadQuery(version = Some(3L)), schema, allRows), + ("read_first_filtered", ReadQuery(version = Some(1L), predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, firstFiltered), + ("read_latest_filtered", ReadQuery(predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, allFiltered), + ("read_optimized_filtered", ReadQuery(version = Some(3L), predicate = Some(predicate), + columns = Some(Seq("value", "id"))), projected, allFiltered)) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$mode $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"$mode $name copied table") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"$mode $name expected parquet") + } + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorPayloads.scala b/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorPayloads.scala new file mode 100644 index 00000000..8b50714a --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorPayloads.scala @@ -0,0 +1,61 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.io.DataInputStream +import java.nio.file.{Files, Path, Paths} + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore +import org.scalatest.Assertions._ + +import io.delta.workload.log.DeletionVector + +/** Reads the exact serialized bitmap of an inline or UUID-relative deletion vector. */ +object DeletionVectorPayloads { + + def descriptor(dv: DeletionVector): DeletionVectorDescriptor = { + // At this DAT pin USE_LONG_FOR_INTS boxes Option[Int] offsets as Long. Normalize + // only this descriptor input, without changing the shared mapper or DV bytes. + val offset = dv.offset.asInstanceOf[Option[Number]].map(n => Math.toIntExact(n.longValue())) + DeletionVectorDescriptor(dv.storageType, dv.pathOrInlineDv, offset, + dv.sizeInBytes, dv.cardinality) + } + + def sidecar(table: Path, dv: DeletionVector): Path = + Paths.get(descriptor(dv).absolutePath(new HadoopPath(table.toUri)).toUri.getPath) + + def payload(table: Path, dv: DeletionVector): Array[Byte] = { + if (dv.storageType == "i") descriptor(dv).inlineData + else { + assert(dv.storageType == "u") + val file = sidecar(table, dv) + assert(file.normalize().startsWith(table.normalize())) + val offset = descriptor(dv).offset.get + assert(offset > 0) + assert(Files.size(file) >= offset.toLong + 4L + dv.sizeInBytes + 4L) + val input = new DataInputStream(Files.newInputStream(file)) + try { + assert(input.readByte() == DeletionVectorStore.DV_FILE_FORMAT_VERSION_ID_V1) + input.skipNBytes(offset.toLong - 1L) + // This OSS reader checks both the payload length and checksum. + DeletionVectorStore.readRangeFromStream(input, dv.sizeInBytes) + } finally input.close() + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorsSuite.scala index d94a5b7f..7e79dc0e 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/DeletionVectorsSuite.scala @@ -16,7 +16,23 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.{Files, Path} + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{Action, AddFile, CommitLog, DeletionVector, Metadata, Protocol, + RemoveFile} +import io.delta.workload.model._ +import io.delta.workload.tables.DeletionVectorPayloads.{payload, sidecar} class DeletionVectorsSuite extends WorkloadTestSuite("deletion_vectors") { @@ -793,4 +809,174 @@ class DeletionVectorsSuite extends WorkloadTestSuite("deletion_vectors") { snapshotSpec(t) } + // Inline and UUID-relative deletion vectors with the same engine-written bitmap. + Seq("inline", "uuid_relative").foreach { form => + test(s"e2e_42_$form") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'remove'), (0, 'keep'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (9, 'remove'), (12, 'keep'), (18, CAST(NULL AS STRING)), (23, 'edge'), + (31, 'keep'), (44, CAST(NULL AS STRING)), (57, 'remove'), (88, 'last') AS v(id, value)""") + sql("DELETE FROM tbl WHERE id IN (-7, 9, 57)") + sql("DELETE FROM tbl WHERE id = 4") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val baseRows = Seq(Row(-7, "remove"), Row(0, "keep"), Row(4, null), Row(4, null), + Row(9, "remove"), Row(12, "keep"), Row(18, null), Row(23, "edge"), + Row(31, "keep"), Row(44, null), Row(57, "remove"), Row(88, "last")) + val firstRows = baseRows.filterNot(r => Set(-7, 9, 57).contains(r.getInt(0))) + val latestRows = firstRows.filterNot(_.getInt(0) == 4) + val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val protocol = original(0L).collect { case p: Protocol => p }.ensuring(_.size == 1).head + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + assert(protocol == Protocol(3, 7, Some(Set("deletionVectors")), + Some(Set("appendOnly", "deletionVectors", "invariants")))) + info("DV original protocol: " + protocol.toJson) + info("DV original metadata: " + metadata.toJson) + val adds = (1L to 3L).map { version => + val found = original(version).collect { case a: AddFile => a } + assert(found.size == 1) + info(s"DV original add v$version: " + found.head.toJson) + version -> found.head + }.toMap + assert(adds.values.map(_.path).toSet.size == 1) // Deletes must retain the data file. + assert(adds(1L).deletionVector.isEmpty) + val physical = spark.read.parquet(table.sourcePath.resolve(adds(1L).path).toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(baseRows.asJava, schema), physical, + s"$form physical input") + val physicalRows = physical.collect().toSeq + val sourceDvs = (2L to 3L).map(v => v -> adds(v).deletionVector.get).toMap + sourceDvs.foreach { case (version, dv) => + assert(dv.storageType == "u") + assert(dv.cardinality == (if (version == 2L) 3L else 5L)) + val expectedPositions = physicalRows.zipWithIndex.collect { + case (row, index) if Set(-7, 9, 57).contains(row.getInt(0)) || + (version == 3L && row.getInt(0) == 4) => index.toLong + } + assert(RoaringBitmapArray.readFrom(payload(table.sourcePath, dv)).toArray.toSeq == + expectedPositions) + } + val converted = sourceDvs.values.map { dv => + val bytes = payload(table.sourcePath, dv) + // Use the verified OSS helper with exact engine-generated payload bytes. + val inline = DeletionVectorDescriptor.inlineInLog(bytes, dv.cardinality) + assert(inline.storageType == "i" && inline.offset.isEmpty) + assert(inline.sizeInBytes == bytes.length && inline.cardinality == dv.cardinality) + assert(inline.inlineData.sameElements(bytes)) + dv -> DeletionVector(inline.storageType, inline.pathOrInlineDv, inline.offset, + inline.sizeInBytes, inline.cardinality) + }.toMap + if (form == "inline") { + (2L to 3L).foreach { version => + modifyCommitActions(table, version) { actions => + actions.map { + case add: AddFile => add.copy(deletionVector = add.deletionVector.map(converted)) + // The second delete removes the first DV identity; convert that tombstone too. + case remove: RemoveFile => + remove.copy(deletionVector = remove.deletionVector.map(converted)) + case other => other + } + } + } + } + val expectedAdds = adds.map { case (version, add) => + version -> (if (form == "inline") { + add.copy(deletionVector = add.deletionVector.map(converted)) + } else add) + } + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + val sidecars = sourceDvs.values.map(dv => sidecar(dir, dv)).toSet + if (form == "inline") sidecars.foreach(Files.delete) + else sidecars.foreach(p => assert(Files.isRegularFile(p))) + // Version checksums embed the original descriptors and become stale after conversion. + val logs = Files.list(dir.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + (1L to 3L).foreach { version => + val actions = CommitLog.read(dir, version) + assert(actions.collect { case a: AddFile => a } == Seq(expectedAdds(version))) + expectedAdds(version).deletionVector.foreach { dv => + assert(dv.storageType == (if (form == "inline") "i" else "u")) + assert(dv.cardinality == (if (version == 2L) 3L else 5L)) + assert(payload(dir, dv).sameElements(payload(table.sourcePath, sourceDvs(version)))) + assert(RoaringBitmapArray.readFrom(payload(dir, dv)).cardinality == dv.cardinality) + } + } + val oldIdentity = CommitLog.read(dir, 3L).collect { case r: RemoveFile => r } + assert(oldIdentity.size == 1) + assert(oldIdentity.head.path == adds(1L).path) + assert(oldIdentity.head.deletionVector == expectedAdds(2L).deletionVector) + assert(Files.size(dir.resolve(adds(1L).path)) == adds(1L).size) + if (form == "inline") assert(sidecars.forall(p => !Files.exists(p))) + } + val snapshot = SnapshotResult( + ProtocolInfo(3, 7, Some(Seq("deletionVectors")), + Some(Seq("appendOnly", "deletionVectors", "invariants"))), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), metadata.schemaString, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](1L, 2L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(snapshot)) + } + } + val projected = new StructType().add("value", StringType) + val reordered = projected.add("id", IntegerType) + val filter = "id IN (4, 12, 18, 57)" + val cases = Seq( + ("read_base", ReadQuery(version = Some(1L)), schema, baseRows), + ("read_first_delete", ReadQuery(version = Some(2L)), schema, firstRows), + ("read_latest", ReadQuery(), schema, latestRows), + ("read_v2_filtered_projected", ReadQuery(version = Some(2L), + predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered, + Seq(Row(null, 4), Row(null, 4), Row("keep", 12), Row(null, 18))), + ("read_v3_filtered_projected", ReadQuery(version = Some(3L), + predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered, + Seq(Row("keep", 12), Row(null, 18))), + ("read_projected", ReadQuery(columns = Some(Seq("value"))), projected, + Seq(Row("keep"), Row("keep"), Row(null), Row("edge"), Row("keep"), Row(null), + Row("last"))), + ("read_deleted_only", ReadQuery(predicate = Some("id IN (-7, 4, 9, 57)")), schema, + Seq.empty[Row])) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + // All predicates overlap physical min/max; the single data AddFile remains scanned. + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, 1, 0L))) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, expectedSchema), actual, + s"$form $name") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_metadata").toString) + val scanned = captured.collect().toSeq.map { row => + val node = JsonUtil.mapper.readTree(row.getString(0)) + val wrapped = if (node.has("add")) node.toString else s"""{"add":$node}""" + Action.parse(wrapped) + } + // OSS InMemoryLogReplay stores active AddFiles with dataChange=false; + // ReadCapture obtains these snapshot actions through DeltaSparkHarness.allFiles. + assert(scanned == Seq(expectedAdds(query.version.getOrElse(3L)) + .copy(dataChange = false))) + } + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/LiteralOracleChecks.scala b/workload-generator/src/test/scala/io/delta/workload/tables/LiteralOracleChecks.scala new file mode 100644 index 00000000..7d39bf5f --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/LiteralOracleChecks.scala @@ -0,0 +1,89 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, StructType} + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.model.Succeeded + +/** + * Read and snapshot checks against literal rows and an independent schema, shared by the + * temporal statistics and partition-value workloads. Row oracles are never read from Delta. + */ +trait LiteralOracleChecks { this: WorkloadTestSuite => + + protected def outputDir: Path + + /** + * Declares a read whose captured rows must equal the literal `rows`. The `all` oracle is + * also written to `oracle-parquet/` next to the output directory, for physical + * Parquet checks in later assertions. + */ + protected def checkLiteralRead( + table: TableHandle, name: String, expectedSchema: StructType, rows: Seq[Row], + predicate: String = null, version: java.lang.Long = null): Unit = { + val workload = WorkloadContext.current.workloadName + readSpec(table, predicate = predicate, version = version, name = Some(name)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == rows.size) + case other => fail(s"$name must succeed against the independent oracle: $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$name").resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name) + if (name == "all") { + expected.coalesce(1).write.mode("overwrite") + .parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString) + } + println(s"Verified independent literal rows/schema: $workload/$name rows=${rows.size}") + } + } + + protected def checkLiteralSnapshot( + table: TableHandle, expectedSchema: StructType, partitions: Seq[String] = Seq.empty, + version: java.lang.Long = null): Unit = { + snapshotSpec(table, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == partitions) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + val ntz = expectedSchema.fields.exists(_.dataType == + org.apache.spark.sql.types.TimestampNTZType) + require(result.protocol.minReaderVersion == (if (ntz) 3 else 1)) + require(result.protocol.minWriterVersion == (if (ntz) 7 else 2)) + case other => fail(s"Snapshot must succeed against independent metadata: $other") + } + } + } + + /** Removes version checksums, which can carry file actions from before a mutation. */ + protected def removeVersionChecksums(dir: Path): Unit = { + val files = Files.list(dir.resolve("_delta_log")) + try files.iterator().asScala.filter(_.toString.endsWith(".crc")) + .foreach(Files.delete(_)) finally files.close() + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/LogReplaySuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/LogReplaySuite.scala index ae228096..d3397f15 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/LogReplaySuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/LogReplaySuite.scala @@ -16,7 +16,20 @@ package io.delta.workload.tables +import java.nio.file.{Files, Path} + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + import io.delta.workload.WorkloadTestSuite +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{Action, AddFile, CommitLog, Metadata, Protocol, RemoveFile} +import io.delta.workload.model._ /** * Log Replay + Production Edge Cases workloads (lr_* + log_* + lc_* + prod_*). @@ -24,7 +37,8 @@ import io.delta.workload.WorkloadTestSuite * checkpoint, metadata latest wins, dataChange=false from compaction, missing * protocol/metadata errors, add-remove-readd, DV key dedup, last checkpoint info, * production edge cases (empty table, many commits, version gaps, truncated logs, - * unknown reader features, varchar metadata, duplicate file refs, external checkpoints). + * unknown reader features, varchar metadata, duplicate file refs, external checkpoints), + * minor log compaction and automatic compaction. * */ class LogReplaySuite extends WorkloadTestSuite("log_replay") { @@ -263,4 +277,297 @@ class LogReplaySuite extends WorkloadTestSuite("log_replay") { readSpec(t) } + // Minor log compaction and automatic compaction. + test("e2e_56_minor_log_compaction") { + import org.apache.spark.sql.delta.actions.{Action => SparkAction, InMemoryLogReplay} + import org.apache.spark.sql.delta.util.FileNames + + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'initial'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing'), (12, 'keep'), (12, 'keep') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (6, 'replacement'), (6, 'replacement'), (CAST(NULL AS INT), 'changed'), + (8, CAST(NULL AS STRING)) AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), (23, 'end') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (31, 'tail'), (31, 'tail'), (CAST(NULL AS INT), 'tail-null'), + (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)) AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (44, 'last'), (44, 'last') AS v(id, value)""") + val table = registerTable("tbl") + val original = (0L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(original(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val adds = (1L to 5L).map { v => + v -> original(v).collect { case a: AddFile => a }.ensuring(_.size == 1).head + }.toMap + assert(adds.values.map(_.path).toSet.size == 5) + assert(original(2L).collect { case r: RemoveFile => r.path } == Seq(adds(1L).path)) + assert(original(4L).collect { case r: RemoveFile => r.path }.toSet == + Set(adds(2L).path, adds(3L).path)) + + // Follow the pinned OSS DeltaLogMinorCompactionSuite: reconcile [0, 3], with + // no retention pruning, then serialize the existing OSS checkpoint iterator. + // Concatenating commits would retain cancelled adds and commitInfo, which is invalid. + val replay = new InMemoryLogReplay(None, None) + (0L to 3L).foreach { v => + val lines = Files.readAllLines(table.sourcePath.resolve(f"_delta_log/$v%020d.json")) + replay.append(v, lines.asScala.iterator.map(SparkAction.fromJson)) + } + val sourceLog = DeltaLog.forTable(spark, table.sourcePath.toString) + val compactedPath = FileNames.compactedDeltaFile(sourceLog.logPath, 0L, 3L) + val compactedName = "00000000000000000000.00000000000000000003.compacted.json" + assert(compactedPath.getName == compactedName) + sourceLog.store.write(compactedPath, replay.checkpoint.map(_.json), + overwrite = false, sourceLog.newDeltaHadoopConf()) + val compactedBytes = Files.readAllBytes(table.sourcePath.resolve(s"_delta_log/$compactedName")) + val compacted = Files.readAllLines(table.sourcePath.resolve(s"_delta_log/$compactedName")) + .asScala.map(Action.parse).toSeq + assert(compacted.size == 5) + assert(compacted.collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(compacted.collect { case m: Metadata => m } == Seq(metadata)) + // OSS Action.fromJson materializes absent optional tags as an empty map. + // FileAction.tagsOrEmpty defines their equivalence; preserve every nonempty tag. + val expectedAdds = Set(adds(2L), adds(3L)).map(a => + a.copy(dataChange = false, tags = a.tags.filter(_.nonEmpty))) + assert(compacted.collect { case a: AddFile => a.copy(tags = a.tags.filter(_.nonEmpty)) } + .toSet == expectedAdds) + assert(compacted.collect { case r: RemoveFile => r.copy(tags = r.tags.filter(_.nonEmpty)) } == + original(2L).collect { case r: RemoveFile => + r.copy(dataChange = false, tags = r.tags.filter(_.nonEmpty)) }) + info("compact compacted [0,3] actions: " + compacted.map(_.toJson).mkString("\n")) + + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + assert(Files.readAllBytes(dir.resolve(s"_delta_log/$compactedName")) + .sameElements(compactedBytes)) + // Compactions are optional accelerators. OSS validates original commit continuity; + // neither compaction nor protocol cleanup authorizes removing this interval. + (0L to 5L).foreach { v => + assert(CommitLog.read(dir, v) == original(v)) + assert(Files.readAllBytes(dir.resolve(f"_delta_log/$v%020d.json")) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(f"_delta_log/$v%020d.json")))) + } + adds.values.foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + val listing = Files.list(dir.resolve("_delta_log")) + try assert(!listing.iterator().asScala.exists(_.getFileName.toString.contains("checkpoint"))) + finally listing.close() + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + Seq[java.lang.Long](0L, 1L, 2L, 3L, 4L, 5L, null).foreach { version => + val v = Option(version).fold(5L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + val selected = snapshot.logSegment.deltas.map(_.getPath.getName) + val expected = if (v < 3L) (0L to v).map(n => f"$n%020d.json") + else Seq(compactedName) ++ (4L to v).map(n => f"$n%020d.json") + assert(selected == expected) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + info(s"compact v$v selected log files: " + selected.mkString(",")) + } + } + val initialRows = Seq(Row(-7, "initial"), Row(4, null), Row(4, null), + Row(null, "missing"), Row(12, "keep"), Row(12, "keep")) + val replacementRows = Seq(Row(6, "replacement"), Row(6, "replacement"), + Row(null, "changed"), Row(8, null)) + val endRows = replacementRows ++ Seq(Row(4, null), Row(4, null), Row(23, "end")) + val tailRows = Seq(Row(31, "tail"), Row(31, "tail"), Row(null, "tail-null"), + Row(4, null), Row(4, null)) + val latestRows = tailRows ++ Seq(Row(44, "last"), Row(44, "last")) + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_start_v0", ReadQuery(version = Some(0L)), schema, Seq.empty[Row]), + ("read_inside_v1", ReadQuery(version = Some(1L)), schema, initialRows), + ("read_inside_v2", ReadQuery(version = Some(2L)), schema, replacementRows), + ("read_end_v3", ReadQuery(version = Some(3L)), schema, endRows), + ("read_tail_v4", ReadQuery(version = Some(4L)), schema, tailRows), + ("read_tail_v5", ReadQuery(version = Some(5L)), schema, latestRows), + ("read_latest", ReadQuery(), schema, latestRows), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("tail-null", null)))) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"compact $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"compact $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"compact $name expected parquet") + } + } + } + test("e2e_57_auto_compaction") { + // These settings affect fixture generation only. Ordinary SQL inserts invoke + // OSS AutoCompact's post-commit hook; no explicit compaction command is called. + val settings = Map( + "spark.databricks.delta.autoCompact.enabled" -> "true", + "spark.databricks.delta.autoCompact.minNumFiles" -> "3", + "spark.databricks.delta.autoCompact.minFileSize" -> "1048576", + "spark.databricks.delta.autoCompact.maxFileSize" -> "2097152") + val saved = settings.keys.map(k => k -> spark.conf.getOption(k)).toMap + try { + settings.foreach { case (key, value) => sql(s"SET $key=$value") } + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.autoOptimize.autoCompact' = 'true', 'delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '1000', 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'initial'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (12, 'keep'), (12, 'keep'), (23, 'end') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (44, 'last'), (44, 'last'), (CAST(NULL AS INT), CAST(NULL AS STRING)), + (88, 'edge') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (99, 'tail'), (99, 'tail') AS v(id, value)""") + } finally { + saved.foreach { + case (key, Some(value)) => spark.conf.set(key, value) + case (key, None) => spark.conf.unset(key) + } + } + val table = registerTable("tbl") + val commits = (0L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + assert(!Files.exists(CommitLog.commitFile(table.sourcePath, 6L))) + val inputAdds = (1L to 3L).flatMap { v => + val adds = commits(v).collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.dataChange) + assert(adds.head.size > 0L && adds.head.size < 1048576L) + assert(commits(v).collect { case r: RemoveFile => r }.isEmpty) + adds + } + val rewritten = commits(4L).collect { case a: AddFile => a } + val removed = commits(4L).collect { case r: RemoveFile => r } + assert(rewritten.size == 1 && !rewritten.head.dataChange) + assert(removed.size == 3 && removed.forall(r => !r.dataChange)) + assert(removed.map(_.path).toSet == inputAdds.map(_.path).toSet) + assert(!inputAdds.exists(_.path == rewritten.head.path)) + val tail = commits(5L).collect { case a: AddFile => a } + assert(tail.size == 1 && tail.head.dataChange) + assert(commits(5L).collect { case r: RemoveFile => r }.isEmpty) + val history = (0L to 5L).map { v => + val info = commits(v).map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + v -> info + }.toMap + assert(history(0L).get("operation").asText() == "CREATE TABLE") + Seq(1L, 2L, 3L, 5L).foreach { v => + assert(history(v).get("operation").asText() == "WRITE") + } + assert(history(4L).get("operation").asText() == "OPTIMIZE") + assert(history(4L).get("operationParameters").get("auto").asText() == "true") + info("auto automatic compaction commit v4: " + history(4L).toString) + + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val metadata = commits(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val configuration = Map("delta.autoOptimize.autoCompact" -> "true", + "delta.enableDeletionVectors" -> "false", "delta.checkpointInterval" -> "1000", + "delta.checkpointPolicy" -> "classic") + assert(commits(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + (0L to 5L).foreach { v => + assert(CommitLog.read(dir, v) == commits(v)) + assert(Files.readAllBytes(CommitLog.commitFile(dir, v)) + .sameElements(Files.readAllBytes(CommitLog.commitFile(table.sourcePath, v)))) + } + (inputAdds ++ rewritten ++ tail).foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + val logs = Files.list(dir.resolve("_delta_log")) + try assert(!logs.iterator().asScala.exists(_.getFileName.toString.contains("checkpoint"))) + finally logs.close() + } + val activePaths = Map(0L -> Set.empty[String], 1L -> Set(inputAdds(0).path), + 2L -> inputAdds.take(2).map(_.path).toSet, 3L -> inputAdds.map(_.path).toSet, + 4L -> Set(rewritten.head.path), 5L -> Set(rewritten.head.path, tail.head.path)) + Seq[java.lang.Long](0L, 1L, 2L, 3L, 4L, 5L, null).foreach { version => + val v = Option(version).fold(5L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + assert(snapshot.allFiles.collect().map(_.path).toSet == activePaths(v)) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + } + } + // Literal, duplicate-sensitive bags are independent of captured expectations. + val firstRows = Seq(Row(-7, "initial"), Row(4, null), Row(4, null), Row(null, "missing")) + val secondRows = firstRows ++ Seq(Row(12, "keep"), Row(12, "keep"), Row(23, "end")) + val beforeRows = secondRows ++ Seq(Row(44, "last"), Row(44, "last"), + Row(null, null), Row(88, "edge")) + val latestRows = beforeRows ++ Seq(Row(99, "tail"), Row(99, "tail")) + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_first_v1", ReadQuery(version = Some(1L)), schema, firstRows, 1, 0L), + ("read_below_threshold_v2", ReadQuery(version = Some(2L)), schema, secondRows, 2, 0L), + ("read_before_v3", ReadQuery(version = Some(3L)), schema, beforeRows, 3, 0L), + ("read_post_v4", ReadQuery(version = Some(4L)), schema, beforeRows, 1, 0L), + ("read_tail_v5", ReadQuery(version = Some(5L)), schema, latestRows, 2, 0L), + ("read_latest", ReadQuery(), schema, latestRows, 2, 0L), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("missing", null), Row(null, null)), 1, 1L)) + cases.foreach { case (name, query, expectedSchema, rows, fileCount, skipped) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, fileCount, skipped))) + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"auto $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"auto $name expected parquet") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/PartitionValuesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/PartitionValuesSuite.scala index 40d3410d..e8adcb57 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/PartitionValuesSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/PartitionValuesSuite.scala @@ -16,9 +16,29 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.Files +import java.sql.Timestamp +import java.time.Instant -class PartitionValuesSuite extends WorkloadTestSuite("partition_values") { +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.parquet.example.data.Group +import org.apache.parquet.hadoop.{ParquetFileReader, ParquetReader} +import org.apache.parquet.hadoop.example.GroupReadSupport +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.parquet.schema.{LogicalTypeAnnotation, PrimitiveType} +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType, TimestampType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog} +import io.delta.workload.model.Succeeded + +class PartitionValuesSuite extends WorkloadTestSuite("partition_values") + with ScopedSessionConf with LiteralOracleChecks { test("boolean") { sql("""CREATE TABLE tbl (id INT, flag BOOLEAN) USING delta @@ -169,4 +189,432 @@ class PartitionValuesSuite extends WorkloadTestSuite("partition_values") { snapshotSpec(t) } + // Partition values come from AddFile metadata, not from data file directories. + test("metadata_partition_values_ignore_directories") { + sql("""CREATE TABLE tbl (id INT, payload STRING, part STRING) USING delta + PARTITIONED BY (part) TBLPROPERTIES ('delta.checkpointInterval' = '1000')""") + sql("""INSERT INTO tbl VALUES + (1, 'duplicate', 'east'), (1, 'duplicate', 'east'), + (2, 'conflicting', 'west'), (3, 'null_partition', NULL), + (4, 'no_hive_directory', 'plain')""") + val t = registerTable("tbl") + val table = WorkloadContext.current.getTableSpec(t) + val schema = new StructType().add("id", IntegerType).add("payload", StringType) + .add("part", StringType) + + mutateTable(t) { tableDir => + val originals = CommitLog.read(tableDir, 1L).collect { case add: AddFile => add } + assert(originals.nonEmpty) + assert(originals.map(_.partitionValues("part")).toSet == Set("east", "west", null, "plain")) + CommitLog.mutate(tableDir, 1L) { actions => + actions.map { + case add: AddFile => + val directory = if (add.partitionValues("part") == "plain") "opaque/nested" + else "part=directory_lie" + val original = tableDir.resolve(add.path) + val relative = s"$directory/${original.getFileName}" + val target = tableDir.resolve(relative) + Files.createDirectories(target.getParent) + Files.move(original, target) + assert(!Files.exists(original) && Files.isRegularFile(target)) + assert(Files.size(target) == add.size) + add.copy(path = relative) + case other => other + } + } + val moved = CommitLog.read(tableDir, 1L).collect { case add: AddFile => add } + assert(moved.map(_.partitionValues) == originals.map(_.partitionValues)) + assert(moved.zip(originals).forall { case (after, before) => + after.copy(path = before.path) == before + }) + assert(moved.forall(add => Files.isRegularFile(tableDir.resolve(add.path)))) + assert(moved.forall(add => !originals.exists(_.path == add.path))) + // Require JSON replay: no checkpoint or Delta CRC may retain the original file paths. + val stream = Files.list(tableDir.resolve("_delta_log")) + val files = try stream.iterator().asScala.toSeq finally stream.close() + assert(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + println("Metadata-partition layout oracle passed: conflicting and non-Hive directories") + } + + val rows = Seq(Row(1, "duplicate", "east"), Row(1, "duplicate", "east"), + Row(2, "conflicting", "west"), Row(3, "null_partition", null), + Row(4, "no_hive_directory", "plain")) + val reads = Seq( + ("read_all", null, rows.indices, None), + ("read_metadata_east", "part = 'east'", Seq(0, 1), None), + ("read_metadata_west", "part = 'west'", Seq(2), None), + ("read_directory_lie", "part = 'directory_lie'", Seq.empty[Int], None), + ("read_null", "part IS NULL", Seq(3), None), + ("read_no_hive_directory", "part = 'plain'", Seq(4), None), + ("read_projection", null, rows.indices, Some(Seq("part", "id"))), + ("read_null_projection", "part IS NULL", Seq(3), Some(Seq("part", "payload")))) + for ((name, predicate, indices, columns) <- reads) { + readSpec(t, name = Some(name), predicate = predicate, columns = columns).assert { spec => + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == indices.size.toLong) + case other => fail(s"metadata partition/$name must succeed: $other") + } + val literal = spark.createDataFrame(indices.map(rows).asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val expected = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expected.toString) + RowComparison.assertRowsEqual(oracle, captured, s"metadata partition/$name literal bag") + assert(captured.schema == oracle.schema) + println(s"Metadata-partition literal row/schema oracle passed: $name") + } + } + snapshotSpec(t).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(DataType.fromJson(result.metadata.schemaString) == schema) + assert(result.metadata.partitionColumns == Seq("part")) + assert(result.metadata.configuration == Map("delta.checkpointInterval" -> "1000")) + case other => fail(s"metadata partition snapshot must succeed: $other") + } + println("Metadata-partition snapshot schema/partition oracle passed") + } + } + + // Literal AddFile partition strings: empty values, timestamp serializations, and + // canonical DATE calendar bounds. + // Literal empty AddFile partition strings are independent of writer normalization. + // BYTE is numeric TINYINT. Its empty input has no numeric value and is expected to be null. + // STRING and BINARY retain a distinct empty value; writer support was probed separately. + for ((kind, sqlType, valueType, firstValue, secondValue, control, emptyValue) <- Seq( + ("byte", "TINYINT", org.apache.spark.sql.types.ByteType, + "CAST(1 AS TINYINT)", "CAST(2 AS TINYINT)", 2.toByte, null), + ("short", "SMALLINT", org.apache.spark.sql.types.ShortType, + "CAST(1 AS SMALLINT)", "CAST(2 AS SMALLINT)", 2.toShort, null), + ("int", "INT", IntegerType, "1", "2", 2, null), + ("long", "BIGINT", org.apache.spark.sql.types.LongType, "1L", "2L", 2L, null), + ("float", "FLOAT", org.apache.spark.sql.types.FloatType, + "CAST(1.25 AS FLOAT)", "CAST(2.5 AS FLOAT)", 2.5f, null), + ("double", "DOUBLE", org.apache.spark.sql.types.DoubleType, + "CAST(1.25 AS DOUBLE)", "CAST(2.5 AS DOUBLE)", 2.5d, null), + ("decimal", "DECIMAL(10,2)", org.apache.spark.sql.types.DecimalType(10, 2), + "CAST(1.25 AS DECIMAL(10,2))", "CAST(2.50 AS DECIMAL(10,2))", + new java.math.BigDecimal("2.50"), null), + ("boolean", "BOOLEAN", org.apache.spark.sql.types.BooleanType, + "false", "true", true, null), + ("date", "DATE", org.apache.spark.sql.types.DateType, + "DATE '1969-12-31'", "DATE '2000-02-29'", java.sql.Date.valueOf("2000-02-29"), null), + ("timestamp", "TIMESTAMP", TimestampType, + "TIMESTAMP '1969-12-31T23:59:59Z'", "TIMESTAMP '2024-03-10T10:00:00.123456Z'", + Timestamp.from(Instant.parse("2024-03-10T10:00:00.123456Z")), null), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType, + "TIMESTAMP_NTZ '1969-12-31 23:59:59'", "TIMESTAMP_NTZ '2024-03-10 02:30:00.123456'", + java.time.LocalDateTime.parse("2024-03-10T02:30:00.123456"), null), + ("string", "STRING", org.apache.spark.sql.types.StringType, + "'first'", "'second'", "second", ""), + ("binary", "BINARY", org.apache.spark.sql.types.BinaryType, + "unhex('61')", "unhex('62')", Array[Byte](98), Array.emptyByteArray))) { + test(s"empty_partition_$kind") { + setForTest("spark.sql.session.timeZone", "UTC") + // ANSI predicate casts reject empty numeric input. Leave legacy casts active for capture. + // The existing spec format cannot transport this session setting to an external consumer. + setForTest("spark.sql.ansi.enabled", "false") + val expectedSchema = new StructType().add("id", IntegerType).add("p", valueType) + val rows = Seq(Row(1, emptyValue), Row(2, control), Row(2, control), Row(3, null)) + sql(s"CREATE TABLE tbl (id INT, p $sqlType) USING delta PARTITIONED BY (p)") + sql(s"INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + s"(1, $firstValue), (2, $secondValue), (2, $secondValue), " + + s"(3, CAST(NULL AS $sqlType)) AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile if JsonUtil.mapper.readTree(add.stats.get) + .at("/minValues/id").intValue() == 1 => + add.copy(partitionValues = Map("p" -> "")) + case other => other + } + } + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 3) + require(adds.count(_.partitionValues("p") == "") == 1) + require(adds.count(_.partitionValues("p") == null) == 1) + println(s"Verified serialized empty partition: $kind empty=1 null=1 control=1") + } + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + checkLiteralRead(table, "is_null", expectedSchema, rows.filter(_.isNullAt(1)), + predicate = "p IS NULL") + checkLiteralRead(table, "is_not_null", expectedSchema, rows.filterNot(_.isNullAt(1)), + predicate = "p IS NOT NULL") + checkLiteralRead(table, "eq_control", expectedSchema, rows.filter(_.getInt(0) == 2), + predicate = s"p = $secondValue") + if (kind == "string" || kind == "binary") { + val emptyLiteral = if (kind == "string") "''" else "unhex('')" + checkLiteralRead(table, "eq_empty", expectedSchema, rows.filter(_.getInt(0) == 1), + predicate = s"p = $emptyLiteral") + } + snapshotSpec(table).assert { spec => + require(spark.conf.get("spark.sql.ansi.enabled") == "false") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == Seq("p")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.protocol.minReaderVersion == (if (kind == "timestamp_ntz") 3 else 1)) + require(result.protocol.minWriterVersion == (if (kind == "timestamp_ntz") 7 else 2)) + case other => fail(s"Empty partition snapshot must succeed: $other") + } + println(s"Verified capture mode: ${WorkloadContext.current.workloadName} ansi=false") + } + checkLiteralSnapshot(table, expectedSchema, Seq("p"), version = 1L) + } + } + + // Inputs are literal AddFile strings, not SQL timestamp literal spellings. + // The pinned Spark parser permits zone suffixes for NTZ casts and ignores their offsets; + // expected NTZ wall clocks below are separate from the corresponding absolute instants. + private def timestampPartitionFormats: Vector[(String, String, String, String)] = { + val fractions = Seq( + ("0", "", "000000"), ("1", ".1", "100000"), ("2", ".12", "120000"), + ("3", ".123", "123000"), ("4", ".1234", "123400"), + ("5", ".12345", "123450"), ("6", ".123456", "123456"), + ("7", ".1234567", "123456"), ("8", ".12345678", "123456"), + ("9", ".123456789", "123456"), ("dot", ".", "000000"), + ("10", ".1234567891", "123456"), ("18", ".123456789123456789", "123456")) + val zones = Seq( + ("z", "12:34:56.123456Z", "12:34:56.123456"), + ("zero_colon", "12:34:56.123456+00:00", "12:34:56.123456"), + ("zero_basic", "12:34:56.123456+0000", "12:34:56.123456"), + ("zero_hour", "12:34:56.123456+00", "12:34:56.123456"), + ("minus_zero", "12:34:56.123456-00:00", "12:34:56.123456"), + ("utc_name", "12:34:56.123456 UTC", "12:34:56.123456"), + ("gmt_name", "12:34:56.123456 GMT", "12:34:56.123456"), + ("east", "18:04:56.123456+05:30", "18:04:56.123456"), + ("west", "04:34:56.123456-08:00", "04:34:56.123456"), + ("region", "04:34:56.123456 America/Los_Angeles", "04:34:56.123456"), + ("east_basic", "18:04:56.123456+0530", "18:04:56.123456"), + ("east_hour", "17:34:56.123456+05", "17:34:56.123456"), + ("west_basic", "04:34:56.123456-0800", "04:34:56.123456"), + ("west_hour", "04:34:56.123456-08", "04:34:56.123456"), + ("west_half", "09:04:56.123456-03:30", "09:04:56.123456"), + ("single_hour", "18:04:56.123456+5:30", "18:04:56.123456"), + ("single_minute", "17:37:56.123456+05:3", "17:37:56.123456"), + ("offset_seconds", "18:05:26.123456+05:30:30", "18:05:26.123456")) + (for ((separatorName, separator) <- Seq("space" -> " ", "T" -> "T"); + entry <- fractions.map { case (label, fraction, micros) => + (s"${separatorName}_fraction_$label", s"2024-02-29${separator}12:34:56$fraction", + s"2024-02-29T12:34:56.$micros" + "Z", s"2024-02-29T12:34:56.$micros") + } ++ zones.map { case (label, raw, wall) => + (s"${separatorName}_$label", s"2024-02-29$separator$raw", + "2024-02-29T12:34:56.123456Z", s"2024-02-29T$wall") + }) yield entry).toVector + } + + for ((kind, sqlType, valueType) <- Seq( + ("timestamp", "TIMESTAMP", TimestampType), + ("timestamp_ntz", "TIMESTAMP_NTZ", org.apache.spark.sql.types.TimestampNTZType))) { + test(s"partition_serializations_$kind") { + setForTest("spark.sql.session.timeZone", "UTC") + setForTest("spark.sql.ansi.enabled", "false") + val formats = timestampPartitionFormats + require(formats.size == 62) + val expectedSchema = new StructType().add("id", IntegerType).add("p", valueType) + val keys = formats.zipWithIndex.map { case ((_, _, instant, wall), index) => + (index + 1) -> (if (kind == "timestamp") instant else wall) + }.toMap + val rows0 = keys.toSeq.sortBy(_._1).map { case (id, key) => + val value = if (kind == "timestamp") Timestamp.from(Instant.parse(key)) + else java.time.LocalDateTime.parse(key) + Row(id, value) + } + val rows = rows0 :+ rows0.head + sql(s"CREATE TABLE tbl (id INT, p $sqlType) USING delta PARTITIONED BY (p)") + // Distinct seed partitions create one data file per raw representation. Only id is + // physically stored; the typed AddFile mutation supplies the independently checked p. + val seeds = formats.indices.map { index => + val seed = Instant.parse("2030-01-01T00:00:00Z").plusSeconds(index.toLong).toString + val literal = if (kind == "timestamp") seed else seed.stripSuffix("Z") + s"(${index + 1}, $sqlType '$literal')" + } + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + (seeds :+ seeds.head).mkString(", ") + " AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val id = JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + add.copy(partitionValues = Map("p" -> formats(id - 1)._2)) + case other => other + } + } + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == formats.size) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + require(stats.at("/maxValues/id").intValue() == id) + require(stats.get("numRecords").asLong() == (if (id == 1) 2L else 1L)) + require(add.partitionValues == Map("p" -> formats(id - 1)._2)) + println(s"Verified raw serialization: $kind ${formats(id - 1)._1} " + + s"value=${formats(id - 1)._2}") + } + } + checkLiteralRead(table, "all", expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + keys.values.toSet.toSeq.sorted.zipWithIndex.foreach { case (key, index) => + val ids = keys.collect { case (id, value) if value == key => id }.toSet + checkLiteralRead(table, s"eq_$index", expectedSchema, + rows.filter(r => ids.contains(r.getInt(0))), predicate = s"p = $sqlType '$key'") + } + def selected(lower: String, upper: String): Seq[Row] = rows.filter { row => + val key = keys(row.getInt(0)) + key >= lower && key < upper + } + checkLiteralRead(table, "fraction_range", expectedSchema, + selected("2024-02-29T12:34:56.100000", "2024-02-29T12:34:56.123456"), + predicate = s"p >= $sqlType '2024-02-29 12:34:56.100000' AND " + + s"p < $sqlType '2024-02-29 12:34:56.123456'") + checkLiteralRead(table, "before_noon", expectedSchema, + selected("2024-02-29T00:00:00", "2024-02-29T12:00:00"), + predicate = s"p < $sqlType '2024-02-29 12:00:00'") + checkLiteralRead(table, "miss", expectedSchema, Seq.empty, + predicate = s"p < $sqlType '2024-02-29 00:00:00'") + checkLiteralSnapshot(table, expectedSchema, Seq("p")) + checkLiteralSnapshot(table, expectedSchema, Seq("p"), version = 1L) + } + } + + // Canonical zero-padded DATE strings use proleptic Gregorian epoch-day oracles. + test("date_partition_canonical_calendar_bounds") { + setForTest("spark.sql.session.timeZone", "UTC") + setForTest("spark.sql.ansi.enabled", "false") + // Expected Parquet must retain proleptic Gregorian days even before the 1582 cutover. + setForTest("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED") + setForTest("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED") + val dates = Seq( + (1, "0001-01-01", -719162), (2, "1582-10-04", -141438), + (3, "1582-10-15", -141427), (4, "1969-12-31", -1), + (5, "1970-01-01", 0), (6, "2000-02-29", 11016), (7, "9999-12-31", 2932896)) + val expectedSchema = new StructType().add("id", IntegerType) + .add("p", org.apache.spark.sql.types.DateType) + val rows = dates.map { case (id, raw, _) => Row(id, java.sql.Date.valueOf(raw)) } ++ + Seq(Row(6, java.sql.Date.valueOf("2000-02-29")), Row(8, null)) + val days: Map[Int, Option[Int]] = dates.map { case (id, _, day) => + id -> Some(day) + }.toMap + (8 -> None) + sql("CREATE TABLE tbl (id INT, p DATE) USING delta PARTITIONED BY (p)") + val inputs = dates.map { case (id, raw, _) => s"($id, DATE '$raw')" } ++ + Seq("(6, DATE '2000-02-29')", "(8, CAST(NULL AS DATE))") + sql("INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES " + + inputs.mkString(", ") + " AS rows(id, p)") + val table = registerTable("tbl") + modifyCommitActions(table, 1L) { actions => + actions.map { + case add: AddFile => + val id = JsonUtil.mapper.readTree(add.stats.get).at("/minValues/id").intValue() + val raw = if (id == 8) null else dates.find(_._1 == id).get._2 + add.copy(partitionValues = Map("p" -> raw)) + case other => other + } + } + mutateTable(table) { dir => + removeVersionChecksums(dir) + val adds = CommitLog.read(dir, 1L).collect { case add: AddFile => add } + require(adds.size == 8) + adds.foreach { add => + val stats = JsonUtil.mapper.readTree(add.stats.get) + val id = stats.at("/minValues/id").intValue() + val raw = if (id == 8) null else dates.find(_._1 == id).get._2 + require(add.partitionValues == Map("p" -> raw)) + require(stats.at("/maxValues/id").intValue() == id) + require(stats.get("numRecords").asLong() == (if (id == 6) 2L else 1L)) + if (raw != null) require(raw.matches("[0-9]{4}-[0-9]{2}-[0-9]{2}")) + println(s"Verified canonical DATE partition: id=$id raw=$raw epochDay=${days(id)}") + } + } + val allName = "all" + checkLiteralRead(table, allName, expectedSchema, rows) + checkLiteralRead(table, "all_v1", expectedSchema, rows, version = 1L) + dates.foreach { case (id, raw, _) => + checkLiteralRead(table, s"eq_$id", expectedSchema, rows.filter(_.getInt(0) == id), + predicate = s"p = DATE '$raw'") + } + checkLiteralRead(table, "is_null", expectedSchema, rows.filter(_.isNullAt(1)), + predicate = "p IS NULL") + checkLiteralRead(table, "pre_epoch", expectedSchema, rows.filter(r => r.getInt(0) <= 4), + predicate = "p < DATE '1970-01-01'") + checkLiteralRead(table, "calendar_transition", expectedSchema, + rows.filter(r => Set(2, 3).contains(r.getInt(0))), + predicate = "p >= DATE '1582-10-04' AND p <= DATE '1582-10-15'") + checkLiteralRead(table, "modern_range", expectedSchema, + rows.filter(r => Set(5, 6).contains(r.getInt(0))), + predicate = "p >= DATE '1970-01-01' AND p <= DATE '2000-02-29'") + val workload = WorkloadContext.current.workloadName + snapshotSpec(table).assert { spec => + require(spark.conf.get("spark.sql.parquet.datetimeRebaseModeInWrite") == "CORRECTED") + require(spark.conf.get("spark.sql.parquet.datetimeRebaseModeInRead") == "CORRECTED") + spec.expectation match { + case Succeeded(result) => + require(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + require(result.metadata.partitionColumns == Seq("p")) + require(result.metadata.format.provider == "parquet") + require(result.metadata.format.options.isEmpty) + require(result.protocol.minReaderVersion == 1 && result.protocol.minWriterVersion == 2) + case other => fail(s"Canonical DATE snapshot must succeed: $other") + } + + } + checkLiteralSnapshot(table, expectedSchema, Seq("p"), version = 1L) + // Snapshots capture before reads at this pin. Inspect Parquet in the last read callback. + val outsideName = "outside_bounds" + readSpec(table, predicate = "p < DATE '0001-01-01' OR p > DATE '9999-12-31'", + name = Some(outsideName)).assert { spec => + spec.expectation match { + case Succeeded(result) => require(result.rowCount == 0L) + case other => fail(s"Outside DATE bounds must succeed with no rows: $other") + } + val expected = spark.createDataFrame(Seq.empty[Row].asJava, expectedSchema) + val outside = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$outsideName").resolve("expected_data") + RowComparison.assertRowsEqual(expected, spark.read.parquet(outside.toString), outsideName) + println(s"Verified independent literal rows/schema: $workload/$outsideName rows=0") + val captured = outputDir.resolve(workload).resolve("expected") + .resolve(s"${workload}_$allName").resolve("expected_data") + val literalOracle = outputDir.resolveSibling("oracle-parquet").resolve(workload) + val expectedUnits = days.toVector :+ (6 -> Some(11016)) + for ((label, directory) <- Seq("captured" -> captured, "literal" -> literalOracle)) { + val files = Files.list(directory) + val parquet = try files.iterator().asScala.filter(_.toString.endsWith(".parquet")) + .toVector finally files.close() + require(parquet.nonEmpty) + var units = Vector.empty[(Int, Option[Int])] + parquet.foreach { file => + val path = new HadoopPath(file.toUri) + val conf = spark.sparkContext.hadoopConfiguration + val footer = ParquetFileReader.open(HadoopInputFile.fromPath(path, conf)) + try { + val physical = footer.getFooter.getFileMetaData.getSchema.getFields.asScala + .find(_.getName == "p").get.asPrimitiveType() + require(physical.getPrimitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) + require(physical.getLogicalTypeAnnotation == LogicalTypeAnnotation.dateType()) + } finally footer.close() + val reader = ParquetReader.builder[Group](new GroupReadSupport(), path) + .withConf(conf).build() + try { + var record = reader.read() + while (record != null) { + val day = if (record.getFieldRepetitionCount("p") == 0) None + else Some(record.getInteger("p", 0)) + units :+= record.getInteger("id", 0) -> day + record = reader.read() + } + } finally reader.close() + } + require(units.sorted == expectedUnits.sorted) + println(s"Verified physical DATE oracle: $workload $label units=${units.sorted}") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ReadsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ReadsSuite.scala index 61e701f2..c19ed18e 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/ReadsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ReadsSuite.scala @@ -16,7 +16,21 @@ package io.delta.workload.tables +import java.nio.file.{Files, Path, Paths} + +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.{Path => HadoopPath} +import org.apache.spark.sql.Row +import org.apache.spark.sql.delta.DeltaLog +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + import io.delta.workload.WorkloadTestSuite +import io.delta.workload.deltaharness.{Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog, Metadata, Protocol} +import io.delta.workload.model._ /** * Consolidated read workloads. @@ -540,4 +554,138 @@ class ReadsSuite extends WorkloadTestSuite("reads") { snapshotSpec(t) } + // Relative Unicode file paths in a relocated table. + test("e2e_58_unicode_路径") { + // The generator copies contents into /delta. Relative log paths must + // survive both the Unicode source root and that different destination root. + val parent = Files.createTempDirectory("paths-source-") + val source = parent.resolve("源_café_表_😀") + sql(s"""CREATE TABLE tbl (id INT, value STRING) USING delta LOCATION '$source' + TBLPROPERTIES ('delta.enableDeletionVectors' = 'false', + 'delta.checkpointInterval' = '1000', 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'base'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (4, CAST(NULL AS STRING)), (23, 'end'), + (CAST(NULL AS INT), CAST(NULL AS STRING)) AS v(id, value)""") + // Canonically equivalent NFC/NFD basenames must remain distinct byte names. + val basenames = Seq("données_café_表_😀.parquet", "données_cafe\u0301_表_😀.parquet") + assert(basenames.distinct.size == 2) + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val first = Seq(Row(-7, "base"), Row(4, null), Row(4, null), Row(null, "missing")) + val second = Seq(Row(4, null), Row(23, "end"), Row(null, null)) + val inputRows = Seq(first, second) + val renamedAdds = (1L to 2L).map { v => + val add = CommitLog.read(source, v).collect { case a: AddFile => a } + .ensuring(_.size == 1).head + assert(!add.path.contains("/") && add.deletionVector.isEmpty) + val basename = basenames((v - 1).toInt) + val file = source.resolve(basename) + Files.move(source.resolve(add.path), file) + // Change only the file reference in raw JSON; preserve every other action field. + val commit = CommitLog.commitFile(source, v) + val lines = Files.readAllLines(commit).asScala.map { line => + val node = JsonUtil.mapper.readTree(line) + if (node.has("add")) { + node.get("add").asInstanceOf[com.fasterxml.jackson.databind.node.ObjectNode] + .put("path", new HadoopPath(basename).toUri.toString) + } + node.toString + } + Files.write(commit, lines.asJava) + val physical = spark.read.parquet(file.toString) + assert(physical.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(inputRows((v - 1).toInt).asJava, + schema), physical, s"paths source physical v$v") + val renamed = CommitLog.read(source, v).collect { case a: AddFile => a }.head + assert(new HadoopPath(renamed.path).toUri.getPath == basename) + assert(Files.size(file) == renamed.size) + renamed + } + // Version checksums can embed the old paths; do not transport stale checksums. + val logs = Files.list(source.resolve("_delta_log")) + try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc")) + .foreach(Files.delete) + finally logs.close() + val commits = (0L to 2L).map(v => v -> Files.readAllBytes( + CommitLog.commitFile(source, v))).toMap + val metadata = CommitLog.read(source, 0L).collect { case m: Metadata => m } + .ensuring(_.size == 1).head + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(CommitLog.read(source, 0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + val table = registerTableFromPath(source.toString) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir != source && dir.getParent.getFileName.toString.endsWith("路径")) + commits.foreach { case (v, bytes) => + assert(Files.readAllBytes(CommitLog.commitFile(dir, v)).sameElements(bytes)) + assert(!new String(bytes, java.nio.charset.StandardCharsets.UTF_8) + .contains(source.toString)) + } + basenames.foreach { basename => + val file = dir.resolve(basename) + assert(file.getFileName.toString.getBytes(java.nio.charset.StandardCharsets.UTF_8) + .sameElements(basename.getBytes(java.nio.charset.StandardCharsets.UTF_8))) + assert(Files.readAllBytes(file).sameElements(Files.readAllBytes(source.resolve(basename)))) + } + // Remove the entire source before any snapshot/read capture. An absolute or + // stale reference to it cannot make the relocated fixture pass by accident. + org.apache.commons.io.FileUtils.deleteDirectory(parent.toFile) + assert(!Files.exists(parent) && !Files.exists(source)) + DeltaLog.clearCache() + info(s"paths source removed: $source; relocated root: $dir") + } + Seq[java.lang.Long](0L, 1L, 2L, null).foreach { version => + val v = Option(version).fold(2L)(_.toLong) + snapshotSpec(table, version = version).assert { spec => + assert(!Files.exists(source)) + assert(spec.query == SnapshotQuery(version = Some(v))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + val snapshot = DeltaLog.forTable(spark, copiedTable.toString).getSnapshotAt(v) + assert(snapshot.allFiles.collect().map(_.path).toSet == + renamedAdds.take(v.toInt).map(_.path).toSet) + assert(snapshot.logSegment.checkpointProvider.version == -1L) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_v1", ReadQuery(version = Some(1L)), schema, first, 1), + ("read_v2", ReadQuery(version = Some(2L)), schema, first ++ second, 2), + ("read_latest", ReadQuery(), schema, first ++ second, 2), + ("read_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row(null, 4), Row("missing", null), Row(null, null)), 2)) + cases.foreach { case (name, query, expectedSchema, rows, files) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(!Files.exists(source)) + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, files, 0L))) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + val paths = actual.inputFiles.map(p => Paths.get(java.net.URI.create(p))).toSet + val expectedFiles = basenames.take(files).map(copiedTable.resolve).toSet + assert(paths == expectedFiles && paths.forall(_.startsWith(copiedTable))) + assert(actual.schema == expectedSchema) + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"paths $name relocated table") + val fixture = copiedTable.getParent + val captured = spark.read.parquet(fixture.resolve( + s"expected/${fixture.getFileName}_$name/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"paths $name expected parquet") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/SchemaEvolutionSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/SchemaEvolutionSuite.scala index 9feef2e6..d27d015d 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/SchemaEvolutionSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/SchemaEvolutionSuite.scala @@ -16,7 +16,19 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.Files + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.types.{ByteType, DataType, DoubleType, IntegerType, Metadata, + StringType, StructType} + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.CommitLog +import io.delta.workload.model.Succeeded class SchemaEvolutionSuite extends WorkloadTestSuite("schema_evolution") { @@ -454,4 +466,462 @@ class SchemaEvolutionSuite extends WorkloadTestSuite("schema_evolution") { snapshotSpec(t) } + // CREATE OR REPLACE with same-name columns of a different type, changed partition and + // file sets, and fresh column-mapping identities. + // Shared by the stats-column and partition-column variants of the same replacement scenario. + private def replaceSameName( + caseName: String, + partitioned: Boolean, + fromType: DataType, + toType: DataType, + seedValues: String, + replacementValues: String, + beforeRows: Seq[Row], + afterRows: Seq[Row], + matchingPredicate: String, + missingPredicate: String): Unit = { + val partitionClause = if (partitioned) "PARTITIONED BY (c)" else "" + val beforeSchema = new StructType().add("id", IntegerType) + .add("c", fromType).add("payload", StringType) + val afterSchema = new StructType().add("id", IntegerType) + .add("c", toType).add("payload", StringType) + + sql(s"CREATE TABLE tbl (${beforeSchema.toDDL}) USING delta $partitionClause") + sql(s"INSERT INTO tbl VALUES $seedValues") + // This is a REPLACE TABLE commit, rather than DROP/ADD COLUMN or an append with evolution. + sql(s"CREATE OR REPLACE TABLE tbl (${afterSchema.toDDL}) USING delta $partitionClause") + sql(s"INSERT INTO tbl VALUES $replacementValues") + val t = registerTable("tbl") + val table = io.delta.workload.WorkloadContext.current.getTableSpec(t) + + // Literal rows form an oracle independent of reading the generated Delta table. + def frame(rows: Seq[Row], schema: StructType): DataFrame = + spark.createDataFrame(rows.asJava, schema) + + def checkRead( + name: String, + rows: Seq[Row], + schema: StructType, + version: java.lang.Long = null, + predicate: String = null, + columns: Option[Seq[String]] = None): Unit = { + val oracle = frame(rows, schema) + val projectedOracle = columns.map(cs => oracle.selectExpr(cs: _*)).getOrElse(oracle) + readSpec(t, name = Some(name), version = version, predicate = predicate, + columns = columns).assert { spec => + val workloadName = table.outputName + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$workloadName/$name must succeed: $other") + } + val expectedPath = outputDir.resolve(workloadName).resolve("expected") + .resolve(s"${workloadName}_$name").resolve("expected_data") + val captured = spark.read.parquet(expectedPath.toString) + assert(captured.schema == projectedOracle.schema) + RowComparison.assertRowsEqual( + projectedOracle, captured, s"$workloadName/$name literal oracle") + println(s"Independent row oracle passed: $workloadName/$name (${rows.size} rows)") + } + } + + checkRead("read_all", afterRows, afterSchema) + checkRead("read_before", beforeRows, beforeSchema, version = 1L) + checkRead("read_replaced_empty", Seq.empty, afterSchema, version = 2L) + checkRead("read_match", afterRows.take(2), afterSchema, predicate = matchingPredicate) + checkRead("read_missing", Seq.empty, afterSchema, predicate = missingPredicate) + checkRead("read_null", afterRows.slice(3, 4), afterSchema, predicate = "c IS NULL") + checkRead("read_non_null", afterRows.take(3) ++ afterRows.drop(4), afterSchema, + predicate = "c IS NOT NULL") + checkRead("read_projection", afterRows, afterSchema, columns = Some(Seq("c", "payload"))) + checkRead("read_before_projection", beforeRows, beforeSchema, version = 1L, + columns = Some(Seq("c", "payload"))) + checkRead("read_before_null", beforeRows.takeRight(1), beforeSchema, version = 1L, + predicate = "c IS NULL") + + for (v <- 0L to 3L) { + val expectedSchema = if (v < 2L) beforeSchema else afterSchema + snapshotSpec(t, version = v).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(DataType.fromJson(result.metadata.schemaString) == expectedSchema) + assert(result.metadata.partitionColumns == (if (partitioned) Seq("c") else Seq.empty)) + case other => fail(s"${table.outputName} snapshot v$v must succeed: $other") + } + } + } + } + + test("string_to_int_stats") { + replaceSameName( + "string_to_int_stats", partitioned = false, StringType, IntegerType, + seedValues = """(1, 'not-an-int', 'old'), (1, 'not-an-int', 'old'), """ + + """(2, '007', 'old-padded'), (3, NULL, 'old-null')""", + replacementValues = """(101, 7, 'new-seven'), (101, 7, 'new-seven'), """ + + """(102, -5, 'new-negative'), (103, NULL, 'new-null'), (104, 0, 'new-zero')""", + beforeRows = Seq( + Row(1, "not-an-int", "old"), + Row(1, "not-an-int", "old"), + Row(2, "007", "old-padded"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, 7, "new-seven"), + Row(101, 7, "new-seven"), + Row(102, -5, "new-negative"), + Row(103, null, "new-null"), + Row(104, 0, "new-zero")), + matchingPredicate = "c = 7", missingPredicate = "c = 999") + } + + test("int_to_string_stats") { + replaceSameName( + "int_to_string_stats", partitioned = false, IntegerType, StringType, + seedValues = """(1, 7, 'old'), (1, 7, 'old'), (2, -5, 'old-negative'), """ + + """(3, NULL, 'old-null')""", + replacementValues = """(101, 'seven', 'new-word'), (101, 'seven', 'new-word'), """ + + """(102, '007', 'new-padded'), (103, NULL, 'new-null'), (104, '', 'new-empty')""", + beforeRows = Seq( + Row(1, 7, "old"), + Row(1, 7, "old"), + Row(2, -5, "old-negative"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "seven", "new-word"), + Row(101, "seven", "new-word"), + Row(102, "007", "new-padded"), + Row(103, null, "new-null"), + Row(104, "", "new-empty")), + matchingPredicate = "c = 'seven'", missingPredicate = "c = 'missing'") + } + + test("byte_to_string_stats") { + replaceSameName( + "byte_to_string_stats", partitioned = false, ByteType, StringType, + seedValues = """(1, CAST(-128 AS BYTE), 'old-min'), (1, CAST(-128 AS BYTE), 'old-min'), """ + + """(2, CAST(127 AS BYTE), 'old-max'), (3, CAST(NULL AS BYTE), 'old-null')""", + replacementValues = """(101, 'byte-min', 'new-word'), (101, 'byte-min', 'new-word'), """ + + """(102, '127', 'new-numeric'), (103, NULL, 'new-null'), (104, '', 'new-empty')""", + beforeRows = Seq( + Row(1, -128.toByte, "old-min"), + Row(1, -128.toByte, "old-min"), + Row(2, 127.toByte, "old-max"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "byte-min", "new-word"), + Row(101, "byte-min", "new-word"), + Row(102, "127", "new-numeric"), + Row(103, null, "new-null"), + Row(104, "", "new-empty")), + matchingPredicate = "c = 'byte-min'", missingPredicate = "c = 'missing'") + } + + test("string_to_int_partition") { + replaceSameName( + "string_to_int_partition", partitioned = true, StringType, IntegerType, + seedValues = """(1, 'not-an-int', 'old'), (1, 'not-an-int', 'old'), """ + + """(2, '007', 'old-padded'), (3, NULL, 'old-null')""", + replacementValues = """(101, 7, 'new-seven'), (101, 7, 'new-seven'), """ + + """(102, -5, 'new-negative'), (103, NULL, 'new-null'), (104, 0, 'new-zero')""", + beforeRows = Seq( + Row(1, "not-an-int", "old"), + Row(1, "not-an-int", "old"), + Row(2, "007", "old-padded"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, 7, "new-seven"), + Row(101, 7, "new-seven"), + Row(102, -5, "new-negative"), + Row(103, null, "new-null"), + Row(104, 0, "new-zero")), + matchingPredicate = "c = 7", missingPredicate = "c = 999") + } + + test("int_to_string_partition") { + replaceSameName( + "int_to_string_partition", partitioned = true, IntegerType, StringType, + seedValues = """(1, 7, 'old'), (1, 7, 'old'), (2, -5, 'old-negative'), """ + + """(3, NULL, 'old-null')""", + replacementValues = """(101, 'seven', 'new-word'), (101, 'seven', 'new-word'), """ + + """(102, '007', 'new-padded'), (103, NULL, 'new-null'), (104, 'zero', 'new-zero')""", + beforeRows = Seq( + Row(1, 7, "old"), + Row(1, 7, "old"), + Row(2, -5, "old-negative"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "seven", "new-word"), + Row(101, "seven", "new-word"), + Row(102, "007", "new-padded"), + Row(103, null, "new-null"), + Row(104, "zero", "new-zero")), + matchingPredicate = "c = 'seven'", missingPredicate = "c = 'missing'") + } + + test("byte_to_string_partition") { + replaceSameName( + "byte_to_string_partition", partitioned = true, ByteType, StringType, + seedValues = """(1, CAST(-128 AS BYTE), 'old-min'), (1, CAST(-128 AS BYTE), 'old-min'), """ + + """(2, CAST(127 AS BYTE), 'old-max'), (3, CAST(NULL AS BYTE), 'old-null')""", + replacementValues = """(101, 'byte-min', 'new-word'), (101, 'byte-min', 'new-word'), """ + + """(102, '127', 'new-numeric'), (103, NULL, 'new-null'), (104, 'zero', 'new-zero')""", + beforeRows = Seq( + Row(1, -128.toByte, "old-min"), + Row(1, -128.toByte, "old-min"), + Row(2, 127.toByte, "old-max"), + Row(3, null, "old-null")), + afterRows = Seq( + Row(101, "byte-min", "new-word"), + Row(101, "byte-min", "new-word"), + Row(102, "127", "new-numeric"), + Row(103, null, "new-null"), + Row(104, "zero", "new-zero")), + matchingPredicate = "c = 'byte-min'", missingPredicate = "c = 'missing'") + } + + // Changed partition/file sets and fresh mapped identities share the same replacement oracle. + private def replacementRead( + t: TableHandle, + name: String, + rows: Seq[Row], + schema: StructType, + version: java.lang.Long = null, + predicate: String = null, + columns: Option[Seq[String]] = None): Unit = { + val table = io.delta.workload.WorkloadContext.current.getTableSpec(t) + val ref = readSpec(t, name = Some(name), version = version, predicate = predicate, + columns = columns) + ref.assert { spec => + val workloadName = table.outputName + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"$workloadName/$name must succeed: $other") + } + val literal = spark.createDataFrame(rows.asJava, schema) + val oracle = columns.map(cs => literal.selectExpr(cs: _*)).getOrElse(literal) + val path = outputDir.resolve(workloadName).resolve("expected") + .resolve(s"${workloadName}_${name}").resolve("expected_data") + val captured = spark.read.parquet(path.toString) + // RowComparison checks exact names/types and duplicate-sensitive row bags. + RowComparison.assertRowsEqual(oracle, captured, s"$workloadName/$name literal oracle") + assert(captured.schema.fields.forall(_.nullable)) + println(s"Independent row oracle passed: $workloadName/$name (${rows.size} rows)") + } + } + + private def replacementLayout( + caseName: String, + layout: String, + mappingMode: Option[String] = None): Unit = { + val mapped = mappingMode.isDefined + val parsed = layout == "checkpoint_parsed" + val checkpointed = layout != "commits" + val beforeNames = if (mapped) Seq("id", "label", "score", "region") + else Seq("id", "old_label", "old_score", "old_region") + val afterNames = Seq("id", "label", "score", "region") + def logicalSchema(names: Seq[String]): StructType = new StructType() + .add(names(0), IntegerType).add(names(1), StringType) + .add(names(2), DoubleType).add(names(3), StringType) + val beforeSchema = logicalSchema(beforeNames) + val afterSchema = logicalSchema(afterNames) + val beforeRows = Seq( + Row(1, "old", 1000.0, "old-east"), Row(1, "old", 1000.0, "old-east"), + Row(2, "old-negative", -1000.0, "old-west"), Row(3, null, null, null)) + val afterRows = Seq( + Row(101, "new-a", 7.5, "east"), Row(101, "new-a", 7.5, "east"), + Row(102, "new-b", -2.5, "west"), Row(103, null, null, null), + Row(104, "new-zero", 0.0, "east")) + val properties = Map( + "delta.checkpointInterval" -> "1000", + "delta.checkpoint.writeStatsAsJson" -> (!parsed).toString, + "delta.checkpoint.writeStatsAsStruct" -> parsed.toString) ++ + mappingMode.map("delta.columnMapping.mode" -> _).toMap + val propertySql = properties.toSeq.sortBy(_._1) + .map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + sql(s"""CREATE TABLE tbl (${beforeSchema.toDDL}) USING delta + PARTITIONED BY (${beforeNames.last}) TBLPROPERTIES ($propertySql)""") + sql("""INSERT INTO tbl VALUES + (1, 'old', 1000D, 'old-east'), (1, 'old', 1000D, 'old-east'), + (2, 'old-negative', -1000D, 'old-west'), (3, NULL, NULL, NULL)""") + + // The setting exists in OSS Delta 4.1.0 as well as Runtime. Restore after the DDL; + // it controls authoring only, and is not needed while queued read specs are captured. + val reuseKey = "spark.databricks.delta.columnMapping.reuseColumnMetadataDuringReplace" + val oldReuse = spark.conf.getOption(reuseKey) + try { + if (mapped) spark.conf.set(reuseKey, "false") + sql(s"""CREATE OR REPLACE TABLE tbl (${afterSchema.toDDL}) USING delta + PARTITIONED BY (region) TBLPROPERTIES ($propertySql)""") + } finally { + if (mapped) oldReuse match { + case Some(value) => spark.conf.set(reuseKey, value) + case None => spark.conf.unset(reuseKey) + } + } + sql("""INSERT INTO tbl VALUES + (101, 'new-a', 7.5D, 'east'), (101, 'new-a', 7.5D, 'east'), + (102, 'new-b', -2.5D, 'west'), (103, NULL, NULL, NULL), + (104, 'new-zero', 0D, 'east')""") + if (checkpointed) forceCheckpoint("tbl") + val t = registerTable("tbl") + val rawBefore = CommitLog.readRaw(t.sourcePath, 0L).map(JsonUtil.mapper.readTree) + .find(_.has("metaData")).get.get("metaData") + val rawAfter = CommitLog.readRaw(t.sourcePath, 2L).map(JsonUtil.mapper.readTree) + .find(_.has("metaData")).get.get("metaData") + val mappedBefore = DataType.fromJson(rawBefore.get("schemaString").asText) + .asInstanceOf[StructType] + val mappedAfter = DataType.fromJson(rawAfter.get("schemaString").asText) + .asInstanceOf[StructType] + def withoutMapping(schema: StructType): StructType = + StructType(schema.fields.map(_.copy(metadata = Metadata.empty))) + assert(withoutMapping(mappedBefore) == beforeSchema) + assert(withoutMapping(mappedAfter) == afterSchema) + if (mapped) { + val idKey = "delta.columnMapping.id" + val physicalKey = "delta.columnMapping.physicalName" + assert(mappedBefore.fields.map(_.metadata.getLong(idKey)).toSeq == Seq(1L, 2L, 3L, 4L)) + assert(mappedAfter.fields.map(_.metadata.getLong(idKey)).toSeq == Seq(5L, 6L, 7L, 8L)) + val oldPhysical = mappedBefore.fields.map(_.metadata.getString(physicalKey)).toSet + val newPhysical = mappedAfter.fields.map(_.metadata.getString(physicalKey)).toSet + assert(oldPhysical.size == 4 && newPhysical.size == 4) + assert(oldPhysical.intersect(newPhysical).isEmpty) + assert(newPhysical.intersect(afterNames.toSet).isEmpty) + } + val physicalNames = mappedAfter.fields.map { field => + if (mapped) field.metadata.getString("delta.columnMapping.physicalName") else field.name + }.toSeq + val statsNames = physicalNames.init.toSet + val partitionName = physicalNames.last + // Inspect new commit actions directly too, including the commit-only mapping variants. + val newAdds = CommitLog.readRaw(t.sourcePath, 3L).map(JsonUtil.mapper.readTree) + .filter(_.has("add")).map(_.get("add")) + assert(newAdds.nonEmpty) + for (add <- newAdds) { + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val stats = JsonUtil.mapper.readTree(add.get("stats").asText) + assert(stats.get("nullCount").fieldNames().asScala.toSet == statsNames) + for (kind <- Seq("minValues", "maxValues")) { + assert(stats.get(kind).fieldNames().asScala.toSet.subsetOf(statsNames)) + } + } + + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + // Retain exact pre-truncation actions in command evidence, including operation/removes. + for (v <- 0L to 3L; line <- CommitLog.readRaw(tableDir, v)) { + println(s"Replacement source action: $caseName v$v $line") + } + val stream = Files.list(logDir) + val files = try stream.iterator().asScala.toSeq finally stream.close() + if (checkpointed) { + val checkpoints = files.filter(_.getFileName.toString.endsWith(".checkpoint.parquet")) + assert(checkpoints.size == 1) + assert(checkpoints.head.getFileName.toString == "00000000000000000003.checkpoint.parquet") + val checkpoint = spark.read.parquet(checkpoints.head.toString) + val addSchema = checkpoint.schema("add").dataType.asInstanceOf[StructType] + assert(addSchema.fieldNames.contains("stats") == !parsed) + assert(addSchema.fieldNames.contains("stats_parsed") == parsed) + assert(addSchema.fieldNames.contains("partitionValues_parsed") == parsed) + if (parsed) { + val partitions = addSchema("partitionValues_parsed").dataType.asInstanceOf[StructType] + assert(partitions.fieldNames.toSeq == Seq(partitionName)) + assert(partitions.fields.head.dataType == StringType) + val stats = addSchema("stats_parsed").dataType.asInstanceOf[StructType] + for (kind <- Seq("minValues", "maxValues", "nullCount")) { + assert(stats(kind).dataType.asInstanceOf[StructType].fieldNames.toSet == statsNames) + } + } + val adds = checkpoint.filter("add IS NOT NULL").select("add").toJSON.collect() + assert(adds.nonEmpty) + for (json <- adds) { + val add = JsonUtil.mapper.readTree(json).get("add") + assert(add.get("partitionValues").fieldNames().asScala.toSet == Set(partitionName)) + val stats = if (parsed) add.get("stats_parsed") + else JsonUtil.mapper.readTree(add.get("stats").asText) + assert(stats.get("nullCount").fieldNames().asScala.toSet == statsNames) + for (kind <- Seq("minValues", "maxValues")) { + assert(stats.get(kind).fieldNames().asScala.toSet.subsetOf(statsNames)) + } + } + // Keep only the checkpoint-era JSON for version discovery. Earlier metadata and file + // state must come from the checkpoint; remove protocol CRC shortcuts as well. + files.filter { path => + val name = path.getFileName.toString + name.endsWith(".crc") || + (name.matches("[0-9]{20}\\.json") && name != "00000000000000000003.json") + }.foreach(Files.delete) + } else { + assert(!files.exists(_.getFileName.toString.contains("checkpoint"))) + files.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + } + println(s"Replacement layout oracle passed: $caseName ($layout)") + } + + replacementRead(t, "read_all", afterRows, afterSchema) + replacementRead(t, "read_projection", afterRows, afterSchema, + columns = Some(Seq("region", "score", "label"))) + replacementRead(t, "read_partition", afterRows.take(2) ++ afterRows.drop(4), + afterSchema, predicate = "region = 'east'") + replacementRead(t, "read_stats", afterRows.take(2), afterSchema, + predicate = "score > 7D") + replacementRead(t, "read_missing", Seq.empty, afterSchema, + predicate = "region = 'missing' OR score > 999D") + replacementRead(t, "read_null", afterRows.slice(3, 4), afterSchema, + predicate = "region IS NULL AND score IS NULL AND label IS NULL") + replacementRead(t, "read_old_stats", Seq.empty, afterSchema, + predicate = "score = 1000D OR label = 'old'") + if (!checkpointed) { + replacementRead(t, "read_before", beforeRows, beforeSchema, version = 1L) + replacementRead(t, "read_replaced_empty", Seq.empty, afterSchema, version = 2L) + } + for (v <- (if (checkpointed) Seq(3L) else Seq(1L, 2L, 3L))) { + val metadata = if (v < 2L) rawBefore else rawAfter + val expectedProperties = properties ++ mappingMode.map { mode => + Map("delta.columnMapping.mode" -> mode, + "delta.columnMapping.maxColumnId" -> (if (v < 2L) "4" else "8")) + }.getOrElse(Map.empty[String, String]) + snapshotSpec(t, version = v).assert { spec => + spec.expectation match { + case Succeeded(result) => + assert(result.metadata.schemaString == metadata.get("schemaString").asText) + assert(result.metadata.partitionColumns == + Seq(if (v < 2L) beforeNames.last else "region")) + assert(result.metadata.configuration == expectedProperties) + case other => fail(s"$caseName snapshot v$v must succeed: $other") + } + println(s"Independent metadata oracle passed: $caseName v$v") + } + } + } + + test("changed_sets_checkpoint_json") { + replacementLayout("changed_sets_checkpoint_json", "checkpoint_json") + } + + test("changed_sets_checkpoint_parsed") { + replacementLayout("changed_sets_checkpoint_parsed", "checkpoint_parsed") + } + + test("mapped_name_commits") { + replacementLayout("mapped_name_commits", "commits", Some("name")) + } + + test("mapped_name_checkpoint_json") { + replacementLayout("mapped_name_checkpoint_json", "checkpoint_json", Some("name")) + } + + test("mapped_name_checkpoint_parsed") { + replacementLayout("mapped_name_checkpoint_parsed", "checkpoint_parsed", Some("name")) + } + + test("mapped_id_commits") { + replacementLayout("mapped_id_commits", "commits", Some("id")) + } + + test("mapped_id_checkpoint_json") { + replacementLayout("mapped_id_checkpoint_json", "checkpoint_json", Some("id")) + } + + test("mapped_id_checkpoint_parsed") { + replacementLayout("mapped_id_checkpoint_parsed", "checkpoint_parsed", Some("id")) + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/ScopedSessionConf.scala b/workload-generator/src/test/scala/io/delta/workload/tables/ScopedSessionConf.scala new file mode 100644 index 00000000..587567ff --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/ScopedSessionConf.scala @@ -0,0 +1,55 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import scala.collection.mutable + +import org.apache.spark.sql.SparkSession +import org.scalatest.BeforeAndAfterEach + +import io.delta.workload.WorkloadTestSuite + +/** + * Session settings that a single workload test needs while its specs are captured. + * + * `WorkloadTestSuite` generates and asserts specs after the test body returns, so a + * try/finally inside the body would restore a setting before capture. A setting applied + * with [[setForTest]] stays active through generation and spec assertions, and is restored + * after the suite's per-test cleanup, including when the body, generation or an assertion + * fails. Other tests in the suite keep the session's original settings. + */ +trait ScopedSessionConf extends BeforeAndAfterEach { this: WorkloadTestSuite => + + // Key -> (session, value before this test first set it). + private val savedConf = mutable.LinkedHashMap.empty[String, (SparkSession, Option[String])] + + protected def setForTest(key: String, value: String): Unit = { + val session = spark + if (!savedConf.contains(key)) savedConf(key) = session -> session.conf.getOption(key) + session.conf.set(key, value) + } + + override def afterEach(): Unit = { + try super.afterEach() finally { + savedConf.toSeq.reverse.foreach { + case (key, (session, Some(previous))) => session.conf.set(key, previous) + case (key, (session, None)) => session.conf.unset(key) + } + savedConf.clear() + } + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/TimeTravelSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/TimeTravelSuite.scala index 9197d8c6..e8c3000c 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/TimeTravelSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/TimeTravelSuite.scala @@ -16,8 +16,19 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite -import io.delta.workload.engine.SnapshotResolver +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType} + +import io.delta.workload.{WorkloadContext, WorkloadTestSuite} +import io.delta.workload.deltaharness.{DeltaHarness, Format => HarnessFormat} +import io.delta.workload.engine.{RowComparison, SnapshotResolver} +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.{AddFile, CommitLog, Metadata, Protocol, RemoveFile} +import io.delta.workload.model._ class TimeTravelSuite extends WorkloadTestSuite("time_travel") { @@ -536,4 +547,399 @@ class TimeTravelSuite extends WorkloadTestSuite("time_travel") { snapshotSpec(t) } + // Truncated history, VACUUM, and RESTORE. + test("e2e_54_truncated_history") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-1, 'old'), (2, 'old'), (2, 'old'), (CAST(NULL AS INT), 'old') AS v(id, value)""") + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (7, 'keep'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (9, 'boundary'), (7, 'keep') AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val boundaryRows = Seq(Row(7, "keep"), Row(3, null), Row(3, null), Row(null, "missing"), + Row(9, "boundary"), Row(7, "keep")) + val latestRows = boundaryRows ++ Seq(Row(3, null), Row(11, "after"), Row(null, "after")) + val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + assert(original(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inputs = (1L to 3L).map { version => + val adds = original(version).collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.deletionVector.isEmpty) + version -> adds.head + }.toMap + assert(inputs.values.map(_.path).toSet.size == 3) + assert(original(2L).collect { case r: RemoveFile => r.path } == Seq(inputs(1L).path)) + DeltaHarness.get.openLog(spark, table.sourcePath.toString).checkpoint() + val checkpointName = "00000000000000000003.checkpoint.parquet" + val checkpointBytes = Files.readAllBytes(table.sourcePath.resolve("_delta_log") + .resolve(checkpointName)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + val logDir = dir.resolve("_delta_log") + (0L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + assert(Files.readAllBytes(logDir.resolve(checkpointName)).sameElements(checkpointBytes)) + // Retain v2 and v3 JSON for OSS historical lookup, plus the oldest checkpoint at v3. + // The missing prefix makes v1 unavailable; no table data is deleted. + (0L to 1L).foreach { version => + val commit = logDir.resolve(f"$version%020d.json") + assert(Files.isRegularFile(commit)) + Files.delete(commit) + assert(!Files.exists(commit)) + } + (2L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + inputs.values.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + val boundary = SnapshotResolver.buildDeltaReader(spark, dir, Some(3L), None) + assert(boundary.schema == schema) + RowComparison.assertRowsEqual(spark.createDataFrame(boundaryRows.asJava, schema), boundary, + "history boundary after JSON removal, before append") + // Append to the already truncated copy: v4 must be replayed after the v3 checkpoint. + spark.sql(s"""INSERT INTO delta.`$dir` SELECT /*+ COALESCE(1) */ * FROM VALUES + (3, CAST(NULL AS STRING)), (11, 'after'), (CAST(NULL AS INT), 'after') AS v(id, value)""") + .collect() + // The generator strips Hadoop checksum sidecars while copying. The append's LogStore + // may recreate this sidecar; keep the same portable fixture form after the mutation. + Files.deleteIfExists(logDir.resolve(".00000000000000000004.json.crc")) + val appended = CommitLog.read(dir, 4L) + val adds = appended.collect { case a: AddFile => a } + assert(adds.size == 1 && adds.head.deletionVector.isEmpty) + assert(!inputs.values.exists(_.path == adds.head.path)) + assert(appended.forall(a => !a.isInstanceOf[RemoveFile] && !a.isInstanceOf[Metadata] && + !a.isInstanceOf[Protocol])) + val commitInfo = appended.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "WRITE") + assert(commitInfo.get("operationParameters").get("mode").asText() == "Append") + assert(Files.size(dir.resolve(adds.head.path)) == adds.head.size) + assert(Files.readAllBytes(logDir.resolve(checkpointName)).sameElements(checkpointBytes)) + val files = Files.list(logDir) + try { + val names = files.iterator().asScala.map(_.getFileName.toString).toSeq + assert(names.filter(_.endsWith(".checkpoint.parquet")) == Seq(checkpointName)) + assert(names.filter(_.endsWith(".json")).sorted == (2L to 4L).map(v => f"$v%020d.json")) + } finally files.close() + assert(JsonUtil.mapper.readTree(Files.readAllBytes(logDir.resolve("_last_checkpoint"))) + .get("version").asLong() == 3L) + (2L to 3L).foreach(v => assert(CommitLog.read(dir, v) == original(v))) + inputs.values.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + info("history removed JSON v0/v1; retained v2/v3 and checkpoint v3; real append v4: " + + commitInfo.toString) + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](3L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(4L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val cases = Seq( + ("read_boundary_v3", ReadQuery(version = Some(3L)), boundaryRows, 2), + ("read_latest_after_append", ReadQuery(), latestRows, 3)) + cases.foreach { case (name, query, rows, fileCount) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull) + .assert { spec => + assert(spec.query == query) + assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, fileCount, 0L))) + val expected = spark.createDataFrame(rows.asJava, schema) + val actual = SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, None) + assert(actual.schema == schema) + RowComparison.assertRowsEqual(expected, actual, s"history $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == schema) + RowComparison.assertRowsEqual(expected, captured, s"history $name expected parquet") + } + } + // No checkpoint or JSON exists at or before v1. DAT resolves getSnapshotAt first; + // OSS raises DELTA_LOG_FILE_NOT_FOUND for that empty log segment, before scanning data. + readSpec(table, name = Some("read_unavailable_v1"), version = 1L, + expectError = ErrorCode("DELTA_LOG_FILE_NOT_FOUND")).assert { spec => + assert(spec.query == ReadQuery(version = Some(1L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode == "DELTA_LOG_FILE_NOT_FOUND") + assert(error.errorMessage.contains("version 1")) + info("history typed unavailable-history read: " + error.toString) + case other => fail(s"history unavailable-history read expected Failed, got $other") + } + } + snapshotSpec(table, version = 1L, expectError = ErrorCode("DELTA_LOG_FILE_NOT_FOUND")) + .assert { spec => + assert(spec.query == SnapshotQuery(version = Some(1L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode == "DELTA_LOG_FILE_NOT_FOUND") + assert(error.errorMessage.contains("version 1")) + case other => fail(s"history unavailable-history snapshot expected Failed, got $other") + } + } + } + + test("e2e_50_actual_vacuum") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (8, 'keep'), (3, CAST(NULL AS STRING)), (3, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing') AS v(id, value)""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-2, 'early'), (5, 'middle'), (8, 'keep'), (12, CAST(NULL AS STRING)) AS v(id, value)""") + val table = registerTable("tbl") + val schema = new StructType().add("id", IntegerType).add("value", StringType) + val rows = Seq(Row(8, "keep"), Row(3, null), Row(3, null), Row(null, "missing"), + Row(-2, "early"), Row(5, "middle"), Row(8, "keep"), Row(12, null)) + val literal = spark.createDataFrame(rows.asJava, schema) + val before = (0L to 2L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + assert(before(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.format == io.delta.workload.log.Format()) + assert(metadata.partitionColumns.isEmpty) + assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic")) + val inputs = (1L to 2L).flatMap(v => before(v).collect { case a: AddFile => a }) + assert(inputs.size == 2 && inputs.map(_.path).distinct.size == 2) + assert(inputs.forall(_.deletionVector.isEmpty)) + RowComparison.assertRowsEqual(literal, + SnapshotResolver.buildDeltaReader(spark, table.sourcePath, Some(2L), None), + "vacuum historical rows before pruning") + sql("OPTIMIZE tbl") + val rewrite = CommitLog.read(table.sourcePath, 3L) + val active = rewrite.collect { case a: AddFile => a } + val obsolete = rewrite.collect { case r: RemoveFile => r } + assert(active.nonEmpty && obsolete.size == 2) + assert(obsolete.map(_.path).toSet == inputs.map(_.path).toSet) + assert((active.map(_.path).toSet intersect obsolete.map(_.path).toSet).isEmpty) + assert(active.forall(a => !a.dataChange && a.deletionVector.isEmpty)) + assert(obsolete.forall(r => !r.dataChange && r.deletionVector.isEmpty)) + assert(rewrite.collect { case m: Metadata => m }.isEmpty) + assert(rewrite.collect { case p: Protocol => p }.isEmpty) + val optimizeInfo = rewrite.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(optimizeInfo.get("operation").asText() == "OPTIMIZE") + (inputs ++ active).foreach { add => + assert(Files.isRegularFile(table.sourcePath.resolve(add.path))) + assert(Files.size(table.sourcePath.resolve(add.path)) == add.size) + } + info("vacuum pre-VACUUM obsolete paths physically present: " + + obsolete.map(_.path).mkString(",")) + val retentionCheck = "spark.databricks.delta.retentionDurationCheck.enabled" + val originalRetentionCheck = spark.conf.getOption(retentionCheck) + try { + spark.conf.set(retentionCheck, "false") + sql("VACUUM tbl RETAIN 0 HOURS") + } finally { + originalRetentionCheck match { + case Some(value) => spark.conf.set(retentionCheck, value) + case None => spark.conf.unset(retentionCheck) + } + } + assert(spark.conf.getOption(retentionCheck) == originalRetentionCheck) + obsolete.foreach(r => assert(!Files.exists(table.sourcePath.resolve(r.path)))) + active.foreach(a => assert(Files.size(table.sourcePath.resolve(a.path)) == a.size)) + // VACUUM emits audit commits; they must not change the active file set or metadata. + val vacuum = (4L to 5L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + val vacuumInfo = vacuum.toSeq.sortBy(_._1).flatMap { case (_, actions) => + assert(actions.forall(a => !a.isInstanceOf[AddFile] && !a.isInstanceOf[RemoveFile] && + !a.isInstanceOf[Metadata] && !a.isInstanceOf[Protocol])) + actions.map(a => JsonUtil.mapper.readTree(a.toJson)).filter(_.has("commitInfo")) + .map(_.get("commitInfo")) + } + assert(vacuumInfo.map(_.get("operation").asText()) == Seq("VACUUM START", "VACUUM END")) + vacuumInfo.foreach(n => info("vacuum actual vacuum commit: " + n.toString)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + (before ++ Map(3L -> rewrite) ++ vacuum).foreach { case (version, actions) => + assert(CommitLog.read(dir, version) == actions) + } + obsolete.foreach(r => assert(!Files.exists(dir.resolve(r.path)))) + active.foreach(a => assert(Files.size(dir.resolve(a.path)) == a.size)) + // Force row materialization, independently of DAT's typed error capture. + val thrown = intercept[Exception] { + SnapshotResolver.buildDeltaReader(spark, dir, Some(2L), None).collect() + } + val causes = Iterator.iterate(thrown: Throwable)(_.getCause).takeWhile(_ != null).toSeq + assert(causes.exists(_.isInstanceOf[java.io.FileNotFoundException])) + assert(causes.exists(e => obsolete.exists(r => + Option(e.getMessage).exists(_.contains(r.path))))) + info("vacuum independent historical missing-file cause: " + + causes.map(e => e.getClass.getName + ": " + e.getMessage).mkString(" <- ")) + } + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + metadata.configuration, metadata.createdTime)) + Seq[java.lang.Long](3L, 5L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(5L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_latest", ReadQuery(), schema, rows), + ("read_rewritten_v3", ReadQuery(version = Some(3L)), schema, rows), + ("read_vacuumed_v5", ReadQuery(version = Some(5L)), schema, rows), + ("read_latest_filtered", ReadQuery(predicate = Some("id = 3 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 3), Row(null, 3), Row("missing", null)))) + cases.foreach { case (name, query, expectedSchema, expectedRows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == expectedRows.size.toLong) + case other => fail(s"vacuum $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(expectedRows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"vacuum $name copied table") + val fixture = outputDir.resolve(WorkloadContext.current.workloadName) + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"vacuum $name expected parquet") + } + } + readSpec(table, name = Some("read_pruned_v2"), version = 2L, + expectError = AnyError).assert { spec => + assert(spec.query == ReadQuery(version = Some(2L))) + spec.expectation match { + case Failed(error) => + assert(error.errorCode.nonEmpty) + assert(obsolete.exists(r => error.errorMessage.contains(r.path))) + val message = error.errorMessage.toLowerCase(java.util.Locale.ROOT) + assert(message.contains("not exist") || message.contains("not found")) + info("vacuum typed historical error: " + error.toString) + case other => fail(s"vacuum pruned historical read expected Failed, got $other") + } + // An error at the historical version must leave the current table readable. + RowComparison.assertRowsEqual(literal, + SnapshotResolver.buildDeltaReader(spark, copiedTable, None, None), + "vacuum current rows after historical failure") + } + } + + test("e2e_55_restore") { + sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES ( + 'delta.enableDeletionVectors' = 'false', 'delta.checkpointInterval' = '1000', + 'delta.checkpointPolicy' = 'classic')""") + sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (-7, 'before'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)), + (CAST(NULL AS INT), 'missing'), (12, 'keep'), (12, 'keep') AS v(id, value)""") + val table = registerTable("tbl") + val before = (0L to 1L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap + sql("""INSERT OVERWRITE tbl SELECT /*+ COALESCE(1) */ * FROM VALUES + (99, 'mutated'), (99, 'mutated'), (CAST(NULL AS INT), 'changed'), + (6, CAST(NULL AS STRING)) AS v(id, value)""") + val mutated = CommitLog.read(table.sourcePath, 2L) + sql("RESTORE TABLE tbl TO VERSION AS OF 1") + val restored = CommitLog.read(table.sourcePath, 3L) + assert(!Files.exists(CommitLog.commitFile(table.sourcePath, 4L))) + (0L to 1L).foreach(v => assert(CommitLog.read(table.sourcePath, v) == before(v))) + assert(CommitLog.read(table.sourcePath, 2L) == mutated) + val originalAdds = before(1L).collect { case a: AddFile => a } + val mutatedAdds = mutated.collect { case a: AddFile => a } + val restoredAdds = restored.collect { case a: AddFile => a } + assert(originalAdds.nonEmpty && mutatedAdds.nonEmpty) + assert(originalAdds.map(_.path).toSet.intersect(mutatedAdds.map(_.path).toSet).isEmpty) + assert(mutated.collect { case r: RemoveFile => r.path }.toSet == + originalAdds.map(_.path).toSet) + assert(restoredAdds.toSet == originalAdds.map(_.copy(dataChange = true)).toSet) + val restoredRemoves = restored.collect { case r: RemoveFile => r } + assert(restoredRemoves.map(_.path).toSet == mutatedAdds.map(_.path).toSet) + assert(restoredAdds.forall(_.dataChange) && restoredRemoves.forall(_.dataChange)) + val commitInfo = restored.map(a => JsonUtil.mapper.readTree(a.toJson)) + .filter(_.has("commitInfo")).ensuring(_.size == 1).head.get("commitInfo") + assert(commitInfo.get("operation").asText() == "RESTORE") + assert(commitInfo.get("operationParameters").get("version").asText() == "1") + info("restore OSS Delta 4.1.0 RESTORE commit: " + commitInfo.toString) + + val schema = new StructType().add("id", IntegerType).add("value", StringType) + // Literal bags are independent of both the captured result and the Delta reader. + val initialRows = Seq(Row(-7, "before"), Row(4, null), Row(4, null), + Row(null, "missing"), Row(12, "keep"), Row(12, "keep")) + val mutatedRows = Seq(Row(99, "mutated"), Row(99, "mutated"), + Row(null, "changed"), Row(6, null)) + val metadata = before(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head + val configuration = Map("delta.enableDeletionVectors" -> "false", + "delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic") + assert(before(0L).collect { case p: Protocol => p } == Seq(Protocol(1, 2))) + assert(DataType.fromJson(metadata.schemaString) == schema) + assert(metadata.partitionColumns.isEmpty && metadata.configuration == configuration) + val expectedSnapshot = SnapshotResult(ProtocolInfo(1, 2), + MetadataInfo(metadata.id, metadata.name, metadata.description, + HarnessFormat("parquet", Map.empty), schema.json, Seq.empty, + configuration, metadata.createdTime)) + var copiedTable: Path = null + mutateTable(table) { dir => + copiedTable = dir + assert(dir.getParent.getParent == outputDir) + (0L to 1L).foreach(v => assert(CommitLog.read(dir, v) == before(v))) + assert(CommitLog.read(dir, 2L) == mutated) + assert(CommitLog.read(dir, 3L) == restored) + (originalAdds ++ mutatedAdds).foreach { add => + assert(Files.readAllBytes(dir.resolve(add.path)) + .sameElements(Files.readAllBytes(table.sourcePath.resolve(add.path)))) + } + } + Seq[java.lang.Long](1L, 2L, null).foreach { version => + snapshotSpec(table, version = version).assert { spec => + assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong)))) + assert(spec.expectation == Succeeded(expectedSnapshot)) + } + } + val projected = new StructType().add("value", StringType).add("id", IntegerType) + val cases = Seq( + ("read_before_v1", ReadQuery(version = Some(1L)), schema, initialRows), + ("read_mutated_v2", ReadQuery(version = Some(2L)), schema, mutatedRows), + ("read_restored_v3", ReadQuery(version = Some(3L)), schema, initialRows), + ("read_latest", ReadQuery(), schema, initialRows), + ("read_latest_selective", ReadQuery(predicate = Some("id = 4 OR id IS NULL"), + columns = Some(Seq("value", "id"))), projected, + Seq(Row(null, 4), Row(null, 4), Row("missing", null)))) + cases.foreach { case (name, query, expectedSchema, rows) => + readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull, + predicate = query.predicate.orNull, columns = query.columns).assert { spec => + assert(spec.query == query) + spec.expectation match { + case Succeeded(result) => assert(result.rowCount == rows.size.toLong) + case other => fail(s"restore $name expected Succeeded, got $other") + } + val expected = spark.createDataFrame(rows.asJava, expectedSchema) + val actual = SnapshotResolver.applyFilters( + SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp), + query.predicate, query.columns) + assert(actual.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, actual, s"restore $name copied table") + val fixture = copiedTable.getParent + val specName = s"${fixture.getFileName}_$name" + val captured = spark.read.parquet( + fixture.resolve(s"expected/$specName/expected_data").toString) + assert(captured.schema == expectedSchema) + RowComparison.assertRowsEqual(expected, captured, s"restore $name expected parquet") + } + } + } + } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/TypeWideningSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/TypeWideningSuite.scala index 4b8bef5e..62865cb3 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/TypeWideningSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/TypeWideningSuite.scala @@ -16,7 +16,17 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import java.nio.file.Files + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.types.{DataType, StructType} + +import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite} +import io.delta.workload.engine.RowComparison +import io.delta.workload.json.JsonUtil +import io.delta.workload.log.AddFile +import io.delta.workload.model._ /** * Type widening workloads: numeric chains, float->double, decimal precision, @@ -357,4 +367,251 @@ class TypeWideningSuite extends WorkloadTestSuite("type_widening") { snapshotSpec(t) } + // Widened JSON statistics, old-type parsed checkpoint statistics, and partition values + // are separate workloads per transition. + private case class Stage(dataType: String, values: Seq[String]) + private case class Widening(name: String, stages: Seq[Stage], jsonGap: Boolean = true) + + // The numeric chain, decimal physical encodings, and DATE values follow TypeWideningSuite. + // One transition corpus drives all three representations. The chain's SHORT stage also covers + // BYTE -> SHORT; existing full/time-travel JSON reads are not redeclared here. + private val widenings = Seq( + Widening("byte_int", Seq( + Stage("BYTE", Seq("1", "127")), Stage("INT", Seq("128", "2147483647")))), + Widening("byte_long", Seq( + Stage("BYTE", Seq("1", "127")), + Stage("BIGINT", Seq("128", "9223372036854775807")))), + Widening("short_int", Seq( + Stage("SHORT", Seq("10", "100")), Stage("INT", Seq("40000", "50000"))), + jsonGap = false), + Widening("short_long", Seq( + Stage("SHORT", Seq("100", "32767")), + Stage("BIGINT", Seq("40000", "9223372036854775807")))), + Widening("int_long", Seq( + Stage("INT", Seq("100", "200", "300", "400")), + Stage("BIGINT", Seq("3000000000", "9000000000000"))), jsonGap = false), + Widening("byte_double", Seq( + Stage("BYTE", Seq("1", "127")), + Stage("DOUBLE", Seq("127.5", "2147483648.25")))), + Widening("short_double", Seq( + Stage("SHORT", Seq("100", "32767")), + Stage("DOUBLE", Seq("32767.5", "2147483648.25")))), + Widening("int_double", Seq( + Stage("INT", Seq("100", "2147483647")), + Stage("DOUBLE", Seq("2147483647.5", "4294967296.25")))), + Widening("float_double", Seq( + Stage("FLOAT", Seq("1.5", "3.125")), + Stage("DOUBLE", Seq("3.141592653589793", "1.7976931348623157E308"))), + jsonGap = false), + Widening("decimal_precision", Seq( + Stage("DECIMAL(5,2)", Seq("123.45", "999.99")), + Stage("DECIMAL(10,2)", Seq("12345678.90", "99999999.99"))), jsonGap = false), + Widening("decimal_precision_scale", Seq( + Stage("DECIMAL(9,2)", Seq("123.45", "9999999.99")), + Stage("DECIMAL(18,2)", Seq("1234567890123.45", "9999999999999999.99")), + Stage("DECIMAL(28,3)", Seq("1234567890123456789012345.678")))), + Widening("date_timestamp_ntz", Seq( + Stage("DATE", Seq("'2024-01-15'", "'2024-06-30'")), + Stage("TIMESTAMP_NTZ", Seq("'2024-12-31 23:59:59'", "'2025-01-01 12:30:00'")))), + Widening("multiple_steps", Seq( + Stage("BYTE", Seq("1", "100")), Stage("SHORT", Seq("200", "30000")), + Stage("INT", Seq("40000", "2000000000")), + Stage("BIGINT", Seq("3000000000", "9000000000000000000"))))) + + // (stable id, literal, original type). Expected bags are selected explicitly by id, never by + // filtering the table or capturing a fresh query result as its own oracle. + private def rows(w: Widening, stage: Int, partitioned: Boolean): Seq[(Int, String, String)] = + w.stages.take(stage + 1).zipWithIndex.flatMap { case (s, i) => + s.values.zipWithIndex.map { case (v, j) => (i * 10 + j + 1, v, s.dataType) } ++ + (if (partitioned) Seq((i * 10 + 9, "NULL", s.dataType)) else Seq.empty) + } + + private def literal(value: String, from: String, to: String): String = + s"CAST(CAST($value AS $from) AS $to)" + + private def checkedRead( + t: TableHandle, + label: String, + dataType: String, + expected: Seq[(Int, String, String)], + predicate: String, + version: Long): Unit = { + val table = WorkloadContext.current.getTableSpec(t) + readSpec(t, version = version, predicate = predicate, name = Some(label)).assert { spec => + spec.expectation match { + case Succeeded(_) => + case other => fail(s"${table.outputName}/$label must succeed: $other") + } + val selects = expected.map { case (id, value, from) => + s"SELECT CAST($id AS INT) AS id, ${literal(value, from, dataType)} AS value" + } + val oracle = if (selects.nonEmpty) selects.mkString(" UNION ALL ") else + s"SELECT CAST(NULL AS INT) AS id, CAST(NULL AS $dataType) AS value WHERE false" + val expectedPath = outputDir.resolve(table.outputName).resolve("expected") + .resolve(s"${table.outputName}_$label").resolve("expected_data") + RowComparison.assertRowsEqual(spark.sql(oracle), + spark.read.parquet(expectedPath.toString), s"${table.outputName}/$label independent oracle") + } + } + + private def checkedSnapshot(t: TableHandle, w: Widening, stage: Int, + partitioned: Boolean, version: Long): Unit = { + snapshotSpec(t, version = version).assert { spec => + spec.expectation match { + case Succeeded(result) => + val schema = DataType.fromJson(result.metadata.schemaString).asInstanceOf[StructType] + val expected = StructType.fromDDL(s"id INT, value ${w.stages(stage).dataType}") + require(schema.map(f => f.name -> f.dataType) == + expected.map(f => f.name -> f.dataType), s"Unexpected schema: $schema") + require(result.metadata.partitionColumns == + (if (partitioned) Seq("value") else Seq.empty)) + val fieldMetadata = schema("value").metadata + val changes = if (fieldMetadata.contains("delta.typeChanges")) + fieldMetadata.getMetadataArray("delta.typeChanges").toSeq else Seq.empty + val expectedChanges = w.stages.take(stage + 1).sliding(2).collect { + case Seq(from, to) => + (DataType.fromDDL(from.dataType).json, DataType.fromDDL(to.dataType).json) + }.toSeq + require(changes.map(c => (s"\"${c.getString("fromType")}\"", + s"\"${c.getString("toType")}\"")) == expectedChanges, + s"Unexpected widening history at $version: $changes") + case other => fail(s"Snapshot at $version must succeed: $other") + } + } + } + + private def prepareStats(t: TableHandle, representation: String, original: Stage): Unit = { + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val stream = Files.list(logDir) + val paths = try stream.iterator().asScala.toVector finally stream.close() + // CRC file-state snapshots must not substitute for the statistics under test. + paths.filter(_.getFileName.toString.endsWith(".crc")).foreach(Files.delete) + if (representation == "parsed") { + val checkpoint = logDir.resolve("00000000000000000001.checkpoint.parquet") + require(Files.exists(checkpoint), "Missing old-schema checkpoint") + val adds = spark.read.parquet(checkpoint.toString).filter("add IS NOT NULL") + val stats = adds.selectExpr("add.stats_parsed.minValues.value AS minValue", + "add.stats_parsed.maxValues.value AS maxValue", "add.stats_parsed.numRecords AS records") + require(stats.schema.head.dataType == DataType.fromDDL(original.dataType), + s"Checkpoint must retain ${original.dataType} statistics: ${stats.schema}") + val addSchema = adds.schema("add").dataType.asInstanceOf[StructType] + if (addSchema.fieldNames.contains("stats")) { + require(adds.filter("add.stats IS NOT NULL").count() == 0, + "Parsed-stat checkpoint must not contain JSON statistics") + } + val oracle = s"""SELECT CAST(${original.values.head} AS ${original.dataType}) AS minValue, + CAST(${original.values.last} AS ${original.dataType}) AS maxValue, + CAST(${original.values.size} AS BIGINT) AS records""" + RowComparison.assertRowsEqual(spark.sql(oracle), stats.selectExpr( + "MIN(minValue) AS minValue", "MAX(maxValue) AS maxValue", "SUM(records) AS records"), + "old parsed checkpoint min/max/count oracle") + // Keep the checkpoint's boundary commit for version discovery; its adds are removed below. + Files.delete(logDir.resolve("00000000000000000000.json")) + } else { + require(!paths.exists(_.getFileName.toString.contains("checkpoint")), + "JSON/partition workloads must not have checkpoints") + val actions = paths.filter(_.getFileName.toString.endsWith(".json")) + .flatMap(p => Files.readAllLines(p).asScala) + .map(JsonUtil.mapper.readTree(_)) + val adds = actions.filter(_.has("add")).map(_.get("add")) + require(adds.nonEmpty) + if (representation == "json") { + require(adds.forall { a => + a.hasNonNull("stats") && Seq("minValues", "maxValues").forall { bound => + JsonUtil.mapper.readTree(a.get("stats").asText()).path(bound).has("value") + } + }) + } else { + require(adds.forall(_.get("partitionValues").has("value"))) + require(adds.exists(_.get("partitionValues").get("value").isNull), + "Retain a null partition written before widening") + } + } + } + if (representation == "parsed") { + modifyCommitActions(t, version = 1)(_.filterNot(_.isInstanceOf[AddFile])) + } + } + + for { + w <- widenings + representation <- Seq("json", "parsed", "partition") + if representation != "json" || w.jsonGap + } { + test(s"${representation}_${w.name}") { + val partitioned = representation == "partition" + val partition = if (partitioned) "PARTITIONED BY (value)" else "" + val timestampFeature = if (w.name == "date_timestamp_ntz") + ", 'delta.feature.timestampNtz' = 'supported'" else "" + val statsProperties = if (representation == "parsed") + ", 'delta.checkpoint.writeStatsAsStruct' = 'true', " + + "'delta.checkpoint.writeStatsAsJson' = 'false'" else "" + sql(s"""CREATE TABLE tbl (id INT, value ${w.stages.head.dataType}) USING delta + $partition TBLPROPERTIES ('delta.enableTypeWidening' = 'true', + 'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '100', + 'delta.checkpointPolicy' = 'classic' $timestampFeature $statsProperties)""") + + for ((s, i) <- w.stages.zipWithIndex) { + if (i > 0) sql(s"ALTER TABLE tbl ALTER COLUMN value TYPE ${s.dataType}") + val inserted = rows(w, i, partitioned).filter(_._1 / 10 == i) + val values = inserted.map { case (id, value, from) => + s"($id, CAST($value AS $from))" + }.mkString(", ") + sql(s"INSERT INTO tbl VALUES $values") + if (i == 0 && representation == "parsed") forceCheckpoint("tbl") + } + + val t = registerTable("tbl") + prepareStats(t, representation, w.stages.head) + + // Capture the requested latest-schema assertions before historical-stage controls. + for ((s, i) <- w.stages.zipWithIndex.reverse) { + val version = i * 2L + 1 + checkedSnapshot(t, w, i, partitioned, version) + val precisionOnlyStats = representation != "partition" && + w.name == "decimal_precision_scale" && i == 1 + if (i > 0 && !precisionOnlyStats) { + val bag = rows(w, i, partitioned) + val nonNull = bag.filter(_._2 != "NULL") + val original = nonNull.filter(_._1 < 10) + val previous = w.stages(i - 1) + val firstNew = literal(s.values.head, s.dataType, s.dataType) + val lastOld = literal(previous.values.last, previous.dataType, s.dataType) + val lastOriginal = literal(w.stages.head.values.last, + w.stages.head.dataType, s.dataType) + val firstOriginal = literal(w.stages.head.values.head, + w.stages.head.dataType, s.dataType) + val prefix = if (w.name == "multiple_steps" && i == 1) "byte_short" else w.name + val label = s"${prefix}_v$version" + val read = (suffix: String, predicate: String, selected: Seq[(Int, String, String)]) => + checkedRead(t, s"${label}_$suffix", s.dataType, selected, predicate, version) + + // JSON partial cases contribute filters only; TypeWideningSuite owns their full reads. + val partialJson = representation == "json" && + Set("multiple_steps", "decimal_precision_scale", "date_timestamp_ntz").contains(w.name) + if (!partialJson) read("all", null, bag) + read("old_range", s"value <= $lastOriginal", original) + read("new_range", s"value > $lastOld", nonNull.filter(_._1 / 10 == i)) + read("cross_range", s"value >= $lastOld AND value <= $firstNew", + nonNull.filter(r => r._1 == (i - 1) * 10 + previous.values.size || + r._1 == i * 10 + 1)) + read("miss", s"value < $firstOriginal", Seq.empty) + + if (partitioned) { + read("old_equality", s"value = $firstOriginal", original.take(1)) + read("new_equality", s"value = $firstNew", nonNull.filter(_._1 == i * 10 + 1)) + read("null_partitions", "value IS NULL", bag.filter(_._2 == "NULL")) + } + if (w.name == "date_timestamp_ntz") { + read("old_midnight", s"value = $lastOriginal", original.takeRight(1)) + read("new_midnight_miss", "value = TIMESTAMP_NTZ'2024-12-31 00:00:00'", Seq.empty) + read("exclusive_boundary", s"value >= $lastOriginal AND value < $firstNew", + original.takeRight(1)) + } + } + } + } + } + }