From 8deeb013667e19957a6a8a297f78b9d9d040996b Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Mon, 15 Jun 2026 23:22:02 +0000 Subject: [PATCH 1/4] feat: add cdf, checkpoint, and crc spec types --- .../io/delta/workload/SpecDeclarations.scala | 24 +++ .../io/delta/workload/WorkloadContext.scala | 38 +++++ .../io/delta/workload/WorkloadGenerator.scala | 35 +++- .../scala/io/delta/workload/WorkloadOps.scala | 60 +++++++ .../delta/workload/capture/CdfCapture.scala | 155 +++++++++++++++++ .../workload/capture/CheckpointCapture.scala | 121 +++++++++++++ .../delta/workload/capture/CrcCapture.scala | 160 ++++++++++++++++++ .../delta/workload/capture/ReadCapture.scala | 2 +- .../workload/capture/SnapshotCapture.scala | 4 +- .../workload/deltaharness/DeltaHarness.scala | 7 + .../deltaharness/DeltaSparkHarness.scala | 9 + .../workload/engine/SnapshotResolver.scala | 18 ++ .../io/delta/workload/json/JsonUtil.scala | 29 +++- .../io/delta/workload/model/SpecModel.scala | 78 ++++++++- .../io/delta/workload/model/WriteModel.scala | 3 +- .../workload/validate/StatsValidator.scala | 120 +++++++++++++ .../workload/validate/WorkloadValidator.scala | 36 +++- .../workload/WorkloadGeneratorSuite.scala | 4 +- .../io/delta/workload/tables/CdcSuite.scala | 108 ++++++++++++ .../workload/tables/CheckpointsSuite.scala | 66 +++++++- .../io/delta/workload/tables/CrcSuite.scala | 79 +++++++++ .../workload/tables/WriteCommitSuite.scala | 17 ++ .../tables/WriteMetadataSpecsSuite.scala | 96 +++++++++++ 23 files changed, 1244 insertions(+), 25 deletions(-) create mode 100644 workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala create mode 100644 workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala create mode 100644 workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala create mode 100644 workload-generator/src/main/scala/io/delta/workload/validate/StatsValidator.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala diff --git a/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala b/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala index 5dbb951a..6de120e3 100644 --- a/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala +++ b/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala @@ -76,6 +76,9 @@ private[workload] class TableDecl( private[workload] def resolveOutputName(name: String): Unit = { _outputName = name } val readSpecs = mutable.ArrayBuffer[ReadSpecConfig]() val snapshotSpecs = mutable.ArrayBuffer[SnapshotSpecConfig]() + val cdfSpecs = mutable.ArrayBuffer[CdfSpecConfig]() + val checkpointSpecs = mutable.ArrayBuffer[CheckpointSpecConfig]() + val crcSpecs = mutable.ArrayBuffer[CrcSpecConfig]() val mutations = mutable.ArrayBuffer[Path => Unit]() var writeBuilder: Option[WriteSpecBuilder] = None } @@ -93,3 +96,24 @@ private[workload] case class SnapshotSpecConfig( val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => JsonUtil.mapper.treeToValue(n, classOf[SnapshotSpec]) } + +private[workload] case class CdfSpecConfig( + name: String, startVersion: Option[Long], endVersion: Option[Long], + startTimestamp: Option[String], endTimestamp: Option[String], + predicate: Option[String], columns: Option[Seq[String]], + expectError: Option[String] = None) extends HasAssertion[CdfSpec] { + val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => + JsonUtil.mapper.treeToValue(n, classOf[CdfSpec]) +} + +private[workload] case class CheckpointSpecConfig( + name: String, version: Long) extends HasAssertion[CheckpointSpec] { + val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => + JsonUtil.mapper.treeToValue(n, classOf[CheckpointSpec]) +} + +private[workload] case class CrcSpecConfig( + name: String, version: Long) extends HasAssertion[CrcSpec] { + val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => + JsonUtil.mapper.treeToValue(n, classOf[CrcSpec]) +} diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala index 46bc3e51..170fae8c 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala @@ -132,6 +132,44 @@ class WorkloadContext private[workload] ( if (parts.length == 1) "read_all" else parts.mkString("_") } + private[workload] def autoCdfName( + startVersion: Option[Long], + endVersion: Option[Long], + startTimestamp: Option[String], + endTimestamp: Option[String], + predicate: Option[String], + columns: Option[Seq[String]]): String = { + val parts = mutable.ArrayBuffer[String]("cdf") + startVersion.foreach(v => parts += s"v$v") + startTimestamp.foreach(ts => parts += "ts_" + ts.replace(":", "-").replace(" ", "_").take(23)) + parts += "to" + endVersion.foreach(v => parts += s"v$v") + endTimestamp.foreach(ts => parts += "ts_" + ts.replace(":", "-").replace(" ", "_").take(23)) + if (endVersion.isEmpty && endTimestamp.isEmpty) parts += "latest" + predicate.foreach(p => parts += simplifyPredicate(p)) + columns.foreach { cols => + parts += "cols_" + cols.take(3).mkString("_") + if (cols.size > 3) parts += s"plus${cols.size - 3}" + } + parts.mkString("_") + } + + /** Turn a SQL predicate into a filesystem-safe spec-name fragment. */ + private def simplifyPredicate(p: String): String = + p.toLowerCase + .replaceAll(">=", "_gte_").replaceAll("<=", "_lte_") + .replaceAll("<>", "_neq_").replaceAll("!=", "_neq_") + .replaceAll(">", "_gt_").replaceAll("<", "_lt_") + .replaceAll("=", "_eq_") + .replaceAll("\\s+and\\s+", "_and_").replaceAll("\\s+or\\s+", "_or_") + .replaceAll("\\s+is\\s+not\\s+null", "_is_not_null") + .replaceAll("\\s+is\\s+null", "_is_null") + .replaceAll("\\s+in\\s+", "_in_") + .replaceAll("\\s+", "_") + .replaceAll("[^a-z0-9_]", "") + .replaceAll("_+", "_").stripPrefix("_").stripSuffix("_") + .take(50) + // ---- Internal ---- private def ensureTableSpec(handle: TableHandle): Unit = { diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 052f92e9..cfb69a9c 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -23,7 +23,7 @@ import scala.collection.mutable import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession -import io.delta.workload.capture.{ReadCapture, SnapshotCapture, TableInfoCapture} +import io.delta.workload.capture.{CdfCapture, CheckpointCapture, CrcCapture, ReadCapture, SnapshotCapture, TableInfoCapture} import io.delta.workload.json.JsonUtil import io.delta.workload.model._ import io.delta.workload.validate.WorkloadValidator @@ -131,6 +131,37 @@ object WorkloadGenerator { builder.buildSpec(spark, destTablePath, testOutputDir) } + // The cdf/checkpoint/crc specs need no name tracking (no dedup/latest handling), only a count + // for the summary total. + var extraSpecCount = 0 + + // CDF specs + for (cs <- ts.cdfSpecs) { + val specPath = CdfCapture.capture(spark, dirName, destTablePath, testOutputDir, specsDir, + name = cs.name, startVersion = cs.startVersion, endVersion = cs.endVersion, + startTimestamp = cs.startTimestamp, endTimestamp = cs.endTimestamp, + predicate = cs.predicate, columns = cs.columns, expectError = cs.expectError) + extraSpecCount += 1 + checkAssertion(cs, specPath) + } + + // Checkpoint specs + for (cp <- ts.checkpointSpecs) { + val specPath = CheckpointCapture.capture(spark, dirName, destTablePath, specsDir, + version = cp.version, name = cp.name) + extraSpecCount += 1 + checkAssertion(cp, specPath) + } + + // CRC specs + for (crc <- ts.crcSpecs) { + CrcCapture.capture(spark, dirName, destTablePath, specsDir, + version = crc.version, name = crc.name).foreach { p => + extraSpecCount += 1 + checkAssertion(crc, p) + } + } + // table_info.json. An intentionally-mutated table (corruption / deleted-log tests) may be // unreadable and so can't produce it; an unmutated table that fails here is a real bug, so // only a mutated table tolerates the failure. @@ -156,7 +187,7 @@ object WorkloadGenerator { Files.write(reproDir.resolve("generate.scala"), s"// Generated by ${ts.outputName} test\n".getBytes("UTF-8")) - val total = snapshotNames.size + readNames.size + val total = snapshotNames.size + readNames.size + extraSpecCount println(s" $dirName: $total specs") dirName diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala index 2b304cd3..0e1d0020 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala @@ -103,6 +103,66 @@ trait WorkloadOps { new SpecRef(config) } + /** + * Change Data Feed spec over a version/timestamp range. Requires a start bound + * (`startVersion` or `startTimestamp`). + */ + def cdfSpec( + table: TableHandle, + startVersion: java.lang.Long = null, + endVersion: java.lang.Long = null, + startTimestamp: String = null, + endTimestamp: String = null, + predicate: String = null, + columns: Seq[String] = null, + name: String = null, + expectError: String = null): SpecRef[CdfSpec] = { + val ctx = WorkloadContext.current + val specName = if (name != null) name else ctx.autoCdfName( + ctx.toOption(startVersion), ctx.toOption(endVersion), Option(startTimestamp), + Option(endTimestamp), Option(predicate), Option(columns)) + ctx.requireUnique(table, specName) + val config = CdfSpecConfig( + specName, ctx.toOption(startVersion), ctx.toOption(endVersion), Option(startTimestamp), + Option(endTimestamp), Option(predicate), Option(columns), Option(expectError)) + ctx.getTableSpec(table).cdfSpecs += config + new SpecRef(config) + } + + /** + * Checkpoint spec. Forces a checkpoint at `version` and asserts the reconstructed + * protocol/metadata/txn/domain-metadata state, recorded inline in the spec JSON. + * Targets classic V1 single-file checkpoints. + */ + def checkpointSpec( + table: TableHandle, + version: Long, + name: String = null): SpecRef[CheckpointSpec] = { + val ctx = WorkloadContext.current + val specName = if (name != null) name else s"checkpoint_v$version" + ctx.requireUnique(table, specName) + val config = CheckpointSpecConfig(specName, version) + ctx.getTableSpec(table).checkpointSpecs += config + new SpecRef(config) + } + + /** + * CRC (version-checksum) spec. Reads the `.crc` file and asserts its core + * aggregate fields (plus protocol and any present optional fields), recorded inline + * in the spec JSON. + */ + def crcSpec( + table: TableHandle, + version: Long, + name: String = null): SpecRef[CrcSpec] = { + val ctx = WorkloadContext.current + val specName = if (name != null) name else s"crc_v$version" + ctx.requireUnique(table, specName) + val config = CrcSpecConfig(specName, version) + ctx.getTableSpec(table).crcSpecs += config + new SpecRef(config) + } + /** Force Spark to write a checkpoint file for the given SQL table name. */ def forceCheckpoint(tableName: String): Unit = current.forceCheckpoint(tableName) diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala new file mode 100644 index 00000000..99c5b952 --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala @@ -0,0 +1,155 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.capture + +import java.nio.file.{Files, Path} + +import scala.util.control.NonFatal + +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession} + +import io.delta.workload.deltaharness.DeltaHarness +import io.delta.workload.engine.{RowComparison, SnapshotResolver, SpecOutcome} +import io.delta.workload.json.JsonUtil +import io.delta.workload.model.{CdfExpected, CdfSpec, SpecError} + +object CdfCapture { + + def capture( + spark: SparkSession, testId: String, tablePath: Path, + outputDir: Path, specsDir: Path, name: String, + startVersion: Option[Long] = None, endVersion: Option[Long] = None, + startTimestamp: Option[String] = None, endTimestamp: Option[String] = None, + predicate: Option[String] = None, columns: Option[Seq[String]] = None, + expectError: Option[String] = None): Path = { + + val specName = s"${testId}_$name" + require(startVersion.isDefined || startTimestamp.isDefined, + s"CDF $specName: a start bound (startVersion or startTimestamp) is required") + require(!(startVersion.isDefined && startTimestamp.isDefined), + s"CDF $specName: cannot specify both startVersion and startTimestamp") + require(!(endVersion.isDefined && endTimestamp.isDefined), + s"CDF $specName: cannot specify both endVersion and endTimestamp") + + val expectedDir = outputDir.resolve("expected").resolve(specName) + Files.createDirectories(expectedDir) + val specPath = specsDir.resolve(s"$specName.json") + + val (expected, expectedError) = try { + var df = buildReader(spark, tablePath, startVersion, endVersion, + startTimestamp, endTimestamp) + df = SnapshotResolver.applyFilters(df, predicate, columns) + df = df.drop("_commit_timestamp") + df = df.cache() + val count = df.count() + + try { + writeExpectedData(expectedDir, df) + (Some(CdfExpected(count)), None) + } finally { + df.unpersist() + } + } catch { + case NonFatal(e) => + (None, Some(SpecError(SpecOutcome.extractErrorCode(e), Option(e.getMessage).getOrElse("")))) + } + + // If the test author declared `expectError`, the operation MUST throw. + // See ReadCapture.capture for full semantics. + expectError.foreach { expected => + def fail(msg: String): Nothing = { + if (expectedDir.toFile.exists()) FileUtils.deleteDirectory(expectedDir.toFile) + throw new RuntimeException(msg) + } + expectedError match { + case None => + fail(s"CDF $specName: declared expectError=" + + (if (expected.isEmpty) "(any)" else s"'$expected'") + + " but operation succeeded") + case Some(err) if expected.nonEmpty && + SpecOutcome.normalizeErrorCode(err.errorCode) != + SpecOutcome.normalizeErrorCode(expected) => + fail(s"CDF $specName: declared expectError='$expected' but got " + + s"'${err.errorCode}'") + case _ => // matches + } + } + + val spec = CdfSpec(startVersion, endVersion, startTimestamp, endTimestamp, + predicate, columns, expected, expectedError) + JsonUtil.writeSpec(specPath, spec) + validateFromSpec(spark, tablePath, expectedDir, specPath) + + (expected, expectedError) match { + case (Some(exp), _) => + println(s" CDF captured: $specName (${exp.rowCount} change rows)") + case (_, Some(err)) => + println(s" CDF captured (error): $specName [${err.errorCode}] ${err.errorMessage}") + case _ => + } + specPath + } + + def validateFromSpec(spark: SparkSession, tablePath: Path, + expectedDir: Path, specPath: Path): Unit = { + val spec = JsonUtil.readSpecAs(specPath, classOf[CdfSpec]) + val specName = specPath.getFileName.toString.stripSuffix(".json") + + def reader = SnapshotResolver.applyFilters( + buildReader(spark, tablePath, spec.startVersion, spec.endVersion, + spec.startTimestamp, spec.endTimestamp), + spec.predicate, spec.columns).drop("_commit_timestamp") + + if (spec.expectedError.isDefined) { + val actualCode = try { + reader.write.format("noop").mode("overwrite").save() + None + } catch { + case NonFatal(e) => Some(SpecOutcome.extractErrorCode(e)) + } + require(actualCode.isDefined, + s"Error validation FAILED for $specName: expected operation to fail but it succeeded") + } else if (spec.expected.isDefined) { + val rereadDf = reader + + val expectedDataPath = expectedDir.resolve("expected_data") + require(Files.exists(expectedDataPath), + s"Validation FAILED for $specName: expected_data is missing") + RowComparison.assertRowsEqual( + spark.read.parquet(expectedDataPath.toString), rereadDf, specName) + } + } + + private def writeExpectedData(expectedDir: Path, df: DataFrame): Unit = { + val path = expectedDir.resolve("expected_data") + if (path.toFile.exists()) FileUtils.deleteDirectory(path.toFile) + df.write.mode(SaveMode.Overwrite).parquet(path.toString) + } + + /** + * Build the change-feed reader, resolving the table's latest version through the + * neutral [[DeltaHarness]] SPI so an open-ended range stays bounded. + */ + private def buildReader(spark: SparkSession, tablePath: Path, + startVersion: Option[Long], endVersion: Option[Long], + startTimestamp: Option[String], endTimestamp: Option[String]): DataFrame = { + val latestVersion = DeltaHarness.get.openLog(spark, tablePath.toString).update().version + SnapshotResolver.buildCdfReader(spark, tablePath, startVersion, endVersion, + startTimestamp, endTimestamp, latestVersion) + } +} diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala new file mode 100644 index 00000000..1c536e6b --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala @@ -0,0 +1,121 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.capture + +import java.nio.file.{Files, Path} + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.SparkSession + +import io.delta.workload.deltaharness.{DeltaHarness, ResolvedSnapshot} +import io.delta.workload.json.JsonUtil +import io.delta.workload.model.{CheckpointExpected, CheckpointFile, CheckpointSpec, MetadataInfo, ProtocolInfo} + +/** + * Captures and validates checkpoint specs. A checkpoint is forced at the target version via + * the neutral `LogView.checkpoint` SPI, then its reconstructed state (protocol, metadata, + * set-transactions, domain-metadata) is read back through the neutral `ResolvedSnapshot` + * accessors and recorded inline in the spec JSON, mirroring [[SnapshotCapture]]. + * + * Scope: targets classic V1 single-file checkpoints. The `.checkpoint.parquet` produced by + * `checkpoint(version)` is left in place under the table's `_delta_log/` so consumers can exercise + * checkpoint reads. + */ +object CheckpointCapture { + + def capture( + spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, + version: Long, name: String): Path = { + + val specName = s"${testId}_$name" + val specPath = specsDir.resolve(s"$specName.json") + + val harness = DeltaHarness.get + harness.openLog(spark, tablePath.toString).checkpoint(version) + require(checkpointFileExists(tablePath, version), + s"Checkpoint $specName: no checkpoint file present at version $version after checkpoint()") + + val snapshot = harness.openLog(spark, tablePath.toString).getSnapshotAt(version) + val expected = Some(buildExpected(snapshot)) + + val spec = CheckpointSpec(version, expected) + JsonUtil.writeSpec(specPath, spec) + validateFromSpec(spark, tablePath, specPath) + + println(s" Checkpoint captured: $specName (version=$version)") + specPath + } + + /** + * Validate a checkpoint spec Protocol/metadata/txn/domain-metadata against the table at `tablePath`. + */ + def validateFromSpec( + spark: SparkSession, tablePath: Path, specPath: Path, + isWriteValidation: Boolean = false): Unit = { + val spec = JsonUtil.readSpecAs(specPath, classOf[CheckpointSpec]) + val specName = specPath.getFileName.toString.stripSuffix(".json") + val harness = DeltaHarness.get + + if (isWriteValidation) { + harness.openLog(spark, tablePath.toString).checkpoint(spec.version) + require(checkpointFileExists(tablePath, spec.version), + s"Checkpoint validation failed for $specName: no checkpoint file at version " + + s"${spec.version} after checkpoint()") + } + + spec.expected.foreach { exp => + val resolved = harness.openLog(spark, tablePath.toString).getSnapshotAt(spec.version) + val live = resolved.snapshot + SnapshotCapture.assertMatches(exp.protocol, live.protocol, exp.metadata, live.metadata, + isWriteValidation, specName) + require(exp.txn.getOrElse(Seq.empty).toSet == resolved.setTransactions.toSet, + s"Checkpoint validation failed for $specName: txn mismatch") + require(exp.domainMetadata.getOrElse(Seq.empty).toSet == resolved.domainMetadata.toSet, + s"Checkpoint validation failed for $specName: domainMetadata mismatch") + if (!isWriteValidation) { + require(exp.files.toSet == fileSet(resolved).toSet, + s"Checkpoint validation failed for $specName: file set mismatch") + } + } + } + + /** Build the expected checkpoint state from a snapshot via the neutral SPI accessors. */ + private def buildExpected(snapshot: ResolvedSnapshot): CheckpointExpected = { + val txn = snapshot.setTransactions + val domainMetadata = snapshot.domainMetadata + CheckpointExpected( + protocol = ProtocolInfo.from(snapshot.snapshot.protocol), + metadata = MetadataInfo.from(snapshot.snapshot.metadata), + files = fileSet(snapshot), + txn = if (txn.nonEmpty) Some(txn) else None, + domainMetadata = if (domainMetadata.nonEmpty) Some(domainMetadata) else None) + } + + private def fileSet(s: ResolvedSnapshot): Seq[CheckpointFile] = + s.allFiles.select("path", "size").collect() + .map(r => CheckpointFile(r.getString(0), r.getLong(1))).toSeq + + /** True iff a classic checkpoint file (`.checkpoint*`) exists in `_delta_log`. */ + private def checkpointFileExists(tablePath: Path, version: Long): Boolean = { + val logDir = tablePath.resolve("_delta_log") + val prefix = f"$version%020d.checkpoint" + val stream = Files.list(logDir) + try stream.iterator().asScala.exists(_.getFileName.toString.startsWith(prefix)) + finally stream.close() + } +} diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala new file mode 100644 index 00000000..5ceaa65c --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala @@ -0,0 +1,160 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.capture + +import java.nio.file.{Files, Path} + +import org.apache.spark.sql.SparkSession + +import io.delta.workload.deltaharness.DeltaHarness +import io.delta.workload.json.JsonUtil +import io.delta.workload.model.{CrcExpected, CrcSpec, ProtocolInfo} + +/** + * Captures and validates CRC (version-checksum) specs. The `.crc` file is plain JSON + * in `_delta_log/`, so it is read directly with [[java.nio.file.Files]] + [[JsonUtil.mapper]] — + * no Delta class and no SPI method are needed. The protocol is read from the neutral + * `ResolvedSnapshot` so it compares engine-neutrally; all other fields come straight from the + * `.crc`, keyed by delta-spark's `VersionChecksum` field names. Optional/feature-gated fields + * (deletion-vector counts, set transactions, inCommitTimestamp) are recorded only when the `.crc` + * carries them, and validated conditionally. + */ +object CrcCapture { + + /** + * Capture a crc spec for `version`, or skip if no `.crc` exists there. + * + * A crc spec asserts the *contents* of a `.crc` where one is present; absence is a + * skip, not a failure. Some engines flush `.crc` asynchronously, so the latest + * commit's `.crc` is often not on disk at read time. When the target `.crc` is missing this + * logs a SKIP, emits no spec, and returns `None`. + * + * @return `Some(specPath)` if a spec was emitted and self-validated, `None` if skipped (no `.crc`). + */ + def capture( + spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, + version: Long, name: String): Option[Path] = { + + val specName = s"${testId}_$name" + val specPath = specsDir.resolve(s"$specName.json") + + val crcNode = readCrc(tablePath, version) match { + case Some(node) => node + case None => + println(s" CRC skipped: $specName (version=$version): " + + s"no .crc at version $version; engine did not write/flush one") + return None + } + + val harness = DeltaHarness.get + val snapshot = harness.openLog(spark, tablePath.toString).getSnapshotAt(version) + val protocol = ProtocolInfo.from(snapshot.snapshot.protocol) + + val expected = CrcExpected( + tableSizeBytes = JsonUtil.crcLongField(crcNode, "tableSizeBytes"), + numFiles = JsonUtil.crcLongField(crcNode, "numFiles"), + numDeletionVectors = JsonUtil.crcLongField(crcNode, "numDeletionVectorsOpt"), + numDeletedRecords = JsonUtil.crcLongField(crcNode, "numDeletedRecordsOpt"), + inCommitTimestamp = JsonUtil.crcLongField(crcNode, "inCommitTimestampOpt"), + protocol = Some(protocol), + setTransactions = JsonUtil.crcSetTransactions(crcNode)) + + val spec = CrcSpec(version, Some(expected)) + JsonUtil.writeSpec(specPath, spec) + validateFromSpec(spark, tablePath, specPath) + + println(s" CRC captured: $specName (version=$version)") + Some(specPath) + } + + /** + * Validate a crc spec's (deletion-vector / deleted-record counts), protocol, and set transactions + * against the table at `tablePath`. + */ + def validateFromSpec( + spark: SparkSession, tablePath: Path, specPath: Path, + isWriteValidation: Boolean = false): Unit = { + val spec = JsonUtil.readSpecAs(specPath, classOf[CrcSpec]) + val specName = specPath.getFileName.toString.stripSuffix(".json") + val harness = DeltaHarness.get + + spec.expected.foreach { exp => + val crcNode = readCrc(tablePath, spec.version) match { + case Some(node) => node + case None => + // Validate-where-present: a missing `.crc` (e.g. async flush) is a skip, not a failure. + println(s" CRC validation skipped for $specName: " + + s"no .crc at version ${spec.version}; engine did not write/flush one") + return + } + + // === Aggregate fields (VersionChecksum keys) === + val scalarFields = + if (isWriteValidation) ScalarFields.filterNot { case (l, _, _) => NonReproducibleScalarFields(l) } + else ScalarFields + scalarFields.foreach { case (label, key, get) => + requireLong(specName, label, get(exp), JsonUtil.crcLongField(crcNode, key)) + } + + // === Protocol (engine-neutral, via ResolvedSnapshot) === + exp.protocol.foreach { expProtocol => + val snapshot = harness.openLog(spark, tablePath.toString).getSnapshotAt(spec.version) + require(expProtocol == ProtocolInfo.from(snapshot.snapshot.protocol), + s"CRC validation failed for $specName: protocol mismatch") + } + + // === Set transactions === + exp.setTransactions.foreach { expTxn => + require(Some(expTxn.toSet) == JsonUtil.crcSetTransactions(crcNode).map(_.toSet), + s"CRC validation failed for $specName: setTransactions mismatch") + } + } + } + + // Scalar `VersionChecksum` fields checked uniformly: (spec label, `.crc` key, spec accessor). + private val ScalarFields: Seq[(String, String, CrcExpected => Option[Long])] = Seq( + ("tableSizeBytes", "tableSizeBytes", _.tableSizeBytes), + ("numFiles", "numFiles", _.numFiles), + ("numDeletionVectors", "numDeletionVectorsOpt", _.numDeletionVectors), + ("numDeletedRecords", "numDeletedRecordsOpt", _.numDeletedRecords), + ("inCommitTimestamp", "inCommitTimestampOpt", _.inCommitTimestamp)) + + private val NonReproducibleScalarFields: Set[String] = + Set("tableSizeBytes", "numFiles", "inCommitTimestamp") + + /** Read the `.crc` JSON, or `None` when no such file is on disk. */ + private def readCrc( + tablePath: Path, version: Long): Option[com.fasterxml.jackson.databind.JsonNode] = { + val crcFile = tablePath.resolve("_delta_log").resolve(f"$version%020d.crc") + if (!Files.exists(crcFile)) None + else Some(JsonUtil.mapper.readTree(new String(Files.readAllBytes(crcFile), "UTF-8"))) + } + + // A field the spec recorded (Some) must be present in the `.crc` under the same key: a + // spec-Some / crc-None mismatch means the writer dropped a field it should have written. + // A field the spec did not record (None, e.g. a feature that was off) is nothing to check. + private def requireLong(specName: String, field: String, + expected: Option[Long], actual: Option[Long]): Unit = + (expected, actual) match { + case (Some(e), Some(a)) => + require(e == a, s"CRC validation failed for $specName: $field expected=$e actual=$a") + case (Some(e), None) => + require(false, s"CRC validation failed for $specName: $field expected=$e " + + "but the .crc has no such field") + case (None, _) => // spec did not record this field + } +} diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/ReadCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/ReadCapture.scala index 7082ab7a..2f31e8c4 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/ReadCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/ReadCapture.scala @@ -111,7 +111,7 @@ object ReadCapture { def validateFromSpec(spark: SparkSession, tablePath: Path, expectedDir: Path, specPath: Path, checkMetadata: Boolean = true): Unit = { - val spec = JsonUtil.readReadSpec(specPath) + val spec = JsonUtil.readSpecAs(specPath, classOf[ReadSpec]) val specName = specPath.getFileName.toString.stripSuffix(".json") val harness = DeltaHarness.get diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala index ce700a5b..67ad4801 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala @@ -86,7 +86,7 @@ object SnapshotCapture { */ def validateFromSpec( spark: SparkSession, tablePath: Path, specPath: Path, isWriteValidation: Boolean = false): Unit = { - val spec = JsonUtil.readSnapshotSpec(specPath) + val spec = JsonUtil.readSpecAs(specPath, classOf[SnapshotSpec]) val specName = specPath.getFileName.toString.stripSuffix(".json") val harness = DeltaHarness.get def resolve = @@ -109,7 +109,7 @@ object SnapshotCapture { private val NonReproducibleConfigKeys = Set("delta.columnMapping.maxColumnId") private val ColumnMappingFieldKeys = Seq("delta.columnMapping.id", "delta.columnMapping.physicalName") - private def assertMatches( + private[capture] def assertMatches( expProtocol: ProtocolInfo, actProtocol: Protocol, expMeta: MetadataInfo, actMeta: Metadata, isWriteValidation: Boolean, name: String): Unit = { def requireEq(field: String, exp: Any, act: Any): Unit = diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala index 0203fcb9..851113be 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala @@ -99,6 +99,7 @@ trait LogView { def update(): ResolvedSnapshot def getSnapshotAt(version: Long): ResolvedSnapshot def checkpoint(): Unit + def checkpoint(version: Long): Unit } trait ResolvedSnapshot { @@ -116,6 +117,12 @@ trait ResolvedSnapshot { * ordering or extra fields. */ def allFiles: DataFrame + + /** Active domain-metadata entries (tombstones excluded). */ + def domainMetadata: Seq[AddDomainMetadata] + + /** Active set-transaction entries. */ + def setTransactions: Seq[AppTxn] } // Neutral typed view of a snapshot's protocol + metadata. SPI-side types (no dependency on diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala index da337182..4b26dab7 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.delta.actions.{Action, AddFile, DomainMetadata, File // Hide Spark's types.Metadata so the unqualified `Metadata` is this package's typed case class. import org.apache.spark.sql.types.{Metadata => _, _} +import io.delta.workload.model.{AddDomainMetadata, AppTxn} + class DeltaSparkHarness extends SparkRowMaterializer { override def openLog(spark: SparkSession, tablePath: String): LogView = { DeltaLog.clearCache() @@ -86,6 +88,7 @@ private class DeltaSparkLogView(inner: DeltaLog) extends LogView { override def getSnapshotAt(version: Long): ResolvedSnapshot = new DeltaSparkResolvedSnapshot(inner.getSnapshotAt(version)) override def checkpoint(): Unit = inner.checkpoint() + override def checkpoint(version: Long): Unit = inner.checkpoint(inner.getSnapshotAt(version)) } private class DeltaSparkResolvedSnapshot(inner: DeltaSnapshot) extends ResolvedSnapshot { @@ -119,4 +122,10 @@ private class DeltaSparkResolvedSnapshot(inner: DeltaSnapshot) extends ResolvedS ds.map(f => (f.path, f.size, f.partitionValues, f.json)) .toDF("path", "size", "partitionValues", "json") } + + override def domainMetadata: Seq[AddDomainMetadata] = + inner.domainMetadata.toSeq.map(dm => AddDomainMetadata(dm.domain, dm.configuration)) + + override def setTransactions: Seq[AppTxn] = + inner.setTransactions.toSeq.map(t => AppTxn(t.appId, t.version)) } diff --git a/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala b/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala index ca410fa0..19e0b065 100644 --- a/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala +++ b/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala @@ -45,6 +45,24 @@ object SnapshotResolver { reader.load(tablePath.toString) } + /** + * Build a Change Data Feed reader over a version/timestamp range. + */ + def buildCdfReader(spark: SparkSession, tablePath: Path, + startVersion: Option[Long], endVersion: Option[Long], + startTimestamp: Option[String], endTimestamp: Option[String], + latestVersion: Long): DataFrame = { + var reader = spark.read.format("delta").option("readChangeFeed", "true") + startVersion.foreach(v => reader = reader.option("startingVersion", v)) + startTimestamp.foreach(ts => reader = reader.option("startingTimestamp", ts)) + endVersion.foreach(v => reader = reader.option("endingVersion", v)) + endTimestamp.foreach(ts => reader = reader.option("endingTimestamp", ts)) + if (endVersion.isEmpty && endTimestamp.isEmpty) { + reader = reader.option("endingVersion", latestVersion) + } + reader.load(tablePath.toString) + } + def applyFilters(df: DataFrame, predicate: Option[String], columns: Option[Seq[String]]): DataFrame = { var result = df diff --git a/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala b/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala index e288ec40..a6097956 100644 --- a/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala +++ b/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.apache.spark.sql.types.{DataType, StructType} -import io.delta.workload.model.{Failed, ReadQuery, ReadResult, ReadSpec, SnapshotQuery, SnapshotResult, SnapshotSpec, Spec, SpecError, SpecExpectation, Succeeded} +import io.delta.workload.model._ /** * Flattens the outcome to top-level `expected` (success) XOR `error` (failure). `@JsonUnwrapped` @@ -143,14 +143,27 @@ object JsonUtil { Files.write(path, prettyWriter.writeValueAsBytes(spec)) - def readReadSpec(path: Path): ReadSpec = - mapper.readValue(Files.readAllBytes(path), classOf[ReadSpec]) - - def readSnapshotSpec(path: Path): SnapshotSpec = - mapper.readValue(Files.readAllBytes(path), classOf[SnapshotSpec]) + /** Deserialize a spec file as the given concrete type; call sites pass `classOf[...]`. */ + def readSpecAs[T](path: Path, cls: Class[T]): T = + mapper.readValue(Files.readAllBytes(path), cls) /** Deserialize any spec by its `type` tag into the sealed [[Spec]]. */ - def readSpec(path: Path): Spec = - mapper.readValue(Files.readAllBytes(path), classOf[Spec]) + def readSpec(path: Path): Spec = readSpecAs(path, classOf[Spec]) + + // === CRC field accessors (operate on the raw `.crc` JSON tree) === + + def crcLongField(node: JsonNode, field: String): Option[Long] = + Option(node.get(field)).filterNot(_.isNull).map(_.asLong()) + + /** Parse the `setTransactions` field of a `.crc`, which may be a single object or an array. */ + def crcSetTransactions(node: JsonNode): Option[Seq[AppTxn]] = + Option(node.get("setTransactions")).filterNot(_.isNull).flatMap { txnNode => + val actions = + if (txnNode.isArray) txnNode.elements().asScala.toSeq + else if (txnNode.isObject) Seq(txnNode) + else Seq.empty + if (actions.isEmpty) None + else Some(actions.map(t => AppTxn(t.get("appId").asText(), t.get("version").asLong()))) + } } diff --git a/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala b/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala index 1021e8e7..33949db1 100644 --- a/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala +++ b/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala @@ -61,6 +61,48 @@ object MetadataInfo { @JsonPropertyOrder(Array("protocol", "metadata")) case class SnapshotResult(protocol: ProtocolInfo, metadata: MetadataInfo) +// ============================================================================= +// Expected (success-outcome) types for the cdf/checkpoint/crc specs +// +// Each type is the `expected` payload recorded inline in its spec JSON, mirroring the read/snapshot +// specs' outcome. The reconstructed protocol/metadata reuse the shared typed [[ProtocolInfo]] / +// [[MetadataInfo]]; txn pairs reuse [[AppTxn]]. +// ============================================================================= + +/** Change Data Feed success data. */ +case class CdfExpected(rowCount: Long) + +/** A single live Add-file entry in a checkpoint's reconstructed state. */ +case class CheckpointFile(path: String, size: Long) + +/** Checkpoint success data. The `txn`/`domainMetadata` sets are omitted when empty. */ +@JsonPropertyOrder(Array("protocol", "metadata", "files", "txn", "domainMetadata")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class CheckpointExpected( + protocol: ProtocolInfo, + metadata: MetadataInfo, + files: Seq[CheckpointFile] = Nil, + txn: Option[Seq[AppTxn]] = None, + domainMetadata: Option[Seq[AddDomainMetadata]] = None) + +/** + * CRC (version checksum) success data, keyed to the fields delta-spark serializes into a + * `.crc` (`VersionChecksum`). Optional fields appear only when the `.crc` carries them: + * deletion-vector counts require the feature, `inCommitTimestamp` requires ICT, `setTransactions` + * appears only when the table has any. + */ +@JsonPropertyOrder(Array("tableSizeBytes", "numFiles", "numDeletionVectors", "numDeletedRecords", + "inCommitTimestamp", "protocol", "setTransactions")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class CrcExpected( + tableSizeBytes: Option[Long] = None, + numFiles: Option[Long] = None, + numDeletionVectors: Option[Long] = None, + numDeletedRecords: Option[Long] = None, + inCommitTimestamp: Option[Long] = None, + protocol: Option[ProtocolInfo] = None, + setTransactions: Option[Seq[AppTxn]] = None) + // ============================================================================= // Shared read/snapshot query types // @@ -97,7 +139,10 @@ case class SnapshotQuery( @JsonSubTypes(Array( new JsonSubTypes.Type(value = classOf[ReadSpec], name = "read"), new JsonSubTypes.Type(value = classOf[SnapshotSpec], name = "snapshot"), - new JsonSubTypes.Type(value = classOf[WriteSpec], name = "write"))) + new JsonSubTypes.Type(value = classOf[WriteSpec], name = "write"), + new JsonSubTypes.Type(value = classOf[CdfSpec], name = "cdf"), + new JsonSubTypes.Type(value = classOf[CheckpointSpec], name = "checkpoint"), + new JsonSubTypes.Type(value = classOf[CrcSpec], name = "crc"))) sealed trait Spec { def `type`: String } @JsonSerialize(using = classOf[ReadSpecSerializer]) @@ -121,6 +166,37 @@ case class WriteSpec(commits: Seq[WriteCommit]) extends Spec { val `type`: String = "write" } +@JsonPropertyOrder(Array("type", "startVersion", "startTimestamp", "endVersion", "endTimestamp", + "predicate", "columns", "expected", "expectedError")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class CdfSpec( + startVersion: Option[Long] = None, + endVersion: Option[Long] = None, + startTimestamp: Option[String] = None, + endTimestamp: Option[String] = None, + predicate: Option[String] = None, + columns: Option[Seq[String]] = None, + expected: Option[CdfExpected] = None, + expectedError: Option[SpecError] = None) extends Spec { + val `type`: String = "cdf" +} + +@JsonPropertyOrder(Array("type", "version", "expected")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class CheckpointSpec( + version: Long, + expected: Option[CheckpointExpected] = None) extends Spec { + val `type`: String = "checkpoint" +} + +@JsonPropertyOrder(Array("type", "version", "expected")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class CrcSpec( + version: Long, + expected: Option[CrcExpected] = None) extends Spec { + val `type`: String = "crc" +} + /** The spec outcome: `Succeeded { expected } | Failed { error }`. */ sealed trait SpecExpectation[+R] case class Succeeded[+R](result: R) extends SpecExpectation[R] diff --git a/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala b/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala index b1dc6539..9a6dc863 100644 --- a/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala +++ b/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala @@ -53,7 +53,8 @@ case class AddFileInput(rows: Seq[Map[String, Any]]) */ case class CommitOrdinal(value: Int) -// AppTxn/AddDomainMetadata are the minimal spec DTOs: only what a write spec declares and +// AppTxn/AddDomainMetadata are the minimal spec DTOs, shared as BOTH write-spec input AND captured +// output (the checkpoint/crc `expected` payloads reuse them): only what a write spec declares and // serializes. They form their own layer, separate from io.delta.workload.log.{Txn,DomainMetadata} // (log-fidelity parse types carrying lastUpdated/removed) and Spark's SetTransaction/DomainMetadata // (the engine boundary). diff --git a/workload-generator/src/main/scala/io/delta/workload/validate/StatsValidator.scala b/workload-generator/src/main/scala/io/delta/workload/validate/StatsValidator.scala new file mode 100644 index 00000000..df3c3a0a --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/validate/StatsValidator.scala @@ -0,0 +1,120 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.validate + +import java.nio.file.Path + +import scala.collection.mutable + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.types.StructType + +import io.delta.workload.deltaharness.DeltaHarness +import io.delta.workload.json.JsonUtil + +/** + * Validates the per-file statistics an engine wrote, on a replayed (engine-written) table. + * + * This is a PROPERTY check, deliberately not an expected-value compare against the corpus: the + * Delta protocol mandates stats SOUNDNESS and FORMAT, but leaves tightness (and even presence) + * optional, so a sound writer may legitimately emit wider — or no — stats. Comparing to a captured + * expected would false-fail such a writer; asserting the invariants does not. + * + * TODO: also check `numRecords` soundness (== the data file's row count) and per-column min/max + * soundness (stats.min <= real_min <= real_max <= stats.max); both need reading the (possibly + * URL-encoded) data-file paths. + */ +object StatsValidator { + + /** Validate the stats on the table at `tablePath`. Returns findings; empty means compliant. */ + def validate(spark: SparkSession, tablePath: Path): Seq[String] = { + val snapshot = DeltaHarness.get.openLog(spark, tablePath.toString).update() + val timestampCols = timestampColumns(snapshot.snapshot.metadata.schema) + if (timestampCols.isEmpty) return Seq.empty // no timestamp columns to check + + val findings = mutable.ArrayBuffer[String]() + val addJsons = snapshot.allFiles.select("json").collect().map(_.getString(0)) + for (addJson <- addJsons) { + val add = JsonUtil.mapper.readTree(addJson).path("add") + val statsNode = add.path("stats") + if (statsNode.isTextual) { + val stats = JsonUtil.mapper.readTree(statsNode.asText) + checkTimestampPrecision(add.path("path").asText(""), stats, timestampCols, findings) + } + } + findings.toSeq + } + + /** + * Segment lists of every timestamp / timestamp_ntz field, recursing through nested structs + * (a struct-nested timestamp is serialized in stats under the same nesting, e.g. `info` -> `ts`). + * Returning segment lists rather than dotted strings avoids the ambiguity between a top-level + * column literally named `a.b` and a nested `a` -> `b`. + * Descends only into structs: Delta collects no min/max stats inside arrays or maps, and + * partition columns live in `partitionValues` with a different serialization (out of scope). + * (`typeName` avoids depending on the `TimestampNTZType` symbol, which older Spark versions lack.) + */ + private def timestampColumns(schema: StructType): Set[Seq[String]] = { + def collect(struct: StructType, prefix: Seq[String]): Seq[Seq[String]] = + struct.fields.toSeq.flatMap { f => + val path = prefix :+ f.name + f.dataType match { + case nested: StructType => collect(nested, path) + case dt if dt.typeName == "timestamp" || dt.typeName == "timestamp_ntz" => Seq(path) + case _ => Seq.empty + } + } + collect(schema, Nil).toSet + } + + /** + * Timestamp-precision check: a timestamp min/max stat must be millisecond-truncated (<= 3 + * fractional digits). Delta serializes stats timestamps as ISO-8601 at millisecond granularity; + * microsecond (`.298677Z`) or nanosecond values violate the protocol and can break skipping + * under some readers. + */ + private def checkTimestampPrecision( + path: String, + stats: com.fasterxml.jackson.databind.JsonNode, + timestampPaths: Set[Seq[String]], + findings: mutable.ArrayBuffer[String]): Unit = { + for (bound <- Seq("minValues", "maxValues")) { + val boundNode = stats.path(bound) + for (colPath <- timestampPaths) { + // Walk the path segments into the (possibly nested) stats object. + val v = colPath.foldLeft(boundNode)((node, seg) => node.path(seg)) + if (v.isTextual) { + fractionalDigits(v.asText).foreach { n => + if (n > 3) { + val col = colPath.mkString(".") + findings += s"$path: $bound.$col = '${v.asText}' has $n fractional-second digits; " + + "Delta stats timestamps must be millisecond-truncated (<= 3)" + } + } + } + } + } + } + + /** Digit count of the fractional-seconds part of an ISO-8601-ish timestamp, or None if absent. */ + private def fractionalDigits(s: String): Option[Int] = { + val dot = s.indexOf('.') + if (dot < 0) return None + val n = s.drop(dot + 1).takeWhile(_.isDigit).length + if (n > 0) Some(n) else None + } +} diff --git a/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala index 94b73576..5b2e2710 100644 --- a/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala @@ -28,7 +28,7 @@ import scala.util.control.NonFatal import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession -import io.delta.workload.capture.{ReadCapture, SnapshotCapture} +import io.delta.workload.capture.{CdfCapture, CheckpointCapture, CrcCapture, ReadCapture, SnapshotCapture} import io.delta.workload.json.JsonUtil import io.delta.workload.model._ import io.delta.workload.write.WriteReplay @@ -121,9 +121,9 @@ object WorkloadValidator { else SpecFailed(id, s"read-only spec but no captured table at $deltaTable") try spec match { case _: WriteSpec => - // Basic validation: triggering the replay (queryTarget) reconstructs the table and - // asserts replay succeeds + finalVersion == commits.size-1. Row content is checked by - // the baseline 'latest' read spec; per-version metadata by the snapshot spec. + // Triggering the replay reconstructs the table and asserts replay succeeds + + // finalVersion == commits.size-1. Rows are checked by the baseline read spec, per-version + // metadata by the snapshot spec, and per-file stats by the separate stats pass below. val _ = queryTarget; SpecPassed(id) case _: ReadSpec => val expectedDir = testDir.resolve("expected").resolve(bare) @@ -135,13 +135,37 @@ object WorkloadValidator { if (writeDerived) { SnapshotCapture.validateFromSpec(spark, queryTarget, specFile, isWriteValidation = true); SpecPassed(id) } else readOnly(t => SnapshotCapture.validateFromSpec(spark, t, specFile)) + case _: CdfSpec => + val expectedDir = testDir.resolve("expected").resolve(bare) + if (writeDerived) { + CdfCapture.validateFromSpec(spark, queryTarget, expectedDir, specFile); SpecPassed(id) + } else readOnly(t => CdfCapture.validateFromSpec(spark, t, expectedDir, specFile)) + case _: CheckpointSpec => + if (writeDerived) { + CheckpointCapture.validateFromSpec(spark, queryTarget, specFile, isWriteValidation = true) + SpecPassed(id) + } else readOnly(t => CheckpointCapture.validateFromSpec(spark, t, specFile)) + case _: CrcSpec => + if (writeDerived) { + CrcCapture.validateFromSpec(spark, queryTarget, specFile, isWriteValidation = true) + SpecPassed(id) + } else readOnly(t => CrcCapture.validateFromSpec(spark, t, specFile)) } catch { case e: Throwable => SpecFailed(id, e.toString) } } - try outcomes ++= parsed.map { case (f, spec) => validateOne(f, spec) } - finally tempDirs.foreach(td => try FileUtils.deleteDirectory(td.toFile) catch { case NonFatal(_) => }) + try { + outcomes ++= parsed.map { case (f, spec) => validateOne(f, spec) } + if (writeDerived) { + val statsId = s"${testDir.getFileName}/stats" + outcomes += (try { + val findings = StatsValidator.validate(spark, queryTarget) + if (findings.isEmpty) SpecPassed(statsId) + else SpecFailed(statsId, s"stats: ${findings.mkString("; ")}") + } catch { case NonFatal(e) => SpecFailed(statsId, e.toString) }) + } + } finally tempDirs.foreach(td => try FileUtils.deleteDirectory(td.toFile) catch { case NonFatal(_) => }) ValidationResult(outcomes.toSeq) } diff --git a/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala b/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala index 39e02cca..b85a54d7 100644 --- a/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala @@ -318,7 +318,7 @@ class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor readSpec(t, version = 999) } assertPassed(results) - val spec = JsonUtil.readReadSpec(specs("t_e1").resolve("t_e1_read_v999.json")) + val spec = JsonUtil.readSpecAs(specs("t_e1").resolve("t_e1_read_v999.json"), classOf[ReadSpec]) assert(spec.query.version.contains(999L)) spec.expectation match { case Failed(err) => @@ -720,7 +720,7 @@ class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor readSpec(t) // should produce error spec } assertPassed(results) - val spec = JsonUtil.readReadSpec(specs("t_corrupt").resolve("t_corrupt_read_all.json")) + val spec = JsonUtil.readSpecAs(specs("t_corrupt").resolve("t_corrupt_read_all.json"), classOf[ReadSpec]) spec.expectation match { case Failed(err) => // Error can be FileNotFoundException, SparkException wrapping it, or other file-related errors diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala new file mode 100644 index 00000000..dc2938d7 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala @@ -0,0 +1,108 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import io.delta.workload.WorkloadTestSuite + +/** + * Change Data Feed workloads. Each test builds a CDF-enabled table across several + * versions, then declares `cdfSpec`s over version ranges. The captured change rows + * carry the `_change_type`, `_commit_version`, and `_commit_timestamp` columns. + */ +class CdcSuite extends WorkloadTestSuite("cdc") { + + test("inserts") { + sql("""CREATE TABLE tbl (id LONG, name STRING) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,'alice'),(2,'bob')") + sql("INSERT INTO tbl VALUES (3,'charlie'),(4,'dave')") + sql("INSERT INTO tbl VALUES (5,'eve'),(6,'frank')") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 3) + cdfSpec(t, startVersion = 1, endVersion = 1) + snapshotSpec(t) + } + + test("updates") { + sql("""CREATE TABLE tbl (id LONG, value INT) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,100),(2,200),(3,300)") + sql("UPDATE tbl SET value = value * 2 WHERE id = 1") + sql("UPDATE tbl SET value = value + 50 WHERE id IN (2, 3)") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 3) + cdfSpec(t, startVersion = 1, endVersion = 3) + snapshotSpec(t) + } + + test("deletes") { + sql("""CREATE TABLE tbl (id LONG, name STRING) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')") + sql("DELETE FROM tbl WHERE id = 3") + sql("DELETE FROM tbl WHERE id IN (1, 5)") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 3) + cdfSpec(t, startVersion = 1, endVersion = 3) + snapshotSpec(t) + } + + test("mixed_dml") { + sql("""CREATE TABLE tbl (id LONG, value INT) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,100),(2,200),(3,300)") + sql("UPDATE tbl SET value = 999 WHERE id = 2") + sql("DELETE FROM tbl WHERE id = 1") + sql("INSERT INTO tbl VALUES (4,400)") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 4) + cdfSpec(t, startVersion = 1, endVersion = 2) + snapshotSpec(t) + } + + test("predicate_filtered") { + sql("""CREATE TABLE tbl (id LONG, value INT) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,100),(2,200),(3,300),(4,400)") + sql("UPDATE tbl SET value = value + 1 WHERE id <= 2") + sql("DELETE FROM tbl WHERE id = 4") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 3) + cdfSpec(t, startVersion = 0, endVersion = 3, predicate = "value > 200") + cdfSpec(t, startVersion = 0, endVersion = 3, predicate = "_change_type = 'delete'") + snapshotSpec(t) + } + + test("open_ended_to_latest") { + sql("""CREATE TABLE tbl (id LONG) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl SELECT id FROM range(10)") + sql("INSERT INTO tbl SELECT id + 10 FROM range(10)") + sql("INSERT INTO tbl SELECT id + 20 FROM range(10)") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 1) + cdfSpec(t, startVersion = 0, endVersion = 3) + snapshotSpec(t) + } + +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala index b96e8df1..55fba144 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala @@ -16,14 +16,30 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import io.delta.workload.{TableHandle, WorkloadTestSuite} import io.delta.workload.model.ErrorCode class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { private def checkpoint(name: String): Unit = forceCheckpoint(name) - // Existing 5 workloads + /** + * Append one or more action lines to the commit JSON at `version`. Also drops that + * version's `.crc` (now stale w.r.t. the edited commit) so the subsequent + * `checkpoint()` does not trip Delta's checksum integrity check and refuse to write. + */ + private def injectActions(t: TableHandle, version: Int, lines: Seq[String]): Unit = { + mutateTable(t) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val commitFile = logDir.resolve(f"$version%020d.json") + val content = new String(java.nio.file.Files.readAllBytes(commitFile), "UTF-8") + java.nio.file.Files.write(commitFile, + (content.trim + "\n" + lines.mkString("\n") + "\n").getBytes("UTF-8")) + java.nio.file.Files.deleteIfExists(logDir.resolve(f"$version%020d.crc")) + } + } + + // === Read/snapshot workloads over tables that have a checkpoint === test("classic") { sql("CREATE TABLE tbl (id INT) USING delta") @@ -546,4 +562,50 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { snapshotSpec(t, expectError = ErrorCode("DELTA_TABLE_NOT_FOUND")) } + // === checkpointSpec workloads: force a V1 checkpoint and assert its reconstructed state === + + test("cp_basic") { + sql("CREATE TABLE tbl (id INT, name STRING) USING delta") + sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c')") + sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") + val t = registerTable("tbl") + checkpointSpec(t, version = 2) + snapshotSpec(t) + } + + test("cp_partitioned") { + sql("CREATE TABLE tbl (id INT, part INT) USING delta PARTITIONED BY (part)") + sql("INSERT INTO tbl VALUES (1,0),(2,1),(3,0)") + sql("INSERT INTO tbl VALUES (4,1),(5,0)") + val t = registerTable("tbl") + checkpointSpec(t, version = 2) + } + + test("cp_after_many_commits") { + sql("CREATE TABLE tbl (id INT) USING delta") + for (i <- 1 to 6) sql(s"INSERT INTO tbl VALUES ($i)") + val t = registerTable("tbl") + checkpointSpec(t, version = 6) + snapshotSpec(t) + } + + test("cp_with_set_transaction") { + sql("CREATE TABLE tbl (id INT) USING delta") + sql("INSERT INTO tbl VALUES (1)") + sql("INSERT INTO tbl VALUES (2)") + val t = registerTable("tbl") + injectActions(t, 2, Seq("""{"txn":{"appId":"cp-app","version":42,"lastUpdated":1}}""")) + checkpointSpec(t, version = 2) + } + + test("cp_with_domain_metadata") { + sql("""CREATE TABLE tbl (id INT) USING delta + TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')""") + sql("INSERT INTO tbl VALUES (1),(2),(3)") + sql("DELETE FROM tbl WHERE id = 2") + val t = registerTable("tbl") + val dm = """{"domain":"cpDomain","configuration":"{\"k\":\"v\"}","removed":false}""" + injectActions(t, 2, Seq(s"""{"domainMetadata":$dm}""")) + checkpointSpec(t, version = 2) + } } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala new file mode 100644 index 00000000..21171109 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala @@ -0,0 +1,79 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import io.delta.workload.WorkloadTestSuite + +/** + * CRC (version-checksum) read workloads. Delta writes a `.crc` sidecar on each commit; + * each test builds a table with plain SQL and declares a `crcSpec` asserting the core aggregate + * fields (tableSizeBytes, numFiles, protocol) plus any feature-gated fields the `.crc` carries + * (deletion-vector counts, set transactions, inCommitTimestamp). + */ +class CrcSuite extends WorkloadTestSuite("crc") { + + test("crc_basic") { + sql("CREATE TABLE tbl (id INT, name STRING) USING delta") + sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c')") + sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") + val t = registerTable("tbl") + crcSpec(t, version = 1) + snapshotSpec(t) + } + + test("crc_partitioned") { + sql("CREATE TABLE tbl (id INT, part INT) USING delta PARTITIONED BY (part)") + sql("INSERT INTO tbl VALUES (1,0),(2,1),(3,0)") + sql("INSERT INTO tbl VALUES (4,1)") + val t = registerTable("tbl") + crcSpec(t, version = 1) + } + + test("crc_after_delete") { + sql("CREATE TABLE tbl (id INT) USING delta") + sql("INSERT INTO tbl VALUES (1),(2),(3)") + sql("INSERT INTO tbl VALUES (4),(5)") + sql("DELETE FROM tbl WHERE id = 2") + sql("INSERT INTO tbl VALUES (6)") + val t = registerTable("tbl") + // v3 is the delete commit; the trailing insert (v4) makes it non-latest. + crcSpec(t, version = 3) + } + + test("crc_with_deletion_vectors") { + sql("""CREATE TABLE tbl (id INT) USING delta + TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')""") + sql("INSERT INTO tbl VALUES (1),(2),(3),(4),(5)") + sql("DELETE FROM tbl WHERE id IN (2, 4)") + sql("INSERT INTO tbl VALUES (6)") + val t = registerTable("tbl") + // v2 is the deletion-vector commit; the trailing insert (v3) makes it non-latest. + crcSpec(t, version = 2) + } + + test("crc_multiple_versions") { + sql("CREATE TABLE tbl (id INT) USING delta") + sql("INSERT INTO tbl VALUES (1)") + sql("INSERT INTO tbl VALUES (2)") + sql("INSERT INTO tbl VALUES (3)") + sql("INSERT INTO tbl VALUES (4)") + val t = registerTable("tbl") + // Both targets are non-latest (table ends at v4). + crcSpec(t, version = 1, name = "crc_v1") + crcSpec(t, version = 3, name = "crc_v3") + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala index 7178f12c..c18ac7d8 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala @@ -63,6 +63,23 @@ class WriteCommitSuite extends WorkloadTestSuite("write_commit") { snapshotSpec(t) } + // Checkpoint / CRC / CDF specs on a write-derived table: the generator validates these against + // the table REPLAYED by the engine-under-test. + test("checkpoint_crc_cdf_write_derived") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("name", StringType), + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"), Map("id" -> 2, "name" -> "bob"))) // v1 + insertOp(w, Seq(Map("id" -> 3, "name" -> "carol"))) // v2 + val t = endWrite(w) + + readSpec(t) + snapshotSpec(t) + checkpointSpec(t, version = 2) + crcSpec(t, version = 2) + cdfSpec(t, startVersion = 0L) + } + test("commit_with_domain_metadata") { val w = createTableOp("tbl", schema = new StructType().add("id", IntegerType).add("name", StringType), diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala new file mode 100644 index 00000000..2524b619 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala @@ -0,0 +1,96 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite +import io.delta.workload.model.{AddDomainMetadata, AppTxn} + +/** + * Write-derived checkpoint / CRC / CDF coverage. Each case has a write spec, so the generator + * validates these specs against the table REPLAYED by the engine-under-test. + */ +class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { + + private def idName = new StructType().add("id", IntegerType).add("name", StringType) + + test("checkpoint_partitioned") { + val schema = new StructType().add("id", IntegerType).add("part", IntegerType) + val w = createTableOp("tbl", schema, partitionColumns = Seq("part")) + insertOp(w, Seq(Map("id" -> 1, "part" -> 0), Map("id" -> 2, "part" -> 1))) // v1 + insertOp(w, Seq(Map("id" -> 3, "part" -> 0))) // v2 + val t = endWrite(w) + checkpointSpec(t, version = 2) + } + + test("checkpoint_with_set_transaction") { + val w = createTableOp("tbl", idName) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"))) // v1 + commitOp(w, txn = Some(AppTxn("cp-app", 42L))) // v2 + val t = endWrite(w) + checkpointSpec(t, version = 2) + } + + test("checkpoint_with_domain_metadata") { + val w = createTableOp("tbl", idName, + properties = Map("delta.feature.domainMetadata" -> "supported")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"))) // v1 + commitOp(w, addDomainMetadata = Some(Seq(AddDomainMetadata("cpDomain", "{\"k\":\"v\"}")))) // v2 + val t = endWrite(w) + checkpointSpec(t, version = 2) + } + + test("crc_with_deletion_vectors") { + val w = createTableOp("tbl", idName, + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"), Map("id" -> 2, "name" -> "b"), + Map("id" -> 3, "name" -> "c"))) // v1 + deleteOp(w, "id = 2") // v2 -> deletion vector + val t = endWrite(w) + crcSpec(t, version = 2) + } + + test("crc_with_set_transaction") { + val w = createTableOp("tbl", idName) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"))) // v1 + commitOp(w, txn = Some(AppTxn("crc-app", 7L))) // v2 + val t = endWrite(w) + crcSpec(t, version = 2) + } + + test("cdf_updates_and_deletes") { + val w = createTableOp("tbl", idName, + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"), Map("id" -> 2, "name" -> "b"), + Map("id" -> 3, "name" -> "c"))) // v1 + updateOp(w, "id = 1", Map("name" -> "'a2'")) // v2 -> update pre/post image change rows + deleteOp(w, "id = 3") // v3 -> delete change rows + val t = endWrite(w) + cdfSpec(t, startVersion = 0L) + } + + test("cdf_version_range") { + val w = createTableOp("tbl", idName, + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"))) // v1 + insertOp(w, Seq(Map("id" -> 2, "name" -> "b"))) // v2 + insertOp(w, Seq(Map("id" -> 3, "name" -> "c"))) // v3 + val t = endWrite(w) + cdfSpec(t, startVersion = 1L, endVersion = 2L, name = "cdf_v1_to_v2") + } +} From 10e9ebe1c2457dfefc1d9305f9e8439408e80ec4 Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Wed, 16 Sep 2026 20:10:47 +0000 Subject: [PATCH 2/4] refactor: make checkpoint/crc specs bare triggers, snapshot spec owns validation --- .../docs/harness-implementation-guide.md | 28 ++++ .../io/delta/workload/SpecDeclarations.scala | 19 ++- .../io/delta/workload/WorkloadGenerator.scala | 14 +- .../scala/io/delta/workload/WorkloadOps.scala | 8 +- .../delta/workload/capture/CdfCapture.scala | 4 +- .../workload/capture/CheckpointCapture.scala | 77 ++-------- .../delta/workload/capture/CrcCapture.scala | 133 +++--------------- .../workload/capture/SnapshotCapture.scala | 26 +++- .../workload/engine/SnapshotResolver.scala | 3 - .../io/delta/workload/json/JsonUtil.scala | 16 --- .../io/delta/workload/model/SpecModel.scala | 62 ++------ .../workload/validate/StatsValidator.scala | 120 ---------------- .../workload/validate/WorkloadValidator.scala | 71 +++++++--- .../io/delta/workload/tables/CdcSuite.scala | 2 +- .../workload/tables/CheckpointsSuite.scala | 38 +++-- .../io/delta/workload/tables/CrcSuite.scala | 13 +- .../tables/WriteMetadataSpecsSuite.scala | 4 + 17 files changed, 205 insertions(+), 433 deletions(-) delete mode 100644 workload-generator/src/main/scala/io/delta/workload/validate/StatsValidator.scala diff --git a/workload-generator/docs/harness-implementation-guide.md b/workload-generator/docs/harness-implementation-guide.md index b8a424b7..b8e7eb39 100644 --- a/workload-generator/docs/harness-implementation-guide.md +++ b/workload-generator/docs/harness-implementation-guide.md @@ -215,6 +215,34 @@ The write spec has no expected-data artifact of its own. Its own check is **basi Because a write spec is replayed (not byte-compared), comparison is **portable**: rows-only data checks, capability-based protocol comparison (your table may use explicit table features where Spark used legacy versions), and column-mapping-normalized schema. A write spec's presence makes its whole directory *write-derived*: replay it into a fresh table once, then validate every read/snapshot spec in that directory (including the baseline `latest` read) against the replay (portably), rather than against a captured `delta/` directory. +### CDF Specs + +```json +{ "type": "cdf", "startVersion": 0, "endVersion": 2, "expected": { "rowCount": 5 } } +``` + +**Execute:** read the change feed over `[startVersion, endVersion]` (or the timestamp bounds). + +**Validate:** compare the change rows against `expected//expected_data/` as a bag of typed rows, exactly like a read spec. `_change_type` and `_commit_version` are compared; `_commit_timestamp` is not (on non-ICT tables it is the commit file's mtime, which does not survive tarring). Assert `rowCount`. An `"error"` outcome is supported like other specs. + +### Checkpoint and CRC Specs + +```json +{ "type": "checkpoint", "version": 2 } +{ "type": "crc", "version": 2 } +``` + +These are **presence triggers**, not content specs. Validate by asserting a `.checkpoint*` (checkpoint) or `.crc` (crc) file exists in `_delta_log/`. Checkpoint contents are validated indirectly by a paired read spec: reading a version whose snapshot reconstructs through the checkpoint returns wrong rows if the checkpoint dropped or duplicated an Add file, so a content-wrong checkpoint fails that read spec. (A snapshot spec covers only protocol/metadata, not the file set, so it does not catch this on its own.) The `.crc` (version-checksum) file is optional per the protocol, so the crc spec is capability-gated: a harness whose engine does not write version checksums skips crc specs, while a harness whose engine does write them must produce a `.crc` at the spec's version. The spec asserts only that the file was written; an engine that verifies checksums on read additionally validates the contents on load, but that is not assumed. + +### Invariant Validators (row tracking, stats) + +Some correctness properties are not per-spec JSON; they are engine-agnostic invariants a write harness checks against the table the engine under test produced. A write harness implements these directly against the written table: + +- **Stats:** per-file timestamp stats are millisecond-truncated; each file's `minValues`/`maxValues` bound its live rows (deletion vectors applied). When `tightBounds` is true the bounds must equal the live min/max; when false they need only contain them. +- **Row tracking (when the `rowTracking` feature is on):** every live Add carries a `baseRowId`; fresh-id ranges never overlap; the `rowIdHighWaterMark` covers the largest assigned id; and every row's `_metadata.row_id` is non-null and unique. + +These run only in a write harness, against the table the engine under test produced. + --- ## Step 4: Incremental Adoption diff --git a/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala b/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala index 6de120e3..69d2a407 100644 --- a/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala +++ b/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala @@ -59,7 +59,9 @@ class SpecRef[T] private[workload] ( private[workload] trait HasAssertion[T] { var assertion: Option[com.fasterxml.jackson.databind.JsonNode => Unit] = None - def deserialize: com.fasterxml.jackson.databind.JsonNode => T + def specClass: Class[T] + def deserialize: com.fasterxml.jackson.databind.JsonNode => T = + n => JsonUtil.mapper.treeToValue(n, specClass) } // --------------------------------------------------------------------------- @@ -86,15 +88,13 @@ private[workload] class TableDecl( private[workload] case class ReadSpecConfig( name: String, query: ReadQuery, expectError: ErrorExpectation = AutoDetect) extends HasAssertion[ReadSpec] { - val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => - JsonUtil.mapper.treeToValue(n, classOf[ReadSpec]) + val specClass = classOf[ReadSpec] } private[workload] case class SnapshotSpecConfig( query: SnapshotQuery, expectError: ErrorExpectation = AutoDetect) extends HasAssertion[SnapshotSpec] { - val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => - JsonUtil.mapper.treeToValue(n, classOf[SnapshotSpec]) + val specClass = classOf[SnapshotSpec] } private[workload] case class CdfSpecConfig( @@ -102,18 +102,15 @@ private[workload] case class CdfSpecConfig( startTimestamp: Option[String], endTimestamp: Option[String], predicate: Option[String], columns: Option[Seq[String]], expectError: Option[String] = None) extends HasAssertion[CdfSpec] { - val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => - JsonUtil.mapper.treeToValue(n, classOf[CdfSpec]) + val specClass = classOf[CdfSpec] } private[workload] case class CheckpointSpecConfig( name: String, version: Long) extends HasAssertion[CheckpointSpec] { - val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => - JsonUtil.mapper.treeToValue(n, classOf[CheckpointSpec]) + val specClass = classOf[CheckpointSpec] } private[workload] case class CrcSpecConfig( name: String, version: Long) extends HasAssertion[CrcSpec] { - val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => - JsonUtil.mapper.treeToValue(n, classOf[CrcSpec]) + val specClass = classOf[CrcSpec] } diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index cfb69a9c..2892bc81 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -97,16 +97,11 @@ object WorkloadGenerator { Seq(SnapshotSpecConfig(SnapshotQuery())) } else ts.snapshotSpecs for (ss <- explicits) { - SnapshotCapture.capture(spark, dirName, destTablePath, specsDir, + val specPath = SnapshotCapture.capture(spark, dirName, destTablePath, specsDir, query = ss.query, expectError = ss.expectError) - val specName = (ss.query.version, ss.query.timestamp) match { - case (Some(v), _) => s"${dirName}_snapshot_v$v" - case (_, Some(ts)) => - s"${dirName}_snapshot_ts_${ts.replace(":", "-").replace(" ", "_")}" - case _ => s"${dirName}_snapshot" - } + val specName = specPath.getFileName.toString.stripSuffix(".json") if (!snapshotNames.contains(specName)) snapshotNames += specName - checkAssertion(ss, specsDir.resolve(s"$specName.json")) + checkAssertion(ss, specPath) } val readNames = mutable.ArrayBuffer[String]() @@ -135,7 +130,6 @@ object WorkloadGenerator { // for the summary total. var extraSpecCount = 0 - // CDF specs for (cs <- ts.cdfSpecs) { val specPath = CdfCapture.capture(spark, dirName, destTablePath, testOutputDir, specsDir, name = cs.name, startVersion = cs.startVersion, endVersion = cs.endVersion, @@ -145,7 +139,6 @@ object WorkloadGenerator { checkAssertion(cs, specPath) } - // Checkpoint specs for (cp <- ts.checkpointSpecs) { val specPath = CheckpointCapture.capture(spark, dirName, destTablePath, specsDir, version = cp.version, name = cp.name) @@ -153,7 +146,6 @@ object WorkloadGenerator { checkAssertion(cp, specPath) } - // CRC specs for (crc <- ts.crcSpecs) { CrcCapture.capture(spark, dirName, destTablePath, specsDir, version = crc.version, name = crc.name).foreach { p => diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala index 0e1d0020..dc4c3204 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala @@ -130,9 +130,7 @@ trait WorkloadOps { } /** - * Checkpoint spec. Forces a checkpoint at `version` and asserts the reconstructed - * protocol/metadata/txn/domain-metadata state, recorded inline in the spec JSON. - * Targets classic V1 single-file checkpoints. + * Checkpoint spec: a trigger that forces a checkpoint at `version`. */ def checkpointSpec( table: TableHandle, @@ -147,9 +145,7 @@ trait WorkloadOps { } /** - * CRC (version-checksum) spec. Reads the `.crc` file and asserts its core - * aggregate fields (plus protocol and any present optional fields), recorded inline - * in the spec JSON. + * CRC (version-checksum) spec: a trigger asserting the engine wrote a `.crc` at `version`. */ def crcSpec( table: TableHandle, diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala index 99c5b952..1e9fd500 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala @@ -54,6 +54,8 @@ object CdfCapture { var df = buildReader(spark, tablePath, startVersion, endVersion, startTimestamp, endTimestamp) df = SnapshotResolver.applyFilters(df, predicate, columns) + // TODO(cdf-commit-timestamp): validate `_commit_timestamp` once ICT tables are covered; today + // it is dropped because on non-ICT tables it is the commit file's mtime (not reproducible). df = df.drop("_commit_timestamp") df = df.cache() val count = df.count() @@ -69,8 +71,6 @@ object CdfCapture { (None, Some(SpecError(SpecOutcome.extractErrorCode(e), Option(e.getMessage).getOrElse("")))) } - // If the test author declared `expectError`, the operation MUST throw. - // See ReadCapture.capture for full semantics. expectError.foreach { expected => def fail(msg: String): Nothing = { if (expectedDir.toFile.exists()) FileUtils.deleteDirectory(expectedDir.toFile) diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala index 1c536e6b..a3d7fcbf 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala @@ -22,22 +22,17 @@ import scala.collection.JavaConverters._ import org.apache.spark.sql.SparkSession -import io.delta.workload.deltaharness.{DeltaHarness, ResolvedSnapshot} +import io.delta.workload.deltaharness.DeltaHarness import io.delta.workload.json.JsonUtil -import io.delta.workload.model.{CheckpointExpected, CheckpointFile, CheckpointSpec, MetadataInfo, ProtocolInfo} +import io.delta.workload.model.CheckpointSpec /** - * Captures and validates checkpoint specs. A checkpoint is forced at the target version via - * the neutral `LogView.checkpoint` SPI, then its reconstructed state (protocol, metadata, - * set-transactions, domain-metadata) is read back through the neutral `ResolvedSnapshot` - * accessors and recorded inline in the spec JSON, mirroring [[SnapshotCapture]]. - * - * Scope: targets classic V1 single-file checkpoints. The `.checkpoint.parquet` produced by - * `checkpoint(version)` is left in place under the table's `_delta_log/` so consumers can exercise - * checkpoint reads. + * Captures and validates checkpoint specs. A checkpoint spec is a bare trigger: it names a version + * the engine under test must checkpoint. */ object CheckpointCapture { + /** Force a checkpoint at `version` and emit its spec; returns the spec path. */ def capture( spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, version: Long, name: String): Path = { @@ -45,72 +40,26 @@ object CheckpointCapture { val specName = s"${testId}_$name" val specPath = specsDir.resolve(s"$specName.json") - val harness = DeltaHarness.get - harness.openLog(spark, tablePath.toString).checkpoint(version) + DeltaHarness.get.openLog(spark, tablePath.toString).checkpoint(version) require(checkpointFileExists(tablePath, version), s"Checkpoint $specName: no checkpoint file present at version $version after checkpoint()") - val snapshot = harness.openLog(spark, tablePath.toString).getSnapshotAt(version) - val expected = Some(buildExpected(snapshot)) - - val spec = CheckpointSpec(version, expected) - JsonUtil.writeSpec(specPath, spec) - validateFromSpec(spark, tablePath, specPath) - + JsonUtil.writeSpec(specPath, CheckpointSpec(version)) println(s" Checkpoint captured: $specName (version=$version)") specPath } - /** - * Validate a checkpoint spec Protocol/metadata/txn/domain-metadata against the table at `tablePath`. - */ + /** Assert a checkpoint file exists at the spec's version. */ def validateFromSpec( - spark: SparkSession, tablePath: Path, specPath: Path, - isWriteValidation: Boolean = false): Unit = { + spark: SparkSession, tablePath: Path, specPath: Path): Unit = { val spec = JsonUtil.readSpecAs(specPath, classOf[CheckpointSpec]) val specName = specPath.getFileName.toString.stripSuffix(".json") - val harness = DeltaHarness.get - - if (isWriteValidation) { - harness.openLog(spark, tablePath.toString).checkpoint(spec.version) - require(checkpointFileExists(tablePath, spec.version), - s"Checkpoint validation failed for $specName: no checkpoint file at version " + - s"${spec.version} after checkpoint()") - } - - spec.expected.foreach { exp => - val resolved = harness.openLog(spark, tablePath.toString).getSnapshotAt(spec.version) - val live = resolved.snapshot - SnapshotCapture.assertMatches(exp.protocol, live.protocol, exp.metadata, live.metadata, - isWriteValidation, specName) - require(exp.txn.getOrElse(Seq.empty).toSet == resolved.setTransactions.toSet, - s"Checkpoint validation failed for $specName: txn mismatch") - require(exp.domainMetadata.getOrElse(Seq.empty).toSet == resolved.domainMetadata.toSet, - s"Checkpoint validation failed for $specName: domainMetadata mismatch") - if (!isWriteValidation) { - require(exp.files.toSet == fileSet(resolved).toSet, - s"Checkpoint validation failed for $specName: file set mismatch") - } - } - } - - /** Build the expected checkpoint state from a snapshot via the neutral SPI accessors. */ - private def buildExpected(snapshot: ResolvedSnapshot): CheckpointExpected = { - val txn = snapshot.setTransactions - val domainMetadata = snapshot.domainMetadata - CheckpointExpected( - protocol = ProtocolInfo.from(snapshot.snapshot.protocol), - metadata = MetadataInfo.from(snapshot.snapshot.metadata), - files = fileSet(snapshot), - txn = if (txn.nonEmpty) Some(txn) else None, - domainMetadata = if (domainMetadata.nonEmpty) Some(domainMetadata) else None) + require(checkpointFileExists(tablePath, spec.version), + s"Checkpoint validation failed for $specName: no checkpoint file at version ${spec.version}; " + + "the engine under test did not write one") } - private def fileSet(s: ResolvedSnapshot): Seq[CheckpointFile] = - s.allFiles.select("path", "size").collect() - .map(r => CheckpointFile(r.getString(0), r.getLong(1))).toSeq - - /** True iff a classic checkpoint file (`.checkpoint*`) exists in `_delta_log`. */ + /** True iff a checkpoint file (`.checkpoint*`) exists in `_delta_log`. */ private def checkpointFileExists(tablePath: Path, version: Long): Boolean = { val logDir = tablePath.resolve("_delta_log") val prefix = f"$version%020d.checkpoint" diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala index 5ceaa65c..677c7f0f 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala @@ -20,141 +20,42 @@ import java.nio.file.{Files, Path} import org.apache.spark.sql.SparkSession -import io.delta.workload.deltaharness.DeltaHarness import io.delta.workload.json.JsonUtil -import io.delta.workload.model.{CrcExpected, CrcSpec, ProtocolInfo} +import io.delta.workload.model.CrcSpec /** - * Captures and validates CRC (version-checksum) specs. The `.crc` file is plain JSON - * in `_delta_log/`, so it is read directly with [[java.nio.file.Files]] + [[JsonUtil.mapper]] — - * no Delta class and no SPI method are needed. The protocol is read from the neutral - * `ResolvedSnapshot` so it compares engine-neutrally; all other fields come straight from the - * `.crc`, keyed by delta-spark's `VersionChecksum` field names. Optional/feature-gated fields - * (deletion-vector counts, set transactions, inCommitTimestamp) are recorded only when the `.crc` - * carries them, and validated conditionally. + * Captures and validates crc (version-checksum) specs. A crc spec is a bare trigger: it names a + * version at which the engine under test must have written a `.crc` sidecar. */ object CrcCapture { - /** - * Capture a crc spec for `version`, or skip if no `.crc` exists there. - * - * A crc spec asserts the *contents* of a `.crc` where one is present; absence is a - * skip, not a failure. Some engines flush `.crc` asynchronously, so the latest - * commit's `.crc` is often not on disk at read time. When the target `.crc` is missing this - * logs a SKIP, emits no spec, and returns `None`. - * - * @return `Some(specPath)` if a spec was emitted and self-validated, `None` if skipped (no `.crc`). - */ + /** @return `Some(specPath)` if a `.crc` was present and a spec emitted, `None` if none exists. */ def capture( spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, version: Long, name: String): Option[Path] = { val specName = s"${testId}_$name" - val specPath = specsDir.resolve(s"$specName.json") - - val crcNode = readCrc(tablePath, version) match { - case Some(node) => node - case None => - println(s" CRC skipped: $specName (version=$version): " + - s"no .crc at version $version; engine did not write/flush one") - return None + if (!crcFileExists(tablePath, version)) { + println(s" CRC skipped: $specName (version=$version): no .crc present at that version") + return None } - - val harness = DeltaHarness.get - val snapshot = harness.openLog(spark, tablePath.toString).getSnapshotAt(version) - val protocol = ProtocolInfo.from(snapshot.snapshot.protocol) - - val expected = CrcExpected( - tableSizeBytes = JsonUtil.crcLongField(crcNode, "tableSizeBytes"), - numFiles = JsonUtil.crcLongField(crcNode, "numFiles"), - numDeletionVectors = JsonUtil.crcLongField(crcNode, "numDeletionVectorsOpt"), - numDeletedRecords = JsonUtil.crcLongField(crcNode, "numDeletedRecordsOpt"), - inCommitTimestamp = JsonUtil.crcLongField(crcNode, "inCommitTimestampOpt"), - protocol = Some(protocol), - setTransactions = JsonUtil.crcSetTransactions(crcNode)) - - val spec = CrcSpec(version, Some(expected)) - JsonUtil.writeSpec(specPath, spec) - validateFromSpec(spark, tablePath, specPath) - + val specPath = specsDir.resolve(s"$specName.json") + JsonUtil.writeSpec(specPath, CrcSpec(version)) println(s" CRC captured: $specName (version=$version)") Some(specPath) } - /** - * Validate a crc spec's (deletion-vector / deleted-record counts), protocol, and set transactions - * against the table at `tablePath`. - */ + /** Assert a `.crc` exists at the spec's version. Its contents are validated by the engine's on-read + * checksum verification when the version is loaded (via the snapshot spec). */ def validateFromSpec( - spark: SparkSession, tablePath: Path, specPath: Path, - isWriteValidation: Boolean = false): Unit = { + spark: SparkSession, tablePath: Path, specPath: Path): Unit = { val spec = JsonUtil.readSpecAs(specPath, classOf[CrcSpec]) val specName = specPath.getFileName.toString.stripSuffix(".json") - val harness = DeltaHarness.get - - spec.expected.foreach { exp => - val crcNode = readCrc(tablePath, spec.version) match { - case Some(node) => node - case None => - // Validate-where-present: a missing `.crc` (e.g. async flush) is a skip, not a failure. - println(s" CRC validation skipped for $specName: " + - s"no .crc at version ${spec.version}; engine did not write/flush one") - return - } - - // === Aggregate fields (VersionChecksum keys) === - val scalarFields = - if (isWriteValidation) ScalarFields.filterNot { case (l, _, _) => NonReproducibleScalarFields(l) } - else ScalarFields - scalarFields.foreach { case (label, key, get) => - requireLong(specName, label, get(exp), JsonUtil.crcLongField(crcNode, key)) - } - - // === Protocol (engine-neutral, via ResolvedSnapshot) === - exp.protocol.foreach { expProtocol => - val snapshot = harness.openLog(spark, tablePath.toString).getSnapshotAt(spec.version) - require(expProtocol == ProtocolInfo.from(snapshot.snapshot.protocol), - s"CRC validation failed for $specName: protocol mismatch") - } - - // === Set transactions === - exp.setTransactions.foreach { expTxn => - require(Some(expTxn.toSet) == JsonUtil.crcSetTransactions(crcNode).map(_.toSet), - s"CRC validation failed for $specName: setTransactions mismatch") - } - } - } - - // Scalar `VersionChecksum` fields checked uniformly: (spec label, `.crc` key, spec accessor). - private val ScalarFields: Seq[(String, String, CrcExpected => Option[Long])] = Seq( - ("tableSizeBytes", "tableSizeBytes", _.tableSizeBytes), - ("numFiles", "numFiles", _.numFiles), - ("numDeletionVectors", "numDeletionVectorsOpt", _.numDeletionVectors), - ("numDeletedRecords", "numDeletedRecordsOpt", _.numDeletedRecords), - ("inCommitTimestamp", "inCommitTimestampOpt", _.inCommitTimestamp)) - - private val NonReproducibleScalarFields: Set[String] = - Set("tableSizeBytes", "numFiles", "inCommitTimestamp") - - /** Read the `.crc` JSON, or `None` when no such file is on disk. */ - private def readCrc( - tablePath: Path, version: Long): Option[com.fasterxml.jackson.databind.JsonNode] = { - val crcFile = tablePath.resolve("_delta_log").resolve(f"$version%020d.crc") - if (!Files.exists(crcFile)) None - else Some(JsonUtil.mapper.readTree(new String(Files.readAllBytes(crcFile), "UTF-8"))) + require(crcFileExists(tablePath, spec.version), + s"CRC validation failed for $specName: no .crc at version ${spec.version}; " + + "the engine under test did not write one") } - // A field the spec recorded (Some) must be present in the `.crc` under the same key: a - // spec-Some / crc-None mismatch means the writer dropped a field it should have written. - // A field the spec did not record (None, e.g. a feature that was off) is nothing to check. - private def requireLong(specName: String, field: String, - expected: Option[Long], actual: Option[Long]): Unit = - (expected, actual) match { - case (Some(e), Some(a)) => - require(e == a, s"CRC validation failed for $specName: $field expected=$e actual=$a") - case (Some(e), None) => - require(false, s"CRC validation failed for $specName: $field expected=$e " + - "but the .crc has no such field") - case (None, _) => // spec did not record this field - } + private def crcFileExists(tablePath: Path, version: Long): Boolean = + Files.exists(tablePath.resolve("_delta_log").resolve(f"$version%020d.crc")) } diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala index 67ad4801..6d5357dc 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala @@ -31,7 +31,7 @@ object SnapshotCapture { def capture( spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, query: SnapshotQuery = SnapshotQuery(), - expectError: ErrorExpectation = AutoDetect): Unit = { + expectError: ErrorExpectation = AutoDetect): Path = { require(!(query.version.isDefined && query.timestamp.isDefined), "Cannot specify both version and timestamp") @@ -55,9 +55,13 @@ object SnapshotCapture { Failed(SpecError("DELTA_TABLE_NOT_FOUND", s"No valid Delta table at ${tablePath}")) } else { resolvedVersion = Some(snapshot.version) + val txn = snapshot.setTransactions + val dm = snapshot.domainMetadata Succeeded(SnapshotResult( ProtocolInfo.from(snapshot.snapshot.protocol), - MetadataInfo.from(snapshot.snapshot.metadata))) + MetadataInfo.from(snapshot.snapshot.metadata), + setTransactions = if (txn.nonEmpty) Some(txn) else None, + domainMetadata = if (dm.nonEmpty) Some(dm) else None)) } } @@ -74,6 +78,7 @@ object SnapshotCapture { case Failed(err) => println(s" Snapshot captured (error): $specName [${err.errorCode}] ${err.errorMessage}") } + specPath } /** @@ -100,9 +105,13 @@ object SnapshotCapture { if (resolve.version < 0) Some("DELTA_TABLE_NOT_FOUND") else None } } { exp => - val live = resolve.snapshot + val resolved = resolve + val live = resolved.snapshot assertMatches(exp.protocol, live.protocol, exp.metadata, live.metadata, isWriteValidation, specName) + requireTxnSubset(specName, exp.setTransactions.getOrElse(Seq.empty), resolved.setTransactions) + requireDomainMetadataEqual(specName, exp.domainMetadata.getOrElse(Seq.empty), + resolved.domainMetadata) } } @@ -139,6 +148,17 @@ object SnapshotCapture { } } + // Expected app txns must all be present; the engine under test may carry extra writer-owned txns + // (e.g. its own ingestion idempotency txn) the corpus never had, so this is a subset, not equality. + private def requireTxnSubset(name: String, expected: Seq[AppTxn], actual: Seq[AppTxn]): Unit = + require(expected.toSet.subsetOf(actual.toSet), + s"snapshot '$name': setTransactions mismatch (expected all of $expected present, got $actual)") + + private def requireDomainMetadataEqual( + name: String, expected: Seq[AddDomainMetadata], actual: Seq[AddDomainMetadata]): Unit = + require(expected.toSet == actual.toSet, + s"snapshot '$name': domainMetadata mismatch (expected $expected, got $actual)") + private def stripColumnMappingIds(schema: StructType): StructType = stripColumnMapping(schema).asInstanceOf[StructType] diff --git a/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala b/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala index 19e0b065..d1f8cf2e 100644 --- a/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala +++ b/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala @@ -45,9 +45,6 @@ object SnapshotResolver { reader.load(tablePath.toString) } - /** - * Build a Change Data Feed reader over a version/timestamp range. - */ def buildCdfReader(spark: SparkSession, tablePath: Path, startVersion: Option[Long], endVersion: Option[Long], startTimestamp: Option[String], endTimestamp: Option[String], diff --git a/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala b/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala index a6097956..c4c2eebb 100644 --- a/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala +++ b/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala @@ -150,20 +150,4 @@ object JsonUtil { /** Deserialize any spec by its `type` tag into the sealed [[Spec]]. */ def readSpec(path: Path): Spec = readSpecAs(path, classOf[Spec]) - // === CRC field accessors (operate on the raw `.crc` JSON tree) === - - def crcLongField(node: JsonNode, field: String): Option[Long] = - Option(node.get(field)).filterNot(_.isNull).map(_.asLong()) - - /** Parse the `setTransactions` field of a `.crc`, which may be a single object or an array. */ - def crcSetTransactions(node: JsonNode): Option[Seq[AppTxn]] = - Option(node.get("setTransactions")).filterNot(_.isNull).flatMap { txnNode => - val actions = - if (txnNode.isArray) txnNode.elements().asScala.toSeq - else if (txnNode.isObject) Seq(txnNode) - else Seq.empty - if (actions.isEmpty) None - else Some(actions.map(t => AppTxn(t.get("appId").asText(), t.get("version").asLong()))) - } - } diff --git a/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala b/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala index 33949db1..4d8df8a2 100644 --- a/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala +++ b/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala @@ -57,51 +57,19 @@ object MetadataInfo { m.configuration, m.createdTime) } -/** Snapshot success outcome. */ -@JsonPropertyOrder(Array("protocol", "metadata")) -case class SnapshotResult(protocol: ProtocolInfo, metadata: MetadataInfo) - -// ============================================================================= -// Expected (success-outcome) types for the cdf/checkpoint/crc specs -// -// Each type is the `expected` payload recorded inline in its spec JSON, mirroring the read/snapshot -// specs' outcome. The reconstructed protocol/metadata reuse the shared typed [[ProtocolInfo]] / -// [[MetadataInfo]]; txn pairs reuse [[AppTxn]]. -// ============================================================================= - -/** Change Data Feed success data. */ -case class CdfExpected(rowCount: Long) - -/** A single live Add-file entry in a checkpoint's reconstructed state. */ -case class CheckpointFile(path: String, size: Long) - -/** Checkpoint success data. The `txn`/`domainMetadata` sets are omitted when empty. */ -@JsonPropertyOrder(Array("protocol", "metadata", "files", "txn", "domainMetadata")) +/** + * Snapshot success outcome: the reconstructed logical state at a version. + */ +@JsonPropertyOrder(Array("protocol", "metadata", "setTransactions", "domainMetadata")) @JsonInclude(JsonInclude.Include.NON_ABSENT) -case class CheckpointExpected( +case class SnapshotResult( protocol: ProtocolInfo, metadata: MetadataInfo, - files: Seq[CheckpointFile] = Nil, - txn: Option[Seq[AppTxn]] = None, + setTransactions: Option[Seq[AppTxn]] = None, domainMetadata: Option[Seq[AddDomainMetadata]] = None) -/** - * CRC (version checksum) success data, keyed to the fields delta-spark serializes into a - * `.crc` (`VersionChecksum`). Optional fields appear only when the `.crc` carries them: - * deletion-vector counts require the feature, `inCommitTimestamp` requires ICT, `setTransactions` - * appears only when the table has any. - */ -@JsonPropertyOrder(Array("tableSizeBytes", "numFiles", "numDeletionVectors", "numDeletedRecords", - "inCommitTimestamp", "protocol", "setTransactions")) -@JsonInclude(JsonInclude.Include.NON_ABSENT) -case class CrcExpected( - tableSizeBytes: Option[Long] = None, - numFiles: Option[Long] = None, - numDeletionVectors: Option[Long] = None, - numDeletedRecords: Option[Long] = None, - inCommitTimestamp: Option[Long] = None, - protocol: Option[ProtocolInfo] = None, - setTransactions: Option[Seq[AppTxn]] = None) +/** Change Data Feed success data. */ +case class CdfExpected(rowCount: Long) // ============================================================================= // Shared read/snapshot query types @@ -181,19 +149,13 @@ case class CdfSpec( val `type`: String = "cdf" } -@JsonPropertyOrder(Array("type", "version", "expected")) -@JsonInclude(JsonInclude.Include.NON_ABSENT) -case class CheckpointSpec( - version: Long, - expected: Option[CheckpointExpected] = None) extends Spec { +@JsonPropertyOrder(Array("type", "version")) +case class CheckpointSpec(version: Long) extends Spec { val `type`: String = "checkpoint" } -@JsonPropertyOrder(Array("type", "version", "expected")) -@JsonInclude(JsonInclude.Include.NON_ABSENT) -case class CrcSpec( - version: Long, - expected: Option[CrcExpected] = None) extends Spec { +@JsonPropertyOrder(Array("type", "version")) +case class CrcSpec(version: Long) extends Spec { val `type`: String = "crc" } diff --git a/workload-generator/src/main/scala/io/delta/workload/validate/StatsValidator.scala b/workload-generator/src/main/scala/io/delta/workload/validate/StatsValidator.scala deleted file mode 100644 index df3c3a0a..00000000 --- a/workload-generator/src/main/scala/io/delta/workload/validate/StatsValidator.scala +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload.validate - -import java.nio.file.Path - -import scala.collection.mutable - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.types.StructType - -import io.delta.workload.deltaharness.DeltaHarness -import io.delta.workload.json.JsonUtil - -/** - * Validates the per-file statistics an engine wrote, on a replayed (engine-written) table. - * - * This is a PROPERTY check, deliberately not an expected-value compare against the corpus: the - * Delta protocol mandates stats SOUNDNESS and FORMAT, but leaves tightness (and even presence) - * optional, so a sound writer may legitimately emit wider — or no — stats. Comparing to a captured - * expected would false-fail such a writer; asserting the invariants does not. - * - * TODO: also check `numRecords` soundness (== the data file's row count) and per-column min/max - * soundness (stats.min <= real_min <= real_max <= stats.max); both need reading the (possibly - * URL-encoded) data-file paths. - */ -object StatsValidator { - - /** Validate the stats on the table at `tablePath`. Returns findings; empty means compliant. */ - def validate(spark: SparkSession, tablePath: Path): Seq[String] = { - val snapshot = DeltaHarness.get.openLog(spark, tablePath.toString).update() - val timestampCols = timestampColumns(snapshot.snapshot.metadata.schema) - if (timestampCols.isEmpty) return Seq.empty // no timestamp columns to check - - val findings = mutable.ArrayBuffer[String]() - val addJsons = snapshot.allFiles.select("json").collect().map(_.getString(0)) - for (addJson <- addJsons) { - val add = JsonUtil.mapper.readTree(addJson).path("add") - val statsNode = add.path("stats") - if (statsNode.isTextual) { - val stats = JsonUtil.mapper.readTree(statsNode.asText) - checkTimestampPrecision(add.path("path").asText(""), stats, timestampCols, findings) - } - } - findings.toSeq - } - - /** - * Segment lists of every timestamp / timestamp_ntz field, recursing through nested structs - * (a struct-nested timestamp is serialized in stats under the same nesting, e.g. `info` -> `ts`). - * Returning segment lists rather than dotted strings avoids the ambiguity between a top-level - * column literally named `a.b` and a nested `a` -> `b`. - * Descends only into structs: Delta collects no min/max stats inside arrays or maps, and - * partition columns live in `partitionValues` with a different serialization (out of scope). - * (`typeName` avoids depending on the `TimestampNTZType` symbol, which older Spark versions lack.) - */ - private def timestampColumns(schema: StructType): Set[Seq[String]] = { - def collect(struct: StructType, prefix: Seq[String]): Seq[Seq[String]] = - struct.fields.toSeq.flatMap { f => - val path = prefix :+ f.name - f.dataType match { - case nested: StructType => collect(nested, path) - case dt if dt.typeName == "timestamp" || dt.typeName == "timestamp_ntz" => Seq(path) - case _ => Seq.empty - } - } - collect(schema, Nil).toSet - } - - /** - * Timestamp-precision check: a timestamp min/max stat must be millisecond-truncated (<= 3 - * fractional digits). Delta serializes stats timestamps as ISO-8601 at millisecond granularity; - * microsecond (`.298677Z`) or nanosecond values violate the protocol and can break skipping - * under some readers. - */ - private def checkTimestampPrecision( - path: String, - stats: com.fasterxml.jackson.databind.JsonNode, - timestampPaths: Set[Seq[String]], - findings: mutable.ArrayBuffer[String]): Unit = { - for (bound <- Seq("minValues", "maxValues")) { - val boundNode = stats.path(bound) - for (colPath <- timestampPaths) { - // Walk the path segments into the (possibly nested) stats object. - val v = colPath.foldLeft(boundNode)((node, seg) => node.path(seg)) - if (v.isTextual) { - fractionalDigits(v.asText).foreach { n => - if (n > 3) { - val col = colPath.mkString(".") - findings += s"$path: $bound.$col = '${v.asText}' has $n fractional-second digits; " + - "Delta stats timestamps must be millisecond-truncated (<= 3)" - } - } - } - } - } - } - - /** Digit count of the fractional-seconds part of an ISO-8601-ish timestamp, or None if absent. */ - private def fractionalDigits(s: String): Option[Int] = { - val dot = s.indexOf('.') - if (dot < 0) return None - val n = s.drop(dot + 1).takeWhile(_.isDigit).length - if (n > 0) Some(n) else None - } -} diff --git a/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala index 5b2e2710..606a130f 100644 --- a/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala @@ -29,6 +29,7 @@ import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession import io.delta.workload.capture.{CdfCapture, CheckpointCapture, CrcCapture, ReadCapture, SnapshotCapture} +import io.delta.workload.deltaharness.DeltaHarness import io.delta.workload.json.JsonUtil import io.delta.workload.model._ import io.delta.workload.write.WriteReplay @@ -51,10 +52,12 @@ case class ValidationResult(outcomes: Seq[SpecOutcome]) { * and validates each captured spec under `specs/`, dispatching on the `type` field: * * - `write`: replay its `commits` into a fresh table, compare rows to `expected//`. - * - `read` / `snapshot`: the captured-vs-replayed target is chosen once per directory. If the - * directory has a write spec, every read/snapshot spec validates against the table replayed - * from it (a read by rows only; a snapshot leniently, since a replay mints fresh per-table - * ids). Otherwise it is read-only: validate against the captured `delta/` table exactly. + * - `read` / `snapshot` / `cdf`: the captured-vs-replayed target is chosen once per directory. If + * the directory has a write spec, every such spec validates against the table replayed from it + * (a read/cdf by rows only; a snapshot leniently, since a replay mints fresh per-table ids). + * Otherwise it is read-only: validate against the captured `delta/` table exactly. + * - `checkpoint` / `crc`: assert the sidecar file exists at the spec's version; contents are + * covered by a paired snapshot/read spec. * * Used both as the presubmit acceptance entry point ([[validateAll]]) and by the generator to * self-check a freshly generated test dir ([[validateTestDir]]). @@ -111,6 +114,12 @@ object WorkloadValidator { val writeDerived = writeSpecFile.isDefined def queryTarget: Path = writeSpecFile.map(wf => replayOf(wf.getFileName.toString)).getOrElse(deltaTable) + if (writeDerived) { + parsed.collect { case (_, s: CheckpointSpec) => s.version }.distinct.foreach { v => + DeltaHarness.get.openLog(spark, queryTarget.toString).checkpoint(v) + } + } + // Validate one spec to a single outcome. Read-only specs (no write spec in the dir) validate // against the captured `delta/` table exactly; its absence means the corpus is incomplete. def validateOne(specFile: Path, spec: Spec): SpecOutcome = { @@ -122,8 +131,8 @@ object WorkloadValidator { try spec match { case _: WriteSpec => // Triggering the replay reconstructs the table and asserts replay succeeds + - // finalVersion == commits.size-1. Rows are checked by the baseline read spec, per-version - // metadata by the snapshot spec, and per-file stats by the separate stats pass below. + // finalVersion == commits.size-1. Rows are checked by the baseline read spec and + // per-version metadata by the snapshot spec. val _ = queryTarget; SpecPassed(id) case _: ReadSpec => val expectedDir = testDir.resolve("expected").resolve(bare) @@ -142,12 +151,12 @@ object WorkloadValidator { } else readOnly(t => CdfCapture.validateFromSpec(spark, t, expectedDir, specFile)) case _: CheckpointSpec => if (writeDerived) { - CheckpointCapture.validateFromSpec(spark, queryTarget, specFile, isWriteValidation = true) + CheckpointCapture.validateFromSpec(spark, queryTarget, specFile) SpecPassed(id) } else readOnly(t => CheckpointCapture.validateFromSpec(spark, t, specFile)) case _: CrcSpec => if (writeDerived) { - CrcCapture.validateFromSpec(spark, queryTarget, specFile, isWriteValidation = true) + CrcCapture.validateFromSpec(spark, queryTarget, specFile) SpecPassed(id) } else readOnly(t => CrcCapture.validateFromSpec(spark, t, specFile)) } catch { @@ -157,18 +166,48 @@ object WorkloadValidator { try { outcomes ++= parsed.map { case (f, spec) => validateOne(f, spec) } - if (writeDerived) { - val statsId = s"${testDir.getFileName}/stats" - outcomes += (try { - val findings = StatsValidator.validate(spark, queryTarget) - if (findings.isEmpty) SpecPassed(statsId) - else SpecFailed(statsId, s"stats: ${findings.mkString("; ")}") - } catch { case NonFatal(e) => SpecFailed(statsId, e.toString) }) - } } finally tempDirs.foreach(td => try FileUtils.deleteDirectory(td.toFile) catch { case NonFatal(_) => }) ValidationResult(outcomes.toSeq) } + /** + * Validate the metadata specs under `specsDir` against an already-written table at `tablePath`. + */ + def validateAgainstTable(spark: SparkSession, tablePath: Path, specsDir: Path): ValidationResult = { + val outcomes = mutable.ArrayBuffer[SpecOutcome]() + // A table that fails to open is the most severe engine failure; surface it as a failure rather + // than swallowing it into a sentinel that makes every spec skip and the result falsely pass. + val latest = + try DeltaHarness.get.openLog(spark, tablePath.toString).update().version + catch { + case NonFatal(e) => + return ValidationResult(Seq(SpecFailed("table", s"failed to open table at $tablePath: $e"))) + } + listSpecs(specsDir).foreach { specFile => + val id = specFile.getFileName.toString.stripSuffix(".json") + def skip(v: Long): Unit = + println(s" SKIP $id: spec version $v > table latest $latest (not reproduced by the engine under test)") + try { + JsonUtil.readSpec(specFile) match { + case s: CheckpointSpec if s.version > latest => skip(s.version) + case s: CrcSpec if s.version > latest => skip(s.version) + case s: SnapshotSpec if s.query.version.exists(_ > latest) => skip(s.query.version.get) + case _: CheckpointSpec => + CheckpointCapture.validateFromSpec(spark, tablePath, specFile) + outcomes += SpecPassed(id) + case _: CrcSpec => + CrcCapture.validateFromSpec(spark, tablePath, specFile) + outcomes += SpecPassed(id) + case _: SnapshotSpec => + SnapshotCapture.validateFromSpec(spark, tablePath, specFile, isWriteValidation = true) + outcomes += SpecPassed(id) + case _ => // read / cdf / write -- rows are validated by the harness's read-back + } + } catch { case NonFatal(e) => outcomes += SpecFailed(id, e.toString) } + } + ValidationResult(outcomes.toSeq) + } + private def listSpecs(specsDir: Path): Seq[Path] = { val stream = Files.list(specsDir) try stream.iterator().asScala.filter(_.getFileName.toString.endsWith(".json")).toSeq.sorted diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala index dc2938d7..d38ded10 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala @@ -21,7 +21,7 @@ import io.delta.workload.WorkloadTestSuite /** * Change Data Feed workloads. Each test builds a CDF-enabled table across several * versions, then declares `cdfSpec`s over version ranges. The captured change rows - * carry the `_change_type`, `_commit_version`, and `_commit_timestamp` columns. + * carry the `_change_type` and `_commit_version` columns. */ class CdcSuite extends WorkloadTestSuite("cdc") { diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala index 55fba144..4bddb5d1 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala @@ -17,6 +17,7 @@ package io.delta.workload.tables import io.delta.workload.{TableHandle, WorkloadTestSuite} +import io.delta.workload.log.{Action, DomainMetadata, Txn} import io.delta.workload.model.ErrorCode class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { @@ -24,17 +25,17 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { private def checkpoint(name: String): Unit = forceCheckpoint(name) /** - * Append one or more action lines to the commit JSON at `version`. Also drops that - * version's `.crc` (now stale w.r.t. the edited commit) so the subsequent - * `checkpoint()` does not trip Delta's checksum integrity check and refuse to write. + * Append `actions` to the commit JSON at `version`. Also drops that version's `.crc` + * (now stale w.r.t. the edited commit) so the subsequent `checkpoint()` does not trip Delta's + * checksum integrity check and refuse to write. */ - private def injectActions(t: TableHandle, version: Int, lines: Seq[String]): Unit = { + private def injectActions(t: TableHandle, version: Int, actions: Seq[Action]): Unit = { mutateTable(t) { tableDir => val logDir = tableDir.resolve("_delta_log") val commitFile = logDir.resolve(f"$version%020d.json") val content = new String(java.nio.file.Files.readAllBytes(commitFile), "UTF-8") java.nio.file.Files.write(commitFile, - (content.trim + "\n" + lines.mkString("\n") + "\n").getBytes("UTF-8")) + (content.trim + "\n" + actions.map(_.toJson).mkString("\n") + "\n").getBytes("UTF-8")) java.nio.file.Files.deleteIfExists(logDir.resolve(f"$version%020d.crc")) } } @@ -562,7 +563,8 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { snapshotSpec(t, expectError = ErrorCode("DELTA_TABLE_NOT_FOUND")) } - // === checkpointSpec workloads: force a V1 checkpoint and assert its reconstructed state === + // === checkpointSpec workloads: force a checkpoint and assert the file exists (contents are + // covered by the paired snapshot spec) === test("cp_basic") { sql("CREATE TABLE tbl (id INT, name STRING) USING delta") @@ -570,6 +572,19 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") val t = registerTable("tbl") checkpointSpec(t, version = 2) + readSpec(t) + snapshotSpec(t) + } + + test("cp_v2") { + sql("""CREATE TABLE tbl (id INT, name STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = 'v2', + 'delta.enableDeletionVectors' = 'trque')""") + sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c')") + sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") + val t = registerTable("tbl") + checkpointSpec(t, version = 2) + readSpec(t) snapshotSpec(t) } @@ -579,6 +594,7 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { sql("INSERT INTO tbl VALUES (4,1),(5,0)") val t = registerTable("tbl") checkpointSpec(t, version = 2) + readSpec(t) } test("cp_after_many_commits") { @@ -586,6 +602,7 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { for (i <- 1 to 6) sql(s"INSERT INTO tbl VALUES ($i)") val t = registerTable("tbl") checkpointSpec(t, version = 6) + readSpec(t) snapshotSpec(t) } @@ -594,8 +611,10 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { sql("INSERT INTO tbl VALUES (1)") sql("INSERT INTO tbl VALUES (2)") val t = registerTable("tbl") - injectActions(t, 2, Seq("""{"txn":{"appId":"cp-app","version":42,"lastUpdated":1}}""")) + injectActions(t, 2, Seq(Txn("cp-app", version = 42, lastUpdated = Some(1)))) checkpointSpec(t, version = 2) + snapshotSpec(t) + readSpec(t) } test("cp_with_domain_metadata") { @@ -604,8 +623,9 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { sql("INSERT INTO tbl VALUES (1),(2),(3)") sql("DELETE FROM tbl WHERE id = 2") val t = registerTable("tbl") - val dm = """{"domain":"cpDomain","configuration":"{\"k\":\"v\"}","removed":false}""" - injectActions(t, 2, Seq(s"""{"domainMetadata":$dm}""")) + injectActions(t, 2, Seq(DomainMetadata("cpDomain", configuration = "{\"k\":\"v\"}"))) checkpointSpec(t, version = 2) + snapshotSpec(t) + readSpec(t) } } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala index 21171109..426a2c05 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala @@ -19,10 +19,8 @@ package io.delta.workload.tables import io.delta.workload.WorkloadTestSuite /** - * CRC (version-checksum) read workloads. Delta writes a `.crc` sidecar on each commit; - * each test builds a table with plain SQL and declares a `crcSpec` asserting the core aggregate - * fields (tableSizeBytes, numFiles, protocol) plus any feature-gated fields the `.crc` carries - * (deletion-vector counts, set transactions, inCommitTimestamp). + * Version-checksum coverage. `crcSpec` asserts a `.crc` was written; the paired + * `snapshotSpec` at that version drives on-read checksum verification of its contents. */ class CrcSuite extends WorkloadTestSuite("crc") { @@ -32,7 +30,7 @@ class CrcSuite extends WorkloadTestSuite("crc") { sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") val t = registerTable("tbl") crcSpec(t, version = 1) - snapshotSpec(t) + snapshotSpec(t, version = 1L) } test("crc_partitioned") { @@ -41,6 +39,7 @@ class CrcSuite extends WorkloadTestSuite("crc") { sql("INSERT INTO tbl VALUES (4,1)") val t = registerTable("tbl") crcSpec(t, version = 1) + snapshotSpec(t, version = 1L) } test("crc_after_delete") { @@ -52,6 +51,7 @@ class CrcSuite extends WorkloadTestSuite("crc") { val t = registerTable("tbl") // v3 is the delete commit; the trailing insert (v4) makes it non-latest. crcSpec(t, version = 3) + snapshotSpec(t, version = 3L) } test("crc_with_deletion_vectors") { @@ -63,6 +63,7 @@ class CrcSuite extends WorkloadTestSuite("crc") { val t = registerTable("tbl") // v2 is the deletion-vector commit; the trailing insert (v3) makes it non-latest. crcSpec(t, version = 2) + snapshotSpec(t, version = 2L) } test("crc_multiple_versions") { @@ -75,5 +76,7 @@ class CrcSuite extends WorkloadTestSuite("crc") { // Both targets are non-latest (table ends at v4). crcSpec(t, version = 1, name = "crc_v1") crcSpec(t, version = 3, name = "crc_v3") + snapshotSpec(t, version = 1L) + snapshotSpec(t, version = 3L) } } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala index 2524b619..58f62b4c 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala @@ -44,6 +44,7 @@ class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { commitOp(w, txn = Some(AppTxn("cp-app", 42L))) // v2 val t = endWrite(w) checkpointSpec(t, version = 2) + snapshotSpec(t) } test("checkpoint_with_domain_metadata") { @@ -53,6 +54,7 @@ class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { commitOp(w, addDomainMetadata = Some(Seq(AddDomainMetadata("cpDomain", "{\"k\":\"v\"}")))) // v2 val t = endWrite(w) checkpointSpec(t, version = 2) + snapshotSpec(t) } test("crc_with_deletion_vectors") { @@ -63,6 +65,7 @@ class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { deleteOp(w, "id = 2") // v2 -> deletion vector val t = endWrite(w) crcSpec(t, version = 2) + snapshotSpec(t) } test("crc_with_set_transaction") { @@ -71,6 +74,7 @@ class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { commitOp(w, txn = Some(AppTxn("crc-app", 7L))) // v2 val t = endWrite(w) crcSpec(t, version = 2) + snapshotSpec(t) } test("cdf_updates_and_deletes") { From 517c4c7f9919bf689213739973824fc3328581bf Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Wed, 23 Sep 2026 01:26:18 +0000 Subject: [PATCH 3/4] refactor: align cdf spec serde with read/snapshot and fail loud on capture gaps --- .../io/delta/workload/WorkloadGenerator.scala | 15 ++++--- .../scala/io/delta/workload/WorkloadOps.scala | 14 +++++++ .../delta/workload/capture/CdfCapture.scala | 39 ++++++++----------- .../delta/workload/capture/CrcCapture.scala | 6 +-- .../io/delta/workload/json/JsonUtil.scala | 24 ++++++++++++ .../io/delta/workload/model/SpecModel.scala | 10 ++--- .../workload/validate/WorkloadValidator.scala | 15 ++++--- .../io/delta/workload/SpecSerdeSuite.scala | 7 ++++ .../workload/tables/CheckpointsSuite.scala | 20 ++-------- 9 files changed, 88 insertions(+), 62 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 2892bc81..ac2d3748 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -126,30 +126,29 @@ object WorkloadGenerator { builder.buildSpec(spark, destTablePath, testOutputDir) } - // The cdf/checkpoint/crc specs need no name tracking (no dedup/latest handling), only a count - // for the summary total. - var extraSpecCount = 0 + // Running count of all specs generated for this case. + var totalSpecCount = 0 for (cs <- ts.cdfSpecs) { val specPath = CdfCapture.capture(spark, dirName, destTablePath, testOutputDir, specsDir, name = cs.name, startVersion = cs.startVersion, endVersion = cs.endVersion, startTimestamp = cs.startTimestamp, endTimestamp = cs.endTimestamp, predicate = cs.predicate, columns = cs.columns, expectError = cs.expectError) - extraSpecCount += 1 + totalSpecCount += 1 checkAssertion(cs, specPath) } for (cp <- ts.checkpointSpecs) { val specPath = CheckpointCapture.capture(spark, dirName, destTablePath, specsDir, version = cp.version, name = cp.name) - extraSpecCount += 1 + totalSpecCount += 1 checkAssertion(cp, specPath) } for (crc <- ts.crcSpecs) { CrcCapture.capture(spark, dirName, destTablePath, specsDir, version = crc.version, name = crc.name).foreach { p => - extraSpecCount += 1 + totalSpecCount += 1 checkAssertion(crc, p) } } @@ -179,8 +178,8 @@ object WorkloadGenerator { Files.write(reproDir.resolve("generate.scala"), s"// Generated by ${ts.outputName} test\n".getBytes("UTF-8")) - val total = snapshotNames.size + readNames.size + extraSpecCount - println(s" $dirName: $total specs") + totalSpecCount += snapshotNames.size + readNames.size + println(s" $dirName: $totalSpecCount specs") dirName } diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala index dc4c3204..76595ad2 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala @@ -406,4 +406,18 @@ trait WorkloadOps { if (Files.exists(CommitLog.commitFile(tableDir, version))) CommitLog.mutate(tableDir, version)(modifier) } + + /** + * Append `actions` to the commit JSON at `version`. + */ + def injectCommitActions(table: TableHandle, version: Long)(actions: Seq[Action]): Unit = + mutateTable(table) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val commitFile = logDir.resolve(f"$version%020d.json") + val content = new String(Files.readAllBytes(commitFile), "UTF-8") + Files.write( + commitFile, + (content.trim + "\n" + actions.map(_.toJson).mkString("\n") + "\n").getBytes("UTF-8")) + Files.deleteIfExists(logDir.resolve(f"$version%020d.crc")) + } } diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala index 1e9fd500..8de7a930 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession} import io.delta.workload.deltaharness.DeltaHarness import io.delta.workload.engine.{RowComparison, SnapshotResolver, SpecOutcome} import io.delta.workload.json.JsonUtil -import io.delta.workload.model.{CdfExpected, CdfSpec, SpecError} +import io.delta.workload.model.{CdfExpected, CdfSpec, Failed, SpecError, SpecExpectation, Succeeded} object CdfCapture { @@ -50,7 +50,7 @@ object CdfCapture { Files.createDirectories(expectedDir) val specPath = specsDir.resolve(s"$specName.json") - val (expected, expectedError) = try { + val expectation: SpecExpectation[CdfExpected] = try { var df = buildReader(spark, tablePath, startVersion, endVersion, startTimestamp, endTimestamp) df = SnapshotResolver.applyFilters(df, predicate, columns) @@ -62,13 +62,13 @@ object CdfCapture { try { writeExpectedData(expectedDir, df) - (Some(CdfExpected(count)), None) + Succeeded(CdfExpected(count)) } finally { df.unpersist() } } catch { case NonFatal(e) => - (None, Some(SpecError(SpecOutcome.extractErrorCode(e), Option(e.getMessage).getOrElse("")))) + Failed(SpecError(SpecOutcome.extractErrorCode(e), Option(e.getMessage).getOrElse(""))) } expectError.foreach { expected => @@ -76,31 +76,30 @@ object CdfCapture { if (expectedDir.toFile.exists()) FileUtils.deleteDirectory(expectedDir.toFile) throw new RuntimeException(msg) } - expectedError match { - case None => + expectation match { + case Succeeded(_) => fail(s"CDF $specName: declared expectError=" + (if (expected.isEmpty) "(any)" else s"'$expected'") + " but operation succeeded") - case Some(err) if expected.nonEmpty && + case Failed(err) if expected.nonEmpty && SpecOutcome.normalizeErrorCode(err.errorCode) != SpecOutcome.normalizeErrorCode(expected) => fail(s"CDF $specName: declared expectError='$expected' but got " + s"'${err.errorCode}'") - case _ => // matches + case _ => // matches (Failed with matching or any code) } } val spec = CdfSpec(startVersion, endVersion, startTimestamp, endTimestamp, - predicate, columns, expected, expectedError) + predicate, columns, expectation) JsonUtil.writeSpec(specPath, spec) validateFromSpec(spark, tablePath, expectedDir, specPath) - (expected, expectedError) match { - case (Some(exp), _) => + expectation match { + case Succeeded(exp) => println(s" CDF captured: $specName (${exp.rowCount} change rows)") - case (_, Some(err)) => + case Failed(err) => println(s" CDF captured (error): $specName [${err.errorCode}] ${err.errorMessage}") - case _ => } specPath } @@ -115,23 +114,17 @@ object CdfCapture { spec.startTimestamp, spec.endTimestamp), spec.predicate, spec.columns).drop("_commit_timestamp") - if (spec.expectedError.isDefined) { - val actualCode = try { + SpecOutcome.compareExpectation(spec.expectation, specName) { + SpecOutcome.runErrorCode { reader.write.format("noop").mode("overwrite").save() None - } catch { - case NonFatal(e) => Some(SpecOutcome.extractErrorCode(e)) } - require(actualCode.isDefined, - s"Error validation FAILED for $specName: expected operation to fail but it succeeded") - } else if (spec.expected.isDefined) { - val rereadDf = reader - + } { _ => val expectedDataPath = expectedDir.resolve("expected_data") require(Files.exists(expectedDataPath), s"Validation FAILED for $specName: expected_data is missing") RowComparison.assertRowsEqual( - spark.read.parquet(expectedDataPath.toString), rereadDf, specName) + spark.read.parquet(expectedDataPath.toString), reader, specName) } } diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala index 677c7f0f..e337bca8 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala @@ -35,10 +35,8 @@ object CrcCapture { version: Long, name: String): Option[Path] = { val specName = s"${testId}_$name" - if (!crcFileExists(tablePath, version)) { - println(s" CRC skipped: $specName (version=$version): no .crc present at that version") - return None - } + require(crcFileExists(tablePath, version), + s"CRC capture failed for $specName (version=$version): no .crc present at that version") val specPath = specsDir.resolve(s"$specName.json") JsonUtil.writeSpec(specPath, CrcSpec(version)) println(s" CRC captured: $specName (version=$version)") diff --git a/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala b/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala index c4c2eebb..584dd2f9 100644 --- a/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala +++ b/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala @@ -106,6 +106,30 @@ private[workload] class SnapshotSpecDeserializer extends SpecDeserializer[Snapsh SnapshotQuery(SpecExpectationJson.optLong(n, "version"), SpecExpectationJson.optText(n, "timestamp")), SpecExpectationJson.read(n, classOf[SnapshotResult])) } +private[workload] class CdfSpecSerializer extends SpecSerializer[CdfSpec] { + override def serialize(s: CdfSpec, g: JsonGenerator, p: SerializerProvider): Unit = + SpecExpectationJson.writeEnvelope(g, s.`type`, version = None, timestamp = None, s.expectation) { + s.startVersion.foreach(v => g.writeNumberField("startVersion", v)) + s.startTimestamp.foreach(t => g.writeStringField("startTimestamp", t)) + s.endVersion.foreach(v => g.writeNumberField("endVersion", v)) + s.endTimestamp.foreach(t => g.writeStringField("endTimestamp", t)) + s.predicate.foreach(pr => g.writeStringField("predicate", pr)) + s.columns.foreach { cols => + g.writeArrayFieldStart("columns"); cols.foreach(g.writeString); g.writeEndArray() + } + } +} +private[workload] class CdfSpecDeserializer extends SpecDeserializer[CdfSpec] { + override protected def fromNode(n: JsonNode): CdfSpec = + CdfSpec( + startVersion = SpecExpectationJson.optLong(n, "startVersion"), + endVersion = SpecExpectationJson.optLong(n, "endVersion"), + startTimestamp = SpecExpectationJson.optText(n, "startTimestamp"), + endTimestamp = SpecExpectationJson.optText(n, "endTimestamp"), + predicate = SpecExpectationJson.optText(n, "predicate"), + columns = SpecExpectationJson.optStrings(n, "columns"), + expectation = SpecExpectationJson.read(n, classOf[CdfExpected])) +} // ============================================================================= // JSON and DataFrame utilities diff --git a/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala b/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala index 4d8df8a2..97457fe3 100644 --- a/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala +++ b/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation._ import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import io.delta.workload.deltaharness.{Format, Metadata} -import io.delta.workload.json.{ReadSpecDeserializer, ReadSpecSerializer, SnapshotSpecDeserializer, SnapshotSpecSerializer} +import io.delta.workload.json.{CdfSpecDeserializer, CdfSpecSerializer, ReadSpecDeserializer, ReadSpecSerializer, SnapshotSpecDeserializer, SnapshotSpecSerializer} // ============================================================================= // Spec Expected types: success data or error info @@ -134,9 +134,8 @@ case class WriteSpec(commits: Seq[WriteCommit]) extends Spec { val `type`: String = "write" } -@JsonPropertyOrder(Array("type", "startVersion", "startTimestamp", "endVersion", "endTimestamp", - "predicate", "columns", "expected", "expectedError")) -@JsonInclude(JsonInclude.Include.NON_ABSENT) +@JsonSerialize(using = classOf[CdfSpecSerializer]) +@JsonDeserialize(using = classOf[CdfSpecDeserializer]) case class CdfSpec( startVersion: Option[Long] = None, endVersion: Option[Long] = None, @@ -144,8 +143,7 @@ case class CdfSpec( endTimestamp: Option[String] = None, predicate: Option[String] = None, columns: Option[Seq[String]] = None, - expected: Option[CdfExpected] = None, - expectedError: Option[SpecError] = None) extends Spec { + expectation: SpecExpectation[CdfExpected]) extends Spec { val `type`: String = "cdf" } diff --git a/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala index 606a130f..1bfe8a4f 100644 --- a/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala @@ -185,13 +185,18 @@ object WorkloadValidator { } listSpecs(specsDir).foreach { specFile => val id = specFile.getFileName.toString.stripSuffix(".json") - def skip(v: Long): Unit = - println(s" SKIP $id: spec version $v > table latest $latest (not reproduced by the engine under test)") + def underWrite(v: Long): SpecOutcome = + SpecFailed( + id, + s"spec version $v > table latest $latest: engine under test never reproduced that " + + s"version (silent under-write)" + ) try { JsonUtil.readSpec(specFile) match { - case s: CheckpointSpec if s.version > latest => skip(s.version) - case s: CrcSpec if s.version > latest => skip(s.version) - case s: SnapshotSpec if s.query.version.exists(_ > latest) => skip(s.query.version.get) + case s: CheckpointSpec if s.version > latest => outcomes += underWrite(s.version) + case s: CrcSpec if s.version > latest => outcomes += underWrite(s.version) + case s: SnapshotSpec if s.query.version.exists(_ > latest) => + outcomes += underWrite(s.query.version.get) case _: CheckpointSpec => CheckpointCapture.validateFromSpec(spark, tablePath, specFile) outcomes += SpecPassed(id) diff --git a/workload-generator/src/test/scala/io/delta/workload/SpecSerdeSuite.scala b/workload-generator/src/test/scala/io/delta/workload/SpecSerdeSuite.scala index 4bddb603..9b1de399 100644 --- a/workload-generator/src/test/scala/io/delta/workload/SpecSerdeSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/SpecSerdeSuite.scala @@ -70,6 +70,13 @@ class SpecSerdeSuite extends AnyFunSuite { assert(mapper.readValue(str(spec), classOf[SnapshotSpec]) == spec) } + test("CdfSpec: failure writes `error`, not `expected`, and round-trips") { + val spec = CdfSpec(startVersion = Some(0), expectation = Failed(SpecError("DELTA_CDF_ERROR", "no"))) + val n = tree(spec) + assert(n.has("error") && !n.has("expected")) + assert(mapper.readValue(str(spec), classOf[CdfSpec]) == spec) + } + test("polymorphic dispatch: readValue[Spec] picks the subtype from `type`") { val read: Spec = ReadSpec(ReadQuery(), Succeeded(ReadResult(0, 0, 0))) val snap: Spec = SnapshotSpec(SnapshotQuery(), Failed(SpecError("E", "m"))) diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala index 4bddb5d1..1aee2d7e 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala @@ -24,21 +24,8 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { private def checkpoint(name: String): Unit = forceCheckpoint(name) - /** - * Append `actions` to the commit JSON at `version`. Also drops that version's `.crc` - * (now stale w.r.t. the edited commit) so the subsequent `checkpoint()` does not trip Delta's - * checksum integrity check and refuse to write. - */ - private def injectActions(t: TableHandle, version: Int, actions: Seq[Action]): Unit = { - mutateTable(t) { tableDir => - val logDir = tableDir.resolve("_delta_log") - val commitFile = logDir.resolve(f"$version%020d.json") - val content = new String(java.nio.file.Files.readAllBytes(commitFile), "UTF-8") - java.nio.file.Files.write(commitFile, - (content.trim + "\n" + actions.map(_.toJson).mkString("\n") + "\n").getBytes("UTF-8")) - java.nio.file.Files.deleteIfExists(logDir.resolve(f"$version%020d.crc")) - } - } + private def injectActions(t: TableHandle, version: Int, actions: Seq[Action]): Unit = + injectCommitActions(t, version.toLong)(actions) // === Read/snapshot workloads over tables that have a checkpoint === @@ -619,7 +606,8 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { test("cp_with_domain_metadata") { sql("""CREATE TABLE tbl (id INT) USING delta - TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')""") + TBLPROPERTIES ('delta.enableDeletionVectors' = 'true', + 'delta.feature.domainMetadata' = 'supported')""") sql("INSERT INTO tbl VALUES (1),(2),(3)") sql("DELETE FROM tbl WHERE id = 2") val t = registerTable("tbl") From c6e21815027bd85f5bd34ff4b618667ed1d6134c Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Thu, 27 Aug 2026 02:42:07 +0000 Subject: [PATCH 4/4] feat: support nested-value inserts and expand write-path test coverage --- .../deltaharness/SparkRowMaterializer.scala | 29 +- .../workload/tables/CheckpointsSuite.scala | 2 +- .../workload/tables/WriteBoundarySuite.scala | 148 +++++++++ .../tables/WriteColumnMappingSuite.scala | 90 ++++++ .../workload/tables/WriteFeaturesSuite.scala | 82 +++++ .../tables/WriteMetadataSpecsSuite.scala | 48 +++ .../workload/tables/WriteNestedSuite.scala | 163 ++++++++++ .../tables/WritePartitioningSuite.scala | 283 ++++++++++++++++++ .../tables/WriteResolutionSuite.scala | 139 +++++++++ .../workload/tables/WriteStatsSuite.scala | 142 +++++++++ .../workload/tables/WriteTypesSuite.scala | 13 +- .../workload/tables/WriteVolumeSuite.scala | 115 +++++++ 12 files changed, 1234 insertions(+), 20 deletions(-) create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteBoundarySuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteColumnMappingSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteFeaturesSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteNestedSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WritePartitioningSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteResolutionSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteStatsSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteVolumeSuite.scala diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/SparkRowMaterializer.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/SparkRowMaterializer.scala index 4a261d6f..569834e5 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/SparkRowMaterializer.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/SparkRowMaterializer.scala @@ -32,9 +32,7 @@ import org.apache.spark.sql.types.{Metadata => _, _} trait SparkRowMaterializer extends DeltaHarness { override def writeRows( spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]], dest: Path): Unit = { - val sparkRows = rows.map { r => - Row.fromSeq(schema.fields.map(f => coerce(r.get(f.name), f.dataType)).toSeq) - } + val sparkRows = rows.map(r => coerce(Some(r), schema).asInstanceOf[Row]) val tmp = Files.createTempDirectory("row-parquet") try { spark.createDataFrame(sparkRows.asJava, schema) @@ -51,10 +49,8 @@ trait SparkRowMaterializer extends DeltaHarness { } /** - * Coerce a row's value to the column's Spark type. `None`/`null` -> SQL NULL. Unknown types - * fail loud rather than passing the raw value through: a silent passthrough that Spark happens - * to accept could write a wrong-but-symmetric value (wrong on both capture AND replay), which - * the row comparison would not catch: a false pass. + * Coerce a row's value to the column's Spark type. `None`/`null` -> SQL NULL, and struct/array/map + * values recurse. */ private def coerce(value: Option[Any], dataType: DataType): Any = value.flatMap(Option(_)) match { @@ -97,6 +93,25 @@ trait SparkRowMaterializer extends DeltaHarness { case s: String => java.util.Base64.getDecoder.decode(s) case _ => unsupported(v, dataType) } + case st: StructType => v match { + case m: Map[_, _] => + val fields = m.asInstanceOf[Map[String, Any]] + val unknown = fields.keySet -- st.fieldNames.toSet + require(unknown.isEmpty, + s"coerce: struct value has unknown field(s) ${unknown.mkString(", ")}; " + + s"schema fields are ${st.fieldNames.mkString(", ")}") + Row.fromSeq(st.fields.map(f => coerce(fields.get(f.name), f.dataType))) + case _ => unsupported(v, dataType) + } + case at: ArrayType => v match { + case s: Seq[_] => s.map(e => coerce(Option(e), at.elementType)) + case _ => unsupported(v, dataType) + } + case mt: MapType => v match { + case m: Map[_, _] => + m.map { case (k, vv) => coerce(Option(k), mt.keyType) -> coerce(Option(vv), mt.valueType) } + case _ => unsupported(v, dataType) + } case _ => unsupported(v, dataType) } } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala index 1aee2d7e..ded051af 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala @@ -566,7 +566,7 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { test("cp_v2") { sql("""CREATE TABLE tbl (id INT, name STRING) USING delta TBLPROPERTIES ('delta.checkpointPolicy' = 'v2', - 'delta.enableDeletionVectors' = 'trque')""") + 'delta.enableDeletionVectors' = 'true')""") sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c')") sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") val t = registerTable("tbl") diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteBoundarySuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBoundarySuite.scala new file mode 100644 index 00000000..bbe7a681 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBoundarySuite.scala @@ -0,0 +1,148 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite + +/** + * Boundary and special-value inserts: type extremes (int/long min/max), the special doubles + * (NaN, +/-Infinity, signed zero), date/timestamp range ends, string edges, and high-precision + * decimals. + */ +class WriteBoundarySuite extends WorkloadTestSuite("write_boundary") { + + test("integer_and_long_extremes") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("c_int", IntegerType).add("c_long", LongType)) + insertOp(w, Seq( + Map("id" -> 1, "c_int" -> Int.MinValue, "c_long" -> Long.MinValue), + Map("id" -> 2, "c_int" -> Int.MaxValue, "c_long" -> Long.MaxValue), + Map("id" -> 3, "c_int" -> 0, "c_long" -> 0L))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "c_int = -2147483648", name = Some("read_int_min")) + readSpec(t, predicate = "c_long = 9223372036854775807", name = Some("read_long_max")) + snapshotSpec(t) + } + + test("double_special_values") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("d", DoubleType)) + insertOp(w, Seq( + Map("id" -> 1, "d" -> Double.NaN), + Map("id" -> 2, "d" -> Double.PositiveInfinity), + Map("id" -> 3, "d" -> Double.NegativeInfinity), + Map("id" -> 4, "d" -> 0.0), + Map("id" -> 5, "d" -> -0.0))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("date_and_timestamp_range_ends") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("d", DateType).add("ts", TimestampType)) + insertOp(w, Seq( + Map("id" -> 1, "d" -> "0001-01-01", "ts" -> "0001-01-01 00:00:00"), + Map("id" -> 2, "d" -> "9999-12-31", "ts" -> "9999-12-31 23:59:59"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "d = DATE '9999-12-31'", name = Some("read_max_date")) + snapshotSpec(t) + } + + test("empty_and_large_strings") { + val large = "x" * 8192 // multi-KB single value + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("s", StringType)) + insertOp(w, Seq( + Map("id" -> 1, "s" -> ""), + Map("id" -> 2, "s" -> large), + Map("id" -> 3, "s" -> "normal"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "s = ''", name = Some("read_empty")) + snapshotSpec(t) + } + + test("high_precision_decimals") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("d380", DecimalType(38, 0)) + .add("d3818", DecimalType(38, 18)) + .add("d54", DecimalType(5, 4))) + insertOp(w, Seq( + Map("id" -> 1, + "d380" -> "99999999999999999999999999999999999999", + "d3818" -> "12345678901234567890.123456789012345678", + "d54" -> "9.9999"), + Map("id" -> 2, + "d380" -> "-99999999999999999999999999999999999999", + "d3818" -> "-99999999999999999999.999999999999999999", + "d54" -> "-9.9999"), + Map("id" -> 3, "d380" -> "0", "d3818" -> "0.000000000000000001", "d54" -> "0.0001"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "d54 < 0", name = Some("read_negative_d54")) + snapshotSpec(t) + } + + test("decimals_with_nulls") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("amount", DecimalType(20, 6))) + insertOp(w, Seq( + Map("id" -> 1, "amount" -> "123456789012.345678"), + Map("id" -> 2, "amount" -> null), + Map("id" -> 3, "amount" -> "-0.000001"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "amount IS NULL", name = Some("read_null_amount")) + readSpec(t, predicate = "amount IS NOT NULL", name = Some("read_non_null_amount")) + snapshotSpec(t) + } + + test("float_special_values") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("f", FloatType)) + insertOp(w, Seq( + Map("id" -> 1, "f" -> Float.NaN), + Map("id" -> 2, "f" -> Float.PositiveInfinity), + Map("id" -> 3, "f" -> Float.NegativeInfinity), + Map("id" -> 4, "f" -> 0.0f), + Map("id" -> 5, "f" -> -0.0f))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("extreme_values_with_checkpoint") { + // Extreme scalar values across two commits, with a forced checkpoint at the final version. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("c_int", IntegerType) + .add("c_long", LongType).add("ts", TimestampType).add("d", DecimalType(18, 0))) + insertOp(w, Seq(Map("id" -> 1, "c_int" -> Int.MinValue, "c_long" -> Long.MinValue, + "ts" -> "9999-12-31 23:59:59.999999", "d" -> "-999999999999999999"))) + insertOp(w, Seq(Map("id" -> 2, "c_int" -> Int.MaxValue, "c_long" -> Long.MaxValue, + "ts" -> "9999-12-31 23:59:59.999999", "d" -> "999999999999999999"))) + val t = endWrite(w) + readSpec(t) + checkpointSpec(t, version = 2) + snapshotSpec(t) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteColumnMappingSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteColumnMappingSuite.scala new file mode 100644 index 00000000..e6285a0e --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteColumnMappingSuite.scala @@ -0,0 +1,90 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite + +/** + * Inserts into column-mapping tables (`name` and `id` modes), including columns whose logical names + * carry spaces and dots that only column mapping permits. Physical column names differ from logical + * names on disk. The `delta.*` keys below are the public OSS Delta property names. + */ +class WriteColumnMappingSuite extends WorkloadTestSuite("write_column_mapping") { + + test("name_mode_insert") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("name", StringType), + properties = Map("delta.columnMapping.mode" -> "name")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"), Map("id" -> 2, "name" -> "bob"))) + insertOp(w, Seq(Map("id" -> 3, "name" -> "cara"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "name = 'bob'", name = Some("read_bob")) + snapshotSpec(t) + } + + test("id_mode_insert") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("v", IntegerType), + properties = Map("delta.columnMapping.mode" -> "id")) + insertOp(w, (1 to 4).map(i => Map("id" -> i, "v" -> i * 100))) + insertOp(w, Seq(Map("id" -> 5, "v" -> 500))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "v > 250", name = Some("read_large_v")) + snapshotSpec(t) + } + + test("special_character_column_names") { + // Spaces and dots in logical names are only legal under column mapping; the physical names on + // disk are mapping-generated. Reads reference the logical names with backtick escaping. + val w = createTableOp("tbl", + schema = new StructType() + .add("id", IntegerType) + .add("first name", StringType) + .add("addr.city", StringType), + properties = Map("delta.columnMapping.mode" -> "name")) + insertOp(w, Seq( + Map("id" -> 1, "first name" -> "ada", "addr.city" -> "london"), + Map("id" -> 2, "first name" -> "grace", "addr.city" -> "new york"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "`first name` = 'ada'", name = Some("read_ada")) + readSpec(t, predicate = "`addr.city` = 'london'", name = Some("read_london")) + snapshotSpec(t) + } + + test("upgrade_none_to_name_then_insert") { + // Start with mapping off, upgrade the mode to `name` in place, then insert under the upgraded + // table. The upgrade also raises the reader/writer protocol versions. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("v", IntegerType)) + insertOp(w, Seq(Map("id" -> 1, "v" -> 10), Map("id" -> 2, "v" -> 20))) + setPropertiesOp(w, Map( + "delta.columnMapping.mode" -> "name", + "delta.minReaderVersion" -> "2", + "delta.minWriterVersion" -> "5")) + insertOp(w, Seq(Map("id" -> 3, "v" -> 30))) + val t = endWrite(w) + readSpec(t) + readSpec(t, version = 1, name = Some("read_before_upgrade")) + readSpec(t, predicate = "v >= 20", name = Some("read_ge_20")) + snapshotSpec(t) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteFeaturesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteFeaturesSuite.scala new file mode 100644 index 00000000..820432d2 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteFeaturesSuite.scala @@ -0,0 +1,82 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite + +/** + * Inserts into tables created with insert-relevant table features enabled. Each case proves plain + * appends succeed while the feature is listed in the table's protocol. The feature flags below + * (`delta.*`) are the public OSS Delta property names. + */ +class WriteFeaturesSuite extends WorkloadTestSuite("write_features") { + + test("append_only_inserts") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("v", StringType), + properties = Map("delta.appendOnly" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "v" -> "a"), Map("id" -> 2, "v" -> "b"))) + insertOp(w, Seq(Map("id" -> 3, "v" -> "c"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, version = 1, name = Some("read_first_append")) + snapshotSpec(t) + } + + test("deletion_vectors_plain_inserts") { + // DVs enabled at create; plain inserts only (no delete needed to exercise the enabled feature). + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("v", IntegerType), + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, (1 to 4).map(i => Map("id" -> i, "v" -> i * 10))) + insertOp(w, Seq(Map("id" -> 5, "v" -> 50))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "v > 25", name = Some("read_large_v")) + snapshotSpec(t) + } + + test("type_widening_inserts") { + // Type widening enabled at create; plain appends only. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("v", IntegerType), + properties = Map("delta.enableTypeWidening" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "v" -> 10), Map("id" -> 2, "v" -> 20))) + insertOp(w, Seq(Map("id" -> 3, "v" -> 30))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "v >= 20", name = Some("read_ge_20")) + snapshotSpec(t) + } + + test("change_data_feed_plain_inserts") { + // CDF enabled at create; plain appends only. Appends emit no change-data files (CDF records + // changes for updates/deletes/merges), but the feature stays listed in the protocol throughout. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("v", IntegerType), + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "v" -> 10), Map("id" -> 2, "v" -> 20))) + insertOp(w, Seq(Map("id" -> 3, "v" -> 30))) + val t = endWrite(w) + readSpec(t) + readSpec(t, version = 1, name = Some("read_first_append")) + readSpec(t, predicate = "v >= 20", name = Some("read_ge_20")) + snapshotSpec(t) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala index 58f62b4c..21dfa2e5 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala @@ -29,6 +29,50 @@ class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { private def idName = new StructType().add("id", IntegerType).add("name", StringType) + // Runnable checkpoint cases: create + high-level DML only (no low-level commit), so a write + // harness can reproduce the whole table and then force + verify a checkpoint at the tip version. + test("checkpoint_basic") { + val w = createTableOp("tbl", idName) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"), Map("id" -> 2, "name" -> "b"))) // v1 + insertOp(w, Seq(Map("id" -> 3, "name" -> "c"))) // v2 + val t = endWrite(w) + checkpointSpec(t, version = 2) + snapshotSpec(t) + readSpec(t) + } + + test("checkpoint_v2") { + val w = createTableOp("tbl", idName, + properties = Map("delta.checkpointPolicy" -> "v2", "delta.enableDeletionVectors" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"), Map("id" -> 2, "name" -> "b"))) // v1 + insertOp(w, Seq(Map("id" -> 3, "name" -> "c"))) // v2 + val t = endWrite(w) + checkpointSpec(t, version = 2) + snapshotSpec(t) + readSpec(t) + } + + test("checkpoint_after_dml") { + val w = createTableOp("tbl", idName, + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, (1 to 4).map(i => Map("id" -> i, "name" -> s"n$i"))) // v1 + updateOp(w, predicate = "id <= 2", set = Map("name" -> "'updated'")) // v2 + deleteOp(w, predicate = "id = 4") // v3 + val t = endWrite(w) + checkpointSpec(t, version = 3) + snapshotSpec(t) + readSpec(t) + } + + test("checkpoint_many_commits") { + val w = createTableOp("tbl", idName) + for (i <- 1 to 6) insertOp(w, Seq(Map("id" -> i, "name" -> s"n$i"))) // v1..v6 + val t = endWrite(w) + checkpointSpec(t, version = 6) + snapshotSpec(t) + readSpec(t) + } + test("checkpoint_partitioned") { val schema = new StructType().add("id", IntegerType).add("part", IntegerType) val w = createTableOp("tbl", schema, partitionColumns = Seq("part")) @@ -36,6 +80,8 @@ class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { insertOp(w, Seq(Map("id" -> 3, "part" -> 0))) // v2 val t = endWrite(w) checkpointSpec(t, version = 2) + snapshotSpec(t) + readSpec(t) } test("checkpoint_with_set_transaction") { @@ -45,6 +91,7 @@ class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { val t = endWrite(w) checkpointSpec(t, version = 2) snapshotSpec(t) + readSpec(t) } test("checkpoint_with_domain_metadata") { @@ -55,6 +102,7 @@ class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { val t = endWrite(w) checkpointSpec(t, version = 2) snapshotSpec(t) + readSpec(t) } test("crc_with_deletion_vectors") { diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteNestedSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteNestedSuite.scala new file mode 100644 index 00000000..022aae89 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteNestedSuite.scala @@ -0,0 +1,163 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite + +/** + * Nested-value inserts: struct/array/map columns, their combinations, nulls at every level, and a + * nested column alongside partitioning. + */ +class WriteNestedSuite extends WorkloadTestSuite("write_nested") { + + test("struct_column") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("s", new StructType().add("x", IntegerType).add("y", StringType))) + insertOp(w, Seq( + Map("id" -> 1, "s" -> Map("x" -> 10, "y" -> "a")), + Map("id" -> 2, "s" -> Map("x" -> 20, "y" -> "b")))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("array_column") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("nums", ArrayType(IntegerType))) + insertOp(w, Seq( + Map("id" -> 1, "nums" -> Seq(1, 2, 3)), + Map("id" -> 2, "nums" -> Seq(4, 5)))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("map_column") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("m", MapType(StringType, IntegerType))) + insertOp(w, Seq( + Map("id" -> 1, "m" -> Map("a" -> 1, "b" -> 2)), + Map("id" -> 2, "m" -> Map("c" -> 3)))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("array_of_struct") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("items", ArrayType(new StructType().add("id", IntegerType).add("name", StringType)))) + insertOp(w, Seq( + Map("id" -> 1, "items" -> Seq(Map("id" -> 1, "name" -> "a"), Map("id" -> 2, "name" -> "b"))), + Map("id" -> 2, "items" -> Seq(Map("id" -> 3, "name" -> "c"))))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("map_with_struct_value") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("m", MapType(StringType, new StructType().add("a", IntegerType).add("b", IntegerType)))) + insertOp(w, Seq( + Map("id" -> 1, "m" -> Map("k1" -> Map("a" -> 1, "b" -> 2))), + Map("id" -> 2, "m" -> Map("k2" -> Map("a" -> 3, "b" -> 4))))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("deeply_nested_struct") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("s", new StructType().add("a", IntegerType) + .add("inner", new StructType().add("b", IntegerType).add("c", StringType)))) + insertOp(w, Seq( + Map("id" -> 1, "s" -> Map("a" -> 1, "inner" -> Map("b" -> 2, "c" -> "x"))), + Map("id" -> 2, "s" -> Map("a" -> 3, "inner" -> Map("b" -> 4, "c" -> "y"))))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("null_struct") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("s", new StructType().add("x", IntegerType).add("y", StringType))) + insertOp(w, Seq( + Map("id" -> 1, "s" -> Map("x" -> 10, "y" -> "a")), + Map("id" -> 2, "s" -> null))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("null_nested_field") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("s", new StructType().add("x", IntegerType).add("y", StringType))) + insertOp(w, Seq( + Map("id" -> 1, "s" -> Map("x" -> 10, "y" -> null)), + Map("id" -> 2, "s" -> Map("x" -> null, "y" -> "b")))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("null_array_and_map") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("nums", ArrayType(IntegerType)).add("m", MapType(StringType, IntegerType))) + insertOp(w, Seq( + Map("id" -> 1, "nums" -> Seq(1, 2), "m" -> Map("a" -> 1)), + Map("id" -> 2, "nums" -> null, "m" -> null))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("nested_in_partitioned_table") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("region", StringType) + .add("s", new StructType().add("x", IntegerType).add("y", StringType)), + partitionColumns = Seq("region")) + insertOp(w, Seq( + Map("id" -> 1, "region" -> "east", "s" -> Map("x" -> 10, "y" -> "a")), + Map("id" -> 2, "region" -> "west", "s" -> Map("x" -> 20, "y" -> "b")))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "region = 'east'", name = Some("read_east")) + snapshotSpec(t) + } + + test("nested_struct_stats") { + // A struct with leaf fields over multiple rows exercises per-leaf statistics generation. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("s", new StructType().add("n", IntegerType).add("label", StringType))) + insertOp(w, Seq( + Map("id" -> 1, "s" -> Map("n" -> 5, "label" -> "a")), + Map("id" -> 2, "s" -> Map("n" -> 15, "label" -> "b")), + Map("id" -> 3, "s" -> Map("n" -> 25, "label" -> "c")))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WritePartitioningSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WritePartitioningSuite.scala new file mode 100644 index 00000000..55ba11e2 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WritePartitioningSuite.scala @@ -0,0 +1,283 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite + +/** + * Partitioning-focused inserts: multi-column partition keys, high-cardinality partitioning, a + * string partition column, and NULL partition values. Each partition combination lands in its own + * directory, so these also exercise partitionValues encoding on the resulting Add actions. + */ +class WritePartitioningSuite extends WorkloadTestSuite("write_partitioning") { + + test("multi_column_partition") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("year", IntegerType) + .add("month", IntegerType).add("day", IntegerType).add("v", StringType), + partitionColumns = Seq("year", "month", "day")) + insertOp(w, Seq( + Map("id" -> 1, "year" -> 2023, "month" -> 1, "day" -> 15, "v" -> "a"), + Map("id" -> 2, "year" -> 2023, "month" -> 1, "day" -> 16, "v" -> "b"), + Map("id" -> 3, "year" -> 2023, "month" -> 2, "day" -> 1, "v" -> "c"))) + insertOp(w, Seq( + Map("id" -> 4, "year" -> 2024, "month" -> 12, "day" -> 31, "v" -> "d"), + Map("id" -> 5, "year" -> 2023, "month" -> 1, "day" -> 15, "v" -> "e"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "year = 2023 AND month = 1", name = Some("read_jan_2023")) + readSpec(t, predicate = "year = 2024", name = Some("read_2024")) + snapshotSpec(t) + } + + test("high_cardinality_partition") { + // 120 distinct partition values written in a single insert. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("p", IntegerType).add("v", IntegerType), + partitionColumns = Seq("p")) + insertOp(w, (0 until 120).map(i => Map("id" -> i, "p" -> i, "v" -> (i * 10)))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "p = 0", name = Some("read_p0")) + readSpec(t, predicate = "p = 119", name = Some("read_p119")) + snapshotSpec(t) + } + + test("string_partition_column") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("country", StringType).add("v", IntegerType), + partitionColumns = Seq("country")) + insertOp(w, Seq( + Map("id" -> 1, "country" -> "US", "v" -> 10), + Map("id" -> 2, "country" -> "DE", "v" -> 20), + Map("id" -> 3, "country" -> "JP", "v" -> 30), + Map("id" -> 4, "country" -> "US", "v" -> 40))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "country = 'US'", name = Some("read_us")) + snapshotSpec(t) + } + + test("null_partition_value") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("region", StringType).add("v", IntegerType), + partitionColumns = Seq("region")) + insertOp(w, Seq( + Map("id" -> 1, "region" -> "east", "v" -> 1), + Map("id" -> 2, "region" -> null, "v" -> 2), + Map("id" -> 3, "region" -> null, "v" -> 3))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "region IS NULL", name = Some("read_null_region")) + readSpec(t, predicate = "region IS NOT NULL", name = Some("read_named_region")) + snapshotSpec(t) + } + + test("append_into_existing_partition") { + // Several appends target the same partition value, so the partition accumulates files across + // commits. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("region", StringType).add("v", IntegerType), + partitionColumns = Seq("region")) + insertOp(w, Seq(Map("id" -> 1, "region" -> "east", "v" -> 10))) + insertOp(w, Seq(Map("id" -> 2, "region" -> "east", "v" -> 20))) + insertOp(w, Seq(Map("id" -> 3, "region" -> "east", "v" -> 30))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "region = 'east'", name = Some("read_east")) + snapshotSpec(t) + } + + test("append_creating_new_partition") { + // A later append introduces a partition value not seen in earlier commits. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("region", StringType).add("v", IntegerType), + partitionColumns = Seq("region")) + insertOp(w, Seq(Map("id" -> 1, "region" -> "east", "v" -> 1))) + insertOp(w, Seq(Map("id" -> 2, "region" -> "west", "v" -> 2))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "region = 'west'", name = Some("read_new_partition")) + snapshotSpec(t) + } + + test("int_partition_column") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("p", IntegerType).add("v", StringType), + partitionColumns = Seq("p")) + insertOp(w, Seq( + Map("id" -> 1, "p" -> 10, "v" -> "a"), + Map("id" -> 2, "p" -> 20, "v" -> "b"), + Map("id" -> 3, "p" -> 10, "v" -> "c"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "p = 10", name = Some("read_p10")) + snapshotSpec(t) + } + + test("date_partition_column") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("d", DateType).add("v", IntegerType), + partitionColumns = Seq("d")) + insertOp(w, Seq( + Map("id" -> 1, "d" -> "2021-01-01", "v" -> 1), + Map("id" -> 2, "d" -> "2021-01-01", "v" -> 2), + Map("id" -> 3, "d" -> "2022-12-31", "v" -> 3))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "d = DATE '2021-01-01'", name = Some("read_jan")) + snapshotSpec(t) + } + + test("boolean_partition_column") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("flag", BooleanType).add("v", IntegerType), + partitionColumns = Seq("flag")) + insertOp(w, Seq( + Map("id" -> 1, "flag" -> true, "v" -> 1), + Map("id" -> 2, "flag" -> false, "v" -> 2), + Map("id" -> 3, "flag" -> true, "v" -> 3))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "flag = true", name = Some("read_true")) + snapshotSpec(t) + } + + test("timestamp_partition_column") { + // Partitioning by TIMESTAMP. The declared wall-clock values are interpreted in the session time + // zone on write and read; the round-trip proves that normalization is consistent end to end. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("ts", TimestampType).add("v", IntegerType), + partitionColumns = Seq("ts")) + insertOp(w, Seq( + Map("id" -> 1, "ts" -> "2021-01-01 12:00:00", "v" -> 1), + Map("id" -> 2, "ts" -> "2021-01-01 12:00:00", "v" -> 2), + Map("id" -> 3, "ts" -> "2021-06-15 08:30:00", "v" -> 3))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "ts = TIMESTAMP '2021-01-01 12:00:00'", name = Some("read_jan_noon")) + snapshotSpec(t) + } + + test("timestamp_ntz_partition_column") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("ntz", TimestampNTZType).add("v", IntegerType), + partitionColumns = Seq("ntz")) + insertOp(w, Seq( + Map("id" -> 1, "ntz" -> "2021-01-01 00:00:00", "v" -> 1), + Map("id" -> 2, "ntz" -> "2021-01-01 00:00:00", "v" -> 2), + Map("id" -> 3, "ntz" -> "2022-02-02 02:02:02", "v" -> 3))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("microsecond_precision_timestamp_partition_values") { + // Sub-second precision in the partition value must survive the partitionValues string encoding. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("ts", TimestampType).add("v", IntegerType), + partitionColumns = Seq("ts")) + insertOp(w, Seq( + Map("id" -> 1, "ts" -> "2021-01-01 00:00:00.000001", "v" -> 1), + Map("id" -> 2, "ts" -> "2021-01-01 00:00:00.123456", "v" -> 2), + Map("id" -> 3, "ts" -> "2021-01-01 00:00:00.999999", "v" -> 3))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("special_character_string_partition_value") { + // Partition values containing slashes, spaces, and quotes must be escaped in the partition + // directory path and preserved through the partitionValues encoding. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("label", StringType).add("v", IntegerType), + partitionColumns = Seq("label")) + insertOp(w, Seq( + Map("id" -> 1, "label" -> "a/ b", "v" -> 1), + Map("id" -> 2, "label" -> "quote's", "v" -> 2), + Map("id" -> 3, "label" -> "eq=sign", "v" -> 3))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "label = 'a/ b'", name = Some("read_slash_space")) + snapshotSpec(t) + } + + test("mixed_type_multi_column_partition") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("part_id", IntegerType) + .add("part_status", StringType).add("v", IntegerType), + partitionColumns = Seq("part_id", "part_status")) + insertOp(w, Seq( + Map("id" -> 1, "part_id" -> 1, "part_status" -> "open", "v" -> 10), + Map("id" -> 2, "part_id" -> 1, "part_status" -> "closed", "v" -> 20), + Map("id" -> 3, "part_id" -> 2, "part_status" -> "open", "v" -> 30))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "part_id = 1 AND part_status = 'open'", name = Some("read_open_1")) + snapshotSpec(t) + } + + test("reorder_partition_and_data_columns") { + // Partition columns are declared out of schema order and the row maps list keys in yet another + // order; name-based resolution places every value correctly regardless. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("region", StringType) + .add("year", IntegerType).add("v", IntegerType), + partitionColumns = Seq("year", "region")) + insertOp(w, Seq( + Map("v" -> 1, "region" -> "east", "id" -> 1, "year" -> 2023), + Map("year" -> 2024, "id" -> 2, "v" -> 2, "region" -> "west"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "year = 2023 AND region = 'east'", name = Some("read_2023_east")) + snapshotSpec(t) + } + + test("empty_string_partition_value") { + // A string partition column with an empty-string value alongside a normal value and NULL; the + // empty string must round-trip through the partition-directory / partitionValues encoding. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("p", StringType).add("v", IntegerType), + partitionColumns = Seq("p")) + insertOp(w, Seq( + Map("id" -> 1, "p" -> "", "v" -> 1), + Map("id" -> 2, "p" -> "normal", "v" -> 2), + Map("id" -> 3, "p" -> null, "v" -> 3))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "p = ''", name = Some("read_empty_partition")) + snapshotSpec(t) + } + + test("decimal_partition_column") { + // Partition by DECIMAL(10,2) with negative, zero, and positive values. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("p", DecimalType(10, 2)).add("v", IntegerType), + partitionColumns = Seq("p")) + insertOp(w, Seq( + Map("id" -> 1, "p" -> "-12.34", "v" -> 1), + Map("id" -> 2, "p" -> "0.00", "v" -> 2), + Map("id" -> 3, "p" -> "99.99", "v" -> 3), + Map("id" -> 4, "p" -> "-12.34", "v" -> 4))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "p < 0", name = Some("read_negative_partition")) + snapshotSpec(t) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteResolutionSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteResolutionSuite.scala new file mode 100644 index 00000000..c569b017 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteResolutionSuite.scala @@ -0,0 +1,139 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite + +/** + * Insert-resolution inserts: how values map onto the target schema. The row-value API resolves each + * column by name (not by map iteration order), fills omitted nullable columns with NULL, and picks + * up columns added by schema evolution. + */ +class WriteResolutionSuite extends WorkloadTestSuite("write_resolution") { + + test("all_columns_present") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("name", StringType).add("v", IntegerType)) + insertOp(w, Seq( + Map("id" -> 1, "name" -> "a", "v" -> 10), + Map("id" -> 2, "name" -> "b", "v" -> 20))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "v >= 20", name = Some("read_ge_20")) + snapshotSpec(t) + } + + test("reordered_row_keys_resolve_by_name") { + // Row maps list keys in an order unrelated to the schema; values still land in the right column + // because resolution is by name. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("name", StringType).add("v", IntegerType)) + insertOp(w, Seq( + Map("v" -> 100, "id" -> 1, "name" -> "x"), + Map("name" -> "y", "v" -> 200, "id" -> 2))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "name = 'x'", name = Some("read_x")) + snapshotSpec(t) + } + + test("missing_nullable_column_filled_with_null") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("name", StringType).add("score", IntegerType)) + // Second row omits `score`; the omitted nullable column is written as NULL. + insertOp(w, Seq( + Map("id" -> 1, "name" -> "a", "score" -> 50), + Map("id" -> 2, "name" -> "b"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "score IS NULL", name = Some("read_null_score")) + readSpec(t, predicate = "score IS NOT NULL", name = Some("read_scored")) + snapshotSpec(t) + } + + test("insert_into_evolved_schema_new_column") { + // A new nullable column is added via schema evolution, then a later insert supplies it. Rows + // written before the evolution read back NULL for the new column. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("name", StringType)) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"), Map("id" -> 2, "name" -> "b"))) + addColumnsOp(w, new StructType().add("tag", StringType)) + insertOp(w, Seq(Map("id" -> 3, "name" -> "c", "tag" -> "t3"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, version = 1, name = Some("read_before_evolution")) + readSpec(t, predicate = "tag IS NULL", name = Some("read_null_tag")) + readSpec(t, predicate = "tag = 't3'", name = Some("read_tag3")) + snapshotSpec(t) + } + + test("multi_row_single_insert") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("v", IntegerType)) + insertOp(w, (1 to 8).map(i => Map("id" -> i, "v" -> i * 5))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "v > 20", name = Some("read_big_v")) + snapshotSpec(t) + } + + test("column_with_comment_in_schema") { + // A column carrying a schema COMMENT round-trips through create + insert. + val w = createTableOp("tbl", + schema = new StructType() + .add("id", IntegerType) + .add("note", StringType, nullable = true, "free-form note about the row")) + insertOp(w, Seq( + Map("id" -> 1, "note" -> "first"), + Map("id" -> 2, "note" -> "second"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "note = 'first'", name = Some("read_first_note")) + snapshotSpec(t) + } + + test("char_column") { + // A fixed-width CHAR(n) column: inserted values are space-padded to the declared width, and the + // padded form must round-trip through capture->replay. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("code", "char(5)")) + insertOp(w, Seq( + Map("id" -> 1, "code" -> "ab"), + Map("id" -> 2, "code" -> "abcde"))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("empty_insert_is_rejected") { + // A 0-row insert produces no commit, so the append DSL rejects it. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType)) + val ex = intercept[IllegalArgumentException] { + insertOp(w, Seq.empty) + } + assert(ex.getMessage.contains("at least one row"), s"message: ${ex.getMessage}") + // A real (non-empty) insert follows so the table has a captured commit history. + insertOp(w, Seq(Map("id" -> 1))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteStatsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteStatsSuite.scala new file mode 100644 index 00000000..680b910d --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteStatsSuite.scala @@ -0,0 +1,142 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite + +/** + * Stats-generation inserts: each append drives the writer's per-file statistics (min/max/nullCount/ + * numRecords) down interesting paths -- all-null columns, string min/max prefix truncation, a + * narrowed indexed-column count, and timestamp/decimal ranges. + */ +class WriteStatsSuite extends WorkloadTestSuite("write_stats") { + + test("partitioned_insert_stats_per_file") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("region", StringType).add("amount", IntegerType), + partitionColumns = Seq("region")) + insertOp(w, Seq( + Map("id" -> 1, "region" -> "east", "amount" -> 10), + Map("id" -> 2, "region" -> "east", "amount" -> 40), + Map("id" -> 3, "region" -> "west", "amount" -> 25))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "amount > 20", name = Some("read_over_20")) + snapshotSpec(t) + } + + test("all_null_column") { + // A column that is NULL in every row: nullCount equals numRecords and min/max are null. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("maybe", IntegerType)) + insertOp(w, Seq( + Map("id" -> 1, "maybe" -> null), + Map("id" -> 2, "maybe" -> null), + Map("id" -> 3, "maybe" -> null))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "maybe IS NULL", name = Some("read_all_null")) + snapshotSpec(t) + } + + test("string_min_max_prefix_truncation") { + // Strings longer than the default indexed prefix length share a common prefix and differ only + // past it, so the min/max stats must be prefix-truncated by the writer. + val prefix = "z" * 40 + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("s", StringType)) + insertOp(w, Seq( + Map("id" -> 1, "s" -> (prefix + "aaaa")), + Map("id" -> 2, "s" -> (prefix + "mmmm")), + Map("id" -> 3, "s" -> (prefix + "zzzz")))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("utf8_boundary_string_min_max") { + val head = "a" * 31 + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("s", StringType)) + insertOp(w, Seq( + Map("id" -> 1, "s" -> (head + "étail")), // e-acute: 2 bytes + Map("id" -> 2, "s" -> (head + "中tail")), // CJK char: 3 bytes + Map("id" -> 3, "s" -> (head + "😀")))) // emoji: 4 bytes + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } + + test("narrowed_indexed_columns") { + // With dataSkippingNumIndexedCols narrowed to 3, only the first three columns get stats; the + // remaining columns are present but unindexed. + val w = createTableOp("tbl", + schema = new StructType().add("c1", IntegerType).add("c2", IntegerType).add("c3", IntegerType) + .add("c4", IntegerType).add("c5", StringType), + properties = Map("delta.dataSkippingNumIndexedCols" -> "3")) + insertOp(w, (1 to 5).map(i => + Map("c1" -> i, "c2" -> i * 2, "c3" -> i * 3, "c4" -> i * 4, "c5" -> s"v$i"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "c1 >= 3", name = Some("read_indexed")) + readSpec(t, predicate = "c4 >= 12", name = Some("read_unindexed")) + snapshotSpec(t) + } + + test("timestamp_stats_across_precisions") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("ts", TimestampType)) + insertOp(w, Seq( + Map("id" -> 1, "ts" -> "2021-01-01 00:00:00"), // second precision + Map("id" -> 2, "ts" -> "2021-01-01 00:00:00.123"), // milli precision + Map("id" -> 3, "ts" -> "2021-01-01 00:00:00.123456"))) // micro precision + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "ts > TIMESTAMP '2021-01-01 00:00:00.1'", name = Some("read_after_first")) + snapshotSpec(t) + } + + test("decimal_stats") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("amount", DecimalType(12, 4))) + insertOp(w, Seq( + Map("id" -> 1, "amount" -> "1.0001"), + Map("id" -> 2, "amount" -> "99999999.9999"), + Map("id" -> 3, "amount" -> "-500.5000"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "amount > 0", name = Some("read_positive")) + snapshotSpec(t) + } + + test("timestamp_ntz_and_boundary_subms_stats") { + // Sub-millisecond fractions on both TIMESTAMP_NTZ and TIMESTAMP columns exercise micro-precision + // min/max stats generation. + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType) + .add("ntz", TimestampNTZType).add("ts", TimestampType)) + insertOp(w, Seq( + Map("id" -> 1, "ntz" -> "2024-01-01 00:00:00.000500", "ts" -> "2024-01-01 00:00:00.000500"), + Map("id" -> 2, "ntz" -> "2024-01-01 00:00:00.000001", "ts" -> "2024-01-01 00:00:00.999999"), + Map("id" -> 3, "ntz" -> "2024-01-01 00:00:00.999999", "ts" -> "2024-01-01 00:00:00.000500"))) + val t = endWrite(w) + readSpec(t) + snapshotSpec(t) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala index b4a3cbda..2688f153 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala @@ -22,8 +22,7 @@ import io.delta.workload.WorkloadTestSuite /** * Type-coverage write workloads: exercises the harness's row-value coercion and the schema round-trip across - * every supported scalar type, nulls, and string edge cases. Each test captures then the - * framework replays + re-validates, so a passing test proves the type survives capture->replay. + * every supported scalar type, nulls, and string edge cases. */ class WriteTypesSuite extends WorkloadTestSuite("write_types") { @@ -109,16 +108,6 @@ class WriteTypesSuite extends WorkloadTestSuite("write_types") { snapshotSpec(t) } - test("nested_type_data_is_unsupported") { - // Nested column TYPES round-trip in the schema, but nested DATA via the rows API is not - // supported — the harness's row-value coercion fails loud rather than silently writing a wrong value. - val w = createTableOp("tbl", schema = new StructType().add("id", IntegerType).add("s", new StructType().add("a", IntegerType))) - val ex = intercept[IllegalArgumentException] { - insertOp(w, Seq(Map("id" -> 1, "s" -> Map("a" -> 1)))) - } - assert(ex.getMessage.toLowerCase.contains("unsupported"), s"message: ${ex.getMessage}") - } - test("partitioned_by_typed_column") { val w = createTableOp("tbl", schema = new StructType().add("id", IntegerType).add("d", DateType).add("amount", DecimalType(8, 2)), partitionColumns = Seq("d")) diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteVolumeSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteVolumeSuite.scala new file mode 100644 index 00000000..c5e0d16f --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteVolumeSuite.scala @@ -0,0 +1,115 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite +import io.delta.workload.write.SpecLayout + +/** + * Volume-oriented inserts: many sequential commits, a high-row-count single append, and a + * partition-fanned append that provably produces more than one Add file in a single commit. Row + * counts stay in the thousands to keep the suite fast. + */ +class WriteVolumeSuite extends WorkloadTestSuite("write_volume") { + + test("many_sequential_appends") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("batch", IntegerType)) + // 12 sequential single-row appends -> 12 commits on top of the create. + (1 to 12).foreach(b => insertOp(w, Seq(Map("id" -> b, "batch" -> b)))) + val t = endWrite(w) + readSpec(t) + readSpec(t, version = 1, name = Some("read_after_first_append")) + readSpec(t, version = 12, name = Some("read_after_last_append")) + readSpec(t, predicate = "batch >= 10", name = Some("read_late_batches")) + snapshotSpec(t) + } + + test("high_volume_single_append") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("v", IntegerType).add("s", StringType)) + insertOp(w, (0 until 5000).map(i => Map("id" -> i, "v" -> (i % 100), "s" -> s"row_$i"))) + val t = endWrite(w) + readSpec(t) + readSpec(t, predicate = "v = 50", name = Some("read_v50")) + snapshotSpec(t) + } + + test("multi_file_append_via_partitions") { + // A single insert spanning multiple partition values yields one Add per partition directory, + // giving a deterministic multi-file commit (each partition is written to its own file). + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("p", IntegerType).add("v", IntegerType), + partitionColumns = Seq("p")) + insertOp(w, (0 until 10).map(i => Map("id" -> i, "p" -> (i % 5), "v" -> i))) + val t = endWrite(w) + + // The insert is version 1 (create is version 0); assert it produced more than one Add. + val adds = SpecLayout.addPathsAt(t.sourcePath, 1).size + assert(adds > 1, s"expected a multi-file append (>1 Add) at v1, got $adds") + + readSpec(t) + readSpec(t, predicate = "p = 2", name = Some("read_p2")) + snapshotSpec(t) + } + + test("wide_nested_high_volume_append") { + // A wide schema (40+ columns) mixing scalar and nested types, filled with a few hundred rows in + // one insert, to exercise wide+nested column encoding. + val scalarCount = 36 + var schema = new StructType().add("id", IntegerType) + (0 until scalarCount).foreach { c => + val tpe: DataType = (c % 4) match { + case 0 => IntegerType + case 1 => LongType + case 2 => StringType + case _ => DoubleType + } + schema = schema.add(s"c$c", tpe) + } + schema = schema + .add("s1", new StructType().add("a", IntegerType).add("b", StringType)) + .add("s2", new StructType().add("x", LongType).add("y", DoubleType)) + .add("s3", new StructType().add("flag", BooleanType).add("label", StringType)) + .add("arr", ArrayType(IntegerType)) + .add("m", MapType(StringType, IntegerType)) + val w = createTableOp("tbl", schema = schema) + insertOp(w, (0 until 300).map { i => + val row = Map.newBuilder[String, Any] + row += "id" -> i + (0 until scalarCount).foreach { c => + val v: Any = (c % 4) match { + case 0 => i + c + case 1 => (i + c).toLong + case 2 => s"v${i}_$c" + case _ => (i + c) * 0.5 + } + row += s"c$c" -> v + } + row += "s1" -> Map("a" -> i, "b" -> s"b$i") + row += "s2" -> Map("x" -> i.toLong, "y" -> i * 1.5) + row += "s3" -> Map("flag" -> (i % 2 == 0), "label" -> s"l$i") + row += "arr" -> Seq(i, i + 1, i + 2) + row += "m" -> Map("k1" -> i, "k2" -> i * 2) + row.result() + }) + val t = endWrite(w) + readSpec(t) + } +}