diff --git a/workload-generator/docs/harness-implementation-guide.md b/workload-generator/docs/harness-implementation-guide.md index b8a424b7..b8e7eb39 100644 --- a/workload-generator/docs/harness-implementation-guide.md +++ b/workload-generator/docs/harness-implementation-guide.md @@ -215,6 +215,34 @@ The write spec has no expected-data artifact of its own. Its own check is **basi Because a write spec is replayed (not byte-compared), comparison is **portable**: rows-only data checks, capability-based protocol comparison (your table may use explicit table features where Spark used legacy versions), and column-mapping-normalized schema. A write spec's presence makes its whole directory *write-derived*: replay it into a fresh table once, then validate every read/snapshot spec in that directory (including the baseline `latest` read) against the replay (portably), rather than against a captured `delta/` directory. +### CDF Specs + +```json +{ "type": "cdf", "startVersion": 0, "endVersion": 2, "expected": { "rowCount": 5 } } +``` + +**Execute:** read the change feed over `[startVersion, endVersion]` (or the timestamp bounds). + +**Validate:** compare the change rows against `expected//expected_data/` as a bag of typed rows, exactly like a read spec. `_change_type` and `_commit_version` are compared; `_commit_timestamp` is not (on non-ICT tables it is the commit file's mtime, which does not survive tarring). Assert `rowCount`. An `"error"` outcome is supported like other specs. + +### Checkpoint and CRC Specs + +```json +{ "type": "checkpoint", "version": 2 } +{ "type": "crc", "version": 2 } +``` + +These are **presence triggers**, not content specs. Validate by asserting a `.checkpoint*` (checkpoint) or `.crc` (crc) file exists in `_delta_log/`. Checkpoint contents are validated indirectly by a paired read spec: reading a version whose snapshot reconstructs through the checkpoint returns wrong rows if the checkpoint dropped or duplicated an Add file, so a content-wrong checkpoint fails that read spec. (A snapshot spec covers only protocol/metadata, not the file set, so it does not catch this on its own.) The `.crc` (version-checksum) file is optional per the protocol, so the crc spec is capability-gated: a harness whose engine does not write version checksums skips crc specs, while a harness whose engine does write them must produce a `.crc` at the spec's version. The spec asserts only that the file was written; an engine that verifies checksums on read additionally validates the contents on load, but that is not assumed. + +### Invariant Validators (row tracking, stats) + +Some correctness properties are not per-spec JSON; they are engine-agnostic invariants a write harness checks against the table the engine under test produced. A write harness implements these directly against the written table: + +- **Stats:** per-file timestamp stats are millisecond-truncated; each file's `minValues`/`maxValues` bound its live rows (deletion vectors applied). When `tightBounds` is true the bounds must equal the live min/max; when false they need only contain them. +- **Row tracking (when the `rowTracking` feature is on):** every live Add carries a `baseRowId`; fresh-id ranges never overlap; the `rowIdHighWaterMark` covers the largest assigned id; and every row's `_metadata.row_id` is non-null and unique. + +These run only in a write harness, against the table the engine under test produced. + --- ## Step 4: Incremental Adoption diff --git a/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala b/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala index 5dbb951a..69d2a407 100644 --- a/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala +++ b/workload-generator/src/main/scala/io/delta/workload/SpecDeclarations.scala @@ -59,7 +59,9 @@ class SpecRef[T] private[workload] ( private[workload] trait HasAssertion[T] { var assertion: Option[com.fasterxml.jackson.databind.JsonNode => Unit] = None - def deserialize: com.fasterxml.jackson.databind.JsonNode => T + def specClass: Class[T] + def deserialize: com.fasterxml.jackson.databind.JsonNode => T = + n => JsonUtil.mapper.treeToValue(n, specClass) } // --------------------------------------------------------------------------- @@ -76,6 +78,9 @@ private[workload] class TableDecl( private[workload] def resolveOutputName(name: String): Unit = { _outputName = name } val readSpecs = mutable.ArrayBuffer[ReadSpecConfig]() val snapshotSpecs = mutable.ArrayBuffer[SnapshotSpecConfig]() + val cdfSpecs = mutable.ArrayBuffer[CdfSpecConfig]() + val checkpointSpecs = mutable.ArrayBuffer[CheckpointSpecConfig]() + val crcSpecs = mutable.ArrayBuffer[CrcSpecConfig]() val mutations = mutable.ArrayBuffer[Path => Unit]() var writeBuilder: Option[WriteSpecBuilder] = None } @@ -83,13 +88,29 @@ private[workload] class TableDecl( private[workload] case class ReadSpecConfig( name: String, query: ReadQuery, expectError: ErrorExpectation = AutoDetect) extends HasAssertion[ReadSpec] { - val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => - JsonUtil.mapper.treeToValue(n, classOf[ReadSpec]) + val specClass = classOf[ReadSpec] } private[workload] case class SnapshotSpecConfig( query: SnapshotQuery, expectError: ErrorExpectation = AutoDetect) extends HasAssertion[SnapshotSpec] { - val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => - JsonUtil.mapper.treeToValue(n, classOf[SnapshotSpec]) + val specClass = classOf[SnapshotSpec] +} + +private[workload] case class CdfSpecConfig( + name: String, startVersion: Option[Long], endVersion: Option[Long], + startTimestamp: Option[String], endTimestamp: Option[String], + predicate: Option[String], columns: Option[Seq[String]], + expectError: Option[String] = None) extends HasAssertion[CdfSpec] { + val specClass = classOf[CdfSpec] +} + +private[workload] case class CheckpointSpecConfig( + name: String, version: Long) extends HasAssertion[CheckpointSpec] { + val specClass = classOf[CheckpointSpec] +} + +private[workload] case class CrcSpecConfig( + name: String, version: Long) extends HasAssertion[CrcSpec] { + val specClass = classOf[CrcSpec] } diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala index 46bc3e51..170fae8c 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala @@ -132,6 +132,44 @@ class WorkloadContext private[workload] ( if (parts.length == 1) "read_all" else parts.mkString("_") } + private[workload] def autoCdfName( + startVersion: Option[Long], + endVersion: Option[Long], + startTimestamp: Option[String], + endTimestamp: Option[String], + predicate: Option[String], + columns: Option[Seq[String]]): String = { + val parts = mutable.ArrayBuffer[String]("cdf") + startVersion.foreach(v => parts += s"v$v") + startTimestamp.foreach(ts => parts += "ts_" + ts.replace(":", "-").replace(" ", "_").take(23)) + parts += "to" + endVersion.foreach(v => parts += s"v$v") + endTimestamp.foreach(ts => parts += "ts_" + ts.replace(":", "-").replace(" ", "_").take(23)) + if (endVersion.isEmpty && endTimestamp.isEmpty) parts += "latest" + predicate.foreach(p => parts += simplifyPredicate(p)) + columns.foreach { cols => + parts += "cols_" + cols.take(3).mkString("_") + if (cols.size > 3) parts += s"plus${cols.size - 3}" + } + parts.mkString("_") + } + + /** Turn a SQL predicate into a filesystem-safe spec-name fragment. */ + private def simplifyPredicate(p: String): String = + p.toLowerCase + .replaceAll(">=", "_gte_").replaceAll("<=", "_lte_") + .replaceAll("<>", "_neq_").replaceAll("!=", "_neq_") + .replaceAll(">", "_gt_").replaceAll("<", "_lt_") + .replaceAll("=", "_eq_") + .replaceAll("\\s+and\\s+", "_and_").replaceAll("\\s+or\\s+", "_or_") + .replaceAll("\\s+is\\s+not\\s+null", "_is_not_null") + .replaceAll("\\s+is\\s+null", "_is_null") + .replaceAll("\\s+in\\s+", "_in_") + .replaceAll("\\s+", "_") + .replaceAll("[^a-z0-9_]", "") + .replaceAll("_+", "_").stripPrefix("_").stripSuffix("_") + .take(50) + // ---- Internal ---- private def ensureTableSpec(handle: TableHandle): Unit = { diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 052f92e9..ac2d3748 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -23,7 +23,7 @@ import scala.collection.mutable import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession -import io.delta.workload.capture.{ReadCapture, SnapshotCapture, TableInfoCapture} +import io.delta.workload.capture.{CdfCapture, CheckpointCapture, CrcCapture, ReadCapture, SnapshotCapture, TableInfoCapture} import io.delta.workload.json.JsonUtil import io.delta.workload.model._ import io.delta.workload.validate.WorkloadValidator @@ -97,16 +97,11 @@ object WorkloadGenerator { Seq(SnapshotSpecConfig(SnapshotQuery())) } else ts.snapshotSpecs for (ss <- explicits) { - SnapshotCapture.capture(spark, dirName, destTablePath, specsDir, + val specPath = SnapshotCapture.capture(spark, dirName, destTablePath, specsDir, query = ss.query, expectError = ss.expectError) - val specName = (ss.query.version, ss.query.timestamp) match { - case (Some(v), _) => s"${dirName}_snapshot_v$v" - case (_, Some(ts)) => - s"${dirName}_snapshot_ts_${ts.replace(":", "-").replace(" ", "_")}" - case _ => s"${dirName}_snapshot" - } + val specName = specPath.getFileName.toString.stripSuffix(".json") if (!snapshotNames.contains(specName)) snapshotNames += specName - checkAssertion(ss, specsDir.resolve(s"$specName.json")) + checkAssertion(ss, specPath) } val readNames = mutable.ArrayBuffer[String]() @@ -131,6 +126,33 @@ object WorkloadGenerator { builder.buildSpec(spark, destTablePath, testOutputDir) } + // Running count of all specs generated for this case. + var totalSpecCount = 0 + + for (cs <- ts.cdfSpecs) { + val specPath = CdfCapture.capture(spark, dirName, destTablePath, testOutputDir, specsDir, + name = cs.name, startVersion = cs.startVersion, endVersion = cs.endVersion, + startTimestamp = cs.startTimestamp, endTimestamp = cs.endTimestamp, + predicate = cs.predicate, columns = cs.columns, expectError = cs.expectError) + totalSpecCount += 1 + checkAssertion(cs, specPath) + } + + for (cp <- ts.checkpointSpecs) { + val specPath = CheckpointCapture.capture(spark, dirName, destTablePath, specsDir, + version = cp.version, name = cp.name) + totalSpecCount += 1 + checkAssertion(cp, specPath) + } + + for (crc <- ts.crcSpecs) { + CrcCapture.capture(spark, dirName, destTablePath, specsDir, + version = crc.version, name = crc.name).foreach { p => + totalSpecCount += 1 + checkAssertion(crc, p) + } + } + // table_info.json. An intentionally-mutated table (corruption / deleted-log tests) may be // unreadable and so can't produce it; an unmutated table that fails here is a real bug, so // only a mutated table tolerates the failure. @@ -156,8 +178,8 @@ object WorkloadGenerator { Files.write(reproDir.resolve("generate.scala"), s"// Generated by ${ts.outputName} test\n".getBytes("UTF-8")) - val total = snapshotNames.size + readNames.size - println(s" $dirName: $total specs") + totalSpecCount += snapshotNames.size + readNames.size + println(s" $dirName: $totalSpecCount specs") dirName } diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala index 2b304cd3..76595ad2 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala @@ -103,6 +103,62 @@ trait WorkloadOps { new SpecRef(config) } + /** + * Change Data Feed spec over a version/timestamp range. Requires a start bound + * (`startVersion` or `startTimestamp`). + */ + def cdfSpec( + table: TableHandle, + startVersion: java.lang.Long = null, + endVersion: java.lang.Long = null, + startTimestamp: String = null, + endTimestamp: String = null, + predicate: String = null, + columns: Seq[String] = null, + name: String = null, + expectError: String = null): SpecRef[CdfSpec] = { + val ctx = WorkloadContext.current + val specName = if (name != null) name else ctx.autoCdfName( + ctx.toOption(startVersion), ctx.toOption(endVersion), Option(startTimestamp), + Option(endTimestamp), Option(predicate), Option(columns)) + ctx.requireUnique(table, specName) + val config = CdfSpecConfig( + specName, ctx.toOption(startVersion), ctx.toOption(endVersion), Option(startTimestamp), + Option(endTimestamp), Option(predicate), Option(columns), Option(expectError)) + ctx.getTableSpec(table).cdfSpecs += config + new SpecRef(config) + } + + /** + * Checkpoint spec: a trigger that forces a checkpoint at `version`. + */ + def checkpointSpec( + table: TableHandle, + version: Long, + name: String = null): SpecRef[CheckpointSpec] = { + val ctx = WorkloadContext.current + val specName = if (name != null) name else s"checkpoint_v$version" + ctx.requireUnique(table, specName) + val config = CheckpointSpecConfig(specName, version) + ctx.getTableSpec(table).checkpointSpecs += config + new SpecRef(config) + } + + /** + * CRC (version-checksum) spec: a trigger asserting the engine wrote a `.crc` at `version`. + */ + def crcSpec( + table: TableHandle, + version: Long, + name: String = null): SpecRef[CrcSpec] = { + val ctx = WorkloadContext.current + val specName = if (name != null) name else s"crc_v$version" + ctx.requireUnique(table, specName) + val config = CrcSpecConfig(specName, version) + ctx.getTableSpec(table).crcSpecs += config + new SpecRef(config) + } + /** Force Spark to write a checkpoint file for the given SQL table name. */ def forceCheckpoint(tableName: String): Unit = current.forceCheckpoint(tableName) @@ -350,4 +406,18 @@ trait WorkloadOps { if (Files.exists(CommitLog.commitFile(tableDir, version))) CommitLog.mutate(tableDir, version)(modifier) } + + /** + * Append `actions` to the commit JSON at `version`. + */ + def injectCommitActions(table: TableHandle, version: Long)(actions: Seq[Action]): Unit = + mutateTable(table) { tableDir => + val logDir = tableDir.resolve("_delta_log") + val commitFile = logDir.resolve(f"$version%020d.json") + val content = new String(Files.readAllBytes(commitFile), "UTF-8") + Files.write( + commitFile, + (content.trim + "\n" + actions.map(_.toJson).mkString("\n") + "\n").getBytes("UTF-8")) + Files.deleteIfExists(logDir.resolve(f"$version%020d.crc")) + } } diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala new file mode 100644 index 00000000..8de7a930 --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CdfCapture.scala @@ -0,0 +1,148 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.capture + +import java.nio.file.{Files, Path} + +import scala.util.control.NonFatal + +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession} + +import io.delta.workload.deltaharness.DeltaHarness +import io.delta.workload.engine.{RowComparison, SnapshotResolver, SpecOutcome} +import io.delta.workload.json.JsonUtil +import io.delta.workload.model.{CdfExpected, CdfSpec, Failed, SpecError, SpecExpectation, Succeeded} + +object CdfCapture { + + def capture( + spark: SparkSession, testId: String, tablePath: Path, + outputDir: Path, specsDir: Path, name: String, + startVersion: Option[Long] = None, endVersion: Option[Long] = None, + startTimestamp: Option[String] = None, endTimestamp: Option[String] = None, + predicate: Option[String] = None, columns: Option[Seq[String]] = None, + expectError: Option[String] = None): Path = { + + val specName = s"${testId}_$name" + require(startVersion.isDefined || startTimestamp.isDefined, + s"CDF $specName: a start bound (startVersion or startTimestamp) is required") + require(!(startVersion.isDefined && startTimestamp.isDefined), + s"CDF $specName: cannot specify both startVersion and startTimestamp") + require(!(endVersion.isDefined && endTimestamp.isDefined), + s"CDF $specName: cannot specify both endVersion and endTimestamp") + + val expectedDir = outputDir.resolve("expected").resolve(specName) + Files.createDirectories(expectedDir) + val specPath = specsDir.resolve(s"$specName.json") + + val expectation: SpecExpectation[CdfExpected] = try { + var df = buildReader(spark, tablePath, startVersion, endVersion, + startTimestamp, endTimestamp) + df = SnapshotResolver.applyFilters(df, predicate, columns) + // TODO(cdf-commit-timestamp): validate `_commit_timestamp` once ICT tables are covered; today + // it is dropped because on non-ICT tables it is the commit file's mtime (not reproducible). + df = df.drop("_commit_timestamp") + df = df.cache() + val count = df.count() + + try { + writeExpectedData(expectedDir, df) + Succeeded(CdfExpected(count)) + } finally { + df.unpersist() + } + } catch { + case NonFatal(e) => + Failed(SpecError(SpecOutcome.extractErrorCode(e), Option(e.getMessage).getOrElse(""))) + } + + expectError.foreach { expected => + def fail(msg: String): Nothing = { + if (expectedDir.toFile.exists()) FileUtils.deleteDirectory(expectedDir.toFile) + throw new RuntimeException(msg) + } + expectation match { + case Succeeded(_) => + fail(s"CDF $specName: declared expectError=" + + (if (expected.isEmpty) "(any)" else s"'$expected'") + + " but operation succeeded") + case Failed(err) if expected.nonEmpty && + SpecOutcome.normalizeErrorCode(err.errorCode) != + SpecOutcome.normalizeErrorCode(expected) => + fail(s"CDF $specName: declared expectError='$expected' but got " + + s"'${err.errorCode}'") + case _ => // matches (Failed with matching or any code) + } + } + + val spec = CdfSpec(startVersion, endVersion, startTimestamp, endTimestamp, + predicate, columns, expectation) + JsonUtil.writeSpec(specPath, spec) + validateFromSpec(spark, tablePath, expectedDir, specPath) + + expectation match { + case Succeeded(exp) => + println(s" CDF captured: $specName (${exp.rowCount} change rows)") + case Failed(err) => + println(s" CDF captured (error): $specName [${err.errorCode}] ${err.errorMessage}") + } + specPath + } + + def validateFromSpec(spark: SparkSession, tablePath: Path, + expectedDir: Path, specPath: Path): Unit = { + val spec = JsonUtil.readSpecAs(specPath, classOf[CdfSpec]) + val specName = specPath.getFileName.toString.stripSuffix(".json") + + def reader = SnapshotResolver.applyFilters( + buildReader(spark, tablePath, spec.startVersion, spec.endVersion, + spec.startTimestamp, spec.endTimestamp), + spec.predicate, spec.columns).drop("_commit_timestamp") + + SpecOutcome.compareExpectation(spec.expectation, specName) { + SpecOutcome.runErrorCode { + reader.write.format("noop").mode("overwrite").save() + None + } + } { _ => + val expectedDataPath = expectedDir.resolve("expected_data") + require(Files.exists(expectedDataPath), + s"Validation FAILED for $specName: expected_data is missing") + RowComparison.assertRowsEqual( + spark.read.parquet(expectedDataPath.toString), reader, specName) + } + } + + private def writeExpectedData(expectedDir: Path, df: DataFrame): Unit = { + val path = expectedDir.resolve("expected_data") + if (path.toFile.exists()) FileUtils.deleteDirectory(path.toFile) + df.write.mode(SaveMode.Overwrite).parquet(path.toString) + } + + /** + * Build the change-feed reader, resolving the table's latest version through the + * neutral [[DeltaHarness]] SPI so an open-ended range stays bounded. + */ + private def buildReader(spark: SparkSession, tablePath: Path, + startVersion: Option[Long], endVersion: Option[Long], + startTimestamp: Option[String], endTimestamp: Option[String]): DataFrame = { + val latestVersion = DeltaHarness.get.openLog(spark, tablePath.toString).update().version + SnapshotResolver.buildCdfReader(spark, tablePath, startVersion, endVersion, + startTimestamp, endTimestamp, latestVersion) + } +} diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala new file mode 100644 index 00000000..a3d7fcbf --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CheckpointCapture.scala @@ -0,0 +1,70 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.capture + +import java.nio.file.{Files, Path} + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.SparkSession + +import io.delta.workload.deltaharness.DeltaHarness +import io.delta.workload.json.JsonUtil +import io.delta.workload.model.CheckpointSpec + +/** + * Captures and validates checkpoint specs. A checkpoint spec is a bare trigger: it names a version + * the engine under test must checkpoint. + */ +object CheckpointCapture { + + /** Force a checkpoint at `version` and emit its spec; returns the spec path. */ + def capture( + spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, + version: Long, name: String): Path = { + + val specName = s"${testId}_$name" + val specPath = specsDir.resolve(s"$specName.json") + + DeltaHarness.get.openLog(spark, tablePath.toString).checkpoint(version) + require(checkpointFileExists(tablePath, version), + s"Checkpoint $specName: no checkpoint file present at version $version after checkpoint()") + + JsonUtil.writeSpec(specPath, CheckpointSpec(version)) + println(s" Checkpoint captured: $specName (version=$version)") + specPath + } + + /** Assert a checkpoint file exists at the spec's version. */ + def validateFromSpec( + spark: SparkSession, tablePath: Path, specPath: Path): Unit = { + val spec = JsonUtil.readSpecAs(specPath, classOf[CheckpointSpec]) + val specName = specPath.getFileName.toString.stripSuffix(".json") + require(checkpointFileExists(tablePath, spec.version), + s"Checkpoint validation failed for $specName: no checkpoint file at version ${spec.version}; " + + "the engine under test did not write one") + } + + /** True iff a checkpoint file (`.checkpoint*`) exists in `_delta_log`. */ + private def checkpointFileExists(tablePath: Path, version: Long): Boolean = { + val logDir = tablePath.resolve("_delta_log") + val prefix = f"$version%020d.checkpoint" + val stream = Files.list(logDir) + try stream.iterator().asScala.exists(_.getFileName.toString.startsWith(prefix)) + finally stream.close() + } +} diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala new file mode 100644 index 00000000..e337bca8 --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/capture/CrcCapture.scala @@ -0,0 +1,59 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.capture + +import java.nio.file.{Files, Path} + +import org.apache.spark.sql.SparkSession + +import io.delta.workload.json.JsonUtil +import io.delta.workload.model.CrcSpec + +/** + * Captures and validates crc (version-checksum) specs. A crc spec is a bare trigger: it names a + * version at which the engine under test must have written a `.crc` sidecar. + */ +object CrcCapture { + + /** @return `Some(specPath)` if a `.crc` was present and a spec emitted, `None` if none exists. */ + def capture( + spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, + version: Long, name: String): Option[Path] = { + + val specName = s"${testId}_$name" + require(crcFileExists(tablePath, version), + s"CRC capture failed for $specName (version=$version): no .crc present at that version") + val specPath = specsDir.resolve(s"$specName.json") + JsonUtil.writeSpec(specPath, CrcSpec(version)) + println(s" CRC captured: $specName (version=$version)") + Some(specPath) + } + + /** Assert a `.crc` exists at the spec's version. Its contents are validated by the engine's on-read + * checksum verification when the version is loaded (via the snapshot spec). */ + def validateFromSpec( + spark: SparkSession, tablePath: Path, specPath: Path): Unit = { + val spec = JsonUtil.readSpecAs(specPath, classOf[CrcSpec]) + val specName = specPath.getFileName.toString.stripSuffix(".json") + require(crcFileExists(tablePath, spec.version), + s"CRC validation failed for $specName: no .crc at version ${spec.version}; " + + "the engine under test did not write one") + } + + private def crcFileExists(tablePath: Path, version: Long): Boolean = + Files.exists(tablePath.resolve("_delta_log").resolve(f"$version%020d.crc")) +} diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/ReadCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/ReadCapture.scala index 7082ab7a..2f31e8c4 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/ReadCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/ReadCapture.scala @@ -111,7 +111,7 @@ object ReadCapture { def validateFromSpec(spark: SparkSession, tablePath: Path, expectedDir: Path, specPath: Path, checkMetadata: Boolean = true): Unit = { - val spec = JsonUtil.readReadSpec(specPath) + val spec = JsonUtil.readSpecAs(specPath, classOf[ReadSpec]) val specName = specPath.getFileName.toString.stripSuffix(".json") val harness = DeltaHarness.get diff --git a/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala b/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala index ce700a5b..6d5357dc 100644 --- a/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/capture/SnapshotCapture.scala @@ -31,7 +31,7 @@ object SnapshotCapture { def capture( spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, query: SnapshotQuery = SnapshotQuery(), - expectError: ErrorExpectation = AutoDetect): Unit = { + expectError: ErrorExpectation = AutoDetect): Path = { require(!(query.version.isDefined && query.timestamp.isDefined), "Cannot specify both version and timestamp") @@ -55,9 +55,13 @@ object SnapshotCapture { Failed(SpecError("DELTA_TABLE_NOT_FOUND", s"No valid Delta table at ${tablePath}")) } else { resolvedVersion = Some(snapshot.version) + val txn = snapshot.setTransactions + val dm = snapshot.domainMetadata Succeeded(SnapshotResult( ProtocolInfo.from(snapshot.snapshot.protocol), - MetadataInfo.from(snapshot.snapshot.metadata))) + MetadataInfo.from(snapshot.snapshot.metadata), + setTransactions = if (txn.nonEmpty) Some(txn) else None, + domainMetadata = if (dm.nonEmpty) Some(dm) else None)) } } @@ -74,6 +78,7 @@ object SnapshotCapture { case Failed(err) => println(s" Snapshot captured (error): $specName [${err.errorCode}] ${err.errorMessage}") } + specPath } /** @@ -86,7 +91,7 @@ object SnapshotCapture { */ def validateFromSpec( spark: SparkSession, tablePath: Path, specPath: Path, isWriteValidation: Boolean = false): Unit = { - val spec = JsonUtil.readSnapshotSpec(specPath) + val spec = JsonUtil.readSpecAs(specPath, classOf[SnapshotSpec]) val specName = specPath.getFileName.toString.stripSuffix(".json") val harness = DeltaHarness.get def resolve = @@ -100,16 +105,20 @@ object SnapshotCapture { if (resolve.version < 0) Some("DELTA_TABLE_NOT_FOUND") else None } } { exp => - val live = resolve.snapshot + val resolved = resolve + val live = resolved.snapshot assertMatches(exp.protocol, live.protocol, exp.metadata, live.metadata, isWriteValidation, specName) + requireTxnSubset(specName, exp.setTransactions.getOrElse(Seq.empty), resolved.setTransactions) + requireDomainMetadataEqual(specName, exp.domainMetadata.getOrElse(Seq.empty), + resolved.domainMetadata) } } private val NonReproducibleConfigKeys = Set("delta.columnMapping.maxColumnId") private val ColumnMappingFieldKeys = Seq("delta.columnMapping.id", "delta.columnMapping.physicalName") - private def assertMatches( + private[capture] def assertMatches( expProtocol: ProtocolInfo, actProtocol: Protocol, expMeta: MetadataInfo, actMeta: Metadata, isWriteValidation: Boolean, name: String): Unit = { def requireEq(field: String, exp: Any, act: Any): Unit = @@ -139,6 +148,17 @@ object SnapshotCapture { } } + // Expected app txns must all be present; the engine under test may carry extra writer-owned txns + // (e.g. its own ingestion idempotency txn) the corpus never had, so this is a subset, not equality. + private def requireTxnSubset(name: String, expected: Seq[AppTxn], actual: Seq[AppTxn]): Unit = + require(expected.toSet.subsetOf(actual.toSet), + s"snapshot '$name': setTransactions mismatch (expected all of $expected present, got $actual)") + + private def requireDomainMetadataEqual( + name: String, expected: Seq[AddDomainMetadata], actual: Seq[AddDomainMetadata]): Unit = + require(expected.toSet == actual.toSet, + s"snapshot '$name': domainMetadata mismatch (expected $expected, got $actual)") + private def stripColumnMappingIds(schema: StructType): StructType = stripColumnMapping(schema).asInstanceOf[StructType] diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala index 0203fcb9..851113be 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala @@ -99,6 +99,7 @@ trait LogView { def update(): ResolvedSnapshot def getSnapshotAt(version: Long): ResolvedSnapshot def checkpoint(): Unit + def checkpoint(version: Long): Unit } trait ResolvedSnapshot { @@ -116,6 +117,12 @@ trait ResolvedSnapshot { * ordering or extra fields. */ def allFiles: DataFrame + + /** Active domain-metadata entries (tombstones excluded). */ + def domainMetadata: Seq[AddDomainMetadata] + + /** Active set-transaction entries. */ + def setTransactions: Seq[AppTxn] } // Neutral typed view of a snapshot's protocol + metadata. SPI-side types (no dependency on diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala index da337182..4b26dab7 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.delta.actions.{Action, AddFile, DomainMetadata, File // Hide Spark's types.Metadata so the unqualified `Metadata` is this package's typed case class. import org.apache.spark.sql.types.{Metadata => _, _} +import io.delta.workload.model.{AddDomainMetadata, AppTxn} + class DeltaSparkHarness extends SparkRowMaterializer { override def openLog(spark: SparkSession, tablePath: String): LogView = { DeltaLog.clearCache() @@ -86,6 +88,7 @@ private class DeltaSparkLogView(inner: DeltaLog) extends LogView { override def getSnapshotAt(version: Long): ResolvedSnapshot = new DeltaSparkResolvedSnapshot(inner.getSnapshotAt(version)) override def checkpoint(): Unit = inner.checkpoint() + override def checkpoint(version: Long): Unit = inner.checkpoint(inner.getSnapshotAt(version)) } private class DeltaSparkResolvedSnapshot(inner: DeltaSnapshot) extends ResolvedSnapshot { @@ -119,4 +122,10 @@ private class DeltaSparkResolvedSnapshot(inner: DeltaSnapshot) extends ResolvedS ds.map(f => (f.path, f.size, f.partitionValues, f.json)) .toDF("path", "size", "partitionValues", "json") } + + override def domainMetadata: Seq[AddDomainMetadata] = + inner.domainMetadata.toSeq.map(dm => AddDomainMetadata(dm.domain, dm.configuration)) + + override def setTransactions: Seq[AppTxn] = + inner.setTransactions.toSeq.map(t => AppTxn(t.appId, t.version)) } diff --git a/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala b/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala index ca410fa0..d1f8cf2e 100644 --- a/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala +++ b/workload-generator/src/main/scala/io/delta/workload/engine/SnapshotResolver.scala @@ -45,6 +45,21 @@ object SnapshotResolver { reader.load(tablePath.toString) } + def buildCdfReader(spark: SparkSession, tablePath: Path, + startVersion: Option[Long], endVersion: Option[Long], + startTimestamp: Option[String], endTimestamp: Option[String], + latestVersion: Long): DataFrame = { + var reader = spark.read.format("delta").option("readChangeFeed", "true") + startVersion.foreach(v => reader = reader.option("startingVersion", v)) + startTimestamp.foreach(ts => reader = reader.option("startingTimestamp", ts)) + endVersion.foreach(v => reader = reader.option("endingVersion", v)) + endTimestamp.foreach(ts => reader = reader.option("endingTimestamp", ts)) + if (endVersion.isEmpty && endTimestamp.isEmpty) { + reader = reader.option("endingVersion", latestVersion) + } + reader.load(tablePath.toString) + } + def applyFilters(df: DataFrame, predicate: Option[String], columns: Option[Seq[String]]): DataFrame = { var result = df diff --git a/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala b/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala index e288ec40..584dd2f9 100644 --- a/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala +++ b/workload-generator/src/main/scala/io/delta/workload/json/JsonUtil.scala @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.apache.spark.sql.types.{DataType, StructType} -import io.delta.workload.model.{Failed, ReadQuery, ReadResult, ReadSpec, SnapshotQuery, SnapshotResult, SnapshotSpec, Spec, SpecError, SpecExpectation, Succeeded} +import io.delta.workload.model._ /** * Flattens the outcome to top-level `expected` (success) XOR `error` (failure). `@JsonUnwrapped` @@ -106,6 +106,30 @@ private[workload] class SnapshotSpecDeserializer extends SpecDeserializer[Snapsh SnapshotQuery(SpecExpectationJson.optLong(n, "version"), SpecExpectationJson.optText(n, "timestamp")), SpecExpectationJson.read(n, classOf[SnapshotResult])) } +private[workload] class CdfSpecSerializer extends SpecSerializer[CdfSpec] { + override def serialize(s: CdfSpec, g: JsonGenerator, p: SerializerProvider): Unit = + SpecExpectationJson.writeEnvelope(g, s.`type`, version = None, timestamp = None, s.expectation) { + s.startVersion.foreach(v => g.writeNumberField("startVersion", v)) + s.startTimestamp.foreach(t => g.writeStringField("startTimestamp", t)) + s.endVersion.foreach(v => g.writeNumberField("endVersion", v)) + s.endTimestamp.foreach(t => g.writeStringField("endTimestamp", t)) + s.predicate.foreach(pr => g.writeStringField("predicate", pr)) + s.columns.foreach { cols => + g.writeArrayFieldStart("columns"); cols.foreach(g.writeString); g.writeEndArray() + } + } +} +private[workload] class CdfSpecDeserializer extends SpecDeserializer[CdfSpec] { + override protected def fromNode(n: JsonNode): CdfSpec = + CdfSpec( + startVersion = SpecExpectationJson.optLong(n, "startVersion"), + endVersion = SpecExpectationJson.optLong(n, "endVersion"), + startTimestamp = SpecExpectationJson.optText(n, "startTimestamp"), + endTimestamp = SpecExpectationJson.optText(n, "endTimestamp"), + predicate = SpecExpectationJson.optText(n, "predicate"), + columns = SpecExpectationJson.optStrings(n, "columns"), + expectation = SpecExpectationJson.read(n, classOf[CdfExpected])) +} // ============================================================================= // JSON and DataFrame utilities @@ -143,14 +167,11 @@ object JsonUtil { Files.write(path, prettyWriter.writeValueAsBytes(spec)) - def readReadSpec(path: Path): ReadSpec = - mapper.readValue(Files.readAllBytes(path), classOf[ReadSpec]) - - def readSnapshotSpec(path: Path): SnapshotSpec = - mapper.readValue(Files.readAllBytes(path), classOf[SnapshotSpec]) + /** Deserialize a spec file as the given concrete type; call sites pass `classOf[...]`. */ + def readSpecAs[T](path: Path, cls: Class[T]): T = + mapper.readValue(Files.readAllBytes(path), cls) /** Deserialize any spec by its `type` tag into the sealed [[Spec]]. */ - def readSpec(path: Path): Spec = - mapper.readValue(Files.readAllBytes(path), classOf[Spec]) + def readSpec(path: Path): Spec = readSpecAs(path, classOf[Spec]) } diff --git a/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala b/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala index 1021e8e7..97457fe3 100644 --- a/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala +++ b/workload-generator/src/main/scala/io/delta/workload/model/SpecModel.scala @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation._ import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import io.delta.workload.deltaharness.{Format, Metadata} -import io.delta.workload.json.{ReadSpecDeserializer, ReadSpecSerializer, SnapshotSpecDeserializer, SnapshotSpecSerializer} +import io.delta.workload.json.{CdfSpecDeserializer, CdfSpecSerializer, ReadSpecDeserializer, ReadSpecSerializer, SnapshotSpecDeserializer, SnapshotSpecSerializer} // ============================================================================= // Spec Expected types: success data or error info @@ -57,9 +57,19 @@ object MetadataInfo { m.configuration, m.createdTime) } -/** Snapshot success outcome. */ -@JsonPropertyOrder(Array("protocol", "metadata")) -case class SnapshotResult(protocol: ProtocolInfo, metadata: MetadataInfo) +/** + * Snapshot success outcome: the reconstructed logical state at a version. + */ +@JsonPropertyOrder(Array("protocol", "metadata", "setTransactions", "domainMetadata")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class SnapshotResult( + protocol: ProtocolInfo, + metadata: MetadataInfo, + setTransactions: Option[Seq[AppTxn]] = None, + domainMetadata: Option[Seq[AddDomainMetadata]] = None) + +/** Change Data Feed success data. */ +case class CdfExpected(rowCount: Long) // ============================================================================= // Shared read/snapshot query types @@ -97,7 +107,10 @@ case class SnapshotQuery( @JsonSubTypes(Array( new JsonSubTypes.Type(value = classOf[ReadSpec], name = "read"), new JsonSubTypes.Type(value = classOf[SnapshotSpec], name = "snapshot"), - new JsonSubTypes.Type(value = classOf[WriteSpec], name = "write"))) + new JsonSubTypes.Type(value = classOf[WriteSpec], name = "write"), + new JsonSubTypes.Type(value = classOf[CdfSpec], name = "cdf"), + new JsonSubTypes.Type(value = classOf[CheckpointSpec], name = "checkpoint"), + new JsonSubTypes.Type(value = classOf[CrcSpec], name = "crc"))) sealed trait Spec { def `type`: String } @JsonSerialize(using = classOf[ReadSpecSerializer]) @@ -121,6 +134,29 @@ case class WriteSpec(commits: Seq[WriteCommit]) extends Spec { val `type`: String = "write" } +@JsonSerialize(using = classOf[CdfSpecSerializer]) +@JsonDeserialize(using = classOf[CdfSpecDeserializer]) +case class CdfSpec( + startVersion: Option[Long] = None, + endVersion: Option[Long] = None, + startTimestamp: Option[String] = None, + endTimestamp: Option[String] = None, + predicate: Option[String] = None, + columns: Option[Seq[String]] = None, + expectation: SpecExpectation[CdfExpected]) extends Spec { + val `type`: String = "cdf" +} + +@JsonPropertyOrder(Array("type", "version")) +case class CheckpointSpec(version: Long) extends Spec { + val `type`: String = "checkpoint" +} + +@JsonPropertyOrder(Array("type", "version")) +case class CrcSpec(version: Long) extends Spec { + val `type`: String = "crc" +} + /** The spec outcome: `Succeeded { expected } | Failed { error }`. */ sealed trait SpecExpectation[+R] case class Succeeded[+R](result: R) extends SpecExpectation[R] diff --git a/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala b/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala index b1dc6539..9a6dc863 100644 --- a/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala +++ b/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala @@ -53,7 +53,8 @@ case class AddFileInput(rows: Seq[Map[String, Any]]) */ case class CommitOrdinal(value: Int) -// AppTxn/AddDomainMetadata are the minimal spec DTOs: only what a write spec declares and +// AppTxn/AddDomainMetadata are the minimal spec DTOs, shared as BOTH write-spec input AND captured +// output (the checkpoint/crc `expected` payloads reuse them): only what a write spec declares and // serializes. They form their own layer, separate from io.delta.workload.log.{Txn,DomainMetadata} // (log-fidelity parse types carrying lastUpdated/removed) and Spark's SetTransaction/DomainMetadata // (the engine boundary). diff --git a/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala index 94b73576..1bfe8a4f 100644 --- a/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/validate/WorkloadValidator.scala @@ -28,7 +28,8 @@ import scala.util.control.NonFatal import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession -import io.delta.workload.capture.{ReadCapture, SnapshotCapture} +import io.delta.workload.capture.{CdfCapture, CheckpointCapture, CrcCapture, ReadCapture, SnapshotCapture} +import io.delta.workload.deltaharness.DeltaHarness import io.delta.workload.json.JsonUtil import io.delta.workload.model._ import io.delta.workload.write.WriteReplay @@ -51,10 +52,12 @@ case class ValidationResult(outcomes: Seq[SpecOutcome]) { * and validates each captured spec under `specs/`, dispatching on the `type` field: * * - `write`: replay its `commits` into a fresh table, compare rows to `expected//`. - * - `read` / `snapshot`: the captured-vs-replayed target is chosen once per directory. If the - * directory has a write spec, every read/snapshot spec validates against the table replayed - * from it (a read by rows only; a snapshot leniently, since a replay mints fresh per-table - * ids). Otherwise it is read-only: validate against the captured `delta/` table exactly. + * - `read` / `snapshot` / `cdf`: the captured-vs-replayed target is chosen once per directory. If + * the directory has a write spec, every such spec validates against the table replayed from it + * (a read/cdf by rows only; a snapshot leniently, since a replay mints fresh per-table ids). + * Otherwise it is read-only: validate against the captured `delta/` table exactly. + * - `checkpoint` / `crc`: assert the sidecar file exists at the spec's version; contents are + * covered by a paired snapshot/read spec. * * Used both as the presubmit acceptance entry point ([[validateAll]]) and by the generator to * self-check a freshly generated test dir ([[validateTestDir]]). @@ -111,6 +114,12 @@ object WorkloadValidator { val writeDerived = writeSpecFile.isDefined def queryTarget: Path = writeSpecFile.map(wf => replayOf(wf.getFileName.toString)).getOrElse(deltaTable) + if (writeDerived) { + parsed.collect { case (_, s: CheckpointSpec) => s.version }.distinct.foreach { v => + DeltaHarness.get.openLog(spark, queryTarget.toString).checkpoint(v) + } + } + // Validate one spec to a single outcome. Read-only specs (no write spec in the dir) validate // against the captured `delta/` table exactly; its absence means the corpus is incomplete. def validateOne(specFile: Path, spec: Spec): SpecOutcome = { @@ -121,9 +130,9 @@ object WorkloadValidator { else SpecFailed(id, s"read-only spec but no captured table at $deltaTable") try spec match { case _: WriteSpec => - // Basic validation: triggering the replay (queryTarget) reconstructs the table and - // asserts replay succeeds + finalVersion == commits.size-1. Row content is checked by - // the baseline 'latest' read spec; per-version metadata by the snapshot spec. + // Triggering the replay reconstructs the table and asserts replay succeeds + + // finalVersion == commits.size-1. Rows are checked by the baseline read spec and + // per-version metadata by the snapshot spec. val _ = queryTarget; SpecPassed(id) case _: ReadSpec => val expectedDir = testDir.resolve("expected").resolve(bare) @@ -135,13 +144,72 @@ object WorkloadValidator { if (writeDerived) { SnapshotCapture.validateFromSpec(spark, queryTarget, specFile, isWriteValidation = true); SpecPassed(id) } else readOnly(t => SnapshotCapture.validateFromSpec(spark, t, specFile)) + case _: CdfSpec => + val expectedDir = testDir.resolve("expected").resolve(bare) + if (writeDerived) { + CdfCapture.validateFromSpec(spark, queryTarget, expectedDir, specFile); SpecPassed(id) + } else readOnly(t => CdfCapture.validateFromSpec(spark, t, expectedDir, specFile)) + case _: CheckpointSpec => + if (writeDerived) { + CheckpointCapture.validateFromSpec(spark, queryTarget, specFile) + SpecPassed(id) + } else readOnly(t => CheckpointCapture.validateFromSpec(spark, t, specFile)) + case _: CrcSpec => + if (writeDerived) { + CrcCapture.validateFromSpec(spark, queryTarget, specFile) + SpecPassed(id) + } else readOnly(t => CrcCapture.validateFromSpec(spark, t, specFile)) } catch { case e: Throwable => SpecFailed(id, e.toString) } } - try outcomes ++= parsed.map { case (f, spec) => validateOne(f, spec) } - finally tempDirs.foreach(td => try FileUtils.deleteDirectory(td.toFile) catch { case NonFatal(_) => }) + try { + outcomes ++= parsed.map { case (f, spec) => validateOne(f, spec) } + } finally tempDirs.foreach(td => try FileUtils.deleteDirectory(td.toFile) catch { case NonFatal(_) => }) + ValidationResult(outcomes.toSeq) + } + + /** + * Validate the metadata specs under `specsDir` against an already-written table at `tablePath`. + */ + def validateAgainstTable(spark: SparkSession, tablePath: Path, specsDir: Path): ValidationResult = { + val outcomes = mutable.ArrayBuffer[SpecOutcome]() + // A table that fails to open is the most severe engine failure; surface it as a failure rather + // than swallowing it into a sentinel that makes every spec skip and the result falsely pass. + val latest = + try DeltaHarness.get.openLog(spark, tablePath.toString).update().version + catch { + case NonFatal(e) => + return ValidationResult(Seq(SpecFailed("table", s"failed to open table at $tablePath: $e"))) + } + listSpecs(specsDir).foreach { specFile => + val id = specFile.getFileName.toString.stripSuffix(".json") + def underWrite(v: Long): SpecOutcome = + SpecFailed( + id, + s"spec version $v > table latest $latest: engine under test never reproduced that " + + s"version (silent under-write)" + ) + try { + JsonUtil.readSpec(specFile) match { + case s: CheckpointSpec if s.version > latest => outcomes += underWrite(s.version) + case s: CrcSpec if s.version > latest => outcomes += underWrite(s.version) + case s: SnapshotSpec if s.query.version.exists(_ > latest) => + outcomes += underWrite(s.query.version.get) + case _: CheckpointSpec => + CheckpointCapture.validateFromSpec(spark, tablePath, specFile) + outcomes += SpecPassed(id) + case _: CrcSpec => + CrcCapture.validateFromSpec(spark, tablePath, specFile) + outcomes += SpecPassed(id) + case _: SnapshotSpec => + SnapshotCapture.validateFromSpec(spark, tablePath, specFile, isWriteValidation = true) + outcomes += SpecPassed(id) + case _ => // read / cdf / write -- rows are validated by the harness's read-back + } + } catch { case NonFatal(e) => outcomes += SpecFailed(id, e.toString) } + } ValidationResult(outcomes.toSeq) } diff --git a/workload-generator/src/test/scala/io/delta/workload/SpecSerdeSuite.scala b/workload-generator/src/test/scala/io/delta/workload/SpecSerdeSuite.scala index 4bddb603..9b1de399 100644 --- a/workload-generator/src/test/scala/io/delta/workload/SpecSerdeSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/SpecSerdeSuite.scala @@ -70,6 +70,13 @@ class SpecSerdeSuite extends AnyFunSuite { assert(mapper.readValue(str(spec), classOf[SnapshotSpec]) == spec) } + test("CdfSpec: failure writes `error`, not `expected`, and round-trips") { + val spec = CdfSpec(startVersion = Some(0), expectation = Failed(SpecError("DELTA_CDF_ERROR", "no"))) + val n = tree(spec) + assert(n.has("error") && !n.has("expected")) + assert(mapper.readValue(str(spec), classOf[CdfSpec]) == spec) + } + test("polymorphic dispatch: readValue[Spec] picks the subtype from `type`") { val read: Spec = ReadSpec(ReadQuery(), Succeeded(ReadResult(0, 0, 0))) val snap: Spec = SnapshotSpec(SnapshotQuery(), Failed(SpecError("E", "m"))) diff --git a/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala b/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala index 39e02cca..b85a54d7 100644 --- a/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala @@ -318,7 +318,7 @@ class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor readSpec(t, version = 999) } assertPassed(results) - val spec = JsonUtil.readReadSpec(specs("t_e1").resolve("t_e1_read_v999.json")) + val spec = JsonUtil.readSpecAs(specs("t_e1").resolve("t_e1_read_v999.json"), classOf[ReadSpec]) assert(spec.query.version.contains(999L)) spec.expectation match { case Failed(err) => @@ -720,7 +720,7 @@ class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor readSpec(t) // should produce error spec } assertPassed(results) - val spec = JsonUtil.readReadSpec(specs("t_corrupt").resolve("t_corrupt_read_all.json")) + val spec = JsonUtil.readSpecAs(specs("t_corrupt").resolve("t_corrupt_read_all.json"), classOf[ReadSpec]) spec.expectation match { case Failed(err) => // Error can be FileNotFoundException, SparkException wrapping it, or other file-related errors diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala new file mode 100644 index 00000000..d38ded10 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CdcSuite.scala @@ -0,0 +1,108 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import io.delta.workload.WorkloadTestSuite + +/** + * Change Data Feed workloads. Each test builds a CDF-enabled table across several + * versions, then declares `cdfSpec`s over version ranges. The captured change rows + * carry the `_change_type` and `_commit_version` columns. + */ +class CdcSuite extends WorkloadTestSuite("cdc") { + + test("inserts") { + sql("""CREATE TABLE tbl (id LONG, name STRING) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,'alice'),(2,'bob')") + sql("INSERT INTO tbl VALUES (3,'charlie'),(4,'dave')") + sql("INSERT INTO tbl VALUES (5,'eve'),(6,'frank')") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 3) + cdfSpec(t, startVersion = 1, endVersion = 1) + snapshotSpec(t) + } + + test("updates") { + sql("""CREATE TABLE tbl (id LONG, value INT) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,100),(2,200),(3,300)") + sql("UPDATE tbl SET value = value * 2 WHERE id = 1") + sql("UPDATE tbl SET value = value + 50 WHERE id IN (2, 3)") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 3) + cdfSpec(t, startVersion = 1, endVersion = 3) + snapshotSpec(t) + } + + test("deletes") { + sql("""CREATE TABLE tbl (id LONG, name STRING) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')") + sql("DELETE FROM tbl WHERE id = 3") + sql("DELETE FROM tbl WHERE id IN (1, 5)") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 3) + cdfSpec(t, startVersion = 1, endVersion = 3) + snapshotSpec(t) + } + + test("mixed_dml") { + sql("""CREATE TABLE tbl (id LONG, value INT) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,100),(2,200),(3,300)") + sql("UPDATE tbl SET value = 999 WHERE id = 2") + sql("DELETE FROM tbl WHERE id = 1") + sql("INSERT INTO tbl VALUES (4,400)") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 4) + cdfSpec(t, startVersion = 1, endVersion = 2) + snapshotSpec(t) + } + + test("predicate_filtered") { + sql("""CREATE TABLE tbl (id LONG, value INT) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl VALUES (1,100),(2,200),(3,300),(4,400)") + sql("UPDATE tbl SET value = value + 1 WHERE id <= 2") + sql("DELETE FROM tbl WHERE id = 4") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 0, endVersion = 3) + cdfSpec(t, startVersion = 0, endVersion = 3, predicate = "value > 200") + cdfSpec(t, startVersion = 0, endVersion = 3, predicate = "_change_type = 'delete'") + snapshotSpec(t) + } + + test("open_ended_to_latest") { + sql("""CREATE TABLE tbl (id LONG) USING delta + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true')""") + sql("INSERT INTO tbl SELECT id FROM range(10)") + sql("INSERT INTO tbl SELECT id + 10 FROM range(10)") + sql("INSERT INTO tbl SELECT id + 20 FROM range(10)") + val t = registerTable("tbl") + readSpec(t) + cdfSpec(t, startVersion = 1) + cdfSpec(t, startVersion = 0, endVersion = 3) + snapshotSpec(t) + } + +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala index b96e8df1..1aee2d7e 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CheckpointsSuite.scala @@ -16,14 +16,18 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import io.delta.workload.{TableHandle, WorkloadTestSuite} +import io.delta.workload.log.{Action, DomainMetadata, Txn} import io.delta.workload.model.ErrorCode class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { private def checkpoint(name: String): Unit = forceCheckpoint(name) - // Existing 5 workloads + private def injectActions(t: TableHandle, version: Int, actions: Seq[Action]): Unit = + injectCommitActions(t, version.toLong)(actions) + + // === Read/snapshot workloads over tables that have a checkpoint === test("classic") { sql("CREATE TABLE tbl (id INT) USING delta") @@ -546,4 +550,70 @@ class CheckpointsSuite extends WorkloadTestSuite("checkpoints") { snapshotSpec(t, expectError = ErrorCode("DELTA_TABLE_NOT_FOUND")) } + // === checkpointSpec workloads: force a checkpoint and assert the file exists (contents are + // covered by the paired snapshot spec) === + + test("cp_basic") { + sql("CREATE TABLE tbl (id INT, name STRING) USING delta") + sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c')") + sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") + val t = registerTable("tbl") + checkpointSpec(t, version = 2) + readSpec(t) + snapshotSpec(t) + } + + test("cp_v2") { + sql("""CREATE TABLE tbl (id INT, name STRING) USING delta + TBLPROPERTIES ('delta.checkpointPolicy' = 'v2', + 'delta.enableDeletionVectors' = 'trque')""") + sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c')") + sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") + val t = registerTable("tbl") + checkpointSpec(t, version = 2) + readSpec(t) + snapshotSpec(t) + } + + test("cp_partitioned") { + sql("CREATE TABLE tbl (id INT, part INT) USING delta PARTITIONED BY (part)") + sql("INSERT INTO tbl VALUES (1,0),(2,1),(3,0)") + sql("INSERT INTO tbl VALUES (4,1),(5,0)") + val t = registerTable("tbl") + checkpointSpec(t, version = 2) + readSpec(t) + } + + test("cp_after_many_commits") { + sql("CREATE TABLE tbl (id INT) USING delta") + for (i <- 1 to 6) sql(s"INSERT INTO tbl VALUES ($i)") + val t = registerTable("tbl") + checkpointSpec(t, version = 6) + readSpec(t) + snapshotSpec(t) + } + + test("cp_with_set_transaction") { + sql("CREATE TABLE tbl (id INT) USING delta") + sql("INSERT INTO tbl VALUES (1)") + sql("INSERT INTO tbl VALUES (2)") + val t = registerTable("tbl") + injectActions(t, 2, Seq(Txn("cp-app", version = 42, lastUpdated = Some(1)))) + checkpointSpec(t, version = 2) + snapshotSpec(t) + readSpec(t) + } + + test("cp_with_domain_metadata") { + sql("""CREATE TABLE tbl (id INT) USING delta + TBLPROPERTIES ('delta.enableDeletionVectors' = 'true', + 'delta.feature.domainMetadata' = 'supported')""") + sql("INSERT INTO tbl VALUES (1),(2),(3)") + sql("DELETE FROM tbl WHERE id = 2") + val t = registerTable("tbl") + injectActions(t, 2, Seq(DomainMetadata("cpDomain", configuration = "{\"k\":\"v\"}"))) + checkpointSpec(t, version = 2) + snapshotSpec(t) + readSpec(t) + } } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala new file mode 100644 index 00000000..426a2c05 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CrcSuite.scala @@ -0,0 +1,82 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import io.delta.workload.WorkloadTestSuite + +/** + * Version-checksum coverage. `crcSpec` asserts a `.crc` was written; the paired + * `snapshotSpec` at that version drives on-read checksum verification of its contents. + */ +class CrcSuite extends WorkloadTestSuite("crc") { + + test("crc_basic") { + sql("CREATE TABLE tbl (id INT, name STRING) USING delta") + sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c')") + sql("INSERT INTO tbl VALUES (4,'d'),(5,'e')") + val t = registerTable("tbl") + crcSpec(t, version = 1) + snapshotSpec(t, version = 1L) + } + + test("crc_partitioned") { + sql("CREATE TABLE tbl (id INT, part INT) USING delta PARTITIONED BY (part)") + sql("INSERT INTO tbl VALUES (1,0),(2,1),(3,0)") + sql("INSERT INTO tbl VALUES (4,1)") + val t = registerTable("tbl") + crcSpec(t, version = 1) + snapshotSpec(t, version = 1L) + } + + test("crc_after_delete") { + sql("CREATE TABLE tbl (id INT) USING delta") + sql("INSERT INTO tbl VALUES (1),(2),(3)") + sql("INSERT INTO tbl VALUES (4),(5)") + sql("DELETE FROM tbl WHERE id = 2") + sql("INSERT INTO tbl VALUES (6)") + val t = registerTable("tbl") + // v3 is the delete commit; the trailing insert (v4) makes it non-latest. + crcSpec(t, version = 3) + snapshotSpec(t, version = 3L) + } + + test("crc_with_deletion_vectors") { + sql("""CREATE TABLE tbl (id INT) USING delta + TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')""") + sql("INSERT INTO tbl VALUES (1),(2),(3),(4),(5)") + sql("DELETE FROM tbl WHERE id IN (2, 4)") + sql("INSERT INTO tbl VALUES (6)") + val t = registerTable("tbl") + // v2 is the deletion-vector commit; the trailing insert (v3) makes it non-latest. + crcSpec(t, version = 2) + snapshotSpec(t, version = 2L) + } + + test("crc_multiple_versions") { + sql("CREATE TABLE tbl (id INT) USING delta") + sql("INSERT INTO tbl VALUES (1)") + sql("INSERT INTO tbl VALUES (2)") + sql("INSERT INTO tbl VALUES (3)") + sql("INSERT INTO tbl VALUES (4)") + val t = registerTable("tbl") + // Both targets are non-latest (table ends at v4). + crcSpec(t, version = 1, name = "crc_v1") + crcSpec(t, version = 3, name = "crc_v3") + snapshotSpec(t, version = 1L) + snapshotSpec(t, version = 3L) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala index 7178f12c..c18ac7d8 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala @@ -63,6 +63,23 @@ class WriteCommitSuite extends WorkloadTestSuite("write_commit") { snapshotSpec(t) } + // Checkpoint / CRC / CDF specs on a write-derived table: the generator validates these against + // the table REPLAYED by the engine-under-test. + test("checkpoint_crc_cdf_write_derived") { + val w = createTableOp("tbl", + schema = new StructType().add("id", IntegerType).add("name", StringType), + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"), Map("id" -> 2, "name" -> "bob"))) // v1 + insertOp(w, Seq(Map("id" -> 3, "name" -> "carol"))) // v2 + val t = endWrite(w) + + readSpec(t) + snapshotSpec(t) + checkpointSpec(t, version = 2) + crcSpec(t, version = 2) + cdfSpec(t, startVersion = 0L) + } + test("commit_with_domain_metadata") { val w = createTableOp("tbl", schema = new StructType().add("id", IntegerType).add("name", StringType), diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala new file mode 100644 index 00000000..58f62b4c --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteMetadataSpecsSuite.scala @@ -0,0 +1,100 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import org.apache.spark.sql.types._ + +import io.delta.workload.WorkloadTestSuite +import io.delta.workload.model.{AddDomainMetadata, AppTxn} + +/** + * Write-derived checkpoint / CRC / CDF coverage. Each case has a write spec, so the generator + * validates these specs against the table REPLAYED by the engine-under-test. + */ +class WriteMetadataSpecsSuite extends WorkloadTestSuite("write_meta") { + + private def idName = new StructType().add("id", IntegerType).add("name", StringType) + + test("checkpoint_partitioned") { + val schema = new StructType().add("id", IntegerType).add("part", IntegerType) + val w = createTableOp("tbl", schema, partitionColumns = Seq("part")) + insertOp(w, Seq(Map("id" -> 1, "part" -> 0), Map("id" -> 2, "part" -> 1))) // v1 + insertOp(w, Seq(Map("id" -> 3, "part" -> 0))) // v2 + val t = endWrite(w) + checkpointSpec(t, version = 2) + } + + test("checkpoint_with_set_transaction") { + val w = createTableOp("tbl", idName) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"))) // v1 + commitOp(w, txn = Some(AppTxn("cp-app", 42L))) // v2 + val t = endWrite(w) + checkpointSpec(t, version = 2) + snapshotSpec(t) + } + + test("checkpoint_with_domain_metadata") { + val w = createTableOp("tbl", idName, + properties = Map("delta.feature.domainMetadata" -> "supported")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"))) // v1 + commitOp(w, addDomainMetadata = Some(Seq(AddDomainMetadata("cpDomain", "{\"k\":\"v\"}")))) // v2 + val t = endWrite(w) + checkpointSpec(t, version = 2) + snapshotSpec(t) + } + + test("crc_with_deletion_vectors") { + val w = createTableOp("tbl", idName, + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"), Map("id" -> 2, "name" -> "b"), + Map("id" -> 3, "name" -> "c"))) // v1 + deleteOp(w, "id = 2") // v2 -> deletion vector + val t = endWrite(w) + crcSpec(t, version = 2) + snapshotSpec(t) + } + + test("crc_with_set_transaction") { + val w = createTableOp("tbl", idName) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"))) // v1 + commitOp(w, txn = Some(AppTxn("crc-app", 7L))) // v2 + val t = endWrite(w) + crcSpec(t, version = 2) + snapshotSpec(t) + } + + test("cdf_updates_and_deletes") { + val w = createTableOp("tbl", idName, + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"), Map("id" -> 2, "name" -> "b"), + Map("id" -> 3, "name" -> "c"))) // v1 + updateOp(w, "id = 1", Map("name" -> "'a2'")) // v2 -> update pre/post image change rows + deleteOp(w, "id = 3") // v3 -> delete change rows + val t = endWrite(w) + cdfSpec(t, startVersion = 0L) + } + + test("cdf_version_range") { + val w = createTableOp("tbl", idName, + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a"))) // v1 + insertOp(w, Seq(Map("id" -> 2, "name" -> "b"))) // v2 + insertOp(w, Seq(Map("id" -> 3, "name" -> "c"))) // v3 + val t = endWrite(w) + cdfSpec(t, startVersion = 1L, endVersion = 2L, name = "cdf_v1_to_v2") + } +}