From 284838df719ef739a72fde602dbe67648e643777 Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Mon, 15 Jun 2026 19:24:09 +0000 Subject: [PATCH 01/25] feat: add write workload generator and spec This PR adds a write spec, which allows DAT workloads to test Delta writers. Any Delta writer can validate their own write APIs through the existing read/snapshot specs. --- workload-generator/docs/spec-reference.md | 275 +++++++++++ .../scala/io/delta/workload/JsonUtil.scala | 71 ++- .../io/delta/workload/LowLevelCommit.scala | 122 +++++ .../io/delta/workload/WorkloadGenerator.scala | 241 +++++++++ .../io/delta/workload/WriteSpecCapture.scala | 238 +++++++++ .../delta/workload/WriteSpecValidator.scala | 460 ++++++++++++++++++ .../workload/tables/WriteBasicSuite.scala | 198 ++++++++ .../workload/tables/WriteCommitSuite.scala | 127 +++++ 8 files changed, 1719 insertions(+), 13 deletions(-) create mode 100644 workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala create mode 100644 workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala create mode 100644 workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala diff --git a/workload-generator/docs/spec-reference.md b/workload-generator/docs/spec-reference.md index 28c68014..7c56768f 100644 --- a/workload-generator/docs/spec-reference.md +++ b/workload-generator/docs/spec-reference.md @@ -22,6 +22,7 @@ Exactly one of `expected` or `expectedError` is present. The other is omitted (n - [Common Types](#common-types) - [Read Spec](#read-spec) - [Snapshot Spec](#snapshot-spec) +- [Write Spec](#write-spec) - [table_info.json](#table_infojson) - [Expected Data Layout](#expected-data-layout) @@ -347,6 +348,280 @@ These are the raw Delta protocol and metadata JSON structures — not simplified --- +## Write Spec + +**Type:** `"write"` + +Tests Delta writer implementations by providing a sequence of write operations to replay. Unlike read specs (which verify reading an existing table), a write spec describes *how to construct* a table from scratch. + +A write spec is a portable, implementation-agnostic recipe: it specifies what operations to perform (create table, insert, delete, update, etc.) declaratively, so any conforming Delta writer can interpret and execute it. After replaying all commits, the resulting table is compared against the expected data under `expected/latest/`. + +The write spec is written to `write_spec.json` at the workload output root (not under `specs/`). The read and snapshot specs declared alongside a write workload are captured normally under `specs/`. + +### Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `string` | yes | Always `"write"` | +| `commits` | `WriteCommit[]` | yes | Ordered list of commits to replay | + +### WriteCommit + +Each commit represents a single Delta transaction. There are two categories: high-level operations (SQL semantics) and the low-level `commit` (raw Delta actions). + +#### High-Level Operations + +These map to SQL-like operations. The writer translates them to appropriate Delta actions. + +| Operation | Fields | Description | +|-----------|--------|-------------| +| `create_table` | `schema`, `partitionColumns?`, `properties?`, `dataFiles?` | Create a new table with the given schema | +| `insert` | `dataFiles?` | Append rows from data files | +| `update` | `predicate`, `set` | Update rows matching predicate | +| `delete` | `predicate` | Delete rows matching predicate | +| `evolve_schema` | `addColumns?`, `renameColumns?`, `dropColumns?` | Modify table schema | +| `update_properties` | `set?`, `remove?` | Modify table properties | +| `restore` | `version` | Restore table to a previous version | + +The reference validator does not replay `restore`: an independent replay generates different file paths, so RESTORE-to-version would reference paths that do not exist in the replayed table. Conforming consumers may implement it. + +#### Low-Level Operation + +| Operation | Fields | Description | +|-----------|--------|-------------| +| `commit` | `schema?`, `tableProperties?`, `txn?`, `addFiles?`, `removeFiles?`, `addDomainMetadata?`, `removeDomainMetadata?` | Directly specify Delta actions. `addFiles[].dataFile` must reference Parquet already in the table (e.g. produced by a prior `insert`); capture copies it under `data/commit_N/`. `removeFiles[].path` must target a file added by a prior low-level `commit`; paths from `insert`/`create_table` are writer-generated and not stable across replay. Deletion vectors are not honored. | + +### WriteCommit Fields + +| Field | Type | Description | +|-------|------|-------------| +| `operation` | `string` | Operation type (see above) | +| `schema` | `object` | Table schema in Delta JSON format | +| `partitionColumns` | `string[]` | Partition column names | +| `properties` | `map` | Table properties (e.g., `delta.enableDeletionVectors`) | +| `dataFiles` | `string[]` | Relative paths to Parquet data files under `data/`. Files for a partitioned commit keep their `col=val/` directory prefix | +| `predicate` | `string` | SQL WHERE clause for update/delete | +| `set` | `map` | Column assignments for update (column → expression) or properties to set | +| `remove` | `string[]` | Property names to remove | +| `addColumns` | `object[]` | Columns to add (each with `name`, `type`, `nullable`) | +| `renameColumns` | `map` | Column renames (old name → new name) | +| `dropColumns` | `string[]` | Column names to drop | +| `version` | `long` | Target version for restore | +| `tableProperties` | `map` | Properties for low-level commit | +| `txn` | `AppTxn` | Application transaction for idempotent writes | +| `addFiles` | `AddFileAction[]` | Files to add (low-level) | +| `removeFiles` | `RemoveFileAction[]` | Files to remove (low-level) | +| `addDomainMetadata` | `AddDomainMetadata[]` | Domain metadata to add | +| `removeDomainMetadata` | `string[]` | Domain names to remove | + +### AddFileAction (Low-Level) + +| Field | Type | Description | +|-------|------|-------------| +| `dataFile` | `string` | Relative path to Parquet file under `data/` | +| `partitionValues` | `map?` | Partition values for this file | +| `dataChange` | `boolean?` | Whether this is a data change (default true) | + +### RemoveFileAction (Low-Level) + +| Field | Type | Description | +|-------|------|-------------| +| `path` | `string` | Path of file to remove | +| `dataChange` | `boolean?` | Whether this is a data change (default true) | + +`path` must target a file added by a prior low-level `commit`; paths from `insert`/`create_table` are writer-generated and not stable across replay. + +### Expected Data + +The final table state after replaying all commits is captured under `expected/latest/`: + +- `table_content/` — Parquet files with the expected rows +- `table_version_metadata.json` — the latest snapshot's `protocol` and `metadata` + +### Comparison Semantics + +The capture comes from one writer, but a write spec must validate against any conforming Delta writer, so a consumer compares the replayed snapshot against `expected` by capability rather than byte equality: + +- **`protocol`** — feature-superset plus version-floor, not exact. The replay must satisfy `minReaderVersion >= expected` and `minWriterVersion >= expected`, and its `readerFeatures`/`writerFeatures` must be supersets of the expected sets (missing arrays count as empty; membership is order-insensitive). A higher version or extra features is allowed. +- **`configuration`** — only the keys the write spec's own commits declared are checked: `create_table.properties` plus `update_properties.set`, minus `update_properties.remove`. Each declared key must be present and equal; each removed key must be absent. Engine-injected default properties are ignored. If the spec declared no properties, nothing is checked. +- **rows** (`table_content/`) — multiset comparison; order is irrelevant, but each row must appear the correct number of times. + +`schemaString`, `partitionColumns`, and `format` are compared exactly. + +### Directory Structure + +``` +/ +├── write_spec.json # The write spec +├── data/ # Data files referenced by commits +│ ├── commit_0/ +│ │ └── part-0000-xxx.parquet +│ ├── commit_1/ +│ │ └── part-0000-yyy.parquet +│ └── ... +├── expected/ +│ ├── latest/ +│ │ ├── table_content/ +│ │ └── table_version_metadata.json +│ └── / # expected_data for each read spec +│ └── expected_data/ +├── specs/ # Read/snapshot specs for this workload +└── table_info.json +``` + +### Examples + +**Create table and insert:** + +```json +{ + "type": "write", + "commits": [ + { + "operation": "create_table", + "schema": { + "type": "struct", + "fields": [ + { "name": "id", "type": "integer", "nullable": false, "metadata": {} }, + { "name": "name", "type": "string", "nullable": true, "metadata": {} } + ] + }, + "properties": { "delta.enableDeletionVectors": "true" } + }, + { + "operation": "insert", + "dataFiles": ["data/commit_1/part-0000-abc.parquet"] + } + ] +} +``` + +**Delete with predicate:** + +```json +{ + "type": "write", + "commits": [ + { "operation": "create_table", "schema": { } }, + { "operation": "insert", "dataFiles": ["data/commit_1/part-0000-abc.parquet"] }, + { "operation": "delete", "predicate": "id > 100" } + ] +} +``` + +**Update with SET:** + +```json +{ + "type": "write", + "commits": [ + { "operation": "create_table", "schema": { } }, + { "operation": "insert", "dataFiles": ["data/commit_1/part-0000-abc.parquet"] }, + { + "operation": "update", + "predicate": "status = 'pending'", + "set": { "status": "'active'", "count": "count + 1" } + } + ] +} +``` + +**Schema evolution (add column):** + +```json +{ + "type": "write", + "commits": [ + { + "operation": "create_table", + "schema": { + "type": "struct", + "fields": [ + { "name": "id", "type": "integer", "nullable": false, "metadata": {} } + ] + } + }, + { "operation": "insert", "dataFiles": ["data/commit_1/part-0000-abc.parquet"] }, + { + "operation": "evolve_schema", + "addColumns": [ { "name": "email", "type": "string", "nullable": true } ] + }, + { "operation": "insert", "dataFiles": ["data/commit_3/part-0000-def.parquet"] } + ] +} +``` + +**Partitioned table:** + +```json +{ + "type": "write", + "commits": [ + { + "operation": "create_table", + "schema": { + "type": "struct", + "fields": [ + { "name": "id", "type": "integer", "nullable": true, "metadata": {} }, + { "name": "region", "type": "string", "nullable": true, "metadata": {} }, + { "name": "revenue", "type": "integer", "nullable": true, "metadata": {} } + ] + }, + "partitionColumns": ["region"] + }, + { + "operation": "insert", + "dataFiles": [ + "data/commit_1/region=east/part-0000-abc.parquet", + "data/commit_1/region=west/part-0000-def.parquet" + ] + } + ] +} +``` + +**Low-level commit with domain metadata:** + +```json +{ + "type": "write", + "commits": [ + { + "operation": "create_table", + "schema": { }, + "properties": { "delta.feature.domainMetadata": "supported" } + }, + { + "operation": "commit", + "addFiles": [ + { "dataFile": "data/commit_1/part-0000-abc.parquet", "partitionValues": {}, "dataChange": true } + ], + "addDomainMetadata": [ + { "domain": "myApp.config", "configuration": "{\"version\": 1}" } + ] + } + ] +} +``` + +**Low-level commit with application transaction:** + +```json +{ + "type": "write", + "commits": [ + { "operation": "create_table", "schema": { } }, + { + "operation": "commit", + "txn": { "appId": "streaming-job-1", "version": 42 }, + "addFiles": [ { "dataFile": "data/commit_1/part-0000-abc.parquet" } ] + } + ] +} +``` + +--- + ## table_info.json Written to the workload output root. Provides metadata about the generated table for discovery and filtering. diff --git a/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala b/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala index cfb48003..b06b55bd 100644 --- a/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala +++ b/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala @@ -118,24 +118,13 @@ case class TableInfo( // Low-level action types // ============================================================================= -/** Deletion vector descriptor for low-level commits. */ -@JsonPropertyOrder(Array("storageType", "pathOrInlineDv", "offset", "sizeInBytes", "cardinality")) -@JsonInclude(JsonInclude.Include.NON_ABSENT) -case class DeletionVectorAction( - storageType: String, - pathOrInlineDv: String, - offset: Option[Int] = None, - sizeInBytes: Int, - cardinality: Long) - /** Add file action for low-level commits. */ -@JsonPropertyOrder(Array("dataFile", "partitionValues", "dataChange", "deletionVector")) +@JsonPropertyOrder(Array("dataFile", "partitionValues", "dataChange")) @JsonInclude(JsonInclude.Include.NON_ABSENT) case class AddFileAction( dataFile: String, partitionValues: Option[Map[String, String]] = None, - dataChange: Option[Boolean] = None, - deletionVector: Option[DeletionVectorAction] = None) + dataChange: Option[Boolean] = None) /** Remove file action for low-level commits. */ @JsonPropertyOrder(Array("path", "dataChange")) @@ -144,6 +133,59 @@ case class RemoveFileAction( path: String, dataChange: Option[Boolean] = None) +/** Application transaction for idempotent low-level commits. */ +@JsonPropertyOrder(Array("appId", "version")) +case class AppTxn(appId: String, version: Long) + +/** Domain metadata entry for low-level commits (added domains). */ +@JsonPropertyOrder(Array("domain", "configuration")) +case class AddDomainMetadata( + domain: String, + configuration: String) + +// ============================================================================= +// Write spec case classes +// ============================================================================= + +/** + * A single commit in a write spec. There are two categories of operation: + * high-level (SQL semantics, e.g. create_table/insert/update/delete) and the + * low-level `commit` (raw Delta actions). Only the fields relevant to a given + * operation are populated; all others are absent in the serialized JSON. + */ +@JsonPropertyOrder(Array("operation", "schema", "partitionColumns", + "properties", "dataFiles", "predicate", "set", "remove", + "addColumns", "renameColumns", "dropColumns", "version", + "tableProperties", "txn", "addFiles", "removeFiles", + "addDomainMetadata", "removeDomainMetadata")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class WriteCommit( + operation: String, + // High-level fields + schema: Option[Any] = None, + partitionColumns: Option[Seq[String]] = None, + properties: Option[Map[String, String]] = None, + dataFiles: Option[Seq[String]] = None, + predicate: Option[String] = None, + set: Option[Map[String, String]] = None, + remove: Option[Seq[String]] = None, + addColumns: Option[Any] = None, + renameColumns: Option[Map[String, String]] = None, + dropColumns: Option[Seq[String]] = None, + version: Option[Long] = None, + // Low-level commit fields (operation = "commit") + tableProperties: Option[Map[String, String]] = None, + txn: Option[AppTxn] = None, + addFiles: Option[Seq[AddFileAction]] = None, + removeFiles: Option[Seq[RemoveFileAction]] = None, + addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, + removeDomainMetadata: Option[Seq[String]] = None) + +@JsonPropertyOrder(Array("type", "commits")) +case class WriteSpec(commits: Seq[WriteCommit]) { + val `type`: String = "write" +} + // ============================================================================= // Delta log action case classes (for parsing commit JSON) // ============================================================================= @@ -228,6 +270,9 @@ object JsonUtil { def readSnapshotSpec(path: Path): SnapshotSpec = mapper.readValue(Files.readAllBytes(path), classOf[SnapshotSpec]) + def readWriteSpec(path: Path): WriteSpec = + mapper.readValue(Files.readAllBytes(path), classOf[WriteSpec]) + def toRowMultiset(df: DataFrame): Map[String, Int] = { // Drop MAP-type columns that Spark can't serialize to JSON val nonMapCols = df.schema.fields diff --git a/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala b/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala new file mode 100644 index 00000000..9f14cc64 --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala @@ -0,0 +1,122 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload + +import java.nio.file.{Files, Path, Paths, StandardCopyOption} + +import scala.collection.mutable + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.types.StructType + +/** + * Applies a low-level `commit` (raw Delta actions) by driving the DeltaLog transaction + * API directly. Shared by capture (`commitOp`, against the live table) and replay + * (`WriteSpecValidator`, against a fresh table) so both produce identical actions. + * + * Capture and replay differ only in where source files come from (`resolveDataFile`) and what + * in-table path each file gets (`assignPath`): capture assigns fresh names and records them so + * the spec stays consistent with the live log; replay reuses those recorded paths so both sides + * commit identical `AddFile.path`s. + * + * Deletion vectors are intentionally not honored: the spec carries no DV descriptor and + * none is emitted here. + */ +object LowLevelCommit { + + /** + * @param spark active session + * @param tablePath table to commit against + * @param resolveDataFile maps an `AddFileAction.dataFile` to the source Parquet on disk + * @param assignPath the in-table path each added file occupies (the emitted `AddFile.path`) + * @return the effective add files, each `dataFile` set to the in-table relative path used + */ + def apply( + spark: SparkSession, + tablePath: String, + schemaDDL: Option[String], + tableProperties: Option[Map[String, String]], + txn: Option[AppTxn], + addFiles: Option[Seq[AddFileAction]], + removeFiles: Option[Seq[RemoveFileAction]], + addDomainMetadata: Option[Seq[AddDomainMetadata]], + removeDomainMetadata: Option[Seq[String]], + resolveDataFile: String => Path, + assignPath: AddFileAction => String): Seq[AddFileAction] = { + import org.apache.spark.sql.delta.DeltaLog + import org.apache.spark.sql.delta.actions._ + + DeltaLog.clearCache() + val deltaLog = DeltaLog.forTable(spark, tablePath) + val transaction = deltaLog.startTransaction() + val actions = mutable.ArrayBuffer[Action]() + + if (schemaDDL.isDefined || tableProperties.isDefined) { + val currentMetadata = transaction.metadata + val newSchemaString = schemaDDL + .map(ddl => StructType.fromDDL(ddl).json) + .getOrElse(currentMetadata.schemaString) + val newConfig = tableProperties + .map(props => currentMetadata.configuration ++ props) + .getOrElse(currentMetadata.configuration) + transaction.updateMetadata( + currentMetadata.copy(schemaString = newSchemaString, configuration = newConfig)) + } + + txn.foreach { t => + actions += SetTransaction(t.appId, t.version, Some(System.currentTimeMillis())) + } + + val effectiveAdds = addFiles.getOrElse(Seq.empty).map { file => + val src = resolveDataFile(file.dataFile) + require(Files.exists(src), s"commit addFiles references missing data file: $src") + val relative = assignPath(file) + val dest = Paths.get(tablePath).resolve(relative) + if (dest.toAbsolutePath.normalize() != src.toAbsolutePath.normalize()) { + Files.createDirectories(dest.getParent) + Files.copy(src, dest, StandardCopyOption.REPLACE_EXISTING) + } + actions += AddFile( + path = relative, + partitionValues = file.partitionValues.getOrElse(Map.empty), + size = Files.size(dest), + modificationTime = System.currentTimeMillis(), + dataChange = file.dataChange.getOrElse(true)) + file.copy(dataFile = relative) + } + + removeFiles.foreach { files => + for (file <- files) { + actions += RemoveFile( + path = file.path, + deletionTimestamp = Some(System.currentTimeMillis()), + dataChange = file.dataChange.getOrElse(true)) + } + } + + addDomainMetadata.foreach { dms => + for (dm <- dms) actions += DomainMetadata(dm.domain, dm.configuration, removed = false) + } + removeDomainMetadata.foreach { domains => + for (domain <- domains) actions += DomainMetadata(domain, "", removed = true) + } + + transaction.commit(actions.toSeq, org.apache.spark.sql.delta.DeltaOperations.ManualUpdate) + DeltaLog.clearCache() + effectiveAdds + } +} diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 3a8f9b36..912a5d2d 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -126,6 +126,11 @@ object WorkloadGenerator { checkAssertion(rs, specsDir.resolve(s"$specName.json")) } + // Write spec - serialize the recorded commit history + expected/latest/ + ts.writeBuilder.foreach { builder => + builder.buildSpec(spark, destTablePath, testOutputDir) + } + // table_info.json - wrap in try/catch for corrupt tables try { TableInfoWriter.write(spark, destTablePath, testOutputDir, @@ -135,6 +140,13 @@ object WorkloadGenerator { System.err.println(s"WARN: Could not write table_info.json for $dirName: ${e.getMessage}") } + // Replay the write spec against a fresh table and re-validate read/snapshot specs. + ts.writeBuilder.foreach { _ => + val replayWarnings = WriteSpecValidator.validate(spark, testOutputDir, dirName) + require(replayWarnings.isEmpty, + s"Write spec validation FAILED for $dirName:\n ${replayWarnings.mkString("\n ")}") + } + // Repro placeholder val reproDir = testOutputDir.resolve("repro") Files.createDirectories(reproDir) @@ -235,6 +247,21 @@ class TableHandle private[workload] ( } } +// --------------------------------------------------------------------------- +// WriteHandle — returned by createTableOp() / writeSpec() +// --------------------------------------------------------------------------- + +/** + * Handle for structured write operations. Obtained from [[WorkloadContext.createTableOp]] + * (which creates the table and records the create) or [[WorkloadContext.writeSpec]] + * (which wraps an already-created [[TableHandle]]). + * + * Accepts write operations (insertOp, updateOp, deleteOp, ...) that both execute SQL + * and record the commit. Call [[WorkloadContext.registerWriteSpec]] to finalize and + * obtain a [[TableHandle]] for declaring read/snapshot specs. + */ +class WriteHandle private[workload] (private[workload] val table: TableHandle) + // --------------------------------------------------------------------------- // WorkloadContext — the user's interface inside a workload() block // --------------------------------------------------------------------------- @@ -346,6 +373,157 @@ class WorkloadContext private[workload] ( DeltaHarness.get.openLog(spark, loc).checkpoint() } + // ---- Write specs ---- + + private val _writeBuilders = mutable.HashMap[String, WriteSpecBuilder]() + + private def getWriteBuilder(table: TableHandle): WriteSpecBuilder = { + val key = s"${workloadName}_${table.tableName}" + val builder = _writeBuilders.getOrElseUpdate(key, new WriteSpecBuilder()) + getTableSpec(table).writeBuilder = Some(builder) + builder + } + + /** Wrap an already-created [[TableHandle]] for structured write operations. */ + def writeSpec(table: TableHandle): WriteHandle = { + getWriteBuilder(table) + new WriteHandle(table) + } + + /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ + def registerWriteSpec(w: WriteHandle): TableHandle = w.table + + /** + * Create a table via SQL and record the create_table operation. `schema` is a SQL + * DDL string (e.g. "id INT, name STRING NOT NULL"). Returns a [[WriteHandle]] for + * further write operations. + */ + def createTableOp( + tableName: String, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty): WriteHandle = { + val partitionClause = + if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" + val propsClause = if (properties.nonEmpty) { + s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" + } else "" + sql(s"CREATE TABLE $tableName ($schema) USING delta$partitionClause$propsClause") + val t = registerTable(tableName) + getWriteBuilder(t).recordCreateTable(schema, properties, partitionColumns) + new WriteHandle(t) + } + + /** + * Insert `rows` (column -> value maps) and record the insert. An empty `rows` is a no-op: + * it produces no commit, so recording it would both desync the commit-index/version mapping + * and let the validator pass a spec with nothing to validate. + */ + def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = { + require(rows.nonEmpty, "insertOp requires at least one row") + val columns = rows.head.keys.toSeq + val colList = columns.map(c => s"`$c`").mkString(", ") + val valuesList = rows.map { row => + val values = columns.map { c => + row.get(c) match { + case Some(s: String) => s"'$s'" + case Some(null) | None => "NULL" + case Some(v) => v.toString + } + } + s"(${values.mkString(", ")})" + }.mkString(", ") + sql(s"INSERT INTO ${w.table.tableName} ($colList) VALUES $valuesList") + getWriteBuilder(w.table).recordInsert() + } + + /** Delete rows matching `predicate` and record the delete. */ + def deleteOp(w: WriteHandle, predicate: String): Unit = { + sql(s"DELETE FROM ${w.table.tableName} WHERE $predicate") + getWriteBuilder(w.table).recordDelete(predicate) + } + + /** Update rows matching `predicate` with `set` (column -> expression) and record it. */ + def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = { + val setClauses = set.map { case (k, v) => s"`$k` = $v" }.mkString(", ") + sql(s"UPDATE ${w.table.tableName} SET $setClauses WHERE $predicate") + getWriteBuilder(w.table).recordUpdate(predicate, set) + } + + /** Restore the table to `version` and record the restore. */ + def restoreOp(w: WriteHandle, version: Long): Unit = { + sql(s"RESTORE TABLE ${w.table.tableName} TO VERSION AS OF $version") + getWriteBuilder(w.table).recordRestore(version) + } + + /** + * Execute a low-level commit of raw Delta actions against the live table and record it. See + * [[LowLevelCommit]] for action semantics; addFiles must reference Parquet already in the table. + */ + def commitOp( + w: WriteHandle, + schemaDDL: Option[String] = None, + tableProperties: Option[Map[String, String]] = None, + txn: Option[AppTxn] = None, + addFiles: Option[Seq[AddFileAction]] = None, + removeFiles: Option[Seq[RemoveFileAction]] = None, + addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, + removeDomainMetadata: Option[Seq[String]] = None): Unit = { + val effectiveAdds = LowLevelCommit.apply( + spark, w.table.sourcePath.toString, + schemaDDL = schemaDDL, tableProperties = tableProperties, txn = txn, + addFiles = addFiles, removeFiles = removeFiles, + addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata, + resolveDataFile = rel => w.table.sourcePath.resolve(rel), + // Copy each source into a fresh in-table file so the added action is independent of the + // source's own add. Recording these paths lets a later removeFiles target them by name. + assignPath = _ => s"part-commit-${java.util.UUID.randomUUID()}-c000.snappy.parquet") + getWriteBuilder(w.table).recordCommit( + schemaDDL = schemaDDL, tableProperties = tableProperties, txn = txn, + addFiles = if (effectiveAdds.nonEmpty) Some(effectiveAdds) else addFiles, + removeFiles = removeFiles, + addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata) + } + + /** Add columns (SQL DDL) and record the schema evolution. */ + def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = { + require(columnsDDL.nonEmpty, "addColumnsOp requires a non-empty DDL string") + sql(s"ALTER TABLE ${w.table.tableName} ADD COLUMNS ($columnsDDL)") + getWriteBuilder(w.table).recordAddColumns(columnsDDL) + } + + /** Rename a column and record the schema evolution. */ + def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = { + sql(s"ALTER TABLE ${w.table.tableName} RENAME COLUMN $oldName TO $newName") + getWriteBuilder(w.table).recordRenameColumn(oldName, newName) + } + + /** Drop columns and record the schema evolution. */ + def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = { + require(columns.nonEmpty, "dropColumnsOp requires at least one column") + if (columns.size == 1) { + sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMN ${columns.head}") + } else { + sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMNS (${columns.mkString(", ")})") + } + getWriteBuilder(w.table).recordDropColumns(columns) + } + + /** Set table properties and record the update_properties operation. */ + def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = { + require(props.nonEmpty, "setPropertiesOp requires at least one property") + val setClause = props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + sql(s"ALTER TABLE ${w.table.tableName} SET TBLPROPERTIES ($setClause)") + getWriteBuilder(w.table).recordSetProperties(props) + } + + /** Unset table properties and record the update_properties operation. */ + def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = { + require(props.nonEmpty, "unsetPropertiesOp requires at least one property") + sql(s"ALTER TABLE ${w.table.tableName} UNSET TBLPROPERTIES (${props.map(k => s"'$k'").mkString(", ")})") + getWriteBuilder(w.table).recordUnsetProperties(props) + } + // ---- Table mutations (applied to copied table before spec capture) ---- /** Mutate the copied table's filesystem before specs are captured. */ @@ -567,6 +745,68 @@ trait WorkloadOps { /** Force Spark to write a checkpoint file for the given SQL table name. */ def forceCheckpoint(tableName: String): Unit = current.forceCheckpoint(tableName) + // ---- Write specs ---- + + /** Wrap an already-created [[TableHandle]] for structured write operations. */ + def writeSpec(table: TableHandle): WriteHandle = current.writeSpec(table) + + /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ + def registerWriteSpec(w: WriteHandle): TableHandle = current.registerWriteSpec(w) + + /** Create a table via SQL and record the create_table operation. */ + def createTableOp( + tableName: String, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty): WriteHandle = + current.createTableOp(tableName, schema, properties, partitionColumns) + + /** Insert rows and record the insert. */ + def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = current.insertOp(w, rows) + + /** Delete rows matching `predicate` and record the delete. */ + def deleteOp(w: WriteHandle, predicate: String): Unit = current.deleteOp(w, predicate) + + /** Update rows matching `predicate` with `set` and record the update. */ + def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = + current.updateOp(w, predicate, set) + + /** Restore the table to `version` and record the restore. */ + def restoreOp(w: WriteHandle, version: Long): Unit = current.restoreOp(w, version) + + /** Execute a low-level commit of raw Delta actions and record it. */ + def commitOp( + w: WriteHandle, + schemaDDL: Option[String] = None, + tableProperties: Option[Map[String, String]] = None, + txn: Option[AppTxn] = None, + addFiles: Option[Seq[AddFileAction]] = None, + removeFiles: Option[Seq[RemoveFileAction]] = None, + addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, + removeDomainMetadata: Option[Seq[String]] = None): Unit = + current.commitOp(w, schemaDDL, tableProperties, txn, addFiles, removeFiles, + addDomainMetadata, removeDomainMetadata) + + /** Add columns (SQL DDL) and record the schema evolution. */ + def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = + current.addColumnsOp(w, columnsDDL) + + /** Rename a column and record the schema evolution. */ + def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = + current.renameColumnOp(w, oldName, newName) + + /** Drop columns and record the schema evolution. */ + def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = + current.dropColumnsOp(w, columns) + + /** Set table properties and record the update_properties operation. */ + def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = + current.setPropertiesOp(w, props) + + /** Unset table properties and record the update_properties operation. */ + def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = + current.unsetPropertiesOp(w, props) + /** Mutate the copied table's filesystem before specs are captured. */ def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = current.mutateTable(table)(mutation) @@ -592,6 +832,7 @@ private[workload] class TableSpec( val readSpecs = mutable.ArrayBuffer[ReadSpecConfig]() val snapshotSpecs = mutable.ArrayBuffer[SnapshotSpecConfig]() val mutations = mutable.ArrayBuffer[Path => Unit]() + var writeBuilder: Option[WriteSpecBuilder] = None } case class WorkloadResult( diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala new file mode 100644 index 00000000..e43f45b0 --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala @@ -0,0 +1,238 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload + +import java.nio.file.{Files, Path, Paths, StandardCopyOption} + +import scala.collection.mutable + +import org.apache.spark.sql.{SaveMode, SparkSession} +import org.apache.spark.sql.types.StructType + +import io.delta.workload.deltaharness.DeltaHarness + +/** + * Records structured write operations and serializes them as `write_spec.json`, + * the portable recipe a Delta writer replays to reconstruct a table. + * + * High-level operations carry SQL semantics (create_table, insert, update, ...); + * the low-level `commit` carries raw Delta actions. Data files produced by + * insert/create_table commits are copied under `data/` so the spec is + * self-sufficient for replay. + */ +class WriteSpecBuilder { + + private val commits = mutable.ArrayBuffer[WriteCommit]() + + // === High-level operations (SQL semantics) === + + def recordCreateTable(schemaDDL: String, properties: Map[String, String], + partitionColumns: Seq[String]): Unit = { + commits += WriteCommit( + operation = "create_table", + schema = Some(ddlToSchemaJson(schemaDDL)), + partitionColumns = if (partitionColumns.nonEmpty) Some(partitionColumns) else None, + properties = if (properties.nonEmpty) Some(properties) else None) + } + + def recordInsert(): Unit = commits += WriteCommit(operation = "insert") + + def recordDelete(predicate: String): Unit = + commits += WriteCommit(operation = "delete", predicate = Some(predicate)) + + def recordUpdate(predicate: String, set: Map[String, String]): Unit = + commits += WriteCommit(operation = "update", predicate = Some(predicate), set = Some(set)) + + def recordSetProperties(props: Map[String, String]): Unit = + recordUpdateProperties(props, Seq.empty) + + def recordUnsetProperties(props: Seq[String]): Unit = + recordUpdateProperties(Map.empty, props) + + def recordUpdateProperties(set: Map[String, String], unset: Seq[String]): Unit = { + commits += WriteCommit( + operation = "update_properties", + set = if (set.nonEmpty) Some(set) else None, + remove = if (unset.nonEmpty) Some(unset) else None) + } + + def recordAddColumns(columnsDDL: String): Unit = + recordEvolveSchema(columnsDDL, Map.empty, Seq.empty) + + def recordRenameColumn(oldName: String, newName: String): Unit = + recordEvolveSchema("", Map(oldName -> newName), Seq.empty) + + def recordDropColumns(columns: Seq[String]): Unit = + recordEvolveSchema("", Map.empty, columns) + + def recordEvolveSchema( + addColumnsDDL: String, + renameColumns: Map[String, String], + dropColumns: Seq[String]): Unit = { + val addCols = if (addColumnsDDL.nonEmpty) { + val st = StructType.fromDDL(addColumnsDDL) + Some(st.fields.map { f => + val typeJson = JsonUtil.mapper.readValue(f.dataType.json, classOf[Any]) + Map[String, Any]("name" -> f.name, "type" -> typeJson, "nullable" -> f.nullable) + }.toSeq) + } else None + commits += WriteCommit( + operation = "evolve_schema", + addColumns = addCols, + renameColumns = if (renameColumns.nonEmpty) Some(renameColumns) else None, + dropColumns = if (dropColumns.nonEmpty) Some(dropColumns) else None) + } + + def recordRestore(version: Long): Unit = + commits += WriteCommit(operation = "restore", version = Some(version)) + + // === Low-level operation (raw Delta actions) === + + def recordCommit( + schemaDDL: Option[String] = None, + tableProperties: Option[Map[String, String]] = None, + txn: Option[AppTxn] = None, + addFiles: Option[Seq[AddFileAction]] = None, + removeFiles: Option[Seq[RemoveFileAction]] = None, + addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, + removeDomainMetadata: Option[Seq[String]] = None): Unit = { + commits += WriteCommit( + operation = "commit", + schema = schemaDDL.map(ddlToSchemaJson), + tableProperties = tableProperties, + txn = txn, + addFiles = addFiles, + removeFiles = removeFiles, + addDomainMetadata = addDomainMetadata, + removeDomainMetadata = removeDomainMetadata) + } + + // === Serialization === + + /** + * Serialize recorded operations to `write_spec.json`, enriching the data-producing + * commits (create_table, insert) with the actual data files they added, and writing + * the expected final table state to `expected/latest/`. + * + * New data files are copied to `outputDir/data/commit_/` and referenced via + * relative paths in each commit's `dataFiles`, making the spec self-sufficient. + */ + def buildSpec(spark: SparkSession, tablePath: Path, outputDir: Path): Unit = { + val log = DeltaHarness.get.openLog(spark, tablePath.toString) + val dataDir = outputDir.resolve("data") + Files.createDirectories(dataDir) + + // Commit index lines up with the table version it produced (commit 0 is the create at + // version 0). Guard the coupling: each recorded op must map to exactly one version. + val finalVersion = log.update().version + require(commits.size == finalVersion + 1, + s"write spec records ${commits.size} ops but table has ${finalVersion + 1} versions " + + "(0.." + finalVersion + "); op index can no longer be used as the table version") + + val highLevelDataOps = Set("insert", "create_table") + + // Enrich data-producing commits with the files they added so replay can reconstruct the + // table from copied parquet. High-level ops carry these as `dataFiles`; low-level commits + // already enumerate them in `addFiles`, whose `dataFile` paths we rewrite to the copies. + val enriched = commits.zipWithIndex.map { case (commit, idx) => + if (highLevelDataOps.contains(commit.operation)) { + val newFiles = newFilesAt(tablePath, idx) + if (newFiles.nonEmpty) { + val paths = newFiles.map(f => copyDataFile(tablePath, dataDir, idx, f)) + commit.copy(dataFiles = Some(paths)) + } else commit + } else if (commit.operation == "commit") { + commit.addFiles match { + case Some(files) if files.nonEmpty => + val rewritten = files.map { af => + val copied = copyDataFile(tablePath, dataDir, idx, af.dataFile) + af.copy(dataFile = copied) + } + commit.copy(addFiles = Some(rewritten)) + case _ => commit + } + } else commit + } + + JsonUtil.writeSpec(outputDir.resolve("write_spec.json"), WriteSpec(enriched.toSeq)) + writeExpectedLatest(spark, log, tablePath, outputDir) + } + + /** + * Copy a table-relative data file into `data/commit_/`, preserving any `col=val/` + * partition-directory prefix, and return the spec-relative path to the copy. + */ + private def copyDataFile(tablePath: Path, dataDir: Path, idx: Int, relative: String): String = { + val src = tablePath.resolve(relative) + val dest = dataDir.resolve(s"commit_$idx").resolve(relative) + Files.createDirectories(dest.getParent) + if (Files.exists(src)) Files.copy(src, dest, StandardCopyOption.REPLACE_EXISTING) + s"data/commit_$idx/$relative" + } + + /** + * Paths of the AddFile actions in commit `version`'s OWN log entry. Derived directly from + * that commit's `add` actions rather than diffing adjacent snapshots' active-file sets, + * which would misattribute files when a commit both adds and removes. + */ + private def newFilesAt(tablePath: Path, version: Int): Seq[String] = { + val commitFile = tablePath.resolve("_delta_log").resolve(f"$version%020d.json") + if (!Files.exists(commitFile)) return Seq.empty + val lines = new String(Files.readAllBytes(commitFile), "UTF-8") + .split("\n").filter(_.trim.nonEmpty) + lines.flatMap { line => + val node = JsonUtil.mapper.readTree(line) + Option(node.get("add")).map(_.get("path").asText()) + }.toSeq + } + + /** + * Capture the expected final table state under `expected/latest/`: + * - `table_content/` — Parquet of the latest snapshot's rows + * - `table_version_metadata.json` — protocol + metadata of the latest snapshot + */ + private def writeExpectedLatest( + spark: SparkSession, + log: io.delta.workload.deltaharness.LogView, + tablePath: Path, + outputDir: Path): Unit = { + val latestDir = outputDir.resolve("expected").resolve("latest") + Files.createDirectories(latestDir) + + val contentDir = latestDir.resolve("table_content") + if (contentDir.toFile.exists()) { + org.apache.commons.io.FileUtils.deleteDirectory(contentDir.toFile) + } + spark.read.format("delta").load(tablePath.toString) + .write.mode(SaveMode.Overwrite).parquet(contentDir.toString) + + val snapshot = log.update() + val protocol = JsonUtil.mapper.treeToValue( + JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol"), classOf[Any]) + val metadata = JsonUtil.mapper.treeToValue( + JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData"), classOf[Any]) + JsonUtil.writeSpec( + latestDir.resolve("table_version_metadata.json"), + SnapshotExpected(protocol, metadata)) + } + + /** Parse a SQL DDL string into Delta schema JSON using Spark's type parser. */ + private def ddlToSchemaJson(ddl: String): Any = { + val st = StructType.fromDDL(ddl) + JsonUtil.mapper.readValue(st.json, classOf[Any]) + } +} diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala new file mode 100644 index 00000000..b9a09b5a --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala @@ -0,0 +1,460 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload + +import java.nio.file.{Files, Path} + +import scala.collection.mutable +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.{SaveMode, SparkSession} +import org.apache.spark.sql.types.{DataType, StructField, StructType} + +import io.delta.workload.deltaharness.DeltaHarness + +/** + * Validates a write spec by: + * 1. Replaying its commits against a fresh table using the data files in `data/`. + * 2. Running every read/snapshot spec in `specs/` against the replayed table, + * comparing against `expected/`. + * + * CDF, checkpoint, CRC, domain-metadata, and appTxn specs are out of scope for the + * write spec and are ignored here. + */ +object WriteSpecValidator { + + /** + * Replay the write spec and validate read/snapshot specs against the result. + * + * @return warning/error messages (empty = all passed). A returned warning fails the + * generated workload; the orchestrator deletes the output so it auto-retries. + */ + def validate(spark: SparkSession, outputDir: Path, specName: String): Seq[String] = { + val writeSpecFile = outputDir.resolve("write_spec.json") + if (!Files.exists(writeSpecFile)) return Seq.empty + + val warnings = mutable.ArrayBuffer[String]() + val writeSpec = JsonUtil.readWriteSpec(writeSpecFile) + if (writeSpec.commits.isEmpty) return Seq.empty + + // An op the reference replayer intentionally does not replay (e.g. `restore`, whose replayed + // file paths differ from capture) means we skip REPLAY validation for the whole spec, but + // this is not a failure. A data op recorded with no data files is a capture bug and IS a + // failure. Check both before replaying anything. + writeSpec.commits.collectFirst { case c if isReplayUnsupported(c) => c.operation }.foreach { op => + println(s" Write spec validation skipping replay validation for $specName: " + + s"'$op' not supported by reference replayer") + return Seq.empty + } + writeSpec.commits.find(captureBugMissingDataFiles).foreach { c => + warnings += s"capture bug: '${c.operation}' recorded with no data files (not replayable)" + return warnings.toSeq + } + + val tempDir = Files.createTempDirectory(s"write_replay_$specName") + val replayTablePath = tempDir.resolve("replay_table") + + try { + val replayRef = s"delta.`${replayTablePath.toAbsolutePath}`" + writeSpec.commits.foreach(c => + replayCommit(spark, c, replayRef, replayTablePath.toAbsolutePath.toString, outputDir)) + + validateExpectedLatest(spark, replayTablePath, outputDir, specName, warnings) + + val specsDir = outputDir.resolve("specs") + if (Files.isDirectory(specsDir)) { + val specFiles = { + val stream = Files.list(specsDir) + try stream.iterator().asScala.filter(_.toString.endsWith(".json")).toSeq.sorted + finally stream.close() + } + for (specFile <- specFiles) { + val specJson = JsonUtil.mapper.readTree(Files.readAllBytes(specFile)) + val specType = Option(specJson.get("type")).map(_.asText()).getOrElse("") + val specFileName = specFile.getFileName.toString.stripSuffix(".json") + specType match { + case "read" => + try { + ReadCapture.validateFromSpec(spark, replayTablePath, + outputDir.resolve("expected").resolve(specFileName), specFile) + } catch { + case e: Throwable => warnings += s"read '$specFileName': ${e.getMessage}" + } + case "snapshot" => + validateSnapshotStructurally(spark, replayTablePath, specFile, specFileName, + writeSpec, warnings) + case _ => // out of scope (write/cdf/checkpoint/crc/...) + } + } + } + + if (warnings.isEmpty) { + println(s" Write spec validation PASSED for $specName") + } + } finally { + try org.apache.commons.io.FileUtils.deleteDirectory(tempDir.toFile) + catch { case _: Throwable => } + } + + warnings.toSeq + } + + /** Compare the replayed latest snapshot against `expected/latest/` (rows + protocol/metadata). */ + private def validateExpectedLatest( + spark: SparkSession, + replayTablePath: Path, + outputDir: Path, + specName: String, + warnings: mutable.ArrayBuffer[String]): Unit = { + val latestDir = outputDir.resolve("expected").resolve("latest") + val contentDir = latestDir.resolve("table_content") + if (Files.exists(contentDir)) { + val expected = JsonUtil.toRowMultiset(spark.read.parquet(contentDir.toString)) + val actual = JsonUtil.toRowMultiset( + spark.read.format("delta").load(replayTablePath.toAbsolutePath.toString)) + try JsonUtil.assertMultisetsEqual(expected, actual, s"${specName}_latest") + catch { case e: Throwable => warnings += e.getMessage } + } + } + + /** + * Validate a snapshot spec against the replayed table structurally. A replay is an + * independently created table, so its `id` and `createdTime` will never match the + * original. + * + * `protocol` and `configuration` are compared with portable, engine-neutral semantics so a + * spec validates against ANY conforming Delta writer, not just the one used to capture it: + * - `protocol`: capability check, not equality. The replay must meet every required floor + * (version >= expected, features superset of expected). A higher version or extra + * features on the replay are fine. + * - `configuration`: only the properties the write spec's own commits declared are checked + * (declared keys present + equal, removed keys absent). Engine-injected defaults are + * ignored. + * `schemaString`, `partitionColumns`, and `format` are still compared exactly. + */ + private def validateSnapshotStructurally( + spark: SparkSession, + replayTablePath: Path, + specFile: Path, + specFileName: String, + writeSpec: WriteSpec, + warnings: mutable.ArrayBuffer[String]): Unit = { + val spec = JsonUtil.readSnapshotSpec(specFile) + val (declaredKeys, removedKeys) = declaredConfiguration(writeSpec) + spec.expected.foreach { exp => + val log = DeltaHarness.get.openLog(spark, replayTablePath.toString) + val snapshot = (spec.version, spec.timestamp) match { + case (Some(v), _) => log.getSnapshotAt(v) + case _ => log.update() + } + val expProtocol = JsonUtil.mapper.valueToTree[com.fasterxml.jackson.databind.JsonNode]( + exp.protocol) + val replayProtocol = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") + protocolViolation(expProtocol, replayProtocol).foreach { reason => + warnings += s"snapshot '$specFileName': protocol mismatch ($reason)" + } + + val expectedMeta = JsonUtil.mapper.valueToTree[com.fasterxml.jackson.databind.JsonNode]( + exp.metadata) + val replayMeta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") + configurationViolation(expectedMeta, replayMeta, declaredKeys, removedKeys).foreach { + reason => warnings += s"snapshot '$specFileName': metadata.configuration mismatch ($reason)" + } + for (field <- Seq("schemaString", "partitionColumns", "format")) { + val expVal = canonicalField(expectedMeta, field) + val repVal = canonicalField(replayMeta, field) + if (expVal != repVal) warnings += s"snapshot '$specFileName': metadata.$field mismatch" + } + } + } + + /** + * Collect the configuration properties the write spec's commits explicitly declared. + * @return (declared keys still in effect, keys explicitly removed). A key removed after being + * set is reported only as removed. + */ + private def declaredConfiguration(writeSpec: WriteSpec): (Set[String], Set[String]) = { + val declared = mutable.LinkedHashSet[String]() + val removed = mutable.LinkedHashSet[String]() + for (commit <- writeSpec.commits) commit.operation match { + case "create_table" => + commit.properties.foreach(declared ++= _.keys) + case "update_properties" => + commit.set.foreach { s => declared ++= s.keys; removed --= s.keys } + commit.remove.foreach { r => removed ++= r; declared --= r } + case _ => () + } + (declared.toSet, removed.toSet) + } + + /** + * Check the replay protocol meets the expected one by capability: version floors and feature + * supersets. Returns a human-readable reason on the first violation, or None if compatible. + */ + private def protocolViolation( + expected: com.fasterxml.jackson.databind.JsonNode, + replay: com.fasterxml.jackson.databind.JsonNode): Option[String] = { + def intField(node: com.fasterxml.jackson.databind.JsonNode, name: String): Int = + if (node != null && node.has(name)) node.get(name).asInt() else 0 + def featureSet( + node: com.fasterxml.jackson.databind.JsonNode, name: String): Set[String] = + if (node != null && node.has(name) && node.get(name).isArray) { + node.get(name).elements().asScala.map(_.asText()).toSet + } else Set.empty + + val expReader = intField(expected, "minReaderVersion") + val expWriter = intField(expected, "minWriterVersion") + val repReader = intField(replay, "minReaderVersion") + val repWriter = intField(replay, "minWriterVersion") + val missingReaderFeatures = + featureSet(expected, "readerFeatures") -- featureSet(replay, "readerFeatures") + val missingWriterFeatures = + featureSet(expected, "writerFeatures") -- featureSet(replay, "writerFeatures") + + if (repReader < expReader) Some(s"minReaderVersion $repReader < $expReader") + else if (repWriter < expWriter) Some(s"minWriterVersion $repWriter < $expWriter") + else if (missingReaderFeatures.nonEmpty) + Some(s"missing readerFeatures ${missingReaderFeatures.mkString(",")}") + else if (missingWriterFeatures.nonEmpty) + Some(s"missing writerFeatures ${missingWriterFeatures.mkString(",")}") + else None + } + + /** + * Check the replay configuration against only the author-declared keys: each declared key must + * be present and equal on the replay, and each removed key must be absent. Engine-injected + * default properties are ignored. Returns a reason on the first violation, or None. + */ + private def configurationViolation( + expectedMeta: com.fasterxml.jackson.databind.JsonNode, + replayMeta: com.fasterxml.jackson.databind.JsonNode, + declaredKeys: Set[String], + removedKeys: Set[String]): Option[String] = { + val expConfig = Option(expectedMeta.get("configuration")).filterNot(_.isNull) + val repConfig = Option(replayMeta.get("configuration")).filterNot(_.isNull) + def get( + config: Option[com.fasterxml.jackson.databind.JsonNode], + key: String): Option[String] = + config.flatMap(c => Option(c.get(key))).filterNot(_.isNull).map(_.asText()) + + val mismatch = declaredKeys.toSeq.sorted.collectFirst { + case key if get(repConfig, key) != get(expConfig, key) => + s"key '$key' expected ${get(expConfig, key)}, got ${get(repConfig, key)}" + } + mismatch.orElse { + removedKeys.toSeq.sorted.collectFirst { + case key if get(repConfig, key).isDefined => s"removed key '$key' still present" + } + } + } + + /** Canonicalize a single metadata field (or "null" if absent) for comparison. */ + private def canonicalField( + node: com.fasterxml.jackson.databind.JsonNode, field: String): String = + if (node.has(field) && !node.get(field).isNull) JsonUtil.canonicalJson(node.get(field)) + else "null" + + /** Ops the reference replayer intentionally does not replay; their presence skips replay. */ + private def isReplayUnsupported(commit: WriteCommit): Boolean = + commit.operation == "restore" + + /** + * A data op recorded with no/empty data files. Capture should always record the files an + * `insert`/`create_table` wrote, so an empty record is a capture bug, not a missing feature. + */ + private def captureBugMissingDataFiles(commit: WriteCommit): Boolean = commit.operation match { + case "insert" => commit.dataFiles.forall(_.isEmpty) + case _ => false + } + + /** Replay a single commit against the replay table. */ + private def replayCommit( + spark: SparkSession, + commit: WriteCommit, + tableRef: String, + tablePath: String, + outputDir: Path): Unit = { + commit.operation match { + case "create_table" => + commit.schema.foreach { schema => + var sql = s"CREATE TABLE $tableRef (${schemaToColDefs(schema)}) USING delta" + commit.partitionColumns.filter(_.nonEmpty).foreach { parts => + sql += s" PARTITIONED BY (${parts.mkString(", ")})" + } + commit.properties.filter(_.nonEmpty).foreach { props => + sql += s" TBLPROPERTIES (${props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" + } + spark.sql(sql) + commit.dataFiles.foreach(loadDataFiles(spark, _, tablePath, outputDir)) + } + + case "insert" => + commit.dataFiles.foreach(loadDataFiles(spark, _, tablePath, outputDir)) + + case "delete" => + spark.sql(s"DELETE FROM $tableRef WHERE ${commit.predicate.getOrElse("true")}") + + case "update" => + commit.set.filter(_.nonEmpty).foreach { setMap => + val setClauses = setMap.map { case (k, v) => s"`$k` = $v" }.mkString(", ") + spark.sql(s"UPDATE $tableRef SET $setClauses WHERE ${commit.predicate.getOrElse("true")}") + } + + case "update_properties" => + val setClause = commit.set.getOrElse(Map.empty) + .map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + if (setClause.nonEmpty) spark.sql(s"ALTER TABLE $tableRef SET TBLPROPERTIES ($setClause)") + val unset = commit.remove.getOrElse(Seq.empty) + if (unset.nonEmpty) { + spark.sql(s"ALTER TABLE $tableRef UNSET TBLPROPERTIES (${unset.map(k => s"'$k'").mkString(", ")})") + } + + case "evolve_schema" => replayEvolveSchema(spark, commit, tablePath) + + case "commit" => replayLowLevelCommit(spark, commit, tablePath, outputDir) + + case _ => () + } + } + + /** Replay schema add/rename/drop via a single metadata-update transaction. */ + private def replayEvolveSchema(spark: SparkSession, commit: WriteCommit, tablePath: String): Unit = { + import org.apache.spark.sql.delta.DeltaLog + DeltaLog.clearCache() + val deltaLog = DeltaLog.forTable(spark, tablePath) + val txn = deltaLog.startTransaction() + val currentMetadata = txn.metadata + var fields = currentMetadata.schema.fields.toBuffer + + commit.addColumns.foreach { cols => + cols.asInstanceOf[Seq[Map[String, Any]]].foreach { col => + val name = col("name").toString + val nullable = col.get("nullable").forall(_.asInstanceOf[Boolean]) + fields += StructField(name, DataType.fromDDL(typeToSql(col("type"))), nullable) + } + } + commit.renameColumns.foreach { renames => + for ((oldName, newName) <- renames) { + val idx = fields.indexWhere(_.name == oldName) + if (idx >= 0) fields(idx) = fields(idx).copy(name = newName) + } + } + commit.dropColumns.foreach { drops => + fields = fields.filterNot(f => drops.contains(f.name)) + } + + txn.updateMetadata(currentMetadata.copy(schemaString = StructType(fields.toSeq).json)) + txn.commit(Seq.empty, org.apache.spark.sql.delta.DeltaOperations.ManualUpdate) + DeltaLog.clearCache() + } + + /** + * Replay a low-level commit. `addFiles[].dataFile` was rewritten at capture to + * `data/commit_N/`; we read from there and strip that prefix to recover + * the table-relative path. Schema for a low-level commit is stored as Delta schema JSON, + * so we round-trip it through `schemaToColDefs` to a DDL string for `LowLevelCommit`. + */ + private def replayLowLevelCommit( + spark: SparkSession, + commit: WriteCommit, + tablePath: String, + outputDir: Path): Unit = { + LowLevelCommit.apply( + spark, tablePath, + schemaDDL = commit.schema.map(schemaToColDefs), + tableProperties = commit.tableProperties, + txn = commit.txn, + addFiles = commit.addFiles, + removeFiles = commit.removeFiles, + addDomainMetadata = commit.addDomainMetadata, + removeDomainMetadata = commit.removeDomainMetadata, + resolveDataFile = rel => outputDir.resolve(rel), + assignPath = af => stripDataCommitPrefix(af.dataFile)) + } + + /** Strip the `data/commit_N/` prefix a captured commit prepends to its `addFiles` paths. */ + private def stripDataCommitPrefix(dataFile: String): String = { + val p = java.nio.file.Paths.get(dataFile) + if (p.getNameCount > 2) p.subpath(2, p.getNameCount).toString else dataFile + } + + /** + * Read parquet data files and append them to the replay table. Files for a partitioned + * table are stored under `col=val/` directories within their `data/commit_N/` base; we + * read with that base as `basePath` so Spark recovers the partition columns, otherwise + * the appended DataFrame would be missing them. + */ + private def loadDataFiles( + spark: SparkSession, + files: Seq[String], + tablePath: String, + outputDir: Path): Unit = { + val resolved = files.map(outputDir.resolve).filter(Files.exists(_)) + if (resolved.isEmpty) return + + // All files in one insert commit share a `data/commit_N/` base. + val basePath = outputDir.resolve( + java.nio.file.Paths.get(files.head).subpath(0, 2)).toAbsolutePath + val isPartitioned = resolved.exists(_.getParent != basePath) + val df = if (isPartitioned) { + spark.read.option("basePath", basePath.toString) + .parquet(resolved.map(_.toAbsolutePath.toString): _*) + } else { + spark.read.parquet(resolved.map(_.toAbsolutePath.toString): _*) + } + df.write.format("delta").mode(SaveMode.Append).save(tablePath) + } + + private def schemaToColDefs(schema: Any): String = { + val schemaMap = schema.asInstanceOf[Map[String, Any]] + val fields = schemaMap("fields").asInstanceOf[Seq[Map[String, Any]]] + fields.map { f => + val name = f("name").toString + val nullable = f.get("nullable").forall(_.asInstanceOf[Boolean]) + s"`$name` ${typeToSql(f("type"))}${if (!nullable) " NOT NULL" else ""}" + }.mkString(", ") + } + + private def typeToSql(typeValue: Any): String = typeValue match { + case s: String => s match { + case "integer" => "INT" + case "long" => "BIGINT" + case "short" => "SMALLINT" + case "byte" => "TINYINT" + case "float" => "FLOAT" + case "double" => "DOUBLE" + case "boolean" => "BOOLEAN" + case "string" => "STRING" + case "binary" => "BINARY" + case "date" => "DATE" + case "timestamp" => "TIMESTAMP" + case "timestamp_ntz" => "TIMESTAMP_NTZ" + case other => other.toUpperCase + } + case m: Map[_, _] => + val typeMap = m.asInstanceOf[Map[String, Any]] + typeMap("type") match { + case "struct" => + val fields = typeMap("fields").asInstanceOf[Seq[Map[String, Any]]] + s"STRUCT<${fields.map(f => s"${f("name")}: ${typeToSql(f("type"))}").mkString(", ")}>" + case "array" => s"ARRAY<${typeToSql(typeMap("elementType"))}>" + case "map" => s"MAP<${typeToSql(typeMap("keyType"))}, ${typeToSql(typeMap("valueType"))}>" + case other => other.toString.toUpperCase + } + case _ => typeValue.toString.toUpperCase + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala new file mode 100644 index 00000000..393ffc3e --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala @@ -0,0 +1,198 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import io.delta.workload.WorkloadTestSuite + +/** Basic write workloads producing write_spec.json for Delta writer testing. */ +class WriteBasicSuite extends WorkloadTestSuite("write_basic") { + + test("create_and_read") { + val w = createTableOp("tbl", schema = "id INT, name STRING, score DOUBLE") + insertOp(w, Seq( + Map("id" -> 1, "name" -> "alice", "score" -> 95.5), + Map("id" -> 2, "name" -> "bob", "score" -> 87.3), + Map("id" -> 3, "name" -> "charlie", "score" -> 72.1))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "score > 90.0", name = "read_high_score") + snapshotSpec(t) + } + + test("create_with_properties") { + val w = createTableOp("tbl", + schema = "id INT, value STRING, amount INT", + properties = Map( + "delta.enableDeletionVectors" -> "true", + "delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq( + Map("id" -> 1, "value" -> "alpha", "amount" -> 100), + Map("id" -> 2, "value" -> "beta", "amount" -> 200), + Map("id" -> 3, "value" -> "gamma", "amount" -> 300))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("insert_multiple") { + val w = createTableOp("tbl", schema = "id INT, category STRING") + insertOp(w, Seq( + Map("id" -> 1, "category" -> "a"), + Map("id" -> 2, "category" -> "a"))) + insertOp(w, Seq( + Map("id" -> 3, "category" -> "b"), + Map("id" -> 4, "category" -> "b"), + Map("id" -> 5, "category" -> "b"))) + insertOp(w, Seq(Map("id" -> 6, "category" -> "c"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, version = 1, name = "read_v1") + readSpec(t, version = 2, name = "read_v2") + readSpec(t, version = 3, name = "read_v3") + readSpec(t, predicate = "category = 'b'", name = "read_category_b") + snapshotSpec(t) + } + + test("delete_basic") { + val w = createTableOp("tbl", + schema = "id INT, value STRING, amount INT", + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, Seq( + Map("id" -> 1, "value" -> "keep", "amount" -> 10), + Map("id" -> 2, "value" -> "remove", "amount" -> 20), + Map("id" -> 3, "value" -> "keep", "amount" -> 30), + Map("id" -> 4, "value" -> "remove", "amount" -> 40), + Map("id" -> 5, "value" -> "keep", "amount" -> 50))) + deleteOp(w, predicate = "value = 'remove'") + val t = registerWriteSpec(w) + readSpec(t, name = "read_after_delete") + readSpec(t, version = 1, name = "read_before_delete") + readSpec(t, predicate = "amount > 25", name = "read_large_amount") + snapshotSpec(t) + } + + test("update_basic") { + val w = createTableOp("tbl", + schema = "id INT, status STRING, count INT", + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, Seq( + Map("id" -> 1, "status" -> "pending", "count" -> 0), + Map("id" -> 2, "status" -> "pending", "count" -> 0), + Map("id" -> 3, "status" -> "active", "count" -> 5), + Map("id" -> 4, "status" -> "pending", "count" -> 0))) + updateOp(w, predicate = "status = 'pending'", + set = Map("status" -> "'active'", "count" -> "count + 1")) + val t = registerWriteSpec(w) + readSpec(t, name = "read_after_update") + readSpec(t, version = 1, name = "read_before_update") + readSpec(t, predicate = "status = 'active'", name = "read_active") + snapshotSpec(t) + } + + test("alter_add_column") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq( + Map("id" -> 1, "name" -> "alice"), + Map("id" -> 2, "name" -> "bob"))) + addColumnsOp(w, "email STRING") + insertOp(w, Seq( + Map("id" -> 3, "name" -> "charlie", "email" -> "charlie@test.com"), + Map("id" -> 4, "name" -> "diana", "email" -> "diana@test.com"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, version = 1, name = "read_before_alter") + readSpec(t, predicate = "email IS NULL", name = "read_null_email") + readSpec(t, predicate = "email IS NOT NULL", name = "read_with_email") + snapshotSpec(t) + } + + test("alter_set_properties") { + val w = createTableOp("tbl", schema = "id INT, value INT") + insertOp(w, Seq( + Map("id" -> 1, "value" -> 100), + Map("id" -> 2, "value" -> 200), + Map("id" -> 3, "value" -> 300))) + setPropertiesOp(w, Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq( + Map("id" -> 4, "value" -> 400), + Map("id" -> 5, "value" -> 500))) + updateOp(w, predicate = "id <= 2", set = Map("value" -> "value + 1000")) + val t = registerWriteSpec(w) + readSpec(t, version = 1, name = "read_initial_insert") + readSpec(t, version = 3, name = "read_before_update") + readSpec(t, name = "read_all") + readSpec(t, predicate = "value > 1000", name = "read_updated") + snapshotSpec(t) + } + + test("delete_all_rows") { + val w = createTableOp("tbl", + schema = "id INT, data STRING", + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, Seq( + Map("id" -> 1, "data" -> "first"), + Map("id" -> 2, "data" -> "second"), + Map("id" -> 3, "data" -> "third"), + Map("id" -> 4, "data" -> "fourth"))) + deleteOp(w, predicate = "true") + val t = registerWriteSpec(w) + readSpec(t, name = "read_after_delete_all") + readSpec(t, version = 1, name = "read_before_delete_all") + snapshotSpec(t) + } + + test("partitioned_insert") { + val w = createTableOp("tbl", + schema = "id INT, region STRING, revenue INT", + partitionColumns = Seq("region")) + insertOp(w, Seq( + Map("id" -> 1, "region" -> "east", "revenue" -> 100), + Map("id" -> 2, "region" -> "east", "revenue" -> 150))) + insertOp(w, Seq( + Map("id" -> 3, "region" -> "west", "revenue" -> 200), + Map("id" -> 4, "region" -> "west", "revenue" -> 250), + Map("id" -> 5, "region" -> "west", "revenue" -> 300))) + insertOp(w, Seq(Map("id" -> 6, "region" -> "north", "revenue" -> 400))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "region = 'east'", name = "read_east") + readSpec(t, predicate = "region = 'west'", name = "read_west") + readSpec(t, predicate = "region = 'north'", name = "read_north") + readSpec(t, predicate = "revenue >= 200", name = "read_high_revenue") + snapshotSpec(t) + } + + test("delete_with_dvs") { + val w = createTableOp("tbl", + schema = "id INT, name STRING, score INT", + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, Seq( + Map("id" -> 1, "name" -> "alice", "score" -> 90), + Map("id" -> 2, "name" -> "bob", "score" -> 75), + Map("id" -> 3, "name" -> "charlie", "score" -> 88), + Map("id" -> 4, "name" -> "diana", "score" -> 92), + Map("id" -> 5, "name" -> "eve", "score" -> 60), + Map("id" -> 6, "name" -> "frank", "score" -> 85))) + deleteOp(w, predicate = "score < 80") + val t = registerWriteSpec(w) + readSpec(t, name = "read_after_delete") + readSpec(t, version = 1, name = "read_before_delete") + readSpec(t, predicate = "score >= 85", name = "read_high_score") + snapshotSpec(t) + } + +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala new file mode 100644 index 00000000..90105d9d --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala @@ -0,0 +1,127 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ + +import io.delta.workload.{AddDomainMetadata, AddFileAction, AppTxn, JsonUtil, + RemoveFileAction, TableHandle, WorkloadTestSuite} + +/** + * Low-level `commit` write workloads. Each case drives [[commitOp]] to produce a real + * `_delta_log` entry, asserts the produced action independently of replay, and relies on + * the generator's round-trip validation (replay + re-check specs) for end-to-end coverage. + */ +class WriteCommitSuite extends WorkloadTestSuite("write_commit") { + + /** Active add-file relative paths in the live table's latest version. */ + private def liveAddPaths(t: TableHandle): Seq[String] = { + val log = io.delta.workload.deltaharness.DeltaHarness.get.openLog(spark, t.sourcePath.toString) + log.update().allFiles.select("path").collect().map(_.getString(0)).toSeq + } + + /** Action inner-objects of a given type in a commit's `_delta_log/NNN.json`. */ + private def actionsAt(t: TableHandle, version: Long, action: String): Seq[com.fasterxml.jackson.databind.JsonNode] = { + val commitFile = t.sourcePath.resolve("_delta_log").resolve(f"$version%020d.json") + Files.readAllLines(commitFile).asScala.toSeq + .filter(_.trim.nonEmpty) + .map(JsonUtil.mapper.readTree) + .flatMap(n => Option(n.get(action))) + } + + private def latestVersion(t: TableHandle): Long = + io.delta.workload.deltaharness.DeltaHarness.get.openLog(spark, t.sourcePath.toString) + .update().version + + test("commit_with_txn") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + commitOp(w, txn = Some(AppTxn("app-1", 7L))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + val txns = actionsAt(t, v, "txn") + assert(txns.size == 1, s"expected one txn action at v$v, got ${txns.size}") + assert(txns.head.get("appId").asText() == "app-1") + assert(txns.head.get("version").asLong() == 7L) + + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("commit_with_domain_metadata") { + val w = createTableOp("tbl", + schema = "id INT, name STRING", + properties = Map("delta.feature.domainMetadata" -> "supported")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + commitOp(w, addDomainMetadata = Some(Seq( + AddDomainMetadata("delta.test", """{"k":"v"}""")))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + val dms = actionsAt(t, v, "domainMetadata") + assert(dms.size == 1, s"expected one domainMetadata action at v$v, got ${dms.size}") + assert(dms.head.get("domain").asText() == "delta.test") + assert(dms.head.get("configuration").asText() == """{"k":"v"}""") + assert(!dms.head.get("removed").asBoolean()) + + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("commit_add_file") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + val t0 = registerWriteSpec(w) + // Add a second data file via a low-level commit. Its bytes come from the existing insert + // file, but commitOp copies them to a fresh in-table path, so the table now holds two + // adds (the alice row appears twice). + val source = liveAddPaths(t0).head + commitOp(w, addFiles = Some(Seq(AddFileAction(dataFile = source)))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + val adds = actionsAt(t, v, "add") + assert(adds.size == 1, s"expected one add action at v$v, got ${adds.size}") + + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("commit_remove_file") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + val t0 = registerWriteSpec(w) + // Add a low-level file (deterministic, capture-assigned path), then tombstone it. Removing + // a low-level-added file (rather than an insert file) keeps the path stable across replay. + val source = liveAddPaths(t0).head + commitOp(w, addFiles = Some(Seq(AddFileAction(dataFile = source)))) + val added = actionsAt(t0, latestVersion(t0), "add").head.get("path").asText() + commitOp(w, removeFiles = Some(Seq(RemoveFileAction(path = added)))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + val removes = actionsAt(t, v, "remove") + assert(removes.size == 1, s"expected one remove action at v$v, got ${removes.size}") + assert(removes.head.get("path").asText() == added) + + readSpec(t, name = "read_after_remove") + snapshotSpec(t) + } +} From f527c17719c0e4af93ef0e830fe332f984813a29 Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Mon, 15 Jun 2026 21:27:27 +0000 Subject: [PATCH 02/25] refactor: route write-spec replay through the DeltaHarness SPI --- .../io/delta/workload/LowLevelCommit.scala | 64 +++++-------------- .../delta/workload/WriteSpecValidator.scala | 25 ++++---- .../workload/deltaharness/DeltaHarness.scala | 47 ++++++++++++++ .../deltaharness/DeltaSparkHarness.scala | 49 +++++++++++++- 4 files changed, 125 insertions(+), 60 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala b/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala index 9f14cc64..330f83e0 100644 --- a/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala +++ b/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala @@ -18,11 +18,11 @@ package io.delta.workload import java.nio.file.{Files, Path, Paths, StandardCopyOption} -import scala.collection.mutable - import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types.StructType +import io.delta.workload.deltaharness.{CommitAddFile, CommitRemoveFile, CommitRequest, DeltaHarness} + /** * Applies a low-level `commit` (raw Delta actions) by driving the DeltaLog transaction * API directly. Shared by capture (`commitOp`, against the live table) and replay @@ -57,31 +57,7 @@ object LowLevelCommit { removeDomainMetadata: Option[Seq[String]], resolveDataFile: String => Path, assignPath: AddFileAction => String): Seq[AddFileAction] = { - import org.apache.spark.sql.delta.DeltaLog - import org.apache.spark.sql.delta.actions._ - - DeltaLog.clearCache() - val deltaLog = DeltaLog.forTable(spark, tablePath) - val transaction = deltaLog.startTransaction() - val actions = mutable.ArrayBuffer[Action]() - - if (schemaDDL.isDefined || tableProperties.isDefined) { - val currentMetadata = transaction.metadata - val newSchemaString = schemaDDL - .map(ddl => StructType.fromDDL(ddl).json) - .getOrElse(currentMetadata.schemaString) - val newConfig = tableProperties - .map(props => currentMetadata.configuration ++ props) - .getOrElse(currentMetadata.configuration) - transaction.updateMetadata( - currentMetadata.copy(schemaString = newSchemaString, configuration = newConfig)) - } - - txn.foreach { t => - actions += SetTransaction(t.appId, t.version, Some(System.currentTimeMillis())) - } - - val effectiveAdds = addFiles.getOrElse(Seq.empty).map { file => + val resolvedAdds = addFiles.getOrElse(Seq.empty).map { file => val src = resolveDataFile(file.dataFile) require(Files.exists(src), s"commit addFiles references missing data file: $src") val relative = assignPath(file) @@ -90,33 +66,25 @@ object LowLevelCommit { Files.createDirectories(dest.getParent) Files.copy(src, dest, StandardCopyOption.REPLACE_EXISTING) } - actions += AddFile( + val add = CommitAddFile( path = relative, partitionValues = file.partitionValues.getOrElse(Map.empty), size = Files.size(dest), - modificationTime = System.currentTimeMillis(), dataChange = file.dataChange.getOrElse(true)) - file.copy(dataFile = relative) - } - - removeFiles.foreach { files => - for (file <- files) { - actions += RemoveFile( - path = file.path, - deletionTimestamp = Some(System.currentTimeMillis()), - dataChange = file.dataChange.getOrElse(true)) - } + (add, file.copy(dataFile = relative)) } - addDomainMetadata.foreach { dms => - for (dm <- dms) actions += DomainMetadata(dm.domain, dm.configuration, removed = false) - } - removeDomainMetadata.foreach { domains => - for (domain <- domains) actions += DomainMetadata(domain, "", removed = true) - } + val req = CommitRequest( + schemaJson = schemaDDL.map(ddl => StructType.fromDDL(ddl).json), + properties = tableProperties, + setTransaction = txn, + addFiles = resolvedAdds.map(_._1), + removeFiles = removeFiles.getOrElse(Seq.empty) + .map(f => CommitRemoveFile(f.path, f.dataChange.getOrElse(true))), + addDomainMetadata = addDomainMetadata.getOrElse(Seq.empty), + removeDomainMetadata = removeDomainMetadata.getOrElse(Seq.empty)) + DeltaHarness.get.commit(spark, tablePath, req) - transaction.commit(actions.toSeq, org.apache.spark.sql.delta.DeltaOperations.ManualUpdate) - DeltaLog.clearCache() - effectiveAdds + resolvedAdds.map(_._2) } } diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala index b9a09b5a..0654c90b 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala @@ -24,7 +24,7 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.types.{DataType, StructField, StructType} -import io.delta.workload.deltaharness.DeltaHarness +import io.delta.workload.deltaharness.{CommitRequest, DeltaHarness} /** * Validates a write spec by: @@ -331,14 +331,18 @@ object WriteSpecValidator { } } - /** Replay schema add/rename/drop via a single metadata-update transaction. */ + /** Replay schema add/rename/drop via a single metadata-update commit. */ private def replayEvolveSchema(spark: SparkSession, commit: WriteCommit, tablePath: String): Unit = { - import org.apache.spark.sql.delta.DeltaLog - DeltaLog.clearCache() - val deltaLog = DeltaLog.forTable(spark, tablePath) - val txn = deltaLog.startTransaction() - val currentMetadata = txn.metadata - var fields = currentMetadata.schema.fields.toBuffer + val log = DeltaHarness.get.openLog(spark, tablePath) + val metaNode = Option(JsonUtil.mapper.readTree(log.update().metadataJson).get("metaData")) + .getOrElse(throw new IllegalStateException( + s"evolve_schema replay: no metaData action in current snapshot of $tablePath")) + val schemaString = Option(metaNode.get("schemaString")) + .getOrElse(throw new IllegalStateException( + s"evolve_schema replay: metaData has no schemaString in $tablePath")) + val currentSchema = + DataType.fromJson(schemaString.asText()).asInstanceOf[StructType] + var fields = currentSchema.fields.toBuffer commit.addColumns.foreach { cols => cols.asInstanceOf[Seq[Map[String, Any]]].foreach { col => @@ -357,9 +361,8 @@ object WriteSpecValidator { fields = fields.filterNot(f => drops.contains(f.name)) } - txn.updateMetadata(currentMetadata.copy(schemaString = StructType(fields.toSeq).json)) - txn.commit(Seq.empty, org.apache.spark.sql.delta.DeltaOperations.ManualUpdate) - DeltaLog.clearCache() + DeltaHarness.get.commit(spark, tablePath, + CommitRequest(schemaJson = Some(StructType(fields.toSeq).json))) } /** diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala index 87fae79a..24152bc3 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala @@ -18,6 +18,44 @@ package io.delta.workload.deltaharness import org.apache.spark.sql.{DataFrame, SparkSession} +import io.delta.workload.{AddDomainMetadata, AppTxn} + +/** + * A single add-file action for a low-level commit, fully resolved by the caller: `path` is the + * in-table relative path and `size` is the on-disk size of the copied file. + */ +case class CommitAddFile( + path: String, + partitionValues: Map[String, String], + size: Long, + dataChange: Boolean) + +/** A single remove-file action for a low-level commit: `path` is the in-table relative path. */ +case class CommitRemoveFile(path: String, dataChange: Boolean) + +/** + * A platform-neutral description of a low-level commit. Carries only plain Scala / Spark-SQL + * types and the generator's neutral action case classes — never `org.apache.spark.sql.delta.*`. + * + * @param schemaJson `StructType.json` to set as `metadata.schemaString`, or None to keep + * the current schema + * @param properties properties merged into the current `metadata.configuration`, or None + * to keep the current configuration + * @param setTransaction optional `SetTransaction` (appId/version) action + * @param addFiles resolved add-file actions (in-table path + computed size) + * @param removeFiles resolved remove-file actions to tombstone + * @param addDomainMetadata domain-metadata entries to add + * @param removeDomainMetadata domain names to tombstone + */ +case class CommitRequest( + schemaJson: Option[String] = None, + properties: Option[Map[String, String]] = None, + setTransaction: Option[AppTxn] = None, + addFiles: Seq[CommitAddFile] = Nil, + removeFiles: Seq[CommitRemoveFile] = Nil, + addDomainMetadata: Seq[AddDomainMetadata] = Nil, + removeDomainMetadata: Seq[String] = Nil) + /** * Platform-specific backing for Delta-internal access. * @@ -33,6 +71,15 @@ trait DeltaHarness { * cache management an adapter-internal concern. */ def openLog(spark: SparkSession, tablePath: String): LogView + + /** + * Open a transaction at `tablePath`, apply the metadata update (schema/properties) if present, + * and commit the described actions using the platform's `DeltaOperations.ManualUpdate`. + * + * Implementations MUST clear any internal DeltaLog cache before and after committing, mirroring + * the `openLog` cache-clearing contract so consumers always see a fresh view. + */ + def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Unit } trait LogView { diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala index 108befc6..f8641d45 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala @@ -16,14 +16,61 @@ package io.delta.workload.deltaharness +import scala.collection.mutable + import org.apache.spark.sql.{DataFrame, SparkSession} -import org.apache.spark.sql.delta.{DeltaLog, Snapshot} +import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, Snapshot} +import org.apache.spark.sql.delta.actions.{Action, AddFile, DomainMetadata, RemoveFile, SetTransaction} class DeltaSparkHarness extends DeltaHarness { override def openLog(spark: SparkSession, tablePath: String): LogView = { DeltaLog.clearCache() new DeltaSparkLogView(DeltaLog.forTable(spark, tablePath)) } + + override def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Unit = { + DeltaLog.clearCache() + val deltaLog = DeltaLog.forTable(spark, tablePath) + val transaction = deltaLog.startTransaction() + val actions = mutable.ArrayBuffer[Action]() + + if (req.schemaJson.isDefined || req.properties.isDefined) { + val currentMetadata = transaction.metadata + val newSchemaString = req.schemaJson.getOrElse(currentMetadata.schemaString) + val newConfig = req.properties + .map(props => currentMetadata.configuration ++ props) + .getOrElse(currentMetadata.configuration) + transaction.updateMetadata( + currentMetadata.copy(schemaString = newSchemaString, configuration = newConfig)) + } + + req.setTransaction.foreach { t => + actions += SetTransaction(t.appId, t.version, Some(System.currentTimeMillis())) + } + req.addFiles.foreach { f => + actions += AddFile( + path = f.path, + partitionValues = f.partitionValues, + size = f.size, + modificationTime = System.currentTimeMillis(), + dataChange = f.dataChange) + } + req.removeFiles.foreach { f => + actions += RemoveFile( + path = f.path, + deletionTimestamp = Some(System.currentTimeMillis()), + dataChange = f.dataChange) + } + req.addDomainMetadata.foreach { dm => + actions += DomainMetadata(dm.domain, dm.configuration, removed = false) + } + req.removeDomainMetadata.foreach { domain => + actions += DomainMetadata(domain, "", removed = true) + } + + transaction.commit(actions.toSeq, DeltaOperations.ManualUpdate) + DeltaLog.clearCache() + } } private class DeltaSparkLogView(inner: DeltaLog) extends LogView { From 93c83eb9fa9af5bb85f13eb0235ac96eb81a7ba3 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 04:58:19 +0000 Subject: [PATCH 03/25] refactor: unify write into a single spec + validator, replay low-level commits via the engine Make "everything a spec": one WorkloadValidator validates read/snapshot/write, dispatching on `type`. Read/snapshot specs carry an optional `writeSpec` pointer marking them write-derived (reconstruct the table by replaying the sibling write spec, then validate portably) vs read-only (validate the captured delta/ exactly). Deletes the separate WriteSpecValidator. Model the write spec as a sealed WriteCommit ADT (one *Commit case class per op), adding replace_table (incl. replace-as-select) and removing restore. Low-level `commit` adds carry logical rows materialized to Parquet and written through the engine's OptimisticTransaction.writeFiles, so column mapping / partitioning / stats are engine-derived, not caller-supplied; removes reference a prior add by commit ordinal (== table version), resolved by reading _delta_log. The DeltaHarness SPI gains commitWithData to hide delta-spark behind a neutral CommitRequest. Review fixes folded in: - replay schema handling reuses Spark's StructType.toDDL/DataType.fromJson instead of a hand-rolled type printer (lossless decimal/nested/quoting; deletes ~45 LOC) - enforce commit-index == table-version at the commitOp that establishes it, plus a non-empty-commit guard, so desync fails at the offending op - columnMapping is credited to the reader axis only when reader>=2 AND writer>=5 jointly (mirrors delta-spark implicitlySupportedFeatures) - read-only specs with no captured table now error instead of silently passing; per-spec errors keep the full throwable; cleanup narrows to NonFatal - document the write spec type and DSL (README, harness/authoring guides); fix stale scaladoc Co-authored-by: Ryan Liao Co-authored-by: Isaac --- workload-generator/README.md | 5 +- workload-generator/docs/authoring-guide.md | 17 + .../docs/harness-implementation-guide.md | 6 + workload-generator/docs/spec-reference.md | 95 ++-- .../scala/io/delta/workload/JsonUtil.scala | 175 ++++++- .../io/delta/workload/LowLevelCommit.scala | 90 ---- .../scala/io/delta/workload/ReadCapture.scala | 8 +- .../scala/io/delta/workload/RowParquet.scala | 142 +++++ .../io/delta/workload/SnapshotCapture.scala | 4 +- .../io/delta/workload/WorkloadGenerator.scala | 180 +++++-- .../io/delta/workload/WorkloadValidator.scala | 495 ++++++++++++++++-- .../io/delta/workload/WriteSpecCapture.scala | 167 +++--- .../delta/workload/WriteSpecValidator.scala | 463 ---------------- .../workload/deltaharness/DeltaHarness.scala | 40 +- .../deltaharness/DeltaSparkHarness.scala | 69 +-- .../workload/WorkloadValidatorSuite.scala | 20 + .../workload/WorkloadValidatorUnitSuite.scala | 139 +++++ .../workload/tables/WriteBasicSuite.scala | 86 ++- .../workload/tables/WriteCommitSuite.scala | 152 +++++- .../workload/tables/WriteSequencesSuite.scala | 118 +++++ .../workload/tables/WriteTypesSuite.scala | 131 +++++ 21 files changed, 1726 insertions(+), 876 deletions(-) delete mode 100644 workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala create mode 100644 workload-generator/src/main/scala/io/delta/workload/RowParquet.scala delete mode 100644 workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorUnitSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteSequencesSuite.scala create mode 100644 workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala diff --git a/workload-generator/README.md b/workload-generator/README.md index 9e50f31b..213b4a7e 100644 --- a/workload-generator/README.md +++ b/workload-generator/README.md @@ -8,7 +8,7 @@ Write a script that creates Delta tables with normal SQL, declare what specs to | Document | Description | |----------|-------------| -| **[Spec Format Reference](docs/spec-reference.md)** | JSON schema for the `read` and `snapshot` spec types, with examples | +| **[Spec Format Reference](docs/spec-reference.md)** | JSON schema for the `read`, `snapshot`, and `write` spec types, with examples | | **[Harness Implementation Guide](docs/harness-implementation-guide.md)** | Step-by-step guide to build a test harness that runs workloads against your engine, with a Rust example | | **[Authoring Guide](docs/authoring-guide.md)** | How to write new workload suites, patterns, recipes, and debugging tips | @@ -157,6 +157,9 @@ See the [Spec Format Reference](docs/spec-reference.md) for the complete JSON sc |------|--------------|---------| | **Read** | Data reads with time travel, predicates, column projection, data skipping | [Reference](docs/spec-reference.md#read-spec) | | **Snapshot** | Protocol and metadata reconstruction from log replay | [Reference](docs/spec-reference.md#snapshot-spec) | +| **Write** | Table construction via a portable sequence of commits (create/replace/insert/delete/update/schema-evolution/properties and low-level `commit`s), replayed into a fresh table | [Reference](docs/spec-reference.md#write-spec) | + +The write DSL (`createTableOp`, `insertOp`, `commitOp`, …) is exposed by `WorkloadOps`; see the [Authoring Guide](docs/authoring-guide.md) for how to author write workloads. A `read`/`snapshot` spec captured against a write-built table carries a `writeSpec` pointer and is validated *portably* (replay the sibling write spec into a fresh table, then compare). ## Workload Suites diff --git a/workload-generator/docs/authoring-guide.md b/workload-generator/docs/authoring-guide.md index dab09d1e..ea3af2e7 100644 --- a/workload-generator/docs/authoring-guide.md +++ b/workload-generator/docs/authoring-guide.md @@ -92,9 +92,26 @@ Inside a test body, these methods are available directly (via `WorkloadOps` trai | SQL | `sql(statement)` | — | | Table handles | `registerTable(name)`, `registerTableFromPath(path)` | → `TableHandle` | | Read specs | `readSpec(t, ...)`, `snapshotSpec(t, ...)` | `TableHandle` → `SpecRef` | +| Write specs | `createTableOp`, `replaceTableOp`, `insertOp`, `deleteOp`, `updateOp`, `addColumnsOp`, `renameColumnOp`, `dropColumnsOp`, `setPropertiesOp`, `unsetPropertiesOp`, `commitOp` (low-level) | → `WriteHandle`; `registerWriteSpec(w)` → `TableHandle` | | Checkpointing | `forceCheckpoint(tableName)` — triggers a checkpoint via DeltaLog | — | | Mutations | `mutateTable(t) { dir => ... }`, `modifyCommitActions(t, version) { ... }` | `TableHandle` | +### Authoring a write workload + +A write workload builds a table through DSL ops instead of raw `sql(...)`, producing a portable `write` spec (`specs/_write.json`) plus any `read`/`snapshot` specs derived from the built table. Start with `createTableOp` (or `replaceTableOp`), apply ops, then `registerWriteSpec(w)` to get a `TableHandle` you can attach reads/snapshots to: + +```scala +val w = createTableOp("tbl", schema = "id INT, name STRING") +insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) +// Low-level commit: logical rows are written through the engine (stats/partitionValues derived). +commitOp(w, addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) +val t = registerWriteSpec(w) +readSpec(t, name = "read_all") +snapshotSpec(t) +``` + +`commitOp` returns the commit's ordinal (== table version); a later `removeFiles = Some(Seq(ordinal))` tombstones the files that commit added. See `WriteBasicSuite`/`WriteCommitSuite`/`WriteSequencesSuite` for worked examples and the [Write Spec reference](spec-reference.md#write-spec) for the on-disk format. + --- ## Creating Tables diff --git a/workload-generator/docs/harness-implementation-guide.md b/workload-generator/docs/harness-implementation-guide.md index a24ac1b2..f04bde52 100644 --- a/workload-generator/docs/harness-implementation-guide.md +++ b/workload-generator/docs/harness-implementation-guide.md @@ -207,6 +207,12 @@ Any spec type can have `"expectedError"` instead of `"expected"`: Run the operation and assert it fails. Matching the exact error code is ideal but optional — just asserting failure is a valid starting point. +### Write Specs + +A `write` spec (`specs/_write.json`) is a portable recipe for *building* a table: an ordered list of `commits` your harness replays into a fresh table, then compares against `expected/_write/`. High-level commits (create/replace/insert/delete/update/schema-evolution/properties) are replayed through your engine's normal write APIs; low-level `commit`s carry raw actions (the data files hold logical rows your engine writes through its own write path, so physical names/stats are engine-derived, and removes reference a prior add by commit ordinal == table version). See the [Write Spec reference](spec-reference.md#write-spec) for the field-level contract. + +Because a write spec is replayed (not byte-compared), comparison is **portable**: rows-only data checks, capability-based protocol comparison (your table may use explicit table features where Spark used legacy versions), and column-mapping-normalized schema. A `read`/`snapshot` spec carrying a `writeSpec` pointer is *write-derived* — reconstruct the table by replaying that sibling write spec first, then validate the read/snapshot against it (portably), rather than against a captured `delta/` directory. + --- ## Step 4: Incremental Adoption diff --git a/workload-generator/docs/spec-reference.md b/workload-generator/docs/spec-reference.md index 7c56768f..5609ad20 100644 --- a/workload-generator/docs/spec-reference.md +++ b/workload-generator/docs/spec-reference.md @@ -81,6 +81,7 @@ Tests reading data from a Delta table with optional time travel, predicate pushd | Field | Type | Required | Description | |-------|------|----------|-------------| | `type` | `string` | yes | Always `"read"` | +| `writeSpec` | `string` | no | If present, this read is *write-derived*: the named sibling write spec (`specs/`) must be replayed into a fresh table first, then this read validated against it. Absent ⇒ read-only (validate against the captured table). | | `version` | `long` | no | Time travel to this version | | `timestamp` | `string` | no | Time travel to this timestamp (format: `yyyy-MM-dd HH:mm:ss.SSS`) | | `predicate` | `string` | no | SQL WHERE clause to apply (e.g., `"id > 5"`) | @@ -244,6 +245,7 @@ Tests constructing a table snapshot at a given version, validating the protocol | Field | Type | Required | Description | |-------|------|----------|-------------| | `type` | `string` | yes | Always `"snapshot"` | +| `writeSpec` | `string` | no | If present, write-derived: replay the named sibling write spec first, then validate this snapshot against it (portable comparison). Absent ⇒ validate against the captured table. | | `version` | `long` | no | Snapshot at this version (latest if omitted and no timestamp) | | `timestamp` | `string` | no | Snapshot at this timestamp | | `expected` | `SnapshotExpected` | no | Present on success | @@ -354,9 +356,9 @@ These are the raw Delta protocol and metadata JSON structures — not simplified Tests Delta writer implementations by providing a sequence of write operations to replay. Unlike read specs (which verify reading an existing table), a write spec describes *how to construct* a table from scratch. -A write spec is a portable, implementation-agnostic recipe: it specifies what operations to perform (create table, insert, delete, update, etc.) declaratively, so any conforming Delta writer can interpret and execute it. After replaying all commits, the resulting table is compared against the expected data under `expected/latest/`. +A write spec is a portable, implementation-agnostic recipe: it specifies what operations to perform (create table, insert, delete, update, etc.) declaratively, so any conforming Delta writer can interpret and execute it. After replaying all commits, the resulting table is compared against the expected data under `expected/_write/`. -The write spec is written to `write_spec.json` at the workload output root (not under `specs/`). The read and snapshot specs declared alongside a write workload are captured normally under `specs/`. +A write spec is **just another spec**: it lives at `specs/_write.json` alongside the read and snapshot specs of the same workload, and is dispatched by its `type` field like any other. The read/snapshot specs of a write workload carry a `writeSpec` pointer back to it (see below), marking them *write-derived*. ### Fields @@ -375,21 +377,27 @@ These map to SQL-like operations. The writer translates them to appropriate Delt | Operation | Fields | Description | |-----------|--------|-------------| -| `create_table` | `schema`, `partitionColumns?`, `properties?`, `dataFiles?` | Create a new table with the given schema | -| `insert` | `dataFiles?` | Append rows from data files | +| `create_table` | `schema`, `partitionColumns?`, `properties?` | Create a new table with the given schema | +| `replace_table` | `schema`, `partitionColumns?`, `properties?`, `dataFiles?` | Replace the table's schema/partitioning/properties (and all data). With `dataFiles` it is a replace-as-select: a single commit that also writes the bundled data. | +| `insert` | `dataFiles?` | Append the rows in the bundled Parquet data files | | `update` | `predicate`, `set` | Update rows matching predicate | | `delete` | `predicate` | Delete rows matching predicate | | `evolve_schema` | `addColumns?`, `renameColumns?`, `dropColumns?` | Modify table schema | | `update_properties` | `set?`, `remove?` | Modify table properties | -| `restore` | `version` | Restore table to a previous version | -The reference validator does not replay `restore`: an independent replay generates different file paths, so RESTORE-to-version would reference paths that do not exist in the replayed table. Conforming consumers may implement it. +The data for `insert` and `replace_table` is bundled as Parquet under `data/commit_N/` (the +generator's authoring API accepts in-memory rows, but the spec always stores Parquet). A +consumer replays by reading those files — `insert` appends them; `replace_table` performs a +single replace-as-select from them. + +`RENAME COLUMN` / `DROP COLUMN` in `evolve_schema` require the table to have column mapping +enabled (a Delta/Spark constraint). #### Low-Level Operation | Operation | Fields | Description | |-----------|--------|-------------| -| `commit` | `schema?`, `tableProperties?`, `txn?`, `addFiles?`, `removeFiles?`, `addDomainMetadata?`, `removeDomainMetadata?` | Directly specify Delta actions. `addFiles[].dataFile` must reference Parquet already in the table (e.g. produced by a prior `insert`); capture copies it under `data/commit_N/`. `removeFiles[].path` must target a file added by a prior low-level `commit`; paths from `insert`/`create_table` are writer-generated and not stable across replay. Deletion vectors are not honored. | +| `commit` | `schema?`, `tableProperties?`, `txn?`, `addFiles?`, `removeFiles?`, `addDomainMetadata?`, `removeDomainMetadata?` | Bundle raw Delta actions in one commit. Each `addFiles[]` entry's logical rows are bundled as Parquet (`dataFile`); a consumer writes them **through its own engine write path** (so column mapping, partitioning, and stats are handled — physical names/stats are not in the spec) and commits the resulting `AddFile`s alongside the `txn`/`DomainMetadata`/schema/property changes. `removeFiles[]` tombstone a file added by a prior low-level `commit`, referenced by that commit's ordinal (`addedAtCommit`); since the engine assigns paths per table, the consumer resolves the ordinal to its own table's path. Deletion vectors are out of scope. | ### WriteCommit Fields @@ -399,14 +407,13 @@ The reference validator does not replay `restore`: an independent replay generat | `schema` | `object` | Table schema in Delta JSON format | | `partitionColumns` | `string[]` | Partition column names | | `properties` | `map` | Table properties (e.g., `delta.enableDeletionVectors`) | -| `dataFiles` | `string[]` | Relative paths to Parquet data files under `data/`. Files for a partitioned commit keep their `col=val/` directory prefix | +| `dataFiles` | `string[]` | Relative paths under `data/` to the Parquet holding this op's rows (full rows incl. partition columns). For `insert` and `replace_table` | | `predicate` | `string` | SQL WHERE clause for update/delete | | `set` | `map` | Column assignments for update (column → expression) or properties to set | | `remove` | `string[]` | Property names to remove | | `addColumns` | `object[]` | Columns to add (each with `name`, `type`, `nullable`) | | `renameColumns` | `map` | Column renames (old name → new name) | | `dropColumns` | `string[]` | Column names to drop | -| `version` | `long` | Target version for restore | | `tableProperties` | `map` | Properties for low-level commit | | `txn` | `AppTxn` | Application transaction for idempotent writes | | `addFiles` | `AddFileAction[]` | Files to add (low-level) | @@ -416,24 +423,24 @@ The reference validator does not replay `restore`: an independent replay generat ### AddFileAction (Low-Level) +Carries a pointer to a Parquet of LOGICAL rows; the consumer writes it through its own engine +write path (physical names, partitioning, and stats are derived per table, not stored). + | Field | Type | Description | |-------|------|-------------| -| `dataFile` | `string` | Relative path to Parquet file under `data/` | -| `partitionValues` | `map?` | Partition values for this file | +| `dataFile` | `string` | Relative path under `data/commit_N/` to the Parquet of logical rows (full rows incl. partition columns) | | `dataChange` | `boolean?` | Whether this is a data change (default true) | ### RemoveFileAction (Low-Level) | Field | Type | Description | |-------|------|-------------| -| `path` | `string` | Path of file to remove | +| `addedAtCommit` | `int` | Ordinal (== table version) of the prior low-level `commit` that added the file(s) to tombstone; the consumer resolves it to its own table's path(s) | | `dataChange` | `boolean?` | Whether this is a data change (default true) | -`path` must target a file added by a prior low-level `commit`; paths from `insert`/`create_table` are writer-generated and not stable across replay. - ### Expected Data -The final table state after replaying all commits is captured under `expected/latest/`: +The final table state after replaying all commits is captured under `expected/_write/`: - `table_content/` — Parquet files with the expected rows - `table_version_metadata.json` — the latest snapshot's `protocol` and `metadata` @@ -442,30 +449,31 @@ The final table state after replaying all commits is captured under `expected/la The capture comes from one writer, but a write spec must validate against any conforming Delta writer, so a consumer compares the replayed snapshot against `expected` by capability rather than byte equality: -- **`protocol`** — feature-superset plus version-floor, not exact. The replay must satisfy `minReaderVersion >= expected` and `minWriterVersion >= expected`, and its `readerFeatures`/`writerFeatures` must be supersets of the expected sets (missing arrays count as empty; membership is order-insensitive). A higher version or extra features is allowed. -- **`configuration`** — only the keys the write spec's own commits declared are checked: `create_table.properties` plus `update_properties.set`, minus `update_properties.remove`. Each declared key must be present and equal; each removed key must be absent. Engine-injected default properties are ignored. If the spec declared no properties, nothing is checked. +- **`protocol`** — feature-superset plus version-floor, not exact, over the EFFECTIVE feature sets (explicit `readerFeatures`/`writerFeatures` unioned with the features implied by the protocol version). The replay must satisfy `minReaderVersion >= expected` and `minWriterVersion >= expected` and be a feature superset. A higher version or extra features is allowed. +- **`configuration`** — only the keys the write spec's own commits declared are checked: `create_table`/`replace_table` `properties` plus `update_properties.set`, minus `update_properties.remove` (a `replace_table` resets the declared set). Each declared key must be present and equal; each removed key must be absent. Engine-injected defaults are ignored. - **rows** (`table_content/`) — multiset comparison; order is irrelevant, but each row must appear the correct number of times. -`schemaString`, `partitionColumns`, and `format` are compared exactly. +`schemaString` (with per-field column-mapping `physicalName`/`id` normalized out, since those are minted per table), `partitionColumns`, and `format` are compared. ### Directory Structure ``` / -├── write_spec.json # The write spec -├── data/ # Data files referenced by commits -│ ├── commit_0/ -│ │ └── part-0000-xxx.parquet +├── delta/ # the captured table (for read-only validation) +├── specs/ +│ ├── _read*.json +│ ├── _snapshot.json +│ └── _write.json # the write spec — just another spec +├── data/ # Parquet referenced by write commits │ ├── commit_1/ -│ │ └── part-0000-yyy.parquet +│ │ └── part-00000.parquet │ └── ... ├── expected/ -│ ├── latest/ +│ ├── _write/ # write spec's final state │ │ ├── table_content/ │ │ └── table_version_metadata.json │ └── / # expected_data for each read spec │ └── expected_data/ -├── specs/ # Read/snapshot specs for this workload └── table_info.json ``` @@ -571,10 +579,33 @@ The capture comes from one writer, but a write spec must validate against any co }, { "operation": "insert", - "dataFiles": [ - "data/commit_1/region=east/part-0000-abc.parquet", - "data/commit_1/region=west/part-0000-def.parquet" - ] + "dataFiles": ["data/commit_1/part-00000.parquet"] + } + ] +} +``` + +The bundled Parquet contains the full rows including the partition columns, so a consumer +appends it and lets its own writer lay out the partition directories. + +**Replace-as-select (replace schema + data in one commit):** + +```json +{ + "type": "write", + "commits": [ + { "operation": "create_table", "schema": { } }, + { "operation": "insert", "dataFiles": ["data/commit_1/part-00000.parquet"] }, + { + "operation": "replace_table", + "schema": { + "type": "struct", + "fields": [ + { "name": "id", "type": "integer", "nullable": true, "metadata": {} }, + { "name": "label", "type": "string", "nullable": true, "metadata": {} } + ] + }, + "dataFiles": ["data/commit_2/part-00000.parquet"] } ] } @@ -593,9 +624,7 @@ The capture comes from one writer, but a write spec must validate against any co }, { "operation": "commit", - "addFiles": [ - { "dataFile": "data/commit_1/part-0000-abc.parquet", "partitionValues": {}, "dataChange": true } - ], + "addFiles": [ { "dataFile": "data/commit_1/add_0.parquet" } ], "addDomainMetadata": [ { "domain": "myApp.config", "configuration": "{\"version\": 1}" } ] @@ -614,7 +643,7 @@ The capture comes from one writer, but a write spec must validate against any co { "operation": "commit", "txn": { "appId": "streaming-job-1", "version": 42 }, - "addFiles": [ { "dataFile": "data/commit_1/part-0000-abc.parquet" } ] + "addFiles": [ { "dataFile": "data/commit_1/add_0.parquet" } ] } ] } diff --git a/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala b/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala index b06b55bd..de080559 100644 --- a/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala +++ b/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala @@ -49,9 +49,11 @@ case class SnapshotExpected(protocol: Any, metadata: Any) // Spec case classes // ============================================================================= -@JsonPropertyOrder(Array("type", "version", "timestamp", "predicate", "columns", "expected", "expectedError")) +@JsonPropertyOrder(Array("type", "writeSpec", "version", "timestamp", "predicate", "columns", + "expected", "expectedError")) @JsonInclude(JsonInclude.Include.NON_ABSENT) case class ReadSpec( + writeSpec: Option[String] = None, version: Option[Long] = None, timestamp: Option[String] = None, predicate: Option[String] = None, @@ -61,9 +63,10 @@ case class ReadSpec( val `type`: String = "read" } -@JsonPropertyOrder(Array("type", "version", "timestamp", "expected", "expectedError")) +@JsonPropertyOrder(Array("type", "writeSpec", "version", "timestamp", "expected", "expectedError")) @JsonInclude(JsonInclude.Include.NON_ABSENT) case class SnapshotSpec( + writeSpec: Option[String] = None, version: Option[Long] = None, timestamp: Option[String] = None, expected: Option[SnapshotExpected] = None, @@ -118,19 +121,37 @@ case class TableInfo( // Low-level action types // ============================================================================= -/** Add file action for low-level commits. */ -@JsonPropertyOrder(Array("dataFile", "partitionValues", "dataChange")) +/** + * Add-file action serialized in a low-level `commit`. `dataFile` points to a Parquet of LOGICAL + * rows under `data/commit_N/`; on replay the engine writes it into the table (column-mapping- and + * partition-aware, stats computed), so physical names/stats are derived per table, never stored. + */ +@JsonPropertyOrder(Array("dataFile", "dataChange")) @JsonInclude(JsonInclude.Include.NON_ABSENT) case class AddFileAction( dataFile: String, - partitionValues: Option[Map[String, String]] = None, dataChange: Option[Boolean] = None) -/** Remove file action for low-level commits. */ -@JsonPropertyOrder(Array("path", "dataChange")) +/** + * Remove-file action (tombstone) for low-level commits. References a prior low-level add by the + * commit ordinal that produced it (`addedAtCommit`); the engine assigns file paths per table, so + * the tombstone is resolved to the actual path(s) at replay. The tombstone inherits the live + * add's partitionValues/size/stats (extendedFileMetadata). + */ +@JsonPropertyOrder(Array("addedAtCommit", "dataChange")) @JsonInclude(JsonInclude.Include.NON_ABSENT) case class RemoveFileAction( - path: String, + addedAtCommit: Int, + dataChange: Option[Boolean] = None) + +/** + * Capture-time input for a low-level add: the caller supplies LOGICAL `rows` (matching the table's + * schema, including any partition columns). The framework materializes them and writes them + * through the engine; physical layout / stats are engine-derived. Not serialized — it only + * crosses the `commitOp` boundary. + */ +case class AddFileInput( + rows: Seq[Map[String, Any]], dataChange: Option[Boolean] = None) /** Application transaction for idempotent low-level commits. */ @@ -148,44 +169,140 @@ case class AddDomainMetadata( // ============================================================================= /** - * A single commit in a write spec. There are two categories of operation: - * high-level (SQL semantics, e.g. create_table/insert/update/delete) and the - * low-level `commit` (raw Delta actions). Only the fields relevant to a given - * operation are populated; all others are absent in the serialized JSON. + * A single commit in a write spec. Each operation is its own case class carrying only the + * fields it uses, so illegal field combinations are unrepresentable and `match`es over the + * hierarchy are exhaustive. The `operation` discriminator is written by Jackson as the leading + * JSON property (`@JsonTypeInfo`); the in-memory `operation` accessor is `@JsonIgnore`d so it is + * not also serialized as a duplicate field. + * + * High-level ops (create_table/replace_table/insert/update/delete/evolve_schema/ + * update_properties) replay from their parameters; the low-level [[LowLevelCommitOp]] replays its + * recorded raw actions. */ -@JsonPropertyOrder(Array("operation", "schema", "partitionColumns", - "properties", "dataFiles", "predicate", "set", "remove", - "addColumns", "renameColumns", "dropColumns", "version", - "tableProperties", "txn", "addFiles", "removeFiles", - "addDomainMetadata", "removeDomainMetadata")) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, + property = "operation") +@JsonSubTypes(Array( + new JsonSubTypes.Type(value = classOf[CreateTableCommit], name = "create_table"), + new JsonSubTypes.Type(value = classOf[ReplaceTableCommit], name = "replace_table"), + new JsonSubTypes.Type(value = classOf[InsertCommit], name = "insert"), + new JsonSubTypes.Type(value = classOf[DeleteCommit], name = "delete"), + new JsonSubTypes.Type(value = classOf[UpdateCommit], name = "update"), + new JsonSubTypes.Type(value = classOf[EvolveSchemaCommit], name = "evolve_schema"), + new JsonSubTypes.Type(value = classOf[UpdatePropertiesCommit], name = "update_properties"), + new JsonSubTypes.Type(value = classOf[LowLevelCommitOp], name = "commit"))) +sealed trait WriteCommit { + @JsonIgnore def operation: String +} + +// ---- High-level operations (replayed from parameters) ---- + +@JsonPropertyOrder(Array("schema", "partitionColumns", "properties")) @JsonInclude(JsonInclude.Include.NON_ABSENT) -case class WriteCommit( - operation: String, - // High-level fields - schema: Option[Any] = None, +case class CreateTableCommit( + schema: Any, + partitionColumns: Option[Seq[String]] = None, + properties: Option[Map[String, String]] = None) extends WriteCommit { + @JsonIgnore val operation = "create_table" +} + +/** + * `CREATE OR REPLACE TABLE` — replaces schema/partitioning/properties and all data. With `rows` + * it is a replace-as-select (RTAS): a single commit that replaces the table and writes the rows + * (replayed via `CREATE OR REPLACE TABLE … USING delta … AS SELECT … FROM VALUES`). + */ +@JsonPropertyOrder(Array("schema", "partitionColumns", "properties", "dataFiles")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class ReplaceTableCommit( + schema: Any, partitionColumns: Option[Seq[String]] = None, properties: Option[Map[String, String]] = None, - dataFiles: Option[Seq[String]] = None, - predicate: Option[String] = None, - set: Option[Map[String, String]] = None, - remove: Option[Seq[String]] = None, + dataFiles: Option[Seq[String]] = None) extends WriteCommit { + @JsonIgnore val operation = "replace_table" +} + +@JsonPropertyOrder(Array("dataFiles")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class InsertCommit(dataFiles: Option[Seq[String]] = None) extends WriteCommit { + @JsonIgnore val operation = "insert" +} + +@JsonPropertyOrder(Array("predicate")) +case class DeleteCommit(predicate: String) extends WriteCommit { + @JsonIgnore val operation = "delete" +} + +@JsonPropertyOrder(Array("predicate", "set")) +case class UpdateCommit(predicate: String, set: Map[String, String]) extends WriteCommit { + @JsonIgnore val operation = "update" +} + +@JsonPropertyOrder(Array("addColumns", "renameColumns", "dropColumns")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class EvolveSchemaCommit( addColumns: Option[Any] = None, renameColumns: Option[Map[String, String]] = None, - dropColumns: Option[Seq[String]] = None, - version: Option[Long] = None, - // Low-level commit fields (operation = "commit") + dropColumns: Option[Seq[String]] = None) extends WriteCommit { + @JsonIgnore val operation = "evolve_schema" +} + +@JsonPropertyOrder(Array("set", "remove")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class UpdatePropertiesCommit( + set: Option[Map[String, String]] = None, + remove: Option[Seq[String]] = None) extends WriteCommit { + @JsonIgnore val operation = "update_properties" +} + +// ---- Low-level operation (raw Delta actions; data written via the engine on replay) ---- + +@JsonPropertyOrder(Array("schema", "tableProperties", "txn", "addFiles", "removeFiles", + "addDomainMetadata", "removeDomainMetadata")) +@JsonInclude(JsonInclude.Include.NON_ABSENT) +case class LowLevelCommitOp( + schema: Option[Any] = None, tableProperties: Option[Map[String, String]] = None, txn: Option[AppTxn] = None, addFiles: Option[Seq[AddFileAction]] = None, removeFiles: Option[Seq[RemoveFileAction]] = None, addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, - removeDomainMetadata: Option[Seq[String]] = None) + removeDomainMetadata: Option[Seq[String]] = None) extends WriteCommit { + @JsonIgnore val operation = "commit" +} @JsonPropertyOrder(Array("type", "commits")) case class WriteSpec(commits: Seq[WriteCommit]) { val `type`: String = "write" } +/** + * Layout conventions for a workload output directory, shared by capture and replay so the + * "commit index -> data directory" decision lives in one place (not duplicated across + * [[WriteSpecBuilder]] and [[WorkloadValidator]]). + */ +private[workload] object SpecLayout { + /** Relative path (from the output dir) of the data directory for commit `idx`. */ + def commitDataDir(idx: Int): String = s"data/commit_$idx" + + /** Relative path (from the output dir) of `name` under commit `idx`'s data directory. */ + def commitDataFile(idx: Int, name: String): String = s"${commitDataDir(idx)}/$name" + + /** + * The in-table `AddFile.path`s a commit produced, read from its `_delta_log/.json`. + * Because commit index == table version, a low-level remove's `addedAtCommit` ordinal is the + * version whose adds it tombstones — resolved here against the actual (engine-assigned) paths. + */ + def addPathsAt(tablePath: java.nio.file.Path, version: Int): Seq[String] = { + val commitFile = tablePath.resolve("_delta_log").resolve(f"$version%020d.json") + if (!java.nio.file.Files.exists(commitFile)) return Seq.empty + new String(java.nio.file.Files.readAllBytes(commitFile), "UTF-8") + .split("\n").filter(_.trim.nonEmpty) + .flatMap { line => + val node = JsonUtil.mapper.readTree(line) + Option(node.get("add")).map(_.get("path").asText()) + }.toSeq + } +} + // ============================================================================= // Delta log action case classes (for parsing commit JSON) // ============================================================================= diff --git a/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala b/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala deleted file mode 100644 index 330f83e0..00000000 --- a/workload-generator/src/main/scala/io/delta/workload/LowLevelCommit.scala +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload - -import java.nio.file.{Files, Path, Paths, StandardCopyOption} - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.types.StructType - -import io.delta.workload.deltaharness.{CommitAddFile, CommitRemoveFile, CommitRequest, DeltaHarness} - -/** - * Applies a low-level `commit` (raw Delta actions) by driving the DeltaLog transaction - * API directly. Shared by capture (`commitOp`, against the live table) and replay - * (`WriteSpecValidator`, against a fresh table) so both produce identical actions. - * - * Capture and replay differ only in where source files come from (`resolveDataFile`) and what - * in-table path each file gets (`assignPath`): capture assigns fresh names and records them so - * the spec stays consistent with the live log; replay reuses those recorded paths so both sides - * commit identical `AddFile.path`s. - * - * Deletion vectors are intentionally not honored: the spec carries no DV descriptor and - * none is emitted here. - */ -object LowLevelCommit { - - /** - * @param spark active session - * @param tablePath table to commit against - * @param resolveDataFile maps an `AddFileAction.dataFile` to the source Parquet on disk - * @param assignPath the in-table path each added file occupies (the emitted `AddFile.path`) - * @return the effective add files, each `dataFile` set to the in-table relative path used - */ - def apply( - spark: SparkSession, - tablePath: String, - schemaDDL: Option[String], - tableProperties: Option[Map[String, String]], - txn: Option[AppTxn], - addFiles: Option[Seq[AddFileAction]], - removeFiles: Option[Seq[RemoveFileAction]], - addDomainMetadata: Option[Seq[AddDomainMetadata]], - removeDomainMetadata: Option[Seq[String]], - resolveDataFile: String => Path, - assignPath: AddFileAction => String): Seq[AddFileAction] = { - val resolvedAdds = addFiles.getOrElse(Seq.empty).map { file => - val src = resolveDataFile(file.dataFile) - require(Files.exists(src), s"commit addFiles references missing data file: $src") - val relative = assignPath(file) - val dest = Paths.get(tablePath).resolve(relative) - if (dest.toAbsolutePath.normalize() != src.toAbsolutePath.normalize()) { - Files.createDirectories(dest.getParent) - Files.copy(src, dest, StandardCopyOption.REPLACE_EXISTING) - } - val add = CommitAddFile( - path = relative, - partitionValues = file.partitionValues.getOrElse(Map.empty), - size = Files.size(dest), - dataChange = file.dataChange.getOrElse(true)) - (add, file.copy(dataFile = relative)) - } - - val req = CommitRequest( - schemaJson = schemaDDL.map(ddl => StructType.fromDDL(ddl).json), - properties = tableProperties, - setTransaction = txn, - addFiles = resolvedAdds.map(_._1), - removeFiles = removeFiles.getOrElse(Seq.empty) - .map(f => CommitRemoveFile(f.path, f.dataChange.getOrElse(true))), - addDomainMetadata = addDomainMetadata.getOrElse(Seq.empty), - removeDomainMetadata = removeDomainMetadata.getOrElse(Seq.empty)) - DeltaHarness.get.commit(spark, tablePath, req) - - resolvedAdds.map(_._2) - } -} diff --git a/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala b/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala index 90cb999a..030fe94d 100644 --- a/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala @@ -32,7 +32,7 @@ object ReadCapture { outputDir: Path, specsDir: Path, name: String, predicate: Option[String] = None, version: Option[Long] = None, timestamp: Option[String] = None, columns: Option[Seq[String]] = None, - expectError: Option[String] = None): Unit = { + expectError: Option[String] = None, writeSpec: Option[String] = None): Unit = { val specName = s"${testId}_$name" require(!(version.isDefined && timestamp.isDefined), @@ -99,7 +99,7 @@ object ReadCapture { } } - val spec = ReadSpec(version, timestamp, predicate, columns, expected, expectedError) + val spec = ReadSpec(writeSpec, version, timestamp, predicate, columns, expected, expectedError) JsonUtil.writeSpec(specPath, spec) validateFromSpec(spark, tablePath, expectedDir, specPath) @@ -132,7 +132,7 @@ object ReadCapture { } def validateFromSpec(spark: SparkSession, tablePath: Path, - expectedDir: Path, specPath: Path): Unit = { + expectedDir: Path, specPath: Path, checkMetadata: Boolean = true): Unit = { val spec = JsonUtil.readReadSpec(specPath) val specName = specPath.getFileName.toString.stripSuffix(".json") @@ -176,7 +176,7 @@ object ReadCapture { specName) val expectedMetaPath = expectedDir.resolve("expected_metadata") - if (Files.exists(expectedMetaPath)) { + if (checkMetadata && Files.exists(expectedMetaPath)) { // Re-derive which AddFiles the current scan would touch, so we can // compare against what was captured at generation time. val harness = DeltaHarness.get diff --git a/workload-generator/src/main/scala/io/delta/workload/RowParquet.scala b/workload-generator/src/main/scala/io/delta/workload/RowParquet.scala new file mode 100644 index 00000000..4f48596f --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/RowParquet.scala @@ -0,0 +1,142 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload + +import java.nio.file.{Files, Path, StandardCopyOption} + +import scala.jdk.CollectionConverters._ + +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.types._ + +import io.delta.workload.deltaharness.DeltaHarness + +/** + * Materializes in-memory `rows` (the workload-generator's authoring surface) into Parquet + * files for the write spec. The API takes rows; the spec stores Parquet. Capture writes rows + * here at generation time — no Delta-log scan — and replay reads the produced files back. + */ +object RowParquet { + + /** + * The table's schema at `version` (latest if None). When `includePartition` is false the + * partition columns are dropped, matching the layout of a raw data file referenced by a + * low-level `AddFile` (partition values ride on the action, not the file). + */ + def schemaAt( + spark: SparkSession, tablePath: String, + version: Option[Long], includePartition: Boolean): StructType = { + val log = DeltaHarness.get.openLog(spark, tablePath) + val snapshot = version.map(log.getSnapshotAt).getOrElse(log.update()) + val meta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") + val full = DataType.fromJson(meta.get("schemaString").asText()).asInstanceOf[StructType] + if (includePartition) full + else { + val partCols = Option(meta.get("partitionColumns")) + .map(_.elements().asScala.map(_.asText()).toSet).getOrElse(Set.empty[String]) + StructType(full.fields.filterNot(f => partCols.contains(f.name))) + } + } + + /** + * Write `rows` to a single Parquet file at `dest`. Spark writes a directory of part-files, so + * we write to a temp dir with `coalesce(1)` and move the one part-file into place. + */ + def writeSingle( + spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]], dest: Path): Unit = { + val sparkRows = rows.map { r => + Row.fromSeq(schema.fields.map(f => coerce(r.get(f.name), f.dataType)).toSeq) + } + val tmp = Files.createTempDirectory("row-parquet") + try { + spark.createDataFrame(sparkRows.asJava, schema) + .coalesce(1).write.mode("overwrite").parquet(tmp.toString) + val part = Files.list(tmp).iterator().asScala.find { p => + val n = p.getFileName.toString + n.endsWith(".parquet") && !n.startsWith(".") + }.getOrElse(throw new IllegalStateException(s"no parquet part-file written under $tmp")) + Files.createDirectories(dest.getParent) + Files.move(part, dest, StandardCopyOption.REPLACE_EXISTING) + } finally { + try FileUtils.deleteDirectory(tmp.toFile) catch { case _: Throwable => } + } + } + + /** Write `rows` to a single Parquet file in a fresh temp directory; returns the file path. */ + def writeTemp(spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]]): Path = { + val dir = Files.createTempDirectory("row-parquet-rtas") + val dest = dir.resolve("part-00000.parquet") + writeSingle(spark, schema, rows, dest) + dest + } + + /** + * Coerce a row's value to the column's Spark type. `None`/`null` -> SQL NULL. Unknown types + * fail loud rather than passing the raw value through: a silent passthrough that Spark happens + * to accept could write a wrong-but-symmetric value (wrong on both capture AND replay), which + * the row comparison would not catch — a false pass. + */ + private def coerce(value: Option[Any], dataType: DataType): Any = + value.flatMap(Option(_)) match { + case None => null + case Some(v) => dataType match { + case _: IntegerType => v.asInstanceOf[Number].intValue() + case _: LongType => v.asInstanceOf[Number].longValue() + case _: ShortType => v.asInstanceOf[Number].shortValue() + case _: ByteType => v.asInstanceOf[Number].byteValue() + case _: DoubleType => v.asInstanceOf[Number].doubleValue() + case _: FloatType => v.asInstanceOf[Number].floatValue() + case _: BooleanType => v.asInstanceOf[Boolean] + case _: StringType => v.toString + case _: DateType => v match { + case s: String => java.sql.Date.valueOf(s) + case d: java.sql.Date => d + case n: Number => new java.sql.Date(n.longValue()) + case _ => unsupported(v, dataType) + } + case _: TimestampType => v match { + case s: String => java.sql.Timestamp.valueOf(s) + case t: java.sql.Timestamp => t + case n: Number => new java.sql.Timestamp(n.longValue()) + case _ => unsupported(v, dataType) + } + case _: TimestampNTZType => v match { + case s: String => java.time.LocalDateTime.parse(s.replace(' ', 'T')) + case ldt: java.time.LocalDateTime => ldt + case _ => unsupported(v, dataType) + } + case _: DecimalType => v match { + case bd: java.math.BigDecimal => bd + case bd: scala.math.BigDecimal => bd.bigDecimal + case s: String => new java.math.BigDecimal(s) + case n: Number => new java.math.BigDecimal(n.toString) + case _ => unsupported(v, dataType) + } + case _: BinaryType => v match { + case b: Array[Byte] => b + case s: String => java.util.Base64.getDecoder.decode(s) + case _ => unsupported(v, dataType) + } + case _ => unsupported(v, dataType) + } + } + + private def unsupported(v: Any, dataType: DataType): Nothing = + throw new IllegalArgumentException( + s"RowParquet.coerce: unsupported value '$v' (${v.getClass.getName}) for column type $dataType") +} diff --git a/workload-generator/src/main/scala/io/delta/workload/SnapshotCapture.scala b/workload-generator/src/main/scala/io/delta/workload/SnapshotCapture.scala index 9e07ccd5..a0fde145 100644 --- a/workload-generator/src/main/scala/io/delta/workload/SnapshotCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/SnapshotCapture.scala @@ -27,7 +27,7 @@ object SnapshotCapture { def capture( spark: SparkSession, testId: String, tablePath: Path, specsDir: Path, version: Option[Long] = None, timestamp: Option[String] = None, - expectError: Option[String] = None): Unit = { + expectError: Option[String] = None, writeSpec: Option[String] = None): Unit = { require(!(version.isDefined && timestamp.isDefined), "Cannot specify both version and timestamp") @@ -79,7 +79,7 @@ object SnapshotCapture { } val specVersion = version.orElse(if (timestamp.isEmpty) resolvedVersion else None) - val spec = SnapshotSpec(specVersion, timestamp, expected, expectedError) + val spec = SnapshotSpec(writeSpec, specVersion, timestamp, expected, expectedError) JsonUtil.writeSpec(specPath, spec) validateFromSpec(spark, tablePath, specPath) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 912a5d2d..5e56c13c 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -24,7 +24,8 @@ import scala.jdk.CollectionConverters._ import com.fasterxml.jackson.databind.node.ObjectNode import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession -import io.delta.workload.deltaharness.DeltaHarness +import org.apache.spark.sql.types.StructType +import io.delta.workload.deltaharness.{CommitRemoveFile, CommitRequest, DeltaHarness} /** * Internal workload generation engine. Use [[WorkloadTestSuite]] as the public API: @@ -95,6 +96,10 @@ object WorkloadGenerator { mutate(destTablePath) } + // When this workload has a write spec, read/snapshot specs point to it (making them + // write-derived: a consumer reconstructs the table from the write spec before validating). + val writePtr = ts.writeBuilder.map(_ => s"${dirName}_write.json") + // Snapshot specs val snapshotNames = mutable.ArrayBuffer[String]() val explicits = if (ts.snapshotSpecs.isEmpty) { @@ -103,7 +108,7 @@ object WorkloadGenerator { for (ss <- explicits) { SnapshotCapture.capture(spark, dirName, destTablePath, specsDir, version = ss.version, timestamp = ss.timestamp, - expectError = ss.expectError) + expectError = ss.expectError, writeSpec = writePtr) val specName = (ss.version, ss.timestamp) match { case (Some(v), _) => s"${dirName}_snapshot_v$v" case (_, Some(t)) => @@ -120,13 +125,13 @@ object WorkloadGenerator { ReadCapture.capture(spark, dirName, destTablePath, testOutputDir, specsDir, name = rs.name, predicate = rs.predicate, version = rs.version, timestamp = rs.timestamp, columns = rs.columns, - expectError = rs.expectError) + expectError = rs.expectError, writeSpec = writePtr) val specName = s"${dirName}_${rs.name}" readNames += specName checkAssertion(rs, specsDir.resolve(s"$specName.json")) } - // Write spec - serialize the recorded commit history + expected/latest/ + // Write spec - serialize the recorded commit history into specs/ + expected/_write/ ts.writeBuilder.foreach { builder => builder.buildSpec(spark, destTablePath, testOutputDir) } @@ -140,11 +145,12 @@ object WorkloadGenerator { System.err.println(s"WARN: Could not write table_info.json for $dirName: ${e.getMessage}") } - // Replay the write spec against a fresh table and re-validate read/snapshot specs. + // Self-validate the write workload: the single WorkloadValidator replays the write spec into + // a fresh table and re-checks the write-derived read/snapshot specs against it. ts.writeBuilder.foreach { _ => - val replayWarnings = WriteSpecValidator.validate(spark, testOutputDir, dirName) - require(replayWarnings.isEmpty, - s"Write spec validation FAILED for $dirName:\n ${replayWarnings.mkString("\n ")}") + val result = WorkloadValidator.validateTestDir(spark, testOutputDir) + require(result.success, + s"Write spec validation FAILED for $dirName:\n ${result.errors.mkString("\n ")}") } // Repro placeholder @@ -414,6 +420,40 @@ class WorkloadContext private[workload] ( new WriteHandle(t) } + /** + * Replace an existing table's schema/partitioning/properties and all data via + * `CREATE OR REPLACE TABLE`, and record the replace_table operation. `schema` is a SQL DDL + * string. With non-empty `rows` it is a replace-as-select (RTAS) — a single commit that also + * writes the rows. Operates on the table behind `w`; subsequent ops see the new schema. + */ + def replaceTableOp( + w: WriteHandle, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty, + rows: Seq[Map[String, Any]] = Seq.empty): Unit = { + val partitionClause = + if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" + val propsClause = if (properties.nonEmpty) { + s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" + } else "" + if (rows.nonEmpty) { + // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so + // the resulting schema matches what the bundled spec Parquet will reproduce on replay. + val parquet = RowParquet.writeTemp(spark, StructType.fromDDL(schema), rows) + try { + sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} USING delta$partitionClause" + + s"$propsClause AS SELECT * FROM parquet.`${parquet.toAbsolutePath}`") + } finally { + FileUtils.deleteDirectory(parquet.getParent.toFile) + } + } else { + sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} ($schema) USING delta" + + s"$partitionClause$propsClause") + } + getWriteBuilder(w.table).recordReplaceTable(schema, properties, partitionColumns, rows) + } + /** * Insert `rows` (column -> value maps) and record the insert. An empty `rows` is a no-op: * it produces no commit, so recording it would both desync the commit-index/version mapping @@ -421,20 +461,18 @@ class WorkloadContext private[workload] ( */ def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = { require(rows.nonEmpty, "insertOp requires at least one row") - val columns = rows.head.keys.toSeq - val colList = columns.map(c => s"`$c`").mkString(", ") - val valuesList = rows.map { row => - val values = columns.map { c => - row.get(c) match { - case Some(s: String) => s"'$s'" - case Some(null) | None => "NULL" - case Some(v) => v.toString - } - } - s"(${values.mkString(", ")})" - }.mkString(", ") - sql(s"INSERT INTO ${w.table.tableName} ($colList) VALUES $valuesList") - getWriteBuilder(w.table).recordInsert() + // Drive the live insert from the SAME materialized Parquet the spec will bundle (via + // RowParquet), so the captured table and the replayed table use one value encoder and + // cannot diverge by type. The spec's own copy is re-materialized in buildSpec. + val schema = RowParquet.schemaAt(spark, w.table.sourcePath.toString, + version = None, includePartition = true) + val parquet = RowParquet.writeTemp(spark, schema, rows) + try { + sql(s"INSERT INTO ${w.table.tableName} SELECT * FROM parquet.`${parquet.toAbsolutePath}`") + } finally { + FileUtils.deleteDirectory(parquet.getParent.toFile) + } + getWriteBuilder(w.table).recordInsert(rows) } /** Delete rows matching `predicate` and record the delete. */ @@ -450,39 +488,74 @@ class WorkloadContext private[workload] ( getWriteBuilder(w.table).recordUpdate(predicate, set) } - /** Restore the table to `version` and record the restore. */ - def restoreOp(w: WriteHandle, version: Long): Unit = { - sql(s"RESTORE TABLE ${w.table.tableName} TO VERSION AS OF $version") - getWriteBuilder(w.table).recordRestore(version) - } /** - * Execute a low-level commit of raw Delta actions against the live table and record it. See - * [[LowLevelCommit]] for action semantics; addFiles must reference Parquet already in the table. + * Execute a low-level commit of raw Delta actions against the live table and record it. `addFiles` + * supply LOGICAL rows; the engine writes them (column-mapping/partition/stats handled) and the + * actions are bundled with the raw `txn`/`addDomainMetadata`/`removeDomainMetadata`/schema/property + * changes in one commit. `removeFiles` reference a prior low-level add by the ordinal that + * [[commitOp]] returned (the commit/version that produced it). Returns this commit's ordinal so a + * later `removeFiles` can target it. */ def commitOp( w: WriteHandle, schemaDDL: Option[String] = None, tableProperties: Option[Map[String, String]] = None, txn: Option[AppTxn] = None, - addFiles: Option[Seq[AddFileAction]] = None, - removeFiles: Option[Seq[RemoveFileAction]] = None, + addFiles: Option[Seq[AddFileInput]] = None, + removeFiles: Option[Seq[Int]] = None, addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, - removeDomainMetadata: Option[Seq[String]] = None): Unit = { - val effectiveAdds = LowLevelCommit.apply( - spark, w.table.sourcePath.toString, - schemaDDL = schemaDDL, tableProperties = tableProperties, txn = txn, - addFiles = addFiles, removeFiles = removeFiles, - addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata, - resolveDataFile = rel => w.table.sourcePath.resolve(rel), - // Copy each source into a fresh in-table file so the added action is independent of the - // source's own add. Recording these paths lets a later removeFiles target them by name. - assignPath = _ => s"part-commit-${java.util.UUID.randomUUID()}-c000.snappy.parquet") - getWriteBuilder(w.table).recordCommit( + removeDomainMetadata: Option[Seq[String]] = None): Int = { + require( + schemaDDL.isDefined || tableProperties.exists(_.nonEmpty) || txn.isDefined || + addFiles.exists(_.nonEmpty) || removeFiles.exists(_.nonEmpty) || + addDomainMetadata.exists(_.nonEmpty) || removeDomainMetadata.exists(_.nonEmpty), + "commitOp requires at least one action (schema/properties/txn/addFiles/removeFiles/" + + "domainMetadata); an empty commit would not advance the table version") + val builder = getWriteBuilder(w.table) + val idx = builder.nextOrdinal + val livePath = w.table.sourcePath.toString + val adds = addFiles.getOrElse(Seq.empty) + + // Materialize each add's logical rows to a temp Parquet for the live write. + val schema = RowParquet.schemaAt(spark, livePath, version = None, includePartition = true) + val tempParquet = adds.map(in => RowParquet.writeTemp(spark, schema, in.rows)) + // A remove targets all files added at the referenced commit (== that table version). + val removePaths = removeFiles.getOrElse(Seq.empty) + .flatMap(k => SpecLayout.addPathsAt(w.table.sourcePath, k)) + try { + DeltaHarness.get.commitWithData(spark, livePath, tempParquet.map(_.toAbsolutePath.toString), + CommitRequest( + schemaJson = schemaDDL.map(ddl => StructType.fromDDL(ddl).json), + properties = tableProperties, + setTransaction = txn, + removeFiles = removePaths.map(p => CommitRemoveFile(p, dataChange = true)), + addDomainMetadata = addDomainMetadata.getOrElse(Seq.empty), + removeDomainMetadata = removeDomainMetadata.getOrElse(Seq.empty))) + } finally { + tempParquet.foreach(p => FileUtils.deleteDirectory(p.getParent.toFile)) + } + + // Enforce the load-bearing commit-index == table-version invariant at the point it is + // established: ordinal-based remove resolution (SpecLayout.addPathsAt) keys off it. Fail here, + // at the offending op, rather than deferring to buildSpec's whole-spec count check. + val newVersion = DeltaHarness.get.openLog(spark, livePath).update().version + require(newVersion == idx, + s"commitOp produced version $newVersion but expected ordinal $idx; commit index can no " + + "longer be used as the table version (ordinal-based removes would mis-resolve)") + + // Record the spec: adds become logical-Parquet pointers (materialized to data/ in buildSpec); + // removes become ordinal references. + val addActions = adds.zipWithIndex.map { case (in, i) => + AddFileAction(dataFile = SpecLayout.commitDataFile(idx, s"add_$i.parquet"), dataChange = in.dataChange) + } + builder.recordCommit( schemaDDL = schemaDDL, tableProperties = tableProperties, txn = txn, - addFiles = if (effectiveAdds.nonEmpty) Some(effectiveAdds) else addFiles, - removeFiles = removeFiles, + addFiles = if (addActions.nonEmpty) Some(addActions) else None, + removeFiles = removeFiles.map(_.map(k => RemoveFileAction(k))), addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata) + builder.recordLowLevelRows(idx, adds.map(_.rows)) + idx } /** Add columns (SQL DDL) and record the schema evolution. */ @@ -761,6 +834,15 @@ trait WorkloadOps { partitionColumns: Seq[String] = Seq.empty): WriteHandle = current.createTableOp(tableName, schema, properties, partitionColumns) + /** Replace an existing table (schema/partitioning/properties + data) and record it. */ + def replaceTableOp( + w: WriteHandle, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty, + rows: Seq[Map[String, Any]] = Seq.empty): Unit = + current.replaceTableOp(w, schema, properties, partitionColumns, rows) + /** Insert rows and record the insert. */ def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = current.insertOp(w, rows) @@ -771,19 +853,17 @@ trait WorkloadOps { def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = current.updateOp(w, predicate, set) - /** Restore the table to `version` and record the restore. */ - def restoreOp(w: WriteHandle, version: Long): Unit = current.restoreOp(w, version) - /** Execute a low-level commit of raw Delta actions and record it. */ + /** Execute a low-level commit of raw Delta actions and record it. Returns the commit ordinal. */ def commitOp( w: WriteHandle, schemaDDL: Option[String] = None, tableProperties: Option[Map[String, String]] = None, txn: Option[AppTxn] = None, - addFiles: Option[Seq[AddFileAction]] = None, - removeFiles: Option[Seq[RemoveFileAction]] = None, + addFiles: Option[Seq[AddFileInput]] = None, + removeFiles: Option[Seq[Int]] = None, addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, - removeDomainMetadata: Option[Seq[String]] = None): Unit = + removeDomainMetadata: Option[Seq[String]] = None): Int = current.commitOp(w, schemaDDL, tableProperties, txn, addFiles, removeFiles, addDomainMetadata, removeDomainMetadata) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index 8755f084..f6cb0654 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -18,10 +18,17 @@ package io.delta.workload import java.nio.file.{Files, Path} -import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal -import org.apache.spark.sql.SparkSession +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.{SaveMode, SparkSession} +import org.apache.spark.sql.types.{DataType, StructField, StructType} + +import io.delta.workload.deltaharness.{CommitRemoveFile, CommitRequest, DeltaHarness} /** Result of validating a workload tree: per-spec pass/fail tallies + error details. */ case class ValidationResult(passed: Int, errors: Seq[String]) { @@ -29,73 +36,467 @@ case class ValidationResult(passed: Int, errors: Seq[String]) { } /** - * Walks a generated workload tree and re-validates every captured spec against - * the tables on disk. Used as the presubmit acceptance-test entry point: given - * a committed tarball extracted locally, verify that the current Delta - * implementation can still reproduce the captured specifications. + * The single validator for every spec type. Walks a generated workload tree (or one test dir) + * and validates each captured spec under `specs/`, dispatching on the `type` field: * - * Expected tree layout (produced by [[WorkloadGenerator]]): - * {{{ - * // - * delta/ # the Delta table - * specs/.json # flat — dispatched by the "type" field - * expected// - * expected_data # parquet - * expected_metadata # parquet - * }}} + * - `write` — replay its `commits` into a fresh table, compare to `expected//`. + * - `read` / `snapshot` — if the spec carries a `writeSpec` pointer it is *write-derived*: + * reconstruct the table by replaying that sibling write spec, then validate **portably** + * (rows-only reads, capability-protocol, column-mapping-normalized schema, declared-config + * only). Otherwise it is *read-only*: validate against the captured `delta/` table exactly. * - * Each `/delta/` is opened and each spec under `/specs/` - * is replayed via [[ReadCapture.validateFromSpec]] or - * [[SnapshotCapture.validateFromSpec]] based on the spec's `type` field. - * A missing `delta/` is skipped (corruption tests may omit the table). + * Used both as the presubmit acceptance entry point ([[validateAll]]) and by the generator to + * self-check a freshly generated test dir ([[validateTestDir]]). */ object WorkloadValidator { def validateAll(spark: SparkSession, workloadRoot: Path): ValidationResult = { require(Files.isDirectory(workloadRoot), s"Not a directory: $workloadRoot") + val errors = mutable.ArrayBuffer[String]() + var passed = 0 + for (testDir <- listChildren(workloadRoot).filter(Files.isDirectory(_))) { + val r = validateTestDir(spark, testDir) + passed += r.passed + errors ++= r.errors + } + ValidationResult(passed, errors.toSeq) + } + + /** + * Validate every spec in one test dir. Write-derived read/snapshot specs and the write spec + * share a single replayed table per write spec (memoized). Returns pass count + error details. + */ + def validateTestDir(spark: SparkSession, testDir: Path): ValidationResult = { + val specsDir = testDir.resolve("specs") + if (!Files.isDirectory(specsDir)) return ValidationResult(0, Seq.empty) + val deltaTable = testDir.resolve("delta") val errors = mutable.ArrayBuffer[String]() var passed = 0 + val replays = mutable.Map[String, Path]() // write-spec file name -> replayed table path + val tempDirs = mutable.ArrayBuffer[Path]() + def replayOf(writeSpecName: String): Path = replays.getOrElseUpdate(writeSpecName, { + val td = Files.createTempDirectory("wl_replay") + tempDirs += td + val table = td.resolve("replay_table") + replayWriteInto(spark, testDir, specsDir.resolve(writeSpecName), table) + table + }) + + try { + for (specFile <- listSpecs(specsDir)) { + val name = specFile.getFileName.toString.stripSuffix(".json") + val node = JsonUtil.mapper.readTree(Files.readAllBytes(specFile)) + val specType = Option(node.get("type")).map(_.asText()).getOrElse("") + val writePtr = Option(node.get("writeSpec")).filterNot(_.isNull).map(_.asText()) + try { + specType match { + case "write" => + val w = mutable.ArrayBuffer[String]() + validateExpectedLatest(spark, replayOf(specFile.getFileName.toString), testDir, name, w) + if (w.isEmpty) passed += 1 else errors ++= w.map(m => s"${testDir.getFileName}/$name: $m") + + case "read" => + val expectedDir = testDir.resolve("expected").resolve(name) + writePtr match { + case Some(ws) => // write-derived: rows-only against the replayed table + ReadCapture.validateFromSpec(spark, replayOf(ws), expectedDir, specFile, + checkMetadata = false) + passed += 1 + case None => // read-only: exact, against the captured table + if (Files.isDirectory(deltaTable)) { + ReadCapture.validateFromSpec(spark, deltaTable, expectedDir, specFile) + passed += 1 + } else { + errors += s"${testDir.getFileName}/$name: read-only spec but no captured " + + s"table at $deltaTable" + } + } + + case "snapshot" => + writePtr match { + case Some(ws) => // write-derived: structural, against the replayed table + val w = mutable.ArrayBuffer[String]() + validateSnapshotStructurally(spark, replayOf(ws), specFile, name, + JsonUtil.readWriteSpec(specsDir.resolve(ws)), w) + if (w.isEmpty) passed += 1 + else errors ++= w.map(m => s"${testDir.getFileName}/$name: $m") + case None => + if (Files.isDirectory(deltaTable)) { + SnapshotCapture.validateFromSpec(spark, deltaTable, specFile) + passed += 1 + } else { + errors += s"${testDir.getFileName}/$name: read-only spec but no captured " + + s"table at $deltaTable" + } + } - val testDirs = listChildren(workloadRoot).filter(Files.isDirectory(_)) - - for (testDir <- testDirs) { - val tablePath = testDir.resolve("delta") - val specsDir = testDir.resolve("specs") - if (!Files.isDirectory(tablePath) || !Files.isDirectory(specsDir)) { - // Nothing to validate — corruption tests may legitimately omit these. - } else { - for (specPath <- listChildren(specsDir).filter(_.getFileName.toString.endsWith(".json"))) { - val specName = specPath.getFileName.toString.stripSuffix(".json") - val expectedDir = testDir.resolve("expected").resolve(specName) - try { - dispatch(spark, tablePath, expectedDir, specPath) - passed += 1 - } catch { - case e: Throwable => - errors += s"${testDir.getFileName}/$specName: ${e.getMessage}" + case other => throw new IllegalArgumentException( + s"Unknown spec type '$other' in ${specFile.getFileName}") } + } catch { + case e: Throwable => errors += s"${testDir.getFileName}/$name: $e" } } + } finally { + tempDirs.foreach(td => try FileUtils.deleteDirectory(td.toFile) catch { case NonFatal(_) => }) } - ValidationResult(passed, errors.toSeq) } - /** Dispatch to the correct validator based on the spec's `type` field. */ - private def dispatch( - spark: SparkSession, tablePath: Path, expectedDir: Path, specPath: Path): Unit = { - val node = JsonUtil.mapper.readTree(Files.readAllBytes(specPath)) - Option(node.get("type")).map(_.asText()).getOrElse("") match { - case "read" => ReadCapture.validateFromSpec(spark, tablePath, expectedDir, specPath) - case "snapshot" => SnapshotCapture.validateFromSpec(spark, tablePath, specPath) - case other => throw new IllegalArgumentException( - s"Unknown spec type '$other' in ${specPath.getFileName}") - } + private def listSpecs(specsDir: Path): Seq[Path] = { + val stream = Files.list(specsDir) + try stream.iterator().asScala.filter(_.getFileName.toString.endsWith(".json")).toSeq.sorted + finally stream.close() } private def listChildren(dir: Path): List[Path] = { val stream = Files.list(dir) try stream.iterator().asScala.toList finally stream.close() } + + // =========================================================================== + // Write-spec replay (reconstruct a fresh table from a write spec's commits) + // =========================================================================== + + /** Replay a write spec's commits into a fresh table at `replayTablePath`. */ + private def replayWriteInto( + spark: SparkSession, testDir: Path, writeSpecFile: Path, replayTablePath: Path): Unit = { + val writeSpec = JsonUtil.readWriteSpec(writeSpecFile) + val ref = s"delta.`${replayTablePath.toAbsolutePath}`" + val path = replayTablePath.toAbsolutePath.toString + writeSpec.commits.zipWithIndex.foreach { case (c, idx) => + replayCommit(spark, c, idx, ref, path, testDir) + } + // Each commit must map to exactly one version, or ordinal-based remove resolution (which keys + // off commit index == table version) silently mis-resolves. Fail loud if replay desynced. + val finalVersion = DeltaHarness.get.openLog(spark, path).update().version + require(finalVersion == writeSpec.commits.size - 1, + s"replay produced ${finalVersion + 1} versions but the write spec has ${writeSpec.commits.size} " + + "commits; commit index can no longer be used as the table version") + } + + /** Compare the replayed table's rows against the write spec's `expected//table_content`. */ + private def validateExpectedLatest( + spark: SparkSession, replayTablePath: Path, testDir: Path, writeName: String, + warnings: mutable.ArrayBuffer[String]): Unit = { + val contentDir = testDir.resolve("expected").resolve(writeName).resolve("table_content") + // A write spec MUST ship its expected rows; a missing dir is a capture bug, not a pass. + if (!Files.exists(contentDir)) { + warnings += s"write spec '$writeName': expected/$writeName/table_content is missing" + return + } + val expected = JsonUtil.toRowMultiset(spark.read.parquet(contentDir.toString)) + val actual = JsonUtil.toRowMultiset( + spark.read.format("delta").load(replayTablePath.toAbsolutePath.toString)) + try JsonUtil.assertMultisetsEqual(expected, actual, s"${writeName}_latest") + catch { case e: Throwable => warnings += e.getMessage } + } + + /** + * Validate a snapshot spec against a write-replayed (independently created) table. `protocol` + * is a capability check (version floors + feature supersets), `configuration` checks only the + * write spec's declared keys, and `schemaString` is compared with column-mapping ids stripped; + * `partitionColumns`/`format` are compared exactly. + */ + private def validateSnapshotStructurally( + spark: SparkSession, replayTablePath: Path, specFile: Path, specFileName: String, + writeSpec: WriteSpec, warnings: mutable.ArrayBuffer[String]): Unit = { + val spec = JsonUtil.readSnapshotSpec(specFile) + val (declaredKeys, removedKeys) = declaredConfiguration(writeSpec) + spec.expected.foreach { exp => + val log = DeltaHarness.get.openLog(spark, replayTablePath.toString) + val snapshot = (spec.version, spec.timestamp) match { + case (Some(v), _) => log.getSnapshotAt(v) + case _ => log.update() + } + val expProtocol = JsonUtil.mapper.valueToTree[JsonNode](exp.protocol) + val replayProtocol = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") + protocolViolation(expProtocol, replayProtocol).foreach { reason => + warnings += s"snapshot '$specFileName': protocol mismatch ($reason)" + } + val expectedMeta = JsonUtil.mapper.valueToTree[JsonNode](exp.metadata) + val replayMeta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") + configurationViolation(expectedMeta, replayMeta, declaredKeys, removedKeys).foreach { + reason => warnings += s"snapshot '$specFileName': metadata.configuration mismatch ($reason)" + } + if (normalizedSchema(expectedMeta) != normalizedSchema(replayMeta)) { + warnings += s"snapshot '$specFileName': metadata.schemaString mismatch" + } + for (field <- Seq("partitionColumns", "format")) { + if (canonicalField(expectedMeta, field) != canonicalField(replayMeta, field)) { + warnings += s"snapshot '$specFileName': metadata.$field mismatch" + } + } + } + } + + // =========================================================================== + // Portable comparison helpers (also unit-tested directly) + // =========================================================================== + + /** + * Canonicalize `metadata.schemaString`, stripping per-field column-mapping identifiers + * (`delta.columnMapping.physicalName`/`.id`) which are minted per-table and so are not a + * meaningful equality target across an independently-replayed table. + */ + private[workload] def normalizedSchema(meta: JsonNode): String = + Option(meta.get("schemaString")).filterNot(_.isNull) match { + case None => "null" + case Some(s) => + val tree = JsonUtil.mapper.readTree(s.asText()) + stripColumnMapping(tree) + JsonUtil.canonicalJson(tree) + } + + private def stripColumnMapping(node: JsonNode): Unit = node match { + case obj: ObjectNode => + obj.get("metadata") match { + case m: ObjectNode => + m.remove("delta.columnMapping.physicalName") + m.remove("delta.columnMapping.id") + case _ => () + } + obj.elements().asScala.foreach(stripColumnMapping) + case arr: ArrayNode => arr.elements().asScala.foreach(stripColumnMapping) + case _ => () + } + + /** + * The configuration keys the write spec's commits explicitly declared. + * @return (declared keys still in effect, keys explicitly removed). `replace_table` resets both. + */ + private[workload] def declaredConfiguration(writeSpec: WriteSpec): (Set[String], Set[String]) = { + val declared = mutable.LinkedHashSet[String]() + val removed = mutable.LinkedHashSet[String]() + for (commit <- writeSpec.commits) commit match { + case c: CreateTableCommit => c.properties.foreach(declared ++= _.keys) + case c: ReplaceTableCommit => + declared.clear(); removed.clear() + c.properties.foreach(declared ++= _.keys) + case c: UpdatePropertiesCommit => + c.set.foreach { s => declared ++= s.keys; removed --= s.keys } + c.remove.foreach { r => removed ++= r; declared --= r } + case _ => () + } + (declared.toSet, removed.toSet) + } + + // Features implied by a protocol version when table features are NOT enumerated (reader < 3 / + // writer < 7). Cumulative — a version implies its own and all lower versions' features. + // columnMapping is the only reader-axis legacy feature; it is implied jointly (reader>=2 AND + // writer>=5), so it lives in the writer map and is credited to the reader axis only when the + // writer floor is also met (see effectiveReader) — a reader=2/writer<5 protocol does NOT imply it. + private val writerImpliedFeatures: Map[Int, Set[String]] = Map( + 1 -> Set.empty, + 2 -> Set("appendOnly", "invariants"), + 3 -> Set("appendOnly", "invariants", "checkConstraints"), + 4 -> Set("appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns"), + 5 -> Set("appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns", + "columnMapping"), + 6 -> Set("appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns", + "columnMapping", "identityColumns")) + + /** + * Capability check: version floors + feature supersets over EFFECTIVE feature sets (explicit + * features unioned with those implied by the protocol version). Returns the first violation. + */ + private[workload] def protocolViolation(expected: JsonNode, replay: JsonNode): Option[String] = { + def intField(node: JsonNode, name: String): Int = + if (node != null && node.has(name)) node.get(name).asInt() else 0 + def explicitFeatures(node: JsonNode, name: String): Set[String] = + if (node != null && node.has(name) && node.get(name).isArray) + node.get(name).elements().asScala.map(_.asText()).toSet + else Set.empty + def effectiveReader(node: JsonNode): Set[String] = { + val r = intField(node, "minReaderVersion") + val w = intField(node, "minWriterVersion") + if (r >= 3) explicitFeatures(node, "readerFeatures") + // Legacy: columnMapping is the only reader-implied feature, and only when reader>=2 AND + // writer>=5 jointly (mirrors delta-spark's implicitlySupportedFeatures). + else if (r >= 2 && w >= 5) Set("columnMapping") + else Set.empty + } + def effectiveWriter(node: JsonNode): Set[String] = { + val v = intField(node, "minWriterVersion") + if (v >= 7) explicitFeatures(node, "writerFeatures") + else writerImpliedFeatures.getOrElse(v, Set.empty) + } + + val expReader = intField(expected, "minReaderVersion") + val expWriter = intField(expected, "minWriterVersion") + val repReader = intField(replay, "minReaderVersion") + val repWriter = intField(replay, "minWriterVersion") + val missingReaderFeatures = effectiveReader(expected) -- effectiveReader(replay) + val missingWriterFeatures = effectiveWriter(expected) -- effectiveWriter(replay) + + if (repReader < expReader) Some(s"minReaderVersion $repReader < $expReader") + else if (repWriter < expWriter) Some(s"minWriterVersion $repWriter < $expWriter") + else if (missingReaderFeatures.nonEmpty) + Some(s"missing readerFeatures ${missingReaderFeatures.mkString(",")}") + else if (missingWriterFeatures.nonEmpty) + Some(s"missing writerFeatures ${missingWriterFeatures.mkString(",")}") + else None + } + + /** + * Check only the author-declared config keys: each declared key present + equal on the replay, + * each removed key absent. Engine-injected defaults are ignored. Returns the first violation. + */ + private[workload] def configurationViolation( + expectedMeta: JsonNode, replayMeta: JsonNode, + declaredKeys: Set[String], removedKeys: Set[String]): Option[String] = { + val expConfig = Option(expectedMeta.get("configuration")).filterNot(_.isNull) + val repConfig = Option(replayMeta.get("configuration")).filterNot(_.isNull) + def get(config: Option[JsonNode], key: String): Option[String] = + config.flatMap(c => Option(c.get(key))).filterNot(_.isNull).map(_.asText()) + + declaredKeys.toSeq.sorted.collectFirst { + case key if get(repConfig, key) != get(expConfig, key) => + s"key '$key' expected ${get(expConfig, key)}, got ${get(repConfig, key)}" + }.orElse { + removedKeys.toSeq.sorted.collectFirst { + case key if get(repConfig, key).isDefined => s"removed key '$key' still present" + } + } + } + + private def canonicalField(node: JsonNode, field: String): String = + if (node.has(field) && !node.get(field).isNull) JsonUtil.canonicalJson(node.get(field)) + else "null" + + // =========================================================================== + // Per-commit replay + // =========================================================================== + + private def replayCommit( + spark: SparkSession, commit: WriteCommit, idx: Int, + tableRef: String, tablePath: String, testDir: Path): Unit = commit match { + case c: CreateTableCommit => + var sql = s"CREATE TABLE $tableRef (${structOf(c.schema).toDDL}) USING delta" + c.partitionColumns.filter(_.nonEmpty).foreach { p => + sql += s" PARTITIONED BY (${p.mkString(", ")})" + } + c.properties.filter(_.nonEmpty).foreach { props => + sql += s" TBLPROPERTIES (${props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" + } + spark.sql(sql) + + case c: ReplaceTableCommit => + val parts = c.partitionColumns.filter(_.nonEmpty) + .map(p => s" PARTITIONED BY (${p.mkString(", ")})").getOrElse("") + val props = c.properties.filter(_.nonEmpty) + .map(m => s" TBLPROPERTIES (${m.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})") + .getOrElse("") + // Always replace via AS SELECT: `CREATE OR REPLACE TABLE delta.`path` (cols)` is treated as + // create-and-validate by Delta; the query path honors the new schema. Data-less = 0 rows. + val select = c.dataFiles.filter(_.nonEmpty) match { + case Some(files) => + val resolved = files.map(f => testDir.resolve(f).toAbsolutePath.toString) + val src = if (resolved.size == 1) s"parquet.`${resolved.head}`" + else s"parquet.`${testDir.resolve(SpecLayout.commitDataDir(idx)).toAbsolutePath}`" + s"SELECT * FROM $src" + case None => + val nulls = structOf(c.schema).fields + .map(f => s"CAST(NULL AS ${f.dataType.sql}) AS `${f.name}`").mkString(", ") + s"SELECT * FROM (SELECT $nulls) WHERE false" + } + spark.sql(s"CREATE OR REPLACE TABLE $tableRef USING delta$parts$props AS $select") + + case c: InsertCommit => + c.dataFiles.foreach(loadDataFiles(spark, _, tablePath, testDir)) + + case c: DeleteCommit => + spark.sql(s"DELETE FROM $tableRef WHERE ${c.predicate}") + + case c: UpdateCommit => + val setClauses = c.set.map { case (k, v) => s"`$k` = $v" }.mkString(", ") + spark.sql(s"UPDATE $tableRef SET $setClauses WHERE ${c.predicate}") + + case c: UpdatePropertiesCommit => + c.set.filter(_.nonEmpty).foreach { s => + val setClause = s.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + spark.sql(s"ALTER TABLE $tableRef SET TBLPROPERTIES ($setClause)") + } + c.remove.filter(_.nonEmpty).foreach { r => + spark.sql(s"ALTER TABLE $tableRef UNSET TBLPROPERTIES (${r.map(k => s"'$k'").mkString(", ")})") + } + + case c: EvolveSchemaCommit => replayEvolveSchema(spark, c, tablePath) + + case c: LowLevelCommitOp => replayLowLevelCommit(spark, c, idx, tablePath, testDir) + } + + /** Replay schema add/rename/drop via a single metadata-update commit. */ + private def replayEvolveSchema( + spark: SparkSession, commit: EvolveSchemaCommit, tablePath: String): Unit = { + val log = DeltaHarness.get.openLog(spark, tablePath) + val metaNode = Option(JsonUtil.mapper.readTree(log.update().metadataJson).get("metaData")) + .getOrElse(throw new IllegalStateException( + s"evolve_schema replay: no metaData action in current snapshot of $tablePath")) + val schemaString = Option(metaNode.get("schemaString")) + .getOrElse(throw new IllegalStateException( + s"evolve_schema replay: metaData has no schemaString in $tablePath")) + val currentSchema = DataType.fromJson(schemaString.asText()).asInstanceOf[StructType] + var fields = currentSchema.fields.toBuffer + + commit.addColumns.foreach { cols => + cols.asInstanceOf[Seq[Map[String, Any]]].foreach { col => + val name = col("name").toString + val nullable = col.get("nullable").forall(_.asInstanceOf[Boolean]) + fields += StructField( + name, DataType.fromJson(JsonUtil.mapper.writeValueAsString(col("type"))), nullable) + } + } + commit.renameColumns.foreach { renames => + for ((oldName, newName) <- renames) { + val i = fields.indexWhere(_.name == oldName) + if (i >= 0) fields(i) = fields(i).copy(name = newName) + } + } + commit.dropColumns.foreach { drops => fields = fields.filterNot(f => drops.contains(f.name)) } + + DeltaHarness.get.commit(spark, tablePath, + CommitRequest(schemaJson = Some(StructType(fields.toSeq).json))) + } + + /** + * Replay a low-level commit: write each add's bundled logical Parquet through the engine + * (`commitWithData` → column-mapping/partition/stats handled), and tombstone each remove's + * referenced prior add — resolved to this replay table's own paths at that commit/version. + */ + private def replayLowLevelCommit( + spark: SparkSession, commit: LowLevelCommitOp, idx: Int, + tablePath: String, testDir: Path): Unit = { + val addDataParquet = commit.addFiles.getOrElse(Seq.empty) + .map(af => testDir.resolve(af.dataFile).toAbsolutePath.toString) + val removePaths = commit.removeFiles.getOrElse(Seq.empty) + .flatMap(rf => SpecLayout.addPathsAt(java.nio.file.Paths.get(tablePath), rf.addedAtCommit)) + DeltaHarness.get.commitWithData(spark, tablePath, addDataParquet, + CommitRequest( + schemaJson = commit.schema.map(s => structOf(s).json), + properties = commit.tableProperties, + setTransaction = commit.txn, + removeFiles = removePaths.map(p => CommitRemoveFile(p, dataChange = true)), + addDomainMetadata = commit.addDomainMetadata.getOrElse(Seq.empty), + removeDomainMetadata = commit.removeDomainMetadata.getOrElse(Seq.empty))) + } + + /** Read the bundled Parquet (full rows, incl. partition columns) and Append it. */ + private def loadDataFiles( + spark: SparkSession, files: Seq[String], tablePath: String, testDir: Path): Unit = { + val resolved = files.map(testDir.resolve).filter(Files.exists(_)).map(_.toAbsolutePath.toString) + if (resolved.nonEmpty) { + spark.read.parquet(resolved: _*).write.format("delta").mode(SaveMode.Append).save(tablePath) + } + } + + /** + * Reconstruct a Spark [[StructType]] from a stored Delta schema value. The spec stores schemas + * as `StructType.json` deserialized to a Map (see `WriteSpecCapture.ddlToSchemaJson`), so this + * re-serializes and parses it through Spark's own reader — `.toDDL`/`.sql`/`.json` on the result + * are then exact and lossless (decimal precision, nested nullability, quoting all preserved). + */ + private def structOf(schema: Any): StructType = + DataType.fromJson(JsonUtil.mapper.writeValueAsString(schema)).asInstanceOf[StructType] } diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala index e43f45b0..eeab2fe7 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala @@ -16,7 +16,7 @@ package io.delta.workload -import java.nio.file.{Files, Path, Paths, StandardCopyOption} +import java.nio.file.{Files, Path} import scala.collection.mutable @@ -26,36 +26,62 @@ import org.apache.spark.sql.types.StructType import io.delta.workload.deltaharness.DeltaHarness /** - * Records structured write operations and serializes them as `write_spec.json`, - * the portable recipe a Delta writer replays to reconstruct a table. + * Records structured write operations and serializes them as a `write` spec + * (`specs/_write.json`), the portable recipe a Delta writer replays to reconstruct a table. * - * High-level operations carry SQL semantics (create_table, insert, update, ...); - * the low-level `commit` carries raw Delta actions. Data files produced by - * insert/create_table commits are copied under `data/` so the spec is - * self-sufficient for replay. + * High-level operations carry SQL semantics (create_table, insert, update, ...); the low-level + * `commit` carries raw Delta actions. Data-bearing ops (insert, replace-as-select, low-level adds) + * record their LOGICAL rows, which `buildSpec` materializes to Parquet under `data/` so the spec + * is self-sufficient for replay. */ class WriteSpecBuilder { private val commits = mutable.ArrayBuffer[WriteCommit]() - // === High-level operations (SQL semantics) === + // Rows for data-bearing high-level ops (insert / replace-as-select), keyed by commit index. + // The API takes rows; `buildSpec` materializes them to Parquet under `data/` and the spec + // serializes only the resulting file paths. + private val rowData = mutable.HashMap[Int, Seq[Map[String, Any]]]() + + // Per-add rows for low-level commits, keyed by commit index (a low-level commit may add several + // files, each its own row-set). `buildSpec` materializes each to `data/commit_/add_.parquet`. + private val lowLevelRows = mutable.HashMap[Int, Seq[Seq[Map[String, Any]]]]() + + /** The ordinal (== table version) the next recorded commit will occupy. */ + def nextOrdinal: Int = commits.size + + /** Store a low-level commit's per-add rows for materialization in [[buildSpec]]. */ + def recordLowLevelRows(idx: Int, addRows: Seq[Seq[Map[String, Any]]]): Unit = + if (addRows.nonEmpty) lowLevelRows(idx) = addRows + + // === High-level operations (replayed from parameters) === def recordCreateTable(schemaDDL: String, properties: Map[String, String], partitionColumns: Seq[String]): Unit = { - commits += WriteCommit( - operation = "create_table", - schema = Some(ddlToSchemaJson(schemaDDL)), + commits += CreateTableCommit( + schema = ddlToSchemaJson(schemaDDL), partitionColumns = if (partitionColumns.nonEmpty) Some(partitionColumns) else None, properties = if (properties.nonEmpty) Some(properties) else None) } - def recordInsert(): Unit = commits += WriteCommit(operation = "insert") + def recordReplaceTable(schemaDDL: String, properties: Map[String, String], + partitionColumns: Seq[String], rows: Seq[Map[String, Any]]): Unit = { + if (rows.nonEmpty) rowData(commits.size) = rows + commits += ReplaceTableCommit( + schema = ddlToSchemaJson(schemaDDL), + partitionColumns = if (partitionColumns.nonEmpty) Some(partitionColumns) else None, + properties = if (properties.nonEmpty) Some(properties) else None) + } - def recordDelete(predicate: String): Unit = - commits += WriteCommit(operation = "delete", predicate = Some(predicate)) + def recordInsert(rows: Seq[Map[String, Any]]): Unit = { + rowData(commits.size) = rows + commits += InsertCommit() + } + + def recordDelete(predicate: String): Unit = commits += DeleteCommit(predicate) def recordUpdate(predicate: String, set: Map[String, String]): Unit = - commits += WriteCommit(operation = "update", predicate = Some(predicate), set = Some(set)) + commits += UpdateCommit(predicate, set) def recordSetProperties(props: Map[String, String]): Unit = recordUpdateProperties(props, Seq.empty) @@ -64,8 +90,7 @@ class WriteSpecBuilder { recordUpdateProperties(Map.empty, props) def recordUpdateProperties(set: Map[String, String], unset: Seq[String]): Unit = { - commits += WriteCommit( - operation = "update_properties", + commits += UpdatePropertiesCommit( set = if (set.nonEmpty) Some(set) else None, remove = if (unset.nonEmpty) Some(unset) else None) } @@ -90,18 +115,17 @@ class WriteSpecBuilder { Map[String, Any]("name" -> f.name, "type" -> typeJson, "nullable" -> f.nullable) }.toSeq) } else None - commits += WriteCommit( - operation = "evolve_schema", + commits += EvolveSchemaCommit( addColumns = addCols, renameColumns = if (renameColumns.nonEmpty) Some(renameColumns) else None, dropColumns = if (dropColumns.nonEmpty) Some(dropColumns) else None) } - def recordRestore(version: Long): Unit = - commits += WriteCommit(operation = "restore", version = Some(version)) - // === Low-level operation (raw Delta actions) === + // === Low-level operation (recorded raw Delta actions) === + /** Records a low-level commit's actions. `addFiles` are logical-Parquet pointers (materialized + * in [[buildSpec]]); `removeFiles` reference prior adds by commit ordinal. */ def recordCommit( schemaDDL: Option[String] = None, tableProperties: Option[Map[String, String]] = None, @@ -110,8 +134,7 @@ class WriteSpecBuilder { removeFiles: Option[Seq[RemoveFileAction]] = None, addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, removeDomainMetadata: Option[Seq[String]] = None): Unit = { - commits += WriteCommit( - operation = "commit", + commits += LowLevelCommitOp( schema = schemaDDL.map(ddlToSchemaJson), tableProperties = tableProperties, txn = txn, @@ -124,17 +147,15 @@ class WriteSpecBuilder { // === Serialization === /** - * Serialize recorded operations to `write_spec.json`, enriching the data-producing + * Serialize recorded operations to `specs/_write.json`, enriching the data-producing * commits (create_table, insert) with the actual data files they added, and writing - * the expected final table state to `expected/latest/`. + * the expected final table state to `expected/_write/`. * * New data files are copied to `outputDir/data/commit_/` and referenced via * relative paths in each commit's `dataFiles`, making the spec self-sufficient. */ def buildSpec(spark: SparkSession, tablePath: Path, outputDir: Path): Unit = { val log = DeltaHarness.get.openLog(spark, tablePath.toString) - val dataDir = outputDir.resolve("data") - Files.createDirectories(dataDir) // Commit index lines up with the table version it produced (commit 0 is the create at // version 0). Guard the coupling: each recorded op must map to exactly one version. @@ -143,65 +164,54 @@ class WriteSpecBuilder { s"write spec records ${commits.size} ops but table has ${finalVersion + 1} versions " + "(0.." + finalVersion + "); op index can no longer be used as the table version") - val highLevelDataOps = Set("insert", "create_table") - - // Enrich data-producing commits with the files they added so replay can reconstruct the - // table from copied parquet. High-level ops carry these as `dataFiles`; low-level commits - // already enumerate them in `addFiles`, whose `dataFile` paths we rewrite to the copies. - val enriched = commits.zipWithIndex.map { case (commit, idx) => - if (highLevelDataOps.contains(commit.operation)) { - val newFiles = newFilesAt(tablePath, idx) - if (newFiles.nonEmpty) { - val paths = newFiles.map(f => copyDataFile(tablePath, dataDir, idx, f)) - commit.copy(dataFiles = Some(paths)) - } else commit - } else if (commit.operation == "commit") { - commit.addFiles match { - case Some(files) if files.nonEmpty => - val rewritten = files.map { af => - val copied = copyDataFile(tablePath, dataDir, idx, af.dataFile) - af.copy(dataFile = copied) - } - commit.copy(addFiles = Some(rewritten)) - case _ => commit + // Materialize the spec's data files. The API took `rows`; the spec stores Parquet. High-level + // insert/replace-as-select rows are written to `data/commit_/part-00000.parquet` (full + // schema incl. partition columns, so replay can Append/RTAS). Low-level commits copy each + // add's already-written Parquet to `data/commit_/` (same in-table path). + val enriched = commits.zipWithIndex.map { + case (ins: InsertCommit, idx) => + ins.copy(dataFiles = materializeRows(spark, tablePath.toString, outputDir, idx)) + case (rep: ReplaceTableCommit, idx) => + rep.copy(dataFiles = materializeRows(spark, tablePath.toString, outputDir, idx)) + case (ll: LowLevelCommitOp, idx) => + // Materialize each low-level add's logical rows to the `dataFile` path recorded in the + // action (full schema incl. partition columns, so replay's writeFiles can partition/map). + lowLevelRows.get(idx).foreach { addRows => + val schema = RowParquet.schemaAt(spark, tablePath.toString, Some(idx.toLong), + includePartition = true) + ll.addFiles.getOrElse(Seq.empty).zip(addRows).foreach { case (af, rows) => + RowParquet.writeSingle(spark, schema, rows, outputDir.resolve(af.dataFile)) + } } - } else commit + ll + case (other, _) => other } - JsonUtil.writeSpec(outputDir.resolve("write_spec.json"), WriteSpec(enriched.toSeq)) - writeExpectedLatest(spark, log, tablePath, outputDir) - } - - /** - * Copy a table-relative data file into `data/commit_/`, preserving any `col=val/` - * partition-directory prefix, and return the spec-relative path to the copy. - */ - private def copyDataFile(tablePath: Path, dataDir: Path, idx: Int, relative: String): String = { - val src = tablePath.resolve(relative) - val dest = dataDir.resolve(s"commit_$idx").resolve(relative) - Files.createDirectories(dest.getParent) - if (Files.exists(src)) Files.copy(src, dest, StandardCopyOption.REPLACE_EXISTING) - s"data/commit_$idx/$relative" + // `write` is just another spec: it lives in `specs/_write.json` alongside read/snapshot + // specs, with its final-state under `expected/_write/`. + val writeName = s"${outputDir.getFileName.toString}_write" + JsonUtil.writeSpec( + outputDir.resolve("specs").resolve(s"$writeName.json"), WriteSpec(enriched.toSeq)) + writeExpectedLatest(spark, log, tablePath, outputDir.resolve("expected").resolve(writeName)) } /** - * Paths of the AddFile actions in commit `version`'s OWN log entry. Derived directly from - * that commit's `add` actions rather than diffing adjacent snapshots' active-file sets, - * which would misattribute files when a commit both adds and removes. + * Write the rows recorded for commit `idx` (if any) to `data/commit_/part-00000.parquet` + * using the table's schema at that version, and return the spec-relative path. None if the op + * recorded no rows. */ - private def newFilesAt(tablePath: Path, version: Int): Seq[String] = { - val commitFile = tablePath.resolve("_delta_log").resolve(f"$version%020d.json") - if (!Files.exists(commitFile)) return Seq.empty - val lines = new String(Files.readAllBytes(commitFile), "UTF-8") - .split("\n").filter(_.trim.nonEmpty) - lines.flatMap { line => - val node = JsonUtil.mapper.readTree(line) - Option(node.get("add")).map(_.get("path").asText()) - }.toSeq + private def materializeRows( + spark: SparkSession, tablePath: String, outputDir: Path, idx: Int): Option[Seq[String]] = { + rowData.get(idx).filter(_.nonEmpty).map { rows => + val schema = RowParquet.schemaAt(spark, tablePath, Some(idx.toLong), includePartition = true) + val rel = SpecLayout.commitDataFile(idx, "part-00000.parquet") + RowParquet.writeSingle(spark, schema, rows, outputDir.resolve(rel)) + Seq(rel) + } } /** - * Capture the expected final table state under `expected/latest/`: + * Capture the expected final table state under the write spec's `expected/_write/`: * - `table_content/` — Parquet of the latest snapshot's rows * - `table_version_metadata.json` — protocol + metadata of the latest snapshot */ @@ -209,8 +219,7 @@ class WriteSpecBuilder { spark: SparkSession, log: io.delta.workload.deltaharness.LogView, tablePath: Path, - outputDir: Path): Unit = { - val latestDir = outputDir.resolve("expected").resolve("latest") + latestDir: Path): Unit = { Files.createDirectories(latestDir) val contentDir = latestDir.resolve("table_content") diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala deleted file mode 100644 index 0654c90b..00000000 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecValidator.scala +++ /dev/null @@ -1,463 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload - -import java.nio.file.{Files, Path} - -import scala.collection.mutable -import scala.jdk.CollectionConverters._ - -import org.apache.spark.sql.{SaveMode, SparkSession} -import org.apache.spark.sql.types.{DataType, StructField, StructType} - -import io.delta.workload.deltaharness.{CommitRequest, DeltaHarness} - -/** - * Validates a write spec by: - * 1. Replaying its commits against a fresh table using the data files in `data/`. - * 2. Running every read/snapshot spec in `specs/` against the replayed table, - * comparing against `expected/`. - * - * CDF, checkpoint, CRC, domain-metadata, and appTxn specs are out of scope for the - * write spec and are ignored here. - */ -object WriteSpecValidator { - - /** - * Replay the write spec and validate read/snapshot specs against the result. - * - * @return warning/error messages (empty = all passed). A returned warning fails the - * generated workload; the orchestrator deletes the output so it auto-retries. - */ - def validate(spark: SparkSession, outputDir: Path, specName: String): Seq[String] = { - val writeSpecFile = outputDir.resolve("write_spec.json") - if (!Files.exists(writeSpecFile)) return Seq.empty - - val warnings = mutable.ArrayBuffer[String]() - val writeSpec = JsonUtil.readWriteSpec(writeSpecFile) - if (writeSpec.commits.isEmpty) return Seq.empty - - // An op the reference replayer intentionally does not replay (e.g. `restore`, whose replayed - // file paths differ from capture) means we skip REPLAY validation for the whole spec, but - // this is not a failure. A data op recorded with no data files is a capture bug and IS a - // failure. Check both before replaying anything. - writeSpec.commits.collectFirst { case c if isReplayUnsupported(c) => c.operation }.foreach { op => - println(s" Write spec validation skipping replay validation for $specName: " + - s"'$op' not supported by reference replayer") - return Seq.empty - } - writeSpec.commits.find(captureBugMissingDataFiles).foreach { c => - warnings += s"capture bug: '${c.operation}' recorded with no data files (not replayable)" - return warnings.toSeq - } - - val tempDir = Files.createTempDirectory(s"write_replay_$specName") - val replayTablePath = tempDir.resolve("replay_table") - - try { - val replayRef = s"delta.`${replayTablePath.toAbsolutePath}`" - writeSpec.commits.foreach(c => - replayCommit(spark, c, replayRef, replayTablePath.toAbsolutePath.toString, outputDir)) - - validateExpectedLatest(spark, replayTablePath, outputDir, specName, warnings) - - val specsDir = outputDir.resolve("specs") - if (Files.isDirectory(specsDir)) { - val specFiles = { - val stream = Files.list(specsDir) - try stream.iterator().asScala.filter(_.toString.endsWith(".json")).toSeq.sorted - finally stream.close() - } - for (specFile <- specFiles) { - val specJson = JsonUtil.mapper.readTree(Files.readAllBytes(specFile)) - val specType = Option(specJson.get("type")).map(_.asText()).getOrElse("") - val specFileName = specFile.getFileName.toString.stripSuffix(".json") - specType match { - case "read" => - try { - ReadCapture.validateFromSpec(spark, replayTablePath, - outputDir.resolve("expected").resolve(specFileName), specFile) - } catch { - case e: Throwable => warnings += s"read '$specFileName': ${e.getMessage}" - } - case "snapshot" => - validateSnapshotStructurally(spark, replayTablePath, specFile, specFileName, - writeSpec, warnings) - case _ => // out of scope (write/cdf/checkpoint/crc/...) - } - } - } - - if (warnings.isEmpty) { - println(s" Write spec validation PASSED for $specName") - } - } finally { - try org.apache.commons.io.FileUtils.deleteDirectory(tempDir.toFile) - catch { case _: Throwable => } - } - - warnings.toSeq - } - - /** Compare the replayed latest snapshot against `expected/latest/` (rows + protocol/metadata). */ - private def validateExpectedLatest( - spark: SparkSession, - replayTablePath: Path, - outputDir: Path, - specName: String, - warnings: mutable.ArrayBuffer[String]): Unit = { - val latestDir = outputDir.resolve("expected").resolve("latest") - val contentDir = latestDir.resolve("table_content") - if (Files.exists(contentDir)) { - val expected = JsonUtil.toRowMultiset(spark.read.parquet(contentDir.toString)) - val actual = JsonUtil.toRowMultiset( - spark.read.format("delta").load(replayTablePath.toAbsolutePath.toString)) - try JsonUtil.assertMultisetsEqual(expected, actual, s"${specName}_latest") - catch { case e: Throwable => warnings += e.getMessage } - } - } - - /** - * Validate a snapshot spec against the replayed table structurally. A replay is an - * independently created table, so its `id` and `createdTime` will never match the - * original. - * - * `protocol` and `configuration` are compared with portable, engine-neutral semantics so a - * spec validates against ANY conforming Delta writer, not just the one used to capture it: - * - `protocol`: capability check, not equality. The replay must meet every required floor - * (version >= expected, features superset of expected). A higher version or extra - * features on the replay are fine. - * - `configuration`: only the properties the write spec's own commits declared are checked - * (declared keys present + equal, removed keys absent). Engine-injected defaults are - * ignored. - * `schemaString`, `partitionColumns`, and `format` are still compared exactly. - */ - private def validateSnapshotStructurally( - spark: SparkSession, - replayTablePath: Path, - specFile: Path, - specFileName: String, - writeSpec: WriteSpec, - warnings: mutable.ArrayBuffer[String]): Unit = { - val spec = JsonUtil.readSnapshotSpec(specFile) - val (declaredKeys, removedKeys) = declaredConfiguration(writeSpec) - spec.expected.foreach { exp => - val log = DeltaHarness.get.openLog(spark, replayTablePath.toString) - val snapshot = (spec.version, spec.timestamp) match { - case (Some(v), _) => log.getSnapshotAt(v) - case _ => log.update() - } - val expProtocol = JsonUtil.mapper.valueToTree[com.fasterxml.jackson.databind.JsonNode]( - exp.protocol) - val replayProtocol = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") - protocolViolation(expProtocol, replayProtocol).foreach { reason => - warnings += s"snapshot '$specFileName': protocol mismatch ($reason)" - } - - val expectedMeta = JsonUtil.mapper.valueToTree[com.fasterxml.jackson.databind.JsonNode]( - exp.metadata) - val replayMeta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") - configurationViolation(expectedMeta, replayMeta, declaredKeys, removedKeys).foreach { - reason => warnings += s"snapshot '$specFileName': metadata.configuration mismatch ($reason)" - } - for (field <- Seq("schemaString", "partitionColumns", "format")) { - val expVal = canonicalField(expectedMeta, field) - val repVal = canonicalField(replayMeta, field) - if (expVal != repVal) warnings += s"snapshot '$specFileName': metadata.$field mismatch" - } - } - } - - /** - * Collect the configuration properties the write spec's commits explicitly declared. - * @return (declared keys still in effect, keys explicitly removed). A key removed after being - * set is reported only as removed. - */ - private def declaredConfiguration(writeSpec: WriteSpec): (Set[String], Set[String]) = { - val declared = mutable.LinkedHashSet[String]() - val removed = mutable.LinkedHashSet[String]() - for (commit <- writeSpec.commits) commit.operation match { - case "create_table" => - commit.properties.foreach(declared ++= _.keys) - case "update_properties" => - commit.set.foreach { s => declared ++= s.keys; removed --= s.keys } - commit.remove.foreach { r => removed ++= r; declared --= r } - case _ => () - } - (declared.toSet, removed.toSet) - } - - /** - * Check the replay protocol meets the expected one by capability: version floors and feature - * supersets. Returns a human-readable reason on the first violation, or None if compatible. - */ - private def protocolViolation( - expected: com.fasterxml.jackson.databind.JsonNode, - replay: com.fasterxml.jackson.databind.JsonNode): Option[String] = { - def intField(node: com.fasterxml.jackson.databind.JsonNode, name: String): Int = - if (node != null && node.has(name)) node.get(name).asInt() else 0 - def featureSet( - node: com.fasterxml.jackson.databind.JsonNode, name: String): Set[String] = - if (node != null && node.has(name) && node.get(name).isArray) { - node.get(name).elements().asScala.map(_.asText()).toSet - } else Set.empty - - val expReader = intField(expected, "minReaderVersion") - val expWriter = intField(expected, "minWriterVersion") - val repReader = intField(replay, "minReaderVersion") - val repWriter = intField(replay, "minWriterVersion") - val missingReaderFeatures = - featureSet(expected, "readerFeatures") -- featureSet(replay, "readerFeatures") - val missingWriterFeatures = - featureSet(expected, "writerFeatures") -- featureSet(replay, "writerFeatures") - - if (repReader < expReader) Some(s"minReaderVersion $repReader < $expReader") - else if (repWriter < expWriter) Some(s"minWriterVersion $repWriter < $expWriter") - else if (missingReaderFeatures.nonEmpty) - Some(s"missing readerFeatures ${missingReaderFeatures.mkString(",")}") - else if (missingWriterFeatures.nonEmpty) - Some(s"missing writerFeatures ${missingWriterFeatures.mkString(",")}") - else None - } - - /** - * Check the replay configuration against only the author-declared keys: each declared key must - * be present and equal on the replay, and each removed key must be absent. Engine-injected - * default properties are ignored. Returns a reason on the first violation, or None. - */ - private def configurationViolation( - expectedMeta: com.fasterxml.jackson.databind.JsonNode, - replayMeta: com.fasterxml.jackson.databind.JsonNode, - declaredKeys: Set[String], - removedKeys: Set[String]): Option[String] = { - val expConfig = Option(expectedMeta.get("configuration")).filterNot(_.isNull) - val repConfig = Option(replayMeta.get("configuration")).filterNot(_.isNull) - def get( - config: Option[com.fasterxml.jackson.databind.JsonNode], - key: String): Option[String] = - config.flatMap(c => Option(c.get(key))).filterNot(_.isNull).map(_.asText()) - - val mismatch = declaredKeys.toSeq.sorted.collectFirst { - case key if get(repConfig, key) != get(expConfig, key) => - s"key '$key' expected ${get(expConfig, key)}, got ${get(repConfig, key)}" - } - mismatch.orElse { - removedKeys.toSeq.sorted.collectFirst { - case key if get(repConfig, key).isDefined => s"removed key '$key' still present" - } - } - } - - /** Canonicalize a single metadata field (or "null" if absent) for comparison. */ - private def canonicalField( - node: com.fasterxml.jackson.databind.JsonNode, field: String): String = - if (node.has(field) && !node.get(field).isNull) JsonUtil.canonicalJson(node.get(field)) - else "null" - - /** Ops the reference replayer intentionally does not replay; their presence skips replay. */ - private def isReplayUnsupported(commit: WriteCommit): Boolean = - commit.operation == "restore" - - /** - * A data op recorded with no/empty data files. Capture should always record the files an - * `insert`/`create_table` wrote, so an empty record is a capture bug, not a missing feature. - */ - private def captureBugMissingDataFiles(commit: WriteCommit): Boolean = commit.operation match { - case "insert" => commit.dataFiles.forall(_.isEmpty) - case _ => false - } - - /** Replay a single commit against the replay table. */ - private def replayCommit( - spark: SparkSession, - commit: WriteCommit, - tableRef: String, - tablePath: String, - outputDir: Path): Unit = { - commit.operation match { - case "create_table" => - commit.schema.foreach { schema => - var sql = s"CREATE TABLE $tableRef (${schemaToColDefs(schema)}) USING delta" - commit.partitionColumns.filter(_.nonEmpty).foreach { parts => - sql += s" PARTITIONED BY (${parts.mkString(", ")})" - } - commit.properties.filter(_.nonEmpty).foreach { props => - sql += s" TBLPROPERTIES (${props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" - } - spark.sql(sql) - commit.dataFiles.foreach(loadDataFiles(spark, _, tablePath, outputDir)) - } - - case "insert" => - commit.dataFiles.foreach(loadDataFiles(spark, _, tablePath, outputDir)) - - case "delete" => - spark.sql(s"DELETE FROM $tableRef WHERE ${commit.predicate.getOrElse("true")}") - - case "update" => - commit.set.filter(_.nonEmpty).foreach { setMap => - val setClauses = setMap.map { case (k, v) => s"`$k` = $v" }.mkString(", ") - spark.sql(s"UPDATE $tableRef SET $setClauses WHERE ${commit.predicate.getOrElse("true")}") - } - - case "update_properties" => - val setClause = commit.set.getOrElse(Map.empty) - .map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") - if (setClause.nonEmpty) spark.sql(s"ALTER TABLE $tableRef SET TBLPROPERTIES ($setClause)") - val unset = commit.remove.getOrElse(Seq.empty) - if (unset.nonEmpty) { - spark.sql(s"ALTER TABLE $tableRef UNSET TBLPROPERTIES (${unset.map(k => s"'$k'").mkString(", ")})") - } - - case "evolve_schema" => replayEvolveSchema(spark, commit, tablePath) - - case "commit" => replayLowLevelCommit(spark, commit, tablePath, outputDir) - - case _ => () - } - } - - /** Replay schema add/rename/drop via a single metadata-update commit. */ - private def replayEvolveSchema(spark: SparkSession, commit: WriteCommit, tablePath: String): Unit = { - val log = DeltaHarness.get.openLog(spark, tablePath) - val metaNode = Option(JsonUtil.mapper.readTree(log.update().metadataJson).get("metaData")) - .getOrElse(throw new IllegalStateException( - s"evolve_schema replay: no metaData action in current snapshot of $tablePath")) - val schemaString = Option(metaNode.get("schemaString")) - .getOrElse(throw new IllegalStateException( - s"evolve_schema replay: metaData has no schemaString in $tablePath")) - val currentSchema = - DataType.fromJson(schemaString.asText()).asInstanceOf[StructType] - var fields = currentSchema.fields.toBuffer - - commit.addColumns.foreach { cols => - cols.asInstanceOf[Seq[Map[String, Any]]].foreach { col => - val name = col("name").toString - val nullable = col.get("nullable").forall(_.asInstanceOf[Boolean]) - fields += StructField(name, DataType.fromDDL(typeToSql(col("type"))), nullable) - } - } - commit.renameColumns.foreach { renames => - for ((oldName, newName) <- renames) { - val idx = fields.indexWhere(_.name == oldName) - if (idx >= 0) fields(idx) = fields(idx).copy(name = newName) - } - } - commit.dropColumns.foreach { drops => - fields = fields.filterNot(f => drops.contains(f.name)) - } - - DeltaHarness.get.commit(spark, tablePath, - CommitRequest(schemaJson = Some(StructType(fields.toSeq).json))) - } - - /** - * Replay a low-level commit. `addFiles[].dataFile` was rewritten at capture to - * `data/commit_N/`; we read from there and strip that prefix to recover - * the table-relative path. Schema for a low-level commit is stored as Delta schema JSON, - * so we round-trip it through `schemaToColDefs` to a DDL string for `LowLevelCommit`. - */ - private def replayLowLevelCommit( - spark: SparkSession, - commit: WriteCommit, - tablePath: String, - outputDir: Path): Unit = { - LowLevelCommit.apply( - spark, tablePath, - schemaDDL = commit.schema.map(schemaToColDefs), - tableProperties = commit.tableProperties, - txn = commit.txn, - addFiles = commit.addFiles, - removeFiles = commit.removeFiles, - addDomainMetadata = commit.addDomainMetadata, - removeDomainMetadata = commit.removeDomainMetadata, - resolveDataFile = rel => outputDir.resolve(rel), - assignPath = af => stripDataCommitPrefix(af.dataFile)) - } - - /** Strip the `data/commit_N/` prefix a captured commit prepends to its `addFiles` paths. */ - private def stripDataCommitPrefix(dataFile: String): String = { - val p = java.nio.file.Paths.get(dataFile) - if (p.getNameCount > 2) p.subpath(2, p.getNameCount).toString else dataFile - } - - /** - * Read parquet data files and append them to the replay table. Files for a partitioned - * table are stored under `col=val/` directories within their `data/commit_N/` base; we - * read with that base as `basePath` so Spark recovers the partition columns, otherwise - * the appended DataFrame would be missing them. - */ - private def loadDataFiles( - spark: SparkSession, - files: Seq[String], - tablePath: String, - outputDir: Path): Unit = { - val resolved = files.map(outputDir.resolve).filter(Files.exists(_)) - if (resolved.isEmpty) return - - // All files in one insert commit share a `data/commit_N/` base. - val basePath = outputDir.resolve( - java.nio.file.Paths.get(files.head).subpath(0, 2)).toAbsolutePath - val isPartitioned = resolved.exists(_.getParent != basePath) - val df = if (isPartitioned) { - spark.read.option("basePath", basePath.toString) - .parquet(resolved.map(_.toAbsolutePath.toString): _*) - } else { - spark.read.parquet(resolved.map(_.toAbsolutePath.toString): _*) - } - df.write.format("delta").mode(SaveMode.Append).save(tablePath) - } - - private def schemaToColDefs(schema: Any): String = { - val schemaMap = schema.asInstanceOf[Map[String, Any]] - val fields = schemaMap("fields").asInstanceOf[Seq[Map[String, Any]]] - fields.map { f => - val name = f("name").toString - val nullable = f.get("nullable").forall(_.asInstanceOf[Boolean]) - s"`$name` ${typeToSql(f("type"))}${if (!nullable) " NOT NULL" else ""}" - }.mkString(", ") - } - - private def typeToSql(typeValue: Any): String = typeValue match { - case s: String => s match { - case "integer" => "INT" - case "long" => "BIGINT" - case "short" => "SMALLINT" - case "byte" => "TINYINT" - case "float" => "FLOAT" - case "double" => "DOUBLE" - case "boolean" => "BOOLEAN" - case "string" => "STRING" - case "binary" => "BINARY" - case "date" => "DATE" - case "timestamp" => "TIMESTAMP" - case "timestamp_ntz" => "TIMESTAMP_NTZ" - case other => other.toUpperCase - } - case m: Map[_, _] => - val typeMap = m.asInstanceOf[Map[String, Any]] - typeMap("type") match { - case "struct" => - val fields = typeMap("fields").asInstanceOf[Seq[Map[String, Any]]] - s"STRUCT<${fields.map(f => s"${f("name")}: ${typeToSql(f("type"))}").mkString(", ")}>" - case "array" => s"ARRAY<${typeToSql(typeMap("elementType"))}>" - case "map" => s"MAP<${typeToSql(typeMap("keyType"))}, ${typeToSql(typeMap("valueType"))}>" - case other => other.toString.toUpperCase - } - case _ => typeValue.toString.toUpperCase - } -} diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala index 24152bc3..579f6d63 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala @@ -21,29 +21,23 @@ import org.apache.spark.sql.{DataFrame, SparkSession} import io.delta.workload.{AddDomainMetadata, AppTxn} /** - * A single add-file action for a low-level commit, fully resolved by the caller: `path` is the - * in-table relative path and `size` is the on-disk size of the copied file. + * A remove-file action (tombstone) for a low-level commit, by in-table `path`. The implementation + * tombstones the matching active file via the engine, so the tombstone inherits its + * `partitionValues`/`size`/`stats` (column-mapping- and partition-correct) — the caller supplies + * only the path and the `dataChange` flag. */ -case class CommitAddFile( - path: String, - partitionValues: Map[String, String], - size: Long, - dataChange: Boolean) - -/** A single remove-file action for a low-level commit: `path` is the in-table relative path. */ case class CommitRemoveFile(path: String, dataChange: Boolean) /** - * A platform-neutral description of a low-level commit. Carries only plain Scala / Spark-SQL - * types and the generator's neutral action case classes — never `org.apache.spark.sql.delta.*`. + * A platform-neutral description of a low-level commit's NON-data actions. Carries only plain + * Scala types and the generator's neutral action case classes — never `org.apache.spark.sql.delta.*`. + * Data files are added separately, via [[DeltaHarness.commitWithData]], so the engine computes + * their physical layout / stats (column-mapping aware). * * @param schemaJson `StructType.json` to set as `metadata.schemaString`, or None to keep - * the current schema - * @param properties properties merged into the current `metadata.configuration`, or None - * to keep the current configuration + * @param properties properties merged into `metadata.configuration`, or None to keep * @param setTransaction optional `SetTransaction` (appId/version) action - * @param addFiles resolved add-file actions (in-table path + computed size) - * @param removeFiles resolved remove-file actions to tombstone + * @param removeFiles remove-file actions to tombstone (by in-table path) * @param addDomainMetadata domain-metadata entries to add * @param removeDomainMetadata domain names to tombstone */ @@ -51,7 +45,6 @@ case class CommitRequest( schemaJson: Option[String] = None, properties: Option[Map[String, String]] = None, setTransaction: Option[AppTxn] = None, - addFiles: Seq[CommitAddFile] = Nil, removeFiles: Seq[CommitRemoveFile] = Nil, addDomainMetadata: Seq[AddDomainMetadata] = Nil, removeDomainMetadata: Seq[String] = Nil) @@ -80,6 +73,19 @@ trait DeltaHarness { * the `openLog` cache-clearing contract so consumers always see a fresh view. */ def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Unit + + /** + * Like [[commit]], but also writes data files through the engine's own write path. Each path in + * `addDataParquet` points to a Parquet file of LOGICAL rows; the implementation reads it and + * writes it into the table via the engine (honoring column mapping and partitioning, computing + * stats), then commits the produced AddFile actions together with `req`'s actions in one commit. + * + * @return the in-table `AddFile.path`s the engine produced (in `addDataParquet` order; a file + * may yield several adds when partitioned), so callers can reference them later. + */ + def commitWithData( + spark: SparkSession, tablePath: String, + addDataParquet: Seq[String], req: CommitRequest): Seq[String] } trait LogView { diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala index f8641d45..b2e34fa1 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala @@ -20,7 +20,7 @@ import scala.collection.mutable import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, Snapshot} -import org.apache.spark.sql.delta.actions.{Action, AddFile, DomainMetadata, RemoveFile, SetTransaction} +import org.apache.spark.sql.delta.actions.{Action, AddFile, DomainMetadata, FileAction, SetTransaction} class DeltaSparkHarness extends DeltaHarness { override def openLog(spark: SparkSession, tablePath: String): LogView = { @@ -29,47 +29,52 @@ class DeltaSparkHarness extends DeltaHarness { } override def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Unit = { + commitWithData(spark, tablePath, Seq.empty, req) + } + + override def commitWithData( + spark: SparkSession, tablePath: String, + addDataParquet: Seq[String], req: CommitRequest): Seq[String] = { DeltaLog.clearCache() val deltaLog = DeltaLog.forTable(spark, tablePath) - val transaction = deltaLog.startTransaction() - val actions = mutable.ArrayBuffer[Action]() + val txn = deltaLog.startTransaction() + val now = System.currentTimeMillis() + // Metadata (schema/properties) is updated first so writeFiles writes against the new schema. if (req.schemaJson.isDefined || req.properties.isDefined) { - val currentMetadata = transaction.metadata - val newSchemaString = req.schemaJson.getOrElse(currentMetadata.schemaString) - val newConfig = req.properties - .map(props => currentMetadata.configuration ++ props) - .getOrElse(currentMetadata.configuration) - transaction.updateMetadata( - currentMetadata.copy(schemaString = newSchemaString, configuration = newConfig)) + val current = txn.metadata + txn.updateMetadata(current.copy( + schemaString = req.schemaJson.getOrElse(current.schemaString), + configuration = req.properties.map(current.configuration ++ _).getOrElse(current.configuration))) } - req.setTransaction.foreach { t => - actions += SetTransaction(t.appId, t.version, Some(System.currentTimeMillis())) - } - req.addFiles.foreach { f => - actions += AddFile( - path = f.path, - partitionValues = f.partitionValues, - size = f.size, - modificationTime = System.currentTimeMillis(), - dataChange = f.dataChange) - } - req.removeFiles.foreach { f => - actions += RemoveFile( - path = f.path, - deletionTimestamp = Some(System.currentTimeMillis()), - dataChange = f.dataChange) + // Engine-written data files: column mapping / partitioning / stats all handled by writeFiles. + // Commit ALL file actions it returns (AddFile + AddCDCFile when change data feed is enabled); + // only the AddFile paths are returned (for ordinal-based remove resolution). + val fileActions: Seq[FileAction] = addDataParquet.flatMap { p => + txn.writeFiles(spark.read.parquet(p)) } - req.addDomainMetadata.foreach { dm => - actions += DomainMetadata(dm.domain, dm.configuration, removed = false) - } - req.removeDomainMetadata.foreach { domain => - actions += DomainMetadata(domain, "", removed = true) + + val manual = mutable.ArrayBuffer[Action]() + req.setTransaction.foreach { t => manual += SetTransaction(t.appId, t.version, Some(now)) } + + if (req.removeFiles.nonEmpty) { + // Tombstone the matching ACTIVE file so the RemoveFile inherits its partitionValues/size/ + // stats (extendedFileMetadata) — column-mapping- and partition-correct, derived not guessed. + val active = txn.snapshot.allFiles.collect().map(a => a.path -> a).toMap + req.removeFiles.foreach { rf => + val add = active.getOrElse(rf.path, throw new IllegalStateException( + s"removeFiles references a path not active in the table: ${rf.path}")) + manual += add.removeWithTimestamp(now, rf.dataChange) + } } - transaction.commit(actions.toSeq, DeltaOperations.ManualUpdate) + req.addDomainMetadata.foreach { dm => manual += DomainMetadata(dm.domain, dm.configuration, removed = false) } + req.removeDomainMetadata.foreach { domain => manual += DomainMetadata(domain, "", removed = true) } + + txn.commit(fileActions ++ manual.toSeq, DeltaOperations.ManualUpdate) DeltaLog.clearCache() + fileActions.collect { case a: AddFile => a.path } } } diff --git a/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorSuite.scala b/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorSuite.scala index 74f4c809..8864730b 100644 --- a/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorSuite.scala @@ -96,6 +96,26 @@ class WorkloadValidatorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor s"expected >= 2 specs validated, got ${result.passed}") } + test("validates a write workload (write-derived dispatch + replay)") { + spark.sql("DROP TABLE IF EXISTS wtbl") + generate("val_write") { + val w = createTableOp("wtbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + commitOp(w, addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + // Validate the write workload standalone (post-generation): the write spec replays into a + // fresh table and the write-derived read/snapshot specs validate against it. + val result = WorkloadValidator.validateTestDir(spark, outputDir.resolve("val_write")) + assert(result.success, + s"write workload should validate; errors:\n ${result.errors.mkString("\n ")}") + assert(result.passed >= 3, + s"expected >= 3 specs (write + read + snapshot), got ${result.passed}") + } + test("validator reports failures when the table diverges from the spec") { spark.sql("DROP TABLE IF EXISTS tbl2") generate("val_divergence") { diff --git a/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorUnitSuite.scala b/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorUnitSuite.scala new file mode 100644 index 00000000..bd6f58fb --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorUnitSuite.scala @@ -0,0 +1,139 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload + +import com.fasterxml.jackson.databind.JsonNode +import org.scalatest.funsuite.AnyFunSuite + +/** + * Direct unit tests for [[WorkloadValidator]]'s pure comparison functions. These branches (esp. + * the implicit-vs-explicit protocol-feature reconciliation and the column-mapping schema + * normalization) are hard for the capture->replay oracle to provoke, because replay produces a + * conforming table — so they are exercised here with hand-built inputs. No Spark. + */ +class WorkloadValidatorUnitSuite extends AnyFunSuite { + + private def node(json: String): JsonNode = JsonUtil.mapper.readTree(json) + + // ---- protocolViolation: version floors + effective (implied + explicit) feature supersets ---- + + test("protocol: identical legacy protocols are compatible") { + val p = node("""{"minReaderVersion":1,"minWriterVersion":2}""") + assert(WorkloadValidator.protocolViolation(p, p).isEmpty) + } + + test("protocol: lower replay version is a violation") { + val exp = node("""{"minReaderVersion":1,"minWriterVersion":5}""") + val rep = node("""{"minReaderVersion":1,"minWriterVersion":3}""") + assert(WorkloadValidator.protocolViolation(exp, rep).exists(_.contains("minWriterVersion"))) + } + + test("protocol: version-implied feature on expected satisfied by explicit feature on replay") { + // Expected writer v2 implies {appendOnly, invariants}; replay (3,7) lists them explicitly. + val exp = node("""{"minReaderVersion":1,"minWriterVersion":2}""") + val rep = node("""{"minReaderVersion":3,"minWriterVersion":7, + "readerFeatures":[],"writerFeatures":["appendOnly","invariants","deletionVectors"]}""") + assert(WorkloadValidator.protocolViolation(exp, rep).isEmpty) + } + + test("protocol: explicit expected feature missing from replay is a violation") { + val exp = node("""{"minReaderVersion":3,"minWriterVersion":7, + "readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}""") + val rep = node("""{"minReaderVersion":3,"minWriterVersion":7, + "readerFeatures":[],"writerFeatures":[]}""") + assert(WorkloadValidator.protocolViolation(exp, rep).exists(_.contains("deletionVectors"))) + } + + test("protocol: higher replay version / extra features is allowed (capability floor)") { + val exp = node("""{"minReaderVersion":1,"minWriterVersion":2}""") + val rep = node("""{"minReaderVersion":3,"minWriterVersion":7, + "readerFeatures":["columnMapping"],"writerFeatures":["appendOnly","invariants","columnMapping"]}""") + assert(WorkloadValidator.protocolViolation(exp, rep).isEmpty) + } + + // ---- configurationViolation: only declared keys checked; removed keys must be absent ---- + + private val emptyMeta = node("""{"configuration":{}}""") + + test("config: declared key present and equal is compatible") { + val exp = node("""{"configuration":{"delta.enableChangeDataFeed":"true"}}""") + val rep = node("""{"configuration":{"delta.enableChangeDataFeed":"true","x":"injected"}}""") + assert(WorkloadValidator.configurationViolation( + exp, rep, Set("delta.enableChangeDataFeed"), Set.empty).isEmpty) + } + + test("config: declared key missing on replay is a violation") { + val exp = node("""{"configuration":{"delta.enableChangeDataFeed":"true"}}""") + assert(WorkloadValidator.configurationViolation( + exp, emptyMeta, Set("delta.enableChangeDataFeed"), Set.empty) + .exists(_.contains("delta.enableChangeDataFeed"))) + } + + test("config: removed key still present on replay is a violation") { + val rep = node("""{"configuration":{"delta.enableChangeDataFeed":"true"}}""") + assert(WorkloadValidator.configurationViolation( + emptyMeta, rep, Set.empty, Set("delta.enableChangeDataFeed")) + .exists(_.contains("delta.enableChangeDataFeed"))) + } + + test("config: engine-injected (undeclared) keys are ignored") { + val exp = node("""{"configuration":{}}""") + val rep = node("""{"configuration":{"delta.columnMapping.maxColumnId":"3"}}""") + assert(WorkloadValidator.configurationViolation(exp, rep, Set.empty, Set.empty).isEmpty) + } + + // ---- normalizedSchema: per-table column-mapping ids are not a comparison target ---- + + test("schema: differing column-mapping physicalName/id normalize to equal") { + def schema(phys: String, id: Int) = + s"""{"schemaString":"{\\"type\\":\\"struct\\",\\"fields\\":[{\\"name\\":\\"id\\",""" + + s"""\\"type\\":\\"integer\\",\\"nullable\\":true,\\"metadata\\":""" + + s"""{\\"delta.columnMapping.physicalName\\":\\"$phys\\",\\"delta.columnMapping.id\\":$id}}]}"}""" + val a = node(schema("col-aaaa", 1)) + val b = node(schema("col-bbbb", 1)) + assert(WorkloadValidator.normalizedSchema(a) == WorkloadValidator.normalizedSchema(b)) + } + + test("schema: a real difference (column name) is NOT masked") { + val a = node("""{"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",""" + + """\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"}""") + val b = node("""{"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"renamed\",""" + + """\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"}""") + assert(WorkloadValidator.normalizedSchema(a) != WorkloadValidator.normalizedSchema(b)) + } + + // ---- declaredConfiguration: replace_table resets, update_properties tracks set/remove ---- + + test("declaredConfiguration: replace_table resets prior declarations") { + val spec = WriteSpec(Seq( + CreateTableCommit(schema = Map("fields" -> Seq.empty[Any]), properties = Some(Map("a" -> "1"))), + UpdatePropertiesCommit(set = Some(Map("b" -> "2"))), + ReplaceTableCommit(schema = Map("fields" -> Seq.empty[Any]), properties = Some(Map("c" -> "3"))))) + val (declared, removed) = WorkloadValidator.declaredConfiguration(spec) + assert(declared == Set("c"), s"declared=$declared") + assert(removed.isEmpty, s"removed=$removed") + } + + test("declaredConfiguration: unset moves a key from declared to removed") { + val spec = WriteSpec(Seq( + CreateTableCommit(schema = Map("fields" -> Seq.empty[Any]), properties = Some(Map("a" -> "1"))), + UpdatePropertiesCommit(remove = Some(Seq("a"))))) + val (declared, removed) = WorkloadValidator.declaredConfiguration(spec) + assert(declared.isEmpty, s"declared=$declared") + assert(removed == Set("a"), s"removed=$removed") + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala index 393ffc3e..a8f5ef0f 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala @@ -18,7 +18,7 @@ package io.delta.workload.tables import io.delta.workload.WorkloadTestSuite -/** Basic write workloads producing write_spec.json for Delta writer testing. */ +/** Basic write workloads producing a write spec (specs/<name>_write.json) for Delta writer testing. */ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { test("create_and_read") { @@ -176,6 +176,90 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { snapshotSpec(t) } + test("replace_table_as_select") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq( + Map("id" -> 1, "name" -> "alice"), + Map("id" -> 2, "name" -> "bob"))) + // Replace-as-select: new schema + new data in a single commit. + replaceTableOp(w, + schema = "id INT, label STRING, score DOUBLE", + rows = Seq( + Map("id" -> 10, "label" -> "x", "score" -> 1.5), + Map("id" -> 20, "label" -> "y", "score" -> 2.5))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, version = 1, name = "read_before_replace") + readSpec(t, predicate = "score > 2.0", name = "read_high_score") + snapshotSpec(t) + } + + test("replace_table_schema_only") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + // Pure replace (no data) then insert under the new schema. + replaceTableOp(w, + schema = "a BIGINT, b STRING", + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("a" -> 100L, "b" -> "hello"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("replace_table_as_select_partitioned") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + replaceTableOp(w, + schema = "id INT, region STRING, revenue INT", + partitionColumns = Seq("region"), + rows = Seq( + Map("id" -> 10, "region" -> "east", "revenue" -> 100), + Map("id" -> 20, "region" -> "west", "revenue" -> 200), + Map("id" -> 30, "region" -> "east", "revenue" -> 300))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "region = 'east'", name = "read_east") + readSpec(t, predicate = "revenue > 150", name = "read_high") + snapshotSpec(t) + } + + test("alter_rename_column") { + val w = createTableOp("tbl", schema = "id INT, name STRING", + properties = Map("delta.columnMapping.mode" -> "name")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"), Map("id" -> 2, "name" -> "bob"))) + renameColumnOp(w, "name", "full_name") + insertOp(w, Seq(Map("id" -> 3, "full_name" -> "charlie"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, version = 1, name = "read_before_rename") + readSpec(t, predicate = "full_name = 'alice'", name = "read_by_renamed") + snapshotSpec(t) + } + + test("alter_drop_column") { + val w = createTableOp("tbl", schema = "id INT, name STRING, scratch STRING", + properties = Map("delta.columnMapping.mode" -> "name")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "a", "scratch" -> "x"))) + dropColumnsOp(w, Seq("scratch")) + insertOp(w, Seq(Map("id" -> 2, "name" -> "b"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, version = 1, name = "read_before_drop") + snapshotSpec(t) + } + + test("alter_unset_properties") { + val w = createTableOp("tbl", schema = "id INT, value INT", + properties = Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 1, "value" -> 10), Map("id" -> 2, "value" -> 20))) + setPropertiesOp(w, Map("delta.enableDeletionVectors" -> "true")) + unsetPropertiesOp(w, Seq("delta.enableChangeDataFeed")) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + snapshotSpec(t) + } + test("delete_with_dvs") { val w = createTableOp("tbl", schema = "id INT, name STRING, score INT", diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala index 90105d9d..abfcaa64 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteCommitSuite.scala @@ -16,26 +16,21 @@ package io.delta.workload.tables -import java.nio.file.{Files, Path} +import java.nio.file.Files import scala.jdk.CollectionConverters._ -import io.delta.workload.{AddDomainMetadata, AddFileAction, AppTxn, JsonUtil, - RemoveFileAction, TableHandle, WorkloadTestSuite} +import io.delta.workload.{AddDomainMetadata, AddFileInput, AppTxn, JsonUtil, TableHandle, + WorkloadTestSuite} /** - * Low-level `commit` write workloads. Each case drives [[commitOp]] to produce a real - * `_delta_log` entry, asserts the produced action independently of replay, and relies on - * the generator's round-trip validation (replay + re-check specs) for end-to-end coverage. + * Low-level `commit` write workloads. Each case drives [[commitOp]] to produce a real `_delta_log` + * entry, asserts the produced action independently of replay, and relies on the generator's + * round-trip validation (replay + re-check specs) for end-to-end coverage. Low-level adds supply + * LOGICAL rows; the engine writes the files (paths/stats/partitionValues are engine-derived). */ class WriteCommitSuite extends WorkloadTestSuite("write_commit") { - /** Active add-file relative paths in the live table's latest version. */ - private def liveAddPaths(t: TableHandle): Seq[String] = { - val log = io.delta.workload.deltaharness.DeltaHarness.get.openLog(spark, t.sourcePath.toString) - log.update().allFiles.select("path").collect().map(_.getString(0)).toSeq - } - /** Action inner-objects of a given type in a commit's `_delta_log/NNN.json`. */ private def actionsAt(t: TableHandle, version: Long, action: String): Seq[com.fasterxml.jackson.databind.JsonNode] = { val commitFile = t.sourcePath.resolve("_delta_log").resolve(f"$version%020d.json") @@ -88,17 +83,12 @@ class WriteCommitSuite extends WorkloadTestSuite("write_commit") { test("commit_add_file") { val w = createTableOp("tbl", schema = "id INT, name STRING") insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) - val t0 = registerWriteSpec(w) - // Add a second data file via a low-level commit. Its bytes come from the existing insert - // file, but commitOp copies them to a fresh in-table path, so the table now holds two - // adds (the alice row appears twice). - val source = liveAddPaths(t0).head - commitOp(w, addFiles = Some(Seq(AddFileAction(dataFile = source)))) + // Add a second data file via a low-level commit; the engine writes it and assigns the path. + commitOp(w, addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) val t = registerWriteSpec(w) val v = latestVersion(t) - val adds = actionsAt(t, v, "add") - assert(adds.size == 1, s"expected one add action at v$v, got ${adds.size}") + assert(actionsAt(t, v, "add").size == 1, s"expected one add action at v$v") readSpec(t, name = "read_all") snapshotSpec(t) @@ -107,21 +97,127 @@ class WriteCommitSuite extends WorkloadTestSuite("write_commit") { test("commit_remove_file") { val w = createTableOp("tbl", schema = "id INT, name STRING") insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) - val t0 = registerWriteSpec(w) - // Add a low-level file (deterministic, capture-assigned path), then tombstone it. Removing - // a low-level-added file (rather than an insert file) keeps the path stable across replay. - val source = liveAddPaths(t0).head - commitOp(w, addFiles = Some(Seq(AddFileAction(dataFile = source)))) - val added = actionsAt(t0, latestVersion(t0), "add").head.get("path").asText() - commitOp(w, removeFiles = Some(Seq(RemoveFileAction(path = added)))) + // Add a file via a low-level commit, then tombstone it by referencing that commit's ordinal. + val added = commitOp(w, addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) val t = registerWriteSpec(w) + val addedPath = actionsAt(t, added.toLong, "add").head.get("path").asText() + commitOp(w, removeFiles = Some(Seq(added))) val v = latestVersion(t) val removes = actionsAt(t, v, "remove") assert(removes.size == 1, s"expected one remove action at v$v, got ${removes.size}") - assert(removes.head.get("path").asText() == added) + assert(removes.head.get("path").asText() == addedPath, "remove should tombstone the added file") readSpec(t, name = "read_after_remove") snapshotSpec(t) } + + test("commit_add_file_with_stats") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + // Stats are engine-computed from the rows (numRecords + per-column min/max/nullCount). + commitOp(w, addFiles = Some(Seq(AddFileInput( + rows = Seq(Map("id" -> 2, "name" -> "bob"), Map("id" -> 3, "name" -> "cara")))))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + val add = actionsAt(t, v, "add").head + val stats = JsonUtil.mapper.readTree(add.get("stats").asText()) + assert(stats.get("numRecords").asLong() == 2, s"stats: ${add.get("stats").asText()}") + assert(stats.get("minValues").get("id").asInt() == 2) + assert(stats.get("maxValues").get("id").asInt() == 3) + + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("commit_partitioned_add") { + val w = createTableOp("tbl", schema = "id INT, region STRING, value INT", + partitionColumns = Seq("region")) + insertOp(w, Seq(Map("id" -> 1, "region" -> "east", "value" -> 10))) + // The rows carry the partition column; the engine partitions and sets partitionValues. + commitOp(w, addFiles = Some(Seq(AddFileInput( + rows = Seq(Map("id" -> 2, "region" -> "west", "value" -> 20)))))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + val add = actionsAt(t, v, "add").head + assert(add.get("partitionValues").get("region").asText() == "west") + + readSpec(t, name = "read_all") + readSpec(t, predicate = "region = 'west'", name = "read_west") + snapshotSpec(t) + } + + test("commit_multi_file") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + commitOp(w, addFiles = Some(Seq( + AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob"))), + AddFileInput(rows = Seq(Map("id" -> 3, "name" -> "cara")))))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + assert(actionsAt(t, v, "add").size == 2, s"expected two add actions at v$v") + + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("commit_with_schema_and_properties") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + // A single low-level commit that evolves the schema, sets a property, AND adds a file. + commitOp(w, + schemaDDL = Some("id INT, name STRING, extra STRING"), + tableProperties = Some(Map("delta.appendOnly" -> "true")), + addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + val meta = actionsAt(t, v, "metaData").head + assert(meta.get("schemaString").asText().contains("extra"), + s"schema: ${meta.get("schemaString").asText()}") + assert(meta.get("configuration").get("delta.appendOnly").asText() == "true") + + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("commit_remove_multi_file_add") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + // One low-level commit adds TWO files; removing by that ordinal must tombstone BOTH. + val added = commitOp(w, addFiles = Some(Seq( + AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob"))), + AddFileInput(rows = Seq(Map("id" -> 3, "name" -> "cara")))))) + val t = registerWriteSpec(w) + val addedPaths = actionsAt(t, added.toLong, "add").map(_.get("path").asText()).toSet + assert(addedPaths.size == 2, s"expected two adds at v$added, got $addedPaths") + commitOp(w, removeFiles = Some(Seq(added))) + + val v = latestVersion(t) + val removed = actionsAt(t, v, "remove").map(_.get("path").asText()).toSet + assert(removed == addedPaths, s"both files tombstoned: removed=$removed added=$addedPaths") + + readSpec(t, name = "read_all") // only id=1 remains + snapshotSpec(t) + } + + test("commit_with_column_mapping") { + // Low-level adds go through the engine write path, so column mapping is handled: the logical + // rows are written under the table's physical names with engine-computed (physical-keyed) stats. + val w = createTableOp("tbl", schema = "id INT, name STRING", + properties = Map("delta.columnMapping.mode" -> "name")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + commitOp(w, addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) + val t = registerWriteSpec(w) + + val v = latestVersion(t) + assert(actionsAt(t, v, "add").size == 1, s"expected one add action at v$v") + + readSpec(t, name = "read_all") + readSpec(t, predicate = "name = 'bob'", name = "read_bob") + snapshotSpec(t) + } } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteSequencesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteSequencesSuite.scala new file mode 100644 index 00000000..a5aa2023 --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteSequencesSuite.scala @@ -0,0 +1,118 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import io.delta.workload.{AddDomainMetadata, AddFileInput, AppTxn, WorkloadTestSuite} + +/** + * Multi-operation write workloads: long op sequences, property churn, schema evolution chains, + * and high-level/low-level mixes. Stresses the commit-index==version invariant and time-travel + * across many versions. + */ +class WriteSequencesSuite extends WorkloadTestSuite("write_sequences") { + + test("insert_update_delete_chain") { + val w = createTableOp("tbl", schema = "id INT, status STRING, n INT", + properties = Map("delta.enableDeletionVectors" -> "true")) + insertOp(w, Seq( + Map("id" -> 1, "status" -> "new", "n" -> 0), + Map("id" -> 2, "status" -> "new", "n" -> 0), + Map("id" -> 3, "status" -> "new", "n" -> 0))) + updateOp(w, predicate = "id <= 2", set = Map("status" -> "'active'", "n" -> "n + 1")) + deleteOp(w, predicate = "id = 3") + insertOp(w, Seq(Map("id" -> 4, "status" -> "new", "n" -> 5))) + updateOp(w, predicate = "status = 'new'", set = Map("n" -> "99")) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, version = 1, name = "read_v1") + readSpec(t, version = 2, name = "read_after_update") + readSpec(t, version = 3, name = "read_after_delete") + readSpec(t, predicate = "status = 'active'", name = "read_active") + readSpec(t, predicate = "n = 99", name = "read_n99") + snapshotSpec(t) + } + + test("property_churn") { + val w = createTableOp("tbl", schema = "id INT, v INT") + insertOp(w, Seq(Map("id" -> 1, "v" -> 1))) + setPropertiesOp(w, Map("delta.enableChangeDataFeed" -> "true")) + insertOp(w, Seq(Map("id" -> 2, "v" -> 2))) + unsetPropertiesOp(w, Seq("delta.enableChangeDataFeed")) + setPropertiesOp(w, Map("delta.enableChangeDataFeed" -> "true", + "delta.enableDeletionVectors" -> "true")) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("schema_evolution_chain") { + // Column mapping enables rename/drop; chain add -> rename -> drop with data between each. + val w = createTableOp("tbl", schema = "id INT, a STRING", + properties = Map("delta.columnMapping.mode" -> "name")) + insertOp(w, Seq(Map("id" -> 1, "a" -> "x"))) + addColumnsOp(w, "b INT") + insertOp(w, Seq(Map("id" -> 2, "a" -> "y", "b" -> 10))) + renameColumnOp(w, "a", "label") + insertOp(w, Seq(Map("id" -> 3, "label" -> "z", "b" -> 20))) + dropColumnsOp(w, Seq("b")) + insertOp(w, Seq(Map("id" -> 4, "label" -> "w"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, version = 1, name = "read_initial") + readSpec(t, version = 3, name = "read_after_add") + readSpec(t, predicate = "label = 'x'", name = "read_renamed") + snapshotSpec(t) + } + + + test("domain_metadata_add_then_remove") { + val w = createTableOp("tbl", schema = "id INT, name STRING", + properties = Map("delta.feature.domainMetadata" -> "supported")) + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + commitOp(w, addDomainMetadata = Some(Seq( + AddDomainMetadata("delta.test", """{"v":1}""")))) + commitOp(w, removeDomainMetadata = Some(Seq("delta.test"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("low_and_high_level_mix") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + val added = commitOp(w, addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) + insertOp(w, Seq(Map("id" -> 3, "name" -> "cara"))) + deleteOp(w, predicate = "id = 1") + commitOp(w, removeFiles = Some(Seq(added))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, version = 2, name = "read_after_lowlevel_add") + snapshotSpec(t) + } + + test("commit_txn_and_files") { + val w = createTableOp("tbl", schema = "id INT, name STRING") + insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) + // A single low-level commit carrying both a SetTransaction and an AddFile. + commitOp(w, + txn = Some(AppTxn("streaming-app", 42L)), + addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + snapshotSpec(t) + } +} diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala new file mode 100644 index 00000000..ddcb480f --- /dev/null +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala @@ -0,0 +1,131 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload.tables + +import io.delta.workload.WorkloadTestSuite + +/** + * Type-coverage write workloads: exercises `RowParquet.coerce` and the schema round-trip across + * every supported scalar type, nulls, and string edge cases. Each test captures then the + * framework replays + re-validates, so a passing test proves the type survives capture->replay. + */ +class WriteTypesSuite extends WorkloadTestSuite("write_types") { + + test("scalar_types") { + val w = createTableOp("tbl", + schema = "id INT, c_long BIGINT, c_short SMALLINT, c_byte TINYINT, " + + "c_float FLOAT, c_double DOUBLE, c_bool BOOLEAN, c_str STRING") + insertOp(w, Seq( + Map("id" -> 1, "c_long" -> 10000000000L, "c_short" -> 100, "c_byte" -> 5, + "c_float" -> 1.5f, "c_double" -> 2.5, "c_bool" -> true, "c_str" -> "a"), + Map("id" -> 2, "c_long" -> 20000000000L, "c_short" -> 200, "c_byte" -> 7, + "c_float" -> 3.5f, "c_double" -> 9.5, "c_bool" -> false, "c_str" -> "b"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "c_bool = true", name = "read_true") + readSpec(t, predicate = "c_double > 5.0", name = "read_big_double") + readSpec(t, predicate = "c_long > 15000000000", name = "read_big_long") + snapshotSpec(t) + } + + test("date_and_timestamp") { + val w = createTableOp("tbl", schema = "id INT, d DATE, ts TIMESTAMP, ntz TIMESTAMP_NTZ") + insertOp(w, Seq( + Map("id" -> 1, "d" -> "2021-01-15", "ts" -> "2021-01-15 10:30:00", + "ntz" -> "2021-06-01 08:00:00"), + Map("id" -> 2, "d" -> "2022-07-04", "ts" -> "2022-07-04 23:59:59", + "ntz" -> "2022-12-31 00:00:00"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "d > DATE '2021-06-01'", name = "read_after_date") + readSpec(t, predicate = "ts < TIMESTAMP '2022-01-01 00:00:00'", name = "read_before_ts") + snapshotSpec(t) + } + + test("decimal_type") { + val w = createTableOp("tbl", schema = "id INT, amount DECIMAL(10,2)") + insertOp(w, Seq( + Map("id" -> 1, "amount" -> "123.45"), + Map("id" -> 2, "amount" -> "99.99"), + Map("id" -> 3, "amount" -> "1000.00"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "amount > 100.00", name = "read_over_100") + snapshotSpec(t) + } + + test("binary_type") { + val w = createTableOp("tbl", schema = "id INT, data BINARY") + insertOp(w, Seq( + Map("id" -> 1, "data" -> "aGVsbG8="), // "hello" + Map("id" -> 2, "data" -> "d29ybGQ="))) // "world" + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + snapshotSpec(t) + } + + test("nullable_columns") { + val w = createTableOp("tbl", schema = "id INT, name STRING, score INT, d DATE") + insertOp(w, Seq( + Map("id" -> 1, "name" -> "alice", "score" -> 90, "d" -> "2021-01-01"), + Map("id" -> 2, "name" -> null, "score" -> null, "d" -> null), + Map("id" -> 3, "name" -> "bob", "score" -> 75, "d" -> null))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "name IS NULL", name = "read_null_name") + readSpec(t, predicate = "name IS NOT NULL", name = "read_named") + readSpec(t, predicate = "d IS NULL", name = "read_null_date") + snapshotSpec(t) + } + + test("string_edge_cases") { + val w = createTableOp("tbl", schema = "id INT, s STRING") + insertOp(w, Seq( + Map("id" -> 1, "s" -> "it's a \"quoted\" value"), + Map("id" -> 2, "s" -> "unicode: café ☕ — ünïcödé"), + Map("id" -> 3, "s" -> ""), + Map("id" -> 4, "s" -> "line1\nline2\ttab"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "s = ''", name = "read_empty") + readSpec(t, predicate = "id = 1", name = "read_apostrophe") + snapshotSpec(t) + } + + test("nested_type_data_is_unsupported") { + // Nested column TYPES round-trip in the schema, but nested DATA via the rows API is not + // supported — `RowParquet.coerce` fails loud rather than silently writing a wrong value. + val w = createTableOp("tbl", schema = "id INT, s STRUCT") + val ex = intercept[IllegalArgumentException] { + insertOp(w, Seq(Map("id" -> 1, "s" -> Map("a" -> 1)))) + } + assert(ex.getMessage.toLowerCase.contains("unsupported"), s"message: ${ex.getMessage}") + } + + test("partitioned_by_typed_column") { + val w = createTableOp("tbl", + schema = "id INT, d DATE, amount DECIMAL(8,2)", partitionColumns = Seq("d")) + insertOp(w, Seq( + Map("id" -> 1, "d" -> "2021-01-01", "amount" -> "10.00"), + Map("id" -> 2, "d" -> "2021-01-01", "amount" -> "20.00"), + Map("id" -> 3, "d" -> "2021-02-01", "amount" -> "30.00"))) + val t = registerWriteSpec(w) + readSpec(t, name = "read_all") + readSpec(t, predicate = "d = DATE '2021-01-01'", name = "read_jan") + snapshotSpec(t) + } +} From 11915e048e0d998b3e9ba590cc66fb9070bcf39d Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 15:30:20 +0000 Subject: [PATCH 04/25] refactor: stop re-validating Spark's own output; enforce invariants at the source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes in one theme — remove checks that only re-assert what the engine already guarantees, and move invariant enforcement to where the invariant is established: - Delete the write-derived snapshot structural comparison (validateSnapshotStructurally) and its helper cluster (declaredConfiguration, configurationViolation, normalizedSchema, stripColumnMapping, canonicalField). WorkloadValidator replays via Spark from the same commits Spark captured, so a table's protocol/metadata/schema is Spark's deterministic output; re-comparing it only asserts Spark agrees with itself. Write-derived snapshots are validated at capture time and trusted standalone. WorkloadValidator is now orchestration-only; snapshot comparison lives solely in SnapshotCapture. - Delete the protocol-capability check (protocolViolation + the version->implied -feature map) for the same reason; it also re-encoded delta-spark's protocol rules by hand and got the columnMapping rule wrong. - Enforce commit-index == table-version inside commitOp (the op that establishes it) and guard UpdatePropertiesCommit to set XOR remove (SQL replays SET and UNSET as separate commits, which would break the invariant). validateExpectedLatest now throws-with-context instead of accumulating into a "warnings" buffer. Row comparison is the only corpus check that validates the spec rather than Spark's determinism. Deletes the now-obsolete WorkloadValidatorUnitSuite (it only tested the removed helpers). Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/WorkloadValidator.scala | 210 ++---------------- .../io/delta/workload/WriteSpecCapture.scala | 4 + .../workload/WorkloadValidatorUnitSuite.scala | 139 ------------ 3 files changed, 21 insertions(+), 332 deletions(-) delete mode 100644 workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorUnitSuite.scala diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index f6cb0654..196b7650 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -22,8 +22,6 @@ import scala.collection.mutable import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} import org.apache.commons.io.FileUtils import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.types.{DataType, StructField, StructType} @@ -92,9 +90,8 @@ object WorkloadValidator { try { specType match { case "write" => - val w = mutable.ArrayBuffer[String]() - validateExpectedLatest(spark, replayOf(specFile.getFileName.toString), testDir, name, w) - if (w.isEmpty) passed += 1 else errors ++= w.map(m => s"${testDir.getFileName}/$name: $m") + validateExpectedLatest(spark, replayOf(specFile.getFileName.toString), testDir, name) + passed += 1 case "read" => val expectedDir = testDir.resolve("expected").resolve(name) @@ -115,12 +112,11 @@ object WorkloadValidator { case "snapshot" => writePtr match { - case Some(ws) => // write-derived: structural, against the replayed table - val w = mutable.ArrayBuffer[String]() - validateSnapshotStructurally(spark, replayOf(ws), specFile, name, - JsonUtil.readWriteSpec(specsDir.resolve(ws)), w) - if (w.isEmpty) passed += 1 - else errors ++= w.map(m => s"${testDir.getFileName}/$name: $m") + case Some(ws) => + // Write-derived: protocol/metadata are Spark's deterministic output and were + // validated at capture time. Standalone, we only confirm the write spec replays. + replayOf(ws) + passed += 1 case None => if (Files.isDirectory(deltaTable)) { SnapshotCapture.validateFromSpec(spark, deltaTable, specFile) @@ -178,193 +174,17 @@ object WorkloadValidator { /** Compare the replayed table's rows against the write spec's `expected//table_content`. */ private def validateExpectedLatest( - spark: SparkSession, replayTablePath: Path, testDir: Path, writeName: String, - warnings: mutable.ArrayBuffer[String]): Unit = { + spark: SparkSession, replayTablePath: Path, testDir: Path, writeName: String): Unit = { val contentDir = testDir.resolve("expected").resolve(writeName).resolve("table_content") - // A write spec MUST ship its expected rows; a missing dir is a capture bug, not a pass. - if (!Files.exists(contentDir)) { - warnings += s"write spec '$writeName': expected/$writeName/table_content is missing" - return - } + // A write spec MUST ship its expected rows; a missing dir is a capture bug. + require(Files.exists(contentDir), + s"write spec '$writeName': expected/$writeName/table_content is missing") val expected = JsonUtil.toRowMultiset(spark.read.parquet(contentDir.toString)) val actual = JsonUtil.toRowMultiset( spark.read.format("delta").load(replayTablePath.toAbsolutePath.toString)) - try JsonUtil.assertMultisetsEqual(expected, actual, s"${writeName}_latest") - catch { case e: Throwable => warnings += e.getMessage } - } - - /** - * Validate a snapshot spec against a write-replayed (independently created) table. `protocol` - * is a capability check (version floors + feature supersets), `configuration` checks only the - * write spec's declared keys, and `schemaString` is compared with column-mapping ids stripped; - * `partitionColumns`/`format` are compared exactly. - */ - private def validateSnapshotStructurally( - spark: SparkSession, replayTablePath: Path, specFile: Path, specFileName: String, - writeSpec: WriteSpec, warnings: mutable.ArrayBuffer[String]): Unit = { - val spec = JsonUtil.readSnapshotSpec(specFile) - val (declaredKeys, removedKeys) = declaredConfiguration(writeSpec) - spec.expected.foreach { exp => - val log = DeltaHarness.get.openLog(spark, replayTablePath.toString) - val snapshot = (spec.version, spec.timestamp) match { - case (Some(v), _) => log.getSnapshotAt(v) - case _ => log.update() - } - val expProtocol = JsonUtil.mapper.valueToTree[JsonNode](exp.protocol) - val replayProtocol = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") - protocolViolation(expProtocol, replayProtocol).foreach { reason => - warnings += s"snapshot '$specFileName': protocol mismatch ($reason)" - } - val expectedMeta = JsonUtil.mapper.valueToTree[JsonNode](exp.metadata) - val replayMeta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") - configurationViolation(expectedMeta, replayMeta, declaredKeys, removedKeys).foreach { - reason => warnings += s"snapshot '$specFileName': metadata.configuration mismatch ($reason)" - } - if (normalizedSchema(expectedMeta) != normalizedSchema(replayMeta)) { - warnings += s"snapshot '$specFileName': metadata.schemaString mismatch" - } - for (field <- Seq("partitionColumns", "format")) { - if (canonicalField(expectedMeta, field) != canonicalField(replayMeta, field)) { - warnings += s"snapshot '$specFileName': metadata.$field mismatch" - } - } - } + JsonUtil.assertMultisetsEqual(expected, actual, s"${writeName}_latest") } - // =========================================================================== - // Portable comparison helpers (also unit-tested directly) - // =========================================================================== - - /** - * Canonicalize `metadata.schemaString`, stripping per-field column-mapping identifiers - * (`delta.columnMapping.physicalName`/`.id`) which are minted per-table and so are not a - * meaningful equality target across an independently-replayed table. - */ - private[workload] def normalizedSchema(meta: JsonNode): String = - Option(meta.get("schemaString")).filterNot(_.isNull) match { - case None => "null" - case Some(s) => - val tree = JsonUtil.mapper.readTree(s.asText()) - stripColumnMapping(tree) - JsonUtil.canonicalJson(tree) - } - - private def stripColumnMapping(node: JsonNode): Unit = node match { - case obj: ObjectNode => - obj.get("metadata") match { - case m: ObjectNode => - m.remove("delta.columnMapping.physicalName") - m.remove("delta.columnMapping.id") - case _ => () - } - obj.elements().asScala.foreach(stripColumnMapping) - case arr: ArrayNode => arr.elements().asScala.foreach(stripColumnMapping) - case _ => () - } - - /** - * The configuration keys the write spec's commits explicitly declared. - * @return (declared keys still in effect, keys explicitly removed). `replace_table` resets both. - */ - private[workload] def declaredConfiguration(writeSpec: WriteSpec): (Set[String], Set[String]) = { - val declared = mutable.LinkedHashSet[String]() - val removed = mutable.LinkedHashSet[String]() - for (commit <- writeSpec.commits) commit match { - case c: CreateTableCommit => c.properties.foreach(declared ++= _.keys) - case c: ReplaceTableCommit => - declared.clear(); removed.clear() - c.properties.foreach(declared ++= _.keys) - case c: UpdatePropertiesCommit => - c.set.foreach { s => declared ++= s.keys; removed --= s.keys } - c.remove.foreach { r => removed ++= r; declared --= r } - case _ => () - } - (declared.toSet, removed.toSet) - } - - // Features implied by a protocol version when table features are NOT enumerated (reader < 3 / - // writer < 7). Cumulative — a version implies its own and all lower versions' features. - // columnMapping is the only reader-axis legacy feature; it is implied jointly (reader>=2 AND - // writer>=5), so it lives in the writer map and is credited to the reader axis only when the - // writer floor is also met (see effectiveReader) — a reader=2/writer<5 protocol does NOT imply it. - private val writerImpliedFeatures: Map[Int, Set[String]] = Map( - 1 -> Set.empty, - 2 -> Set("appendOnly", "invariants"), - 3 -> Set("appendOnly", "invariants", "checkConstraints"), - 4 -> Set("appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns"), - 5 -> Set("appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns", - "columnMapping"), - 6 -> Set("appendOnly", "invariants", "checkConstraints", "changeDataFeed", "generatedColumns", - "columnMapping", "identityColumns")) - - /** - * Capability check: version floors + feature supersets over EFFECTIVE feature sets (explicit - * features unioned with those implied by the protocol version). Returns the first violation. - */ - private[workload] def protocolViolation(expected: JsonNode, replay: JsonNode): Option[String] = { - def intField(node: JsonNode, name: String): Int = - if (node != null && node.has(name)) node.get(name).asInt() else 0 - def explicitFeatures(node: JsonNode, name: String): Set[String] = - if (node != null && node.has(name) && node.get(name).isArray) - node.get(name).elements().asScala.map(_.asText()).toSet - else Set.empty - def effectiveReader(node: JsonNode): Set[String] = { - val r = intField(node, "minReaderVersion") - val w = intField(node, "minWriterVersion") - if (r >= 3) explicitFeatures(node, "readerFeatures") - // Legacy: columnMapping is the only reader-implied feature, and only when reader>=2 AND - // writer>=5 jointly (mirrors delta-spark's implicitlySupportedFeatures). - else if (r >= 2 && w >= 5) Set("columnMapping") - else Set.empty - } - def effectiveWriter(node: JsonNode): Set[String] = { - val v = intField(node, "minWriterVersion") - if (v >= 7) explicitFeatures(node, "writerFeatures") - else writerImpliedFeatures.getOrElse(v, Set.empty) - } - - val expReader = intField(expected, "minReaderVersion") - val expWriter = intField(expected, "minWriterVersion") - val repReader = intField(replay, "minReaderVersion") - val repWriter = intField(replay, "minWriterVersion") - val missingReaderFeatures = effectiveReader(expected) -- effectiveReader(replay) - val missingWriterFeatures = effectiveWriter(expected) -- effectiveWriter(replay) - - if (repReader < expReader) Some(s"minReaderVersion $repReader < $expReader") - else if (repWriter < expWriter) Some(s"minWriterVersion $repWriter < $expWriter") - else if (missingReaderFeatures.nonEmpty) - Some(s"missing readerFeatures ${missingReaderFeatures.mkString(",")}") - else if (missingWriterFeatures.nonEmpty) - Some(s"missing writerFeatures ${missingWriterFeatures.mkString(",")}") - else None - } - - /** - * Check only the author-declared config keys: each declared key present + equal on the replay, - * each removed key absent. Engine-injected defaults are ignored. Returns the first violation. - */ - private[workload] def configurationViolation( - expectedMeta: JsonNode, replayMeta: JsonNode, - declaredKeys: Set[String], removedKeys: Set[String]): Option[String] = { - val expConfig = Option(expectedMeta.get("configuration")).filterNot(_.isNull) - val repConfig = Option(replayMeta.get("configuration")).filterNot(_.isNull) - def get(config: Option[JsonNode], key: String): Option[String] = - config.flatMap(c => Option(c.get(key))).filterNot(_.isNull).map(_.asText()) - - declaredKeys.toSeq.sorted.collectFirst { - case key if get(repConfig, key) != get(expConfig, key) => - s"key '$key' expected ${get(expConfig, key)}, got ${get(repConfig, key)}" - }.orElse { - removedKeys.toSeq.sorted.collectFirst { - case key if get(repConfig, key).isDefined => s"removed key '$key' still present" - } - } - } - - private def canonicalField(node: JsonNode, field: String): String = - if (node.has(field) && !node.get(field).isNull) JsonUtil.canonicalJson(node.get(field)) - else "null" - // =========================================================================== // Per-commit replay // =========================================================================== @@ -414,6 +234,10 @@ object WorkloadValidator { spark.sql(s"UPDATE $tableRef SET $setClauses WHERE ${c.predicate}") case c: UpdatePropertiesCommit => + // SET and UNSET replay as separate commits, so one UpdatePropertiesCommit must not do both + // (it would advance the table by two versions and break commit index == version). + require(!(c.set.exists(_.nonEmpty) && c.remove.exists(_.nonEmpty)), + s"UpdatePropertiesCommit at commit $idx both sets and removes properties") c.set.filter(_.nonEmpty).foreach { s => val setClause = s.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") spark.sql(s"ALTER TABLE $tableRef SET TBLPROPERTIES ($setClause)") @@ -494,7 +318,7 @@ object WorkloadValidator { /** * Reconstruct a Spark [[StructType]] from a stored Delta schema value. The spec stores schemas * as `StructType.json` deserialized to a Map (see `WriteSpecCapture.ddlToSchemaJson`), so this - * re-serializes and parses it through Spark's own reader — `.toDDL`/`.sql`/`.json` on the result + * re-serializes and parses it through Spark's own reader. `.toDDL`/`.sql`/`.json` on the result * are then exact and lossless (decimal precision, nested nullability, quoting all preserved). */ private def structOf(schema: Any): StructType = diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala index eeab2fe7..4231161b 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala @@ -90,6 +90,10 @@ class WriteSpecBuilder { recordUpdateProperties(Map.empty, props) def recordUpdateProperties(set: Map[String, String], unset: Seq[String]): Unit = { + // SQL replays SET and UNSET TBLPROPERTIES as separate ALTER statements, so a single commit + // cannot do both without producing two table versions and breaking commit index == version. + require(!(set.nonEmpty && unset.nonEmpty), + "an UpdatePropertiesCommit sets or removes properties, not both in one commit") commits += UpdatePropertiesCommit( set = if (set.nonEmpty) Some(set) else None, remove = if (unset.nonEmpty) Some(unset) else None) diff --git a/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorUnitSuite.scala b/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorUnitSuite.scala deleted file mode 100644 index bd6f58fb..00000000 --- a/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorUnitSuite.scala +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload - -import com.fasterxml.jackson.databind.JsonNode -import org.scalatest.funsuite.AnyFunSuite - -/** - * Direct unit tests for [[WorkloadValidator]]'s pure comparison functions. These branches (esp. - * the implicit-vs-explicit protocol-feature reconciliation and the column-mapping schema - * normalization) are hard for the capture->replay oracle to provoke, because replay produces a - * conforming table — so they are exercised here with hand-built inputs. No Spark. - */ -class WorkloadValidatorUnitSuite extends AnyFunSuite { - - private def node(json: String): JsonNode = JsonUtil.mapper.readTree(json) - - // ---- protocolViolation: version floors + effective (implied + explicit) feature supersets ---- - - test("protocol: identical legacy protocols are compatible") { - val p = node("""{"minReaderVersion":1,"minWriterVersion":2}""") - assert(WorkloadValidator.protocolViolation(p, p).isEmpty) - } - - test("protocol: lower replay version is a violation") { - val exp = node("""{"minReaderVersion":1,"minWriterVersion":5}""") - val rep = node("""{"minReaderVersion":1,"minWriterVersion":3}""") - assert(WorkloadValidator.protocolViolation(exp, rep).exists(_.contains("minWriterVersion"))) - } - - test("protocol: version-implied feature on expected satisfied by explicit feature on replay") { - // Expected writer v2 implies {appendOnly, invariants}; replay (3,7) lists them explicitly. - val exp = node("""{"minReaderVersion":1,"minWriterVersion":2}""") - val rep = node("""{"minReaderVersion":3,"minWriterVersion":7, - "readerFeatures":[],"writerFeatures":["appendOnly","invariants","deletionVectors"]}""") - assert(WorkloadValidator.protocolViolation(exp, rep).isEmpty) - } - - test("protocol: explicit expected feature missing from replay is a violation") { - val exp = node("""{"minReaderVersion":3,"minWriterVersion":7, - "readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}""") - val rep = node("""{"minReaderVersion":3,"minWriterVersion":7, - "readerFeatures":[],"writerFeatures":[]}""") - assert(WorkloadValidator.protocolViolation(exp, rep).exists(_.contains("deletionVectors"))) - } - - test("protocol: higher replay version / extra features is allowed (capability floor)") { - val exp = node("""{"minReaderVersion":1,"minWriterVersion":2}""") - val rep = node("""{"minReaderVersion":3,"minWriterVersion":7, - "readerFeatures":["columnMapping"],"writerFeatures":["appendOnly","invariants","columnMapping"]}""") - assert(WorkloadValidator.protocolViolation(exp, rep).isEmpty) - } - - // ---- configurationViolation: only declared keys checked; removed keys must be absent ---- - - private val emptyMeta = node("""{"configuration":{}}""") - - test("config: declared key present and equal is compatible") { - val exp = node("""{"configuration":{"delta.enableChangeDataFeed":"true"}}""") - val rep = node("""{"configuration":{"delta.enableChangeDataFeed":"true","x":"injected"}}""") - assert(WorkloadValidator.configurationViolation( - exp, rep, Set("delta.enableChangeDataFeed"), Set.empty).isEmpty) - } - - test("config: declared key missing on replay is a violation") { - val exp = node("""{"configuration":{"delta.enableChangeDataFeed":"true"}}""") - assert(WorkloadValidator.configurationViolation( - exp, emptyMeta, Set("delta.enableChangeDataFeed"), Set.empty) - .exists(_.contains("delta.enableChangeDataFeed"))) - } - - test("config: removed key still present on replay is a violation") { - val rep = node("""{"configuration":{"delta.enableChangeDataFeed":"true"}}""") - assert(WorkloadValidator.configurationViolation( - emptyMeta, rep, Set.empty, Set("delta.enableChangeDataFeed")) - .exists(_.contains("delta.enableChangeDataFeed"))) - } - - test("config: engine-injected (undeclared) keys are ignored") { - val exp = node("""{"configuration":{}}""") - val rep = node("""{"configuration":{"delta.columnMapping.maxColumnId":"3"}}""") - assert(WorkloadValidator.configurationViolation(exp, rep, Set.empty, Set.empty).isEmpty) - } - - // ---- normalizedSchema: per-table column-mapping ids are not a comparison target ---- - - test("schema: differing column-mapping physicalName/id normalize to equal") { - def schema(phys: String, id: Int) = - s"""{"schemaString":"{\\"type\\":\\"struct\\",\\"fields\\":[{\\"name\\":\\"id\\",""" + - s"""\\"type\\":\\"integer\\",\\"nullable\\":true,\\"metadata\\":""" + - s"""{\\"delta.columnMapping.physicalName\\":\\"$phys\\",\\"delta.columnMapping.id\\":$id}}]}"}""" - val a = node(schema("col-aaaa", 1)) - val b = node(schema("col-bbbb", 1)) - assert(WorkloadValidator.normalizedSchema(a) == WorkloadValidator.normalizedSchema(b)) - } - - test("schema: a real difference (column name) is NOT masked") { - val a = node("""{"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",""" + - """\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"}""") - val b = node("""{"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"renamed\",""" + - """\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"}""") - assert(WorkloadValidator.normalizedSchema(a) != WorkloadValidator.normalizedSchema(b)) - } - - // ---- declaredConfiguration: replace_table resets, update_properties tracks set/remove ---- - - test("declaredConfiguration: replace_table resets prior declarations") { - val spec = WriteSpec(Seq( - CreateTableCommit(schema = Map("fields" -> Seq.empty[Any]), properties = Some(Map("a" -> "1"))), - UpdatePropertiesCommit(set = Some(Map("b" -> "2"))), - ReplaceTableCommit(schema = Map("fields" -> Seq.empty[Any]), properties = Some(Map("c" -> "3"))))) - val (declared, removed) = WorkloadValidator.declaredConfiguration(spec) - assert(declared == Set("c"), s"declared=$declared") - assert(removed.isEmpty, s"removed=$removed") - } - - test("declaredConfiguration: unset moves a key from declared to removed") { - val spec = WriteSpec(Seq( - CreateTableCommit(schema = Map("fields" -> Seq.empty[Any]), properties = Some(Map("a" -> "1"))), - UpdatePropertiesCommit(remove = Some(Seq("a"))))) - val (declared, removed) = WorkloadValidator.declaredConfiguration(spec) - assert(declared.isEmpty, s"declared=$declared") - assert(removed == Set("a"), s"removed=$removed") - } -} From b28d39ecf1fe294543a4bbdb9497a3d8e7194b3c Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 15:54:15 +0000 Subject: [PATCH 05/25] refactor: route commit-log mutation through the typed log/ module The log/ package (CommitLog + the typed Action ADT) was dead except for its own test, while three places hand-rolled the same raw-JSON manipulation it provides. Make it load-bearing: - modifyCommitActions now delegates to CommitLog.mutate and hands the modifier typed Actions (AddFile, Metadata, ...) instead of (String, ObjectNode) pairs. The ~18-line parse/wrap/IO body collapses to a 4-line delegation; callers filter by type and `.copy` fields (AddFile.stats stays Option[String], so the stats-corruption tests still express empty/garbage stats; commitInfo/cdc surface as RawAction and pass through verbatim). Byte-level corruption still uses mutateTable directly. - TableInfoWriter counts removes via CommitLog.read + isInstanceOf[RemoveFile] instead of the fragile startsWith("{\"remove") substring match. - Migrated all 8 modifyCommitActions call sites (CorruptionSuite, DataSkippingSuite, WorkloadGeneratorSuite) to the typed modifier. WriteCommit (spec) / Action (raw log) / CommitRequest (SPI) remain deliberately separate: different abstractions, no shared information. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/TableInfoWriter.scala | 8 +++-- .../io/delta/workload/WorkloadGenerator.scala | 35 +++++------------- .../workload/WorkloadGeneratorSuite.scala | 15 ++++---- .../workload/tables/CorruptionSuite.scala | 5 +-- .../workload/tables/DataSkippingSuite.scala | 36 ++++++++----------- 5 files changed, 39 insertions(+), 60 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala b/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala index e8e59474..66a3ef39 100644 --- a/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala +++ b/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.{col, to_json} import io.delta.workload.deltaharness.DeltaHarness +import io.delta.workload.log.{CommitLog, RemoveFile} object TableInfoWriter { @@ -66,11 +67,12 @@ object TableInfoWriter { val numCommits = commitFiles.size - // Count actions and remove actions in a single pass over each commit. + // Count actions and remove actions in a single pass over each commit, via the typed log. val (numActions, numRemoveFiles) = commitFiles.foldLeft((0L, 0L)) { case ((actions, removes), f) => - val lines = Files.readAllLines(f).asScala - (actions + lines.size, removes + lines.count(_.startsWith("""{"remove"""))) + val version = f.getFileName.toString.stripSuffix(".json").toLong + val as = CommitLog.read(tablePath, version) + (actions + as.size, removes + as.count(_.isInstanceOf[RemoveFile])) } val lastCrcVersion = listLog(_.map(_.getFileName.toString) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 5e56c13c..2045e0c6 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -21,11 +21,11 @@ import java.nio.file.{Files, Path, Paths} import scala.collection.mutable import scala.jdk.CollectionConverters._ -import com.fasterxml.jackson.databind.node.ObjectNode import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types.StructType import io.delta.workload.deltaharness.{CommitRemoveFile, CommitRequest, DeltaHarness} +import io.delta.workload.log.{Action, CommitLog} /** * Internal workload generation engine. Use [[WorkloadTestSuite]] as the public API: @@ -605,33 +605,16 @@ class WorkloadContext private[workload] ( } /** - * Modify actions in a specific commit version. - * The modifier receives the full list of `(actionType, innerNode)` pairs - * (e.g. `("add", )`) and returns the (possibly reordered, - * filtered, or modified) list to write back. + * Modify the typed [[Action]]s of a specific commit version: filter, reorder, or `.copy` fields. + * Unknown/malformed lines surface as [[io.delta.workload.log.RawAction]] and pass through + * unchanged. For byte-level corruption (truncation, garbage), use [[mutateTable]] directly. */ def modifyCommitActions(table: TableHandle, version: Long)( - modifier: Seq[(String, ObjectNode)] => Seq[(String, ObjectNode)]): Unit = { + modifier: Seq[Action] => Seq[Action]): Unit = mutateTable(table) { tableDir => - val commitFile = tableDir.resolve("_delta_log").resolve(f"$version%020d.json") - if (Files.exists(commitFile)) { - val lines = new String(Files.readAllBytes(commitFile), "UTF-8").split("\n") - .filter(_.trim.nonEmpty) - val actions = lines.map { line => - val node = JsonUtil.mapper.readTree(line) - val actionType = node.fieldNames().next() - (actionType, node.get(actionType).asInstanceOf[ObjectNode]) - }.toSeq - val result = modifier(actions) - val newLines = result.map { case (actionType, innerNode) => - val wrapper = JsonUtil.mapper.createObjectNode() - wrapper.set(actionType, innerNode) - JsonUtil.mapper.writeValueAsString(wrapper) - } - Files.write(commitFile, newLines.mkString("\n").getBytes("UTF-8")) - } + if (Files.exists(CommitLog.commitFile(tableDir, version))) + CommitLog.mutate(tableDir, version)(modifier) } - } // ---- Auto-naming ---- @@ -891,9 +874,9 @@ trait WorkloadOps { def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = current.mutateTable(table)(mutation) - /** Modify actions in a specific commit version. */ + /** Modify the typed actions of a specific commit version. */ def modifyCommitActions(table: TableHandle, version: Long)( - modifier: Seq[(String, ObjectNode)] => Seq[(String, ObjectNode)]): Unit = + modifier: Seq[Action] => Seq[Action]): Unit = current.modifyCommitActions(table, version)(modifier) } diff --git a/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala b/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala index 2d34cff4..54fa8953 100644 --- a/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala @@ -25,6 +25,8 @@ import org.apache.spark.sql.SparkSession import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite +import io.delta.workload.log.{AddFile, Metadata} + class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with WorkloadOps { private var _spark: SparkSession = _ @@ -609,8 +611,8 @@ class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor sql("INSERT INTO tbl VALUES (1),(2)") val t = registerTable("tbl") modifyCommitActions(t, version = 1) { actions => - actions.map { case ("add", node) => - node.put("stats", """{"numRecords":999}"""); ("add", node) + actions.map { + case a: AddFile => a.copy(stats = Some("""{"numRecords":999}""")) case other => other } } @@ -639,7 +641,7 @@ class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor sql("INSERT INTO tbl VALUES (1),(2)") val t = registerTable("tbl") modifyCommitActions(t, version = 1) { actions => - actions.filter(_._1 != "add") + actions.filterNot(_.isInstanceOf[AddFile]) } snapshotSpec(t) } @@ -660,11 +662,8 @@ class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor val t = registerTable("tbl") // Modify the metaData action in the CREATE commit (v0) modifyCommitActions(t, version = 0) { actions => - actions.map { case ("metaData", node) => - val configNode = node.get("configuration").asInstanceOf[ - com.fasterxml.jackson.databind.node.ObjectNode] - configNode.put("test.property", "hello") - ("metaData", node) + actions.map { + case m: Metadata => m.copy(configuration = m.configuration + ("test.property" -> "hello")) case other => other } } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/CorruptionSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/CorruptionSuite.scala index d6b056e5..c0fba1db 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/CorruptionSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/CorruptionSuite.scala @@ -17,6 +17,7 @@ package io.delta.workload.tables import io.delta.workload.WorkloadTestSuite +import io.delta.workload.log.AddFile class CorruptionSuite extends WorkloadTestSuite("corruption") { @@ -80,8 +81,8 @@ class CorruptionSuite extends WorkloadTestSuite("corruption") { sql("INSERT INTO tbl VALUES (1,'a'),(2,'b'),(3,'c')") val t = registerTable("tbl") modifyCommitActions(t, version = 1) { actions => - actions.map { case ("add", node) => - node.put("stats", """{"numRecords":999}"""); ("add", node) + actions.map { + case a: AddFile => a.copy(stats = Some("""{"numRecords":999}""")) case other => other } } diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala index 0fb987d9..dbc96d90 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/DataSkippingSuite.scala @@ -16,7 +16,8 @@ package io.delta.workload.tables -import io.delta.workload.WorkloadTestSuite +import io.delta.workload.{JsonUtil, WorkloadTestSuite} +import io.delta.workload.log.AddFile /** * Data skipping, statistics, and partitioning workloads. @@ -880,8 +881,8 @@ class DataSkippingSuite extends WorkloadTestSuite("data_skipping") { val t = registerTable("tbl") // Strip stats from commit modifyCommitActions(t, version = 1) { actions => - actions.map { case ("add", node) => - node.remove("stats"); ("add", node) + actions.map { + case a: AddFile => a.copy(stats = None) case other => other } } @@ -1190,19 +1191,13 @@ class DataSkippingSuite extends WorkloadTestSuite("data_skipping") { val t = registerTable("tbl") // Strip min/max stats, keep only numRecords modifyCommitActions(t, 0) { actions => - actions.map { case ("add", node) => - if (node.has("stats")) { - val stats = node.get("stats").asText() - if (stats.contains("numRecords")) { - import com.fasterxml.jackson.databind.ObjectMapper - val mapper = new ObjectMapper() - val statsNode = mapper.readTree(stats) - val newStats = mapper.createObjectNode() - newStats.set("numRecords", statsNode.get("numRecords")) - node.put("stats", mapper.writeValueAsString(newStats)) - } - } - ("add", node) + actions.map { + case a: AddFile if a.stats.exists(_.contains("numRecords")) => + val statsNode = JsonUtil.mapper.readTree(a.stats.get) + val newStats = JsonUtil.mapper.createObjectNode() + newStats.set[com.fasterxml.jackson.databind.JsonNode]( + "numRecords", statsNode.get("numRecords")) + a.copy(stats = Some(JsonUtil.mapper.writeValueAsString(newStats))) case other => other } } @@ -1253,9 +1248,8 @@ class DataSkippingSuite extends WorkloadTestSuite("data_skipping") { val t = registerTable("tbl") // Strip stats completely to simulate empty stats string modifyCommitActions(t, 0) { actions => - actions.map { case ("add", node) => - if (node.has("stats")) node.put("stats", "") - ("add", node) + actions.map { + case a: AddFile if a.stats.isDefined => a.copy(stats = Some("")) case other => other } } @@ -1269,8 +1263,8 @@ class DataSkippingSuite extends WorkloadTestSuite("data_skipping") { val t = registerTable("tbl") // Remove stats field entirely modifyCommitActions(t, 0) { actions => - actions.map { case ("add", node) => - node.remove("stats"); ("add", node) + actions.map { + case a: AddFile => a.copy(stats = None) case other => other } } From ab293280926ea5b2f1434d92ebc0fbbcd48061bf Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 16:23:28 +0000 Subject: [PATCH 06/25] refactor: split the WorkloadGenerator god-file by concern WorkloadGenerator.scala held the generation engine, the DSL state machine, the handles, and the test-facing facade in one 938-line file. Split by abstraction (pure move, no behavior change): - WorkloadGenerator.scala: the generation engine (generateTable) plus the internal model it consumes (TableSpec, ReadSpecConfig, SnapshotSpecConfig, HasAssertion, WorkloadResult). - WorkloadContext.scala: the DSL state machine (WorkloadContext + companion) and the handles (SpecRef, TableHandle, WriteHandle). - WorkloadOps.scala: the facade trait mixed into WorkloadTestSuite. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/WorkloadContext.scala | 588 ++++++++++++++ .../io/delta/workload/WorkloadGenerator.scala | 715 +----------------- .../scala/io/delta/workload/WorkloadOps.scala | 150 ++++ 3 files changed, 740 insertions(+), 713 deletions(-) create mode 100644 workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala create mode 100644 workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala new file mode 100644 index 00000000..15b6670a --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala @@ -0,0 +1,588 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload + +import java.nio.file.{Files, Path, Paths} + +import scala.collection.mutable + +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.types.StructType + +import io.delta.workload.deltaharness.{CommitRemoveFile, CommitRequest, DeltaHarness} +import io.delta.workload.log.{Action, CommitLog} + +/** + * Handle to a declared spec. Attach assertions that are checked after capture. + * `T` is the typed spec class (e.g. [[ReadSpec]], [[SnapshotSpec]]). + */ +class SpecRef[T] private[workload] ( + private[workload] val config: HasAssertion[T]) { + + /** + * Assert that the captured spec is an error (has `expectedError`, no `expected`). + * Works for any spec type that follows the `expected`/`expectedError` convention. + */ + def assertError(): SpecRef[T] = { + config.assertion = Some { node => + require(node.has("expectedError") && !node.get("expectedError").isNull, + s"Expected an error spec but got a success result") + } + this + } + + /** Assert conditions on the captured spec using the typed case class. */ + def assert(check: T => Unit): SpecRef[T] = { + val deserialize = config.deserialize + config.assertion = Some { node => + check(deserialize(node)) + } + this + } +} + +/** + * Handle to a Delta table created via SQL. Used for declaring read specs + * (read, snapshot) and table mutations (mutateTable, modifyCommitActions). + */ +class TableHandle private[workload] ( + private[workload] val tableName: String, + private[workload] val sourcePath: Path, + private[workload] val ctx: WorkloadContext) { + + /** + * Get the commit timestamp for a specific version as a timestampAsOf-safe + * string (yyyy-MM-dd HH:mm:ss.SSS in session TZ). + * + * Mirrors what Delta's time-travel resolution actually keys off: + * - ICT-enabled tables: `commitInfo.inCommitTimestamp` in the commit JSON. + * - Non-ICT tables: the commit JSON file's mtime. + * + * DESCRIBE HISTORY and `commitInfo.timestamp` both look right at a glance + * but diverge by a few milliseconds from the file mtime on some engines — feeding + * those back as `timestampAsOf` raises DELTA_TIMESTAMP_GREATER_THAN_COMMIT + * / DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION. + */ + def getTimestampForVersion(version: Long): String = { + val padded = f"$version%020d" + val commitPath = sourcePath.resolve(s"_delta_log/$padded.json") + require(Files.exists(commitPath), s"No commit file for version $version: $commitPath") + val ictOpt = scala.collection.JavaConverters + .asScalaBufferConverter(Files.readAllLines(commitPath)).asScala.iterator + .flatMap { line => + val node = JsonUtil.mapper.readTree(line) + Option(node.get("commitInfo")).flatMap(ci => Option(ci.get("inCommitTimestamp"))) + } + .find(_ => true) + .map(_.asLong()) + val tsMillis = ictOpt.getOrElse( + Files.getLastModifiedTime(commitPath).toMillis) + val sessionTz = ctx.spark.conf.get("spark.sql.session.timeZone", "UTC") + val fmt = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + fmt.setTimeZone(java.util.TimeZone.getTimeZone(sessionTz)) + fmt.format(new java.util.Date(tsMillis)) + } +} + +/** + * Handle for structured write operations. Obtained from [[WorkloadContext.createTableOp]] + * (which creates the table and records the create) or [[WorkloadContext.writeSpec]] + * (which wraps an already-created [[TableHandle]]). + * + * Accepts write operations (insertOp, updateOp, deleteOp, ...) that both execute SQL + * and record the commit. Call [[WorkloadContext.registerWriteSpec]] to finalize and + * obtain a [[TableHandle]] for declaring read/snapshot specs. + */ +class WriteHandle private[workload] (private[workload] val table: TableHandle) + +class WorkloadContext private[workload] ( + val spark: SparkSession, + val workloadName: String, + private[workload] val tags: Seq[String] = Seq.empty) { + + private val _createdTables = mutable.ArrayBuffer[String]() + private[workload] val tableSpecs = mutable.ArrayBuffer[TableSpec]() + + /** Convert nullable java.lang.Long to Option[Long]. */ + private def opt(v: java.lang.Long): Option[Long] = Option(v).map(_.longValue()) + + // Per-table state: spec names must be unique within each table + private val _tableSpecNames = mutable.HashMap[String, mutable.HashSet[String]]() + + // ---- SQL ---- + + private val _createTableRegex = + """(?i)(?:CREATE|REPLACE)\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?(\w+)`?""".r + + /** Execute SQL. Tracks CREATE TABLE statements for cleanup. */ + def sql(statement: String): Unit = { + _createTableRegex.findFirstMatchIn(statement).foreach { m => + _createdTables += m.group(1) + } + spark.sql(statement) + } + + // ---- Table handles ---- + + /** Register a managed Spark table for spec capture. */ + def registerTable(name: String): TableHandle = { + val path = resolveTablePath(name) + val handle = new TableHandle(name, path, this) + ensureTableSpec(handle) + handle + } + + /** Register a table at a filesystem path for spec capture. */ + def registerTableFromPath(path: String): TableHandle = { + val p = Paths.get(path) + require(Files.exists(p.resolve("_delta_log")), + s"No Delta table at path: $path") + val name = p.getFileName.toString + val handle = new TableHandle(name, p, this) + ensureTableSpec(handle) + handle + } + + // ---- Spec declaration ---- + + /** + * Read spec. Name auto-generated from parameters. + * + * If `expectError` is set, capture asserts that Spark MUST throw when the + * read is executed: + * - non-null + non-empty: the thrown error's code must match (after + * normalization). Capture fails if the operation succeeds or the code + * doesn't match. + * - non-null + empty string `""`: any error is accepted; capture fails + * only if the operation succeeds. + * - null (default): preserve legacy auto-detect behavior — record + * whatever Spark does (success or error). + */ + def readSpec( + table: TableHandle, + predicate: String = null, + version: java.lang.Long = null, + timestamp: String = null, + columns: Seq[String] = null, + name: String = null, + expectError: String = null): SpecRef[ReadSpec] = { + val specName = if (name != null) name else autoReadName( + Option(predicate), opt(version), + Option(timestamp), Option(columns)) + requireUnique(table, specName) + val config = ReadSpecConfig( + specName, Option(predicate), opt(version), + Option(timestamp), Option(columns), Option(expectError)) + getTableSpec(table).readSpecs += config + new SpecRef(config) + } + + /** + * Snapshot spec. + * + * `expectError` has the same semantics as on [[readSpec]]: declaring it + * asserts that Spark must throw, and capture fails loudly otherwise. + */ + def snapshotSpec( + table: TableHandle, + version: java.lang.Long = null, + timestamp: String = null, + expectError: String = null): SpecRef[SnapshotSpec] = { + val config = SnapshotSpecConfig(opt(version), Option(timestamp), Option(expectError)) + getTableSpec(table).snapshotSpecs += config + new SpecRef(config) + } + + /** + * Force Spark to write a checkpoint file for the given SQL table name. + * Convenience wrapper around `DeltaHarness.openLog(...).checkpoint()`. + */ + def forceCheckpoint(tableName: String): Unit = { + val loc = spark.sql(s"DESCRIBE DETAIL `$tableName`").collect()(0).getAs[String]("location") + DeltaHarness.get.openLog(spark, loc).checkpoint() + } + + // ---- Write specs ---- + + private val _writeBuilders = mutable.HashMap[String, WriteSpecBuilder]() + + private def getWriteBuilder(table: TableHandle): WriteSpecBuilder = { + val key = s"${workloadName}_${table.tableName}" + val builder = _writeBuilders.getOrElseUpdate(key, new WriteSpecBuilder()) + getTableSpec(table).writeBuilder = Some(builder) + builder + } + + /** Wrap an already-created [[TableHandle]] for structured write operations. */ + def writeSpec(table: TableHandle): WriteHandle = { + getWriteBuilder(table) + new WriteHandle(table) + } + + /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ + def registerWriteSpec(w: WriteHandle): TableHandle = w.table + + /** + * Create a table via SQL and record the create_table operation. `schema` is a SQL + * DDL string (e.g. "id INT, name STRING NOT NULL"). Returns a [[WriteHandle]] for + * further write operations. + */ + def createTableOp( + tableName: String, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty): WriteHandle = { + val partitionClause = + if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" + val propsClause = if (properties.nonEmpty) { + s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" + } else "" + sql(s"CREATE TABLE $tableName ($schema) USING delta$partitionClause$propsClause") + val t = registerTable(tableName) + getWriteBuilder(t).recordCreateTable(schema, properties, partitionColumns) + new WriteHandle(t) + } + + /** + * Replace an existing table's schema/partitioning/properties and all data via + * `CREATE OR REPLACE TABLE`, and record the replace_table operation. `schema` is a SQL DDL + * string. With non-empty `rows` it is a replace-as-select (RTAS) — a single commit that also + * writes the rows. Operates on the table behind `w`; subsequent ops see the new schema. + */ + def replaceTableOp( + w: WriteHandle, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty, + rows: Seq[Map[String, Any]] = Seq.empty): Unit = { + val partitionClause = + if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" + val propsClause = if (properties.nonEmpty) { + s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" + } else "" + if (rows.nonEmpty) { + // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so + // the resulting schema matches what the bundled spec Parquet will reproduce on replay. + val parquet = RowParquet.writeTemp(spark, StructType.fromDDL(schema), rows) + try { + sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} USING delta$partitionClause" + + s"$propsClause AS SELECT * FROM parquet.`${parquet.toAbsolutePath}`") + } finally { + FileUtils.deleteDirectory(parquet.getParent.toFile) + } + } else { + sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} ($schema) USING delta" + + s"$partitionClause$propsClause") + } + getWriteBuilder(w.table).recordReplaceTable(schema, properties, partitionColumns, rows) + } + + /** + * Insert `rows` (column -> value maps) and record the insert. An empty `rows` is a no-op: + * it produces no commit, so recording it would both desync the commit-index/version mapping + * and let the validator pass a spec with nothing to validate. + */ + def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = { + require(rows.nonEmpty, "insertOp requires at least one row") + // Drive the live insert from the SAME materialized Parquet the spec will bundle (via + // RowParquet), so the captured table and the replayed table use one value encoder and + // cannot diverge by type. The spec's own copy is re-materialized in buildSpec. + val schema = RowParquet.schemaAt(spark, w.table.sourcePath.toString, + version = None, includePartition = true) + val parquet = RowParquet.writeTemp(spark, schema, rows) + try { + sql(s"INSERT INTO ${w.table.tableName} SELECT * FROM parquet.`${parquet.toAbsolutePath}`") + } finally { + FileUtils.deleteDirectory(parquet.getParent.toFile) + } + getWriteBuilder(w.table).recordInsert(rows) + } + + /** Delete rows matching `predicate` and record the delete. */ + def deleteOp(w: WriteHandle, predicate: String): Unit = { + sql(s"DELETE FROM ${w.table.tableName} WHERE $predicate") + getWriteBuilder(w.table).recordDelete(predicate) + } + + /** Update rows matching `predicate` with `set` (column -> expression) and record it. */ + def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = { + val setClauses = set.map { case (k, v) => s"`$k` = $v" }.mkString(", ") + sql(s"UPDATE ${w.table.tableName} SET $setClauses WHERE $predicate") + getWriteBuilder(w.table).recordUpdate(predicate, set) + } + + + /** + * Execute a low-level commit of raw Delta actions against the live table and record it. `addFiles` + * supply LOGICAL rows; the engine writes them (column-mapping/partition/stats handled) and the + * actions are bundled with the raw `txn`/`addDomainMetadata`/`removeDomainMetadata`/schema/property + * changes in one commit. `removeFiles` reference a prior low-level add by the ordinal that + * [[commitOp]] returned (the commit/version that produced it). Returns this commit's ordinal so a + * later `removeFiles` can target it. + */ + def commitOp( + w: WriteHandle, + schemaDDL: Option[String] = None, + tableProperties: Option[Map[String, String]] = None, + txn: Option[AppTxn] = None, + addFiles: Option[Seq[AddFileInput]] = None, + removeFiles: Option[Seq[Int]] = None, + addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, + removeDomainMetadata: Option[Seq[String]] = None): Int = { + require( + schemaDDL.isDefined || tableProperties.exists(_.nonEmpty) || txn.isDefined || + addFiles.exists(_.nonEmpty) || removeFiles.exists(_.nonEmpty) || + addDomainMetadata.exists(_.nonEmpty) || removeDomainMetadata.exists(_.nonEmpty), + "commitOp requires at least one action (schema/properties/txn/addFiles/removeFiles/" + + "domainMetadata); an empty commit would not advance the table version") + val builder = getWriteBuilder(w.table) + val idx = builder.nextOrdinal + val livePath = w.table.sourcePath.toString + val adds = addFiles.getOrElse(Seq.empty) + + // Materialize each add's logical rows to a temp Parquet for the live write. + val schema = RowParquet.schemaAt(spark, livePath, version = None, includePartition = true) + val tempParquet = adds.map(in => RowParquet.writeTemp(spark, schema, in.rows)) + // A remove targets all files added at the referenced commit (== that table version). + val removePaths = removeFiles.getOrElse(Seq.empty) + .flatMap(k => SpecLayout.addPathsAt(w.table.sourcePath, k)) + try { + DeltaHarness.get.commitWithData(spark, livePath, tempParquet.map(_.toAbsolutePath.toString), + CommitRequest( + schemaJson = schemaDDL.map(ddl => StructType.fromDDL(ddl).json), + properties = tableProperties, + setTransaction = txn, + removeFiles = removePaths.map(p => CommitRemoveFile(p, dataChange = true)), + addDomainMetadata = addDomainMetadata.getOrElse(Seq.empty), + removeDomainMetadata = removeDomainMetadata.getOrElse(Seq.empty))) + } finally { + tempParquet.foreach(p => FileUtils.deleteDirectory(p.getParent.toFile)) + } + + // Enforce the load-bearing commit-index == table-version invariant at the point it is + // established: ordinal-based remove resolution (SpecLayout.addPathsAt) keys off it. Fail here, + // at the offending op, rather than deferring to buildSpec's whole-spec count check. + val newVersion = DeltaHarness.get.openLog(spark, livePath).update().version + require(newVersion == idx, + s"commitOp produced version $newVersion but expected ordinal $idx; commit index can no " + + "longer be used as the table version (ordinal-based removes would mis-resolve)") + + // Record the spec: adds become logical-Parquet pointers (materialized to data/ in buildSpec); + // removes become ordinal references. + val addActions = adds.zipWithIndex.map { case (in, i) => + AddFileAction(dataFile = SpecLayout.commitDataFile(idx, s"add_$i.parquet"), dataChange = in.dataChange) + } + builder.recordCommit( + schemaDDL = schemaDDL, tableProperties = tableProperties, txn = txn, + addFiles = if (addActions.nonEmpty) Some(addActions) else None, + removeFiles = removeFiles.map(_.map(k => RemoveFileAction(k))), + addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata) + builder.recordLowLevelRows(idx, adds.map(_.rows)) + idx + } + + /** Add columns (SQL DDL) and record the schema evolution. */ + def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = { + require(columnsDDL.nonEmpty, "addColumnsOp requires a non-empty DDL string") + sql(s"ALTER TABLE ${w.table.tableName} ADD COLUMNS ($columnsDDL)") + getWriteBuilder(w.table).recordAddColumns(columnsDDL) + } + + /** Rename a column and record the schema evolution. */ + def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = { + sql(s"ALTER TABLE ${w.table.tableName} RENAME COLUMN $oldName TO $newName") + getWriteBuilder(w.table).recordRenameColumn(oldName, newName) + } + + /** Drop columns and record the schema evolution. */ + def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = { + require(columns.nonEmpty, "dropColumnsOp requires at least one column") + if (columns.size == 1) { + sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMN ${columns.head}") + } else { + sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMNS (${columns.mkString(", ")})") + } + getWriteBuilder(w.table).recordDropColumns(columns) + } + + /** Set table properties and record the update_properties operation. */ + def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = { + require(props.nonEmpty, "setPropertiesOp requires at least one property") + val setClause = props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + sql(s"ALTER TABLE ${w.table.tableName} SET TBLPROPERTIES ($setClause)") + getWriteBuilder(w.table).recordSetProperties(props) + } + + /** Unset table properties and record the update_properties operation. */ + def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = { + require(props.nonEmpty, "unsetPropertiesOp requires at least one property") + sql(s"ALTER TABLE ${w.table.tableName} UNSET TBLPROPERTIES (${props.map(k => s"'$k'").mkString(", ")})") + getWriteBuilder(w.table).recordUnsetProperties(props) + } + + // ---- Table mutations (applied to copied table before spec capture) ---- + + /** Mutate the copied table's filesystem before specs are captured. */ + def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = { + getTableSpec(table).mutations += mutation + } + + /** + * Modify the typed [[Action]]s of a specific commit version: filter, reorder, or `.copy` fields. + * Unknown/malformed lines surface as [[io.delta.workload.log.RawAction]] and pass through + * unchanged. For byte-level corruption (truncation, garbage), use [[mutateTable]] directly. + */ + def modifyCommitActions(table: TableHandle, version: Long)( + modifier: Seq[Action] => Seq[Action]): Unit = + mutateTable(table) { tableDir => + if (Files.exists(CommitLog.commitFile(tableDir, version))) + CommitLog.mutate(tableDir, version)(modifier) + } + + // ---- Auto-naming ---- + + private def autoReadName( + predicate: Option[String], + version: Option[Long], + timestamp: Option[String], + columns: Option[Seq[String]]): String = { + val parts = mutable.ArrayBuffer[String]("read") + version.foreach(v => parts += s"v$v") + timestamp.foreach { ts => + parts += "ts_" + ts.replace(":", "-").replace(" ", "_").take(23) + } + predicate.foreach { p => + val simplified = p.toLowerCase + .replaceAll(">=", "_gte_").replaceAll("<=", "_lte_") + .replaceAll("<>", "_neq_").replaceAll("!=", "_neq_") + .replaceAll(">", "_gt_").replaceAll("<", "_lt_") + .replaceAll("=", "_eq_") + .replaceAll("\\s+and\\s+", "_and_").replaceAll("\\s+or\\s+", "_or_") + .replaceAll("\\s+is\\s+not\\s+null", "_is_not_null") + .replaceAll("\\s+is\\s+null", "_is_null") + .replaceAll("\\s+in\\s+", "_in_") + .replaceAll("\\s+", "_") + .replaceAll("[^a-z0-9_]", "") + .replaceAll("_+", "_").stripPrefix("_").stripSuffix("_") + .take(50) + parts += simplified + } + columns.foreach { cols => + parts += "cols_" + cols.take(3).mkString("_") + if (cols.size > 3) parts += s"plus${cols.size - 3}" + } + parts.mkString("_") + } + + // ---- Internal ---- + + private def ensureTableSpec(handle: TableHandle): Unit = { + val outputName = s"${workloadName}_${handle.tableName}" + if (!tableSpecs.exists(_.outputName == outputName)) { + tableSpecs += new TableSpec( + _outputName = outputName, + description = s"$workloadName — ${handle.tableName}", + tags = tags, + sourcePath = handle.sourcePath + ) + _tableSpecNames(outputName) = mutable.HashSet[String]() + } + } + + private def getTableSpec(handle: TableHandle): TableSpec = { + val outputName = s"${workloadName}_${handle.tableName}" + tableSpecs.find(_.outputName == outputName).getOrElse( + throw new RuntimeException(s"No table spec for ${handle.tableName}. Call registerTable() first.")) + } + + private def requireUnique(handle: TableHandle, specName: String): Unit = { + val outputName = s"${workloadName}_${handle.tableName}" + val names = _tableSpecNames.getOrElseUpdate(outputName, mutable.HashSet[String]()) + require(names.add(specName), + s"Duplicate spec name '$specName' for table '$outputName'") + } + + private def resolveTablePath(tableName: String): Path = { + try { + val detail = spark.sql(s"DESCRIBE DETAIL $tableName").collect() + require(detail.nonEmpty, s"DESCRIBE DETAIL returned no rows for '$tableName'") + val location = detail(0).getAs[String]("location") + require(location != null && location.nonEmpty, s"'$tableName' has no location") + val path = if (location.startsWith("file:")) { + Paths.get(new java.net.URI(location)) + } else Paths.get(location) + require(Files.exists(path.resolve("_delta_log")), + s"'$tableName' at $path has no _delta_log") + path + } catch { + case e: org.apache.spark.sql.AnalysisException => + throw new RuntimeException( + s"Table '$tableName' not found. Created tables: ${_createdTables.mkString(", ")}", e) + } + } + + private[workload] def cleanup(): Unit = { + val warehouseDir = spark.conf.get("spark.sql.warehouse.dir", "") + val errors = mutable.ArrayBuffer[String]() + + _createdTables.foreach { t => + // Get location before dropping (table may not exist in catalog, which is OK) + val location = try { + val tableId = spark.sessionState.catalog.getTableMetadata( + org.apache.spark.sql.catalyst.TableIdentifier(t)) + Option(tableId.location).map(_.toString) + } catch { + case _: org.apache.spark.sql.catalyst.analysis.NoSuchTableException => None + case _: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException => None + } + + // Drop from catalog + spark.sql(s"DROP TABLE IF EXISTS `$t`") + + // Delete directory: try catalog location, then fall back to warehouse/tableName + val pathsToDelete = location.toSeq.map { loc => + if (loc.startsWith("file:")) Paths.get(new java.net.URI(loc)) else Paths.get(loc) + } ++ (if (warehouseDir.nonEmpty) { + val base = if (warehouseDir.startsWith("file:")) + Paths.get(new java.net.URI(warehouseDir)) else Paths.get(warehouseDir) + Seq(base.resolve(t)) + } else Seq.empty) + + pathsToDelete.distinct.foreach { path => + if (Files.exists(path)) { + org.apache.commons.io.FileUtils.deleteDirectory(path.toFile) + } + } + } + } + +} + +object WorkloadContext { + private val _current = new scala.util.DynamicVariable[WorkloadContext](null) + + /** Get the current WorkloadContext. Throws if called outside a test body. */ + def current: WorkloadContext = { + val ctx = _current.value + require(ctx != null, "No active WorkloadContext. This method must be called inside a test body.") + ctx + } + + /** Execute a block with the given context as current. */ + def withContext[T](ctx: WorkloadContext)(body: => T): T = _current.withValue(ctx)(body) +} diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 2045e0c6..8f59472d 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -16,16 +16,12 @@ package io.delta.workload -import java.nio.file.{Files, Path, Paths} +import java.nio.file.{Files, Path} import scala.collection.mutable -import scala.jdk.CollectionConverters._ import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.types.StructType -import io.delta.workload.deltaharness.{CommitRemoveFile, CommitRequest, DeltaHarness} -import io.delta.workload.log.{Action, CommitLog} /** * Internal workload generation engine. Use [[WorkloadTestSuite]] as the public API: @@ -168,720 +164,13 @@ object WorkloadGenerator { } -// --------------------------------------------------------------------------- -// SpecRef — returned by spec declaration methods for optional assertions -// --------------------------------------------------------------------------- - -/** - * Handle to a declared spec. Attach assertions that are checked after capture. - * `T` is the typed spec class (e.g. [[ReadSpec]], [[SnapshotSpec]]). - */ -class SpecRef[T] private[workload] ( - private[workload] val config: HasAssertion[T]) { - - /** - * Assert that the captured spec is an error (has `expectedError`, no `expected`). - * Works for any spec type that follows the `expected`/`expectedError` convention. - */ - def assertError(): SpecRef[T] = { - config.assertion = Some { node => - require(node.has("expectedError") && !node.get("expectedError").isNull, - s"Expected an error spec but got a success result") - } - this - } - - /** Assert conditions on the captured spec using the typed case class. */ - def assert(check: T => Unit): SpecRef[T] = { - val deserialize = config.deserialize - config.assertion = Some { node => - check(deserialize(node)) - } - this - } -} - private[workload] trait HasAssertion[T] { var assertion: Option[com.fasterxml.jackson.databind.JsonNode => Unit] = None def deserialize: com.fasterxml.jackson.databind.JsonNode => T } // --------------------------------------------------------------------------- -// TableHandle — returned by registerTable() / registerTableFromPath() -// --------------------------------------------------------------------------- - -/** - * Handle to a Delta table created via SQL. Used for declaring read specs - * (read, snapshot) and table mutations (mutateTable, modifyCommitActions). - */ -class TableHandle private[workload] ( - private[workload] val tableName: String, - private[workload] val sourcePath: Path, - private[workload] val ctx: WorkloadContext) { - - /** - * Get the commit timestamp for a specific version as a timestampAsOf-safe - * string (yyyy-MM-dd HH:mm:ss.SSS in session TZ). - * - * Mirrors what Delta's time-travel resolution actually keys off: - * - ICT-enabled tables: `commitInfo.inCommitTimestamp` in the commit JSON. - * - Non-ICT tables: the commit JSON file's mtime. - * - * DESCRIBE HISTORY and `commitInfo.timestamp` both look right at a glance - * but diverge by a few milliseconds from the file mtime on some engines — feeding - * those back as `timestampAsOf` raises DELTA_TIMESTAMP_GREATER_THAN_COMMIT - * / DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION. - */ - def getTimestampForVersion(version: Long): String = { - val padded = f"$version%020d" - val commitPath = sourcePath.resolve(s"_delta_log/$padded.json") - require(Files.exists(commitPath), s"No commit file for version $version: $commitPath") - val ictOpt = scala.collection.JavaConverters - .asScalaBufferConverter(Files.readAllLines(commitPath)).asScala.iterator - .flatMap { line => - val node = JsonUtil.mapper.readTree(line) - Option(node.get("commitInfo")).flatMap(ci => Option(ci.get("inCommitTimestamp"))) - } - .find(_ => true) - .map(_.asLong()) - val tsMillis = ictOpt.getOrElse( - Files.getLastModifiedTime(commitPath).toMillis) - val sessionTz = ctx.spark.conf.get("spark.sql.session.timeZone", "UTC") - val fmt = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") - fmt.setTimeZone(java.util.TimeZone.getTimeZone(sessionTz)) - fmt.format(new java.util.Date(tsMillis)) - } -} - -// --------------------------------------------------------------------------- -// WriteHandle — returned by createTableOp() / writeSpec() -// --------------------------------------------------------------------------- - -/** - * Handle for structured write operations. Obtained from [[WorkloadContext.createTableOp]] - * (which creates the table and records the create) or [[WorkloadContext.writeSpec]] - * (which wraps an already-created [[TableHandle]]). - * - * Accepts write operations (insertOp, updateOp, deleteOp, ...) that both execute SQL - * and record the commit. Call [[WorkloadContext.registerWriteSpec]] to finalize and - * obtain a [[TableHandle]] for declaring read/snapshot specs. - */ -class WriteHandle private[workload] (private[workload] val table: TableHandle) - -// --------------------------------------------------------------------------- -// WorkloadContext — the user's interface inside a workload() block -// --------------------------------------------------------------------------- - -class WorkloadContext private[workload] ( - val spark: SparkSession, - val workloadName: String, - private[workload] val tags: Seq[String] = Seq.empty) { - - private val _createdTables = mutable.ArrayBuffer[String]() - private[workload] val tableSpecs = mutable.ArrayBuffer[TableSpec]() - - /** Convert nullable java.lang.Long to Option[Long]. */ - private def opt(v: java.lang.Long): Option[Long] = Option(v).map(_.longValue()) - - // Per-table state: spec names must be unique within each table - private val _tableSpecNames = mutable.HashMap[String, mutable.HashSet[String]]() - - // ---- SQL ---- - - private val _createTableRegex = - """(?i)(?:CREATE|REPLACE)\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?(\w+)`?""".r - - /** Execute SQL. Tracks CREATE TABLE statements for cleanup. */ - def sql(statement: String): Unit = { - _createTableRegex.findFirstMatchIn(statement).foreach { m => - _createdTables += m.group(1) - } - spark.sql(statement) - } - - // ---- Table handles ---- - - /** Register a managed Spark table for spec capture. */ - def registerTable(name: String): TableHandle = { - val path = resolveTablePath(name) - val handle = new TableHandle(name, path, this) - ensureTableSpec(handle) - handle - } - - /** Register a table at a filesystem path for spec capture. */ - def registerTableFromPath(path: String): TableHandle = { - val p = Paths.get(path) - require(Files.exists(p.resolve("_delta_log")), - s"No Delta table at path: $path") - val name = p.getFileName.toString - val handle = new TableHandle(name, p, this) - ensureTableSpec(handle) - handle - } - - // ---- Spec declaration ---- - - /** - * Read spec. Name auto-generated from parameters. - * - * If `expectError` is set, capture asserts that Spark MUST throw when the - * read is executed: - * - non-null + non-empty: the thrown error's code must match (after - * normalization). Capture fails if the operation succeeds or the code - * doesn't match. - * - non-null + empty string `""`: any error is accepted; capture fails - * only if the operation succeeds. - * - null (default): preserve legacy auto-detect behavior — record - * whatever Spark does (success or error). - */ - def readSpec( - table: TableHandle, - predicate: String = null, - version: java.lang.Long = null, - timestamp: String = null, - columns: Seq[String] = null, - name: String = null, - expectError: String = null): SpecRef[ReadSpec] = { - val specName = if (name != null) name else autoReadName( - Option(predicate), opt(version), - Option(timestamp), Option(columns)) - requireUnique(table, specName) - val config = ReadSpecConfig( - specName, Option(predicate), opt(version), - Option(timestamp), Option(columns), Option(expectError)) - getTableSpec(table).readSpecs += config - new SpecRef(config) - } - - /** - * Snapshot spec. - * - * `expectError` has the same semantics as on [[readSpec]]: declaring it - * asserts that Spark must throw, and capture fails loudly otherwise. - */ - def snapshotSpec( - table: TableHandle, - version: java.lang.Long = null, - timestamp: String = null, - expectError: String = null): SpecRef[SnapshotSpec] = { - val config = SnapshotSpecConfig(opt(version), Option(timestamp), Option(expectError)) - getTableSpec(table).snapshotSpecs += config - new SpecRef(config) - } - - /** - * Force Spark to write a checkpoint file for the given SQL table name. - * Convenience wrapper around `DeltaHarness.openLog(...).checkpoint()`. - */ - def forceCheckpoint(tableName: String): Unit = { - val loc = spark.sql(s"DESCRIBE DETAIL `$tableName`").collect()(0).getAs[String]("location") - DeltaHarness.get.openLog(spark, loc).checkpoint() - } - - // ---- Write specs ---- - - private val _writeBuilders = mutable.HashMap[String, WriteSpecBuilder]() - - private def getWriteBuilder(table: TableHandle): WriteSpecBuilder = { - val key = s"${workloadName}_${table.tableName}" - val builder = _writeBuilders.getOrElseUpdate(key, new WriteSpecBuilder()) - getTableSpec(table).writeBuilder = Some(builder) - builder - } - - /** Wrap an already-created [[TableHandle]] for structured write operations. */ - def writeSpec(table: TableHandle): WriteHandle = { - getWriteBuilder(table) - new WriteHandle(table) - } - - /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ - def registerWriteSpec(w: WriteHandle): TableHandle = w.table - - /** - * Create a table via SQL and record the create_table operation. `schema` is a SQL - * DDL string (e.g. "id INT, name STRING NOT NULL"). Returns a [[WriteHandle]] for - * further write operations. - */ - def createTableOp( - tableName: String, - schema: String, - properties: Map[String, String] = Map.empty, - partitionColumns: Seq[String] = Seq.empty): WriteHandle = { - val partitionClause = - if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" - val propsClause = if (properties.nonEmpty) { - s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" - } else "" - sql(s"CREATE TABLE $tableName ($schema) USING delta$partitionClause$propsClause") - val t = registerTable(tableName) - getWriteBuilder(t).recordCreateTable(schema, properties, partitionColumns) - new WriteHandle(t) - } - - /** - * Replace an existing table's schema/partitioning/properties and all data via - * `CREATE OR REPLACE TABLE`, and record the replace_table operation. `schema` is a SQL DDL - * string. With non-empty `rows` it is a replace-as-select (RTAS) — a single commit that also - * writes the rows. Operates on the table behind `w`; subsequent ops see the new schema. - */ - def replaceTableOp( - w: WriteHandle, - schema: String, - properties: Map[String, String] = Map.empty, - partitionColumns: Seq[String] = Seq.empty, - rows: Seq[Map[String, Any]] = Seq.empty): Unit = { - val partitionClause = - if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" - val propsClause = if (properties.nonEmpty) { - s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" - } else "" - if (rows.nonEmpty) { - // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so - // the resulting schema matches what the bundled spec Parquet will reproduce on replay. - val parquet = RowParquet.writeTemp(spark, StructType.fromDDL(schema), rows) - try { - sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} USING delta$partitionClause" + - s"$propsClause AS SELECT * FROM parquet.`${parquet.toAbsolutePath}`") - } finally { - FileUtils.deleteDirectory(parquet.getParent.toFile) - } - } else { - sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} ($schema) USING delta" + - s"$partitionClause$propsClause") - } - getWriteBuilder(w.table).recordReplaceTable(schema, properties, partitionColumns, rows) - } - - /** - * Insert `rows` (column -> value maps) and record the insert. An empty `rows` is a no-op: - * it produces no commit, so recording it would both desync the commit-index/version mapping - * and let the validator pass a spec with nothing to validate. - */ - def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = { - require(rows.nonEmpty, "insertOp requires at least one row") - // Drive the live insert from the SAME materialized Parquet the spec will bundle (via - // RowParquet), so the captured table and the replayed table use one value encoder and - // cannot diverge by type. The spec's own copy is re-materialized in buildSpec. - val schema = RowParquet.schemaAt(spark, w.table.sourcePath.toString, - version = None, includePartition = true) - val parquet = RowParquet.writeTemp(spark, schema, rows) - try { - sql(s"INSERT INTO ${w.table.tableName} SELECT * FROM parquet.`${parquet.toAbsolutePath}`") - } finally { - FileUtils.deleteDirectory(parquet.getParent.toFile) - } - getWriteBuilder(w.table).recordInsert(rows) - } - - /** Delete rows matching `predicate` and record the delete. */ - def deleteOp(w: WriteHandle, predicate: String): Unit = { - sql(s"DELETE FROM ${w.table.tableName} WHERE $predicate") - getWriteBuilder(w.table).recordDelete(predicate) - } - - /** Update rows matching `predicate` with `set` (column -> expression) and record it. */ - def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = { - val setClauses = set.map { case (k, v) => s"`$k` = $v" }.mkString(", ") - sql(s"UPDATE ${w.table.tableName} SET $setClauses WHERE $predicate") - getWriteBuilder(w.table).recordUpdate(predicate, set) - } - - - /** - * Execute a low-level commit of raw Delta actions against the live table and record it. `addFiles` - * supply LOGICAL rows; the engine writes them (column-mapping/partition/stats handled) and the - * actions are bundled with the raw `txn`/`addDomainMetadata`/`removeDomainMetadata`/schema/property - * changes in one commit. `removeFiles` reference a prior low-level add by the ordinal that - * [[commitOp]] returned (the commit/version that produced it). Returns this commit's ordinal so a - * later `removeFiles` can target it. - */ - def commitOp( - w: WriteHandle, - schemaDDL: Option[String] = None, - tableProperties: Option[Map[String, String]] = None, - txn: Option[AppTxn] = None, - addFiles: Option[Seq[AddFileInput]] = None, - removeFiles: Option[Seq[Int]] = None, - addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, - removeDomainMetadata: Option[Seq[String]] = None): Int = { - require( - schemaDDL.isDefined || tableProperties.exists(_.nonEmpty) || txn.isDefined || - addFiles.exists(_.nonEmpty) || removeFiles.exists(_.nonEmpty) || - addDomainMetadata.exists(_.nonEmpty) || removeDomainMetadata.exists(_.nonEmpty), - "commitOp requires at least one action (schema/properties/txn/addFiles/removeFiles/" + - "domainMetadata); an empty commit would not advance the table version") - val builder = getWriteBuilder(w.table) - val idx = builder.nextOrdinal - val livePath = w.table.sourcePath.toString - val adds = addFiles.getOrElse(Seq.empty) - - // Materialize each add's logical rows to a temp Parquet for the live write. - val schema = RowParquet.schemaAt(spark, livePath, version = None, includePartition = true) - val tempParquet = adds.map(in => RowParquet.writeTemp(spark, schema, in.rows)) - // A remove targets all files added at the referenced commit (== that table version). - val removePaths = removeFiles.getOrElse(Seq.empty) - .flatMap(k => SpecLayout.addPathsAt(w.table.sourcePath, k)) - try { - DeltaHarness.get.commitWithData(spark, livePath, tempParquet.map(_.toAbsolutePath.toString), - CommitRequest( - schemaJson = schemaDDL.map(ddl => StructType.fromDDL(ddl).json), - properties = tableProperties, - setTransaction = txn, - removeFiles = removePaths.map(p => CommitRemoveFile(p, dataChange = true)), - addDomainMetadata = addDomainMetadata.getOrElse(Seq.empty), - removeDomainMetadata = removeDomainMetadata.getOrElse(Seq.empty))) - } finally { - tempParquet.foreach(p => FileUtils.deleteDirectory(p.getParent.toFile)) - } - - // Enforce the load-bearing commit-index == table-version invariant at the point it is - // established: ordinal-based remove resolution (SpecLayout.addPathsAt) keys off it. Fail here, - // at the offending op, rather than deferring to buildSpec's whole-spec count check. - val newVersion = DeltaHarness.get.openLog(spark, livePath).update().version - require(newVersion == idx, - s"commitOp produced version $newVersion but expected ordinal $idx; commit index can no " + - "longer be used as the table version (ordinal-based removes would mis-resolve)") - - // Record the spec: adds become logical-Parquet pointers (materialized to data/ in buildSpec); - // removes become ordinal references. - val addActions = adds.zipWithIndex.map { case (in, i) => - AddFileAction(dataFile = SpecLayout.commitDataFile(idx, s"add_$i.parquet"), dataChange = in.dataChange) - } - builder.recordCommit( - schemaDDL = schemaDDL, tableProperties = tableProperties, txn = txn, - addFiles = if (addActions.nonEmpty) Some(addActions) else None, - removeFiles = removeFiles.map(_.map(k => RemoveFileAction(k))), - addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata) - builder.recordLowLevelRows(idx, adds.map(_.rows)) - idx - } - - /** Add columns (SQL DDL) and record the schema evolution. */ - def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = { - require(columnsDDL.nonEmpty, "addColumnsOp requires a non-empty DDL string") - sql(s"ALTER TABLE ${w.table.tableName} ADD COLUMNS ($columnsDDL)") - getWriteBuilder(w.table).recordAddColumns(columnsDDL) - } - - /** Rename a column and record the schema evolution. */ - def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = { - sql(s"ALTER TABLE ${w.table.tableName} RENAME COLUMN $oldName TO $newName") - getWriteBuilder(w.table).recordRenameColumn(oldName, newName) - } - - /** Drop columns and record the schema evolution. */ - def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = { - require(columns.nonEmpty, "dropColumnsOp requires at least one column") - if (columns.size == 1) { - sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMN ${columns.head}") - } else { - sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMNS (${columns.mkString(", ")})") - } - getWriteBuilder(w.table).recordDropColumns(columns) - } - - /** Set table properties and record the update_properties operation. */ - def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = { - require(props.nonEmpty, "setPropertiesOp requires at least one property") - val setClause = props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") - sql(s"ALTER TABLE ${w.table.tableName} SET TBLPROPERTIES ($setClause)") - getWriteBuilder(w.table).recordSetProperties(props) - } - - /** Unset table properties and record the update_properties operation. */ - def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = { - require(props.nonEmpty, "unsetPropertiesOp requires at least one property") - sql(s"ALTER TABLE ${w.table.tableName} UNSET TBLPROPERTIES (${props.map(k => s"'$k'").mkString(", ")})") - getWriteBuilder(w.table).recordUnsetProperties(props) - } - - // ---- Table mutations (applied to copied table before spec capture) ---- - - /** Mutate the copied table's filesystem before specs are captured. */ - def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = { - getTableSpec(table).mutations += mutation - } - - /** - * Modify the typed [[Action]]s of a specific commit version: filter, reorder, or `.copy` fields. - * Unknown/malformed lines surface as [[io.delta.workload.log.RawAction]] and pass through - * unchanged. For byte-level corruption (truncation, garbage), use [[mutateTable]] directly. - */ - def modifyCommitActions(table: TableHandle, version: Long)( - modifier: Seq[Action] => Seq[Action]): Unit = - mutateTable(table) { tableDir => - if (Files.exists(CommitLog.commitFile(tableDir, version))) - CommitLog.mutate(tableDir, version)(modifier) - } - - // ---- Auto-naming ---- - - private def autoReadName( - predicate: Option[String], - version: Option[Long], - timestamp: Option[String], - columns: Option[Seq[String]]): String = { - val parts = mutable.ArrayBuffer[String]("read") - version.foreach(v => parts += s"v$v") - timestamp.foreach { ts => - parts += "ts_" + ts.replace(":", "-").replace(" ", "_").take(23) - } - predicate.foreach { p => - val simplified = p.toLowerCase - .replaceAll(">=", "_gte_").replaceAll("<=", "_lte_") - .replaceAll("<>", "_neq_").replaceAll("!=", "_neq_") - .replaceAll(">", "_gt_").replaceAll("<", "_lt_") - .replaceAll("=", "_eq_") - .replaceAll("\\s+and\\s+", "_and_").replaceAll("\\s+or\\s+", "_or_") - .replaceAll("\\s+is\\s+not\\s+null", "_is_not_null") - .replaceAll("\\s+is\\s+null", "_is_null") - .replaceAll("\\s+in\\s+", "_in_") - .replaceAll("\\s+", "_") - .replaceAll("[^a-z0-9_]", "") - .replaceAll("_+", "_").stripPrefix("_").stripSuffix("_") - .take(50) - parts += simplified - } - columns.foreach { cols => - parts += "cols_" + cols.take(3).mkString("_") - if (cols.size > 3) parts += s"plus${cols.size - 3}" - } - parts.mkString("_") - } - - // ---- Internal ---- - - private def ensureTableSpec(handle: TableHandle): Unit = { - val outputName = s"${workloadName}_${handle.tableName}" - if (!tableSpecs.exists(_.outputName == outputName)) { - tableSpecs += new TableSpec( - _outputName = outputName, - description = s"$workloadName — ${handle.tableName}", - tags = tags, - sourcePath = handle.sourcePath - ) - _tableSpecNames(outputName) = mutable.HashSet[String]() - } - } - - private def getTableSpec(handle: TableHandle): TableSpec = { - val outputName = s"${workloadName}_${handle.tableName}" - tableSpecs.find(_.outputName == outputName).getOrElse( - throw new RuntimeException(s"No table spec for ${handle.tableName}. Call registerTable() first.")) - } - - private def requireUnique(handle: TableHandle, specName: String): Unit = { - val outputName = s"${workloadName}_${handle.tableName}" - val names = _tableSpecNames.getOrElseUpdate(outputName, mutable.HashSet[String]()) - require(names.add(specName), - s"Duplicate spec name '$specName' for table '$outputName'") - } - - private def resolveTablePath(tableName: String): Path = { - try { - val detail = spark.sql(s"DESCRIBE DETAIL $tableName").collect() - require(detail.nonEmpty, s"DESCRIBE DETAIL returned no rows for '$tableName'") - val location = detail(0).getAs[String]("location") - require(location != null && location.nonEmpty, s"'$tableName' has no location") - val path = if (location.startsWith("file:")) { - Paths.get(new java.net.URI(location)) - } else Paths.get(location) - require(Files.exists(path.resolve("_delta_log")), - s"'$tableName' at $path has no _delta_log") - path - } catch { - case e: org.apache.spark.sql.AnalysisException => - throw new RuntimeException( - s"Table '$tableName' not found. Created tables: ${_createdTables.mkString(", ")}", e) - } - } - - private[workload] def cleanup(): Unit = { - val warehouseDir = spark.conf.get("spark.sql.warehouse.dir", "") - val errors = mutable.ArrayBuffer[String]() - - _createdTables.foreach { t => - // Get location before dropping (table may not exist in catalog, which is OK) - val location = try { - val tableId = spark.sessionState.catalog.getTableMetadata( - org.apache.spark.sql.catalyst.TableIdentifier(t)) - Option(tableId.location).map(_.toString) - } catch { - case _: org.apache.spark.sql.catalyst.analysis.NoSuchTableException => None - case _: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException => None - } - - // Drop from catalog - spark.sql(s"DROP TABLE IF EXISTS `$t`") - - // Delete directory: try catalog location, then fall back to warehouse/tableName - val pathsToDelete = location.toSeq.map { loc => - if (loc.startsWith("file:")) Paths.get(new java.net.URI(loc)) else Paths.get(loc) - } ++ (if (warehouseDir.nonEmpty) { - val base = if (warehouseDir.startsWith("file:")) - Paths.get(new java.net.URI(warehouseDir)) else Paths.get(warehouseDir) - Seq(base.resolve(t)) - } else Seq.empty) - - pathsToDelete.distinct.foreach { path => - if (Files.exists(path)) { - org.apache.commons.io.FileUtils.deleteDirectory(path.toFile) - } - } - } - } - -} - -object WorkloadContext { - private val _current = new scala.util.DynamicVariable[WorkloadContext](null) - - /** Get the current WorkloadContext. Throws if called outside a test body. */ - def current: WorkloadContext = { - val ctx = _current.value - require(ctx != null, "No active WorkloadContext. This method must be called inside a test body.") - ctx - } - - /** Execute a block with the given context as current. */ - def withContext[T](ctx: WorkloadContext)(body: => T): T = _current.withValue(ctx)(body) -} - -// --------------------------------------------------------------------------- -// WorkloadOps — DSL trait mixed into WorkloadSuite for clean syntax -// --------------------------------------------------------------------------- - -/** - * Provides DSL methods for workload generation. Mixed into WorkloadSuite - * so that test bodies can call methods like `sql()`, `registerTable()`, `readSpec()` - * directly without a context prefix. - */ -trait WorkloadOps { - import WorkloadContext.current - - /** The active SparkSession. */ - def spark: SparkSession = current.spark - - /** Execute SQL. Tracks CREATE TABLE statements for cleanup. */ - def sql(statement: String): Unit = current.sql(statement) - - /** Register a managed Spark table for spec capture. */ - def registerTable(name: String): TableHandle = current.registerTable(name) - - /** Register a table at a filesystem path for spec capture. */ - def registerTableFromPath(path: String): TableHandle = current.registerTableFromPath(path) - - /** - * Read spec. Name auto-generated from parameters. See - * [[WorkloadContext.readSpec]] for the semantics of `expectError`. - */ - def readSpec( - table: TableHandle, - predicate: String = null, - version: java.lang.Long = null, - timestamp: String = null, - columns: Seq[String] = null, - name: String = null, - expectError: String = null): SpecRef[ReadSpec] = - current.readSpec(table, predicate, version, timestamp, columns, name, expectError) - - /** - * Snapshot spec. See [[WorkloadContext.snapshotSpec]] for the semantics of - * `expectError`. - */ - def snapshotSpec( - table: TableHandle, - version: java.lang.Long = null, - timestamp: String = null, - expectError: String = null): SpecRef[SnapshotSpec] = - current.snapshotSpec(table, version, timestamp, expectError) - - /** Force Spark to write a checkpoint file for the given SQL table name. */ - def forceCheckpoint(tableName: String): Unit = current.forceCheckpoint(tableName) - - // ---- Write specs ---- - - /** Wrap an already-created [[TableHandle]] for structured write operations. */ - def writeSpec(table: TableHandle): WriteHandle = current.writeSpec(table) - - /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ - def registerWriteSpec(w: WriteHandle): TableHandle = current.registerWriteSpec(w) - - /** Create a table via SQL and record the create_table operation. */ - def createTableOp( - tableName: String, - schema: String, - properties: Map[String, String] = Map.empty, - partitionColumns: Seq[String] = Seq.empty): WriteHandle = - current.createTableOp(tableName, schema, properties, partitionColumns) - - /** Replace an existing table (schema/partitioning/properties + data) and record it. */ - def replaceTableOp( - w: WriteHandle, - schema: String, - properties: Map[String, String] = Map.empty, - partitionColumns: Seq[String] = Seq.empty, - rows: Seq[Map[String, Any]] = Seq.empty): Unit = - current.replaceTableOp(w, schema, properties, partitionColumns, rows) - - /** Insert rows and record the insert. */ - def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = current.insertOp(w, rows) - - /** Delete rows matching `predicate` and record the delete. */ - def deleteOp(w: WriteHandle, predicate: String): Unit = current.deleteOp(w, predicate) - - /** Update rows matching `predicate` with `set` and record the update. */ - def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = - current.updateOp(w, predicate, set) - - - /** Execute a low-level commit of raw Delta actions and record it. Returns the commit ordinal. */ - def commitOp( - w: WriteHandle, - schemaDDL: Option[String] = None, - tableProperties: Option[Map[String, String]] = None, - txn: Option[AppTxn] = None, - addFiles: Option[Seq[AddFileInput]] = None, - removeFiles: Option[Seq[Int]] = None, - addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, - removeDomainMetadata: Option[Seq[String]] = None): Int = - current.commitOp(w, schemaDDL, tableProperties, txn, addFiles, removeFiles, - addDomainMetadata, removeDomainMetadata) - - /** Add columns (SQL DDL) and record the schema evolution. */ - def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = - current.addColumnsOp(w, columnsDDL) - - /** Rename a column and record the schema evolution. */ - def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = - current.renameColumnOp(w, oldName, newName) - - /** Drop columns and record the schema evolution. */ - def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = - current.dropColumnsOp(w, columns) - - /** Set table properties and record the update_properties operation. */ - def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = - current.setPropertiesOp(w, props) - - /** Unset table properties and record the update_properties operation. */ - def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = - current.unsetPropertiesOp(w, props) - - /** Mutate the copied table's filesystem before specs are captured. */ - def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = - current.mutateTable(table)(mutation) - - /** Modify the typed actions of a specific commit version. */ - def modifyCommitActions(table: TableHandle, version: Long)( - modifier: Seq[Action] => Seq[Action]): Unit = - current.modifyCommitActions(table, version)(modifier) -} - -// --------------------------------------------------------------------------- -// Internal data structures +// Internal data structures — the model the DSL builds and the engine consumes // --------------------------------------------------------------------------- private[workload] class TableSpec( diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala new file mode 100644 index 00000000..9470de8e --- /dev/null +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala @@ -0,0 +1,150 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.workload + +import java.nio.file.Path + +import org.apache.spark.sql.SparkSession + +import io.delta.workload.log.Action + +/** + * Provides DSL methods for workload generation. Mixed into WorkloadSuite + * so that test bodies can call methods like `sql()`, `registerTable()`, `readSpec()` + * directly without a context prefix. + */ +trait WorkloadOps { + import WorkloadContext.current + + /** The active SparkSession. */ + def spark: SparkSession = current.spark + + /** Execute SQL. Tracks CREATE TABLE statements for cleanup. */ + def sql(statement: String): Unit = current.sql(statement) + + /** Register a managed Spark table for spec capture. */ + def registerTable(name: String): TableHandle = current.registerTable(name) + + /** Register a table at a filesystem path for spec capture. */ + def registerTableFromPath(path: String): TableHandle = current.registerTableFromPath(path) + + /** + * Read spec. Name auto-generated from parameters. See + * [[WorkloadContext.readSpec]] for the semantics of `expectError`. + */ + def readSpec( + table: TableHandle, + predicate: String = null, + version: java.lang.Long = null, + timestamp: String = null, + columns: Seq[String] = null, + name: String = null, + expectError: String = null): SpecRef[ReadSpec] = + current.readSpec(table, predicate, version, timestamp, columns, name, expectError) + + /** + * Snapshot spec. See [[WorkloadContext.snapshotSpec]] for the semantics of + * `expectError`. + */ + def snapshotSpec( + table: TableHandle, + version: java.lang.Long = null, + timestamp: String = null, + expectError: String = null): SpecRef[SnapshotSpec] = + current.snapshotSpec(table, version, timestamp, expectError) + + /** Force Spark to write a checkpoint file for the given SQL table name. */ + def forceCheckpoint(tableName: String): Unit = current.forceCheckpoint(tableName) + + // ---- Write specs ---- + + /** Wrap an already-created [[TableHandle]] for structured write operations. */ + def writeSpec(table: TableHandle): WriteHandle = current.writeSpec(table) + + /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ + def registerWriteSpec(w: WriteHandle): TableHandle = current.registerWriteSpec(w) + + /** Create a table via SQL and record the create_table operation. */ + def createTableOp( + tableName: String, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty): WriteHandle = + current.createTableOp(tableName, schema, properties, partitionColumns) + + /** Replace an existing table (schema/partitioning/properties + data) and record it. */ + def replaceTableOp( + w: WriteHandle, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty, + rows: Seq[Map[String, Any]] = Seq.empty): Unit = + current.replaceTableOp(w, schema, properties, partitionColumns, rows) + + /** Insert rows and record the insert. */ + def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = current.insertOp(w, rows) + + /** Delete rows matching `predicate` and record the delete. */ + def deleteOp(w: WriteHandle, predicate: String): Unit = current.deleteOp(w, predicate) + + /** Update rows matching `predicate` with `set` and record the update. */ + def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = + current.updateOp(w, predicate, set) + + + /** Execute a low-level commit of raw Delta actions and record it. Returns the commit ordinal. */ + def commitOp( + w: WriteHandle, + schemaDDL: Option[String] = None, + tableProperties: Option[Map[String, String]] = None, + txn: Option[AppTxn] = None, + addFiles: Option[Seq[AddFileInput]] = None, + removeFiles: Option[Seq[Int]] = None, + addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, + removeDomainMetadata: Option[Seq[String]] = None): Int = + current.commitOp(w, schemaDDL, tableProperties, txn, addFiles, removeFiles, + addDomainMetadata, removeDomainMetadata) + + /** Add columns (SQL DDL) and record the schema evolution. */ + def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = + current.addColumnsOp(w, columnsDDL) + + /** Rename a column and record the schema evolution. */ + def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = + current.renameColumnOp(w, oldName, newName) + + /** Drop columns and record the schema evolution. */ + def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = + current.dropColumnsOp(w, columns) + + /** Set table properties and record the update_properties operation. */ + def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = + current.setPropertiesOp(w, props) + + /** Unset table properties and record the update_properties operation. */ + def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = + current.unsetPropertiesOp(w, props) + + /** Mutate the copied table's filesystem before specs are captured. */ + def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = + current.mutateTable(table)(mutation) + + /** Modify the typed actions of a specific commit version. */ + def modifyCommitActions(table: TableHandle, version: Long)( + modifier: Seq[Action] => Seq[Action]): Unit = + current.modifyCommitActions(table, version)(modifier) +} From 095f43dd708654c2e14a5d7e8b402879d58daa75 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 16:33:03 +0000 Subject: [PATCH 07/25] perf: validate test dirs concurrently in validateAll validateAll walked test dirs sequentially. Each validateTestDir is independent (its own temp replay tables, path-based table refs, no shared catalog names) and SparkSession is thread-safe, so run them on a bounded thread pool and let Spark's scheduler interleave the per-dir jobs. Commits are per-table, so the global DeltaLog cache only thrashes; the result is unchanged. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/WorkloadValidator.scala | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index 196b7650..b8931262 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -17,8 +17,11 @@ package io.delta.workload import java.nio.file.{Files, Path} +import java.util.concurrent.Executors import scala.collection.mutable +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.duration.Duration import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal @@ -50,14 +53,18 @@ object WorkloadValidator { def validateAll(spark: SparkSession, workloadRoot: Path): ValidationResult = { require(Files.isDirectory(workloadRoot), s"Not a directory: $workloadRoot") - val errors = mutable.ArrayBuffer[String]() - var passed = 0 - for (testDir <- listChildren(workloadRoot).filter(Files.isDirectory(_))) { - val r = validateTestDir(spark, testDir) - passed += r.passed - errors ++= r.errors - } - ValidationResult(passed, errors.toSeq) + val testDirs = listChildren(workloadRoot).filter(Files.isDirectory(_)) + // Each test dir validates independently (its own temp replay tables, path-based table refs), + // and SparkSession is thread-safe, so validate them concurrently and let Spark's scheduler + // interleave the per-dir jobs. (DeltaLog's global cache only thrashes under this; commits are + // per-table, so the result stays correct.) + val pool = Executors.newFixedThreadPool( + math.min(8, math.max(1, Runtime.getRuntime.availableProcessors))) + implicit val ec: ExecutionContext = ExecutionContext.fromExecutorService(pool) + val results = + try Await.result(Future.sequence(testDirs.map(d => Future(validateTestDir(spark, d)))), Duration.Inf) + finally pool.shutdown() + ValidationResult(results.map(_.passed).sum, results.flatMap(_.errors)) } /** From fe1d92cd71369e7b4dc56dea3006baca53ab38a6 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 16:44:02 +0000 Subject: [PATCH 08/25] style: drop em-dashes from comments per house style Comment etiquette pass over the files this change touches: replace em-dashes with plain punctuation (colon/semicolon/period). Comments only; no behavior change. Pre-existing untouched files and data literals are left alone. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../main/scala/io/delta/workload/WorkloadContext.scala | 6 +++--- .../main/scala/io/delta/workload/WorkloadGenerator.scala | 8 ++++---- .../main/scala/io/delta/workload/WorkloadValidator.scala | 6 +++--- .../main/scala/io/delta/workload/WriteSpecCapture.scala | 4 ++-- .../scala/io/delta/workload/WorkloadGeneratorSuite.scala | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala index 15b6670a..d9b4913b 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala @@ -74,7 +74,7 @@ class TableHandle private[workload] ( * - Non-ICT tables: the commit JSON file's mtime. * * DESCRIBE HISTORY and `commitInfo.timestamp` both look right at a glance - * but diverge by a few milliseconds from the file mtime on some engines — feeding + * but diverge by a few milliseconds from the file mtime on some engines. Feeding * those back as `timestampAsOf` raises DELTA_TIMESTAMP_GREATER_THAN_COMMIT * / DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION. */ @@ -170,7 +170,7 @@ class WorkloadContext private[workload] ( * doesn't match. * - non-null + empty string `""`: any error is accepted; capture fails * only if the operation succeeds. - * - null (default): preserve legacy auto-detect behavior — record + * - null (default): preserve legacy auto-detect behavior, recording * whatever Spark does (success or error). */ def readSpec( @@ -261,7 +261,7 @@ class WorkloadContext private[workload] ( /** * Replace an existing table's schema/partitioning/properties and all data via * `CREATE OR REPLACE TABLE`, and record the replace_table operation. `schema` is a SQL DDL - * string. With non-empty `rows` it is a replace-as-select (RTAS) — a single commit that also + * string. With non-empty `rows` it is a replace-as-select (RTAS): a single commit that also * writes the rows. Operates on the table behind `w`; subsequent ops see the new schema. */ def replaceTableOp( diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 8f59472d..4e70f6f1 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -69,15 +69,15 @@ object WorkloadGenerator { // Copy Delta table. Skip: // - transient files that async engine hooks may be mid-cleaning (e.g. - // some engines write `.crc..tmp` files and delete them shortly after — + // some engines write `.crc..tmp` files and delete them shortly after; // they can disappear between listing and copying) - // - Hadoop CRC sidecars (dot-prefixed `.NAME.crc`) — these would go + // - Hadoop CRC sidecars (dot-prefixed `.NAME.crc`): these would go // stale on any subsequent `mutateTable` / `modifyCommitActions` and // cause `ChecksumFileSystem` to reject the (intentionally) corrupted // file. Workloads use `RawLocalFileSystem` for reads anyway, so the // sidecars carry no information for our consumers. // NOTE: Delta version-checksums (`_delta_log/NNN.crc`, no dot prefix) are - // NOT excluded — those are protocol-level artifacts. + // NOT excluded; those are protocol-level artifacts. val destTablePath = testOutputDir.resolve("delta") val copyFilter: java.io.FileFilter = (f: java.io.File) => { val n = f.getName @@ -170,7 +170,7 @@ private[workload] trait HasAssertion[T] { } // --------------------------------------------------------------------------- -// Internal data structures — the model the DSL builds and the engine consumes +// Internal data structures: the model the DSL builds and the engine consumes // --------------------------------------------------------------------------- private[workload] class TableSpec( diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index b8931262..3ecf546a 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -40,8 +40,8 @@ case class ValidationResult(passed: Int, errors: Seq[String]) { * The single validator for every spec type. Walks a generated workload tree (or one test dir) * and validates each captured spec under `specs/`, dispatching on the `type` field: * - * - `write` — replay its `commits` into a fresh table, compare to `expected//`. - * - `read` / `snapshot` — if the spec carries a `writeSpec` pointer it is *write-derived*: + * - `write`: replay its `commits` into a fresh table, compare to `expected//`. + * - `read` / `snapshot`: if the spec carries a `writeSpec` pointer it is *write-derived*: * reconstruct the table by replaying that sibling write spec, then validate **portably** * (rows-only reads, capability-protocol, column-mapping-normalized schema, declared-config * only). Otherwise it is *read-only*: validate against the captured `delta/` table exactly. @@ -294,7 +294,7 @@ object WorkloadValidator { /** * Replay a low-level commit: write each add's bundled logical Parquet through the engine * (`commitWithData` → column-mapping/partition/stats handled), and tombstone each remove's - * referenced prior add — resolved to this replay table's own paths at that commit/version. + * referenced prior add, resolved to this replay table's own paths at that commit/version. */ private def replayLowLevelCommit( spark: SparkSession, commit: LowLevelCommitOp, idx: Int, diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala index 4231161b..e4f3fb0f 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala @@ -216,8 +216,8 @@ class WriteSpecBuilder { /** * Capture the expected final table state under the write spec's `expected/_write/`: - * - `table_content/` — Parquet of the latest snapshot's rows - * - `table_version_metadata.json` — protocol + metadata of the latest snapshot + * - `table_content/`: Parquet of the latest snapshot's rows + * - `table_version_metadata.json`: protocol + metadata of the latest snapshot */ private def writeExpectedLatest( spark: SparkSession, diff --git a/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala b/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala index 54fa8953..d0d9f523 100644 --- a/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/WorkloadGeneratorSuite.scala @@ -752,7 +752,7 @@ class WorkloadGeneratorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor test("framework: zero-table test warns") { val results = run("t_fw3") { _ => sql("CREATE TABLE tbl (id INT) USING delta") - // Never call registerTable() — should warn + // Never call registerTable(); should warn } assert(results.size == 1) assert(results.head.passed) // passes but with warning From b8af51671e069b5998e9c208bf64f1fb2ece1f67 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 16:44:52 +0000 Subject: [PATCH 09/25] fix: validate row count when expected_data exceeds the materialization limit writeExpectedData skips materializing expected_data when the row count exceeds MaxExpectedDataRows (5M), but validateFromSpec still required the directory, so any large-table read spec (e.g. DeletionVectorsSuite.DV-017, ~2.1B rows) failed with "expected_data is missing". When the directory is absent, verify the re-read row count against the captured count instead; a missing dir at or under the limit remains a hard failure (incomplete corpus). Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../scala/io/delta/workload/ReadCapture.scala | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala b/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala index 030fe94d..0edf65a4 100644 --- a/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala @@ -160,20 +160,27 @@ object ReadCapture { require(actualCode.isDefined, s"Error validation FAILED for $specName: expected operation to fail but it succeeded") } else if (spec.expected.isDefined) { + val exp = spec.expected.get val rereadDf = JsonUtil.applyFilters( JsonUtil.buildDeltaReader(spark, tablePath, spec.version, spec.timestamp), spec.predicate, spec.columns) - // A spec with `expected` MUST have captured expected_data. A missing directory means - // the corpus is incomplete (or one engine produced data and another did not), so fail - // loudly rather than silently skipping the row check. val expectedDataPath = expectedDir.resolve("expected_data") - require(Files.exists(expectedDataPath), - s"Validation FAILED for $specName: expected_data is missing") - JsonUtil.assertMultisetsEqual( - JsonUtil.toRowMultiset(spark.read.parquet(expectedDataPath.toString)), - JsonUtil.toRowMultiset(rereadDf), - specName) + if (Files.exists(expectedDataPath)) { + JsonUtil.assertMultisetsEqual( + JsonUtil.toRowMultiset(spark.read.parquet(expectedDataPath.toString)), + JsonUtil.toRowMultiset(rereadDf), + specName) + } else { + // expected_data is only skipped at capture when the row count exceeds the + // materialization limit (see writeExpectedData), so verify the row count for those + // specs. A missing dir at or under the limit means the corpus is incomplete: fail. + require(exp.rowCount > MaxExpectedDataRows, + s"Validation FAILED for $specName: expected_data is missing") + val actual = rereadDf.count() + require(actual == exp.rowCount, + s"Validation FAILED for $specName: row count $actual != expected ${exp.rowCount}") + } val expectedMetaPath = expectedDir.resolve("expected_metadata") if (checkMetadata && Files.exists(expectedMetaPath)) { From 8fa6e67b4cd0410e1498a3a772156bbdf248c686 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 18:06:25 +0000 Subject: [PATCH 10/25] refactor: address review follow-ups (addPathsAt via log/, doc fidelity, simplify) Post-sweep review batch: - SpecLayout.addPathsAt now resolves remove targets via CommitLog.read + typed AddFile instead of hand-parsing commit JSON (the one reimplementation the log/ unification missed; robust to JSON key ordering). - CommitLog.mutate doc now states honestly that it re-serializes through the typed model and is not byte-exact (matches behavior; use the raw mutateTable path for byte-level corruption). - Docs: fix the WorkloadValidator scaladoc that still described the deleted capability/normalized-schema validation; rewrite the authoring-guide modifyCommitActions examples to the typed Action API (the old ones no longer compiled); add WorkloadContext/WorkloadValidator to the README components; voice fix in spec-reference. - Simplify pass: drop a dead tuple element in ReadCapture; extract partitionedByClause/tblPropertiesClause helpers in WorkloadContext; collapse the duplicated read-only validation into a validateReadOnly helper; fix TableInfoWriter indentation. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- workload-generator/README.md | 6 +- workload-generator/docs/authoring-guide.md | 17 +- workload-generator/docs/spec-reference.md | 2 +- .../scala/io/delta/workload/JsonUtil.scala | 14 +- .../scala/io/delta/workload/ReadCapture.scala | 6 +- .../io/delta/workload/TableInfoWriter.scala | 216 +++++++++--------- .../io/delta/workload/WorkloadContext.scala | 24 +- .../io/delta/workload/WorkloadValidator.scala | 37 +-- .../io/delta/workload/log/CommitLog.scala | 8 +- 9 files changed, 168 insertions(+), 162 deletions(-) diff --git a/workload-generator/README.md b/workload-generator/README.md index 213b4a7e..ba38a05c 100644 --- a/workload-generator/README.md +++ b/workload-generator/README.md @@ -81,8 +81,10 @@ WORKLOAD_FORCE=true WORKLOAD_OUTPUT_DIR=/tmp/workloads sbt "testOnly *ReadsSuite | Component | Purpose | |-----------|---------| | `WorkloadTestSuite` | ScalaTest base class with workload generation integration | -| `WorkloadOps` | DSL trait: `sql()`, `registerTable()`, `readSpec()`, `snapshotSpec()` | -| `WorkloadGenerator` | Orchestrates table copy, spec capture, and validation | +| `WorkloadOps` | DSL trait (pass-through over `WorkloadContext`): `sql()`, `registerTable()`, `readSpec()`, write ops | +| `WorkloadContext` | DSL state: tracks created tables, declared specs, and write builders | +| `WorkloadGenerator` | Orchestrates table copy, spec capture, and self-validation | +| `WorkloadValidator` | Replays/validates a generated workload tree; presubmit acceptance entry point | | `ReadCapture` | Captures read specs with expected row data | | `SnapshotCapture` | Captures snapshot specs with protocol/metadata | | `TableInfoWriter` | Writes table metadata (schema, protocol, stats) | diff --git a/workload-generator/docs/authoring-guide.md b/workload-generator/docs/authoring-guide.md index ea3af2e7..1b35a0bc 100644 --- a/workload-generator/docs/authoring-guide.md +++ b/workload-generator/docs/authoring-guide.md @@ -328,23 +328,22 @@ test("missing_file", "Test missing data file") { ### modifyCommitActions — Action-Level Manipulation -The modifier receives the full list of `(actionType, innerNode)` pairs for a commit and returns the (possibly reordered, filtered, or modified) list to write back. +The modifier receives the commit's typed `Seq[Action]` (subtypes `AddFile`, `RemoveFile`, `Metadata`, `Protocol`, `Txn`, `DomainMetadata`, …; unrecognized or malformed lines arrive as `RawAction` and pass through unchanged) and returns the reordered, filtered, or modified list to write back. Use `.copy` to edit fields. For byte-exact corruption, use the raw `mutateTable` path below instead. ```scala +import io.delta.workload.log.{AddFile, Metadata} + // Modify add actions -modifyCommitActions(t, version = 1) { actions => - actions.map { case ("add", node) => - node.put("stats", """{"numRecords":999}"""); ("add", node) - case other => other - } +modifyCommitActions(t, version = 1) { + _.map { case a: AddFile => a.copy(stats = Some("""{"numRecords":999}""")); case other => other } } // Drop all add actions -modifyCommitActions(t, version = 1) { _.filter(_._1 != "add") } +modifyCommitActions(t, version = 1) { _.filterNot(_.isInstanceOf[AddFile]) } -// Reorder — put metaData last +// Reorder: put metaData last modifyCommitActions(t, version = 0) { actions => - val (meta, rest) = actions.partition(_._1 == "metaData") + val (meta, rest) = actions.partition(_.isInstanceOf[Metadata]) rest ++ meta } ``` diff --git a/workload-generator/docs/spec-reference.md b/workload-generator/docs/spec-reference.md index 5609ad20..b0b73259 100644 --- a/workload-generator/docs/spec-reference.md +++ b/workload-generator/docs/spec-reference.md @@ -453,7 +453,7 @@ The capture comes from one writer, but a write spec must validate against any co - **`configuration`** — only the keys the write spec's own commits declared are checked: `create_table`/`replace_table` `properties` plus `update_properties.set`, minus `update_properties.remove` (a `replace_table` resets the declared set). Each declared key must be present and equal; each removed key must be absent. Engine-injected defaults are ignored. - **rows** (`table_content/`) — multiset comparison; order is irrelevant, but each row must appear the correct number of times. -`schemaString` (with per-field column-mapping `physicalName`/`id` normalized out, since those are minted per table), `partitionColumns`, and `format` are compared. +`schemaString` should be compared with per-field column-mapping `physicalName`/`id` normalized out (those are minted per table); `partitionColumns` and `format` compared exactly. ### Directory Structure diff --git a/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala b/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala index de080559..a018caae 100644 --- a/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala +++ b/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.{Column, DataFrame, SparkSession} import org.apache.spark.sql.functions._ import io.delta.workload.deltaharness.{LogView, SnapshotView} +import io.delta.workload.log.{AddFile, CommitLog} // ============================================================================= // Spec Expected types - success data or error info @@ -291,16 +292,9 @@ private[workload] object SpecLayout { * Because commit index == table version, a low-level remove's `addedAtCommit` ordinal is the * version whose adds it tombstones — resolved here against the actual (engine-assigned) paths. */ - def addPathsAt(tablePath: java.nio.file.Path, version: Int): Seq[String] = { - val commitFile = tablePath.resolve("_delta_log").resolve(f"$version%020d.json") - if (!java.nio.file.Files.exists(commitFile)) return Seq.empty - new String(java.nio.file.Files.readAllBytes(commitFile), "UTF-8") - .split("\n").filter(_.trim.nonEmpty) - .flatMap { line => - val node = JsonUtil.mapper.readTree(line) - Option(node.get("add")).map(_.get("path").asText()) - }.toSeq - } + def addPathsAt(tablePath: Path, version: Int): Seq[String] = + if (!Files.exists(CommitLog.commitFile(tablePath, version))) Seq.empty + else CommitLog.read(tablePath, version).collect { case a: AddFile => a.path } } // ============================================================================= diff --git a/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala b/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala index 0edf65a4..c6dd7cec 100644 --- a/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/ReadCapture.scala @@ -44,7 +44,7 @@ object ReadCapture { val harness = DeltaHarness.get - val (expected, expectedError, addFilesJson) = try { + val (expected, expectedError) = try { val log = harness.openLog(spark, tablePath.toString) val snapshot = JsonUtil.resolveSnapshot(spark, log, tablePath.toString, version, timestamp) @@ -65,13 +65,13 @@ object ReadCapture { try { writeExpectedData(expectedDir, df, count, specName) writeExpectedMetadata(spark, expectedDir, addFilesJson) - (Some(ReadExpected(count, addFilesJson.length, totalFileCount - addFilesJson.length)), None, addFilesJson) + (Some(ReadExpected(count, addFilesJson.length, totalFileCount - addFilesJson.length)), None) } finally { df.unpersist() } } catch { case e: Throwable => - (None, Some(SpecError(JsonUtil.extractErrorCode(e), Option(e.getMessage).getOrElse(""))), Seq.empty[String]) + (None, Some(SpecError(JsonUtil.extractErrorCode(e), Option(e.getMessage).getOrElse("")))) } // If the test author declared `expectError`, the operation MUST throw. diff --git a/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala b/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala index 66a3ef39..3f0f7ba8 100644 --- a/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala +++ b/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala @@ -36,115 +36,115 @@ object TableInfoWriter { description: String, tags: Seq[String] = Seq.empty): Unit = { val log = DeltaHarness.get.openLog(spark, tablePath.toString) - val snapshot = log.update() - - val metadataJson = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") - val schemaObj = JsonUtil.mapper.readValue(metadataJson.get("schemaString").asText(), classOf[Any]) - - val protocolJson = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") - val minReaderV = protocolJson.get("minReaderVersion").asInt() - val minWriterV = protocolJson.get("minWriterVersion").asInt() - val readerFeatures = Option(protocolJson.get("readerFeatures")) - .filter(!_.isNull).map(_.elements().asScala.map(_.asText()).toSeq.sorted) - val writerFeatures = Option(protocolJson.get("writerFeatures")) - .filter(!_.isNull).map(_.elements().asScala.map(_.asText()).toSeq.sorted) - val protocol = ProtocolInfo( - minReaderVersion = minReaderV, - minWriterVersion = minWriterV, - readerFeatures = readerFeatures.filter(_.nonEmpty), - writerFeatures = writerFeatures.filter(_.nonEmpty)) - - val deltaLogDir = tablePath.resolve("_delta_log") - require(Files.isDirectory(deltaLogDir), s"Missing _delta_log at $tablePath") - - def listLog[T](fn: Iterator[Path] => T): T = { - val stream = Files.list(deltaLogDir) - try fn(stream.iterator().asScala) finally stream.close() - } - - val commitFiles = listLog(_.filter(p => - p.toString.endsWith(".json") && !p.toString.contains("checkpoint")).toSeq) - - val numCommits = commitFiles.size - - // Count actions and remove actions in a single pass over each commit, via the typed log. - val (numActions, numRemoveFiles) = commitFiles.foldLeft((0L, 0L)) { - case ((actions, removes), f) => - val version = f.getFileName.toString.stripSuffix(".json").toLong - val as = CommitLog.read(tablePath, version) - (actions + as.size, removes + as.count(_.isInstanceOf[RemoveFile])) - } - - val lastCrcVersion = listLog(_.map(_.getFileName.toString) - .filter(_.endsWith(".crc")) - .flatMap { n => - try Some(n.stripSuffix(".crc").toLong) + val snapshot = log.update() + + val metadataJson = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") + val schemaObj = JsonUtil.mapper.readValue(metadataJson.get("schemaString").asText(), classOf[Any]) + + val protocolJson = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") + val minReaderV = protocolJson.get("minReaderVersion").asInt() + val minWriterV = protocolJson.get("minWriterVersion").asInt() + val readerFeatures = Option(protocolJson.get("readerFeatures")) + .filter(!_.isNull).map(_.elements().asScala.map(_.asText()).toSeq.sorted) + val writerFeatures = Option(protocolJson.get("writerFeatures")) + .filter(!_.isNull).map(_.elements().asScala.map(_.asText()).toSeq.sorted) + val protocol = ProtocolInfo( + minReaderVersion = minReaderV, + minWriterVersion = minWriterV, + readerFeatures = readerFeatures.filter(_.nonEmpty), + writerFeatures = writerFeatures.filter(_.nonEmpty)) + + val deltaLogDir = tablePath.resolve("_delta_log") + require(Files.isDirectory(deltaLogDir), s"Missing _delta_log at $tablePath") + + def listLog[T](fn: Iterator[Path] => T): T = { + val stream = Files.list(deltaLogDir) + try fn(stream.iterator().asScala) finally stream.close() + } + + val commitFiles = listLog(_.filter(p => + p.toString.endsWith(".json") && !p.toString.contains("checkpoint")).toSeq) + + val numCommits = commitFiles.size + + // Count actions and remove actions in a single pass over each commit, via the typed log. + val (numActions, numRemoveFiles) = commitFiles.foldLeft((0L, 0L)) { + case ((actions, removes), f) => + val version = f.getFileName.toString.stripSuffix(".json").toLong + val as = CommitLog.read(tablePath, version) + (actions + as.size, removes + as.count(_.isInstanceOf[RemoveFile])) + } + + val lastCrcVersion = listLog(_.map(_.getFileName.toString) + .filter(_.endsWith(".crc")) + .flatMap { n => + try Some(n.stripSuffix(".crc").toLong) + catch { case _: NumberFormatException => None } + }.toSeq.sorted.lastOption.getOrElse(-1L)) + + val (lastCheckpointVersion, numCheckpointFiles) = { + val lastCheckpointPath = deltaLogDir.resolve("_last_checkpoint") + if (Files.exists(lastCheckpointPath)) { + val cpNode = JsonUtil.mapper.readTree(Files.readAllBytes(lastCheckpointPath)) + val ver = cpNode.get("version").asLong() + val numParts = Option(cpNode.get("parts")).map(_.asInt()).getOrElse(1) + (ver, numParts) + } else { + val cpFiles = listLog(_.map(_.getFileName.toString) + .filter(n => n.contains("checkpoint") && n.endsWith(".parquet")).toSeq) + val maxVersion = cpFiles.flatMap { n => + try Some(n.split("\\.")(0).toLong) catch { case _: NumberFormatException => None } - }.toSeq.sorted.lastOption.getOrElse(-1L)) - - val (lastCheckpointVersion, numCheckpointFiles) = { - val lastCheckpointPath = deltaLogDir.resolve("_last_checkpoint") - if (Files.exists(lastCheckpointPath)) { - val cpNode = JsonUtil.mapper.readTree(Files.readAllBytes(lastCheckpointPath)) - val ver = cpNode.get("version").asLong() - val numParts = Option(cpNode.get("parts")).map(_.asInt()).getOrElse(1) - (ver, numParts) - } else { - val cpFiles = listLog(_.map(_.getFileName.toString) - .filter(n => n.contains("checkpoint") && n.endsWith(".parquet")).toSeq) - val maxVersion = cpFiles.flatMap { n => - try Some(n.split("\\.")(0).toLong) - catch { case _: NumberFormatException => None } - }.sorted.lastOption.getOrElse(-1L) - (maxVersion, cpFiles.count(_.startsWith(f"$maxVersion%020d"))) - } + }.sorted.lastOption.getOrElse(-1L) + (maxVersion, cpFiles.count(_.startsWith(f"$maxVersion%020d"))) } - - val numAddFiles = snapshot.allFiles.count() - val sizeInBytes = if (numAddFiles == 0) 0L - else snapshot.allFiles.selectExpr("sum(size)").first().getLong(0) - - val logInfo = LogInfo( - numAddFiles = numAddFiles, - numRemoveFiles = numRemoveFiles, - sizeInBytes = sizeInBytes, - numCommits = numCommits, - numActions = numActions, - lastCheckpointVersion = lastCheckpointVersion, - lastCrcVersion = lastCrcVersion, - numCheckpointFiles = numCheckpointFiles) - - val partCols = Option(metadataJson.get("partitionColumns")) - .filter(!_.isNull) - .map(_.elements().asScala.map(_.asText()).toSeq) - .getOrElse(Seq.empty) - val numDistinctPartitions = if (partCols.nonEmpty) { - // Convert MAP to JSON string for distinct() since Spark doesn't support set ops on MAP types - snapshot.allFiles.select(to_json(col("partitionValues"))).distinct().count() - } else 0L - - val dataLayout = DataLayoutInfo( - numClusteringColumns = 0, - numPartitionColumns = partCols.size, - numDistinctPartitions = numDistinctPartitions) - - val properties = Option(metadataJson.get("configuration")) - .filter(!_.isNull) - .map { node => - node.fields().asScala.map(e => e.getKey -> e.getValue.asText()).toMap - } - .getOrElse(Map.empty) - - val tableInfo = TableInfo( - name = name, - description = description, - schema = schemaObj, - protocol = protocol, - logInfo = logInfo, - properties = properties, - dataLayout = dataLayout, - tags = if (tags.nonEmpty) Some(tags) else None) - - JsonUtil.writeSpec(outputDir.resolve("table_info.json"), tableInfo) + } + + val numAddFiles = snapshot.allFiles.count() + val sizeInBytes = if (numAddFiles == 0) 0L + else snapshot.allFiles.selectExpr("sum(size)").first().getLong(0) + + val logInfo = LogInfo( + numAddFiles = numAddFiles, + numRemoveFiles = numRemoveFiles, + sizeInBytes = sizeInBytes, + numCommits = numCommits, + numActions = numActions, + lastCheckpointVersion = lastCheckpointVersion, + lastCrcVersion = lastCrcVersion, + numCheckpointFiles = numCheckpointFiles) + + val partCols = Option(metadataJson.get("partitionColumns")) + .filter(!_.isNull) + .map(_.elements().asScala.map(_.asText()).toSeq) + .getOrElse(Seq.empty) + val numDistinctPartitions = if (partCols.nonEmpty) { + // Convert MAP to JSON string for distinct() since Spark doesn't support set ops on MAP types + snapshot.allFiles.select(to_json(col("partitionValues"))).distinct().count() + } else 0L + + val dataLayout = DataLayoutInfo( + numClusteringColumns = 0, + numPartitionColumns = partCols.size, + numDistinctPartitions = numDistinctPartitions) + + val properties = Option(metadataJson.get("configuration")) + .filter(!_.isNull) + .map { node => + node.fields().asScala.map(e => e.getKey -> e.getValue.asText()).toMap + } + .getOrElse(Map.empty) + + val tableInfo = TableInfo( + name = name, + description = description, + schema = schemaObj, + protocol = protocol, + logInfo = logInfo, + properties = properties, + dataLayout = dataLayout, + tags = if (tags.nonEmpty) Some(tags) else None) + + JsonUtil.writeSpec(outputDir.resolve("table_info.json"), tableInfo) } } diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala index d9b4913b..a2cbeea9 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala @@ -219,6 +219,16 @@ class WorkloadContext private[workload] ( // ---- Write specs ---- + /** ` PARTITIONED BY (a, b)` clause, or empty when there are no partition columns. */ + private def partitionedByClause(partitionColumns: Seq[String]): String = + if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" + + /** ` TBLPROPERTIES ('k' = 'v', ...)` clause, or empty when there are no properties. */ + private def tblPropertiesClause(properties: Map[String, String]): String = + if (properties.nonEmpty) { + s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" + } else "" + private val _writeBuilders = mutable.HashMap[String, WriteSpecBuilder]() private def getWriteBuilder(table: TableHandle): WriteSpecBuilder = { @@ -247,11 +257,8 @@ class WorkloadContext private[workload] ( schema: String, properties: Map[String, String] = Map.empty, partitionColumns: Seq[String] = Seq.empty): WriteHandle = { - val partitionClause = - if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" - val propsClause = if (properties.nonEmpty) { - s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" - } else "" + val partitionClause = partitionedByClause(partitionColumns) + val propsClause = tblPropertiesClause(properties) sql(s"CREATE TABLE $tableName ($schema) USING delta$partitionClause$propsClause") val t = registerTable(tableName) getWriteBuilder(t).recordCreateTable(schema, properties, partitionColumns) @@ -270,11 +277,8 @@ class WorkloadContext private[workload] ( properties: Map[String, String] = Map.empty, partitionColumns: Seq[String] = Seq.empty, rows: Seq[Map[String, Any]] = Seq.empty): Unit = { - val partitionClause = - if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" - val propsClause = if (properties.nonEmpty) { - s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" - } else "" + val partitionClause = partitionedByClause(partitionColumns) + val propsClause = tblPropertiesClause(properties) if (rows.nonEmpty) { // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so // the resulting schema matches what the bundled spec Parquet will reproduce on replay. diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index 3ecf546a..a5735b8c 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -40,11 +40,12 @@ case class ValidationResult(passed: Int, errors: Seq[String]) { * The single validator for every spec type. Walks a generated workload tree (or one test dir) * and validates each captured spec under `specs/`, dispatching on the `type` field: * - * - `write`: replay its `commits` into a fresh table, compare to `expected//`. + * - `write`: replay its `commits` into a fresh table, compare rows to `expected//`. * - `read` / `snapshot`: if the spec carries a `writeSpec` pointer it is *write-derived*: - * reconstruct the table by replaying that sibling write spec, then validate **portably** - * (rows-only reads, capability-protocol, column-mapping-normalized schema, declared-config - * only). Otherwise it is *read-only*: validate against the captured `delta/` table exactly. + * reconstruct the table by replaying that sibling write spec, then validate a read spec by + * rows only; a write-derived snapshot only confirms the write spec replays (its + * protocol/metadata were validated against the real table at capture time). Otherwise it is + * *read-only*: validate against the captured `delta/` table exactly. * * Used both as the presubmit acceptance entry point ([[validateAll]]) and by the generator to * self-check a freshly generated test dir ([[validateTestDir]]). @@ -94,6 +95,18 @@ object WorkloadValidator { val node = JsonUtil.mapper.readTree(Files.readAllBytes(specFile)) val specType = Option(node.get("type")).map(_.asText()).getOrElse("") val writePtr = Option(node.get("writeSpec")).filterNot(_.isNull).map(_.asText()) + + // Read-only specs validate against the captured `delta/` table; if it is absent the corpus + // is incomplete. Run `validate` against it (counting a pass), else record the missing table. + def validateReadOnly(validate: => Unit): Unit = + if (Files.isDirectory(deltaTable)) { + validate + passed += 1 + } else { + errors += s"${testDir.getFileName}/$name: read-only spec but no captured " + + s"table at $deltaTable" + } + try { specType match { case "write" => @@ -108,13 +121,7 @@ object WorkloadValidator { checkMetadata = false) passed += 1 case None => // read-only: exact, against the captured table - if (Files.isDirectory(deltaTable)) { - ReadCapture.validateFromSpec(spark, deltaTable, expectedDir, specFile) - passed += 1 - } else { - errors += s"${testDir.getFileName}/$name: read-only spec but no captured " + - s"table at $deltaTable" - } + validateReadOnly(ReadCapture.validateFromSpec(spark, deltaTable, expectedDir, specFile)) } case "snapshot" => @@ -125,13 +132,7 @@ object WorkloadValidator { replayOf(ws) passed += 1 case None => - if (Files.isDirectory(deltaTable)) { - SnapshotCapture.validateFromSpec(spark, deltaTable, specFile) - passed += 1 - } else { - errors += s"${testDir.getFileName}/$name: read-only spec but no captured " + - s"table at $deltaTable" - } + validateReadOnly(SnapshotCapture.validateFromSpec(spark, deltaTable, specFile)) } case other => throw new IllegalArgumentException( diff --git a/workload-generator/src/main/scala/io/delta/workload/log/CommitLog.scala b/workload-generator/src/main/scala/io/delta/workload/log/CommitLog.scala index 38d90d25..d1be12dc 100644 --- a/workload-generator/src/main/scala/io/delta/workload/log/CommitLog.scala +++ b/workload-generator/src/main/scala/io/delta/workload/log/CommitLog.scala @@ -60,7 +60,13 @@ object CommitLog { def write(tableDir: Path, version: Long, actions: Seq[Action]): Unit = writeRaw(tableDir, version, actions.map(_.toJson)) - /** Read → transform → write. The common case for corruption / mutation tests. */ + /** + * Read → transform → write. Actions are re-serialized from the typed model, so this is for + * action-level edits (filter, reorder, `.copy` fields); it does not preserve fields the ADT + * does not model or the exact byte layout of untouched lines. For byte-exact corruption, write + * the raw lines directly (e.g. via mutateTable). Unrecognized/malformed lines round-trip as + * [[RawAction]], so they are preserved verbatim. + */ def mutate(tableDir: Path, version: Long)(fn: Seq[Action] => Seq[Action]): Unit = write(tableDir, version, fn(read(tableDir, version))) } From a56f512797a5b7b8c45c92a4117bda1b78921679 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 18:17:01 +0000 Subject: [PATCH 11/25] fix: replay evolve_schema via ALTER TABLE so the engine enforces column mapping replayEvolveSchema rebuilt the StructType by hand and committed a raw schemaString swap via a metadata-only update. On a non-column-mapping table a rename/drop done that way reinterprets the physical Parquet columns by the new logical name, minting a logically-corrupt table. Replay add/rename/drop through ALTER TABLE instead (mirroring the capture-side ops), so Delta enforces the same column-mapping requirements it does at capture and rejects an unsafe rename/drop. Each EvolveSchemaCommit carries exactly one of add/rename/drop, so this still emits exactly one commit (commit-index == version preserved). Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/WorkloadValidator.scala | 48 ++++++++----------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index a5735b8c..a77e3b1a 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -254,42 +254,36 @@ object WorkloadValidator { spark.sql(s"ALTER TABLE $tableRef UNSET TBLPROPERTIES (${r.map(k => s"'$k'").mkString(", ")})") } - case c: EvolveSchemaCommit => replayEvolveSchema(spark, c, tablePath) + case c: EvolveSchemaCommit => replayEvolveSchema(spark, c, tableRef) case c: LowLevelCommitOp => replayLowLevelCommit(spark, c, idx, tablePath, testDir) } - /** Replay schema add/rename/drop via a single metadata-update commit. */ + /** + * Replay a schema add/rename/drop via `ALTER TABLE` (mirroring the capture-side ops), so the + * engine enforces column-mapping requirements — e.g. it rejects a rename/drop on a non-CM table + * that would reinterpret physical columns by the new logical name. Each `EvolveSchemaCommit` + * carries exactly one of add/rename/drop, so this emits exactly one commit. + */ private def replayEvolveSchema( - spark: SparkSession, commit: EvolveSchemaCommit, tablePath: String): Unit = { - val log = DeltaHarness.get.openLog(spark, tablePath) - val metaNode = Option(JsonUtil.mapper.readTree(log.update().metadataJson).get("metaData")) - .getOrElse(throw new IllegalStateException( - s"evolve_schema replay: no metaData action in current snapshot of $tablePath")) - val schemaString = Option(metaNode.get("schemaString")) - .getOrElse(throw new IllegalStateException( - s"evolve_schema replay: metaData has no schemaString in $tablePath")) - val currentSchema = DataType.fromJson(schemaString.asText()).asInstanceOf[StructType] - var fields = currentSchema.fields.toBuffer - + spark: SparkSession, commit: EvolveSchemaCommit, tableRef: String): Unit = { commit.addColumns.foreach { cols => - cols.asInstanceOf[Seq[Map[String, Any]]].foreach { col => - val name = col("name").toString - val nullable = col.get("nullable").forall(_.asInstanceOf[Boolean]) - fields += StructField( - name, DataType.fromJson(JsonUtil.mapper.writeValueAsString(col("type"))), nullable) - } + val st = StructType(cols.asInstanceOf[Seq[Map[String, Any]]].map { col => + StructField( + col("name").toString, + DataType.fromJson(JsonUtil.mapper.writeValueAsString(col("type"))), + col.get("nullable").forall(_.asInstanceOf[Boolean])) + }) + spark.sql(s"ALTER TABLE $tableRef ADD COLUMNS (${st.toDDL})") } commit.renameColumns.foreach { renames => - for ((oldName, newName) <- renames) { - val i = fields.indexWhere(_.name == oldName) - if (i >= 0) fields(i) = fields(i).copy(name = newName) - } + for ((oldName, newName) <- renames) + spark.sql(s"ALTER TABLE $tableRef RENAME COLUMN $oldName TO $newName") + } + commit.dropColumns.foreach { drops => + if (drops.size == 1) spark.sql(s"ALTER TABLE $tableRef DROP COLUMN ${drops.head}") + else spark.sql(s"ALTER TABLE $tableRef DROP COLUMNS (${drops.mkString(", ")})") } - commit.dropColumns.foreach { drops => fields = fields.filterNot(f => drops.contains(f.name)) } - - DeltaHarness.get.commit(spark, tablePath, - CommitRequest(schemaJson = Some(StructType(fields.toSeq).json))) } /** From e0731a393b573814b5a64d8c59b2acb137746a0a Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 18:29:35 +0000 Subject: [PATCH 12/25] docs: fix WriteCommit field reference to match the sealed ADT The "WriteCommit Fields" table read like the pre-refactor monolithic type (a flat union of all fields). Reframe it as a field glossary over the per-operation case classes (the model is a sealed set discriminated by `operation`; each op carries only its own fields, per the tables above). Also add the missing AppTxn and AddDomainMetadata value-type tables, which were referenced but never defined. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- workload-generator/docs/spec-reference.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/workload-generator/docs/spec-reference.md b/workload-generator/docs/spec-reference.md index b0b73259..07406ca0 100644 --- a/workload-generator/docs/spec-reference.md +++ b/workload-generator/docs/spec-reference.md @@ -401,9 +401,14 @@ enabled (a Delta/Spark constraint). ### WriteCommit Fields +`WriteCommit` is a sealed set of per-operation types discriminated by the leading `operation` +property (each operation carries only the fields listed for it in the tables above). The glossary +below describes each field; a field appears only on the operations that list it. Referenced value +types (`AppTxn`, `AddDomainMetadata`, `AddFileAction`, `RemoveFileAction`) are defined below. + | Field | Type | Description | |-------|------|-------------| -| `operation` | `string` | Operation type (see above) | +| `operation` | `string` | Operation type discriminator (see above) | | `schema` | `object` | Table schema in Delta JSON format | | `partitionColumns` | `string[]` | Partition column names | | `properties` | `map` | Table properties (e.g., `delta.enableDeletionVectors`) | @@ -438,6 +443,20 @@ write path (physical names, partitioning, and stats are derived per table, not s | `addedAtCommit` | `int` | Ordinal (== table version) of the prior low-level `commit` that added the file(s) to tombstone; the consumer resolves it to its own table's path(s) | | `dataChange` | `boolean?` | Whether this is a data change (default true) | +### AppTxn (Low-Level) + +| Field | Type | Description | +|-------|------|-------------| +| `appId` | `string` | Application id for idempotent writes (`txn` action) | +| `version` | `long` | Monotonic transaction version for `appId` | + +### AddDomainMetadata (Low-Level) + +| Field | Type | Description | +|-------|------|-------------| +| `domain` | `string` | Domain name | +| `configuration` | `string` | Domain configuration payload | + ### Expected Data The final table state after replaying all commits is captured under `expected/_write/`: From 7cade76c8f80584641047d5737ad1a978b993a3f Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 19:21:05 +0000 Subject: [PATCH 13/25] feat: insertOp/replaceTableOp accept IterableOnce; generate formulaic test rows Loosen the row parameter of insertOp/replaceTableOp from Seq to IterableOnce[Map[String, Any]] (materialized once internally), so callers can pass a range/iterator/view, not just a literal Seq. All existing Seq(...) callers still compile. Convert the genuinely formulaic insert blocks in WriteBasicSuite to ranges (delete_basic, category grouping, value=id*100); left the irregular ones (named rows, per-type values) as explicit Seq, where a range would be less clear. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/WorkloadContext.scala | 20 +++++++------- .../scala/io/delta/workload/WorkloadOps.scala | 7 +++-- .../workload/tables/WriteBasicSuite.scala | 26 +++++-------------- 3 files changed, 22 insertions(+), 31 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala index a2cbeea9..2d82530e 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala @@ -18,7 +18,7 @@ package io.delta.workload import java.nio.file.{Files, Path, Paths} -import scala.collection.mutable +import scala.collection.{IterableOnce, mutable} import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession @@ -276,13 +276,14 @@ class WorkloadContext private[workload] ( schema: String, properties: Map[String, String] = Map.empty, partitionColumns: Seq[String] = Seq.empty, - rows: Seq[Map[String, Any]] = Seq.empty): Unit = { + rows: IterableOnce[Map[String, Any]] = Seq.empty): Unit = { + val rowSeq = rows.iterator.toSeq val partitionClause = partitionedByClause(partitionColumns) val propsClause = tblPropertiesClause(properties) - if (rows.nonEmpty) { + if (rowSeq.nonEmpty) { // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so // the resulting schema matches what the bundled spec Parquet will reproduce on replay. - val parquet = RowParquet.writeTemp(spark, StructType.fromDDL(schema), rows) + val parquet = RowParquet.writeTemp(spark, StructType.fromDDL(schema), rowSeq) try { sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} USING delta$partitionClause" + s"$propsClause AS SELECT * FROM parquet.`${parquet.toAbsolutePath}`") @@ -293,7 +294,7 @@ class WorkloadContext private[workload] ( sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} ($schema) USING delta" + s"$partitionClause$propsClause") } - getWriteBuilder(w.table).recordReplaceTable(schema, properties, partitionColumns, rows) + getWriteBuilder(w.table).recordReplaceTable(schema, properties, partitionColumns, rowSeq) } /** @@ -301,20 +302,21 @@ class WorkloadContext private[workload] ( * it produces no commit, so recording it would both desync the commit-index/version mapping * and let the validator pass a spec with nothing to validate. */ - def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = { - require(rows.nonEmpty, "insertOp requires at least one row") + def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = { + val rowSeq = rows.iterator.toSeq + require(rowSeq.nonEmpty, "insertOp requires at least one row") // Drive the live insert from the SAME materialized Parquet the spec will bundle (via // RowParquet), so the captured table and the replayed table use one value encoder and // cannot diverge by type. The spec's own copy is re-materialized in buildSpec. val schema = RowParquet.schemaAt(spark, w.table.sourcePath.toString, version = None, includePartition = true) - val parquet = RowParquet.writeTemp(spark, schema, rows) + val parquet = RowParquet.writeTemp(spark, schema, rowSeq) try { sql(s"INSERT INTO ${w.table.tableName} SELECT * FROM parquet.`${parquet.toAbsolutePath}`") } finally { FileUtils.deleteDirectory(parquet.getParent.toFile) } - getWriteBuilder(w.table).recordInsert(rows) + getWriteBuilder(w.table).recordInsert(rowSeq) } /** Delete rows matching `predicate` and record the delete. */ diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala index 9470de8e..e629fda1 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala @@ -18,6 +18,8 @@ package io.delta.workload import java.nio.file.Path +import scala.collection.IterableOnce + import org.apache.spark.sql.SparkSession import io.delta.workload.log.Action @@ -92,11 +94,12 @@ trait WorkloadOps { schema: String, properties: Map[String, String] = Map.empty, partitionColumns: Seq[String] = Seq.empty, - rows: Seq[Map[String, Any]] = Seq.empty): Unit = + rows: IterableOnce[Map[String, Any]] = Seq.empty): Unit = current.replaceTableOp(w, schema, properties, partitionColumns, rows) /** Insert rows and record the insert. */ - def insertOp(w: WriteHandle, rows: Seq[Map[String, Any]]): Unit = current.insertOp(w, rows) + def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = + current.insertOp(w, rows) /** Delete rows matching `predicate` and record the delete. */ def deleteOp(w: WriteHandle, predicate: String): Unit = current.deleteOp(w, predicate) diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala index a8f5ef0f..76110a46 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala @@ -50,13 +50,8 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { test("insert_multiple") { val w = createTableOp("tbl", schema = "id INT, category STRING") - insertOp(w, Seq( - Map("id" -> 1, "category" -> "a"), - Map("id" -> 2, "category" -> "a"))) - insertOp(w, Seq( - Map("id" -> 3, "category" -> "b"), - Map("id" -> 4, "category" -> "b"), - Map("id" -> 5, "category" -> "b"))) + insertOp(w, (1 to 2).map(i => Map("id" -> i, "category" -> "a"))) + insertOp(w, (3 to 5).map(i => Map("id" -> i, "category" -> "b"))) insertOp(w, Seq(Map("id" -> 6, "category" -> "c"))) val t = registerWriteSpec(w) readSpec(t, name = "read_all") @@ -71,12 +66,8 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { val w = createTableOp("tbl", schema = "id INT, value STRING, amount INT", properties = Map("delta.enableDeletionVectors" -> "true")) - insertOp(w, Seq( - Map("id" -> 1, "value" -> "keep", "amount" -> 10), - Map("id" -> 2, "value" -> "remove", "amount" -> 20), - Map("id" -> 3, "value" -> "keep", "amount" -> 30), - Map("id" -> 4, "value" -> "remove", "amount" -> 40), - Map("id" -> 5, "value" -> "keep", "amount" -> 50))) + insertOp(w, (1 to 5).map(i => + Map("id" -> i, "value" -> (if (i % 2 == 0) "remove" else "keep"), "amount" -> i * 10))) deleteOp(w, predicate = "value = 'remove'") val t = registerWriteSpec(w) readSpec(t, name = "read_after_delete") @@ -122,14 +113,9 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { test("alter_set_properties") { val w = createTableOp("tbl", schema = "id INT, value INT") - insertOp(w, Seq( - Map("id" -> 1, "value" -> 100), - Map("id" -> 2, "value" -> 200), - Map("id" -> 3, "value" -> 300))) + insertOp(w, (1 to 3).map(i => Map("id" -> i, "value" -> i * 100))) setPropertiesOp(w, Map("delta.enableChangeDataFeed" -> "true")) - insertOp(w, Seq( - Map("id" -> 4, "value" -> 400), - Map("id" -> 5, "value" -> 500))) + insertOp(w, (4 to 5).map(i => Map("id" -> i, "value" -> i * 100))) updateOp(w, predicate = "id <= 2", set = Map("value" -> "value + 1000")) val t = registerWriteSpec(w) readSpec(t, version = 1, name = "read_initial_insert") From 9f896350e9695cbdb010a8f59abe0afc96d3c592 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 21:16:13 +0000 Subject: [PATCH 14/25] test: collapse verbose row literals into iterators (minimize LOC) Convert multi-line `Seq(Map(...))` insert/replace row lists in WriteBasicSuite and WriteSequencesSuite to compact range/zipWithIndex iterators. Generated data is byte-identical (verified: WORKLOAD_FORCE=true suites pass). Single-row inserts and the per-type rows in WriteTypesSuite are left explicit (no LOC win there). Net -28 lines. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../workload/tables/WriteBasicSuite.scala | 71 ++++++------------- .../workload/tables/WriteSequencesSuite.scala | 5 +- 2 files changed, 24 insertions(+), 52 deletions(-) diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala index 76110a46..d969570d 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteBasicSuite.scala @@ -23,10 +23,8 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { test("create_and_read") { val w = createTableOp("tbl", schema = "id INT, name STRING, score DOUBLE") - insertOp(w, Seq( - Map("id" -> 1, "name" -> "alice", "score" -> 95.5), - Map("id" -> 2, "name" -> "bob", "score" -> 87.3), - Map("id" -> 3, "name" -> "charlie", "score" -> 72.1))) + insertOp(w, Seq(("alice", 95.5), ("bob", 87.3), ("charlie", 72.1)).zipWithIndex.map { + case ((name, score), i) => Map("id" -> (i + 1), "name" -> name, "score" -> score) }) val t = registerWriteSpec(w) readSpec(t, name = "read_all") readSpec(t, predicate = "score > 90.0", name = "read_high_score") @@ -39,10 +37,8 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { properties = Map( "delta.enableDeletionVectors" -> "true", "delta.enableChangeDataFeed" -> "true")) - insertOp(w, Seq( - Map("id" -> 1, "value" -> "alpha", "amount" -> 100), - Map("id" -> 2, "value" -> "beta", "amount" -> 200), - Map("id" -> 3, "value" -> "gamma", "amount" -> 300))) + insertOp(w, Seq("alpha", "beta", "gamma").zipWithIndex.map { + case (v, i) => Map("id" -> (i + 1), "value" -> v, "amount" -> (i + 1) * 100) }) val t = registerWriteSpec(w) readSpec(t, name = "read_all") snapshotSpec(t) @@ -80,11 +76,8 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { val w = createTableOp("tbl", schema = "id INT, status STRING, count INT", properties = Map("delta.enableDeletionVectors" -> "true")) - insertOp(w, Seq( - Map("id" -> 1, "status" -> "pending", "count" -> 0), - Map("id" -> 2, "status" -> "pending", "count" -> 0), - Map("id" -> 3, "status" -> "active", "count" -> 5), - Map("id" -> 4, "status" -> "pending", "count" -> 0))) + insertOp(w, Seq(("pending", 0), ("pending", 0), ("active", 5), ("pending", 0)).zipWithIndex.map { + case ((status, count), i) => Map("id" -> (i + 1), "status" -> status, "count" -> count) }) updateOp(w, predicate = "status = 'pending'", set = Map("status" -> "'active'", "count" -> "count + 1")) val t = registerWriteSpec(w) @@ -96,13 +89,10 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { test("alter_add_column") { val w = createTableOp("tbl", schema = "id INT, name STRING") - insertOp(w, Seq( - Map("id" -> 1, "name" -> "alice"), - Map("id" -> 2, "name" -> "bob"))) + insertOp(w, Seq("alice", "bob").zipWithIndex.map { case (n, i) => Map("id" -> (i + 1), "name" -> n) }) addColumnsOp(w, "email STRING") - insertOp(w, Seq( - Map("id" -> 3, "name" -> "charlie", "email" -> "charlie@test.com"), - Map("id" -> 4, "name" -> "diana", "email" -> "diana@test.com"))) + insertOp(w, Seq("charlie", "diana").zipWithIndex.map { + case (n, i) => Map("id" -> (i + 3), "name" -> n, "email" -> s"$n@test.com") }) val t = registerWriteSpec(w) readSpec(t, name = "read_all") readSpec(t, version = 1, name = "read_before_alter") @@ -129,11 +119,8 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { val w = createTableOp("tbl", schema = "id INT, data STRING", properties = Map("delta.enableDeletionVectors" -> "true")) - insertOp(w, Seq( - Map("id" -> 1, "data" -> "first"), - Map("id" -> 2, "data" -> "second"), - Map("id" -> 3, "data" -> "third"), - Map("id" -> 4, "data" -> "fourth"))) + insertOp(w, Seq("first", "second", "third", "fourth").zipWithIndex.map { + case (d, i) => Map("id" -> (i + 1), "data" -> d) }) deleteOp(w, predicate = "true") val t = registerWriteSpec(w) readSpec(t, name = "read_after_delete_all") @@ -145,13 +132,10 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { val w = createTableOp("tbl", schema = "id INT, region STRING, revenue INT", partitionColumns = Seq("region")) - insertOp(w, Seq( - Map("id" -> 1, "region" -> "east", "revenue" -> 100), - Map("id" -> 2, "region" -> "east", "revenue" -> 150))) - insertOp(w, Seq( - Map("id" -> 3, "region" -> "west", "revenue" -> 200), - Map("id" -> 4, "region" -> "west", "revenue" -> 250), - Map("id" -> 5, "region" -> "west", "revenue" -> 300))) + insertOp(w, Seq(100, 150).zipWithIndex.map { + case (rev, i) => Map("id" -> (i + 1), "region" -> "east", "revenue" -> rev) }) + insertOp(w, Seq(200, 250, 300).zipWithIndex.map { + case (rev, i) => Map("id" -> (i + 3), "region" -> "west", "revenue" -> rev) }) insertOp(w, Seq(Map("id" -> 6, "region" -> "north", "revenue" -> 400))) val t = registerWriteSpec(w) readSpec(t, name = "read_all") @@ -164,15 +148,12 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { test("replace_table_as_select") { val w = createTableOp("tbl", schema = "id INT, name STRING") - insertOp(w, Seq( - Map("id" -> 1, "name" -> "alice"), - Map("id" -> 2, "name" -> "bob"))) + insertOp(w, Seq("alice", "bob").zipWithIndex.map { case (n, i) => Map("id" -> (i + 1), "name" -> n) }) // Replace-as-select: new schema + new data in a single commit. replaceTableOp(w, schema = "id INT, label STRING, score DOUBLE", - rows = Seq( - Map("id" -> 10, "label" -> "x", "score" -> 1.5), - Map("id" -> 20, "label" -> "y", "score" -> 2.5))) + rows = Seq(("x", 1.5), ("y", 2.5)).zipWithIndex.map { + case ((label, score), i) => Map("id" -> ((i + 1) * 10), "label" -> label, "score" -> score) }) val t = registerWriteSpec(w) readSpec(t, name = "read_all") readSpec(t, version = 1, name = "read_before_replace") @@ -199,10 +180,8 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { replaceTableOp(w, schema = "id INT, region STRING, revenue INT", partitionColumns = Seq("region"), - rows = Seq( - Map("id" -> 10, "region" -> "east", "revenue" -> 100), - Map("id" -> 20, "region" -> "west", "revenue" -> 200), - Map("id" -> 30, "region" -> "east", "revenue" -> 300))) + rows = Seq("east", "west", "east").zipWithIndex.map { + case (region, i) => Map("id" -> ((i + 1) * 10), "region" -> region, "revenue" -> ((i + 1) * 100)) }) val t = registerWriteSpec(w) readSpec(t, name = "read_all") readSpec(t, predicate = "region = 'east'", name = "read_east") @@ -250,13 +229,9 @@ class WriteBasicSuite extends WorkloadTestSuite("write_basic") { val w = createTableOp("tbl", schema = "id INT, name STRING, score INT", properties = Map("delta.enableDeletionVectors" -> "true")) - insertOp(w, Seq( - Map("id" -> 1, "name" -> "alice", "score" -> 90), - Map("id" -> 2, "name" -> "bob", "score" -> 75), - Map("id" -> 3, "name" -> "charlie", "score" -> 88), - Map("id" -> 4, "name" -> "diana", "score" -> 92), - Map("id" -> 5, "name" -> "eve", "score" -> 60), - Map("id" -> 6, "name" -> "frank", "score" -> 85))) + insertOp(w, Seq(("alice", 90), ("bob", 75), ("charlie", 88), ("diana", 92), ("eve", 60), + ("frank", 85)).zipWithIndex.map { + case ((name, score), i) => Map("id" -> (i + 1), "name" -> name, "score" -> score) }) deleteOp(w, predicate = "score < 80") val t = registerWriteSpec(w) readSpec(t, name = "read_after_delete") diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteSequencesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteSequencesSuite.scala index a5aa2023..3bb953d7 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteSequencesSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteSequencesSuite.scala @@ -28,10 +28,7 @@ class WriteSequencesSuite extends WorkloadTestSuite("write_sequences") { test("insert_update_delete_chain") { val w = createTableOp("tbl", schema = "id INT, status STRING, n INT", properties = Map("delta.enableDeletionVectors" -> "true")) - insertOp(w, Seq( - Map("id" -> 1, "status" -> "new", "n" -> 0), - Map("id" -> 2, "status" -> "new", "n" -> 0), - Map("id" -> 3, "status" -> "new", "n" -> 0))) + insertOp(w, (1 to 3).map(i => Map("id" -> i, "status" -> "new", "n" -> 0))) updateOp(w, predicate = "id <= 2", set = Map("status" -> "'active'", "n" -> "n + 1")) deleteOp(w, predicate = "id = 3") insertOp(w, Seq(Map("id" -> 4, "status" -> "new", "n" -> 5))) From 663958065f0df024bd839d92f1df5fbc8f953c16 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 21:48:17 +0000 Subject: [PATCH 15/25] fix: restore write-derived snapshot validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The write-derived snapshot branch had become a no-op (replay-or-throw only), so a whole class of spec validated nothing. Restore real validation: compare the replayed table's snapshot against the captured spec — protocol exact-equality (replay is Spark from the same commits, deterministic), declared-config keys, column-mapping-normalized schemaString, and partitionColumns/format. Table id and createdTime differ per table instance and are intentionally not compared. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/WorkloadValidator.scala | 113 +++++++++++++++++- 1 file changed, 109 insertions(+), 4 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index a77e3b1a..a17518e3 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -25,6 +25,8 @@ import scala.concurrent.duration.Duration import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} import org.apache.commons.io.FileUtils import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.types.{DataType, StructField, StructType} @@ -126,10 +128,9 @@ object WorkloadValidator { case "snapshot" => writePtr match { - case Some(ws) => - // Write-derived: protocol/metadata are Spark's deterministic output and were - // validated at capture time. Standalone, we only confirm the write spec replays. - replayOf(ws) + case Some(ws) => // write-derived: validate the replayed table's snapshot + validateSnapshotDerived(spark, replayOf(ws), specFile, + JsonUtil.readWriteSpec(specsDir.resolve(ws))) passed += 1 case None => validateReadOnly(SnapshotCapture.validateFromSpec(spark, deltaTable, specFile)) @@ -193,6 +194,110 @@ object WorkloadValidator { JsonUtil.assertMultisetsEqual(expected, actual, s"${writeName}_latest") } + /** + * Validate a write-derived snapshot spec against the table replayed from its write spec: the + * replay must reproduce the captured `protocol` and `metadata`. Protocol is compared exactly + * (replay is Spark from the same commits, so it is deterministic). For metadata we compare the + * fields that are reproducible across an independently-created table: `schemaString` (with + * per-field column-mapping ids stripped, since those are minted per table), `partitionColumns`, + * `format`, and the author-declared `configuration` keys (engine-injected defaults, table `id`, + * and `createdTime` differ between table instances and are not compared). + */ + private def validateSnapshotDerived( + spark: SparkSession, replayTablePath: Path, specFile: Path, writeSpec: WriteSpec): Unit = { + val spec = JsonUtil.readSnapshotSpec(specFile) + val name = specFile.getFileName.toString.stripSuffix(".json") + val (declaredKeys, removedKeys) = declaredConfiguration(writeSpec) + spec.expected.foreach { exp => + val log = DeltaHarness.get.openLog(spark, replayTablePath.toString) + val snapshot = (spec.version, spec.timestamp) match { + case (Some(v), _) => log.getSnapshotAt(v) + case _ => log.update() + } + val expProtocol = JsonUtil.mapper.valueToTree[JsonNode](exp.protocol) + val replayProtocol = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") + require(JsonUtil.canonicalJson(expProtocol) == JsonUtil.canonicalJson(replayProtocol), + s"snapshot '$name': protocol mismatch (expected ${JsonUtil.canonicalJson(expProtocol)}, " + + s"got ${JsonUtil.canonicalJson(replayProtocol)})") + val expectedMeta = JsonUtil.mapper.valueToTree[JsonNode](exp.metadata) + val replayMeta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") + configurationViolation(expectedMeta, replayMeta, declaredKeys, removedKeys).foreach { reason => + throw new IllegalStateException(s"snapshot '$name': metadata.configuration mismatch ($reason)") + } + require(normalizedSchema(expectedMeta) == normalizedSchema(replayMeta), + s"snapshot '$name': metadata.schemaString mismatch") + for (field <- Seq("partitionColumns", "format")) + require(canonicalField(expectedMeta, field) == canonicalField(replayMeta, field), + s"snapshot '$name': metadata.$field mismatch") + } + } + + /** The config keys the write spec's commits declared: (still-in-effect, removed). */ + private def declaredConfiguration(writeSpec: WriteSpec): (Set[String], Set[String]) = { + val declared = mutable.LinkedHashSet[String]() + val removed = mutable.LinkedHashSet[String]() + for (commit <- writeSpec.commits) commit match { + case c: CreateTableCommit => c.properties.foreach(declared ++= _.keys) + case c: ReplaceTableCommit => + declared.clear(); removed.clear() + c.properties.foreach(declared ++= _.keys) + case c: UpdatePropertiesCommit => + c.set.foreach { s => declared ++= s.keys; removed --= s.keys } + c.remove.foreach { r => removed ++= r; declared --= r } + case _ => () + } + (declared.toSet, removed.toSet) + } + + /** Each declared key present + equal on the replay; each removed key absent. First violation. */ + private def configurationViolation( + expectedMeta: JsonNode, replayMeta: JsonNode, + declaredKeys: Set[String], removedKeys: Set[String]): Option[String] = { + val expConfig = Option(expectedMeta.get("configuration")).filterNot(_.isNull) + val repConfig = Option(replayMeta.get("configuration")).filterNot(_.isNull) + def get(config: Option[JsonNode], key: String): Option[String] = + config.flatMap(c => Option(c.get(key))).filterNot(_.isNull).map(_.asText()) + declaredKeys.toSeq.sorted.collectFirst { + case key if get(repConfig, key) != get(expConfig, key) => + s"key '$key' expected ${get(expConfig, key)}, got ${get(repConfig, key)}" + }.orElse { + removedKeys.toSeq.sorted.collectFirst { + case key if get(repConfig, key).isDefined => s"removed key '$key' still present" + } + } + } + + /** + * Canonicalize `metadata.schemaString`, stripping per-field column-mapping identifiers + * (`delta.columnMapping.physicalName`/`.id`) which are minted per table and so are not a + * meaningful equality target across an independently-replayed table. + */ + private def normalizedSchema(meta: JsonNode): String = + Option(meta.get("schemaString")).filterNot(_.isNull) match { + case None => "null" + case Some(s) => + val tree = JsonUtil.mapper.readTree(s.asText()) + stripColumnMapping(tree) + JsonUtil.canonicalJson(tree) + } + + private def stripColumnMapping(node: JsonNode): Unit = node match { + case obj: ObjectNode => + obj.get("metadata") match { + case m: ObjectNode => + m.remove("delta.columnMapping.physicalName") + m.remove("delta.columnMapping.id") + case _ => () + } + obj.elements().asScala.foreach(stripColumnMapping) + case arr: ArrayNode => arr.elements().asScala.foreach(stripColumnMapping) + case _ => () + } + + private def canonicalField(node: JsonNode, field: String): String = + if (node.has(field) && !node.get(field).isNull) JsonUtil.canonicalJson(node.get(field)) + else "null" + // =========================================================================== // Per-commit replay // =========================================================================== From 000658368784e9570c5738cec4226152a9d13910 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 21:48:18 +0000 Subject: [PATCH 16/25] refactor: move row->Parquet materialization onto the DeltaHarness SPI Delete the standalone RowParquet object; add schemaAt / writeRows / writeRowsToTemp to the DeltaHarness trait (implemented in DeltaSparkHarness, with the value coercion kept private and unchanged). Rewire the capture-side callers (WorkloadContext, WriteSpecCapture) to the harness. Consolidates this Spark-only capture logic behind the SPI and removes a file. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../scala/io/delta/workload/RowParquet.scala | 142 ------------------ .../io/delta/workload/WorkloadContext.scala | 14 +- .../io/delta/workload/WriteSpecCapture.scala | 8 +- .../workload/deltaharness/DeltaHarness.scala | 26 ++++ .../deltaharness/DeltaSparkHarness.scala | 107 ++++++++++++- .../workload/tables/WriteTypesSuite.scala | 4 +- 6 files changed, 145 insertions(+), 156 deletions(-) delete mode 100644 workload-generator/src/main/scala/io/delta/workload/RowParquet.scala diff --git a/workload-generator/src/main/scala/io/delta/workload/RowParquet.scala b/workload-generator/src/main/scala/io/delta/workload/RowParquet.scala deleted file mode 100644 index 4f48596f..00000000 --- a/workload-generator/src/main/scala/io/delta/workload/RowParquet.scala +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload - -import java.nio.file.{Files, Path, StandardCopyOption} - -import scala.jdk.CollectionConverters._ - -import org.apache.commons.io.FileUtils -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.types._ - -import io.delta.workload.deltaharness.DeltaHarness - -/** - * Materializes in-memory `rows` (the workload-generator's authoring surface) into Parquet - * files for the write spec. The API takes rows; the spec stores Parquet. Capture writes rows - * here at generation time — no Delta-log scan — and replay reads the produced files back. - */ -object RowParquet { - - /** - * The table's schema at `version` (latest if None). When `includePartition` is false the - * partition columns are dropped, matching the layout of a raw data file referenced by a - * low-level `AddFile` (partition values ride on the action, not the file). - */ - def schemaAt( - spark: SparkSession, tablePath: String, - version: Option[Long], includePartition: Boolean): StructType = { - val log = DeltaHarness.get.openLog(spark, tablePath) - val snapshot = version.map(log.getSnapshotAt).getOrElse(log.update()) - val meta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") - val full = DataType.fromJson(meta.get("schemaString").asText()).asInstanceOf[StructType] - if (includePartition) full - else { - val partCols = Option(meta.get("partitionColumns")) - .map(_.elements().asScala.map(_.asText()).toSet).getOrElse(Set.empty[String]) - StructType(full.fields.filterNot(f => partCols.contains(f.name))) - } - } - - /** - * Write `rows` to a single Parquet file at `dest`. Spark writes a directory of part-files, so - * we write to a temp dir with `coalesce(1)` and move the one part-file into place. - */ - def writeSingle( - spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]], dest: Path): Unit = { - val sparkRows = rows.map { r => - Row.fromSeq(schema.fields.map(f => coerce(r.get(f.name), f.dataType)).toSeq) - } - val tmp = Files.createTempDirectory("row-parquet") - try { - spark.createDataFrame(sparkRows.asJava, schema) - .coalesce(1).write.mode("overwrite").parquet(tmp.toString) - val part = Files.list(tmp).iterator().asScala.find { p => - val n = p.getFileName.toString - n.endsWith(".parquet") && !n.startsWith(".") - }.getOrElse(throw new IllegalStateException(s"no parquet part-file written under $tmp")) - Files.createDirectories(dest.getParent) - Files.move(part, dest, StandardCopyOption.REPLACE_EXISTING) - } finally { - try FileUtils.deleteDirectory(tmp.toFile) catch { case _: Throwable => } - } - } - - /** Write `rows` to a single Parquet file in a fresh temp directory; returns the file path. */ - def writeTemp(spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]]): Path = { - val dir = Files.createTempDirectory("row-parquet-rtas") - val dest = dir.resolve("part-00000.parquet") - writeSingle(spark, schema, rows, dest) - dest - } - - /** - * Coerce a row's value to the column's Spark type. `None`/`null` -> SQL NULL. Unknown types - * fail loud rather than passing the raw value through: a silent passthrough that Spark happens - * to accept could write a wrong-but-symmetric value (wrong on both capture AND replay), which - * the row comparison would not catch — a false pass. - */ - private def coerce(value: Option[Any], dataType: DataType): Any = - value.flatMap(Option(_)) match { - case None => null - case Some(v) => dataType match { - case _: IntegerType => v.asInstanceOf[Number].intValue() - case _: LongType => v.asInstanceOf[Number].longValue() - case _: ShortType => v.asInstanceOf[Number].shortValue() - case _: ByteType => v.asInstanceOf[Number].byteValue() - case _: DoubleType => v.asInstanceOf[Number].doubleValue() - case _: FloatType => v.asInstanceOf[Number].floatValue() - case _: BooleanType => v.asInstanceOf[Boolean] - case _: StringType => v.toString - case _: DateType => v match { - case s: String => java.sql.Date.valueOf(s) - case d: java.sql.Date => d - case n: Number => new java.sql.Date(n.longValue()) - case _ => unsupported(v, dataType) - } - case _: TimestampType => v match { - case s: String => java.sql.Timestamp.valueOf(s) - case t: java.sql.Timestamp => t - case n: Number => new java.sql.Timestamp(n.longValue()) - case _ => unsupported(v, dataType) - } - case _: TimestampNTZType => v match { - case s: String => java.time.LocalDateTime.parse(s.replace(' ', 'T')) - case ldt: java.time.LocalDateTime => ldt - case _ => unsupported(v, dataType) - } - case _: DecimalType => v match { - case bd: java.math.BigDecimal => bd - case bd: scala.math.BigDecimal => bd.bigDecimal - case s: String => new java.math.BigDecimal(s) - case n: Number => new java.math.BigDecimal(n.toString) - case _ => unsupported(v, dataType) - } - case _: BinaryType => v match { - case b: Array[Byte] => b - case s: String => java.util.Base64.getDecoder.decode(s) - case _ => unsupported(v, dataType) - } - case _ => unsupported(v, dataType) - } - } - - private def unsupported(v: Any, dataType: DataType): Nothing = - throw new IllegalArgumentException( - s"RowParquet.coerce: unsupported value '$v' (${v.getClass.getName}) for column type $dataType") -} diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala index 2d82530e..70a62140 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala @@ -283,7 +283,7 @@ class WorkloadContext private[workload] ( if (rowSeq.nonEmpty) { // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so // the resulting schema matches what the bundled spec Parquet will reproduce on replay. - val parquet = RowParquet.writeTemp(spark, StructType.fromDDL(schema), rowSeq) + val parquet = DeltaHarness.get.writeRowsToTemp(spark, StructType.fromDDL(schema), rowSeq) try { sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} USING delta$partitionClause" + s"$propsClause AS SELECT * FROM parquet.`${parquet.toAbsolutePath}`") @@ -305,12 +305,12 @@ class WorkloadContext private[workload] ( def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = { val rowSeq = rows.iterator.toSeq require(rowSeq.nonEmpty, "insertOp requires at least one row") - // Drive the live insert from the SAME materialized Parquet the spec will bundle (via - // RowParquet), so the captured table and the replayed table use one value encoder and + // Drive the live insert from the SAME materialized Parquet the spec will bundle (via the + // harness), so the captured table and the replayed table use one value encoder and // cannot diverge by type. The spec's own copy is re-materialized in buildSpec. - val schema = RowParquet.schemaAt(spark, w.table.sourcePath.toString, + val schema = DeltaHarness.get.schemaAt(spark, w.table.sourcePath.toString, version = None, includePartition = true) - val parquet = RowParquet.writeTemp(spark, schema, rowSeq) + val parquet = DeltaHarness.get.writeRowsToTemp(spark, schema, rowSeq) try { sql(s"INSERT INTO ${w.table.tableName} SELECT * FROM parquet.`${parquet.toAbsolutePath}`") } finally { @@ -362,8 +362,8 @@ class WorkloadContext private[workload] ( val adds = addFiles.getOrElse(Seq.empty) // Materialize each add's logical rows to a temp Parquet for the live write. - val schema = RowParquet.schemaAt(spark, livePath, version = None, includePartition = true) - val tempParquet = adds.map(in => RowParquet.writeTemp(spark, schema, in.rows)) + val schema = DeltaHarness.get.schemaAt(spark, livePath, version = None, includePartition = true) + val tempParquet = adds.map(in => DeltaHarness.get.writeRowsToTemp(spark, schema, in.rows)) // A remove targets all files added at the referenced commit (== that table version). val removePaths = removeFiles.getOrElse(Seq.empty) .flatMap(k => SpecLayout.addPathsAt(w.table.sourcePath, k)) diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala index e4f3fb0f..5063fb40 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala @@ -181,10 +181,10 @@ class WriteSpecBuilder { // Materialize each low-level add's logical rows to the `dataFile` path recorded in the // action (full schema incl. partition columns, so replay's writeFiles can partition/map). lowLevelRows.get(idx).foreach { addRows => - val schema = RowParquet.schemaAt(spark, tablePath.toString, Some(idx.toLong), + val schema = DeltaHarness.get.schemaAt(spark, tablePath.toString, Some(idx.toLong), includePartition = true) ll.addFiles.getOrElse(Seq.empty).zip(addRows).foreach { case (af, rows) => - RowParquet.writeSingle(spark, schema, rows, outputDir.resolve(af.dataFile)) + DeltaHarness.get.writeRows(spark, schema, rows, outputDir.resolve(af.dataFile)) } } ll @@ -207,9 +207,9 @@ class WriteSpecBuilder { private def materializeRows( spark: SparkSession, tablePath: String, outputDir: Path, idx: Int): Option[Seq[String]] = { rowData.get(idx).filter(_.nonEmpty).map { rows => - val schema = RowParquet.schemaAt(spark, tablePath, Some(idx.toLong), includePartition = true) + val schema = DeltaHarness.get.schemaAt(spark, tablePath, Some(idx.toLong), includePartition = true) val rel = SpecLayout.commitDataFile(idx, "part-00000.parquet") - RowParquet.writeSingle(spark, schema, rows, outputDir.resolve(rel)) + DeltaHarness.get.writeRows(spark, schema, rows, outputDir.resolve(rel)) Seq(rel) } } diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala index 579f6d63..5a19eb42 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala @@ -16,7 +16,10 @@ package io.delta.workload.deltaharness +import java.nio.file.Path + import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.types.StructType import io.delta.workload.{AddDomainMetadata, AppTxn} @@ -86,6 +89,29 @@ trait DeltaHarness { def commitWithData( spark: SparkSession, tablePath: String, addDataParquet: Seq[String], req: CommitRequest): Seq[String] + + /** + * The table's schema at `version` (latest if None). When `includePartition` is false the + * partition columns are dropped, matching the layout of a raw data file referenced by a + * low-level `AddFile` (partition values ride on the action, not the file). + */ + def schemaAt( + spark: SparkSession, tablePath: String, + version: Option[Long], includePartition: Boolean): StructType + + /** + * Materialize in-memory `rows` (the workload-generator's authoring surface) into a single Parquet + * file at `dest`, coercing each value to the corresponding column type in `schema`. Capture writes + * rows here at generation time — no Delta-log scan — and replay reads the produced files back. + */ + def writeRows( + spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]], dest: Path): Unit + + /** + * Like [[writeRows]], but writes to a single Parquet file in a fresh temp directory and returns + * the produced file path. + */ + def writeRowsToTemp(spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]]): Path } trait LogView { diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala index b2e34fa1..0ded592a 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala @@ -16,11 +16,18 @@ package io.delta.workload.deltaharness +import java.nio.file.{Files, Path, StandardCopyOption} + import scala.collection.mutable +import scala.jdk.CollectionConverters._ -import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.apache.spark.sql.delta.{DeltaLog, DeltaOperations, Snapshot} import org.apache.spark.sql.delta.actions.{Action, AddFile, DomainMetadata, FileAction, SetTransaction} +import org.apache.spark.sql.types._ + +import io.delta.workload.JsonUtil class DeltaSparkHarness extends DeltaHarness { override def openLog(spark: SparkSession, tablePath: String): LogView = { @@ -76,6 +83,104 @@ class DeltaSparkHarness extends DeltaHarness { DeltaLog.clearCache() fileActions.collect { case a: AddFile => a.path } } + + override def schemaAt( + spark: SparkSession, tablePath: String, + version: Option[Long], includePartition: Boolean): StructType = { + val log = openLog(spark, tablePath) + val snapshot = version.map(log.getSnapshotAt).getOrElse(log.update()) + val meta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") + val full = DataType.fromJson(meta.get("schemaString").asText()).asInstanceOf[StructType] + if (includePartition) full + else { + val partCols = Option(meta.get("partitionColumns")) + .map(_.elements().asScala.map(_.asText()).toSet).getOrElse(Set.empty[String]) + StructType(full.fields.filterNot(f => partCols.contains(f.name))) + } + } + + override def writeRows( + spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]], dest: Path): Unit = { + val sparkRows = rows.map { r => + Row.fromSeq(schema.fields.map(f => coerce(r.get(f.name), f.dataType)).toSeq) + } + val tmp = Files.createTempDirectory("row-parquet") + try { + spark.createDataFrame(sparkRows.asJava, schema) + .coalesce(1).write.mode("overwrite").parquet(tmp.toString) + val part = Files.list(tmp).iterator().asScala.find { p => + val n = p.getFileName.toString + n.endsWith(".parquet") && !n.startsWith(".") + }.getOrElse(throw new IllegalStateException(s"no parquet part-file written under $tmp")) + Files.createDirectories(dest.getParent) + Files.move(part, dest, StandardCopyOption.REPLACE_EXISTING) + } finally { + try FileUtils.deleteDirectory(tmp.toFile) catch { case _: Throwable => } + } + } + + override def writeRowsToTemp( + spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]]): Path = { + val dir = Files.createTempDirectory("row-parquet-rtas") + val dest = dir.resolve("part-00000.parquet") + writeRows(spark, schema, rows, dest) + dest + } + + /** + * Coerce a row's value to the column's Spark type. `None`/`null` -> SQL NULL. Unknown types + * fail loud rather than passing the raw value through: a silent passthrough that Spark happens + * to accept could write a wrong-but-symmetric value (wrong on both capture AND replay), which + * the row comparison would not catch — a false pass. + */ + private def coerce(value: Option[Any], dataType: DataType): Any = + value.flatMap(Option(_)) match { + case None => null + case Some(v) => dataType match { + case _: IntegerType => v.asInstanceOf[Number].intValue() + case _: LongType => v.asInstanceOf[Number].longValue() + case _: ShortType => v.asInstanceOf[Number].shortValue() + case _: ByteType => v.asInstanceOf[Number].byteValue() + case _: DoubleType => v.asInstanceOf[Number].doubleValue() + case _: FloatType => v.asInstanceOf[Number].floatValue() + case _: BooleanType => v.asInstanceOf[Boolean] + case _: StringType => v.toString + case _: DateType => v match { + case s: String => java.sql.Date.valueOf(s) + case d: java.sql.Date => d + case n: Number => new java.sql.Date(n.longValue()) + case _ => unsupported(v, dataType) + } + case _: TimestampType => v match { + case s: String => java.sql.Timestamp.valueOf(s) + case t: java.sql.Timestamp => t + case n: Number => new java.sql.Timestamp(n.longValue()) + case _ => unsupported(v, dataType) + } + case _: TimestampNTZType => v match { + case s: String => java.time.LocalDateTime.parse(s.replace(' ', 'T')) + case ldt: java.time.LocalDateTime => ldt + case _ => unsupported(v, dataType) + } + case _: DecimalType => v match { + case bd: java.math.BigDecimal => bd + case bd: scala.math.BigDecimal => bd.bigDecimal + case s: String => new java.math.BigDecimal(s) + case n: Number => new java.math.BigDecimal(n.toString) + case _ => unsupported(v, dataType) + } + case _: BinaryType => v match { + case b: Array[Byte] => b + case s: String => java.util.Base64.getDecoder.decode(s) + case _ => unsupported(v, dataType) + } + case _ => unsupported(v, dataType) + } + } + + private def unsupported(v: Any, dataType: DataType): Nothing = + throw new IllegalArgumentException( + s"coerce: unsupported value '$v' (${v.getClass.getName}) for column type $dataType") } private class DeltaSparkLogView(inner: DeltaLog) extends LogView { diff --git a/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala b/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala index ddcb480f..f70a2662 100644 --- a/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/tables/WriteTypesSuite.scala @@ -19,7 +19,7 @@ package io.delta.workload.tables import io.delta.workload.WorkloadTestSuite /** - * Type-coverage write workloads: exercises `RowParquet.coerce` and the schema round-trip across + * Type-coverage write workloads: exercises the harness's row-value coercion and the schema round-trip across * every supported scalar type, nulls, and string edge cases. Each test captures then the * framework replays + re-validates, so a passing test proves the type survives capture->replay. */ @@ -108,7 +108,7 @@ class WriteTypesSuite extends WorkloadTestSuite("write_types") { test("nested_type_data_is_unsupported") { // Nested column TYPES round-trip in the schema, but nested DATA via the rows API is not - // supported — `RowParquet.coerce` fails loud rather than silently writing a wrong value. + // supported — the harness's row-value coercion fails loud rather than silently writing a wrong value. val w = createTableOp("tbl", schema = "id INT, s STRUCT") val ex = intercept[IllegalArgumentException] { insertOp(w, Seq(Map("id" -> 1, "s" -> Map("a" -> 1)))) From 3ea19ec70b96f64d4ac50eb7865f8896bdba5af6 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 21:48:19 +0000 Subject: [PATCH 17/25] style: revert TableInfoWriter reindent (whitespace-only churn) A prior cleanup reindented the whole write() body (~200 whitespace-only diff lines). Restore the original indentation, keeping only the real change (typed remove-count via CommitLog.read). Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/TableInfoWriter.scala | 216 +++++++++--------- 1 file changed, 108 insertions(+), 108 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala b/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala index 3f0f7ba8..66a3ef39 100644 --- a/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala +++ b/workload-generator/src/main/scala/io/delta/workload/TableInfoWriter.scala @@ -36,115 +36,115 @@ object TableInfoWriter { description: String, tags: Seq[String] = Seq.empty): Unit = { val log = DeltaHarness.get.openLog(spark, tablePath.toString) - val snapshot = log.update() - - val metadataJson = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") - val schemaObj = JsonUtil.mapper.readValue(metadataJson.get("schemaString").asText(), classOf[Any]) - - val protocolJson = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") - val minReaderV = protocolJson.get("minReaderVersion").asInt() - val minWriterV = protocolJson.get("minWriterVersion").asInt() - val readerFeatures = Option(protocolJson.get("readerFeatures")) - .filter(!_.isNull).map(_.elements().asScala.map(_.asText()).toSeq.sorted) - val writerFeatures = Option(protocolJson.get("writerFeatures")) - .filter(!_.isNull).map(_.elements().asScala.map(_.asText()).toSeq.sorted) - val protocol = ProtocolInfo( - minReaderVersion = minReaderV, - minWriterVersion = minWriterV, - readerFeatures = readerFeatures.filter(_.nonEmpty), - writerFeatures = writerFeatures.filter(_.nonEmpty)) - - val deltaLogDir = tablePath.resolve("_delta_log") - require(Files.isDirectory(deltaLogDir), s"Missing _delta_log at $tablePath") - - def listLog[T](fn: Iterator[Path] => T): T = { - val stream = Files.list(deltaLogDir) - try fn(stream.iterator().asScala) finally stream.close() - } - - val commitFiles = listLog(_.filter(p => - p.toString.endsWith(".json") && !p.toString.contains("checkpoint")).toSeq) - - val numCommits = commitFiles.size - - // Count actions and remove actions in a single pass over each commit, via the typed log. - val (numActions, numRemoveFiles) = commitFiles.foldLeft((0L, 0L)) { - case ((actions, removes), f) => - val version = f.getFileName.toString.stripSuffix(".json").toLong - val as = CommitLog.read(tablePath, version) - (actions + as.size, removes + as.count(_.isInstanceOf[RemoveFile])) - } - - val lastCrcVersion = listLog(_.map(_.getFileName.toString) - .filter(_.endsWith(".crc")) - .flatMap { n => - try Some(n.stripSuffix(".crc").toLong) - catch { case _: NumberFormatException => None } - }.toSeq.sorted.lastOption.getOrElse(-1L)) - - val (lastCheckpointVersion, numCheckpointFiles) = { - val lastCheckpointPath = deltaLogDir.resolve("_last_checkpoint") - if (Files.exists(lastCheckpointPath)) { - val cpNode = JsonUtil.mapper.readTree(Files.readAllBytes(lastCheckpointPath)) - val ver = cpNode.get("version").asLong() - val numParts = Option(cpNode.get("parts")).map(_.asInt()).getOrElse(1) - (ver, numParts) - } else { - val cpFiles = listLog(_.map(_.getFileName.toString) - .filter(n => n.contains("checkpoint") && n.endsWith(".parquet")).toSeq) - val maxVersion = cpFiles.flatMap { n => - try Some(n.split("\\.")(0).toLong) - catch { case _: NumberFormatException => None } - }.sorted.lastOption.getOrElse(-1L) - (maxVersion, cpFiles.count(_.startsWith(f"$maxVersion%020d"))) + val snapshot = log.update() + + val metadataJson = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") + val schemaObj = JsonUtil.mapper.readValue(metadataJson.get("schemaString").asText(), classOf[Any]) + + val protocolJson = JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol") + val minReaderV = protocolJson.get("minReaderVersion").asInt() + val minWriterV = protocolJson.get("minWriterVersion").asInt() + val readerFeatures = Option(protocolJson.get("readerFeatures")) + .filter(!_.isNull).map(_.elements().asScala.map(_.asText()).toSeq.sorted) + val writerFeatures = Option(protocolJson.get("writerFeatures")) + .filter(!_.isNull).map(_.elements().asScala.map(_.asText()).toSeq.sorted) + val protocol = ProtocolInfo( + minReaderVersion = minReaderV, + minWriterVersion = minWriterV, + readerFeatures = readerFeatures.filter(_.nonEmpty), + writerFeatures = writerFeatures.filter(_.nonEmpty)) + + val deltaLogDir = tablePath.resolve("_delta_log") + require(Files.isDirectory(deltaLogDir), s"Missing _delta_log at $tablePath") + + def listLog[T](fn: Iterator[Path] => T): T = { + val stream = Files.list(deltaLogDir) + try fn(stream.iterator().asScala) finally stream.close() + } + + val commitFiles = listLog(_.filter(p => + p.toString.endsWith(".json") && !p.toString.contains("checkpoint")).toSeq) + + val numCommits = commitFiles.size + + // Count actions and remove actions in a single pass over each commit, via the typed log. + val (numActions, numRemoveFiles) = commitFiles.foldLeft((0L, 0L)) { + case ((actions, removes), f) => + val version = f.getFileName.toString.stripSuffix(".json").toLong + val as = CommitLog.read(tablePath, version) + (actions + as.size, removes + as.count(_.isInstanceOf[RemoveFile])) } - } - - val numAddFiles = snapshot.allFiles.count() - val sizeInBytes = if (numAddFiles == 0) 0L - else snapshot.allFiles.selectExpr("sum(size)").first().getLong(0) - - val logInfo = LogInfo( - numAddFiles = numAddFiles, - numRemoveFiles = numRemoveFiles, - sizeInBytes = sizeInBytes, - numCommits = numCommits, - numActions = numActions, - lastCheckpointVersion = lastCheckpointVersion, - lastCrcVersion = lastCrcVersion, - numCheckpointFiles = numCheckpointFiles) - - val partCols = Option(metadataJson.get("partitionColumns")) - .filter(!_.isNull) - .map(_.elements().asScala.map(_.asText()).toSeq) - .getOrElse(Seq.empty) - val numDistinctPartitions = if (partCols.nonEmpty) { - // Convert MAP to JSON string for distinct() since Spark doesn't support set ops on MAP types - snapshot.allFiles.select(to_json(col("partitionValues"))).distinct().count() - } else 0L - - val dataLayout = DataLayoutInfo( - numClusteringColumns = 0, - numPartitionColumns = partCols.size, - numDistinctPartitions = numDistinctPartitions) - - val properties = Option(metadataJson.get("configuration")) - .filter(!_.isNull) - .map { node => - node.fields().asScala.map(e => e.getKey -> e.getValue.asText()).toMap + + val lastCrcVersion = listLog(_.map(_.getFileName.toString) + .filter(_.endsWith(".crc")) + .flatMap { n => + try Some(n.stripSuffix(".crc").toLong) + catch { case _: NumberFormatException => None } + }.toSeq.sorted.lastOption.getOrElse(-1L)) + + val (lastCheckpointVersion, numCheckpointFiles) = { + val lastCheckpointPath = deltaLogDir.resolve("_last_checkpoint") + if (Files.exists(lastCheckpointPath)) { + val cpNode = JsonUtil.mapper.readTree(Files.readAllBytes(lastCheckpointPath)) + val ver = cpNode.get("version").asLong() + val numParts = Option(cpNode.get("parts")).map(_.asInt()).getOrElse(1) + (ver, numParts) + } else { + val cpFiles = listLog(_.map(_.getFileName.toString) + .filter(n => n.contains("checkpoint") && n.endsWith(".parquet")).toSeq) + val maxVersion = cpFiles.flatMap { n => + try Some(n.split("\\.")(0).toLong) + catch { case _: NumberFormatException => None } + }.sorted.lastOption.getOrElse(-1L) + (maxVersion, cpFiles.count(_.startsWith(f"$maxVersion%020d"))) + } } - .getOrElse(Map.empty) - - val tableInfo = TableInfo( - name = name, - description = description, - schema = schemaObj, - protocol = protocol, - logInfo = logInfo, - properties = properties, - dataLayout = dataLayout, - tags = if (tags.nonEmpty) Some(tags) else None) - - JsonUtil.writeSpec(outputDir.resolve("table_info.json"), tableInfo) + + val numAddFiles = snapshot.allFiles.count() + val sizeInBytes = if (numAddFiles == 0) 0L + else snapshot.allFiles.selectExpr("sum(size)").first().getLong(0) + + val logInfo = LogInfo( + numAddFiles = numAddFiles, + numRemoveFiles = numRemoveFiles, + sizeInBytes = sizeInBytes, + numCommits = numCommits, + numActions = numActions, + lastCheckpointVersion = lastCheckpointVersion, + lastCrcVersion = lastCrcVersion, + numCheckpointFiles = numCheckpointFiles) + + val partCols = Option(metadataJson.get("partitionColumns")) + .filter(!_.isNull) + .map(_.elements().asScala.map(_.asText()).toSeq) + .getOrElse(Seq.empty) + val numDistinctPartitions = if (partCols.nonEmpty) { + // Convert MAP to JSON string for distinct() since Spark doesn't support set ops on MAP types + snapshot.allFiles.select(to_json(col("partitionValues"))).distinct().count() + } else 0L + + val dataLayout = DataLayoutInfo( + numClusteringColumns = 0, + numPartitionColumns = partCols.size, + numDistinctPartitions = numDistinctPartitions) + + val properties = Option(metadataJson.get("configuration")) + .filter(!_.isNull) + .map { node => + node.fields().asScala.map(e => e.getKey -> e.getValue.asText()).toMap + } + .getOrElse(Map.empty) + + val tableInfo = TableInfo( + name = name, + description = description, + schema = schemaObj, + protocol = protocol, + logInfo = logInfo, + properties = properties, + dataLayout = dataLayout, + tags = if (tags.nonEmpty) Some(tags) else None) + + JsonUtil.writeSpec(outputDir.resolve("table_info.json"), tableInfo) } } From 227e4ee4f5fbf49d28aea964bd65b5f27bbce113 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 22:04:38 +0000 Subject: [PATCH 18/25] refactor: collapse WriteSpecBuilder record* methods into one generic record() Replace the 11 record* wrappers with a single `record(commit, rows): Int` (append + stage rows + return ordinal); callers in WorkloadContext now build the typed ADT inline. Keep only the genuine converters (ddlToSchemaJson, addColumnsJson) and recordLowLevelRows. Also drop dead code in WorkloadTestSuite. Net ~-64 lines; the SET-XOR-UNSET invariant is preserved structurally and still enforced at replay. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/WorkloadContext.scala | 41 ++++--- .../io/delta/workload/WorkloadTestSuite.scala | 5 - .../io/delta/workload/WriteSpecCapture.scala | 114 ++++-------------- 3 files changed, 48 insertions(+), 112 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala index 70a62140..4f0a0a98 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala @@ -121,6 +121,9 @@ class WorkloadContext private[workload] ( /** Convert nullable java.lang.Long to Option[Long]. */ private def opt(v: java.lang.Long): Option[Long] = Option(v).map(_.longValue()) + /** Wrap a possibly-empty collection: Some(it) when non-empty, else None. */ + private def opt[C <: Iterable[_]](c: C): Option[C] = if (c.nonEmpty) Some(c) else None + // Per-table state: spec names must be unique within each table private val _tableSpecNames = mutable.HashMap[String, mutable.HashSet[String]]() @@ -261,7 +264,11 @@ class WorkloadContext private[workload] ( val propsClause = tblPropertiesClause(properties) sql(s"CREATE TABLE $tableName ($schema) USING delta$partitionClause$propsClause") val t = registerTable(tableName) - getWriteBuilder(t).recordCreateTable(schema, properties, partitionColumns) + val b = getWriteBuilder(t) + b.record(CreateTableCommit( + schema = b.ddlToSchemaJson(schema), + partitionColumns = opt(partitionColumns), + properties = opt(properties))) new WriteHandle(t) } @@ -294,7 +301,11 @@ class WorkloadContext private[workload] ( sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} ($schema) USING delta" + s"$partitionClause$propsClause") } - getWriteBuilder(w.table).recordReplaceTable(schema, properties, partitionColumns, rowSeq) + val b = getWriteBuilder(w.table) + b.record(ReplaceTableCommit( + schema = b.ddlToSchemaJson(schema), + partitionColumns = opt(partitionColumns), + properties = opt(properties)), rowSeq) } /** @@ -316,20 +327,20 @@ class WorkloadContext private[workload] ( } finally { FileUtils.deleteDirectory(parquet.getParent.toFile) } - getWriteBuilder(w.table).recordInsert(rowSeq) + getWriteBuilder(w.table).record(InsertCommit(), rowSeq) } /** Delete rows matching `predicate` and record the delete. */ def deleteOp(w: WriteHandle, predicate: String): Unit = { sql(s"DELETE FROM ${w.table.tableName} WHERE $predicate") - getWriteBuilder(w.table).recordDelete(predicate) + getWriteBuilder(w.table).record(DeleteCommit(predicate)) } /** Update rows matching `predicate` with `set` (column -> expression) and record it. */ def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = { val setClauses = set.map { case (k, v) => s"`$k` = $v" }.mkString(", ") sql(s"UPDATE ${w.table.tableName} SET $setClauses WHERE $predicate") - getWriteBuilder(w.table).recordUpdate(predicate, set) + getWriteBuilder(w.table).record(UpdateCommit(predicate, set)) } @@ -393,11 +404,12 @@ class WorkloadContext private[workload] ( val addActions = adds.zipWithIndex.map { case (in, i) => AddFileAction(dataFile = SpecLayout.commitDataFile(idx, s"add_$i.parquet"), dataChange = in.dataChange) } - builder.recordCommit( - schemaDDL = schemaDDL, tableProperties = tableProperties, txn = txn, - addFiles = if (addActions.nonEmpty) Some(addActions) else None, + builder.record(LowLevelCommitOp( + schema = schemaDDL.map(builder.ddlToSchemaJson), + tableProperties = tableProperties, txn = txn, + addFiles = opt(addActions), removeFiles = removeFiles.map(_.map(k => RemoveFileAction(k))), - addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata) + addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata)) builder.recordLowLevelRows(idx, adds.map(_.rows)) idx } @@ -406,13 +418,14 @@ class WorkloadContext private[workload] ( def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = { require(columnsDDL.nonEmpty, "addColumnsOp requires a non-empty DDL string") sql(s"ALTER TABLE ${w.table.tableName} ADD COLUMNS ($columnsDDL)") - getWriteBuilder(w.table).recordAddColumns(columnsDDL) + val b = getWriteBuilder(w.table) + b.record(EvolveSchemaCommit(addColumns = b.addColumnsJson(columnsDDL))) } /** Rename a column and record the schema evolution. */ def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = { sql(s"ALTER TABLE ${w.table.tableName} RENAME COLUMN $oldName TO $newName") - getWriteBuilder(w.table).recordRenameColumn(oldName, newName) + getWriteBuilder(w.table).record(EvolveSchemaCommit(renameColumns = Some(Map(oldName -> newName)))) } /** Drop columns and record the schema evolution. */ @@ -423,7 +436,7 @@ class WorkloadContext private[workload] ( } else { sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMNS (${columns.mkString(", ")})") } - getWriteBuilder(w.table).recordDropColumns(columns) + getWriteBuilder(w.table).record(EvolveSchemaCommit(dropColumns = opt(columns))) } /** Set table properties and record the update_properties operation. */ @@ -431,14 +444,14 @@ class WorkloadContext private[workload] ( require(props.nonEmpty, "setPropertiesOp requires at least one property") val setClause = props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") sql(s"ALTER TABLE ${w.table.tableName} SET TBLPROPERTIES ($setClause)") - getWriteBuilder(w.table).recordSetProperties(props) + getWriteBuilder(w.table).record(UpdatePropertiesCommit(set = Some(props))) } /** Unset table properties and record the update_properties operation. */ def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = { require(props.nonEmpty, "unsetPropertiesOp requires at least one property") sql(s"ALTER TABLE ${w.table.tableName} UNSET TBLPROPERTIES (${props.map(k => s"'$k'").mkString(", ")})") - getWriteBuilder(w.table).recordUnsetProperties(props) + getWriteBuilder(w.table).record(UpdatePropertiesCommit(remove = Some(props))) } // ---- Table mutations (applied to copied table before spec capture) ---- diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadTestSuite.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadTestSuite.scala index 1960960b..5337aa53 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadTestSuite.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadTestSuite.scala @@ -18,7 +18,6 @@ package io.delta.workload import java.nio.file.{Files, Path, Paths} -import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.scalatest.funsuite.AnyFunSuite @@ -178,8 +177,4 @@ abstract class WorkloadTestSuite(override val suiteName: String) } } } - - private def cleanupDir(dir: Path): Unit = { - if (Files.exists(dir)) FileUtils.deleteDirectory(dir.toFile) - } } diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala index 5063fb40..fc9ff8d8 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala @@ -47,105 +47,39 @@ class WriteSpecBuilder { // files, each its own row-set). `buildSpec` materializes each to `data/commit_/add_.parquet`. private val lowLevelRows = mutable.HashMap[Int, Seq[Seq[Map[String, Any]]]]() + /** + * Append a recorded commit and return its ordinal (== the table version it produces). For + * data-bearing high-level ops (insert / replace-as-select), pass the logical `rows` to + * materialize to Parquet in [[buildSpec]]. Low-level per-add rows go through + * [[recordLowLevelRows]] (a different `Seq[Seq[...]]` shape). + */ /** The ordinal (== table version) the next recorded commit will occupy. */ def nextOrdinal: Int = commits.size + def record(commit: WriteCommit, rows: Seq[Map[String, Any]] = Nil): Int = { + val idx = commits.size + if (rows.nonEmpty) rowData(idx) = rows + commits += commit + idx + } + /** Store a low-level commit's per-add rows for materialization in [[buildSpec]]. */ def recordLowLevelRows(idx: Int, addRows: Seq[Seq[Map[String, Any]]]): Unit = if (addRows.nonEmpty) lowLevelRows(idx) = addRows - // === High-level operations (replayed from parameters) === - - def recordCreateTable(schemaDDL: String, properties: Map[String, String], - partitionColumns: Seq[String]): Unit = { - commits += CreateTableCommit( - schema = ddlToSchemaJson(schemaDDL), - partitionColumns = if (partitionColumns.nonEmpty) Some(partitionColumns) else None, - properties = if (properties.nonEmpty) Some(properties) else None) - } - - def recordReplaceTable(schemaDDL: String, properties: Map[String, String], - partitionColumns: Seq[String], rows: Seq[Map[String, Any]]): Unit = { - if (rows.nonEmpty) rowData(commits.size) = rows - commits += ReplaceTableCommit( - schema = ddlToSchemaJson(schemaDDL), - partitionColumns = if (partitionColumns.nonEmpty) Some(partitionColumns) else None, - properties = if (properties.nonEmpty) Some(properties) else None) - } - - def recordInsert(rows: Seq[Map[String, Any]]): Unit = { - rowData(commits.size) = rows - commits += InsertCommit() - } - - def recordDelete(predicate: String): Unit = commits += DeleteCommit(predicate) - - def recordUpdate(predicate: String, set: Map[String, String]): Unit = - commits += UpdateCommit(predicate, set) - - def recordSetProperties(props: Map[String, String]): Unit = - recordUpdateProperties(props, Seq.empty) - - def recordUnsetProperties(props: Seq[String]): Unit = - recordUpdateProperties(Map.empty, props) - - def recordUpdateProperties(set: Map[String, String], unset: Seq[String]): Unit = { - // SQL replays SET and UNSET TBLPROPERTIES as separate ALTER statements, so a single commit - // cannot do both without producing two table versions and breaking commit index == version. - require(!(set.nonEmpty && unset.nonEmpty), - "an UpdatePropertiesCommit sets or removes properties, not both in one commit") - commits += UpdatePropertiesCommit( - set = if (set.nonEmpty) Some(set) else None, - remove = if (unset.nonEmpty) Some(unset) else None) - } - - def recordAddColumns(columnsDDL: String): Unit = - recordEvolveSchema(columnsDDL, Map.empty, Seq.empty) - - def recordRenameColumn(oldName: String, newName: String): Unit = - recordEvolveSchema("", Map(oldName -> newName), Seq.empty) - - def recordDropColumns(columns: Seq[String]): Unit = - recordEvolveSchema("", Map.empty, columns) - - def recordEvolveSchema( - addColumnsDDL: String, - renameColumns: Map[String, String], - dropColumns: Seq[String]): Unit = { - val addCols = if (addColumnsDDL.nonEmpty) { - val st = StructType.fromDDL(addColumnsDDL) - Some(st.fields.map { f => + /** Parse `ADD COLUMNS` DDL into the spec's `addColumns` JSON (name/type/nullable per field). */ + private[workload] def addColumnsJson(ddl: String): Option[Any] = + if (ddl.nonEmpty) { + Some(StructType.fromDDL(ddl).fields.map { f => val typeJson = JsonUtil.mapper.readValue(f.dataType.json, classOf[Any]) Map[String, Any]("name" -> f.name, "type" -> typeJson, "nullable" -> f.nullable) }.toSeq) } else None - commits += EvolveSchemaCommit( - addColumns = addCols, - renameColumns = if (renameColumns.nonEmpty) Some(renameColumns) else None, - dropColumns = if (dropColumns.nonEmpty) Some(dropColumns) else None) - } - - // === Low-level operation (recorded raw Delta actions) === - - /** Records a low-level commit's actions. `addFiles` are logical-Parquet pointers (materialized - * in [[buildSpec]]); `removeFiles` reference prior adds by commit ordinal. */ - def recordCommit( - schemaDDL: Option[String] = None, - tableProperties: Option[Map[String, String]] = None, - txn: Option[AppTxn] = None, - addFiles: Option[Seq[AddFileAction]] = None, - removeFiles: Option[Seq[RemoveFileAction]] = None, - addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, - removeDomainMetadata: Option[Seq[String]] = None): Unit = { - commits += LowLevelCommitOp( - schema = schemaDDL.map(ddlToSchemaJson), - tableProperties = tableProperties, - txn = txn, - addFiles = addFiles, - removeFiles = removeFiles, - addDomainMetadata = addDomainMetadata, - removeDomainMetadata = removeDomainMetadata) + /** Parse a SQL DDL string into Delta schema JSON using Spark's type parser. */ + private[workload] def ddlToSchemaJson(ddl: String): Any = { + val st = StructType.fromDDL(ddl) + JsonUtil.mapper.readValue(st.json, classOf[Any]) } // === Serialization === @@ -242,10 +176,4 @@ class WriteSpecBuilder { latestDir.resolve("table_version_metadata.json"), SnapshotExpected(protocol, metadata)) } - - /** Parse a SQL DDL string into Delta schema JSON using Spark's type parser. */ - private def ddlToSchemaJson(ddl: String): Any = { - val st = StructType.fromDDL(ddl) - JsonUtil.mapper.readValue(st.json, classOf[Any]) - } } From 4ff5b6ca4432f2c839bf1a7041b3b8b1baf33d81 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 22:24:08 +0000 Subject: [PATCH 19/25] refactor: keep the workload engine/DSL in one file (drop the god-file split) Fold WorkloadContext and WorkloadOps back into WorkloadGenerator.scala so PR #55 is the write-generator feature, not a file reorganization. Pure code movement (no behavior change); the split will land as its own follow-up PR. Cuts ~1k churn lines from the PR. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- .../io/delta/workload/WorkloadContext.scala | 607 --------------- .../io/delta/workload/WorkloadGenerator.scala | 715 +++++++++++++++++- .../scala/io/delta/workload/WorkloadOps.scala | 153 ---- 3 files changed, 713 insertions(+), 762 deletions(-) delete mode 100644 workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala delete mode 100644 workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala deleted file mode 100644 index 4f0a0a98..00000000 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadContext.scala +++ /dev/null @@ -1,607 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload - -import java.nio.file.{Files, Path, Paths} - -import scala.collection.{IterableOnce, mutable} - -import org.apache.commons.io.FileUtils -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.types.StructType - -import io.delta.workload.deltaharness.{CommitRemoveFile, CommitRequest, DeltaHarness} -import io.delta.workload.log.{Action, CommitLog} - -/** - * Handle to a declared spec. Attach assertions that are checked after capture. - * `T` is the typed spec class (e.g. [[ReadSpec]], [[SnapshotSpec]]). - */ -class SpecRef[T] private[workload] ( - private[workload] val config: HasAssertion[T]) { - - /** - * Assert that the captured spec is an error (has `expectedError`, no `expected`). - * Works for any spec type that follows the `expected`/`expectedError` convention. - */ - def assertError(): SpecRef[T] = { - config.assertion = Some { node => - require(node.has("expectedError") && !node.get("expectedError").isNull, - s"Expected an error spec but got a success result") - } - this - } - - /** Assert conditions on the captured spec using the typed case class. */ - def assert(check: T => Unit): SpecRef[T] = { - val deserialize = config.deserialize - config.assertion = Some { node => - check(deserialize(node)) - } - this - } -} - -/** - * Handle to a Delta table created via SQL. Used for declaring read specs - * (read, snapshot) and table mutations (mutateTable, modifyCommitActions). - */ -class TableHandle private[workload] ( - private[workload] val tableName: String, - private[workload] val sourcePath: Path, - private[workload] val ctx: WorkloadContext) { - - /** - * Get the commit timestamp for a specific version as a timestampAsOf-safe - * string (yyyy-MM-dd HH:mm:ss.SSS in session TZ). - * - * Mirrors what Delta's time-travel resolution actually keys off: - * - ICT-enabled tables: `commitInfo.inCommitTimestamp` in the commit JSON. - * - Non-ICT tables: the commit JSON file's mtime. - * - * DESCRIBE HISTORY and `commitInfo.timestamp` both look right at a glance - * but diverge by a few milliseconds from the file mtime on some engines. Feeding - * those back as `timestampAsOf` raises DELTA_TIMESTAMP_GREATER_THAN_COMMIT - * / DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION. - */ - def getTimestampForVersion(version: Long): String = { - val padded = f"$version%020d" - val commitPath = sourcePath.resolve(s"_delta_log/$padded.json") - require(Files.exists(commitPath), s"No commit file for version $version: $commitPath") - val ictOpt = scala.collection.JavaConverters - .asScalaBufferConverter(Files.readAllLines(commitPath)).asScala.iterator - .flatMap { line => - val node = JsonUtil.mapper.readTree(line) - Option(node.get("commitInfo")).flatMap(ci => Option(ci.get("inCommitTimestamp"))) - } - .find(_ => true) - .map(_.asLong()) - val tsMillis = ictOpt.getOrElse( - Files.getLastModifiedTime(commitPath).toMillis) - val sessionTz = ctx.spark.conf.get("spark.sql.session.timeZone", "UTC") - val fmt = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") - fmt.setTimeZone(java.util.TimeZone.getTimeZone(sessionTz)) - fmt.format(new java.util.Date(tsMillis)) - } -} - -/** - * Handle for structured write operations. Obtained from [[WorkloadContext.createTableOp]] - * (which creates the table and records the create) or [[WorkloadContext.writeSpec]] - * (which wraps an already-created [[TableHandle]]). - * - * Accepts write operations (insertOp, updateOp, deleteOp, ...) that both execute SQL - * and record the commit. Call [[WorkloadContext.registerWriteSpec]] to finalize and - * obtain a [[TableHandle]] for declaring read/snapshot specs. - */ -class WriteHandle private[workload] (private[workload] val table: TableHandle) - -class WorkloadContext private[workload] ( - val spark: SparkSession, - val workloadName: String, - private[workload] val tags: Seq[String] = Seq.empty) { - - private val _createdTables = mutable.ArrayBuffer[String]() - private[workload] val tableSpecs = mutable.ArrayBuffer[TableSpec]() - - /** Convert nullable java.lang.Long to Option[Long]. */ - private def opt(v: java.lang.Long): Option[Long] = Option(v).map(_.longValue()) - - /** Wrap a possibly-empty collection: Some(it) when non-empty, else None. */ - private def opt[C <: Iterable[_]](c: C): Option[C] = if (c.nonEmpty) Some(c) else None - - // Per-table state: spec names must be unique within each table - private val _tableSpecNames = mutable.HashMap[String, mutable.HashSet[String]]() - - // ---- SQL ---- - - private val _createTableRegex = - """(?i)(?:CREATE|REPLACE)\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?(\w+)`?""".r - - /** Execute SQL. Tracks CREATE TABLE statements for cleanup. */ - def sql(statement: String): Unit = { - _createTableRegex.findFirstMatchIn(statement).foreach { m => - _createdTables += m.group(1) - } - spark.sql(statement) - } - - // ---- Table handles ---- - - /** Register a managed Spark table for spec capture. */ - def registerTable(name: String): TableHandle = { - val path = resolveTablePath(name) - val handle = new TableHandle(name, path, this) - ensureTableSpec(handle) - handle - } - - /** Register a table at a filesystem path for spec capture. */ - def registerTableFromPath(path: String): TableHandle = { - val p = Paths.get(path) - require(Files.exists(p.resolve("_delta_log")), - s"No Delta table at path: $path") - val name = p.getFileName.toString - val handle = new TableHandle(name, p, this) - ensureTableSpec(handle) - handle - } - - // ---- Spec declaration ---- - - /** - * Read spec. Name auto-generated from parameters. - * - * If `expectError` is set, capture asserts that Spark MUST throw when the - * read is executed: - * - non-null + non-empty: the thrown error's code must match (after - * normalization). Capture fails if the operation succeeds or the code - * doesn't match. - * - non-null + empty string `""`: any error is accepted; capture fails - * only if the operation succeeds. - * - null (default): preserve legacy auto-detect behavior, recording - * whatever Spark does (success or error). - */ - def readSpec( - table: TableHandle, - predicate: String = null, - version: java.lang.Long = null, - timestamp: String = null, - columns: Seq[String] = null, - name: String = null, - expectError: String = null): SpecRef[ReadSpec] = { - val specName = if (name != null) name else autoReadName( - Option(predicate), opt(version), - Option(timestamp), Option(columns)) - requireUnique(table, specName) - val config = ReadSpecConfig( - specName, Option(predicate), opt(version), - Option(timestamp), Option(columns), Option(expectError)) - getTableSpec(table).readSpecs += config - new SpecRef(config) - } - - /** - * Snapshot spec. - * - * `expectError` has the same semantics as on [[readSpec]]: declaring it - * asserts that Spark must throw, and capture fails loudly otherwise. - */ - def snapshotSpec( - table: TableHandle, - version: java.lang.Long = null, - timestamp: String = null, - expectError: String = null): SpecRef[SnapshotSpec] = { - val config = SnapshotSpecConfig(opt(version), Option(timestamp), Option(expectError)) - getTableSpec(table).snapshotSpecs += config - new SpecRef(config) - } - - /** - * Force Spark to write a checkpoint file for the given SQL table name. - * Convenience wrapper around `DeltaHarness.openLog(...).checkpoint()`. - */ - def forceCheckpoint(tableName: String): Unit = { - val loc = spark.sql(s"DESCRIBE DETAIL `$tableName`").collect()(0).getAs[String]("location") - DeltaHarness.get.openLog(spark, loc).checkpoint() - } - - // ---- Write specs ---- - - /** ` PARTITIONED BY (a, b)` clause, or empty when there are no partition columns. */ - private def partitionedByClause(partitionColumns: Seq[String]): String = - if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" - - /** ` TBLPROPERTIES ('k' = 'v', ...)` clause, or empty when there are no properties. */ - private def tblPropertiesClause(properties: Map[String, String]): String = - if (properties.nonEmpty) { - s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" - } else "" - - private val _writeBuilders = mutable.HashMap[String, WriteSpecBuilder]() - - private def getWriteBuilder(table: TableHandle): WriteSpecBuilder = { - val key = s"${workloadName}_${table.tableName}" - val builder = _writeBuilders.getOrElseUpdate(key, new WriteSpecBuilder()) - getTableSpec(table).writeBuilder = Some(builder) - builder - } - - /** Wrap an already-created [[TableHandle]] for structured write operations. */ - def writeSpec(table: TableHandle): WriteHandle = { - getWriteBuilder(table) - new WriteHandle(table) - } - - /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ - def registerWriteSpec(w: WriteHandle): TableHandle = w.table - - /** - * Create a table via SQL and record the create_table operation. `schema` is a SQL - * DDL string (e.g. "id INT, name STRING NOT NULL"). Returns a [[WriteHandle]] for - * further write operations. - */ - def createTableOp( - tableName: String, - schema: String, - properties: Map[String, String] = Map.empty, - partitionColumns: Seq[String] = Seq.empty): WriteHandle = { - val partitionClause = partitionedByClause(partitionColumns) - val propsClause = tblPropertiesClause(properties) - sql(s"CREATE TABLE $tableName ($schema) USING delta$partitionClause$propsClause") - val t = registerTable(tableName) - val b = getWriteBuilder(t) - b.record(CreateTableCommit( - schema = b.ddlToSchemaJson(schema), - partitionColumns = opt(partitionColumns), - properties = opt(properties))) - new WriteHandle(t) - } - - /** - * Replace an existing table's schema/partitioning/properties and all data via - * `CREATE OR REPLACE TABLE`, and record the replace_table operation. `schema` is a SQL DDL - * string. With non-empty `rows` it is a replace-as-select (RTAS): a single commit that also - * writes the rows. Operates on the table behind `w`; subsequent ops see the new schema. - */ - def replaceTableOp( - w: WriteHandle, - schema: String, - properties: Map[String, String] = Map.empty, - partitionColumns: Seq[String] = Seq.empty, - rows: IterableOnce[Map[String, Any]] = Seq.empty): Unit = { - val rowSeq = rows.iterator.toSeq - val partitionClause = partitionedByClause(partitionColumns) - val propsClause = tblPropertiesClause(properties) - if (rowSeq.nonEmpty) { - // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so - // the resulting schema matches what the bundled spec Parquet will reproduce on replay. - val parquet = DeltaHarness.get.writeRowsToTemp(spark, StructType.fromDDL(schema), rowSeq) - try { - sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} USING delta$partitionClause" + - s"$propsClause AS SELECT * FROM parquet.`${parquet.toAbsolutePath}`") - } finally { - FileUtils.deleteDirectory(parquet.getParent.toFile) - } - } else { - sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} ($schema) USING delta" + - s"$partitionClause$propsClause") - } - val b = getWriteBuilder(w.table) - b.record(ReplaceTableCommit( - schema = b.ddlToSchemaJson(schema), - partitionColumns = opt(partitionColumns), - properties = opt(properties)), rowSeq) - } - - /** - * Insert `rows` (column -> value maps) and record the insert. An empty `rows` is a no-op: - * it produces no commit, so recording it would both desync the commit-index/version mapping - * and let the validator pass a spec with nothing to validate. - */ - def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = { - val rowSeq = rows.iterator.toSeq - require(rowSeq.nonEmpty, "insertOp requires at least one row") - // Drive the live insert from the SAME materialized Parquet the spec will bundle (via the - // harness), so the captured table and the replayed table use one value encoder and - // cannot diverge by type. The spec's own copy is re-materialized in buildSpec. - val schema = DeltaHarness.get.schemaAt(spark, w.table.sourcePath.toString, - version = None, includePartition = true) - val parquet = DeltaHarness.get.writeRowsToTemp(spark, schema, rowSeq) - try { - sql(s"INSERT INTO ${w.table.tableName} SELECT * FROM parquet.`${parquet.toAbsolutePath}`") - } finally { - FileUtils.deleteDirectory(parquet.getParent.toFile) - } - getWriteBuilder(w.table).record(InsertCommit(), rowSeq) - } - - /** Delete rows matching `predicate` and record the delete. */ - def deleteOp(w: WriteHandle, predicate: String): Unit = { - sql(s"DELETE FROM ${w.table.tableName} WHERE $predicate") - getWriteBuilder(w.table).record(DeleteCommit(predicate)) - } - - /** Update rows matching `predicate` with `set` (column -> expression) and record it. */ - def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = { - val setClauses = set.map { case (k, v) => s"`$k` = $v" }.mkString(", ") - sql(s"UPDATE ${w.table.tableName} SET $setClauses WHERE $predicate") - getWriteBuilder(w.table).record(UpdateCommit(predicate, set)) - } - - - /** - * Execute a low-level commit of raw Delta actions against the live table and record it. `addFiles` - * supply LOGICAL rows; the engine writes them (column-mapping/partition/stats handled) and the - * actions are bundled with the raw `txn`/`addDomainMetadata`/`removeDomainMetadata`/schema/property - * changes in one commit. `removeFiles` reference a prior low-level add by the ordinal that - * [[commitOp]] returned (the commit/version that produced it). Returns this commit's ordinal so a - * later `removeFiles` can target it. - */ - def commitOp( - w: WriteHandle, - schemaDDL: Option[String] = None, - tableProperties: Option[Map[String, String]] = None, - txn: Option[AppTxn] = None, - addFiles: Option[Seq[AddFileInput]] = None, - removeFiles: Option[Seq[Int]] = None, - addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, - removeDomainMetadata: Option[Seq[String]] = None): Int = { - require( - schemaDDL.isDefined || tableProperties.exists(_.nonEmpty) || txn.isDefined || - addFiles.exists(_.nonEmpty) || removeFiles.exists(_.nonEmpty) || - addDomainMetadata.exists(_.nonEmpty) || removeDomainMetadata.exists(_.nonEmpty), - "commitOp requires at least one action (schema/properties/txn/addFiles/removeFiles/" + - "domainMetadata); an empty commit would not advance the table version") - val builder = getWriteBuilder(w.table) - val idx = builder.nextOrdinal - val livePath = w.table.sourcePath.toString - val adds = addFiles.getOrElse(Seq.empty) - - // Materialize each add's logical rows to a temp Parquet for the live write. - val schema = DeltaHarness.get.schemaAt(spark, livePath, version = None, includePartition = true) - val tempParquet = adds.map(in => DeltaHarness.get.writeRowsToTemp(spark, schema, in.rows)) - // A remove targets all files added at the referenced commit (== that table version). - val removePaths = removeFiles.getOrElse(Seq.empty) - .flatMap(k => SpecLayout.addPathsAt(w.table.sourcePath, k)) - try { - DeltaHarness.get.commitWithData(spark, livePath, tempParquet.map(_.toAbsolutePath.toString), - CommitRequest( - schemaJson = schemaDDL.map(ddl => StructType.fromDDL(ddl).json), - properties = tableProperties, - setTransaction = txn, - removeFiles = removePaths.map(p => CommitRemoveFile(p, dataChange = true)), - addDomainMetadata = addDomainMetadata.getOrElse(Seq.empty), - removeDomainMetadata = removeDomainMetadata.getOrElse(Seq.empty))) - } finally { - tempParquet.foreach(p => FileUtils.deleteDirectory(p.getParent.toFile)) - } - - // Enforce the load-bearing commit-index == table-version invariant at the point it is - // established: ordinal-based remove resolution (SpecLayout.addPathsAt) keys off it. Fail here, - // at the offending op, rather than deferring to buildSpec's whole-spec count check. - val newVersion = DeltaHarness.get.openLog(spark, livePath).update().version - require(newVersion == idx, - s"commitOp produced version $newVersion but expected ordinal $idx; commit index can no " + - "longer be used as the table version (ordinal-based removes would mis-resolve)") - - // Record the spec: adds become logical-Parquet pointers (materialized to data/ in buildSpec); - // removes become ordinal references. - val addActions = adds.zipWithIndex.map { case (in, i) => - AddFileAction(dataFile = SpecLayout.commitDataFile(idx, s"add_$i.parquet"), dataChange = in.dataChange) - } - builder.record(LowLevelCommitOp( - schema = schemaDDL.map(builder.ddlToSchemaJson), - tableProperties = tableProperties, txn = txn, - addFiles = opt(addActions), - removeFiles = removeFiles.map(_.map(k => RemoveFileAction(k))), - addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata)) - builder.recordLowLevelRows(idx, adds.map(_.rows)) - idx - } - - /** Add columns (SQL DDL) and record the schema evolution. */ - def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = { - require(columnsDDL.nonEmpty, "addColumnsOp requires a non-empty DDL string") - sql(s"ALTER TABLE ${w.table.tableName} ADD COLUMNS ($columnsDDL)") - val b = getWriteBuilder(w.table) - b.record(EvolveSchemaCommit(addColumns = b.addColumnsJson(columnsDDL))) - } - - /** Rename a column and record the schema evolution. */ - def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = { - sql(s"ALTER TABLE ${w.table.tableName} RENAME COLUMN $oldName TO $newName") - getWriteBuilder(w.table).record(EvolveSchemaCommit(renameColumns = Some(Map(oldName -> newName)))) - } - - /** Drop columns and record the schema evolution. */ - def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = { - require(columns.nonEmpty, "dropColumnsOp requires at least one column") - if (columns.size == 1) { - sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMN ${columns.head}") - } else { - sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMNS (${columns.mkString(", ")})") - } - getWriteBuilder(w.table).record(EvolveSchemaCommit(dropColumns = opt(columns))) - } - - /** Set table properties and record the update_properties operation. */ - def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = { - require(props.nonEmpty, "setPropertiesOp requires at least one property") - val setClause = props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") - sql(s"ALTER TABLE ${w.table.tableName} SET TBLPROPERTIES ($setClause)") - getWriteBuilder(w.table).record(UpdatePropertiesCommit(set = Some(props))) - } - - /** Unset table properties and record the update_properties operation. */ - def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = { - require(props.nonEmpty, "unsetPropertiesOp requires at least one property") - sql(s"ALTER TABLE ${w.table.tableName} UNSET TBLPROPERTIES (${props.map(k => s"'$k'").mkString(", ")})") - getWriteBuilder(w.table).record(UpdatePropertiesCommit(remove = Some(props))) - } - - // ---- Table mutations (applied to copied table before spec capture) ---- - - /** Mutate the copied table's filesystem before specs are captured. */ - def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = { - getTableSpec(table).mutations += mutation - } - - /** - * Modify the typed [[Action]]s of a specific commit version: filter, reorder, or `.copy` fields. - * Unknown/malformed lines surface as [[io.delta.workload.log.RawAction]] and pass through - * unchanged. For byte-level corruption (truncation, garbage), use [[mutateTable]] directly. - */ - def modifyCommitActions(table: TableHandle, version: Long)( - modifier: Seq[Action] => Seq[Action]): Unit = - mutateTable(table) { tableDir => - if (Files.exists(CommitLog.commitFile(tableDir, version))) - CommitLog.mutate(tableDir, version)(modifier) - } - - // ---- Auto-naming ---- - - private def autoReadName( - predicate: Option[String], - version: Option[Long], - timestamp: Option[String], - columns: Option[Seq[String]]): String = { - val parts = mutable.ArrayBuffer[String]("read") - version.foreach(v => parts += s"v$v") - timestamp.foreach { ts => - parts += "ts_" + ts.replace(":", "-").replace(" ", "_").take(23) - } - predicate.foreach { p => - val simplified = p.toLowerCase - .replaceAll(">=", "_gte_").replaceAll("<=", "_lte_") - .replaceAll("<>", "_neq_").replaceAll("!=", "_neq_") - .replaceAll(">", "_gt_").replaceAll("<", "_lt_") - .replaceAll("=", "_eq_") - .replaceAll("\\s+and\\s+", "_and_").replaceAll("\\s+or\\s+", "_or_") - .replaceAll("\\s+is\\s+not\\s+null", "_is_not_null") - .replaceAll("\\s+is\\s+null", "_is_null") - .replaceAll("\\s+in\\s+", "_in_") - .replaceAll("\\s+", "_") - .replaceAll("[^a-z0-9_]", "") - .replaceAll("_+", "_").stripPrefix("_").stripSuffix("_") - .take(50) - parts += simplified - } - columns.foreach { cols => - parts += "cols_" + cols.take(3).mkString("_") - if (cols.size > 3) parts += s"plus${cols.size - 3}" - } - parts.mkString("_") - } - - // ---- Internal ---- - - private def ensureTableSpec(handle: TableHandle): Unit = { - val outputName = s"${workloadName}_${handle.tableName}" - if (!tableSpecs.exists(_.outputName == outputName)) { - tableSpecs += new TableSpec( - _outputName = outputName, - description = s"$workloadName — ${handle.tableName}", - tags = tags, - sourcePath = handle.sourcePath - ) - _tableSpecNames(outputName) = mutable.HashSet[String]() - } - } - - private def getTableSpec(handle: TableHandle): TableSpec = { - val outputName = s"${workloadName}_${handle.tableName}" - tableSpecs.find(_.outputName == outputName).getOrElse( - throw new RuntimeException(s"No table spec for ${handle.tableName}. Call registerTable() first.")) - } - - private def requireUnique(handle: TableHandle, specName: String): Unit = { - val outputName = s"${workloadName}_${handle.tableName}" - val names = _tableSpecNames.getOrElseUpdate(outputName, mutable.HashSet[String]()) - require(names.add(specName), - s"Duplicate spec name '$specName' for table '$outputName'") - } - - private def resolveTablePath(tableName: String): Path = { - try { - val detail = spark.sql(s"DESCRIBE DETAIL $tableName").collect() - require(detail.nonEmpty, s"DESCRIBE DETAIL returned no rows for '$tableName'") - val location = detail(0).getAs[String]("location") - require(location != null && location.nonEmpty, s"'$tableName' has no location") - val path = if (location.startsWith("file:")) { - Paths.get(new java.net.URI(location)) - } else Paths.get(location) - require(Files.exists(path.resolve("_delta_log")), - s"'$tableName' at $path has no _delta_log") - path - } catch { - case e: org.apache.spark.sql.AnalysisException => - throw new RuntimeException( - s"Table '$tableName' not found. Created tables: ${_createdTables.mkString(", ")}", e) - } - } - - private[workload] def cleanup(): Unit = { - val warehouseDir = spark.conf.get("spark.sql.warehouse.dir", "") - val errors = mutable.ArrayBuffer[String]() - - _createdTables.foreach { t => - // Get location before dropping (table may not exist in catalog, which is OK) - val location = try { - val tableId = spark.sessionState.catalog.getTableMetadata( - org.apache.spark.sql.catalyst.TableIdentifier(t)) - Option(tableId.location).map(_.toString) - } catch { - case _: org.apache.spark.sql.catalyst.analysis.NoSuchTableException => None - case _: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException => None - } - - // Drop from catalog - spark.sql(s"DROP TABLE IF EXISTS `$t`") - - // Delete directory: try catalog location, then fall back to warehouse/tableName - val pathsToDelete = location.toSeq.map { loc => - if (loc.startsWith("file:")) Paths.get(new java.net.URI(loc)) else Paths.get(loc) - } ++ (if (warehouseDir.nonEmpty) { - val base = if (warehouseDir.startsWith("file:")) - Paths.get(new java.net.URI(warehouseDir)) else Paths.get(warehouseDir) - Seq(base.resolve(t)) - } else Seq.empty) - - pathsToDelete.distinct.foreach { path => - if (Files.exists(path)) { - org.apache.commons.io.FileUtils.deleteDirectory(path.toFile) - } - } - } - } - -} - -object WorkloadContext { - private val _current = new scala.util.DynamicVariable[WorkloadContext](null) - - /** Get the current WorkloadContext. Throws if called outside a test body. */ - def current: WorkloadContext = { - val ctx = _current.value - require(ctx != null, "No active WorkloadContext. This method must be called inside a test body.") - ctx - } - - /** Execute a block with the given context as current. */ - def withContext[T](ctx: WorkloadContext)(body: => T): T = _current.withValue(ctx)(body) -} diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 4e70f6f1..5f92869c 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -16,12 +16,16 @@ package io.delta.workload -import java.nio.file.{Files, Path} +import java.nio.file.{Files, Path, Paths} -import scala.collection.mutable +import scala.collection.{IterableOnce, mutable} import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.types.StructType + +import io.delta.workload.deltaharness.{CommitRemoveFile, CommitRequest, DeltaHarness} +import io.delta.workload.log.{Action, CommitLog} /** * Internal workload generation engine. Use [[WorkloadTestSuite]] as the public API: @@ -208,3 +212,710 @@ private[workload] case class SnapshotSpecConfig( val deserialize = (n: com.fasterxml.jackson.databind.JsonNode) => JsonUtil.mapper.treeToValue(n, classOf[SnapshotSpec]) } + +/** + * Handle to a declared spec. Attach assertions that are checked after capture. + * `T` is the typed spec class (e.g. [[ReadSpec]], [[SnapshotSpec]]). + */ +class SpecRef[T] private[workload] ( + private[workload] val config: HasAssertion[T]) { + + /** + * Assert that the captured spec is an error (has `expectedError`, no `expected`). + * Works for any spec type that follows the `expected`/`expectedError` convention. + */ + def assertError(): SpecRef[T] = { + config.assertion = Some { node => + require(node.has("expectedError") && !node.get("expectedError").isNull, + s"Expected an error spec but got a success result") + } + this + } + + /** Assert conditions on the captured spec using the typed case class. */ + def assert(check: T => Unit): SpecRef[T] = { + val deserialize = config.deserialize + config.assertion = Some { node => + check(deserialize(node)) + } + this + } +} + +/** + * Handle to a Delta table created via SQL. Used for declaring read specs + * (read, snapshot) and table mutations (mutateTable, modifyCommitActions). + */ +class TableHandle private[workload] ( + private[workload] val tableName: String, + private[workload] val sourcePath: Path, + private[workload] val ctx: WorkloadContext) { + + /** + * Get the commit timestamp for a specific version as a timestampAsOf-safe + * string (yyyy-MM-dd HH:mm:ss.SSS in session TZ). + * + * Mirrors what Delta's time-travel resolution actually keys off: + * - ICT-enabled tables: `commitInfo.inCommitTimestamp` in the commit JSON. + * - Non-ICT tables: the commit JSON file's mtime. + * + * DESCRIBE HISTORY and `commitInfo.timestamp` both look right at a glance + * but diverge by a few milliseconds from the file mtime on some engines. Feeding + * those back as `timestampAsOf` raises DELTA_TIMESTAMP_GREATER_THAN_COMMIT + * / DELTA_TIMESTAMP_EARLIER_THAN_COMMIT_RETENTION. + */ + def getTimestampForVersion(version: Long): String = { + val padded = f"$version%020d" + val commitPath = sourcePath.resolve(s"_delta_log/$padded.json") + require(Files.exists(commitPath), s"No commit file for version $version: $commitPath") + val ictOpt = scala.collection.JavaConverters + .asScalaBufferConverter(Files.readAllLines(commitPath)).asScala.iterator + .flatMap { line => + val node = JsonUtil.mapper.readTree(line) + Option(node.get("commitInfo")).flatMap(ci => Option(ci.get("inCommitTimestamp"))) + } + .find(_ => true) + .map(_.asLong()) + val tsMillis = ictOpt.getOrElse( + Files.getLastModifiedTime(commitPath).toMillis) + val sessionTz = ctx.spark.conf.get("spark.sql.session.timeZone", "UTC") + val fmt = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + fmt.setTimeZone(java.util.TimeZone.getTimeZone(sessionTz)) + fmt.format(new java.util.Date(tsMillis)) + } +} + +/** + * Handle for structured write operations. Obtained from [[WorkloadContext.createTableOp]] + * (which creates the table and records the create) or [[WorkloadContext.writeSpec]] + * (which wraps an already-created [[TableHandle]]). + * + * Accepts write operations (insertOp, updateOp, deleteOp, ...) that both execute SQL + * and record the commit. Call [[WorkloadContext.registerWriteSpec]] to finalize and + * obtain a [[TableHandle]] for declaring read/snapshot specs. + */ +class WriteHandle private[workload] (private[workload] val table: TableHandle) + +class WorkloadContext private[workload] ( + val spark: SparkSession, + val workloadName: String, + private[workload] val tags: Seq[String] = Seq.empty) { + + private val _createdTables = mutable.ArrayBuffer[String]() + private[workload] val tableSpecs = mutable.ArrayBuffer[TableSpec]() + + /** Convert nullable java.lang.Long to Option[Long]. */ + private def opt(v: java.lang.Long): Option[Long] = Option(v).map(_.longValue()) + + /** Wrap a possibly-empty collection: Some(it) when non-empty, else None. */ + private def opt[C <: Iterable[_]](c: C): Option[C] = if (c.nonEmpty) Some(c) else None + + // Per-table state: spec names must be unique within each table + private val _tableSpecNames = mutable.HashMap[String, mutable.HashSet[String]]() + + // ---- SQL ---- + + private val _createTableRegex = + """(?i)(?:CREATE|REPLACE)\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?(\w+)`?""".r + + /** Execute SQL. Tracks CREATE TABLE statements for cleanup. */ + def sql(statement: String): Unit = { + _createTableRegex.findFirstMatchIn(statement).foreach { m => + _createdTables += m.group(1) + } + spark.sql(statement) + } + + // ---- Table handles ---- + + /** Register a managed Spark table for spec capture. */ + def registerTable(name: String): TableHandle = { + val path = resolveTablePath(name) + val handle = new TableHandle(name, path, this) + ensureTableSpec(handle) + handle + } + + /** Register a table at a filesystem path for spec capture. */ + def registerTableFromPath(path: String): TableHandle = { + val p = Paths.get(path) + require(Files.exists(p.resolve("_delta_log")), + s"No Delta table at path: $path") + val name = p.getFileName.toString + val handle = new TableHandle(name, p, this) + ensureTableSpec(handle) + handle + } + + // ---- Spec declaration ---- + + /** + * Read spec. Name auto-generated from parameters. + * + * If `expectError` is set, capture asserts that Spark MUST throw when the + * read is executed: + * - non-null + non-empty: the thrown error's code must match (after + * normalization). Capture fails if the operation succeeds or the code + * doesn't match. + * - non-null + empty string `""`: any error is accepted; capture fails + * only if the operation succeeds. + * - null (default): preserve legacy auto-detect behavior, recording + * whatever Spark does (success or error). + */ + def readSpec( + table: TableHandle, + predicate: String = null, + version: java.lang.Long = null, + timestamp: String = null, + columns: Seq[String] = null, + name: String = null, + expectError: String = null): SpecRef[ReadSpec] = { + val specName = if (name != null) name else autoReadName( + Option(predicate), opt(version), + Option(timestamp), Option(columns)) + requireUnique(table, specName) + val config = ReadSpecConfig( + specName, Option(predicate), opt(version), + Option(timestamp), Option(columns), Option(expectError)) + getTableSpec(table).readSpecs += config + new SpecRef(config) + } + + /** + * Snapshot spec. + * + * `expectError` has the same semantics as on [[readSpec]]: declaring it + * asserts that Spark must throw, and capture fails loudly otherwise. + */ + def snapshotSpec( + table: TableHandle, + version: java.lang.Long = null, + timestamp: String = null, + expectError: String = null): SpecRef[SnapshotSpec] = { + val config = SnapshotSpecConfig(opt(version), Option(timestamp), Option(expectError)) + getTableSpec(table).snapshotSpecs += config + new SpecRef(config) + } + + /** + * Force Spark to write a checkpoint file for the given SQL table name. + * Convenience wrapper around `DeltaHarness.openLog(...).checkpoint()`. + */ + def forceCheckpoint(tableName: String): Unit = { + val loc = spark.sql(s"DESCRIBE DETAIL `$tableName`").collect()(0).getAs[String]("location") + DeltaHarness.get.openLog(spark, loc).checkpoint() + } + + // ---- Write specs ---- + + /** ` PARTITIONED BY (a, b)` clause, or empty when there are no partition columns. */ + private def partitionedByClause(partitionColumns: Seq[String]): String = + if (partitionColumns.nonEmpty) s" PARTITIONED BY (${partitionColumns.mkString(", ")})" else "" + + /** ` TBLPROPERTIES ('k' = 'v', ...)` clause, or empty when there are no properties. */ + private def tblPropertiesClause(properties: Map[String, String]): String = + if (properties.nonEmpty) { + s" TBLPROPERTIES (${properties.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ")})" + } else "" + + private val _writeBuilders = mutable.HashMap[String, WriteSpecBuilder]() + + private def getWriteBuilder(table: TableHandle): WriteSpecBuilder = { + val key = s"${workloadName}_${table.tableName}" + val builder = _writeBuilders.getOrElseUpdate(key, new WriteSpecBuilder()) + getTableSpec(table).writeBuilder = Some(builder) + builder + } + + /** Wrap an already-created [[TableHandle]] for structured write operations. */ + def writeSpec(table: TableHandle): WriteHandle = { + getWriteBuilder(table) + new WriteHandle(table) + } + + /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ + def registerWriteSpec(w: WriteHandle): TableHandle = w.table + + /** + * Create a table via SQL and record the create_table operation. `schema` is a SQL + * DDL string (e.g. "id INT, name STRING NOT NULL"). Returns a [[WriteHandle]] for + * further write operations. + */ + def createTableOp( + tableName: String, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty): WriteHandle = { + val partitionClause = partitionedByClause(partitionColumns) + val propsClause = tblPropertiesClause(properties) + sql(s"CREATE TABLE $tableName ($schema) USING delta$partitionClause$propsClause") + val t = registerTable(tableName) + val b = getWriteBuilder(t) + b.record(CreateTableCommit( + schema = b.ddlToSchemaJson(schema), + partitionColumns = opt(partitionColumns), + properties = opt(properties))) + new WriteHandle(t) + } + + /** + * Replace an existing table's schema/partitioning/properties and all data via + * `CREATE OR REPLACE TABLE`, and record the replace_table operation. `schema` is a SQL DDL + * string. With non-empty `rows` it is a replace-as-select (RTAS): a single commit that also + * writes the rows. Operates on the table behind `w`; subsequent ops see the new schema. + */ + def replaceTableOp( + w: WriteHandle, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty, + rows: IterableOnce[Map[String, Any]] = Seq.empty): Unit = { + val rowSeq = rows.iterator.toSeq + val partitionClause = partitionedByClause(partitionColumns) + val propsClause = tblPropertiesClause(properties) + if (rowSeq.nonEmpty) { + // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so + // the resulting schema matches what the bundled spec Parquet will reproduce on replay. + val parquet = DeltaHarness.get.writeRowsToTemp(spark, StructType.fromDDL(schema), rowSeq) + try { + sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} USING delta$partitionClause" + + s"$propsClause AS SELECT * FROM parquet.`${parquet.toAbsolutePath}`") + } finally { + FileUtils.deleteDirectory(parquet.getParent.toFile) + } + } else { + sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} ($schema) USING delta" + + s"$partitionClause$propsClause") + } + val b = getWriteBuilder(w.table) + b.record(ReplaceTableCommit( + schema = b.ddlToSchemaJson(schema), + partitionColumns = opt(partitionColumns), + properties = opt(properties)), rowSeq) + } + + /** + * Insert `rows` (column -> value maps) and record the insert. An empty `rows` is a no-op: + * it produces no commit, so recording it would both desync the commit-index/version mapping + * and let the validator pass a spec with nothing to validate. + */ + def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = { + val rowSeq = rows.iterator.toSeq + require(rowSeq.nonEmpty, "insertOp requires at least one row") + // Drive the live insert from the SAME materialized Parquet the spec will bundle (via the + // harness), so the captured table and the replayed table use one value encoder and + // cannot diverge by type. The spec's own copy is re-materialized in buildSpec. + val schema = DeltaHarness.get.schemaAt(spark, w.table.sourcePath.toString, + version = None, includePartition = true) + val parquet = DeltaHarness.get.writeRowsToTemp(spark, schema, rowSeq) + try { + sql(s"INSERT INTO ${w.table.tableName} SELECT * FROM parquet.`${parquet.toAbsolutePath}`") + } finally { + FileUtils.deleteDirectory(parquet.getParent.toFile) + } + getWriteBuilder(w.table).record(InsertCommit(), rowSeq) + } + + /** Delete rows matching `predicate` and record the delete. */ + def deleteOp(w: WriteHandle, predicate: String): Unit = { + sql(s"DELETE FROM ${w.table.tableName} WHERE $predicate") + getWriteBuilder(w.table).record(DeleteCommit(predicate)) + } + + /** Update rows matching `predicate` with `set` (column -> expression) and record it. */ + def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = { + val setClauses = set.map { case (k, v) => s"`$k` = $v" }.mkString(", ") + sql(s"UPDATE ${w.table.tableName} SET $setClauses WHERE $predicate") + getWriteBuilder(w.table).record(UpdateCommit(predicate, set)) + } + + + /** + * Execute a low-level commit of raw Delta actions against the live table and record it. `addFiles` + * supply LOGICAL rows; the engine writes them (column-mapping/partition/stats handled) and the + * actions are bundled with the raw `txn`/`addDomainMetadata`/`removeDomainMetadata`/schema/property + * changes in one commit. `removeFiles` reference a prior low-level add by the ordinal that + * [[commitOp]] returned (the commit/version that produced it). Returns this commit's ordinal so a + * later `removeFiles` can target it. + */ + def commitOp( + w: WriteHandle, + schemaDDL: Option[String] = None, + tableProperties: Option[Map[String, String]] = None, + txn: Option[AppTxn] = None, + addFiles: Option[Seq[AddFileInput]] = None, + removeFiles: Option[Seq[Int]] = None, + addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, + removeDomainMetadata: Option[Seq[String]] = None): Int = { + require( + schemaDDL.isDefined || tableProperties.exists(_.nonEmpty) || txn.isDefined || + addFiles.exists(_.nonEmpty) || removeFiles.exists(_.nonEmpty) || + addDomainMetadata.exists(_.nonEmpty) || removeDomainMetadata.exists(_.nonEmpty), + "commitOp requires at least one action (schema/properties/txn/addFiles/removeFiles/" + + "domainMetadata); an empty commit would not advance the table version") + val builder = getWriteBuilder(w.table) + val idx = builder.nextOrdinal + val livePath = w.table.sourcePath.toString + val adds = addFiles.getOrElse(Seq.empty) + + // Materialize each add's logical rows to a temp Parquet for the live write. + val schema = DeltaHarness.get.schemaAt(spark, livePath, version = None, includePartition = true) + val tempParquet = adds.map(in => DeltaHarness.get.writeRowsToTemp(spark, schema, in.rows)) + // A remove targets all files added at the referenced commit (== that table version). + val removePaths = removeFiles.getOrElse(Seq.empty) + .flatMap(k => SpecLayout.addPathsAt(w.table.sourcePath, k)) + try { + DeltaHarness.get.commitWithData(spark, livePath, tempParquet.map(_.toAbsolutePath.toString), + CommitRequest( + schemaJson = schemaDDL.map(ddl => StructType.fromDDL(ddl).json), + properties = tableProperties, + setTransaction = txn, + removeFiles = removePaths.map(p => CommitRemoveFile(p, dataChange = true)), + addDomainMetadata = addDomainMetadata.getOrElse(Seq.empty), + removeDomainMetadata = removeDomainMetadata.getOrElse(Seq.empty))) + } finally { + tempParquet.foreach(p => FileUtils.deleteDirectory(p.getParent.toFile)) + } + + // Enforce the load-bearing commit-index == table-version invariant at the point it is + // established: ordinal-based remove resolution (SpecLayout.addPathsAt) keys off it. Fail here, + // at the offending op, rather than deferring to buildSpec's whole-spec count check. + val newVersion = DeltaHarness.get.openLog(spark, livePath).update().version + require(newVersion == idx, + s"commitOp produced version $newVersion but expected ordinal $idx; commit index can no " + + "longer be used as the table version (ordinal-based removes would mis-resolve)") + + // Record the spec: adds become logical-Parquet pointers (materialized to data/ in buildSpec); + // removes become ordinal references. + val addActions = adds.zipWithIndex.map { case (in, i) => + AddFileAction(dataFile = SpecLayout.commitDataFile(idx, s"add_$i.parquet"), dataChange = in.dataChange) + } + builder.record(LowLevelCommitOp( + schema = schemaDDL.map(builder.ddlToSchemaJson), + tableProperties = tableProperties, txn = txn, + addFiles = opt(addActions), + removeFiles = removeFiles.map(_.map(k => RemoveFileAction(k))), + addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata)) + builder.recordLowLevelRows(idx, adds.map(_.rows)) + idx + } + + /** Add columns (SQL DDL) and record the schema evolution. */ + def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = { + require(columnsDDL.nonEmpty, "addColumnsOp requires a non-empty DDL string") + sql(s"ALTER TABLE ${w.table.tableName} ADD COLUMNS ($columnsDDL)") + val b = getWriteBuilder(w.table) + b.record(EvolveSchemaCommit(addColumns = b.addColumnsJson(columnsDDL))) + } + + /** Rename a column and record the schema evolution. */ + def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = { + sql(s"ALTER TABLE ${w.table.tableName} RENAME COLUMN $oldName TO $newName") + getWriteBuilder(w.table).record(EvolveSchemaCommit(renameColumns = Some(Map(oldName -> newName)))) + } + + /** Drop columns and record the schema evolution. */ + def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = { + require(columns.nonEmpty, "dropColumnsOp requires at least one column") + if (columns.size == 1) { + sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMN ${columns.head}") + } else { + sql(s"ALTER TABLE ${w.table.tableName} DROP COLUMNS (${columns.mkString(", ")})") + } + getWriteBuilder(w.table).record(EvolveSchemaCommit(dropColumns = opt(columns))) + } + + /** Set table properties and record the update_properties operation. */ + def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = { + require(props.nonEmpty, "setPropertiesOp requires at least one property") + val setClause = props.map { case (k, v) => s"'$k' = '$v'" }.mkString(", ") + sql(s"ALTER TABLE ${w.table.tableName} SET TBLPROPERTIES ($setClause)") + getWriteBuilder(w.table).record(UpdatePropertiesCommit(set = Some(props))) + } + + /** Unset table properties and record the update_properties operation. */ + def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = { + require(props.nonEmpty, "unsetPropertiesOp requires at least one property") + sql(s"ALTER TABLE ${w.table.tableName} UNSET TBLPROPERTIES (${props.map(k => s"'$k'").mkString(", ")})") + getWriteBuilder(w.table).record(UpdatePropertiesCommit(remove = Some(props))) + } + + // ---- Table mutations (applied to copied table before spec capture) ---- + + /** Mutate the copied table's filesystem before specs are captured. */ + def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = { + getTableSpec(table).mutations += mutation + } + + /** + * Modify the typed [[Action]]s of a specific commit version: filter, reorder, or `.copy` fields. + * Unknown/malformed lines surface as [[io.delta.workload.log.RawAction]] and pass through + * unchanged. For byte-level corruption (truncation, garbage), use [[mutateTable]] directly. + */ + def modifyCommitActions(table: TableHandle, version: Long)( + modifier: Seq[Action] => Seq[Action]): Unit = + mutateTable(table) { tableDir => + if (Files.exists(CommitLog.commitFile(tableDir, version))) + CommitLog.mutate(tableDir, version)(modifier) + } + + // ---- Auto-naming ---- + + private def autoReadName( + predicate: Option[String], + version: Option[Long], + timestamp: Option[String], + columns: Option[Seq[String]]): String = { + val parts = mutable.ArrayBuffer[String]("read") + version.foreach(v => parts += s"v$v") + timestamp.foreach { ts => + parts += "ts_" + ts.replace(":", "-").replace(" ", "_").take(23) + } + predicate.foreach { p => + val simplified = p.toLowerCase + .replaceAll(">=", "_gte_").replaceAll("<=", "_lte_") + .replaceAll("<>", "_neq_").replaceAll("!=", "_neq_") + .replaceAll(">", "_gt_").replaceAll("<", "_lt_") + .replaceAll("=", "_eq_") + .replaceAll("\\s+and\\s+", "_and_").replaceAll("\\s+or\\s+", "_or_") + .replaceAll("\\s+is\\s+not\\s+null", "_is_not_null") + .replaceAll("\\s+is\\s+null", "_is_null") + .replaceAll("\\s+in\\s+", "_in_") + .replaceAll("\\s+", "_") + .replaceAll("[^a-z0-9_]", "") + .replaceAll("_+", "_").stripPrefix("_").stripSuffix("_") + .take(50) + parts += simplified + } + columns.foreach { cols => + parts += "cols_" + cols.take(3).mkString("_") + if (cols.size > 3) parts += s"plus${cols.size - 3}" + } + parts.mkString("_") + } + + // ---- Internal ---- + + private def ensureTableSpec(handle: TableHandle): Unit = { + val outputName = s"${workloadName}_${handle.tableName}" + if (!tableSpecs.exists(_.outputName == outputName)) { + tableSpecs += new TableSpec( + _outputName = outputName, + description = s"$workloadName — ${handle.tableName}", + tags = tags, + sourcePath = handle.sourcePath + ) + _tableSpecNames(outputName) = mutable.HashSet[String]() + } + } + + private def getTableSpec(handle: TableHandle): TableSpec = { + val outputName = s"${workloadName}_${handle.tableName}" + tableSpecs.find(_.outputName == outputName).getOrElse( + throw new RuntimeException(s"No table spec for ${handle.tableName}. Call registerTable() first.")) + } + + private def requireUnique(handle: TableHandle, specName: String): Unit = { + val outputName = s"${workloadName}_${handle.tableName}" + val names = _tableSpecNames.getOrElseUpdate(outputName, mutable.HashSet[String]()) + require(names.add(specName), + s"Duplicate spec name '$specName' for table '$outputName'") + } + + private def resolveTablePath(tableName: String): Path = { + try { + val detail = spark.sql(s"DESCRIBE DETAIL $tableName").collect() + require(detail.nonEmpty, s"DESCRIBE DETAIL returned no rows for '$tableName'") + val location = detail(0).getAs[String]("location") + require(location != null && location.nonEmpty, s"'$tableName' has no location") + val path = if (location.startsWith("file:")) { + Paths.get(new java.net.URI(location)) + } else Paths.get(location) + require(Files.exists(path.resolve("_delta_log")), + s"'$tableName' at $path has no _delta_log") + path + } catch { + case e: org.apache.spark.sql.AnalysisException => + throw new RuntimeException( + s"Table '$tableName' not found. Created tables: ${_createdTables.mkString(", ")}", e) + } + } + + private[workload] def cleanup(): Unit = { + val warehouseDir = spark.conf.get("spark.sql.warehouse.dir", "") + val errors = mutable.ArrayBuffer[String]() + + _createdTables.foreach { t => + // Get location before dropping (table may not exist in catalog, which is OK) + val location = try { + val tableId = spark.sessionState.catalog.getTableMetadata( + org.apache.spark.sql.catalyst.TableIdentifier(t)) + Option(tableId.location).map(_.toString) + } catch { + case _: org.apache.spark.sql.catalyst.analysis.NoSuchTableException => None + case _: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException => None + } + + // Drop from catalog + spark.sql(s"DROP TABLE IF EXISTS `$t`") + + // Delete directory: try catalog location, then fall back to warehouse/tableName + val pathsToDelete = location.toSeq.map { loc => + if (loc.startsWith("file:")) Paths.get(new java.net.URI(loc)) else Paths.get(loc) + } ++ (if (warehouseDir.nonEmpty) { + val base = if (warehouseDir.startsWith("file:")) + Paths.get(new java.net.URI(warehouseDir)) else Paths.get(warehouseDir) + Seq(base.resolve(t)) + } else Seq.empty) + + pathsToDelete.distinct.foreach { path => + if (Files.exists(path)) { + org.apache.commons.io.FileUtils.deleteDirectory(path.toFile) + } + } + } + } + +} + +object WorkloadContext { + private val _current = new scala.util.DynamicVariable[WorkloadContext](null) + + /** Get the current WorkloadContext. Throws if called outside a test body. */ + def current: WorkloadContext = { + val ctx = _current.value + require(ctx != null, "No active WorkloadContext. This method must be called inside a test body.") + ctx + } + + /** Execute a block with the given context as current. */ + def withContext[T](ctx: WorkloadContext)(body: => T): T = _current.withValue(ctx)(body) +} + +/** + * Provides DSL methods for workload generation. Mixed into WorkloadSuite + * so that test bodies can call methods like `sql()`, `registerTable()`, `readSpec()` + * directly without a context prefix. + */ +trait WorkloadOps { + import WorkloadContext.current + + /** The active SparkSession. */ + def spark: SparkSession = current.spark + + /** Execute SQL. Tracks CREATE TABLE statements for cleanup. */ + def sql(statement: String): Unit = current.sql(statement) + + /** Register a managed Spark table for spec capture. */ + def registerTable(name: String): TableHandle = current.registerTable(name) + + /** Register a table at a filesystem path for spec capture. */ + def registerTableFromPath(path: String): TableHandle = current.registerTableFromPath(path) + + /** + * Read spec. Name auto-generated from parameters. See + * [[WorkloadContext.readSpec]] for the semantics of `expectError`. + */ + def readSpec( + table: TableHandle, + predicate: String = null, + version: java.lang.Long = null, + timestamp: String = null, + columns: Seq[String] = null, + name: String = null, + expectError: String = null): SpecRef[ReadSpec] = + current.readSpec(table, predicate, version, timestamp, columns, name, expectError) + + /** + * Snapshot spec. See [[WorkloadContext.snapshotSpec]] for the semantics of + * `expectError`. + */ + def snapshotSpec( + table: TableHandle, + version: java.lang.Long = null, + timestamp: String = null, + expectError: String = null): SpecRef[SnapshotSpec] = + current.snapshotSpec(table, version, timestamp, expectError) + + /** Force Spark to write a checkpoint file for the given SQL table name. */ + def forceCheckpoint(tableName: String): Unit = current.forceCheckpoint(tableName) + + // ---- Write specs ---- + + /** Wrap an already-created [[TableHandle]] for structured write operations. */ + def writeSpec(table: TableHandle): WriteHandle = current.writeSpec(table) + + /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ + def registerWriteSpec(w: WriteHandle): TableHandle = current.registerWriteSpec(w) + + /** Create a table via SQL and record the create_table operation. */ + def createTableOp( + tableName: String, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty): WriteHandle = + current.createTableOp(tableName, schema, properties, partitionColumns) + + /** Replace an existing table (schema/partitioning/properties + data) and record it. */ + def replaceTableOp( + w: WriteHandle, + schema: String, + properties: Map[String, String] = Map.empty, + partitionColumns: Seq[String] = Seq.empty, + rows: IterableOnce[Map[String, Any]] = Seq.empty): Unit = + current.replaceTableOp(w, schema, properties, partitionColumns, rows) + + /** Insert rows and record the insert. */ + def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = + current.insertOp(w, rows) + + /** Delete rows matching `predicate` and record the delete. */ + def deleteOp(w: WriteHandle, predicate: String): Unit = current.deleteOp(w, predicate) + + /** Update rows matching `predicate` with `set` and record the update. */ + def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = + current.updateOp(w, predicate, set) + + + /** Execute a low-level commit of raw Delta actions and record it. Returns the commit ordinal. */ + def commitOp( + w: WriteHandle, + schemaDDL: Option[String] = None, + tableProperties: Option[Map[String, String]] = None, + txn: Option[AppTxn] = None, + addFiles: Option[Seq[AddFileInput]] = None, + removeFiles: Option[Seq[Int]] = None, + addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, + removeDomainMetadata: Option[Seq[String]] = None): Int = + current.commitOp(w, schemaDDL, tableProperties, txn, addFiles, removeFiles, + addDomainMetadata, removeDomainMetadata) + + /** Add columns (SQL DDL) and record the schema evolution. */ + def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = + current.addColumnsOp(w, columnsDDL) + + /** Rename a column and record the schema evolution. */ + def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = + current.renameColumnOp(w, oldName, newName) + + /** Drop columns and record the schema evolution. */ + def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = + current.dropColumnsOp(w, columns) + + /** Set table properties and record the update_properties operation. */ + def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = + current.setPropertiesOp(w, props) + + /** Unset table properties and record the update_properties operation. */ + def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = + current.unsetPropertiesOp(w, props) + + /** Mutate the copied table's filesystem before specs are captured. */ + def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = + current.mutateTable(table)(mutation) + + /** Modify the typed actions of a specific commit version. */ + def modifyCommitActions(table: TableHandle, version: Long)( + modifier: Seq[Action] => Seq[Action]): Unit = + current.modifyCommitActions(table, version)(modifier) +} diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala deleted file mode 100644 index e629fda1..00000000 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (2025) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.workload - -import java.nio.file.Path - -import scala.collection.IterableOnce - -import org.apache.spark.sql.SparkSession - -import io.delta.workload.log.Action - -/** - * Provides DSL methods for workload generation. Mixed into WorkloadSuite - * so that test bodies can call methods like `sql()`, `registerTable()`, `readSpec()` - * directly without a context prefix. - */ -trait WorkloadOps { - import WorkloadContext.current - - /** The active SparkSession. */ - def spark: SparkSession = current.spark - - /** Execute SQL. Tracks CREATE TABLE statements for cleanup. */ - def sql(statement: String): Unit = current.sql(statement) - - /** Register a managed Spark table for spec capture. */ - def registerTable(name: String): TableHandle = current.registerTable(name) - - /** Register a table at a filesystem path for spec capture. */ - def registerTableFromPath(path: String): TableHandle = current.registerTableFromPath(path) - - /** - * Read spec. Name auto-generated from parameters. See - * [[WorkloadContext.readSpec]] for the semantics of `expectError`. - */ - def readSpec( - table: TableHandle, - predicate: String = null, - version: java.lang.Long = null, - timestamp: String = null, - columns: Seq[String] = null, - name: String = null, - expectError: String = null): SpecRef[ReadSpec] = - current.readSpec(table, predicate, version, timestamp, columns, name, expectError) - - /** - * Snapshot spec. See [[WorkloadContext.snapshotSpec]] for the semantics of - * `expectError`. - */ - def snapshotSpec( - table: TableHandle, - version: java.lang.Long = null, - timestamp: String = null, - expectError: String = null): SpecRef[SnapshotSpec] = - current.snapshotSpec(table, version, timestamp, expectError) - - /** Force Spark to write a checkpoint file for the given SQL table name. */ - def forceCheckpoint(tableName: String): Unit = current.forceCheckpoint(tableName) - - // ---- Write specs ---- - - /** Wrap an already-created [[TableHandle]] for structured write operations. */ - def writeSpec(table: TableHandle): WriteHandle = current.writeSpec(table) - - /** Finalize write operations and return the [[TableHandle]] for read/snapshot specs. */ - def registerWriteSpec(w: WriteHandle): TableHandle = current.registerWriteSpec(w) - - /** Create a table via SQL and record the create_table operation. */ - def createTableOp( - tableName: String, - schema: String, - properties: Map[String, String] = Map.empty, - partitionColumns: Seq[String] = Seq.empty): WriteHandle = - current.createTableOp(tableName, schema, properties, partitionColumns) - - /** Replace an existing table (schema/partitioning/properties + data) and record it. */ - def replaceTableOp( - w: WriteHandle, - schema: String, - properties: Map[String, String] = Map.empty, - partitionColumns: Seq[String] = Seq.empty, - rows: IterableOnce[Map[String, Any]] = Seq.empty): Unit = - current.replaceTableOp(w, schema, properties, partitionColumns, rows) - - /** Insert rows and record the insert. */ - def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = - current.insertOp(w, rows) - - /** Delete rows matching `predicate` and record the delete. */ - def deleteOp(w: WriteHandle, predicate: String): Unit = current.deleteOp(w, predicate) - - /** Update rows matching `predicate` with `set` and record the update. */ - def updateOp(w: WriteHandle, predicate: String, set: Map[String, String]): Unit = - current.updateOp(w, predicate, set) - - - /** Execute a low-level commit of raw Delta actions and record it. Returns the commit ordinal. */ - def commitOp( - w: WriteHandle, - schemaDDL: Option[String] = None, - tableProperties: Option[Map[String, String]] = None, - txn: Option[AppTxn] = None, - addFiles: Option[Seq[AddFileInput]] = None, - removeFiles: Option[Seq[Int]] = None, - addDomainMetadata: Option[Seq[AddDomainMetadata]] = None, - removeDomainMetadata: Option[Seq[String]] = None): Int = - current.commitOp(w, schemaDDL, tableProperties, txn, addFiles, removeFiles, - addDomainMetadata, removeDomainMetadata) - - /** Add columns (SQL DDL) and record the schema evolution. */ - def addColumnsOp(w: WriteHandle, columnsDDL: String): Unit = - current.addColumnsOp(w, columnsDDL) - - /** Rename a column and record the schema evolution. */ - def renameColumnOp(w: WriteHandle, oldName: String, newName: String): Unit = - current.renameColumnOp(w, oldName, newName) - - /** Drop columns and record the schema evolution. */ - def dropColumnsOp(w: WriteHandle, columns: Seq[String]): Unit = - current.dropColumnsOp(w, columns) - - /** Set table properties and record the update_properties operation. */ - def setPropertiesOp(w: WriteHandle, props: Map[String, String]): Unit = - current.setPropertiesOp(w, props) - - /** Unset table properties and record the update_properties operation. */ - def unsetPropertiesOp(w: WriteHandle, props: Seq[String]): Unit = - current.unsetPropertiesOp(w, props) - - /** Mutate the copied table's filesystem before specs are captured. */ - def mutateTable(table: TableHandle)(mutation: Path => Unit): Unit = - current.mutateTable(table)(mutation) - - /** Modify the typed actions of a specific commit version. */ - def modifyCommitActions(table: TableHandle, version: Long)( - modifier: Seq[Action] => Seq[Action]): Unit = - current.modifyCommitActions(table, version)(modifier) -} From 2380f2249ebd7896fb06dbb8e2fc33ae9555ef5b Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 22:28:06 +0000 Subject: [PATCH 20/25] docs: trim the write-spec reference Delete the obsolete WriteCommit field glossary (per-op tables already cover it), merge the four low-level value-type tables into one, and dedup the examples (8 -> 4). Net ~-117 lines; content stays accurate to the ADT. Co-authored-by: Ryan Liao Co-authored-by: Isaac --- workload-generator/docs/spec-reference.md | 179 ++++------------------ 1 file changed, 31 insertions(+), 148 deletions(-) diff --git a/workload-generator/docs/spec-reference.md b/workload-generator/docs/spec-reference.md index 07406ca0..02b6628f 100644 --- a/workload-generator/docs/spec-reference.md +++ b/workload-generator/docs/spec-reference.md @@ -354,11 +354,9 @@ These are the raw Delta protocol and metadata JSON structures — not simplified **Type:** `"write"` -Tests Delta writer implementations by providing a sequence of write operations to replay. Unlike read specs (which verify reading an existing table), a write spec describes *how to construct* a table from scratch. +Tests Delta *writer* implementations: a portable, engine-agnostic recipe of `commits` to replay into a fresh table, after which the result is compared against `expected/_write/`. -A write spec is a portable, implementation-agnostic recipe: it specifies what operations to perform (create table, insert, delete, update, etc.) declaratively, so any conforming Delta writer can interpret and execute it. After replaying all commits, the resulting table is compared against the expected data under `expected/_write/`. - -A write spec is **just another spec**: it lives at `specs/_write.json` alongside the read and snapshot specs of the same workload, and is dispatched by its `type` field like any other. The read/snapshot specs of a write workload carry a `writeSpec` pointer back to it (see below), marking them *write-derived*. +It is **just another spec** — it lives at `specs/_write.json` and dispatches by `type`. The read/snapshot specs of the same workload carry a `writeSpec` pointer back to it, marking them *write-derived* (replayed from the write spec, then validated against it). ### Fields @@ -377,13 +375,13 @@ These map to SQL-like operations. The writer translates them to appropriate Delt | Operation | Fields | Description | |-----------|--------|-------------| -| `create_table` | `schema`, `partitionColumns?`, `properties?` | Create a new table with the given schema | +| `create_table` | `schema`, `partitionColumns?`, `properties?` | Create a new table with the given schema (`schema` is a Delta-JSON struct) | | `replace_table` | `schema`, `partitionColumns?`, `properties?`, `dataFiles?` | Replace the table's schema/partitioning/properties (and all data). With `dataFiles` it is a replace-as-select: a single commit that also writes the bundled data. | | `insert` | `dataFiles?` | Append the rows in the bundled Parquet data files | -| `update` | `predicate`, `set` | Update rows matching predicate | -| `delete` | `predicate` | Delete rows matching predicate | -| `evolve_schema` | `addColumns?`, `renameColumns?`, `dropColumns?` | Modify table schema | -| `update_properties` | `set?`, `remove?` | Modify table properties | +| `update` | `predicate`, `set` | Update rows matching `predicate`; `set` maps column → SQL expression | +| `delete` | `predicate` | Delete rows matching `predicate` (SQL WHERE clause) | +| `evolve_schema` | `addColumns?`, `renameColumns?`, `dropColumns?` | Modify table schema. `addColumns` entries are `{name, type, nullable}`; `renameColumns` maps old → new | +| `update_properties` | `set?`, `remove?` | Modify table properties (`set` map, `remove` names) | The data for `insert` and `replace_table` is bundled as Parquet under `data/commit_N/` (the generator's authoring API accepts in-memory rows, but the spec always stores Parquet). A @@ -399,63 +397,18 @@ enabled (a Delta/Spark constraint). |-----------|--------|-------------| | `commit` | `schema?`, `tableProperties?`, `txn?`, `addFiles?`, `removeFiles?`, `addDomainMetadata?`, `removeDomainMetadata?` | Bundle raw Delta actions in one commit. Each `addFiles[]` entry's logical rows are bundled as Parquet (`dataFile`); a consumer writes them **through its own engine write path** (so column mapping, partitioning, and stats are handled — physical names/stats are not in the spec) and commits the resulting `AddFile`s alongside the `txn`/`DomainMetadata`/schema/property changes. `removeFiles[]` tombstone a file added by a prior low-level `commit`, referenced by that commit's ordinal (`addedAtCommit`); since the engine assigns paths per table, the consumer resolves the ordinal to its own table's path. Deletion vectors are out of scope. | -### WriteCommit Fields - -`WriteCommit` is a sealed set of per-operation types discriminated by the leading `operation` -property (each operation carries only the fields listed for it in the tables above). The glossary -below describes each field; a field appears only on the operations that list it. Referenced value -types (`AppTxn`, `AddDomainMetadata`, `AddFileAction`, `RemoveFileAction`) are defined below. - -| Field | Type | Description | -|-------|------|-------------| -| `operation` | `string` | Operation type discriminator (see above) | -| `schema` | `object` | Table schema in Delta JSON format | -| `partitionColumns` | `string[]` | Partition column names | -| `properties` | `map` | Table properties (e.g., `delta.enableDeletionVectors`) | -| `dataFiles` | `string[]` | Relative paths under `data/` to the Parquet holding this op's rows (full rows incl. partition columns). For `insert` and `replace_table` | -| `predicate` | `string` | SQL WHERE clause for update/delete | -| `set` | `map` | Column assignments for update (column → expression) or properties to set | -| `remove` | `string[]` | Property names to remove | -| `addColumns` | `object[]` | Columns to add (each with `name`, `type`, `nullable`) | -| `renameColumns` | `map` | Column renames (old name → new name) | -| `dropColumns` | `string[]` | Column names to drop | -| `tableProperties` | `map` | Properties for low-level commit | -| `txn` | `AppTxn` | Application transaction for idempotent writes | -| `addFiles` | `AddFileAction[]` | Files to add (low-level) | -| `removeFiles` | `RemoveFileAction[]` | Files to remove (low-level) | -| `addDomainMetadata` | `AddDomainMetadata[]` | Domain metadata to add | -| `removeDomainMetadata` | `string[]` | Domain names to remove | - -### AddFileAction (Low-Level) - -Carries a pointer to a Parquet of LOGICAL rows; the consumer writes it through its own engine -write path (physical names, partitioning, and stats are derived per table, not stored). - -| Field | Type | Description | -|-------|------|-------------| -| `dataFile` | `string` | Relative path under `data/commit_N/` to the Parquet of logical rows (full rows incl. partition columns) | -| `dataChange` | `boolean?` | Whether this is a data change (default true) | - -### RemoveFileAction (Low-Level) - -| Field | Type | Description | -|-------|------|-------------| -| `addedAtCommit` | `int` | Ordinal (== table version) of the prior low-level `commit` that added the file(s) to tombstone; the consumer resolves it to its own table's path(s) | -| `dataChange` | `boolean?` | Whether this is a data change (default true) | - -### AppTxn (Low-Level) - -| Field | Type | Description | -|-------|------|-------------| -| `appId` | `string` | Application id for idempotent writes (`txn` action) | -| `version` | `long` | Monotonic transaction version for `appId` | +### Low-level value types -### AddDomainMetadata (Low-Level) +Value types referenced by the low-level `commit` operation. An `AddFileAction` carries a pointer to a Parquet of **logical** rows (full rows incl. partition columns); the consumer writes it through its own engine write path, so physical names, partitioning, and stats are derived per table, not stored. -| Field | Type | Description | -|-------|------|-------------| -| `domain` | `string` | Domain name | -| `configuration` | `string` | Domain configuration payload | +| Type | Field | Description | +|------|-------|-------------| +| `AddFileAction` | `dataFile` | Relative path under `data/commit_N/` to the Parquet of logical rows | +| `AddFileAction` | `dataChange?` | Whether this is a data change (default true) | +| `RemoveFileAction` | `addedAtCommit` | Ordinal (== table version) of the prior low-level `commit` that added the file(s) to tombstone; consumer resolves it to its own table's path(s) | +| `RemoveFileAction` | `dataChange?` | Whether this is a data change (default true) | +| `AppTxn` | `appId`, `version` | Application id and its monotonic transaction version for idempotent writes (`txn` action) | +| `AddDomainMetadata` | `domain`, `configuration` | Domain name and its configuration payload | ### Expected Data @@ -466,13 +419,12 @@ The final table state after replaying all commits is captured under `expected/= expected` and `minWriterVersion >= expected` and be a feature superset. A higher version or extra features is allowed. -- **`configuration`** — only the keys the write spec's own commits declared are checked: `create_table`/`replace_table` `properties` plus `update_properties.set`, minus `update_properties.remove` (a `replace_table` resets the declared set). Each declared key must be present and equal; each removed key must be absent. Engine-injected defaults are ignored. -- **rows** (`table_content/`) — multiset comparison; order is irrelevant, but each row must appear the correct number of times. - -`schemaString` should be compared with per-field column-mapping `physicalName`/`id` normalized out (those are minted per table); `partitionColumns` and `format` compared exactly. +- **rows** (`table_content/`) — multiset equality (order-independent). +- **protocol** — the replay must support at least the expected reader/writer versions and features (a stronger protocol is acceptable). +- **configuration** — only the keys the spec's own commits declared: `create_table`/`replace_table` `properties` + `update_properties.set` minus `.remove` (`replace_table` resets the set); each present and equal, removed keys absent. Engine-injected defaults are ignored. +- **schemaString** — equal with per-field column-mapping `physicalName`/`id` normalized out (minted per table); `partitionColumns` and `format` equal. ### Directory Structure @@ -498,7 +450,7 @@ The capture comes from one writer, but a write spec must validate against any co ### Examples -**Create table and insert:** +**High-level sequence (create, insert, delete, update):** ```json { @@ -510,44 +462,17 @@ The capture comes from one writer, but a write spec must validate against any co "type": "struct", "fields": [ { "name": "id", "type": "integer", "nullable": false, "metadata": {} }, - { "name": "name", "type": "string", "nullable": true, "metadata": {} } + { "name": "status", "type": "string", "nullable": true, "metadata": {} } ] }, "properties": { "delta.enableDeletionVectors": "true" } }, - { - "operation": "insert", - "dataFiles": ["data/commit_1/part-0000-abc.parquet"] - } - ] -} -``` - -**Delete with predicate:** - -```json -{ - "type": "write", - "commits": [ - { "operation": "create_table", "schema": { } }, - { "operation": "insert", "dataFiles": ["data/commit_1/part-0000-abc.parquet"] }, - { "operation": "delete", "predicate": "id > 100" } - ] -} -``` - -**Update with SET:** - -```json -{ - "type": "write", - "commits": [ - { "operation": "create_table", "schema": { } }, { "operation": "insert", "dataFiles": ["data/commit_1/part-0000-abc.parquet"] }, + { "operation": "delete", "predicate": "id > 100" }, { "operation": "update", "predicate": "status = 'pending'", - "set": { "status": "'active'", "count": "count + 1" } + "set": { "status": "'active'" } } ] } @@ -578,35 +503,6 @@ The capture comes from one writer, but a write spec must validate against any co } ``` -**Partitioned table:** - -```json -{ - "type": "write", - "commits": [ - { - "operation": "create_table", - "schema": { - "type": "struct", - "fields": [ - { "name": "id", "type": "integer", "nullable": true, "metadata": {} }, - { "name": "region", "type": "string", "nullable": true, "metadata": {} }, - { "name": "revenue", "type": "integer", "nullable": true, "metadata": {} } - ] - }, - "partitionColumns": ["region"] - }, - { - "operation": "insert", - "dataFiles": ["data/commit_1/part-00000.parquet"] - } - ] -} -``` - -The bundled Parquet contains the full rows including the partition columns, so a consumer -appends it and lets its own writer lay out the partition directories. - **Replace-as-select (replace schema + data in one commit):** ```json @@ -630,7 +526,7 @@ appends it and lets its own writer lay out the partition directories. } ``` -**Low-level commit with domain metadata:** +**Low-level commit (addFiles/removeFiles + txn + domainMetadata):** ```json { @@ -641,9 +537,12 @@ appends it and lets its own writer lay out the partition directories. "schema": { }, "properties": { "delta.feature.domainMetadata": "supported" } }, + { "operation": "commit", "addFiles": [ { "dataFile": "data/commit_1/add_0.parquet" } ] }, { "operation": "commit", - "addFiles": [ { "dataFile": "data/commit_1/add_0.parquet" } ], + "txn": { "appId": "streaming-job-1", "version": 42 }, + "addFiles": [ { "dataFile": "data/commit_2/add_0.parquet" } ], + "removeFiles": [ { "addedAtCommit": 1 } ], "addDomainMetadata": [ { "domain": "myApp.config", "configuration": "{\"version\": 1}" } ] @@ -652,22 +551,6 @@ appends it and lets its own writer lay out the partition directories. } ``` -**Low-level commit with application transaction:** - -```json -{ - "type": "write", - "commits": [ - { "operation": "create_table", "schema": { } }, - { - "operation": "commit", - "txn": { "appId": "streaming-job-1", "version": 42 }, - "addFiles": [ { "dataFile": "data/commit_1/add_0.parquet" } ] - } - ] -} -``` - --- ## table_info.json From 67327501ce60973e1cee5e7c8bfff0be8dff9370 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Wed, 17 Jun 2026 22:53:51 +0000 Subject: [PATCH 21/25] refactor: carry write-op rows with the commit instead of side maps Replace the two index-keyed side maps (rowData, lowLevelRows) joined to the commits buffer by ordinal with a single buffer of WriteOpWithData(commit, rows). Rows now travel with the commit; buildSpec reads them by destructuring instead of HashMap lookups. Drops recordLowLevelRows and the implicit "same int" join. Co-authored-by: Isaac --- .../io/delta/workload/WorkloadGenerator.scala | 8 +-- .../io/delta/workload/WriteSpecCapture.scala | 72 +++++++++---------- 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 5f92869c..e9317d88 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -491,7 +491,7 @@ class WorkloadContext private[workload] ( b.record(ReplaceTableCommit( schema = b.ddlToSchemaJson(schema), partitionColumns = opt(partitionColumns), - properties = opt(properties)), rowSeq) + properties = opt(properties)), Seq(rowSeq)) } /** @@ -513,7 +513,7 @@ class WorkloadContext private[workload] ( } finally { FileUtils.deleteDirectory(parquet.getParent.toFile) } - getWriteBuilder(w.table).record(InsertCommit(), rowSeq) + getWriteBuilder(w.table).record(InsertCommit(), Seq(rowSeq)) } /** Delete rows matching `predicate` and record the delete. */ @@ -595,8 +595,8 @@ class WorkloadContext private[workload] ( tableProperties = tableProperties, txn = txn, addFiles = opt(addActions), removeFiles = removeFiles.map(_.map(k => RemoveFileAction(k))), - addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata)) - builder.recordLowLevelRows(idx, adds.map(_.rows)) + addDomainMetadata = addDomainMetadata, removeDomainMetadata = removeDomainMetadata), + adds.map(_.rows)) idx } diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala index fc9ff8d8..8a3c6fdb 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala @@ -36,37 +36,30 @@ import io.delta.workload.deltaharness.DeltaHarness */ class WriteSpecBuilder { - private val commits = mutable.ArrayBuffer[WriteCommit]() + /** + * A recorded commit together with its capture-time logical rows. Rows are NOT part of the + * serialized [[WriteCommit]] (which carries only file paths); [[buildSpec]] materializes them to + * Parquet and writes the resulting paths back into the commit. `rows` holds one row-set per data + * file the commit adds: high-level insert / replace-as-select add a single file (one element); + * a low-level commit adds one row-set per `AddFile`. Data-free ops carry no rows. + */ + private case class WriteOpWithData(commit: WriteCommit, rows: Seq[Seq[Map[String, Any]]]) - // Rows for data-bearing high-level ops (insert / replace-as-select), keyed by commit index. - // The API takes rows; `buildSpec` materializes them to Parquet under `data/` and the spec - // serializes only the resulting file paths. - private val rowData = mutable.HashMap[Int, Seq[Map[String, Any]]]() + private val recorded = mutable.ArrayBuffer[WriteOpWithData]() - // Per-add rows for low-level commits, keyed by commit index (a low-level commit may add several - // files, each its own row-set). `buildSpec` materializes each to `data/commit_/add_.parquet`. - private val lowLevelRows = mutable.HashMap[Int, Seq[Seq[Map[String, Any]]]]() + /** The ordinal (== table version) the next recorded commit will occupy. */ + def nextOrdinal: Int = recorded.size /** - * Append a recorded commit and return its ordinal (== the table version it produces). For - * data-bearing high-level ops (insert / replace-as-select), pass the logical `rows` to - * materialize to Parquet in [[buildSpec]]. Low-level per-add rows go through - * [[recordLowLevelRows]] (a different `Seq[Seq[...]]` shape). + * Append a recorded commit and return its ordinal (== the table version it produces). Pass the + * commit's logical `rows` (one row-set per data file it adds) to materialize to Parquet in + * [[buildSpec]]; data-free ops pass nothing. */ - /** The ordinal (== table version) the next recorded commit will occupy. */ - def nextOrdinal: Int = commits.size - - def record(commit: WriteCommit, rows: Seq[Map[String, Any]] = Nil): Int = { - val idx = commits.size - if (rows.nonEmpty) rowData(idx) = rows - commits += commit - idx + def record(commit: WriteCommit, rows: Seq[Seq[Map[String, Any]]] = Nil): Int = { + recorded += WriteOpWithData(commit, rows) + recorded.size - 1 } - /** Store a low-level commit's per-add rows for materialization in [[buildSpec]]. */ - def recordLowLevelRows(idx: Int, addRows: Seq[Seq[Map[String, Any]]]): Unit = - if (addRows.nonEmpty) lowLevelRows(idx) = addRows - /** Parse `ADD COLUMNS` DDL into the spec's `addColumns` JSON (name/type/nullable per field). */ private[workload] def addColumnsJson(ddl: String): Option[Any] = if (ddl.nonEmpty) { @@ -98,31 +91,31 @@ class WriteSpecBuilder { // Commit index lines up with the table version it produced (commit 0 is the create at // version 0). Guard the coupling: each recorded op must map to exactly one version. val finalVersion = log.update().version - require(commits.size == finalVersion + 1, - s"write spec records ${commits.size} ops but table has ${finalVersion + 1} versions " + + require(recorded.size == finalVersion + 1, + s"write spec records ${recorded.size} ops but table has ${finalVersion + 1} versions " + "(0.." + finalVersion + "); op index can no longer be used as the table version") // Materialize the spec's data files. The API took `rows`; the spec stores Parquet. High-level // insert/replace-as-select rows are written to `data/commit_/part-00000.parquet` (full // schema incl. partition columns, so replay can Append/RTAS). Low-level commits copy each // add's already-written Parquet to `data/commit_/` (same in-table path). - val enriched = commits.zipWithIndex.map { - case (ins: InsertCommit, idx) => - ins.copy(dataFiles = materializeRows(spark, tablePath.toString, outputDir, idx)) - case (rep: ReplaceTableCommit, idx) => - rep.copy(dataFiles = materializeRows(spark, tablePath.toString, outputDir, idx)) - case (ll: LowLevelCommitOp, idx) => + val enriched = recorded.zipWithIndex.map { + case (WriteOpWithData(ins: InsertCommit, rows), idx) => + ins.copy(dataFiles = materializeRows(spark, tablePath.toString, outputDir, idx, rows)) + case (WriteOpWithData(rep: ReplaceTableCommit, rows), idx) => + rep.copy(dataFiles = materializeRows(spark, tablePath.toString, outputDir, idx, rows)) + case (WriteOpWithData(ll: LowLevelCommitOp, rows), idx) => // Materialize each low-level add's logical rows to the `dataFile` path recorded in the // action (full schema incl. partition columns, so replay's writeFiles can partition/map). - lowLevelRows.get(idx).foreach { addRows => + if (rows.nonEmpty) { val schema = DeltaHarness.get.schemaAt(spark, tablePath.toString, Some(idx.toLong), includePartition = true) - ll.addFiles.getOrElse(Seq.empty).zip(addRows).foreach { case (af, rows) => - DeltaHarness.get.writeRows(spark, schema, rows, outputDir.resolve(af.dataFile)) + ll.addFiles.getOrElse(Seq.empty).zip(rows).foreach { case (af, r) => + DeltaHarness.get.writeRows(spark, schema, r, outputDir.resolve(af.dataFile)) } } ll - case (other, _) => other + case (WriteOpWithData(other, _), _) => other } // `write` is just another spec: it lives in `specs/_write.json` alongside read/snapshot @@ -139,11 +132,12 @@ class WriteSpecBuilder { * recorded no rows. */ private def materializeRows( - spark: SparkSession, tablePath: String, outputDir: Path, idx: Int): Option[Seq[String]] = { - rowData.get(idx).filter(_.nonEmpty).map { rows => + spark: SparkSession, tablePath: String, outputDir: Path, idx: Int, + rows: Seq[Seq[Map[String, Any]]]): Option[Seq[String]] = { + rows.headOption.filter(_.nonEmpty).map { r => val schema = DeltaHarness.get.schemaAt(spark, tablePath, Some(idx.toLong), includePartition = true) val rel = SpecLayout.commitDataFile(idx, "part-00000.parquet") - DeltaHarness.get.writeRows(spark, schema, rows, outputDir.resolve(rel)) + DeltaHarness.get.writeRows(spark, schema, r, outputDir.resolve(rel)) Seq(rel) } } From eff054f291e6379471ce7b15ab6171dd4d1b141d Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Thu, 18 Jun 2026 03:05:36 +0000 Subject: [PATCH 22/25] refactor: fold addDataParquet into CommitRequest, collapse to one commit() The harness SPI had a dead no-data commit() and a commitWithData() taking data files as a separate param. Make data files part of CommitRequest (addDataParquet) and expose a single commit(spark, tablePath, req): Seq[String]. Also trim the AddFileInput doc to the two non-obvious facts. Co-authored-by: Isaac --- .../scala/io/delta/workload/JsonUtil.scala | 7 +----- .../io/delta/workload/WorkloadGenerator.scala | 3 ++- .../io/delta/workload/WorkloadValidator.scala | 7 +++--- .../workload/deltaharness/DeltaHarness.scala | 25 +++++++------------ .../deltaharness/DeltaSparkHarness.scala | 10 ++------ 5 files changed, 18 insertions(+), 34 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala b/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala index a018caae..1ec0a1ae 100644 --- a/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala +++ b/workload-generator/src/main/scala/io/delta/workload/JsonUtil.scala @@ -145,12 +145,7 @@ case class RemoveFileAction( addedAtCommit: Int, dataChange: Option[Boolean] = None) -/** - * Capture-time input for a low-level add: the caller supplies LOGICAL `rows` (matching the table's - * schema, including any partition columns). The framework materializes them and writes them - * through the engine; physical layout / stats are engine-derived. Not serialized — it only - * crosses the `commitOp` boundary. - */ +/** Logical `rows` for a low-level add (full schema incl. partition columns); engine-materialized, not serialized. */ case class AddFileInput( rows: Seq[Map[String, Any]], dataChange: Option[Boolean] = None) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index e9317d88..879682ae 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -565,8 +565,9 @@ class WorkloadContext private[workload] ( val removePaths = removeFiles.getOrElse(Seq.empty) .flatMap(k => SpecLayout.addPathsAt(w.table.sourcePath, k)) try { - DeltaHarness.get.commitWithData(spark, livePath, tempParquet.map(_.toAbsolutePath.toString), + DeltaHarness.get.commit(spark, livePath, CommitRequest( + addDataParquet = tempParquet.map(_.toAbsolutePath.toString), schemaJson = schemaDDL.map(ddl => StructType.fromDDL(ddl).json), properties = tableProperties, setTransaction = txn, diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index a17518e3..de6e29ac 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -393,8 +393,8 @@ object WorkloadValidator { /** * Replay a low-level commit: write each add's bundled logical Parquet through the engine - * (`commitWithData` → column-mapping/partition/stats handled), and tombstone each remove's - * referenced prior add, resolved to this replay table's own paths at that commit/version. + * (via `req.addDataParquet`, so column-mapping/partition/stats are engine-handled), and tombstone + * each remove's referenced prior add, resolved to this replay table's own paths at that version. */ private def replayLowLevelCommit( spark: SparkSession, commit: LowLevelCommitOp, idx: Int, @@ -403,8 +403,9 @@ object WorkloadValidator { .map(af => testDir.resolve(af.dataFile).toAbsolutePath.toString) val removePaths = commit.removeFiles.getOrElse(Seq.empty) .flatMap(rf => SpecLayout.addPathsAt(java.nio.file.Paths.get(tablePath), rf.addedAtCommit)) - DeltaHarness.get.commitWithData(spark, tablePath, addDataParquet, + DeltaHarness.get.commit(spark, tablePath, CommitRequest( + addDataParquet = addDataParquet, schemaJson = commit.schema.map(s => structOf(s).json), properties = commit.tableProperties, setTransaction = commit.txn, diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala index 5a19eb42..1e9a8c72 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala @@ -32,11 +32,11 @@ import io.delta.workload.{AddDomainMetadata, AppTxn} case class CommitRemoveFile(path: String, dataChange: Boolean) /** - * A platform-neutral description of a low-level commit's NON-data actions. Carries only plain - * Scala types and the generator's neutral action case classes — never `org.apache.spark.sql.delta.*`. - * Data files are added separately, via [[DeltaHarness.commitWithData]], so the engine computes - * their physical layout / stats (column-mapping aware). + * A platform-neutral description of a low-level commit. Carries only plain Scala types and the + * generator's neutral action case classes, never `org.apache.spark.sql.delta.*`. * + * @param addDataParquet Parquet files of LOGICAL rows to add; the engine reads each and writes + * it into the table (column-mapping aware, computing stats/layout) * @param schemaJson `StructType.json` to set as `metadata.schemaString`, or None to keep * @param properties properties merged into `metadata.configuration`, or None to keep * @param setTransaction optional `SetTransaction` (appId/version) action @@ -45,6 +45,7 @@ case class CommitRemoveFile(path: String, dataChange: Boolean) * @param removeDomainMetadata domain names to tombstone */ case class CommitRequest( + addDataParquet: Seq[String] = Nil, schemaJson: Option[String] = None, properties: Option[Map[String, String]] = None, setTransaction: Option[AppTxn] = None, @@ -70,25 +71,17 @@ trait DeltaHarness { /** * Open a transaction at `tablePath`, apply the metadata update (schema/properties) if present, - * and commit the described actions using the platform's `DeltaOperations.ManualUpdate`. + * write any `req.addDataParquet` data files through the engine's own write path (honoring column + * mapping and partitioning, computing stats), and commit the produced AddFile actions together + * with `req`'s other actions in one `DeltaOperations.ManualUpdate`. * * Implementations MUST clear any internal DeltaLog cache before and after committing, mirroring * the `openLog` cache-clearing contract so consumers always see a fresh view. - */ - def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Unit - - /** - * Like [[commit]], but also writes data files through the engine's own write path. Each path in - * `addDataParquet` points to a Parquet file of LOGICAL rows; the implementation reads it and - * writes it into the table via the engine (honoring column mapping and partitioning, computing - * stats), then commits the produced AddFile actions together with `req`'s actions in one commit. * * @return the in-table `AddFile.path`s the engine produced (in `addDataParquet` order; a file * may yield several adds when partitioned), so callers can reference them later. */ - def commitWithData( - spark: SparkSession, tablePath: String, - addDataParquet: Seq[String], req: CommitRequest): Seq[String] + def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Seq[String] /** * The table's schema at `version` (latest if None). When `includePartition` is false the diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala index 0ded592a..2149e33f 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala @@ -35,13 +35,7 @@ class DeltaSparkHarness extends DeltaHarness { new DeltaSparkLogView(DeltaLog.forTable(spark, tablePath)) } - override def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Unit = { - commitWithData(spark, tablePath, Seq.empty, req) - } - - override def commitWithData( - spark: SparkSession, tablePath: String, - addDataParquet: Seq[String], req: CommitRequest): Seq[String] = { + override def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Seq[String] = { DeltaLog.clearCache() val deltaLog = DeltaLog.forTable(spark, tablePath) val txn = deltaLog.startTransaction() @@ -58,7 +52,7 @@ class DeltaSparkHarness extends DeltaHarness { // Engine-written data files: column mapping / partitioning / stats all handled by writeFiles. // Commit ALL file actions it returns (AddFile + AddCDCFile when change data feed is enabled); // only the AddFile paths are returned (for ordinal-based remove resolution). - val fileActions: Seq[FileAction] = addDataParquet.flatMap { p => + val fileActions: Seq[FileAction] = req.addDataParquet.flatMap { p => txn.writeFiles(spark.read.parquet(p)) } From 580175836a20a0d793bc05d9df8efb7556ab1a27 Mon Sep 17 00:00:00 2001 From: Oussama Saoudi Date: Fri, 19 Jun 2026 04:35:46 +0000 Subject: [PATCH 23/25] refactor: move writeRowsToTemp off the SPI, drop schemaAt includePartition, trim writeExpectedLatest - writeRowsToTemp was pure sugar over writeRows; move it into WorkloadContext (its only callers) so the DeltaHarness SPI exposes just the writeRows primitive. - schemaAt's includePartition param + its false branch were dead (every caller passed true); remove. - writeExpectedLatest wrote a table_version_metadata.json that nothing reads; keep only table_content. Co-authored-by: Isaac --- .../io/delta/workload/WorkloadGenerator.scala | 18 ++++++++----- .../io/delta/workload/WriteSpecCapture.scala | 27 +++++-------------- .../workload/deltaharness/DeltaHarness.scala | 16 ++--------- .../deltaharness/DeltaSparkHarness.scala | 19 ++----------- 4 files changed, 22 insertions(+), 58 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 879682ae..ce2ab9a3 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -476,7 +476,7 @@ class WorkloadContext private[workload] ( if (rowSeq.nonEmpty) { // Single-commit replace-as-select: write the rows to a temp Parquet and RTAS from it, so // the resulting schema matches what the bundled spec Parquet will reproduce on replay. - val parquet = DeltaHarness.get.writeRowsToTemp(spark, StructType.fromDDL(schema), rowSeq) + val parquet = writeRowsToTemp(StructType.fromDDL(schema), rowSeq) try { sql(s"CREATE OR REPLACE TABLE ${w.table.tableName} USING delta$partitionClause" + s"$propsClause AS SELECT * FROM parquet.`${parquet.toAbsolutePath}`") @@ -494,6 +494,13 @@ class WorkloadContext private[workload] ( properties = opt(properties)), Seq(rowSeq)) } + /** Write `rows` to a single Parquet file in a fresh temp dir; the caller deletes it after use. */ + private def writeRowsToTemp(schema: StructType, rows: Seq[Map[String, Any]]): Path = { + val dest = Files.createTempDirectory("row-parquet").resolve("part-00000.parquet") + DeltaHarness.get.writeRows(spark, schema, rows, dest) + dest + } + /** * Insert `rows` (column -> value maps) and record the insert. An empty `rows` is a no-op: * it produces no commit, so recording it would both desync the commit-index/version mapping @@ -505,9 +512,8 @@ class WorkloadContext private[workload] ( // Drive the live insert from the SAME materialized Parquet the spec will bundle (via the // harness), so the captured table and the replayed table use one value encoder and // cannot diverge by type. The spec's own copy is re-materialized in buildSpec. - val schema = DeltaHarness.get.schemaAt(spark, w.table.sourcePath.toString, - version = None, includePartition = true) - val parquet = DeltaHarness.get.writeRowsToTemp(spark, schema, rowSeq) + val schema = DeltaHarness.get.schemaAt(spark, w.table.sourcePath.toString, version = None) + val parquet = writeRowsToTemp(schema, rowSeq) try { sql(s"INSERT INTO ${w.table.tableName} SELECT * FROM parquet.`${parquet.toAbsolutePath}`") } finally { @@ -559,8 +565,8 @@ class WorkloadContext private[workload] ( val adds = addFiles.getOrElse(Seq.empty) // Materialize each add's logical rows to a temp Parquet for the live write. - val schema = DeltaHarness.get.schemaAt(spark, livePath, version = None, includePartition = true) - val tempParquet = adds.map(in => DeltaHarness.get.writeRowsToTemp(spark, schema, in.rows)) + val schema = DeltaHarness.get.schemaAt(spark, livePath, version = None) + val tempParquet = adds.map(in => writeRowsToTemp(schema, in.rows)) // A remove targets all files added at the referenced commit (== that table version). val removePaths = removeFiles.getOrElse(Seq.empty) .flatMap(k => SpecLayout.addPathsAt(w.table.sourcePath, k)) diff --git a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala index 8a3c6fdb..6f04c2d8 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WriteSpecCapture.scala @@ -108,8 +108,7 @@ class WriteSpecBuilder { // Materialize each low-level add's logical rows to the `dataFile` path recorded in the // action (full schema incl. partition columns, so replay's writeFiles can partition/map). if (rows.nonEmpty) { - val schema = DeltaHarness.get.schemaAt(spark, tablePath.toString, Some(idx.toLong), - includePartition = true) + val schema = DeltaHarness.get.schemaAt(spark, tablePath.toString, Some(idx.toLong)) ll.addFiles.getOrElse(Seq.empty).zip(rows).foreach { case (af, r) => DeltaHarness.get.writeRows(spark, schema, r, outputDir.resolve(af.dataFile)) } @@ -123,7 +122,7 @@ class WriteSpecBuilder { val writeName = s"${outputDir.getFileName.toString}_write" JsonUtil.writeSpec( outputDir.resolve("specs").resolve(s"$writeName.json"), WriteSpec(enriched.toSeq)) - writeExpectedLatest(spark, log, tablePath, outputDir.resolve("expected").resolve(writeName)) + writeExpectedLatest(spark, tablePath, outputDir.resolve("expected").resolve(writeName)) } /** @@ -135,7 +134,7 @@ class WriteSpecBuilder { spark: SparkSession, tablePath: String, outputDir: Path, idx: Int, rows: Seq[Seq[Map[String, Any]]]): Option[Seq[String]] = { rows.headOption.filter(_.nonEmpty).map { r => - val schema = DeltaHarness.get.schemaAt(spark, tablePath, Some(idx.toLong), includePartition = true) + val schema = DeltaHarness.get.schemaAt(spark, tablePath, Some(idx.toLong)) val rel = SpecLayout.commitDataFile(idx, "part-00000.parquet") DeltaHarness.get.writeRows(spark, schema, r, outputDir.resolve(rel)) Seq(rel) @@ -143,15 +142,10 @@ class WriteSpecBuilder { } /** - * Capture the expected final table state under the write spec's `expected/_write/`: - * - `table_content/`: Parquet of the latest snapshot's rows - * - `table_version_metadata.json`: protocol + metadata of the latest snapshot + * Capture the write spec's expected final rows under `expected/_write/table_content/`, + * which [[WorkloadValidator]] compares against the replayed table. */ - private def writeExpectedLatest( - spark: SparkSession, - log: io.delta.workload.deltaharness.LogView, - tablePath: Path, - latestDir: Path): Unit = { + private def writeExpectedLatest(spark: SparkSession, tablePath: Path, latestDir: Path): Unit = { Files.createDirectories(latestDir) val contentDir = latestDir.resolve("table_content") @@ -160,14 +154,5 @@ class WriteSpecBuilder { } spark.read.format("delta").load(tablePath.toString) .write.mode(SaveMode.Overwrite).parquet(contentDir.toString) - - val snapshot = log.update() - val protocol = JsonUtil.mapper.treeToValue( - JsonUtil.mapper.readTree(snapshot.protocolJson).get("protocol"), classOf[Any]) - val metadata = JsonUtil.mapper.treeToValue( - JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData"), classOf[Any]) - JsonUtil.writeSpec( - latestDir.resolve("table_version_metadata.json"), - SnapshotExpected(protocol, metadata)) } } diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala index 1e9a8c72..3408da0b 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaHarness.scala @@ -83,14 +83,8 @@ trait DeltaHarness { */ def commit(spark: SparkSession, tablePath: String, req: CommitRequest): Seq[String] - /** - * The table's schema at `version` (latest if None). When `includePartition` is false the - * partition columns are dropped, matching the layout of a raw data file referenced by a - * low-level `AddFile` (partition values ride on the action, not the file). - */ - def schemaAt( - spark: SparkSession, tablePath: String, - version: Option[Long], includePartition: Boolean): StructType + /** The table's schema at `version` (latest if None). */ + def schemaAt(spark: SparkSession, tablePath: String, version: Option[Long]): StructType /** * Materialize in-memory `rows` (the workload-generator's authoring surface) into a single Parquet @@ -99,12 +93,6 @@ trait DeltaHarness { */ def writeRows( spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]], dest: Path): Unit - - /** - * Like [[writeRows]], but writes to a single Parquet file in a fresh temp directory and returns - * the produced file path. - */ - def writeRowsToTemp(spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]]): Path } trait LogView { diff --git a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala index 2149e33f..0e40cd30 100644 --- a/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala +++ b/workload-generator/src/main/scala/io/delta/workload/deltaharness/DeltaSparkHarness.scala @@ -79,18 +79,11 @@ class DeltaSparkHarness extends DeltaHarness { } override def schemaAt( - spark: SparkSession, tablePath: String, - version: Option[Long], includePartition: Boolean): StructType = { + spark: SparkSession, tablePath: String, version: Option[Long]): StructType = { val log = openLog(spark, tablePath) val snapshot = version.map(log.getSnapshotAt).getOrElse(log.update()) val meta = JsonUtil.mapper.readTree(snapshot.metadataJson).get("metaData") - val full = DataType.fromJson(meta.get("schemaString").asText()).asInstanceOf[StructType] - if (includePartition) full - else { - val partCols = Option(meta.get("partitionColumns")) - .map(_.elements().asScala.map(_.asText()).toSet).getOrElse(Set.empty[String]) - StructType(full.fields.filterNot(f => partCols.contains(f.name))) - } + DataType.fromJson(meta.get("schemaString").asText()).asInstanceOf[StructType] } override def writeRows( @@ -113,14 +106,6 @@ class DeltaSparkHarness extends DeltaHarness { } } - override def writeRowsToTemp( - spark: SparkSession, schema: StructType, rows: Seq[Map[String, Any]]): Path = { - val dir = Files.createTempDirectory("row-parquet-rtas") - val dest = dir.resolve("part-00000.parquet") - writeRows(spark, schema, rows, dest) - dest - } - /** * Coerce a row's value to the column's Spark type. `None`/`null` -> SQL NULL. Unknown types * fail loud rather than passing the raw value through: a silent passthrough that Spark happens From 6a827a51ac4d6076155787c82937b4194a950123 Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Mon, 22 Jun 2026 23:10:51 +0000 Subject: [PATCH 24/25] chore: reword comments flagged by the typos linter --- .../src/main/scala/io/delta/workload/WorkloadGenerator.scala | 2 +- .../src/main/scala/io/delta/workload/WorkloadValidator.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index ce2ab9a3..85c329df 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -590,7 +590,7 @@ class WorkloadContext private[workload] ( val newVersion = DeltaHarness.get.openLog(spark, livePath).update().version require(newVersion == idx, s"commitOp produced version $newVersion but expected ordinal $idx; commit index can no " + - "longer be used as the table version (ordinal-based removes would mis-resolve)") + "longer be used as the table version (ordinal-based removes would resolve incorrectly)") // Record the spec: adds become logical-Parquet pointers (materialized to data/ in buildSpec); // removes become ordinal references. diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala index de6e29ac..664e77ea 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadValidator.scala @@ -174,7 +174,7 @@ object WorkloadValidator { replayCommit(spark, c, idx, ref, path, testDir) } // Each commit must map to exactly one version, or ordinal-based remove resolution (which keys - // off commit index == table version) silently mis-resolves. Fail loud if replay desynced. + // off commit index == table version) silently resolves the wrong version. Fail loud if desynced. val finalVersion = DeltaHarness.get.openLog(spark, path).update().version require(finalVersion == writeSpec.commits.size - 1, s"replay produced ${finalVersion + 1} versions but the write spec has ${writeSpec.commits.size} " + From 40c1bad70d07623273666f5469e55af715cf5b4a Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Tue, 23 Jun 2026 19:53:41 +0000 Subject: [PATCH 25/25] fix: use Iterable instead of IterableOnce for scala 2.12 build --- .../scala/io/delta/workload/WorkloadGenerator.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala index 85c329df..ece511e4 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadGenerator.scala @@ -18,7 +18,7 @@ package io.delta.workload import java.nio.file.{Files, Path, Paths} -import scala.collection.{IterableOnce, mutable} +import scala.collection.mutable import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession @@ -469,7 +469,7 @@ class WorkloadContext private[workload] ( schema: String, properties: Map[String, String] = Map.empty, partitionColumns: Seq[String] = Seq.empty, - rows: IterableOnce[Map[String, Any]] = Seq.empty): Unit = { + rows: Iterable[Map[String, Any]] = Seq.empty): Unit = { val rowSeq = rows.iterator.toSeq val partitionClause = partitionedByClause(partitionColumns) val propsClause = tblPropertiesClause(properties) @@ -506,7 +506,7 @@ class WorkloadContext private[workload] ( * it produces no commit, so recording it would both desync the commit-index/version mapping * and let the validator pass a spec with nothing to validate. */ - def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = { + def insertOp(w: WriteHandle, rows: Iterable[Map[String, Any]]): Unit = { val rowSeq = rows.iterator.toSeq require(rowSeq.nonEmpty, "insertOp requires at least one row") // Drive the live insert from the SAME materialized Parquet the spec will bundle (via the @@ -869,11 +869,11 @@ trait WorkloadOps { schema: String, properties: Map[String, String] = Map.empty, partitionColumns: Seq[String] = Seq.empty, - rows: IterableOnce[Map[String, Any]] = Seq.empty): Unit = + rows: Iterable[Map[String, Any]] = Seq.empty): Unit = current.replaceTableOp(w, schema, properties, partitionColumns, rows) /** Insert rows and record the insert. */ - def insertOp(w: WriteHandle, rows: IterableOnce[Map[String, Any]]): Unit = + def insertOp(w: WriteHandle, rows: Iterable[Map[String, Any]]): Unit = current.insertOp(w, rows) /** Delete rows matching `predicate` and record the delete. */