From faf538de0c639cf9a878ed969dd94de04b66b27c Mon Sep 17 00:00:00 2001 From: Ryan Liao Date: Fri, 9 Oct 2026 20:18:41 +0000 Subject: [PATCH 1/2] feat: export create table sql in DAT write specs --- workload-generator/README.md | 6 ++ workload-generator/docs/authoring-guide.md | 14 +++-- .../docs/harness-implementation-guide.md | 21 +++++++ workload-generator/docs/spec-reference.md | 17 +++++- .../scala/io/delta/workload/WorkloadOps.scala | 7 ++- .../io/delta/workload/model/WriteModel.scala | 6 +- .../io/delta/workload/write/TableSql.scala | 10 ++++ .../io/delta/workload/write/WriteReplay.scala | 8 ++- .../workload/WorkloadValidatorSuite.scala | 58 +++++++++++++++++++ .../workload/write/WriteSerdeSuite.scala | 29 +++++++++- 10 files changed, 163 insertions(+), 13 deletions(-) diff --git a/workload-generator/README.md b/workload-generator/README.md index 645d49fa..a8e0e374 100644 --- a/workload-generator/README.md +++ b/workload-generator/README.md @@ -160,6 +160,12 @@ See the [Spec Format Reference](docs/spec-reference.md) for the complete JSON sc The write DSL (`createTableOp`, `insertOp`, `commitOp`, …) is exposed by `WorkloadOps`; see the [Authoring Guide](docs/authoring-guide.md) for how to author write workloads. There is no per-spec `writeSpec` pointer: whether the `read`/`snapshot` specs in a directory validate against the captured `delta/` table or against a replay of the sibling write spec is decided per directory — if a `*_write.json` is present, they are validated *portably* against the replay (replay the write spec into a fresh table, then compare). +`create_table` commits include an optional `createSql` template for Spark/Delta SQL consumers. +It uses the generator's creation SQL with a destination-table placeholder; structured schema, +partition columns, and properties remain available to non-SQL consumers. See the +[creation SQL contract](docs/spec-reference.md#high-level-operations) for target binding and +the fallback when the field is absent. + ## Workload Suites Workload suites are in `src/test/scala/io/delta/workload/tables/`. Each suite extends `WorkloadTestSuite` and covers a specific Delta feature area (reads, deletion vectors, column mapping, time travel, checkpoints, data skipping, protocol versions, merge, schema evolution, type widening, variant, row tracking, in-commit timestamps, and more). diff --git a/workload-generator/docs/authoring-guide.md b/workload-generator/docs/authoring-guide.md index f80ae015..44b2e3db 100644 --- a/workload-generator/docs/authoring-guide.md +++ b/workload-generator/docs/authoring-guide.md @@ -92,13 +92,13 @@ Inside a test body, these methods are available directly (via `WorkloadOps` trai | SQL | `sql(statement)` | — | | Table handles | `registerTable(name)`, `registerTableFromPath(path)` | → `TableHandle` | | Read specs | `readSpec(t, ...)`, `snapshotSpec(t, ...)` | `TableHandle` → `SpecRef` | -| Write specs | `createTableOp`, `replaceTableOp`, `insertOp`, `deleteOp`, `updateOp`, `addColumnsOp`, `renameColumnOp`, `dropColumnsOp`, `setPropertiesOp`, `unsetPropertiesOp`, `commitOp` (low-level) | → `WriteHandle`; `registerWriteSpec(w)` → `TableHandle` | +| Write specs | `createTableOp`, `replaceTableOp`, `insertOp`, `deleteOp`, `updateOp`, `addColumnsOp`, `renameColumnOp`, `dropColumnsOp`, `setPropertiesOp`, `unsetPropertiesOp`, `commitOp` (low-level) | → `WriteHandle`; `endWrite(w)` → `TableHandle` | | Checkpointing | `forceCheckpoint(tableName)` — triggers a checkpoint via DeltaLog | — | | Mutations | `mutateTable(t) { dir => ... }`, `modifyCommitActions(t, version) { ... }` | `TableHandle` | ### Authoring a write workload -A write workload builds a table through DSL ops instead of raw `sql(...)`, producing a portable `write` spec (`specs/_write.json`) plus any `read`/`snapshot` specs derived from the built table. Start with `createTableOp` (or `replaceTableOp`), apply ops, then `registerWriteSpec(w)` to get a `TableHandle` you can attach reads/snapshots to: +A write workload builds a table through DSL ops instead of raw `sql(...)`, producing a portable `write` spec (`specs/_write.json`) plus any `read`/`snapshot` specs derived from the built table. Start with `createTableOp`, apply ops, then `endWrite(w)` to get a `TableHandle` you can attach reads/snapshots to: ```scala import org.apache.spark.sql.types.{IntegerType, StringType, StructType} @@ -108,8 +108,8 @@ val w = createTableOp("tbl", schema = schema) insertOp(w, Seq(Map("id" -> 1, "name" -> "alice"))) // Low-level commit: logical rows are written through the engine (stats/partitionValues derived). commitOp(w, addFiles = Some(Seq(AddFileInput(rows = Seq(Map("id" -> 2, "name" -> "bob")))))) -val t = registerWriteSpec(w) -readSpec(t, name = "read_all") +val t = endWrite(w) +readSpec(t, name = Some("read_all")) snapshotSpec(t) ``` @@ -117,6 +117,12 @@ The write-op schema inputs (`createTableOp`, `replaceTableOp`, `addColumnsOp`, a optional `schema`) are Spark `StructType`s, not DDL strings. The generator stores them as Delta schema JSON in the spec. +`createTableOp` also exports its Spark/Delta creation SQL as `createSql`, with `{{table}}` +in place of the reference table's name. Authors do not need to supply SQL separately. +The schema, partition columns, and properties remain in the commit for consumers that +use their own table-creation API. Calling `sql("CREATE TABLE ...")` alone does not export +a creation template. See the [creation SQL contract](spec-reference.md#high-level-operations). + `commitOp` returns the commit's ordinal (== table version); a later `removeFiles = Some(Seq(ordinal))` tombstones the files that commit added. See `WriteBasicSuite`/`WriteCommitSuite`/`WriteSequencesSuite` for worked examples and the [Write Spec reference](spec-reference.md#write-spec) for the on-disk format. --- diff --git a/workload-generator/docs/harness-implementation-guide.md b/workload-generator/docs/harness-implementation-guide.md index b8e7eb39..b0fce3e9 100644 --- a/workload-generator/docs/harness-implementation-guide.md +++ b/workload-generator/docs/harness-implementation-guide.md @@ -211,6 +211,27 @@ Run the operation and assert it fails. Matching the exact error code is ideal bu A `write` spec (`specs/_write.json`) is a portable recipe for *building* a table: an ordered list of `commits` your harness replays into a fresh table. High-level commits (create/replace/insert/delete/update/schema-evolution/properties) are replayed through your engine's normal write APIs; low-level `commit`s carry raw actions (the data files hold logical rows your engine writes through its own write path, so physical names/stats are engine-derived, and removes reference a prior add by commit ordinal == table version). See the [Write Spec reference](spec-reference.md#write-spec) for the field-level contract. +For `create_table`, a Spark/Delta SQL consumer can execute the optional `createSql` template +instead of converting the structured schema to SQL. Bind only the target placeholder at the +start of the statement; leave placeholder text inside column definitions and literals intact. +For example, given a template and an already-quoted destination table reference: + +```scala +val prefix = "CREATE TABLE {{table}}" +require(template.startsWith(prefix + " (")) +val statement = "CREATE TABLE " + targetTableRef + template.substring(prefix.length) +executeSql(statement) +``` + +Execute this statement through the harness's SQL setup engine. A separate row-ingestion API +still receives rows. If `createSql` is absent, construct the statement from `schema`, +`partitionColumns`, and `properties`; consumers without SQL support use those structured +fields directly. The template dialect and binding rules are specified in the +[creation SQL contract](spec-reference.md#high-level-operations). + +Readback filters and projections use the existing read-spec `predicate` and `columns` +fields. `createSql` does not add SQL templates for reads or other write operations. + The write spec has no expected-data artifact of its own. Its own check is **basic**: the replay must succeed and produce the expected number of versions (`finalVersion == commits.size - 1`). The final-state rows are validated by an auto-generated baseline read spec named `latest` (`specs/_latest.json` with rows under `expected/_latest/expected_data/`), and per-version protocol/metadata by the snapshot spec(s). Because a write spec is replayed (not byte-compared), comparison is **portable**: rows-only data checks, capability-based protocol comparison (your table may use explicit table features where Spark used legacy versions), and column-mapping-normalized schema. A write spec's presence makes its whole directory *write-derived*: replay it into a fresh table once, then validate every read/snapshot spec in that directory (including the baseline `latest` read) against the replay (portably), rather than against a captured `delta/` directory. diff --git a/workload-generator/docs/spec-reference.md b/workload-generator/docs/spec-reference.md index 99203e97..aa68f2c9 100644 --- a/workload-generator/docs/spec-reference.md +++ b/workload-generator/docs/spec-reference.md @@ -375,7 +375,7 @@ These map to SQL-like operations. The writer translates them to appropriate Delt | Operation | Fields | Description | |-----------|--------|-------------| -| `create_table` | `schema`, `partitionColumns?`, `properties?` | Create a new table with the given schema (`schema` is a Delta-JSON struct) | +| `create_table` | `schema`, `partitionColumns?`, `properties?`, `createSql?` | Create a new table with the given schema (`schema` is a Delta-JSON struct); `createSql` optionally supplies the Spark SQL creation template | | `replace_table` | `schema`, `partitionColumns?`, `properties?`, `dataFiles?` | Replace the table's schema/partitioning/properties (and all data). With `dataFiles` it is a replace-as-select: a single commit that also writes the bundled data. | | `insert` | `dataFiles?` | Append the rows in the bundled Parquet data files | | `update` | `predicate`, `set` | Update rows matching `predicate`; `set` maps column → SQL expression | @@ -383,6 +383,18 @@ These map to SQL-like operations. The writer translates them to appropriate Delt | `evolve_schema` | `addColumns?`, `renameColumns?`, `dropColumns?` | Modify table schema. `addColumns` entries are `{name, type, nullable}`; `renameColumns` maps old → new | | `update_properties` | `set?`, `remove?` | Modify table properties (`set` map, `remove` names) | +`createTableOp` exports `createSql` using the same SQL template it executes to create the +reference table. The template starts with `CREATE TABLE {{table}} (`. SQL consumers replace +only that target placeholder with their own quoted table identifier or Delta path reference, +then execute the statement. Placeholder text inside column definitions, comments, or property +values must remain unchanged. The template contains the column definitions, partition clause, +and table properties; it does not contain the reference table's name or storage location. + +The structured `schema`, `partitionColumns`, and `properties` remain available to consumers +that do not execute SQL. Older corpora can omit `createSql`; SQL consumers then construct the +creation statement from the structured fields. The exported template uses Spark/Delta SQL +and must agree with the structured definition. Other operations do not export SQL templates. + The data for `insert` and `replace_table` is bundled as Parquet under `data/commit_N/` (the generator's authoring API accepts in-memory rows, but the spec always stores Parquet). A consumer replays by reading those files — `insert` appends them; `replace_table` performs a @@ -464,7 +476,8 @@ The write spec itself only asserts that the replay succeeds and produces the exp { "name": "status", "type": "string", "nullable": true, "metadata": {} } ] }, - "properties": { "delta.enableDeletionVectors": "true" } + "properties": { "delta.enableDeletionVectors": "true" }, + "createSql": "CREATE TABLE {{table}} (id INT NOT NULL, status STRING) USING delta TBLPROPERTIES ('delta.enableDeletionVectors' = 'true')" }, { "operation": "insert", "dataFiles": ["data/commit_1/part-0000-abc.parquet"] }, { "operation": "delete", "predicate": "id > 100" }, diff --git a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala index 76595ad2..aa020a1e 100644 --- a/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala +++ b/workload-generator/src/main/scala/io/delta/workload/WorkloadOps.scala @@ -184,12 +184,15 @@ trait WorkloadOps { properties: Map[String, String] = Map.empty, partitionColumns: Seq[String] = Seq.empty): WriteHandle = { val ctx = WorkloadContext.current - ctx.sql(TableSql.createTable(tableName, schema, partitionColumns, properties)) + val createSql = TableSql.createTable( + TableSql.tablePlaceholder, schema, partitionColumns, properties) + ctx.sql(TableSql.bindCreateTable(createSql, tableName)) val t = ctx.registerTable(tableName) ctx.getWriteBuilder(t).record(CreateTableCommit( schema = schema, partitionColumns = ctx.toOption(partitionColumns), - properties = ctx.toOption(properties))) + properties = ctx.toOption(properties), + createSql = Some(createSql))) new WriteHandle(t) } diff --git a/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala b/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala index 9a6dc863..6503a30d 100644 --- a/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala +++ b/workload-generator/src/main/scala/io/delta/workload/model/WriteModel.scala @@ -101,12 +101,14 @@ sealed trait WriteCommit { // ---- High-level operations (replayed from parameters) ---- -@JsonPropertyOrder(Array("schema", "partitionColumns", "properties")) +/** createSql optionally supplies Spark SQL for the same definition, with a {{table}} target. */ +@JsonPropertyOrder(Array("schema", "partitionColumns", "properties", "createSql")) @JsonInclude(JsonInclude.Include.NON_ABSENT) case class CreateTableCommit( schema: StructType, partitionColumns: Option[Seq[String]] = None, - properties: Option[Map[String, String]] = None) extends WriteCommit { + properties: Option[Map[String, String]] = None, + createSql: Option[String] = None) extends WriteCommit { @JsonIgnore val operation = "create_table" } diff --git a/workload-generator/src/main/scala/io/delta/workload/write/TableSql.scala b/workload-generator/src/main/scala/io/delta/workload/write/TableSql.scala index 989dab34..5d479ce5 100644 --- a/workload-generator/src/main/scala/io/delta/workload/write/TableSql.scala +++ b/workload-generator/src/main/scala/io/delta/workload/write/TableSql.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.types.StructType */ private[workload] object TableSql { + val tablePlaceholder: String = "{{table}}" + /** `CREATE [OR REPLACE] TABLE ref (cols) USING delta [PARTITIONED BY …] [TBLPROPERTIES …]`. */ def createTable( ref: String, schema: StructType, partitionColumns: Seq[String], @@ -31,6 +33,14 @@ private[workload] object TableSql { s"CREATE ${if (orReplace) "OR REPLACE " else ""}TABLE $ref (${schema.toDDL}) USING delta" + partitionedBy(partitionColumns) + tblProperties(properties) + /** Bind the CREATE TABLE target without replacing placeholder text inside SQL literals. */ + def bindCreateTable(sql: String, ref: String): String = { + val prefix = s"CREATE TABLE $tablePlaceholder" + require(sql.startsWith(prefix + " ("), + s"createSql must start with '$prefix ('") + s"CREATE TABLE $ref${sql.substring(prefix.length)}" + } + /** `CREATE OR REPLACE TABLE ref USING delta [PARTITIONED BY …] [TBLPROPERTIES …] AS ` (RTAS). */ @@ -48,6 +48,9 @@ private[workload] object TableSql { s"CREATE OR REPLACE TABLE $ref USING delta" + partitionedBy(partitionColumns) + tblProperties(properties) + s" AS $select" + private def createTablePrefix(ref: String, orReplace: Boolean = false): String = + s"CREATE ${if (orReplace) "OR REPLACE " else ""}TABLE $ref" + private def partitionedBy(columns: Seq[String]): String = if (columns.nonEmpty) s" PARTITIONED BY (${columns.mkString(", ")})" else "" diff --git a/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorSuite.scala b/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorSuite.scala index 9b930066..f6f2c70e 100644 --- a/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorSuite.scala +++ b/workload-generator/src/test/scala/io/delta/workload/WorkloadValidatorSuite.scala @@ -18,6 +18,8 @@ package io.delta.workload import java.nio.file.{Files, Path} +import scala.collection.immutable.ListMap + import com.fasterxml.jackson.databind.node.ObjectNode import org.apache.commons.io.FileUtils import org.apache.spark.sql.SparkSession @@ -149,7 +151,7 @@ class WorkloadValidatorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor .add("details", new StructType().add("labels", ArrayType(StringType))) val w = createTableOp(tableName, schema, partitionColumns = Seq("region"), - properties = Map("delta.appendOnly" -> "true", "note" -> "{{table}}")) + properties = ListMap("delta.appendOnly" -> "true", "note" -> "{{table}}")) insertOp(w, Seq(Map( "id" -> 1, "region" -> "east", "details" -> Map("labels" -> Seq("a", "b"))))) @@ -163,9 +165,9 @@ class WorkloadValidatorSuite extends AnyFunSuite with BeforeAndAfterAll with Wor val spec = JsonUtil.mapper.readTree(Files.readAllBytes(specPath)) val create = spec.path("commits").get(0).asInstanceOf[ObjectNode] val createSql = create.path("createSql").asText() - assert(createSql.startsWith("CREATE TABLE {{table}} (")) - assert(createSql.contains("PARTITIONED BY (region)")) - assert(createSql.contains("'note' = '{{table}}'")) + assert(createSql == "CREATE TABLE {{table}} (id INT NOT NULL,region STRING," + + "details STRUCT>) USING delta PARTITIONED BY (region) " + + "TBLPROPERTIES ('delta.appendOnly' = 'true', 'note' = '{{table}}')") if (!includeSql) { create.remove("createSql") Files.write(specPath, JsonUtil.mapper.writeValueAsBytes(spec))