Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright (2026) The Delta Lake Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.delta.workload.tables

import java.io.DataInputStream
import java.nio.file.{Files, Path, Paths}

import org.apache.hadoop.fs.{Path => HadoopPath}
import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor
import org.apache.spark.sql.delta.storage.dv.DeletionVectorStore
import org.scalatest.Assertions._

import io.delta.workload.log.DeletionVector

/** Reads the exact serialized bitmap of an inline or UUID-relative deletion vector. */
object DeletionVectorPayloads {

def descriptor(dv: DeletionVector): DeletionVectorDescriptor = {
// At this DAT pin USE_LONG_FOR_INTS boxes Option[Int] offsets as Long. Normalize
// only this descriptor input, without changing the shared mapper or DV bytes.
val offset = dv.offset.asInstanceOf[Option[Number]].map(n => Math.toIntExact(n.longValue()))
DeletionVectorDescriptor(dv.storageType, dv.pathOrInlineDv, offset,
dv.sizeInBytes, dv.cardinality)
}

def sidecar(table: Path, dv: DeletionVector): Path =
Paths.get(descriptor(dv).absolutePath(new HadoopPath(table.toUri)).toUri.getPath)

def payload(table: Path, dv: DeletionVector): Array[Byte] = {
if (dv.storageType == "i") descriptor(dv).inlineData
else {
assert(dv.storageType == "u")
val file = sidecar(table, dv)
assert(file.normalize().startsWith(table.normalize()))
val offset = descriptor(dv).offset.get
assert(offset > 0)
assert(Files.size(file) >= offset.toLong + 4L + dv.sizeInBytes + 4L)
val input = new DataInputStream(Files.newInputStream(file))
try {
assert(input.readByte() == DeletionVectorStore.DV_FILE_FORMAT_VERSION_ID_V1)
input.skipNBytes(offset.toLong - 1L)
// This OSS reader checks both the payload length and checksum.
DeletionVectorStore.readRangeFromStream(input, dv.sizeInBytes)
} finally input.close()
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,23 @@

package io.delta.workload.tables

import io.delta.workload.WorkloadTestSuite
import java.nio.file.{Files, Path}

import scala.collection.JavaConverters._

import org.apache.spark.sql.Row
import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor
import org.apache.spark.sql.delta.deletionvectors.RoaringBitmapArray
import org.apache.spark.sql.types.{DataType, IntegerType, StringType, StructType}

import io.delta.workload.{WorkloadContext, WorkloadTestSuite}
import io.delta.workload.deltaharness.{Format => HarnessFormat}
import io.delta.workload.engine.{RowComparison, SnapshotResolver}
import io.delta.workload.json.JsonUtil
import io.delta.workload.log.{Action, AddFile, CommitLog, DeletionVector, Metadata, Protocol,
RemoveFile}
import io.delta.workload.model._
import io.delta.workload.tables.DeletionVectorPayloads.{payload, sidecar}

class DeletionVectorsSuite extends WorkloadTestSuite("deletion_vectors") {

Expand Down Expand Up @@ -793,4 +809,174 @@ class DeletionVectorsSuite extends WorkloadTestSuite("deletion_vectors") {
snapshotSpec(t)
}

// Inline and UUID-relative deletion vectors with the same engine-written bitmap.
Seq("inline", "uuid_relative").foreach { form =>
test(s"e2e_42_$form") {
sql("""CREATE TABLE tbl (id INT, value STRING) USING delta TBLPROPERTIES (
'delta.enableDeletionVectors' = 'true', 'delta.checkpointInterval' = '1000',
'delta.checkpointPolicy' = 'classic')""")
sql("""INSERT INTO tbl SELECT /*+ COALESCE(1) */ * FROM VALUES
(-7, 'remove'), (0, 'keep'), (4, CAST(NULL AS STRING)), (4, CAST(NULL AS STRING)),
(9, 'remove'), (12, 'keep'), (18, CAST(NULL AS STRING)), (23, 'edge'),
(31, 'keep'), (44, CAST(NULL AS STRING)), (57, 'remove'), (88, 'last') AS v(id, value)""")
sql("DELETE FROM tbl WHERE id IN (-7, 9, 57)")
sql("DELETE FROM tbl WHERE id = 4")
val table = registerTable("tbl")
val schema = new StructType().add("id", IntegerType).add("value", StringType)
val baseRows = Seq(Row(-7, "remove"), Row(0, "keep"), Row(4, null), Row(4, null),
Row(9, "remove"), Row(12, "keep"), Row(18, null), Row(23, "edge"),
Row(31, "keep"), Row(44, null), Row(57, "remove"), Row(88, "last"))
val firstRows = baseRows.filterNot(r => Set(-7, 9, 57).contains(r.getInt(0)))
val latestRows = firstRows.filterNot(_.getInt(0) == 4)
val original = (0L to 3L).map(v => v -> CommitLog.read(table.sourcePath, v)).toMap
val metadata = original(0L).collect { case m: Metadata => m }.ensuring(_.size == 1).head
val protocol = original(0L).collect { case p: Protocol => p }.ensuring(_.size == 1).head
assert(DataType.fromJson(metadata.schemaString) == schema)
assert(metadata.partitionColumns.isEmpty)
assert(metadata.configuration == Map("delta.enableDeletionVectors" -> "true",
"delta.checkpointInterval" -> "1000", "delta.checkpointPolicy" -> "classic"))
assert(protocol == Protocol(3, 7, Some(Set("deletionVectors")),
Some(Set("appendOnly", "deletionVectors", "invariants"))))
info("DV original protocol: " + protocol.toJson)
info("DV original metadata: " + metadata.toJson)
val adds = (1L to 3L).map { version =>
val found = original(version).collect { case a: AddFile => a }
assert(found.size == 1)
info(s"DV original add v$version: " + found.head.toJson)
version -> found.head
}.toMap
assert(adds.values.map(_.path).toSet.size == 1) // Deletes must retain the data file.
assert(adds(1L).deletionVector.isEmpty)
val physical = spark.read.parquet(table.sourcePath.resolve(adds(1L).path).toString)
assert(physical.schema == schema)
RowComparison.assertRowsEqual(spark.createDataFrame(baseRows.asJava, schema), physical,
s"$form physical input")
val physicalRows = physical.collect().toSeq
val sourceDvs = (2L to 3L).map(v => v -> adds(v).deletionVector.get).toMap
sourceDvs.foreach { case (version, dv) =>
assert(dv.storageType == "u")
assert(dv.cardinality == (if (version == 2L) 3L else 5L))
val expectedPositions = physicalRows.zipWithIndex.collect {
case (row, index) if Set(-7, 9, 57).contains(row.getInt(0)) ||
(version == 3L && row.getInt(0) == 4) => index.toLong
}
assert(RoaringBitmapArray.readFrom(payload(table.sourcePath, dv)).toArray.toSeq ==
expectedPositions)
}
val converted = sourceDvs.values.map { dv =>
val bytes = payload(table.sourcePath, dv)
// Use the verified OSS helper with exact engine-generated payload bytes.
val inline = DeletionVectorDescriptor.inlineInLog(bytes, dv.cardinality)
assert(inline.storageType == "i" && inline.offset.isEmpty)
assert(inline.sizeInBytes == bytes.length && inline.cardinality == dv.cardinality)
assert(inline.inlineData.sameElements(bytes))
dv -> DeletionVector(inline.storageType, inline.pathOrInlineDv, inline.offset,
inline.sizeInBytes, inline.cardinality)
}.toMap
if (form == "inline") {
(2L to 3L).foreach { version =>
modifyCommitActions(table, version) { actions =>
actions.map {
case add: AddFile => add.copy(deletionVector = add.deletionVector.map(converted))
// The second delete removes the first DV identity; convert that tombstone too.
case remove: RemoveFile =>
remove.copy(deletionVector = remove.deletionVector.map(converted))
case other => other
}
}
}
}
val expectedAdds = adds.map { case (version, add) =>
version -> (if (form == "inline") {
add.copy(deletionVector = add.deletionVector.map(converted))
} else add)
}
var copiedTable: Path = null
mutateTable(table) { dir =>
copiedTable = dir
val sidecars = sourceDvs.values.map(dv => sidecar(dir, dv)).toSet
if (form == "inline") sidecars.foreach(Files.delete)
else sidecars.foreach(p => assert(Files.isRegularFile(p)))
// Version checksums embed the original descriptors and become stale after conversion.
val logs = Files.list(dir.resolve("_delta_log"))
try logs.iterator().asScala.filter(_.getFileName.toString.endsWith(".crc"))
.foreach(Files.delete)
finally logs.close()
(1L to 3L).foreach { version =>
val actions = CommitLog.read(dir, version)
assert(actions.collect { case a: AddFile => a } == Seq(expectedAdds(version)))
expectedAdds(version).deletionVector.foreach { dv =>
assert(dv.storageType == (if (form == "inline") "i" else "u"))
assert(dv.cardinality == (if (version == 2L) 3L else 5L))
assert(payload(dir, dv).sameElements(payload(table.sourcePath, sourceDvs(version))))
assert(RoaringBitmapArray.readFrom(payload(dir, dv)).cardinality == dv.cardinality)
}
}
val oldIdentity = CommitLog.read(dir, 3L).collect { case r: RemoveFile => r }
assert(oldIdentity.size == 1)
assert(oldIdentity.head.path == adds(1L).path)
assert(oldIdentity.head.deletionVector == expectedAdds(2L).deletionVector)
assert(Files.size(dir.resolve(adds(1L).path)) == adds(1L).size)
if (form == "inline") assert(sidecars.forall(p => !Files.exists(p)))
}
val snapshot = SnapshotResult(
ProtocolInfo(3, 7, Some(Seq("deletionVectors")),
Some(Seq("appendOnly", "deletionVectors", "invariants"))),
MetadataInfo(metadata.id, metadata.name, metadata.description,
HarnessFormat("parquet", Map.empty), metadata.schemaString, Seq.empty,
metadata.configuration, metadata.createdTime))
Seq[java.lang.Long](1L, 2L, null).foreach { version =>
snapshotSpec(table, version = version).assert { spec =>
assert(spec.query == SnapshotQuery(version = Some(Option(version).fold(3L)(_.toLong))))
assert(spec.expectation == Succeeded(snapshot))
}
}
val projected = new StructType().add("value", StringType)
val reordered = projected.add("id", IntegerType)
val filter = "id IN (4, 12, 18, 57)"
val cases = Seq(
("read_base", ReadQuery(version = Some(1L)), schema, baseRows),
("read_first_delete", ReadQuery(version = Some(2L)), schema, firstRows),
("read_latest", ReadQuery(), schema, latestRows),
("read_v2_filtered_projected", ReadQuery(version = Some(2L),
predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered,
Seq(Row(null, 4), Row(null, 4), Row("keep", 12), Row(null, 18))),
("read_v3_filtered_projected", ReadQuery(version = Some(3L),
predicate = Some(filter), columns = Some(Seq("value", "id"))), reordered,
Seq(Row("keep", 12), Row(null, 18))),
("read_projected", ReadQuery(columns = Some(Seq("value"))), projected,
Seq(Row("keep"), Row("keep"), Row(null), Row("edge"), Row("keep"), Row(null),
Row("last"))),
("read_deleted_only", ReadQuery(predicate = Some("id IN (-7, 4, 9, 57)")), schema,
Seq.empty[Row]))
cases.foreach { case (name, query, expectedSchema, rows) =>
readSpec(table, name = Some(name), version = query.version.map(Long.box).orNull,
predicate = query.predicate.orNull, columns = query.columns).assert { spec =>
assert(spec.query == query)
// All predicates overlap physical min/max; the single data AddFile remains scanned.
assert(spec.expectation == Succeeded(ReadResult(rows.size.toLong, 1, 0L)))
val actual = SnapshotResolver.applyFilters(
SnapshotResolver.buildDeltaReader(spark, copiedTable, query.version, query.timestamp),
query.predicate, query.columns)
assert(actual.schema == expectedSchema)
RowComparison.assertRowsEqual(spark.createDataFrame(rows.asJava, expectedSchema), actual,
s"$form $name")
val fixture = outputDir.resolve(WorkloadContext.current.workloadName)
val specName = s"${fixture.getFileName}_$name"
val captured = spark.read.parquet(
fixture.resolve(s"expected/$specName/expected_metadata").toString)
val scanned = captured.collect().toSeq.map { row =>
val node = JsonUtil.mapper.readTree(row.getString(0))
val wrapped = if (node.has("add")) node.toString else s"""{"add":$node}"""
Action.parse(wrapped)
}
// OSS InMemoryLogReplay stores active AddFiles with dataChange=false;
// ReadCapture obtains these snapshot actions through DeltaSparkHarness.allFiles.
assert(scanned == Seq(expectedAdds(query.version.getOrElse(3L))
.copy(dataChange = false)))
}
}
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Copyright (2026) The Delta Lake Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.delta.workload.tables

import java.nio.file.{Files, Path}

import scala.jdk.CollectionConverters._

import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataType, StructType}

import io.delta.workload.{TableHandle, WorkloadContext, WorkloadTestSuite}
import io.delta.workload.engine.RowComparison
import io.delta.workload.model.Succeeded

/**
* Read and snapshot checks against literal rows and an independent schema, shared by the
* temporal statistics and partition-value workloads. Row oracles are never read from Delta.
*/
trait LiteralOracleChecks { this: WorkloadTestSuite =>

protected def outputDir: Path

/**
* Declares a read whose captured rows must equal the literal `rows`. The `all` oracle is
* also written to `oracle-parquet/<workload>` next to the output directory, for physical
* Parquet checks in later assertions.
*/
protected def checkLiteralRead(
table: TableHandle, name: String, expectedSchema: StructType, rows: Seq[Row],
predicate: String = null, version: java.lang.Long = null): Unit = {
val workload = WorkloadContext.current.workloadName
readSpec(table, predicate = predicate, version = version, name = Some(name)).assert { spec =>
spec.expectation match {
case Succeeded(result) => require(result.rowCount == rows.size)
case other => fail(s"$name must succeed against the independent oracle: $other")
}
val expected = spark.createDataFrame(rows.asJava, expectedSchema)
val captured = outputDir.resolve(workload).resolve("expected")
.resolve(s"${workload}_$name").resolve("expected_data")
RowComparison.assertRowsEqual(expected, spark.read.parquet(captured.toString), name)
if (name == "all") {
expected.coalesce(1).write.mode("overwrite")
.parquet(outputDir.resolveSibling("oracle-parquet").resolve(workload).toString)
}
println(s"Verified independent literal rows/schema: $workload/$name rows=${rows.size}")
}
}

protected def checkLiteralSnapshot(
table: TableHandle, expectedSchema: StructType, partitions: Seq[String] = Seq.empty,
version: java.lang.Long = null): Unit = {
snapshotSpec(table, version = version).assert { spec =>
spec.expectation match {
case Succeeded(result) =>
require(DataType.fromJson(result.metadata.schemaString) == expectedSchema)
require(result.metadata.partitionColumns == partitions)
require(result.metadata.format.provider == "parquet")
require(result.metadata.format.options.isEmpty)
val ntz = expectedSchema.fields.exists(_.dataType ==
org.apache.spark.sql.types.TimestampNTZType)
require(result.protocol.minReaderVersion == (if (ntz) 3 else 1))
require(result.protocol.minWriterVersion == (if (ntz) 7 else 2))
case other => fail(s"Snapshot must succeed against independent metadata: $other")
}
}
}

/** Removes version checksums, which can carry file actions from before a mutation. */
protected def removeVersionChecksums(dir: Path): Unit = {
val files = Files.list(dir.resolve("_delta_log"))
try files.iterator().asScala.filter(_.toString.endsWith(".crc"))
.foreach(Files.delete(_)) finally files.close()
}
}
Loading
Loading