Skip to content

Fix PCBS binary column decoding [serial-ut] [databricks] #15936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wjxiz1992 merged 8 commits into from
Sep 11, 2026
Merged

Fix PCBS binary column decoding [serial-ut] [databricks] #15936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6b48ea7
Fix binary columns in Parquet cached batches
wjxiz1992 Sep 9, 2026
63dc7d9
Fix PCBS binary cache review feedback
wjxiz1992 Sep 9, 2026
1653032
Update ColumnCastUtil copyright year
wjxiz1992 Sep 9, 2026
99eaf37
Update generated PCBS support metadata
wjxiz1992 Sep 9, 2026
81731cb
Update versioned PCBS support metadata
wjxiz1992 Sep 9, 2026
dd55be1
Merge current main into PCBS binary repair
wjxiz1992 Sep 10, 2026
4c64370
Cover binary arrays and maps in GPU cache regression
wjxiz1992 Sep 10, 2026
8039820
Simplify ownership dispatch for converted columns
wjxiz1992 Sep 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docs/supported_ops.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -674,11 +674,11 @@ plugin supports are described below.
<td>S</td>
<td>S</td>
<td>S</td>
<td>S</td>
<td><b>NS</b></td>
<td><b>NS</b></td>
<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types BINARY, CALENDAR, UDT</em></td>
<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types BINARY, CALENDAR, UDT</em></td>
<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types BINARY, CALENDAR, UDT</em></td>
<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types CALENDAR, UDT</em></td>
<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types CALENDAR, UDT</em></td>
<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types CALENDAR, UDT</em></td>
<td><b>NS</b></td>
<td>S</td>
<td>S</td>
Expand Down
36 changes: 35 additions & 1 deletion integration_tests/src/main/python/cache_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import pyspark.sql.functions as f
from spark_session import with_cpu_session, with_gpu_session
from join_test import create_df
from marks import incompat, allow_non_gpu, ignore_order, disable_ansi_mode
from marks import incompat, allow_non_gpu, ignore_order, disable_ansi_mode, inject_oom
import pyspark.mllib.linalg as mllib
import pyspark.ml.linalg as ml

Expand Down Expand Up @@ -498,3 +498,37 @@ def func(spark):
'spark.sql.adaptive.enabled': 'false'})
assert_cpu_and_gpu_are_equal_collect_with_capture(
func, exist_classes='GpuInMemoryTableScanExec', conf=dyn_conf)


@pytest.mark.skipif(not _pcbs_enabled,
reason="requires PCBS lane: "
"PYSP_TEST_spark_sql_cache_serializer=com.nvidia.spark.ParquetCachedBatchSerializer")
@allow_non_gpu('ColumnarToRowExec')
@inject_oom
@pytest.mark.parametrize('enable_vectorized_conf', enable_vectorized_confs, ids=idfn)
def test_cache_binary_on_gpu(enable_vectorized_conf):
schema = StructType([
Comment thread
thirtiseven marked this conversation as resolved.
StructField('id', IntegerType(), nullable=False),
StructField('payload', BinaryType(), nullable=True),
StructField('nested', StructType([
StructField('nested_payload', BinaryType(), nullable=True)]), nullable=False),
StructField('array_payload', ArrayType(BinaryType(), containsNull=True), nullable=True),
StructField('map_payload', MapType(IntegerType(), BinaryType(), valueContainsNull=True),
nullable=True)])
values = [
(0, bytes([1]), (bytes([0, 255]),),
[bytes([0, 255]), bytes(), None], {1: bytes([0, 255]), 2: bytes(), 3: None}),
(1, bytes(), (bytes(),), [], {}),
(2, None, (None,), None, None)]

def func(spark):
cached = spark.createDataFrame(
spark.sparkContext.parallelize(values, numSlices=1), schema).cache()
cached.count() # populate the cache so the read hits InMemoryTableScanExec
return cached

dyn_conf = copy_and_update(enable_vectorized_conf,
{'spark.rapids.sql.exec.InMemoryTableScanExec': 'true',
'spark.sql.adaptive.enabled': 'false'})
assert_cpu_and_gpu_are_equal_collect_with_capture(
func, exist_classes='GpuInMemoryTableScanExec', conf=dyn_conf)
10 changes: 4 additions & 6 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2021-2024, NVIDIA CORPORATION.
* Copyright (c) 2021-2026, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -284,11 +284,9 @@ object ColumnCastUtil {
}

withResource(new ArrayBuffer[ColumnView]) { toClose =>
val tmp = convertTypeAToTypeB(cv, dataType, predicate, toClose)
if (tmp != cv) {
tmp.copyToColumnVector()
} else {
tmp.asInstanceOf[ColumnVector].incRefCount()
convertTypeAToTypeB(cv, dataType, predicate, toClose) match {
case vector: ColumnVector => vector.incRefCount()
case view => view.copyToColumnVector()
}
}
}
Expand Down
5 changes: 3 additions & 2 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExecOverrides.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,9 @@ private[rapids] object GpuExecOverrides {
MapInPandasExecConstructorRuleMeta),
exec[InMemoryTableScanExec](
"Implementation of InMemoryTableScanExec to use GPU accelerated caching",
ExecChecks((TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.STRUCT +
TypeSig.ARRAY + TypeSig.MAP + GpuTypeShims.additionalCommonOperatorSupportedTypes)
ExecChecks((TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.BINARY + TypeSig.DECIMAL_128 +
TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP +
GpuTypeShims.additionalCommonOperatorSupportedTypes)
.nested(), TypeSig.all),
InMemoryTableScanExecConstructorRuleMeta),
neverReplaceExec[AlterNamespaceSetPropertiesExec]("Namespace metadata operation"),
Expand Down
68 changes: 37 additions & 31 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import com.nvidia.spark.rapids.{ByteBufferInputStream, ColumnCastUtil, DecimalUt
import com.nvidia.spark.rapids.Arm.withResource
import com.nvidia.spark.rapids.GpuColumnVector.GpuColumnarBatchBuilder
import com.nvidia.spark.rapids.RapidsPluginImplicits._
import com.nvidia.spark.rapids.RmmRapidsRetryIterator.withRetryNoSplit
import com.nvidia.spark.rapids.ScalableTaskCompletion.onTaskCompletion
import com.nvidia.spark.rapids.shims.{LegacyBehaviorPolicyShim, ParquetVariantShims, SparkShimImpl}
import com.nvidia.spark.rapids.shims.parquet.{ParquetFieldIdShims, ParquetLegacyNanoAsLongShims, ParquetTimestampNTZShims}
Expand Down Expand Up @@ -453,38 +454,43 @@ class ParquetCachedBatchSerializer extends GpuCachedBatchSerializer {
GpuSemaphore.acquireIfNecessary(TaskContext.get())
val parquetOptions = ParquetOptions.builder()
.includeColumn(selectedAttributes.map(_.name).asJavaCollection).build()
val table = try {
Table.readParquet(parquetOptions, parquetCB.buffer, 0, parquetCB.sizeInBytes)
} catch {
case e: Exception =>
throw new IOException("Error when processing file " +
s"[range: 0-${parquetCB.sizeInBytes}]", e)
}
withResource(table) { table =>
withResource {
for (i <- 0 until table.getNumberOfColumns) yield {
ColumnCastUtil.ifTrueThenDeepConvertTypeAtoTypeB(table.getColumn(i),
originalSelectedAttributes(i).dataType,
(dataType, _) => dataType match {
case d: DecimalType if d.scale < 0 => true
case _ => false
},
(dataType, cv) => {
dataType match {
case d: DecimalType =>
withResource(cv.bitCastTo(DecimalUtil.createCudfDecimal(d))) {
_.copyToColumnVector()
}
case _ =>
throw new IllegalStateException("We don't cast any type besides Decimal " +
"with scale < 0")
withRetryNoSplit {
val table = try {
Table.readParquet(parquetOptions, parquetCB.buffer, 0, parquetCB.sizeInBytes)
} catch {
case e: Exception =>
throw new IOException("Error when processing file " +
s"[range: 0-${parquetCB.sizeInBytes}]", e)
}
withResource(table) { table =>
withResource(new mutable.ArrayBuffer[ColumnVector]) { columns =>
for (i <- 0 until table.getNumberOfColumns) {
columns += ColumnCastUtil.ifTrueThenDeepConvertTypeAtoTypeB(table.getColumn(i),
originalSelectedAttributes(i).dataType,
(dataType, cv) => dataType match {
case BinaryType if cv.getType == DType.STRING => true
case d: DecimalType if d.scale < 0 => true
case _ => false
},
(dataType, cv) => {
dataType match {
case BinaryType =>
ParquetSchemaUtils.convertStringToBinary(cv)
case d: DecimalType =>
withResource(cv.bitCastTo(DecimalUtil.createCudfDecimal(d))) {
_.copyToColumnVector()
}
case _ =>
throw new IllegalStateException("We only cast STRING-backed BinaryType " +
"and DecimalType with scale < 0")
}
}
}
)
}
} { col =>
withResource(new Table(col: _*)) { t =>
GpuColumnVector.from(t, originalSelectedAttributes.map(_.dataType).toArray)
)
}
withResource(new Table(columns.toArray: _*)) { convertedTable =>
GpuColumnVector.from(convertedTable,
originalSelectedAttributes.map(_.dataType).toArray)
}
}
}
}
Expand Down
31 changes: 16 additions & 15 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetSchemaUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import java.util.{Locale, Optional}

import scala.collection.JavaConverters._

import ai.rapids.cudf.{ColumnView, DType, Table}
import ai.rapids.cudf.{ColumnVector, ColumnView, DType, Table}
import com.nvidia.spark.rapids.{CastOptions, GpuCast, GpuColumnVector, SchemaUtils}
import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource}
import com.nvidia.spark.rapids.shims.parquet.ParquetSchemaClipShims
Expand Down Expand Up @@ -685,6 +685,20 @@ object ParquetSchemaUtils {
}
}

private[parquet] def convertStringToBinary(cv: ColumnView): ColumnVector = {
// Ideally we would bitCast the STRING to a LIST, but that does not work.
// Instead, pull apart the string and put it back together as a list.
val dataBuf = Option(cv.getData)
withResource(new ColumnView(DType.UINT8, dataBuf.map(_.getLength).getOrElse(0),
Optional.of(0L), dataBuf.orNull, null)) { data =>
withResource(new ColumnView(DType.LIST, cv.getRowCount,
Optional.of[java.lang.Long](cv.getNullCount),
cv.getValid, cv.getOffsets, Array(data))) { everything =>
everything.copyToColumnVector()
}
}
}

// Wrap up all required casts for Parquet schema evolution
//
// Note: The behavior of unsigned to signed is decided by the Spark,
Expand All @@ -705,20 +719,7 @@ object ParquetSchemaUtils {
needUpcast(cv, dt)) {
cv.castTo(GpuColumnVector.getNonNestedRapidsType(dt))
} else if (DType.STRING.equals(cv.getType) && dt == BinaryType) {
// Ideally we would bitCast the STRING to a LIST, but that does not work.
// Instead, we are going to have to pull apart the string and put it back together
// as a list.

val dataBuf = Option(cv.getData)
withResource(new ColumnView(DType.UINT8, dataBuf.map(_.getLength).getOrElse(0),
Optional.of(0L),
dataBuf.orNull, null)) { data =>
withResource(new ColumnView(DType.LIST, cv.getRowCount,
Optional.of[java.lang.Long](cv.getNullCount),
cv.getValid, cv.getOffsets, Array(data))) { everything =>
everything.copyToColumnVector()
}
}
convertStringToBinary(cv)
} else {
throw new IllegalStateException("Logical error: no valid casts are found " +
s"${cv.getType} to $dt")
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/320/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/321/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/321cdh/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/322/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/323/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/324/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/330/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S

@greptile-apps greptile-apps Bot Sep 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Versioned Support Metadata Is Stale

The common GpuExecOverrides code now supports top-level and nested binary columns for InMemoryTableScanExec, but only the generic and Spark 3.3.0 capability files were updated. Other supported version snapshots, including Spark 3.4.0 and 4.2.0, still report binary columns as unsupported. This non-blocking inconsistency gives documentation and capability consumers conflicting support information, so the remaining versioned metadata should be regenerated.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

@wjxiz1992 wjxiz1992 Sep 10, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The remaining versioned snapshots were regenerated in 81731cb. I verified that all 37 versioned snapshots and the generic snapshot now report BINARY=S for InMemoryTableScanExec.

DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/330cdh/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,S,S
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/331/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/332/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S
Expand Down
2 changes: 1 addition & 1 deletion tools/generated_files/332cdh/supportedExecs.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS
SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,S,S
InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S
DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S
ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S
AppendDataExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S
Expand Down
Loading
Loading