From 6b48ea76c74f0fac1f5c9d9625bdb5d8400a30e2 Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Wed, 9 Sep 2026 16:00:02 +0800 Subject: [PATCH 1/7] Fix binary columns in Parquet cached batches Signed-off-by: Allen Xu --- .../ParquetCachedBatchSerializer.scala | 9 ++- .../rapids/parquet/ParquetSchemaUtils.scala | 31 +++++----- .../spark/rapids/CachedBatchWriterSuite.scala | 57 +++++++++++++++++++ 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala index 99fa5d39099..2d12014b734 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala @@ -465,19 +465,22 @@ class ParquetCachedBatchSerializer extends GpuCachedBatchSerializer { for (i <- 0 until table.getNumberOfColumns) yield { ColumnCastUtil.ifTrueThenDeepConvertTypeAtoTypeB(table.getColumn(i), originalSelectedAttributes(i).dataType, - (dataType, _) => dataType match { + (dataType, cv) => dataType match { + case BinaryType if cv.getType == DType.STRING => true case d: DecimalType if d.scale < 0 => true case _ => false }, (dataType, cv) => { dataType match { + case BinaryType => + ParquetSchemaUtils.convertStringToBinary(cv) case d: DecimalType => withResource(cv.bitCastTo(DecimalUtil.createCudfDecimal(d))) { _.copyToColumnVector() } case _ => - throw new IllegalStateException("We don't cast any type besides Decimal " + - "with scale < 0") + throw new IllegalStateException("We only cast STRING-backed BinaryType " + + "and DecimalType with scale < 0") } } ) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetSchemaUtils.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetSchemaUtils.scala index 83a7c45113f..5abadfa3192 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetSchemaUtils.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetSchemaUtils.scala @@ -20,7 +20,7 @@ import java.util.{Locale, Optional} import scala.collection.JavaConverters._ -import ai.rapids.cudf.{ColumnView, DType, Table} +import ai.rapids.cudf.{ColumnVector, ColumnView, DType, Table} import com.nvidia.spark.rapids.{CastOptions, GpuCast, GpuColumnVector, SchemaUtils} import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource} import com.nvidia.spark.rapids.shims.parquet.ParquetSchemaClipShims @@ -685,6 +685,20 @@ object ParquetSchemaUtils { } } + private[parquet] def convertStringToBinary(cv: ColumnView): ColumnVector = { + // Ideally we would bitCast the STRING to a LIST, but that does not work. + // Instead, pull apart the string and put it back together as a list. + val dataBuf = Option(cv.getData) + withResource(new ColumnView(DType.UINT8, dataBuf.map(_.getLength).getOrElse(0), + Optional.of(0L), dataBuf.orNull, null)) { data => + withResource(new ColumnView(DType.LIST, cv.getRowCount, + Optional.of[java.lang.Long](cv.getNullCount), + cv.getValid, cv.getOffsets, Array(data))) { everything => + everything.copyToColumnVector() + } + } + } + // Wrap up all required casts for Parquet schema evolution // // Note: The behavior of unsigned to signed is decided by the Spark, @@ -705,20 +719,7 @@ object ParquetSchemaUtils { needUpcast(cv, dt)) { cv.castTo(GpuColumnVector.getNonNestedRapidsType(dt)) } else if (DType.STRING.equals(cv.getType) && dt == BinaryType) { - // Ideally we would bitCast the STRING to a LIST, but that does not work. - // Instead, we are going to have to pull apart the string and put it back together - // as a list. - - val dataBuf = Option(cv.getData) - withResource(new ColumnView(DType.UINT8, dataBuf.map(_.getLength).getOrElse(0), - Optional.of(0L), - dataBuf.orNull, null)) { data => - withResource(new ColumnView(DType.LIST, cv.getRowCount, - Optional.of[java.lang.Long](cv.getNullCount), - cv.getValid, cv.getOffsets, Array(data))) { everything => - everything.copyToColumnVector() - } - } + convertStringToBinary(cv) } else { throw new IllegalStateException("Logical error: no valid casts are found " + s"${cv.getType} to $dt") diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/CachedBatchWriterSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/CachedBatchWriterSuite.scala index 2b2bbca0903..f5d1973910b 100644 --- a/tests/src/test/scala/com/nvidia/spark/rapids/CachedBatchWriterSuite.scala +++ b/tests/src/test/scala/com/nvidia/spark/rapids/CachedBatchWriterSuite.scala @@ -226,6 +226,63 @@ class CachedBatchWriterSuite extends SparkQueryCompareTestSuite { } } + test("PCBS round trip preserves nested and top-level BinaryType columns") { + withGpuSparkSession { spark => + setActiveSession(spark) + val ser = new ParquetCachedBatchSerializer + val nestedType = StructType(Array( + StructField("nested_payload", BinaryType, nullable = true))) + val origAttrs: Seq[Attribute] = Seq( + AttributeReference("payload", BinaryType, nullable = true)(), + AttributeReference("nested", nestedType, nullable = false)()) + val origSchema = origAttrs.toStructType + val cachedSchema = PCBSSchemaHelper.getSupportedSchemaFromUnsupported(origAttrs).toStructType + val values = Seq( + (Array[Byte](1), Array[Byte](0, -1)), + (Array.emptyByteArray, Array.emptyByteArray), + (null.asInstanceOf[Array[Byte]], null.asInstanceOf[Array[Byte]])) + val rows = values.map { case (payload, nestedPayload) => + InternalRow(payload, InternalRow(nestedPayload)) + }.toArray + + val converter = new GpuRowToColumnConverter(origSchema) + val listOfPCB = withResource(converter.convertBatch(rows, origSchema)) { gpuCB => + ser.compressColumnarBatchWithParquet(gpuCB, cachedSchema, origSchema, + BYTES_ALLOWED_PER_BATCH, useCompression = false) + } + + val conf = TrampolineUtil.getSparkConf(spark) + val cachedRdd = spark.sparkContext.parallelize[CachedBatch](listOfPCB, numSlices = 1) + val cbRdd = ser.convertCachedBatchToColumnarBatch(cachedRdd, origAttrs, origAttrs, conf) + val context = new MockTaskContext(taskAttemptId = 1, partitionId = 0) + TrampolineUtil.setTaskContext(context) + try { + val batches = cbRdd.compute(cbRdd.partitions.head, context) + assert(batches.hasNext) + val cb = batches.next() + assert(cb.numRows() == rows.length) + values.zipWithIndex.foreach { case ((payload, nestedPayload), rowIndex) => + if (payload == null) { + assert(cb.column(0).isNullAt(rowIndex)) + } else { + assert(cb.column(0).getBinary(rowIndex).sameElements(payload)) + } + + val nested = cb.column(1).getStruct(rowIndex) + if (nestedPayload == null) { + assert(nested.isNullAt(0)) + } else { + assert(nested.getBinary(0).sameElements(nestedPayload)) + } + } + assert(!batches.hasNext) + } finally { + TrampolineUtil.unsetTaskContext() + context.markTaskComplete() + } + } + } + private def writeAndConsumeEmptyBatch(spark: SparkSession): Unit = { setActiveSession(spark) val schema = Seq(AttributeReference("_col0", IntegerType, true)()) From 63dc7d95edffe3ba3d9c4c826e277f0e7ef8b8c9 Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Wed, 9 Sep 2026 16:48:33 +0800 Subject: [PATCH 2/7] Fix PCBS binary cache review feedback Signed-off-by: Allen Xu --- .../src/main/python/cache_test.py | 32 ++++++++- .../nvidia/spark/rapids/ColumnCastUtil.scala | 5 +- .../spark/rapids/GpuExecOverrides.scala | 5 +- .../ParquetCachedBatchSerializer.scala | 71 ++++++++++--------- .../spark/rapids/CachedBatchWriterSuite.scala | 57 --------------- 5 files changed, 75 insertions(+), 95 deletions(-) diff --git a/integration_tests/src/main/python/cache_test.py b/integration_tests/src/main/python/cache_test.py index 05c5379651b..1f765fc215f 100644 --- a/integration_tests/src/main/python/cache_test.py +++ b/integration_tests/src/main/python/cache_test.py @@ -23,7 +23,7 @@ import pyspark.sql.functions as f from spark_session import with_cpu_session, with_gpu_session from join_test import create_df -from marks import incompat, allow_non_gpu, ignore_order, disable_ansi_mode +from marks import incompat, allow_non_gpu, ignore_order, disable_ansi_mode, inject_oom import pyspark.mllib.linalg as mllib import pyspark.ml.linalg as ml @@ -498,3 +498,33 @@ def func(spark): 'spark.sql.adaptive.enabled': 'false'}) assert_cpu_and_gpu_are_equal_collect_with_capture( func, exist_classes='GpuInMemoryTableScanExec', conf=dyn_conf) + + +@pytest.mark.skipif(not _pcbs_enabled, + reason="requires PCBS lane: " + "PYSP_TEST_spark_sql_cache_serializer=com.nvidia.spark.ParquetCachedBatchSerializer") +@allow_non_gpu('ColumnarToRowExec') +@inject_oom +@pytest.mark.parametrize('enable_vectorized_conf', enable_vectorized_confs, ids=idfn) +def test_cache_binary_on_gpu(enable_vectorized_conf): + schema = StructType([ + StructField('id', IntegerType(), nullable=False), + StructField('payload', BinaryType(), nullable=True), + StructField('nested', StructType([ + StructField('nested_payload', BinaryType(), nullable=True)]), nullable=False)]) + values = [ + (0, bytes([1]), (bytes([0, 255]),)), + (1, bytes(), (bytes(),)), + (2, None, (None,))] + + def func(spark): + cached = spark.createDataFrame( + spark.sparkContext.parallelize(values, numSlices=1), schema).cache() + cached.count() # populate the cache so the read hits InMemoryTableScanExec + return cached + + dyn_conf = copy_and_update(enable_vectorized_conf, + {'spark.rapids.sql.exec.InMemoryTableScanExec': 'true', + 'spark.sql.adaptive.enabled': 'false'}) + assert_cpu_and_gpu_are_equal_collect_with_capture( + func, exist_classes='GpuInMemoryTableScanExec', conf=dyn_conf) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala index af4bc53ab4c..cd47c858bd0 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala @@ -286,7 +286,10 @@ object ColumnCastUtil { withResource(new ArrayBuffer[ColumnView]) { toClose => val tmp = convertTypeAToTypeB(cv, dataType, predicate, toClose) if (tmp != cv) { - tmp.copyToColumnVector() + tmp match { + case vector: ColumnVector => vector.incRefCount() + case _ => tmp.copyToColumnVector() + } } else { tmp.asInstanceOf[ColumnVector].incRefCount() } diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExecOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExecOverrides.scala index 1ae61943511..a6bbefc433f 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExecOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuExecOverrides.scala @@ -279,8 +279,9 @@ private[rapids] object GpuExecOverrides { MapInPandasExecConstructorRuleMeta), exec[InMemoryTableScanExec]( "Implementation of InMemoryTableScanExec to use GPU accelerated caching", - ExecChecks((TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.STRUCT + - TypeSig.ARRAY + TypeSig.MAP + GpuTypeShims.additionalCommonOperatorSupportedTypes) + ExecChecks((TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.BINARY + TypeSig.DECIMAL_128 + + TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP + + GpuTypeShims.additionalCommonOperatorSupportedTypes) .nested(), TypeSig.all), InMemoryTableScanExecConstructorRuleMeta), neverReplaceExec[AlterNamespaceSetPropertiesExec]("Namespace metadata operation"), diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala index 2d12014b734..2aef74cd997 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/parquet/ParquetCachedBatchSerializer.scala @@ -30,6 +30,7 @@ import com.nvidia.spark.rapids.{ByteBufferInputStream, ColumnCastUtil, DecimalUt import com.nvidia.spark.rapids.Arm.withResource import com.nvidia.spark.rapids.GpuColumnVector.GpuColumnarBatchBuilder import com.nvidia.spark.rapids.RapidsPluginImplicits._ +import com.nvidia.spark.rapids.RmmRapidsRetryIterator.withRetryNoSplit import com.nvidia.spark.rapids.ScalableTaskCompletion.onTaskCompletion import com.nvidia.spark.rapids.shims.{LegacyBehaviorPolicyShim, ParquetVariantShims, SparkShimImpl} import com.nvidia.spark.rapids.shims.parquet.{ParquetFieldIdShims, ParquetLegacyNanoAsLongShims, ParquetTimestampNTZShims} @@ -453,41 +454,43 @@ class ParquetCachedBatchSerializer extends GpuCachedBatchSerializer { GpuSemaphore.acquireIfNecessary(TaskContext.get()) val parquetOptions = ParquetOptions.builder() .includeColumn(selectedAttributes.map(_.name).asJavaCollection).build() - val table = try { - Table.readParquet(parquetOptions, parquetCB.buffer, 0, parquetCB.sizeInBytes) - } catch { - case e: Exception => - throw new IOException("Error when processing file " + - s"[range: 0-${parquetCB.sizeInBytes}]", e) - } - withResource(table) { table => - withResource { - for (i <- 0 until table.getNumberOfColumns) yield { - ColumnCastUtil.ifTrueThenDeepConvertTypeAtoTypeB(table.getColumn(i), - originalSelectedAttributes(i).dataType, - (dataType, cv) => dataType match { - case BinaryType if cv.getType == DType.STRING => true - case d: DecimalType if d.scale < 0 => true - case _ => false - }, - (dataType, cv) => { - dataType match { - case BinaryType => - ParquetSchemaUtils.convertStringToBinary(cv) - case d: DecimalType => - withResource(cv.bitCastTo(DecimalUtil.createCudfDecimal(d))) { - _.copyToColumnVector() - } - case _ => - throw new IllegalStateException("We only cast STRING-backed BinaryType " + - "and DecimalType with scale < 0") + withRetryNoSplit { + val table = try { + Table.readParquet(parquetOptions, parquetCB.buffer, 0, parquetCB.sizeInBytes) + } catch { + case e: Exception => + throw new IOException("Error when processing file " + + s"[range: 0-${parquetCB.sizeInBytes}]", e) + } + withResource(table) { table => + withResource(new mutable.ArrayBuffer[ColumnVector]) { columns => + for (i <- 0 until table.getNumberOfColumns) { + columns += ColumnCastUtil.ifTrueThenDeepConvertTypeAtoTypeB(table.getColumn(i), + originalSelectedAttributes(i).dataType, + (dataType, cv) => dataType match { + case BinaryType if cv.getType == DType.STRING => true + case d: DecimalType if d.scale < 0 => true + case _ => false + }, + (dataType, cv) => { + dataType match { + case BinaryType => + ParquetSchemaUtils.convertStringToBinary(cv) + case d: DecimalType => + withResource(cv.bitCastTo(DecimalUtil.createCudfDecimal(d))) { + _.copyToColumnVector() + } + case _ => + throw new IllegalStateException("We only cast STRING-backed BinaryType " + + "and DecimalType with scale < 0") + } } - } - ) - } - } { col => - withResource(new Table(col: _*)) { t => - GpuColumnVector.from(t, originalSelectedAttributes.map(_.dataType).toArray) + ) + } + withResource(new Table(columns.toArray: _*)) { convertedTable => + GpuColumnVector.from(convertedTable, + originalSelectedAttributes.map(_.dataType).toArray) + } } } } diff --git a/tests/src/test/scala/com/nvidia/spark/rapids/CachedBatchWriterSuite.scala b/tests/src/test/scala/com/nvidia/spark/rapids/CachedBatchWriterSuite.scala index f5d1973910b..2b2bbca0903 100644 --- a/tests/src/test/scala/com/nvidia/spark/rapids/CachedBatchWriterSuite.scala +++ b/tests/src/test/scala/com/nvidia/spark/rapids/CachedBatchWriterSuite.scala @@ -226,63 +226,6 @@ class CachedBatchWriterSuite extends SparkQueryCompareTestSuite { } } - test("PCBS round trip preserves nested and top-level BinaryType columns") { - withGpuSparkSession { spark => - setActiveSession(spark) - val ser = new ParquetCachedBatchSerializer - val nestedType = StructType(Array( - StructField("nested_payload", BinaryType, nullable = true))) - val origAttrs: Seq[Attribute] = Seq( - AttributeReference("payload", BinaryType, nullable = true)(), - AttributeReference("nested", nestedType, nullable = false)()) - val origSchema = origAttrs.toStructType - val cachedSchema = PCBSSchemaHelper.getSupportedSchemaFromUnsupported(origAttrs).toStructType - val values = Seq( - (Array[Byte](1), Array[Byte](0, -1)), - (Array.emptyByteArray, Array.emptyByteArray), - (null.asInstanceOf[Array[Byte]], null.asInstanceOf[Array[Byte]])) - val rows = values.map { case (payload, nestedPayload) => - InternalRow(payload, InternalRow(nestedPayload)) - }.toArray - - val converter = new GpuRowToColumnConverter(origSchema) - val listOfPCB = withResource(converter.convertBatch(rows, origSchema)) { gpuCB => - ser.compressColumnarBatchWithParquet(gpuCB, cachedSchema, origSchema, - BYTES_ALLOWED_PER_BATCH, useCompression = false) - } - - val conf = TrampolineUtil.getSparkConf(spark) - val cachedRdd = spark.sparkContext.parallelize[CachedBatch](listOfPCB, numSlices = 1) - val cbRdd = ser.convertCachedBatchToColumnarBatch(cachedRdd, origAttrs, origAttrs, conf) - val context = new MockTaskContext(taskAttemptId = 1, partitionId = 0) - TrampolineUtil.setTaskContext(context) - try { - val batches = cbRdd.compute(cbRdd.partitions.head, context) - assert(batches.hasNext) - val cb = batches.next() - assert(cb.numRows() == rows.length) - values.zipWithIndex.foreach { case ((payload, nestedPayload), rowIndex) => - if (payload == null) { - assert(cb.column(0).isNullAt(rowIndex)) - } else { - assert(cb.column(0).getBinary(rowIndex).sameElements(payload)) - } - - val nested = cb.column(1).getStruct(rowIndex) - if (nestedPayload == null) { - assert(nested.isNullAt(0)) - } else { - assert(nested.getBinary(0).sameElements(nestedPayload)) - } - } - assert(!batches.hasNext) - } finally { - TrampolineUtil.unsetTaskContext() - context.markTaskComplete() - } - } - } - private def writeAndConsumeEmptyBatch(spark: SparkSession): Unit = { setActiveSession(spark) val schema = Seq(AttributeReference("_col0", IntegerType, true)()) From 16530323f60c231e98b364adf396ae8cb08449b4 Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Wed, 9 Sep 2026 17:12:37 +0800 Subject: [PATCH 3/7] Update ColumnCastUtil copyright year Signed-off-by: Allen Xu --- .../src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala index cd47c858bd0..a5d9881e2be 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. + * Copyright (c) 2021-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 99eaf374710548a11fec064454ece862396b332e Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Wed, 9 Sep 2026 18:27:36 +0800 Subject: [PATCH 4/7] Update generated PCBS support metadata Signed-off-by: Allen Xu --- docs/supported_ops.md | 8 ++++---- tools/generated_files/330/supportedExecs.csv | 2 +- tools/generated_files/supportedExecs.csv | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/supported_ops.md b/docs/supported_ops.md index 3600eaf387b..4f8c43bcc87 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -674,11 +674,11 @@ plugin supports are described below. S S S +S NS -NS -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT -PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types BINARY, CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT +PS
UTC is only supported TZ for child TIMESTAMP;
unsupported child types CALENDAR, UDT NS S S diff --git a/tools/generated_files/330/supportedExecs.csv b/tools/generated_files/330/supportedExecs.csv index 22aed1730ba..910530df141 100644 --- a/tools/generated_files/330/supportedExecs.csv +++ b/tools/generated_files/330/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S diff --git a/tools/generated_files/supportedExecs.csv b/tools/generated_files/supportedExecs.csv index 22aed1730ba..910530df141 100644 --- a/tools/generated_files/supportedExecs.csv +++ b/tools/generated_files/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S From 81731cb7f3232ab43ae6caae389807c51521f657 Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Wed, 9 Sep 2026 18:44:16 +0800 Subject: [PATCH 5/7] Update versioned PCBS support metadata Signed-off-by: Allen Xu --- tools/generated_files/320/supportedExecs.csv | 2 +- tools/generated_files/321/supportedExecs.csv | 2 +- tools/generated_files/321cdh/supportedExecs.csv | 2 +- tools/generated_files/322/supportedExecs.csv | 2 +- tools/generated_files/323/supportedExecs.csv | 2 +- tools/generated_files/324/supportedExecs.csv | 2 +- tools/generated_files/330cdh/supportedExecs.csv | 2 +- tools/generated_files/331/supportedExecs.csv | 2 +- tools/generated_files/332/supportedExecs.csv | 2 +- tools/generated_files/332cdh/supportedExecs.csv | 2 +- tools/generated_files/333/supportedExecs.csv | 2 +- tools/generated_files/334/supportedExecs.csv | 2 +- tools/generated_files/340/supportedExecs.csv | 2 +- tools/generated_files/341/supportedExecs.csv | 2 +- tools/generated_files/342/supportedExecs.csv | 2 +- tools/generated_files/343/supportedExecs.csv | 2 +- tools/generated_files/344/supportedExecs.csv | 2 +- tools/generated_files/350/supportedExecs.csv | 2 +- tools/generated_files/351/supportedExecs.csv | 2 +- tools/generated_files/352/supportedExecs.csv | 2 +- tools/generated_files/353/supportedExecs.csv | 2 +- tools/generated_files/354/supportedExecs.csv | 2 +- tools/generated_files/355/supportedExecs.csv | 2 +- tools/generated_files/356/supportedExecs.csv | 2 +- tools/generated_files/357/supportedExecs.csv | 2 +- tools/generated_files/358/supportedExecs.csv | 2 +- tools/generated_files/359/supportedExecs.csv | 2 +- tools/generated_files/400/supportedExecs.csv | 2 +- tools/generated_files/401/supportedExecs.csv | 2 +- tools/generated_files/402/supportedExecs.csv | 2 +- tools/generated_files/403/supportedExecs.csv | 2 +- tools/generated_files/404/supportedExecs.csv | 2 +- tools/generated_files/411/supportedExecs.csv | 2 +- tools/generated_files/412/supportedExecs.csv | 2 +- tools/generated_files/413/supportedExecs.csv | 2 +- tools/generated_files/420/supportedExecs.csv | 2 +- 36 files changed, 36 insertions(+), 36 deletions(-) diff --git a/tools/generated_files/320/supportedExecs.csv b/tools/generated_files/320/supportedExecs.csv index 317fb6c5ca3..836fb4d4ee3 100644 --- a/tools/generated_files/320/supportedExecs.csv +++ b/tools/generated_files/320/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS diff --git a/tools/generated_files/321/supportedExecs.csv b/tools/generated_files/321/supportedExecs.csv index 317fb6c5ca3..836fb4d4ee3 100644 --- a/tools/generated_files/321/supportedExecs.csv +++ b/tools/generated_files/321/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS diff --git a/tools/generated_files/321cdh/supportedExecs.csv b/tools/generated_files/321cdh/supportedExecs.csv index 317fb6c5ca3..836fb4d4ee3 100644 --- a/tools/generated_files/321cdh/supportedExecs.csv +++ b/tools/generated_files/321cdh/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS diff --git a/tools/generated_files/322/supportedExecs.csv b/tools/generated_files/322/supportedExecs.csv index 317fb6c5ca3..836fb4d4ee3 100644 --- a/tools/generated_files/322/supportedExecs.csv +++ b/tools/generated_files/322/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS diff --git a/tools/generated_files/323/supportedExecs.csv b/tools/generated_files/323/supportedExecs.csv index 317fb6c5ca3..836fb4d4ee3 100644 --- a/tools/generated_files/323/supportedExecs.csv +++ b/tools/generated_files/323/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS diff --git a/tools/generated_files/324/supportedExecs.csv b/tools/generated_files/324/supportedExecs.csv index 317fb6c5ca3..836fb4d4ee3 100644 --- a/tools/generated_files/324/supportedExecs.csv +++ b/tools/generated_files/324/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,NS,NS +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,NS,NS ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,NS,NS diff --git a/tools/generated_files/330cdh/supportedExecs.csv b/tools/generated_files/330cdh/supportedExecs.csv index 6e1cc5e8f9b..0535b2df5d9 100644 --- a/tools/generated_files/330cdh/supportedExecs.csv +++ b/tools/generated_files/330cdh/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S diff --git a/tools/generated_files/331/supportedExecs.csv b/tools/generated_files/331/supportedExecs.csv index 22aed1730ba..910530df141 100644 --- a/tools/generated_files/331/supportedExecs.csv +++ b/tools/generated_files/331/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S diff --git a/tools/generated_files/332/supportedExecs.csv b/tools/generated_files/332/supportedExecs.csv index 22aed1730ba..910530df141 100644 --- a/tools/generated_files/332/supportedExecs.csv +++ b/tools/generated_files/332/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S diff --git a/tools/generated_files/332cdh/supportedExecs.csv b/tools/generated_files/332cdh/supportedExecs.csv index 6e1cc5e8f9b..0535b2df5d9 100644 --- a/tools/generated_files/332cdh/supportedExecs.csv +++ b/tools/generated_files/332cdh/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S diff --git a/tools/generated_files/333/supportedExecs.csv b/tools/generated_files/333/supportedExecs.csv index 22aed1730ba..910530df141 100644 --- a/tools/generated_files/333/supportedExecs.csv +++ b/tools/generated_files/333/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S diff --git a/tools/generated_files/334/supportedExecs.csv b/tools/generated_files/334/supportedExecs.csv index 22aed1730ba..910530df141 100644 --- a/tools/generated_files/334/supportedExecs.csv +++ b/tools/generated_files/334/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S AppendDataExecV1,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,NS,S,NS,PS,PS,PS,NS,S,S diff --git a/tools/generated_files/340/supportedExecs.csv b/tools/generated_files/340/supportedExecs.csv index 55f852f7cf2..1656702a4df 100644 --- a/tools/generated_files/340/supportedExecs.csv +++ b/tools/generated_files/340/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/341/supportedExecs.csv b/tools/generated_files/341/supportedExecs.csv index 55f852f7cf2..1656702a4df 100644 --- a/tools/generated_files/341/supportedExecs.csv +++ b/tools/generated_files/341/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/342/supportedExecs.csv b/tools/generated_files/342/supportedExecs.csv index 55f852f7cf2..1656702a4df 100644 --- a/tools/generated_files/342/supportedExecs.csv +++ b/tools/generated_files/342/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/343/supportedExecs.csv b/tools/generated_files/343/supportedExecs.csv index 55f852f7cf2..1656702a4df 100644 --- a/tools/generated_files/343/supportedExecs.csv +++ b/tools/generated_files/343/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/344/supportedExecs.csv b/tools/generated_files/344/supportedExecs.csv index 55f852f7cf2..1656702a4df 100644 --- a/tools/generated_files/344/supportedExecs.csv +++ b/tools/generated_files/344/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/350/supportedExecs.csv b/tools/generated_files/350/supportedExecs.csv index 4ad5747add3..d5ae278f305 100644 --- a/tools/generated_files/350/supportedExecs.csv +++ b/tools/generated_files/350/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,NS,This is disabled by default because there could be complications when using it with AQE with Spark-3.5.0 and Spark-3.5.1. For more details please check https://github.com/NVIDIA/spark-rapids/issues/10603,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,NS,This is disabled by default because there could be complications when using it with AQE with Spark-3.5.0 and Spark-3.5.1. For more details please check https://github.com/NVIDIA/spark-rapids/issues/10603,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/351/supportedExecs.csv b/tools/generated_files/351/supportedExecs.csv index 4ad5747add3..d5ae278f305 100644 --- a/tools/generated_files/351/supportedExecs.csv +++ b/tools/generated_files/351/supportedExecs.csv @@ -18,7 +18,7 @@ AQEShuffleReadExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,NS,This is disabled by default because there could be complications when using it with AQE with Spark-3.5.0 and Spark-3.5.1. For more details please check https://github.com/NVIDIA/spark-rapids/issues/10603,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,NS,This is disabled by default because there could be complications when using it with AQE with Spark-3.5.0 and Spark-3.5.1. For more details please check https://github.com/NVIDIA/spark-rapids/issues/10603,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/352/supportedExecs.csv b/tools/generated_files/352/supportedExecs.csv index a11a1bf209b..00a6fd90905 100644 --- a/tools/generated_files/352/supportedExecs.csv +++ b/tools/generated_files/352/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/353/supportedExecs.csv b/tools/generated_files/353/supportedExecs.csv index a11a1bf209b..00a6fd90905 100644 --- a/tools/generated_files/353/supportedExecs.csv +++ b/tools/generated_files/353/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/354/supportedExecs.csv b/tools/generated_files/354/supportedExecs.csv index a11a1bf209b..00a6fd90905 100644 --- a/tools/generated_files/354/supportedExecs.csv +++ b/tools/generated_files/354/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/355/supportedExecs.csv b/tools/generated_files/355/supportedExecs.csv index a11a1bf209b..00a6fd90905 100644 --- a/tools/generated_files/355/supportedExecs.csv +++ b/tools/generated_files/355/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/356/supportedExecs.csv b/tools/generated_files/356/supportedExecs.csv index a11a1bf209b..00a6fd90905 100644 --- a/tools/generated_files/356/supportedExecs.csv +++ b/tools/generated_files/356/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/357/supportedExecs.csv b/tools/generated_files/357/supportedExecs.csv index a11a1bf209b..00a6fd90905 100644 --- a/tools/generated_files/357/supportedExecs.csv +++ b/tools/generated_files/357/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/358/supportedExecs.csv b/tools/generated_files/358/supportedExecs.csv index a11a1bf209b..00a6fd90905 100644 --- a/tools/generated_files/358/supportedExecs.csv +++ b/tools/generated_files/358/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/359/supportedExecs.csv b/tools/generated_files/359/supportedExecs.csv index a11a1bf209b..00a6fd90905 100644 --- a/tools/generated_files/359/supportedExecs.csv +++ b/tools/generated_files/359/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/400/supportedExecs.csv b/tools/generated_files/400/supportedExecs.csv index d4229adae6f..3ddc9e5b454 100644 --- a/tools/generated_files/400/supportedExecs.csv +++ b/tools/generated_files/400/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/401/supportedExecs.csv b/tools/generated_files/401/supportedExecs.csv index d4229adae6f..3ddc9e5b454 100644 --- a/tools/generated_files/401/supportedExecs.csv +++ b/tools/generated_files/401/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/402/supportedExecs.csv b/tools/generated_files/402/supportedExecs.csv index d4229adae6f..3ddc9e5b454 100644 --- a/tools/generated_files/402/supportedExecs.csv +++ b/tools/generated_files/402/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/403/supportedExecs.csv b/tools/generated_files/403/supportedExecs.csv index d4229adae6f..3ddc9e5b454 100644 --- a/tools/generated_files/403/supportedExecs.csv +++ b/tools/generated_files/403/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/404/supportedExecs.csv b/tools/generated_files/404/supportedExecs.csv index d4229adae6f..3ddc9e5b454 100644 --- a/tools/generated_files/404/supportedExecs.csv +++ b/tools/generated_files/404/supportedExecs.csv @@ -19,7 +19,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/411/supportedExecs.csv b/tools/generated_files/411/supportedExecs.csv index cfc76cc02c1..553780610fd 100644 --- a/tools/generated_files/411/supportedExecs.csv +++ b/tools/generated_files/411/supportedExecs.csv @@ -20,7 +20,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/412/supportedExecs.csv b/tools/generated_files/412/supportedExecs.csv index cfc76cc02c1..553780610fd 100644 --- a/tools/generated_files/412/supportedExecs.csv +++ b/tools/generated_files/412/supportedExecs.csv @@ -20,7 +20,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/413/supportedExecs.csv b/tools/generated_files/413/supportedExecs.csv index cfc76cc02c1..553780610fd 100644 --- a/tools/generated_files/413/supportedExecs.csv +++ b/tools/generated_files/413/supportedExecs.csv @@ -20,7 +20,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/tools/generated_files/420/supportedExecs.csv b/tools/generated_files/420/supportedExecs.csv index d1acffc1841..f09056bdd79 100644 --- a/tools/generated_files/420/supportedExecs.csv +++ b/tools/generated_files/420/supportedExecs.csv @@ -20,7 +20,7 @@ TableCacheQueryStageExec,NS,This is disabled by default because Table cache quer HashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ObjectHashAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,PS,NS,PS,PS,PS,NS,NS,NS SortAggregateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS -InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,S,S +InMemoryTableScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S DataWritingCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,PS,NS,S,NS,PS,PS,PS,NS,S,S ExecutedCommandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S WriteFilesExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S From 4c64370049a8e07f25af57c68fd20e0769fb6571 Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Thu, 10 Sep 2026 15:16:45 +0800 Subject: [PATCH 6/7] Cover binary arrays and maps in GPU cache regression Signed-off-by: Allen Xu --- integration_tests/src/main/python/cache_test.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/integration_tests/src/main/python/cache_test.py b/integration_tests/src/main/python/cache_test.py index 1f765fc215f..95bf4d44595 100644 --- a/integration_tests/src/main/python/cache_test.py +++ b/integration_tests/src/main/python/cache_test.py @@ -511,11 +511,15 @@ def test_cache_binary_on_gpu(enable_vectorized_conf): StructField('id', IntegerType(), nullable=False), StructField('payload', BinaryType(), nullable=True), StructField('nested', StructType([ - StructField('nested_payload', BinaryType(), nullable=True)]), nullable=False)]) + StructField('nested_payload', BinaryType(), nullable=True)]), nullable=False), + StructField('array_payload', ArrayType(BinaryType(), containsNull=True), nullable=True), + StructField('map_payload', MapType(IntegerType(), BinaryType(), valueContainsNull=True), + nullable=True)]) values = [ - (0, bytes([1]), (bytes([0, 255]),)), - (1, bytes(), (bytes(),)), - (2, None, (None,))] + (0, bytes([1]), (bytes([0, 255]),), + [bytes([0, 255]), bytes(), None], {1: bytes([0, 255]), 2: bytes(), 3: None}), + (1, bytes(), (bytes(),), [], {}), + (2, None, (None,), None, None)] def func(spark): cached = spark.createDataFrame( From 8039820e19b7c8dea68bd4dfa7abbec0511d6b8d Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Thu, 10 Sep 2026 16:29:43 +0800 Subject: [PATCH 7/7] Simplify ownership dispatch for converted columns Signed-off-by: Allen Xu --- .../com/nvidia/spark/rapids/ColumnCastUtil.scala | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala index a5d9881e2be..bbf92218406 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnCastUtil.scala @@ -284,14 +284,9 @@ object ColumnCastUtil { } withResource(new ArrayBuffer[ColumnView]) { toClose => - val tmp = convertTypeAToTypeB(cv, dataType, predicate, toClose) - if (tmp != cv) { - tmp match { - case vector: ColumnVector => vector.incRefCount() - case _ => tmp.copyToColumnVector() - } - } else { - tmp.asInstanceOf[ColumnVector].incRefCount() + convertTypeAToTypeB(cv, dataType, predicate, toClose) match { + case vector: ColumnVector => vector.incRefCount() + case view => view.copyToColumnVector() } } }