From 2c64d213fbc84c95449e52df7f2b0f19c10cce5c Mon Sep 17 00:00:00 2001 From: Ahmed Hussein Date: Wed, 9 Sep 2026 11:33:16 -0500 Subject: [PATCH] Fix profiling AutoTuner RAPIDS property lookup Fixes #2142 Include RAPIDS properties in the profiling provider's all-properties map so AutoTuner source lookups see values recorded in the event log. Update the three affected expectations, which previously encoded false missing-property comments. Test profiling RAPIDS property preservation Signed-off-by: Ahmed Hussein --- .../profiling/ApplicationSummaryInfo.scala | 2 +- .../ClusterRecommendationSuite.scala | 28 +++++++++---------- .../tool/tuning/ProfilingAutoTunerSuite.scala | 4 --- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationSummaryInfo.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationSummaryInfo.scala index 0f4b7acdf..fa2820080 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationSummaryInfo.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ApplicationSummaryInfo.scala @@ -157,7 +157,7 @@ class SingleAppSummaryInfoProvider( } override def getAllProperties: Map[String, String] = { - app.sparkProps.collect { + (app.sparkProps ++ app.rapidsProps).collect { case entry: RapidsPropertyProfileResult if entry.rows(1) != null => entry }.map(r => r.key -> r.rows(1)).toMap diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ClusterRecommendationSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ClusterRecommendationSuite.scala index 7fbe7bdf3..d82570f01 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ClusterRecommendationSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/profiling/ClusterRecommendationSuite.scala @@ -252,7 +252,9 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase /** - * Test to validate the cluster shape recommendation with enforced spark properties. + * Test to validate the cluster shape recommendation with enforced and preserved spark + * properties. The event log sets concurrentGpuTasks to 4 while the calculated value is 3, + * making preservation observable in the output. * * Target Cluster YAML file: * {{{ @@ -267,9 +269,12 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase * spark.memory.offHeap.enabled: true * spark.memory.offHeap.size: 2g * spark.sql.shuffle.partitions: 400 + * preserve: + * - spark.rapids.sql.concurrentGpuTasks * }}} */ - test(s"test valid cluster shape recommendation with enforced spark properties on dataproc") { + test(s"test valid cluster shape recommendation with enforced and preserved spark properties " + + s"on dataproc") { val expectedClusterInfo = RecommendedClusterInfo( vendor = PlatformNames.DATAPROC, coresPerExecutor = 8, @@ -291,12 +296,14 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase "spark.memory.offHeap.size" -> "2g", "spark.sql.shuffle.partitions" -> "400" ) + val preservedProperty = "spark.rapids.sql.concurrentGpuTasks" TrampolineUtil.withTempDir { tempDir => val targetClusterInfoFile = ToolTestUtils.createTargetClusterInfoFile( tempDir.getAbsolutePath, driverNodeInstanceType = expectedClusterInfo.driverNodeType, workerNodeInstanceType = expectedClusterInfo.workerNodeType, - enforcedSparkProperties = expectedEnforcedSparkProperties) + enforcedSparkProperties = expectedEnforcedSparkProperties, + preserveSparkProperties = List(preservedProperty)) val appArgs = new ProfileArgs(Array( "--platform", @@ -320,7 +327,7 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase // 1. Verify the recommended cluster info assertRecommendedClusterInfo(actualClusterInfoFile, expectedClusterInfo) - // 2. Verify the enforced spark properties + // 2. Verify the target Spark properties val logFile = getOutputFilePath(tempDir, "profile.log") val profileLogContent = FSUtils.readFileContentAsUTF8(logFile) val actualResults = extractAutoTunerResults(profileLogContent) @@ -340,7 +347,7 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase |--conf spark.rapids.shuffle.multiThreaded.reader.threads=20 |--conf spark.rapids.shuffle.multiThreaded.writer.threads=20 |--conf spark.rapids.sql.batchSizeBytes=2147483647b - |--conf spark.rapids.sql.concurrentGpuTasks=3 + |--conf $preservedProperty=4 |--conf spark.rapids.sql.enabled=true |--conf spark.rapids.sql.multiThreadedRead.numThreads=40 |--conf spark.sql.adaptive.autoBroadcastJoinThreshold=[FILL_IN_VALUE] @@ -358,13 +365,9 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase |- ${getEnforcedPropertyComment("spark.executor.memory")} |- ${getEnforcedPropertyComment("spark.memory.offHeap.enabled")} |- ${getEnforcedPropertyComment("spark.memory.offHeap.size")} - |- 'spark.rapids.memory.pinnedPool.size' was not set. - |- 'spark.rapids.shuffle.multiThreaded.reader.threads' was not set. - |- 'spark.rapids.shuffle.multiThreaded.writer.threads' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. - |- 'spark.rapids.sql.concurrentGpuTasks' was not set. + |- ${getPreservedPropertyComment(preservedProperty)} |- 'spark.rapids.sql.enabled' was not set. - |- 'spark.rapids.sql.multiThreadedRead.numThreads' was not set. |- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set. |- 'spark.sql.adaptive.coalescePartitions.initialPartitionNum' was not set. |- ${getEnforcedPropertyComment("spark.sql.shuffle.partitions")} @@ -477,13 +480,8 @@ class ClusterRecommendationSuite extends ProfilingAutoTunerSuiteBase |- ${getEnforcedPropertyComment("spark.executor.cores")} |- ${getEnforcedPropertyComment("spark.executor.instances")} |- ${getEnforcedPropertyComment("spark.executor.memory")} - |- 'spark.rapids.memory.pinnedPool.size' was not set. - |- 'spark.rapids.shuffle.multiThreaded.reader.threads' was not set. - |- 'spark.rapids.shuffle.multiThreaded.writer.threads' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. - |- 'spark.rapids.sql.concurrentGpuTasks' was not set. |- 'spark.rapids.sql.enabled' was not set. - |- 'spark.rapids.sql.multiThreadedRead.numThreads' was not set. |- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set. |- 'spark.sql.adaptive.coalescePartitions.initialPartitionNum' was not set. |- $shufflePartitionsCommentForSpilling diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala index 6c067fc44..f4199e346 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala @@ -3179,12 +3179,8 @@ class ProfilingAutoTunerSuite extends ProfilingAutoTunerSuiteBase { | |Comments: |- 'spark.executor.instances' was not set. - |- 'spark.rapids.shuffle.multiThreaded.reader.threads' was not set. - |- 'spark.rapids.shuffle.multiThreaded.writer.threads' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. - |- 'spark.rapids.sql.concurrentGpuTasks' was not set. |- 'spark.rapids.sql.enabled' was not set. - |- 'spark.rapids.sql.multiThreadedRead.numThreads' was not set. |- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set. |- 'spark.sql.adaptive.coalescePartitions.initialPartitionNum' was not set. |- ${notEnoughMemCommentForKey("spark.executor.memory")}