From 6f678d72c9c39c4f658a0c1aee0834c8347b59ca Mon Sep 17 00:00:00 2001 From: Thomas Wynne Date: Fri, 11 Sep 2026 12:30:50 -0500 Subject: [PATCH] Map the Photon nodes Databricks 15.4 and 17.3 added to their Spark equivalents The Photon-to-Spark mapping is one file, parser/photon/databricks-13_3.json, loaded for every runtime. A Photon name with no entry falls through the generic parser as an unsupported exec. On a 17.3 Delta MERGE log seven of the 21 Photon node names have no entry and are reported Unsupported, Triage in 39 of 121 SQLs, the write pair under an Execute WriteIntoDeltaCommand the tools already recognise. Seven additive entries: PhotonWriteStage to WholeStageCodegen like the other stage wrappers, PhotonParquetWriter to WriteFiles, PhotonMetadataSubquery and PhotonRuntimeFilterSource to Subquery (removed from the estimate, as GenerateBloomFilter is on EMR), PhotonColumnarToRow to ColumnarToRow, PhotonRange to Range, PhotonJsonScan to Scan. A 13.3 plan never contains these names, so the 13.3 fixture and its expectations are unchanged, and #1384 stays for a mapping that diverges between runtimes. Test: PhotonPlanParserSuite pins each entry through PhotonOssOpMapper.mapContentToOss with the expected name written out. It fails on dev and passes with the change (2 succeeded), scalastyle clean. On the 17.3 log the seven names leave the unsupported report (71 rows to 0), the unsupported stage share moves from 49.26% to 28.45% and the estimate from 1.05x to 1.15x. Fixes #2158 Signed-off-by: Thomas Wynne --- .../parser/photon/databricks-13_3.json | 24 +++++++++++++++- .../planparser/PhotonPlanParserSuite.scala | 28 ++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/core/src/main/resources/parser/photon/databricks-13_3.json b/core/src/main/resources/parser/photon/databricks-13_3.json index 35ffaab3f..437483f93 100644 --- a/core/src/main/resources/parser/photon/databricks-13_3.json +++ b/core/src/main/resources/parser/photon/databricks-13_3.json @@ -4,7 +4,8 @@ "Some entries have one-to-many mappings. For example, 'PhotonAgg' can map to either 'HashAggregate', 'SortAggregate', or 'ObjectHashAggregate'.", "Currently, only the first mapping in the list is used.", "This limitation exists because we cannot differentiate between these operators in the SparkPlan.", - "TODO: Create separate mapping file for different Photon/Databricks versions" + "TODO: Create separate mapping file for different Photon/Databricks versions", + "The entries after PhotonWindow appear in Databricks 15.4 and 17.3 Photon plans and not in 13.3 plans; the mapping is additive, so they are harmless on older runtimes." ], "PhotonAdapter": [ "Scan" @@ -102,5 +103,26 @@ "PhotonWindow": [ "Window", "RunningWindowFunction" + ], + "PhotonWriteStage": [ + "WholeStageCodegen" + ], + "PhotonParquetWriter": [ + "WriteFiles" + ], + "PhotonColumnarToRow": [ + "ColumnarToRow" + ], + "PhotonMetadataSubquery": [ + "Subquery" + ], + "PhotonRuntimeFilterSource": [ + "Subquery" + ], + "PhotonRange": [ + "Range" + ], + "PhotonJsonScan": [ + "Scan" ] } diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/PhotonPlanParserSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/PhotonPlanParserSuite.scala index 9c6da41a0..7962bbf8c 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/PhotonPlanParserSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/PhotonPlanParserSuite.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025, NVIDIA CORPORATION. + * Copyright (c) 2024-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.nvidia.spark.rapids.tool.planparser import com.nvidia.spark.rapids.tool.PlatformNames +import com.nvidia.spark.rapids.tool.planparser.db.PhotonOssOpMapper import com.nvidia.spark.rapids.tool.qualification.PluginTypeChecker @@ -68,4 +69,29 @@ class PhotonPlanParserSuite extends BasePlanParserSuite { s"Failed to parse Photon operator $photonName as Spark operator $sparkName") } } + + // Photon nodes that appear in Databricks 15.4 and 17.3 plans and not in the 13.3 log above. + // Each expected value is written out, so a wrong or missing entry in the mapping file fails + // here rather than passing through the generic parser as an unsupported exec. + val photonOpTestCasesNewerRuntimes: Seq[(String, String)] = Seq( + "PhotonWriteStage" -> "WholeStageCodegen", + "PhotonParquetWriter" -> "WriteFiles", + "PhotonColumnarToRow" -> "ColumnarToRow", + "PhotonMetadataSubquery" -> "Subquery", + "PhotonRuntimeFilterSource" -> "Subquery", + "PhotonRange" -> "Range", + "PhotonJsonScan" -> "Scan" + ) + + test("Photon operators from Databricks 15.4 and 17.3 map to their Spark equivalents") { + photonOpTestCasesNewerRuntimes.foreach { case (photonName, sparkName) => + assert(PhotonOssOpMapper.mapContentToOss(photonName) == sparkName, + s"$photonName should map to $sparkName") + } + // The scan keeps its format suffix, as PhotonScan does, so the read parser sees "Scan json". + assert(PhotonOssOpMapper.mapContentToOss("PhotonJsonScan json") == "Scan json") + // A Photon node with no entry is left as it is; that is what the generic parser then reports + // as unsupported. + assert(PhotonOssOpMapper.mapContentToOss("PhotonNotARealNode") == "PhotonNotARealNode") + } }