diff --git a/core/src/main/resources/parser/photon/databricks-13_3.json b/core/src/main/resources/parser/photon/databricks-13_3.json index 35ffaab3f..437483f93 100644 --- a/core/src/main/resources/parser/photon/databricks-13_3.json +++ b/core/src/main/resources/parser/photon/databricks-13_3.json @@ -4,7 +4,8 @@ "Some entries have one-to-many mappings. For example, 'PhotonAgg' can map to either 'HashAggregate', 'SortAggregate', or 'ObjectHashAggregate'.", "Currently, only the first mapping in the list is used.", "This limitation exists because we cannot differentiate between these operators in the SparkPlan.", - "TODO: Create separate mapping file for different Photon/Databricks versions" + "TODO: Create separate mapping file for different Photon/Databricks versions", + "The entries after PhotonWindow appear in Databricks 15.4 and 17.3 Photon plans and not in 13.3 plans; the mapping is additive, so they are harmless on older runtimes." ], "PhotonAdapter": [ "Scan" @@ -102,5 +103,26 @@ "PhotonWindow": [ "Window", "RunningWindowFunction" + ], + "PhotonWriteStage": [ + "WholeStageCodegen" + ], + "PhotonParquetWriter": [ + "WriteFiles" + ], + "PhotonColumnarToRow": [ + "ColumnarToRow" + ], + "PhotonMetadataSubquery": [ + "Subquery" + ], + "PhotonRuntimeFilterSource": [ + "Subquery" + ], + "PhotonRange": [ + "Range" + ], + "PhotonJsonScan": [ + "Scan" ] } diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/PhotonPlanParserSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/PhotonPlanParserSuite.scala index 9c6da41a0..7962bbf8c 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/PhotonPlanParserSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/PhotonPlanParserSuite.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025, NVIDIA CORPORATION. + * Copyright (c) 2024-2026, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.nvidia.spark.rapids.tool.planparser import com.nvidia.spark.rapids.tool.PlatformNames +import com.nvidia.spark.rapids.tool.planparser.db.PhotonOssOpMapper import com.nvidia.spark.rapids.tool.qualification.PluginTypeChecker @@ -68,4 +69,29 @@ class PhotonPlanParserSuite extends BasePlanParserSuite { s"Failed to parse Photon operator $photonName as Spark operator $sparkName") } } + + // Photon nodes that appear in Databricks 15.4 and 17.3 plans and not in the 13.3 log above. + // Each expected value is written out, so a wrong or missing entry in the mapping file fails + // here rather than passing through the generic parser as an unsupported exec. + val photonOpTestCasesNewerRuntimes: Seq[(String, String)] = Seq( + "PhotonWriteStage" -> "WholeStageCodegen", + "PhotonParquetWriter" -> "WriteFiles", + "PhotonColumnarToRow" -> "ColumnarToRow", + "PhotonMetadataSubquery" -> "Subquery", + "PhotonRuntimeFilterSource" -> "Subquery", + "PhotonRange" -> "Range", + "PhotonJsonScan" -> "Scan" + ) + + test("Photon operators from Databricks 15.4 and 17.3 map to their Spark equivalents") { + photonOpTestCasesNewerRuntimes.foreach { case (photonName, sparkName) => + assert(PhotonOssOpMapper.mapContentToOss(photonName) == sparkName, + s"$photonName should map to $sparkName") + } + // The scan keeps its format suffix, as PhotonScan does, so the read parser sees "Scan json". + assert(PhotonOssOpMapper.mapContentToOss("PhotonJsonScan json") == "Scan json") + // A Photon node with no entry is left as it is; that is what the generic parser then reports + // as unsupported. + assert(PhotonOssOpMapper.mapContentToOss("PhotonNotARealNode") == "PhotonNotARealNode") + } }