Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion core/src/main/resources/parser/photon/databricks-13_3.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
"Some entries have one-to-many mappings. For example, 'PhotonAgg' can map to either 'HashAggregate', 'SortAggregate', or 'ObjectHashAggregate'.",
"Currently, only the first mapping in the list is used.",
"This limitation exists because we cannot differentiate between these operators in the SparkPlan.",
"TODO: Create separate mapping file for different Photon/Databricks versions"
"TODO: Create separate mapping file for different Photon/Databricks versions",
"The entries after PhotonWindow appear in Databricks 15.4 and 17.3 Photon plans and not in 13.3 plans; the mapping is additive, so they are harmless on older runtimes."
],
"PhotonAdapter": [
"Scan"
Expand Down Expand Up @@ -102,5 +103,26 @@
"PhotonWindow": [
"Window",
"RunningWindowFunction"
],
"PhotonWriteStage": [
"WholeStageCodegen"
],
"PhotonParquetWriter": [
"WriteFiles"
],
"PhotonColumnarToRow": [
"ColumnarToRow"
],
"PhotonMetadataSubquery": [
"Subquery"
],
"PhotonRuntimeFilterSource": [
"Subquery"
],
"PhotonRange": [
"Range"
],
"PhotonJsonScan": [
"Scan"
]
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2024-2025, NVIDIA CORPORATION.
* Copyright (c) 2024-2026, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -17,6 +17,7 @@
package com.nvidia.spark.rapids.tool.planparser

import com.nvidia.spark.rapids.tool.PlatformNames
import com.nvidia.spark.rapids.tool.planparser.db.PhotonOssOpMapper
import com.nvidia.spark.rapids.tool.qualification.PluginTypeChecker


Expand Down Expand Up @@ -68,4 +69,29 @@ class PhotonPlanParserSuite extends BasePlanParserSuite {
s"Failed to parse Photon operator $photonName as Spark operator $sparkName")
}
}

// Photon nodes that appear in Databricks 15.4 and 17.3 plans and not in the 13.3 log above.
// Each expected value is written out, so a wrong or missing entry in the mapping file fails
// here rather than passing through the generic parser as an unsupported exec.
val photonOpTestCasesNewerRuntimes: Seq[(String, String)] = Seq(
"PhotonWriteStage" -> "WholeStageCodegen",
"PhotonParquetWriter" -> "WriteFiles",
"PhotonColumnarToRow" -> "ColumnarToRow",
"PhotonMetadataSubquery" -> "Subquery",
"PhotonRuntimeFilterSource" -> "Subquery",
"PhotonRange" -> "Range",
"PhotonJsonScan" -> "Scan"
)

test("Photon operators from Databricks 15.4 and 17.3 map to their Spark equivalents") {
photonOpTestCasesNewerRuntimes.foreach { case (photonName, sparkName) =>
assert(PhotonOssOpMapper.mapContentToOss(photonName) == sparkName,
s"$photonName should map to $sparkName")
}
// The scan keeps its format suffix, as PhotonScan does, so the read parser sees "Scan json".
assert(PhotonOssOpMapper.mapContentToOss("PhotonJsonScan json") == "Scan json")
// A Photon node with no entry is left as it is; that is what the generic parser then reports
// as unsupported.
assert(PhotonOssOpMapper.mapContentToOss("PhotonNotARealNode") == "PhotonNotARealNode")
}
}
Loading