diff --git a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala index fbca93c..58cbdd2 100644 --- a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala +++ b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala @@ -112,6 +112,10 @@ class DataflintSparkUICommonInstaller extends Logging { logInfo("DataFlint UI is already installed, skipping installation...") return ui.webUrl } + if (!pageFactory.isUISupported(ui)) { + logWarning("DataFlint UI is not supported in this environment, skipping UI installation; listeners will still run") + return ui.webUrl + } pageFactory.addStaticHandler(ui, "io/dataflint/spark/static/ui", ui.basePath + "/dataflint") val dataflintStore = new DataflintStore(store = ui.store.store) val tab = pageFactory.createDataFlintTab(ui) diff --git a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala index f3bbc10..bc93b02 100644 --- a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala +++ b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala @@ -27,6 +27,8 @@ abstract class DataflintPageFactory { def createSQLStagesRddPage(ui: SparkUI): WebUIPage def addStaticHandler(ui: SparkUI, resourceBase: String, contextPath: String): Unit - + def getTabs(ui: SparkUI): Seq[WebUITab] + + def isUISupported(ui: SparkUI): Boolean = true } diff --git a/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/Spark4PageFactory.scala b/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/Spark4PageFactory.scala index 7f0529d..e14a7b8 100644 --- a/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/Spark4PageFactory.scala +++ b/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/Spark4PageFactory.scala @@ -44,8 +44,15 @@ class Spark4PageFactory extends DataflintPageFactory { override def addStaticHandler(ui: SparkUI, resourceBase: String, contextPath: String): Unit = { DataflintJettyUtils.addStaticHandler(ui, resourceBase, contextPath) } - + override def getTabs(ui: SparkUI): Seq[WebUITab] = { ui.getTabs.toSeq } + + // Databricks Runtime 17.3 (Spark 4 based) ships javax.servlet instead of jakarta.servlet, + // so any access to jakarta.servlet.* in this module crashes with NoClassDefFoundError. + // Skip the entire DataFlint UI on Databricks; listeners (data export) still run. + override def isUISupported(ui: SparkUI): Boolean = { + !ui.conf.getOption("spark.databricks.clusterUsageTags.cloudProvider").isDefined + } }