From 7b2a0cd35b2e302adb8924165af3ee894b233d06 Mon Sep 17 00:00:00 2001 From: Avi Minsky Date: Wed, 6 May 2026 18:13:16 +0300 Subject: [PATCH] skip dataflint UI on databricks spark 4 to avoid jakarta.servlet crash DBR 17.3 is Spark 4 based but ships javax.servlet instead of jakarta.servlet, causing NoClassDefFoundError when DataflintJettyUtils.createStaticHandler runs and crashing the cluster on startup (issue #47). Add isUISupported on the page factory; Spark4PageFactory returns false on Databricks so the common loader skips UI installation entirely. Listeners and data export still run. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../spark/dataflint/DataflintSparkUICommonLoader.scala | 4 ++++ .../spark/dataflint/api/DataflintPageFactory.scala | 4 +++- .../apache/spark/dataflint/api/Spark4PageFactory.scala | 9 ++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala index fbca93c1..58cbdd23 100644 --- a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala +++ b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala @@ -112,6 +112,10 @@ class DataflintSparkUICommonInstaller extends Logging { logInfo("DataFlint UI is already installed, skipping installation...") return ui.webUrl } + if (!pageFactory.isUISupported(ui)) { + logWarning("DataFlint UI is not supported in this environment, skipping UI installation; listeners will still run") + return ui.webUrl + } pageFactory.addStaticHandler(ui, "io/dataflint/spark/static/ui", ui.basePath + "/dataflint") val dataflintStore = new DataflintStore(store = ui.store.store) val tab = pageFactory.createDataFlintTab(ui) diff --git a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala index f3bbc109..bc93b028 100644 --- a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala +++ b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala @@ -27,6 +27,8 @@ abstract class DataflintPageFactory { def createSQLStagesRddPage(ui: SparkUI): WebUIPage def addStaticHandler(ui: SparkUI, resourceBase: String, contextPath: String): Unit - + def getTabs(ui: SparkUI): Seq[WebUITab] + + def isUISupported(ui: SparkUI): Boolean = true } diff --git a/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/Spark4PageFactory.scala b/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/Spark4PageFactory.scala index 7f0529d0..e14a7b84 100644 --- a/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/Spark4PageFactory.scala +++ b/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/Spark4PageFactory.scala @@ -44,8 +44,15 @@ class Spark4PageFactory extends DataflintPageFactory { override def addStaticHandler(ui: SparkUI, resourceBase: String, contextPath: String): Unit = { DataflintJettyUtils.addStaticHandler(ui, resourceBase, contextPath) } - + override def getTabs(ui: SparkUI): Seq[WebUITab] = { ui.getTabs.toSeq } + + // Databricks Runtime 17.3 (Spark 4 based) ships javax.servlet instead of jakarta.servlet, + // so any access to jakarta.servlet.* in this module crashes with NoClassDefFoundError. + // Skip the entire DataFlint UI on Databricks; listeners (data export) still run. + override def isUISupported(ui: SparkUI): Boolean = { + !ui.conf.getOption("spark.databricks.clusterUsageTags.cloudProvider").isDefined + } }