serve dataflint JSON via reflective Jetty servlet on spark 4

spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUICommonLoader.scala

@@ -119,13 +119,22 @@ class DataflintSparkUICommonInstaller extends Logging {
     pageFactory.addStaticHandler(ui, "io/dataflint/spark/static/ui", ui.basePath + "/dataflint")
     val dataflintStore = new DataflintStore(store = ui.store.store)
     val tab = pageFactory.createDataFlintTab(ui)
-    tab.attachPage(pageFactory.createSQLPlanPage(ui, dataflintStore, sqlListener))
-    tab.attachPage(pageFactory.createSQLMetricsPage(ui, sqlListener))
-    tab.attachPage(pageFactory.createSQLStagesRddPage(ui))
-    tab.attachPage(pageFactory.createApplicationInfoPage(ui, dataflintStore))
-    tab.attachPage(pageFactory.createIcebergPage(ui, dataflintStore))
-    tab.attachPage(pageFactory.createDeltaLakeScanPage(ui, dataflintStore))
-    tab.attachPage(pageFactory.createCachedStoragePage(ui, dataflintStore))
+    if (pageFactory.usesReflectiveEndpoints) {
+      // Spark 4 path: bypass WebUIPage entirely and serve JSON via reflective Jetty
+      // ServletContextHandlers. Required for runtimes like Databricks Runtime 17.3
+      // that ship javax.servlet instead of jakarta.servlet — a WebUIPage subclass
+      // with jakarta-typed render/renderJson would fail JVM verification there.
+      pageFactory.attachReflectiveEndpoints(ui, dataflintStore, sqlListener)
+    } else {
+      // Spark 3 path: attach WebUIPage instances to the DataFlint tab in the usual way.
+      tab.attachPage(pageFactory.createSQLPlanPage(ui, dataflintStore, sqlListener))
+      tab.attachPage(pageFactory.createSQLMetricsPage(ui, sqlListener))
+      tab.attachPage(pageFactory.createSQLStagesRddPage(ui))
+      tab.attachPage(pageFactory.createApplicationInfoPage(ui, dataflintStore))
+      tab.attachPage(pageFactory.createIcebergPage(ui, dataflintStore))
+      tab.attachPage(pageFactory.createDeltaLakeScanPage(ui, dataflintStore))
+      tab.attachPage(pageFactory.createCachedStoragePage(ui, dataflintStore))
+    }
     ui.attachTab(tab)
     ui.webUrl
   }

spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/api/DataflintPageFactory.scala

@@ -7,28 +7,52 @@ import org.apache.spark.ui.{SparkUI, WebUIPage, WebUITab}
 /**
  * Abstract factory for creating Dataflint UI components.
  * This allows version-specific implementations for different Spark versions.
+ *
+ * Spark 3 implementations build a `DataFlintTab` and attach `WebUIPage` instances
+ * to it (see [[createApplicationInfoPage]] etc.). Spark 4 cannot use that flow
+ * because some target runtimes (e.g. Databricks Runtime 17.3) ship javax.servlet
+ * instead of jakarta.servlet, which breaks JVM verification of any class that
+ * extends WebUIPage with jakarta-typed `render`/`renderJson`. The Spark 4 path
+ * therefore overrides [[usesReflectiveEndpoints]] to `true` and supplies all
+ * endpoints through [[attachReflectiveEndpoints]] instead.
  */
 abstract class DataflintPageFactory {
-  
+
   def createDataFlintTab(ui: SparkUI): WebUITab
-  
+
   def createApplicationInfoPage(ui: SparkUI, dataflintStore: DataflintStore): WebUIPage
-  
+
   def createCachedStoragePage(ui: SparkUI, dataflintStore: DataflintStore): WebUIPage
-  
+
   def createIcebergPage(ui: SparkUI, dataflintStore: DataflintStore): WebUIPage
-  
+
   def createDeltaLakeScanPage(ui: SparkUI, dataflintStore: DataflintStore): WebUIPage
-  
+
   def createSQLMetricsPage(ui: SparkUI, sqlListener: () => Option[SQLAppStatusListener]): WebUIPage
-  
+
   def createSQLPlanPage(ui: SparkUI, dataflintStore: DataflintStore, sqlListener: () => Option[SQLAppStatusListener]): WebUIPage
-  
+
   def createSQLStagesRddPage(ui: SparkUI): WebUIPage
 
   def addStaticHandler(ui: SparkUI, resourceBase: String, contextPath: String): Unit
 
   def getTabs(ui: SparkUI): Seq[WebUITab]
 
   def isUISupported(ui: SparkUI): Boolean = true
-}
+
+  /**
+   * If true, the common loader will skip the per-page `attachPage` calls and
+   * rely entirely on [[attachReflectiveEndpoints]] to wire the JSON API. Used
+   * by Spark 4 to bypass `WebUIPage` (which is jakarta.servlet-typed and breaks
+   * on javax-only runtimes like Databricks Runtime 17.3).
+   */
+  def usesReflectiveEndpoints: Boolean = false
+
+  /**
+   * Attach DataFlint JSON endpoints reflectively. Default no-op; overridden by
+   * Spark 4 to install a Proxy-backed servlet for each endpoint.
+   */
+  def attachReflectiveEndpoints(ui: SparkUI,
+                                dataflintStore: DataflintStore,
+                                sqlListener: () => Option[SQLAppStatusListener]): Unit = ()
+}

spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataFlintTab.scala

@@ -1,16 +1,7 @@
 package org.apache.spark.dataflint.api
 
-import org.apache.spark.ui.{SparkUI, UIUtils, WebUITab}
+import org.apache.spark.ui.{SparkUI, WebUITab}
 
-import jakarta.servlet.http.HttpServletRequest
-import scala.xml.Node
-
-class DataFlintTab(parent: SparkUI) extends WebUITab(parent,"dataflint") {
+class DataFlintTab(parent: SparkUI) extends WebUITab(parent, "dataflint") {
   override val name: String = "DataFlint"
-  def render(request: HttpServletRequest): Seq[Node] = {
-    val content =
-          <div>
-          </div>
-    UIUtils.basicSparkPage(request, content, "DataFlint", true)
-  }
-}
+}

spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala

@@ -2,7 +2,6 @@ package org.apache.spark.dataflint.api
 
 import org.apache.spark.ui.SparkUI
 
-import jakarta.servlet.Servlet
 import scala.language.implicitConversions
 
 object DataflintJettyUtils {
@@ -16,7 +15,9 @@ object DataflintJettyUtils {
   // copy of createStaticHandler in core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
   // only difference is we are loading the resources from this class loader which might be different from the spark one
   // with use reflection to support both org.sparkproject.jetty.servlet and org.eclipse.jetty
-  // in spark source code
+  // in spark source code.
+  // Avoids any compile-time reference to javax.servlet or jakarta.servlet so that the
+  // same artifact can run on Databricks Runtime 17.3 (javax) and on stock Spark 4 (jakarta).
   private def createStaticHandler(resourceBase: String, path: String): Any = {
     // Try to load classes from both packages
     def getClassForName(className: String): Class[_] = {
@@ -35,9 +36,15 @@ object DataflintJettyUtils {
     val setInitParameterMethod = contextHandler.getClass.getMethod("setInitParameter", classOf[String], classOf[String])
     setInitParameterMethod.invoke(contextHandler, "org.eclipse.jetty.servlet.Default.gzip", "false")
 
-    val staticHandler = defaultServletClass.getDeclaredConstructor().newInstance()
-    val servletHolderConstructor = servletHolderClass.getConstructor(classOf[Servlet])
-    val holder = servletHolderConstructor.newInstance(staticHandler.asInstanceOf[Object])
+    val staticHandler = defaultServletClass.getDeclaredConstructor().newInstance().asInstanceOf[Object]
+    // Pick the 1-arg ServletHolder constructor whose parameter type accepts our static
+    // handler, regardless of whether the runtime expects jakarta.servlet.Servlet or
+    // javax.servlet.Servlet. Replaces a previous getConstructor(classOf[jakarta.servlet.Servlet])
+    // call that broke load on javax-only runtimes (e.g. DBR 17.3).
+    val servletHolderConstructor = servletHolderClass.getConstructors
+      .find(c => c.getParameterCount == 1 && c.getParameterTypes()(0).isInstance(staticHandler))
+      .getOrElse(sys.error(s"No 1-arg ServletHolder constructor accepting ${staticHandler.getClass.getName}"))
+    val holder = servletHolderConstructor.newInstance(staticHandler)
 
     Option(this.getClass.getClassLoader.getResource(resourceBase)) match {
       case Some(res) =>
@@ -56,4 +63,4 @@ object DataflintJettyUtils {
     contextHandler
   }
 
-}
+}