From 8e06c0cd96ec64547a4d63d955d6babbbf9609c9 Mon Sep 17 00:00:00 2001 From: soumyadeep-roy Date: Thu, 17 Sep 2026 11:47:59 +0530 Subject: [PATCH 1/2] Add config to disable Kubernetes driver pod log polling Livy fetches the driver pod's Kubernetes log (pods//log) on every poll cycle. On large clusters or when driver logs are already collected externally (e.g. via a centralized log pipeline), this is wasted load against the Kubernetes API server for log content that is never read. Add livy.server.kubernetes.driver-log-polling.enabled (default true) to skip this fetch while still polling pod state and diagnostics. Co-Authored-By: Claude Sonnet 5 --- conf/livy.conf.template | 4 +++ .../main/scala/org/apache/livy/LivyConf.scala | 7 +++++ .../livy/utils/SparkKubernetesApp.scala | 22 +++++++++++-- .../livy/utils/SparkKubernetesAppSpec.scala | 31 +++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/conf/livy.conf.template b/conf/livy.conf.template index c3fe73e86..3dd7bfad7 100644 --- a/conf/livy.conf.template +++ b/conf/livy.conf.template @@ -294,6 +294,10 @@ # per-executor diagnostics). Disable on large clusters; driver pod is polled regardless. # livy.server.kubernetes.executor-tracking.enabled = true +# Whether Livy fetches the driver pod's Kubernetes logs each poll cycle. Disable if driver +# logs are collected externally and freshness in /sessions/:id/log, /batches/:id/log is not needed. +# livy.server.kubernetes.driver-log-polling.enabled = true + # Weather to create Kubernetes Nginx Ingress for Spark UI. If set to true, configure the desired # options below # livy.server.kubernetes.ingress.create = false diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala index 73c7ec3c3..799cd05be 100644 --- a/server/src/main/scala/org/apache/livy/LivyConf.scala +++ b/server/src/main/scala/org/apache/livy/LivyConf.scala @@ -318,6 +318,13 @@ object LivyConf { val KUBERNETES_EXECUTOR_TRACKING_ENABLED = Entry("livy.server.kubernetes.executor-tracking.enabled", true) + // Whether Livy fetches the driver pod's Kubernetes logs each poll cycle. When + // disabled, Livy still polls pod state and diagnostics, but /sessions/:id/log and + // /batches/:id/log won't include fresh Kubernetes driver logs. Useful when driver + // logs are collected externally and the repeated pods//log calls are not needed. + val KUBERNETES_DRIVER_LOG_POLLING_ENABLED = + Entry("livy.server.kubernetes.driver-log-polling.enabled", true) + // How long to check livy session leakage. val KUBERNETES_APP_LEAKAGE_CHECK_TIMEOUT = Entry("livy.server.kubernetes.app-leakage.check-timeout", "600s") diff --git a/server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala b/server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala index fec93ca4e..d2beb26be 100644 --- a/server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala +++ b/server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala @@ -182,6 +182,10 @@ object SparkKubernetesApp extends Logging { info("Kubernetes executor tracking is disabled. Per-executor log URLs and " + "per-executor entries in session diagnostics will be omitted.") } + if (!livyConf.getBoolean(LivyConf.KUBERNETES_DRIVER_LOG_POLLING_ENABLED)) { + info("Kubernetes driver log polling is disabled. Livy will keep polling pod state " + + "and diagnostics, but live driver log lines will be omitted from session/batch logs.") + } leakedAppsGCThread.setDaemon(true) leakedAppsGCThread.setName("LeakedAppsGCThread") @@ -693,6 +697,18 @@ private[utils] case class KubernetesAppReport(driver: Option[Pod], executors: Se private[utils] object KubernetesExtensions { import KubernetesConstants._ + // Skips the driver pod's Kubernetes log fetch when driver log polling is disabled, + // rather than paying for the pods//log call just to discard the result. + private[utils] def resolveDriverAppLog( + livyConf: LivyConf, + fetchLog: () => IndexedSeq[String]): IndexedSeq[String] = { + if (livyConf.getBoolean(LivyConf.KUBERNETES_DRIVER_LOG_POLLING_ENABLED)) { + fetchLog() + } else { + IndexedSeq.empty + } + } + implicit class KubernetesClientExtensions(client: KubernetesClient) { import scala.collection.JavaConverters._ @@ -747,11 +763,13 @@ private[utils] object KubernetesExtensions { Seq.empty } - val appLog = Try( + // The driver pod's Kubernetes logs are used only to populate live log lines in + // session/batch responses; skip the pods//log call when disabled. + val appLog = resolveDriverAppLog(livyConf, () => Try( client.pods.inNamespace(app.getApplicationNamespace) .withName(app.getApplicationPod.getMetadata.getName) .tailingLines(cacheLogSize).getLog.split("\n").toIndexedSeq - ).getOrElse(IndexedSeq.empty) + ).getOrElse(IndexedSeq.empty)) val ingress = client.network.v1.ingresses.inNamespace(app.getApplicationNamespace) .withLabel(SPARK_APP_TAG_LABEL, app.getApplicationTag) .list.getItems.asScala.headOption diff --git a/server/src/test/scala/org/apache/livy/utils/SparkKubernetesAppSpec.scala b/server/src/test/scala/org/apache/livy/utils/SparkKubernetesAppSpec.scala index 759cbed10..b47017020 100644 --- a/server/src/test/scala/org/apache/livy/utils/SparkKubernetesAppSpec.scala +++ b/server/src/test/scala/org/apache/livy/utils/SparkKubernetesAppSpec.scala @@ -269,6 +269,37 @@ class SparkKubernetesAppSpec extends AnyFunSpec with LivyBaseUnitTestSuite with // would silently drop executor entries from session diagnostics. assert(new LivyConf(false).getBoolean(LivyConf.KUBERNETES_EXECUTOR_TRACKING_ENABLED)) } + + it("should enable driver log polling by default") { + // Preserve existing behavior unless operators explicitly disable driver log polling. + assert(new LivyConf(false).getBoolean(LivyConf.KUBERNETES_DRIVER_LOG_POLLING_ENABLED)) + } + } + + describe("resolveDriverAppLog") { + it("should skip fetching the log when driver log polling is disabled") { + var fetchCalled = false + val livyConf = new LivyConf(false) + .set(LivyConf.KUBERNETES_DRIVER_LOG_POLLING_ENABLED, false) + + val result = KubernetesExtensions.resolveDriverAppLog(livyConf, () => { + fetchCalled = true + IndexedSeq("should-not-be-returned") + }) + + assert(!fetchCalled) + assert(result.isEmpty) + } + + it("should fetch the log when driver log polling is enabled") { + val livyConf = new LivyConf(false) + .set(LivyConf.KUBERNETES_DRIVER_LOG_POLLING_ENABLED, true) + + val result = KubernetesExtensions.resolveDriverAppLog( + livyConf, () => IndexedSeq("line-1", "line-2")) + + assertResult(IndexedSeq("line-1", "line-2"))(result) + } } describe("KubernetesClientExtensions") { From fe8e84cb57f062ea9091a7b0e8fccd12e4b45aee Mon Sep 17 00:00:00 2001 From: soumyadeep-roy Date: Fri, 18 Sep 2026 21:33:07 +0530 Subject: [PATCH 2/2] Clarify driver log polling doc wording Disabling driver log polling means /sessions/:id/log and /batches/:id/log never include Kubernetes driver log lines, not merely stale ones. Update the config doc comments accordingly. --- conf/livy.conf.template | 6 ++++-- server/src/main/scala/org/apache/livy/LivyConf.scala | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/conf/livy.conf.template b/conf/livy.conf.template index 3dd7bfad7..6187c74d2 100644 --- a/conf/livy.conf.template +++ b/conf/livy.conf.template @@ -294,8 +294,10 @@ # per-executor diagnostics). Disable on large clusters; driver pod is polled regardless. # livy.server.kubernetes.executor-tracking.enabled = true -# Whether Livy fetches the driver pod's Kubernetes logs each poll cycle. Disable if driver -# logs are collected externally and freshness in /sessions/:id/log, /batches/:id/log is not needed. +# Whether Livy fetches the driver pod's Kubernetes logs each poll cycle. When disabled, +# /sessions/:id/log and /batches/:id/log will not include any Kubernetes driver log lines +# at all. Disable if driver logs are collected externally and those repeated log calls +# are not needed. # livy.server.kubernetes.driver-log-polling.enabled = true # Weather to create Kubernetes Nginx Ingress for Spark UI. If set to true, configure the desired diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala index 799cd05be..80a4e8fb8 100644 --- a/server/src/main/scala/org/apache/livy/LivyConf.scala +++ b/server/src/main/scala/org/apache/livy/LivyConf.scala @@ -320,8 +320,9 @@ object LivyConf { // Whether Livy fetches the driver pod's Kubernetes logs each poll cycle. When // disabled, Livy still polls pod state and diagnostics, but /sessions/:id/log and - // /batches/:id/log won't include fresh Kubernetes driver logs. Useful when driver - // logs are collected externally and the repeated pods//log calls are not needed. + // /batches/:id/log will not include any Kubernetes driver log lines at all. Useful + // when driver logs are collected externally and the repeated pods//log calls + // are not needed. val KUBERNETES_DRIVER_LOG_POLLING_ENABLED = Entry("livy.server.kubernetes.driver-log-polling.enabled", true)