From 6699fa60778132740b5af879a8682eb69797c2bc Mon Sep 17 00:00:00 2001 From: soumyadeep-roy Date: Thu, 17 Sep 2026 11:58:44 +0530 Subject: [PATCH 1/2] Coerce batch/session Int fields sent as JSON strings driverCores, executorCores and numExecutors are Option[Int], but Scala type erasure means jackson-module-scala stores a JSON string such as "4" verbatim as Some("4") without error. The bad value only surfaces later as a ClassCastException during unboxing in BatchSession/InteractiveSession app creation, which the servlet returns to callers as an opaque HTTP 500. Annotate the three fields on CreateBatchRequest and CreateInteractiveRequest with @JsonDeserialize(contentAs = classOf[Integer]) so Jackson coerces numeric strings to Int and rejects truly non-numeric strings with a 400 instead of a 500. Co-Authored-By: Claude Sonnet 5 --- .../server/batch/CreateBatchRequest.scala | 6 +++ .../CreateInteractiveRequest.scala | 6 +++ .../server/batch/CreateBatchRequestSpec.scala | 53 +++++++++++++++++++ .../CreateInteractiveRequestSpec.scala | 53 ++++++++++++++++++- 4 files changed, 117 insertions(+), 1 deletion(-) diff --git a/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala b/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala index 53b5e1b76..399886cf8 100644 --- a/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala +++ b/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala @@ -17,6 +17,8 @@ package org.apache.livy.server.batch +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + class CreateBatchRequest { var file: String = _ @@ -27,9 +29,13 @@ class CreateBatchRequest { var pyFiles: List[String] = List() var files: List[String] = List() var driverMemory: Option[String] = None + // contentAs pins the element type past erasure so Jackson coerces numeric strings (e.g. "4"). + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var driverCores: Option[Int] = None var executorMemory: Option[String] = None + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var executorCores: Option[Int] = None + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var numExecutors: Option[Int] = None var archives: List[String] = List() var queue: Option[String] = None diff --git a/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala b/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala index af1996847..6f4ab2828 100644 --- a/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala +++ b/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala @@ -17,6 +17,8 @@ package org.apache.livy.server.interactive +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + import org.apache.livy.sessions.{Kind, Shared} class CreateInteractiveRequest { @@ -26,9 +28,13 @@ class CreateInteractiveRequest { var pyFiles: List[String] = List() var files: List[String] = List() var driverMemory: Option[String] = None + // contentAs pins the element type past erasure so Jackson coerces numeric strings (e.g. "4"). + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var driverCores: Option[Int] = None var executorMemory: Option[String] = None + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var executorCores: Option[Int] = None + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var numExecutors: Option[Int] = None var archives: List[String] = List() var queue: Option[String] = None diff --git a/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala b/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala index 5ba66b1f0..c159c286b 100644 --- a/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala +++ b/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala @@ -50,6 +50,59 @@ class CreateBatchRequestSpec extends AnyFunSpec with LivyBaseUnitTestSuite { assert(req.conf === Map()) } + it("should deserialize numeric fields sent as JSON numbers") { + val json = + """{ "file" : "foo", "driverCores" : 4, "executorCores" : 2, "numExecutors" : 20 }""" + val req = mapper.readValue(json, classOf[CreateBatchRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + assert(req.numExecutors === Some(20)) + } + + it("should coerce numeric fields sent as JSON strings") { + // A mistyped client that sends "4" instead of 4 must not blow up with a + // ClassCastException in BatchSession.createSparkApp -- Jackson coerces the string. + val json = + """{ "file" : "foo", "driverCores" : "4", "executorCores" : "2", "numExecutors" : "20" }""" + val req = mapper.readValue(json, classOf[CreateBatchRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + assert(req.numExecutors === Some(20)) + // The unbox that used to throw at BatchSession.scala:95 now succeeds. + assert(req.driverCores.map(_ + 1) === Some(5)) + } + + it("should reject a non-numeric string for a numeric field with a mapping error") { + val json = """{ "file" : "foo", "driverCores" : "notanumber" }""" + intercept[JsonMappingException] { + mapper.readValue(json, classOf[CreateBatchRequest]) + } + } + + it("should treat an empty string as None") { + // Jackson's scalar coercion maps "" → null, which Option deserializes as None. + val req = mapper.readValue("""{ "file" : "foo", "driverCores" : "" }""", + classOf[CreateBatchRequest]) + assert(req.driverCores === None) + } + + it("should coerce whitespace-padded numeric strings") { + val req = mapper.readValue( + """{ "file" : "foo", "driverCores" : "4 ", "executorCores" : " 2" }""", + classOf[CreateBatchRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + } + + it("should reject fractional numeric strings with a mapping error") { + Seq("""{ "file" : "foo", "driverCores" : "4.5" }""", + """{ "file" : "foo", "driverCores" : "4.0" }""").foreach { json => + intercept[JsonMappingException] { + mapper.readValue(json, classOf[CreateBatchRequest]) + } + } + } + } } diff --git a/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala index b5f6b5e98..a49362829 100644 --- a/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala +++ b/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala @@ -17,7 +17,7 @@ package org.apache.livy.server.interactive -import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.{JsonMappingException, ObjectMapper} import org.scalatest.funspec.AnyFunSpec import org.apache.livy.LivyBaseUnitTestSuite @@ -50,6 +50,57 @@ class CreateInteractiveRequestSpec extends AnyFunSpec with LivyBaseUnitTestSuite assert(req.conf === Map()) } + it("should deserialize numeric fields sent as JSON numbers") { + val json = + """{ "kind" : "pyspark", "driverCores" : 4, "executorCores" : 2, "numExecutors" : 20 }""" + val req = mapper.readValue(json, classOf[CreateInteractiveRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + assert(req.numExecutors === Some(20)) + } + + it("should coerce numeric fields sent as JSON strings") { + // Same Option[Int] erasure fix as CreateBatchRequest — string "4" must coerce to 4. + val json = + """{ "kind" : "pyspark", "driverCores" : "4", "executorCores" : "2", "numExecutors" : "20" }""" + val req = mapper.readValue(json, classOf[CreateInteractiveRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + assert(req.numExecutors === Some(20)) + assert(req.driverCores.map(_ + 1) === Some(5)) + } + + it("should reject a non-numeric string for a numeric field with a mapping error") { + val json = """{ "kind" : "pyspark", "driverCores" : "notanumber" }""" + intercept[JsonMappingException] { + mapper.readValue(json, classOf[CreateInteractiveRequest]) + } + } + + it("should treat an empty string as None") { + // Jackson's scalar coercion maps "" → null, which Option deserializes as None. + val req = mapper.readValue("""{ "kind" : "pyspark", "driverCores" : "" }""", + classOf[CreateInteractiveRequest]) + assert(req.driverCores === None) + } + + it("should coerce whitespace-padded numeric strings") { + val req = mapper.readValue( + """{ "kind" : "pyspark", "driverCores" : "4 ", "executorCores" : " 2" }""", + classOf[CreateInteractiveRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + } + + it("should reject fractional numeric strings with a mapping error") { + Seq("""{ "kind" : "pyspark", "driverCores" : "4.5" }""", + """{ "kind" : "pyspark", "driverCores" : "4.0" }""").foreach { json => + intercept[JsonMappingException] { + mapper.readValue(json, classOf[CreateInteractiveRequest]) + } + } + } + } } From 015c29645c37df46ee817757e683520f94372b57 Mon Sep 17 00:00:00 2001 From: soumyadeep-roy Date: Fri, 18 Sep 2026 12:07:51 +0530 Subject: [PATCH 2/2] [LIVY-1083] Fix scalastyle line-length violation in CreateInteractiveRequestSpec CI failed on a 103-character line introduced by the JSON-string coercion test. Split the literal across two concatenated strings to fit under the 100-character limit. Co-Authored-By: Claude Sonnet 5 --- .../livy/server/interactive/CreateInteractiveRequestSpec.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala index a49362829..dde291aa2 100644 --- a/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala +++ b/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala @@ -62,7 +62,8 @@ class CreateInteractiveRequestSpec extends AnyFunSpec with LivyBaseUnitTestSuite it("should coerce numeric fields sent as JSON strings") { // Same Option[Int] erasure fix as CreateBatchRequest — string "4" must coerce to 4. val json = - """{ "kind" : "pyspark", "driverCores" : "4", "executorCores" : "2", "numExecutors" : "20" }""" + """{ "kind" : "pyspark", "driverCores" : "4", "executorCores" : "2", """ + + """"numExecutors" : "20" }""" val req = mapper.readValue(json, classOf[CreateInteractiveRequest]) assert(req.driverCores === Some(4)) assert(req.executorCores === Some(2))