diff --git a/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala b/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala index 53b5e1b76..399886cf8 100644 --- a/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala +++ b/server/src/main/scala/org/apache/livy/server/batch/CreateBatchRequest.scala @@ -17,6 +17,8 @@ package org.apache.livy.server.batch +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + class CreateBatchRequest { var file: String = _ @@ -27,9 +29,13 @@ class CreateBatchRequest { var pyFiles: List[String] = List() var files: List[String] = List() var driverMemory: Option[String] = None + // contentAs pins the element type past erasure so Jackson coerces numeric strings (e.g. "4"). + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var driverCores: Option[Int] = None var executorMemory: Option[String] = None + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var executorCores: Option[Int] = None + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var numExecutors: Option[Int] = None var archives: List[String] = List() var queue: Option[String] = None diff --git a/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala b/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala index af1996847..6f4ab2828 100644 --- a/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala +++ b/server/src/main/scala/org/apache/livy/server/interactive/CreateInteractiveRequest.scala @@ -17,6 +17,8 @@ package org.apache.livy.server.interactive +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + import org.apache.livy.sessions.{Kind, Shared} class CreateInteractiveRequest { @@ -26,9 +28,13 @@ class CreateInteractiveRequest { var pyFiles: List[String] = List() var files: List[String] = List() var driverMemory: Option[String] = None + // contentAs pins the element type past erasure so Jackson coerces numeric strings (e.g. "4"). + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var driverCores: Option[Int] = None var executorMemory: Option[String] = None + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var executorCores: Option[Int] = None + @JsonDeserialize(contentAs = classOf[java.lang.Integer]) var numExecutors: Option[Int] = None var archives: List[String] = List() var queue: Option[String] = None diff --git a/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala b/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala index 5ba66b1f0..c159c286b 100644 --- a/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala +++ b/server/src/test/scala/org/apache/livy/server/batch/CreateBatchRequestSpec.scala @@ -50,6 +50,59 @@ class CreateBatchRequestSpec extends AnyFunSpec with LivyBaseUnitTestSuite { assert(req.conf === Map()) } + it("should deserialize numeric fields sent as JSON numbers") { + val json = + """{ "file" : "foo", "driverCores" : 4, "executorCores" : 2, "numExecutors" : 20 }""" + val req = mapper.readValue(json, classOf[CreateBatchRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + assert(req.numExecutors === Some(20)) + } + + it("should coerce numeric fields sent as JSON strings") { + // A mistyped client that sends "4" instead of 4 must not blow up with a + // ClassCastException in BatchSession.createSparkApp -- Jackson coerces the string. + val json = + """{ "file" : "foo", "driverCores" : "4", "executorCores" : "2", "numExecutors" : "20" }""" + val req = mapper.readValue(json, classOf[CreateBatchRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + assert(req.numExecutors === Some(20)) + // The unbox that used to throw at BatchSession.scala:95 now succeeds. + assert(req.driverCores.map(_ + 1) === Some(5)) + } + + it("should reject a non-numeric string for a numeric field with a mapping error") { + val json = """{ "file" : "foo", "driverCores" : "notanumber" }""" + intercept[JsonMappingException] { + mapper.readValue(json, classOf[CreateBatchRequest]) + } + } + + it("should treat an empty string as None") { + // Jackson's scalar coercion maps "" → null, which Option deserializes as None. + val req = mapper.readValue("""{ "file" : "foo", "driverCores" : "" }""", + classOf[CreateBatchRequest]) + assert(req.driverCores === None) + } + + it("should coerce whitespace-padded numeric strings") { + val req = mapper.readValue( + """{ "file" : "foo", "driverCores" : "4 ", "executorCores" : " 2" }""", + classOf[CreateBatchRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + } + + it("should reject fractional numeric strings with a mapping error") { + Seq("""{ "file" : "foo", "driverCores" : "4.5" }""", + """{ "file" : "foo", "driverCores" : "4.0" }""").foreach { json => + intercept[JsonMappingException] { + mapper.readValue(json, classOf[CreateBatchRequest]) + } + } + } + } } diff --git a/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala b/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala index b5f6b5e98..dde291aa2 100644 --- a/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala +++ b/server/src/test/scala/org/apache/livy/server/interactive/CreateInteractiveRequestSpec.scala @@ -17,7 +17,7 @@ package org.apache.livy.server.interactive -import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.{JsonMappingException, ObjectMapper} import org.scalatest.funspec.AnyFunSpec import org.apache.livy.LivyBaseUnitTestSuite @@ -50,6 +50,58 @@ class CreateInteractiveRequestSpec extends AnyFunSpec with LivyBaseUnitTestSuite assert(req.conf === Map()) } + it("should deserialize numeric fields sent as JSON numbers") { + val json = + """{ "kind" : "pyspark", "driverCores" : 4, "executorCores" : 2, "numExecutors" : 20 }""" + val req = mapper.readValue(json, classOf[CreateInteractiveRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + assert(req.numExecutors === Some(20)) + } + + it("should coerce numeric fields sent as JSON strings") { + // Same Option[Int] erasure fix as CreateBatchRequest — string "4" must coerce to 4. + val json = + """{ "kind" : "pyspark", "driverCores" : "4", "executorCores" : "2", """ + + """"numExecutors" : "20" }""" + val req = mapper.readValue(json, classOf[CreateInteractiveRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + assert(req.numExecutors === Some(20)) + assert(req.driverCores.map(_ + 1) === Some(5)) + } + + it("should reject a non-numeric string for a numeric field with a mapping error") { + val json = """{ "kind" : "pyspark", "driverCores" : "notanumber" }""" + intercept[JsonMappingException] { + mapper.readValue(json, classOf[CreateInteractiveRequest]) + } + } + + it("should treat an empty string as None") { + // Jackson's scalar coercion maps "" → null, which Option deserializes as None. + val req = mapper.readValue("""{ "kind" : "pyspark", "driverCores" : "" }""", + classOf[CreateInteractiveRequest]) + assert(req.driverCores === None) + } + + it("should coerce whitespace-padded numeric strings") { + val req = mapper.readValue( + """{ "kind" : "pyspark", "driverCores" : "4 ", "executorCores" : " 2" }""", + classOf[CreateInteractiveRequest]) + assert(req.driverCores === Some(4)) + assert(req.executorCores === Some(2)) + } + + it("should reject fractional numeric strings with a mapping error") { + Seq("""{ "kind" : "pyspark", "driverCores" : "4.5" }""", + """{ "kind" : "pyspark", "driverCores" : "4.0" }""").foreach { json => + intercept[JsonMappingException] { + mapper.readValue(json, classOf[CreateInteractiveRequest]) + } + } + } + } }