Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.livy.server.batch

import com.fasterxml.jackson.databind.annotation.JsonDeserialize

class CreateBatchRequest {

var file: String = _
Expand All @@ -27,9 +29,13 @@ class CreateBatchRequest {
var pyFiles: List[String] = List()
var files: List[String] = List()
var driverMemory: Option[String] = None
// contentAs pins the element type past erasure so Jackson coerces numeric strings (e.g. "4").
@JsonDeserialize(contentAs = classOf[java.lang.Integer])
var driverCores: Option[Int] = None
var executorMemory: Option[String] = None
@JsonDeserialize(contentAs = classOf[java.lang.Integer])
var executorCores: Option[Int] = None
@JsonDeserialize(contentAs = classOf[java.lang.Integer])
var numExecutors: Option[Int] = None
var archives: List[String] = List()
var queue: Option[String] = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.livy.server.interactive

import com.fasterxml.jackson.databind.annotation.JsonDeserialize

import org.apache.livy.sessions.{Kind, Shared}

class CreateInteractiveRequest {
Expand All @@ -26,9 +28,13 @@ class CreateInteractiveRequest {
var pyFiles: List[String] = List()
var files: List[String] = List()
var driverMemory: Option[String] = None
// contentAs pins the element type past erasure so Jackson coerces numeric strings (e.g. "4").
@JsonDeserialize(contentAs = classOf[java.lang.Integer])
var driverCores: Option[Int] = None
var executorMemory: Option[String] = None
@JsonDeserialize(contentAs = classOf[java.lang.Integer])
var executorCores: Option[Int] = None
@JsonDeserialize(contentAs = classOf[java.lang.Integer])
var numExecutors: Option[Int] = None
var archives: List[String] = List()
var queue: Option[String] = None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,59 @@ class CreateBatchRequestSpec extends AnyFunSpec with LivyBaseUnitTestSuite {
assert(req.conf === Map())
}

it("should deserialize numeric fields sent as JSON numbers") {
val json =
"""{ "file" : "foo", "driverCores" : 4, "executorCores" : 2, "numExecutors" : 20 }"""
val req = mapper.readValue(json, classOf[CreateBatchRequest])
assert(req.driverCores === Some(4))
assert(req.executorCores === Some(2))
assert(req.numExecutors === Some(20))
}

it("should coerce numeric fields sent as JSON strings") {
// A mistyped client that sends "4" instead of 4 must not blow up with a
// ClassCastException in BatchSession.createSparkApp -- Jackson coerces the string.
val json =
"""{ "file" : "foo", "driverCores" : "4", "executorCores" : "2", "numExecutors" : "20" }"""
val req = mapper.readValue(json, classOf[CreateBatchRequest])
assert(req.driverCores === Some(4))
assert(req.executorCores === Some(2))
assert(req.numExecutors === Some(20))
// The unbox that used to throw at BatchSession.scala:95 now succeeds.
assert(req.driverCores.map(_ + 1) === Some(5))
}

it("should reject a non-numeric string for a numeric field with a mapping error") {
val json = """{ "file" : "foo", "driverCores" : "notanumber" }"""
intercept[JsonMappingException] {
mapper.readValue(json, classOf[CreateBatchRequest])
}
}

it("should treat an empty string as None") {
// Jackson's scalar coercion maps "" → null, which Option deserializes as None.
val req = mapper.readValue("""{ "file" : "foo", "driverCores" : "" }""",
classOf[CreateBatchRequest])
assert(req.driverCores === None)
}

it("should coerce whitespace-padded numeric strings") {
val req = mapper.readValue(
"""{ "file" : "foo", "driverCores" : "4 ", "executorCores" : " 2" }""",
classOf[CreateBatchRequest])
assert(req.driverCores === Some(4))
assert(req.executorCores === Some(2))
}

it("should reject fractional numeric strings with a mapping error") {
Seq("""{ "file" : "foo", "driverCores" : "4.5" }""",
"""{ "file" : "foo", "driverCores" : "4.0" }""").foreach { json =>
intercept[JsonMappingException] {
mapper.readValue(json, classOf[CreateBatchRequest])
}
}
}

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.livy.server.interactive

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.{JsonMappingException, ObjectMapper}
import org.scalatest.funspec.AnyFunSpec

import org.apache.livy.LivyBaseUnitTestSuite
Expand Down Expand Up @@ -50,6 +50,58 @@ class CreateInteractiveRequestSpec extends AnyFunSpec with LivyBaseUnitTestSuite
assert(req.conf === Map())
}

it("should deserialize numeric fields sent as JSON numbers") {
val json =
"""{ "kind" : "pyspark", "driverCores" : 4, "executorCores" : 2, "numExecutors" : 20 }"""
val req = mapper.readValue(json, classOf[CreateInteractiveRequest])
assert(req.driverCores === Some(4))
assert(req.executorCores === Some(2))
assert(req.numExecutors === Some(20))
}

it("should coerce numeric fields sent as JSON strings") {
// Same Option[Int] erasure fix as CreateBatchRequest — string "4" must coerce to 4.
val json =
"""{ "kind" : "pyspark", "driverCores" : "4", "executorCores" : "2", """ +
""""numExecutors" : "20" }"""
val req = mapper.readValue(json, classOf[CreateInteractiveRequest])
assert(req.driverCores === Some(4))
assert(req.executorCores === Some(2))
assert(req.numExecutors === Some(20))
assert(req.driverCores.map(_ + 1) === Some(5))
}

it("should reject a non-numeric string for a numeric field with a mapping error") {
val json = """{ "kind" : "pyspark", "driverCores" : "notanumber" }"""
intercept[JsonMappingException] {
mapper.readValue(json, classOf[CreateInteractiveRequest])
}
}

it("should treat an empty string as None") {
// Jackson's scalar coercion maps "" → null, which Option deserializes as None.
val req = mapper.readValue("""{ "kind" : "pyspark", "driverCores" : "" }""",
classOf[CreateInteractiveRequest])
assert(req.driverCores === None)
}

it("should coerce whitespace-padded numeric strings") {
val req = mapper.readValue(
"""{ "kind" : "pyspark", "driverCores" : "4 ", "executorCores" : " 2" }""",
classOf[CreateInteractiveRequest])
assert(req.driverCores === Some(4))
assert(req.executorCores === Some(2))
}

it("should reject fractional numeric strings with a mapping error") {
Seq("""{ "kind" : "pyspark", "driverCores" : "4.5" }""",
"""{ "kind" : "pyspark", "driverCores" : "4.0" }""").foreach { json =>
intercept[JsonMappingException] {
mapper.readValue(json, classOf[CreateInteractiveRequest])
}
}
}

}

}
Loading