From 44fd5d3921299f93d1aab7fe971078a6bce835a2 Mon Sep 17 00:00:00 2001 From: German Schiavon Date: Tue, 21 Nov 2017 16:32:04 +0100 Subject: [PATCH 1/5] Check submission request parameters --- .../spark/deploy/rest/SubmitRestProtocolRequest.scala | 2 ++ .../spark/deploy/rest/SubmitRestProtocolSuite.scala | 2 ++ .../apache/spark/deploy/rest/mesos/MesosRestServer.scala | 8 ++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolRequest.scala b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolRequest.scala index 0d50a768942ed..86ddf954ca128 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolRequest.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolRequest.scala @@ -46,6 +46,8 @@ private[rest] class CreateSubmissionRequest extends SubmitRestProtocolRequest { super.doValidate() assert(sparkProperties != null, "No Spark properties set!") assertFieldIsSet(appResource, "appResource") + assertFieldIsSet(appArgs, "appArgs") + assertFieldIsSet(environmentVariables, "environmentVariables") assertPropertyIsSet("spark.app.name") assertPropertyIsBoolean("spark.driver.supervise") assertPropertyIsNumeric("spark.driver.cores") diff --git a/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala b/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala index 725b8848bc052..34758b4fcf5e9 100644 --- a/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala @@ -86,6 +86,8 @@ class SubmitRestProtocolSuite extends SparkFunSuite { message.clientSparkVersion = "1.2.3" message.appResource = "honey-walnut-cherry.jar" message.mainClass = "org.apache.spark.examples.SparkPie" + message.appArgs = Array("hdfs://tmp/auth") + message.environmentVariables = Map("SPARK_SCALA_VERSION" -> "2.11") val conf = new SparkConf(false) conf.set("spark.app.name", "SparkPie") message.sparkProperties = conf.getAll.toMap diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala index ff60b88c6d533..1adc91ae73b91 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala @@ -82,6 +82,12 @@ private[mesos] class MesosSubmitRequestServlet( val mainClass = Option(request.mainClass).getOrElse { throw new SubmitRestMissingFieldException("Main class is missing.") } + val appArgs = Option(request.appArgs).getOrElse { + throw new SubmitRestMissingFieldException("Application arguments are missing.") + } + val environmentVariables = Option(request.environmentVariables).getOrElse { + throw new SubmitRestMissingFieldException("Environment variables are missing.") + } // Optional fields val sparkProperties = request.sparkProperties @@ -91,8 +97,6 @@ private[mesos] class MesosSubmitRequestServlet( val superviseDriver = sparkProperties.get("spark.driver.supervise") val driverMemory = sparkProperties.get("spark.driver.memory") val driverCores = sparkProperties.get("spark.driver.cores") - val appArgs = request.appArgs - val environmentVariables = request.environmentVariables val name = request.sparkProperties.getOrElse("spark.app.name", mainClass) // Construct driver description From 14d64172500483f9e984ac28ba5f3b52db33ad9e Mon Sep 17 00:00:00 2001 From: German Schiavon Date: Mon, 27 Nov 2017 09:47:41 +0100 Subject: [PATCH 2/5] change test env var name --- .../org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala b/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala index 34758b4fcf5e9..e871a9e6f51fd 100644 --- a/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala @@ -87,7 +87,7 @@ class SubmitRestProtocolSuite extends SparkFunSuite { message.appResource = "honey-walnut-cherry.jar" message.mainClass = "org.apache.spark.examples.SparkPie" message.appArgs = Array("hdfs://tmp/auth") - message.environmentVariables = Map("SPARK_SCALA_VERSION" -> "2.11") + message.environmentVariables = Map("SPARK_HOME" -> "/test") val conf = new SparkConf(false) conf.set("spark.app.name", "SparkPie") message.sparkProperties = conf.getAll.toMap From 57d52c4917b8cd08e8e73bce9729f8a59afa6ffd Mon Sep 17 00:00:00 2001 From: German Schiavon Date: Tue, 28 Nov 2017 08:20:53 +0100 Subject: [PATCH 3/5] added clearer exception message --- .../apache/spark/deploy/rest/mesos/MesosRestServer.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala index 1adc91ae73b91..bb8dfee165183 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala @@ -77,16 +77,16 @@ private[mesos] class MesosSubmitRequestServlet( private def buildDriverDescription(request: CreateSubmissionRequest): MesosDriverDescription = { // Required fields, including the main class because python is not yet supported val appResource = Option(request.appResource).getOrElse { - throw new SubmitRestMissingFieldException("Application jar is missing.") + throw new SubmitRestMissingFieldException("Application jar 'appResource' is missing.") } val mainClass = Option(request.mainClass).getOrElse { - throw new SubmitRestMissingFieldException("Main class is missing.") + throw new SubmitRestMissingFieldException("Main class 'mainClass' is missing.") } val appArgs = Option(request.appArgs).getOrElse { - throw new SubmitRestMissingFieldException("Application arguments are missing.") + throw new SubmitRestMissingFieldException("Application arguments 'appArgs' are missing.") } val environmentVariables = Option(request.environmentVariables).getOrElse { - throw new SubmitRestMissingFieldException("Environment variables are missing.") + throw new SubmitRestMissingFieldException("Environment variables 'environmentVariables' are missing.") } // Optional fields From ba221bd412c8bec2c6b7f16719556c17b819f2a7 Mon Sep 17 00:00:00 2001 From: German Schiavon Date: Tue, 28 Nov 2017 08:47:42 +0100 Subject: [PATCH 4/5] change test variables names --- .../apache/spark/deploy/rest/SubmitRestProtocolSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala b/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala index e871a9e6f51fd..75c50af23c66a 100644 --- a/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/rest/SubmitRestProtocolSuite.scala @@ -86,8 +86,8 @@ class SubmitRestProtocolSuite extends SparkFunSuite { message.clientSparkVersion = "1.2.3" message.appResource = "honey-walnut-cherry.jar" message.mainClass = "org.apache.spark.examples.SparkPie" - message.appArgs = Array("hdfs://tmp/auth") - message.environmentVariables = Map("SPARK_HOME" -> "/test") + message.appArgs = Array("two slices") + message.environmentVariables = Map("PATH" -> "/dev/null") val conf = new SparkConf(false) conf.set("spark.app.name", "SparkPie") message.sparkProperties = conf.getAll.toMap From 9989471842193962fb373eb415423ed732b7fe0c Mon Sep 17 00:00:00 2001 From: German Schiavon Date: Wed, 13 Dec 2017 08:48:14 +0100 Subject: [PATCH 5/5] Fixed Scalastyle --- .../org/apache/spark/deploy/rest/mesos/MesosRestServer.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala index bb8dfee165183..68f6921153d89 100644 --- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala +++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/rest/mesos/MesosRestServer.scala @@ -86,7 +86,8 @@ private[mesos] class MesosSubmitRequestServlet( throw new SubmitRestMissingFieldException("Application arguments 'appArgs' are missing.") } val environmentVariables = Option(request.environmentVariables).getOrElse { - throw new SubmitRestMissingFieldException("Environment variables 'environmentVariables' are missing.") + throw new SubmitRestMissingFieldException("Environment variables 'environmentVariables' " + + "are missing.") } // Optional fields