From 2352018760c215b2e3bf3e077fc007a2f0538faf Mon Sep 17 00:00:00 2001 From: Yu Gan Date: Wed, 17 Jun 2020 15:35:43 +0800 Subject: [PATCH] #122 standalone cluster manager (#126) Co-authored-by: Yu Gan --- assembly/pom.xml | 2 +- common/kvstore/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- core/pom.xml | 2 +- .../scala/org/apache/spark/deploy/DeployMessage.scala | 6 ++++-- .../scala/org/apache/spark/deploy/master/Master.scala | 4 ++++ .../org/apache/spark/deploy/master/WorkerState.scala | 2 +- .../main/scala/org/apache/spark/rpc/RpcAddress.scala | 4 ++-- core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala | 10 +++++++++- core/src/main/scala/org/apache/spark/util/Utils.scala | 2 +- examples/pom.xml | 2 +- external/avro/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/flume-assembly/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml | 2 +- external/kafka-0-10-assembly/pom.xml | 2 +- external/kafka-0-10-sql/pom.xml | 2 +- external/kafka-0-10/pom.xml | 2 +- external/kafka-0-8-assembly/pom.xml | 2 +- external/kafka-0-8/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- hadoop-cloud/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- resource-managers/kubernetes/core/pom.xml | 2 +- resource-managers/kubernetes/integration-tests/pom.xml | 2 +- resource-managers/mesos/pom.xml | 2 +- resource-managers/yarn/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- 46 files changed, 61 insertions(+), 47 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 8b45f9139742f..02b61d5dc6d59 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index 43173cd66c7a0..2f2864990890d 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index fda95914c1842..d8940ce9cd584 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 4c6d42905ddee..c3a87b095083b 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index c10dde2cc5e0b..ec0c475d18d8e 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 8418e80e57a8e..eb45dba8a8f35 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 87cbcd8f22397..74a8c5ad10d94 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 4e63e21d8a2aa..a55bc7f4444bf 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 931fe4ad26539..e6ad744fd0f1e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala index 49a319abb3238..88ad63d87a71d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala @@ -27,10 +27,10 @@ import org.apache.spark.deploy.worker.{DriverRunner, ExecutorRunner} import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef} import org.apache.spark.util.Utils -private[deploy] sealed trait DeployMessage extends Serializable +sealed trait DeployMessage extends Serializable /** Contains messages sent between Scheduler endpoint nodes. */ -private[deploy] object DeployMessages { +object DeployMessages { // Worker to Master @@ -230,4 +230,6 @@ private[deploy] object DeployMessages { case object SendHeartbeat + case class KillApplication(appId: String) + } diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index e1184248af460..a471c2d0319fe 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -416,6 +416,10 @@ private[deploy] class Master( case CheckForWorkerTimeOut => timeOutDeadWorkers() + case KillApplication(applicationId) => + logInfo(s"Asked to kill application $applicationId") + idToApp.get(applicationId).foreach(removeApplication(_, ApplicationState.KILLED)) + } override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = { diff --git a/core/src/main/scala/org/apache/spark/deploy/master/WorkerState.scala b/core/src/main/scala/org/apache/spark/deploy/master/WorkerState.scala index b60baaadfb4bc..4754a0b832f94 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/WorkerState.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/WorkerState.scala @@ -17,7 +17,7 @@ package org.apache.spark.deploy.master -private[master] object WorkerState extends Enumeration { +object WorkerState extends Enumeration { type WorkerState = Value val ALIVE, DEAD, DECOMMISSIONED, UNKNOWN = Value diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala b/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala index eb0b26947f504..5412f6cd2016e 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala @@ -23,7 +23,7 @@ import org.apache.spark.util.Utils /** * Address for an RPC environment, with hostname and port. */ -private[spark] case class RpcAddress(host: String, port: Int) { +case class RpcAddress(host: String, port: Int) { def hostPort: String = host + ":" + port @@ -34,7 +34,7 @@ private[spark] case class RpcAddress(host: String, port: Int) { } -private[spark] object RpcAddress { +object RpcAddress { /** Return the [[RpcAddress]] represented by `uri`. */ def fromURIString(uri: String): RpcAddress = { diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala index de2cc56bc6b16..eeeb34b0ec0a1 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala @@ -31,7 +31,15 @@ import org.apache.spark.util.RpcUtils * A RpcEnv implementation must have a [[RpcEnvFactory]] implementation with an empty constructor * so that it can be created via Reflection. */ -private[spark] object RpcEnv { +object RpcEnv { + + def create( + name: String, + host: String, + port: Int, + conf: SparkConf): RpcEnv = { + create(name, host, host, port, conf, new SecurityManager(conf), 0, clientMode = true) + } def create( name: String, diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index ca33415eb4187..f6e08983089e1 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -78,7 +78,7 @@ private[spark] object CallSite { /** * Various utility methods used by Spark. */ -private[spark] object Utils extends Logging { +object Utils extends Logging { val random = new Random() private val sparkUncaughtExceptionHandler = new SparkUncaughtExceptionHandler diff --git a/examples/pom.xml b/examples/pom.xml index d60d084715c57..3cff4085f302b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index 74e2d0ee7256b..f20273fcb321a 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 3f8d0b3130d55..8cbb5767464cd 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index 1ab298f6ac0bc..2c14c3676015c 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index d283d4a37eee0..b2fe63f9150aa 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 6e54d055a4aea..73f56f78e4881 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index b09cf95226e0b..60531cfc28cfd 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index cde76d128188f..85919e1a35315 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index b68b5b209cc84..7bfa8ceab2d5d 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml index 3348314b3eaf2..7e253bc8a57c9 100644 --- a/external/kafka-0-8-assembly/pom.xml +++ b/external/kafka-0-8-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml index 4636f22f4c037..2ef14b96b78c0 100644 --- a/external/kafka-0-8/pom.xml +++ b/external/kafka-0-8/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 7c091809a93c5..85ce42326269f 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 870ca4903cf90..b8bb5b17ee2fa 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index 4b75e6f198f26..a115f5ce877c6 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index 989fa0cecba53..0d0ecf503505a 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 42f4473e9bdf8..f9f4dc027af7d 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 7d45402e977b1..00cbb867f9d37 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 285dac08a134d..f98ba9fa49428 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 33e5546962e38..82a7fa248a326 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/pom.xml b/pom.xml index bd42b334135af..0f1bb4fa628dd 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index faebb1fac3e1f..2a56f48bd842d 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index e23a54e1bcfe5..14759655e4042 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index a3aa2fceeb90b..c8f1a07a246d0 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index f626e1c744e21..0a69197ea3888 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index ae6fca751de5f..51d11a6765d77 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 9c1473464a2c2..267fddc897795 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 5cf6e5028d85e..a85900021c8a8 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 6be0203a65138..4974651079dc3 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 6f8cc3df4ad59..26f81c73f23b7 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 898fa98d262f2..54ee1469ccf88 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index cd19d92eae869..37841bf6843b3 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r32 + 2.4.1-kylin-r33 ../pom.xml