From d797d12c99cf8609c54dfae4a4775c308dfc35a3 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Tue, 11 Sep 2018 16:10:08 +0300
Subject: [PATCH 01/30] aws - cloudformation base

---
 cloud_install/aws/template.json | 51 +++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 cloud_install/aws/template.json

diff --git a/cloud_install/aws/template.json b/cloud_install/aws/template.json
new file mode 100644
index 000000000..2cdab0592
--- /dev/null
+++ b/cloud_install/aws/template.json
@@ -0,0 +1,51 @@
+{
+  "AWSTemplateFormatVersion" : "2010-09-09",
+  "Description" : "Hydrosphere/mist setup template",
+  "Metadata" : {
+    "Instances" : {"Description" : "Information about the instances"},
+    "Databases" : {"Description" : "Information about the databases"}
+  },
+  "Parameters" : {
+    "InstanceTypeParameter" : {
+      "Type" : "String",
+      "Default" : "m4.large",
+      "Description" : "Enter t2.micro, m1.small, m1.large ..."
+    },
+    "KeyPairID": {
+      "Type" : "String",
+      "Description" : "AWS access key ID"
+    },
+    "KeyPairSecret": {
+      "Type" : "String",
+      "Description" : "AWS secret access key"
+    },
+    "AZ": {
+       "Description": "Availability Zone of the Subnet",
+       "Type": "AWS::EC2::AvailabilityZone::Name"
+    }
+  },
+  "Resources": {
+    "MasterEc2Instance": {
+      "Type" : "AWS::EC2::Instance",
+      "Metadata": {
+        "AWS::CloudFormation::Init" : {
+          "config" : {
+            "commands" : {
+              "test" : {
+                "command" : "echo \"$CFNTEST\" > test.txt",
+                "env" : { "CFNTEST" : "I come from config1." },
+                "cwd" : "~"
+              }
+            }
+          }
+        }
+      },
+      "Properties" : {
+         "AvailabilityZone" : { "Ref": "AZ" },
+         "InstanceType": { "Ref": "InstanceTypeParameter" },
+         "ImageId" : "ami-027583e616ca104df",
+         "Tags" : [ {"Key" : "Name", "Value" : "mist-server"} ]
+      }
+    }
+  }
+}

From cde30ac8609072eb3ac9c0767af95b5774631aca Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Tue, 11 Sep 2018 22:41:13 +0300
Subject: [PATCH 02/30] cluodformation - fix sec group

---
 cloud_install/aws/install.sh    |  3 +++
 cloud_install/aws/template.json | 41 +++++++++++++++++++++++++++++----
 2 files changed, 39 insertions(+), 5 deletions(-)
 create mode 100644 cloud_install/aws/install.sh

diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
new file mode 100644
index 000000000..116175fa9
--- /dev/null
+++ b/cloud_install/aws/install.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "$HELLO" > ~/.check
diff --git a/cloud_install/aws/template.json b/cloud_install/aws/template.json
index 2cdab0592..516d2741c 100644
--- a/cloud_install/aws/template.json
+++ b/cloud_install/aws/template.json
@@ -11,11 +11,15 @@
       "Default" : "m4.large",
       "Description" : "Enter t2.micro, m1.small, m1.large ..."
     },
-    "KeyPairID": {
+    "KeyPairName": {
+      "Type": "AWS::EC2::KeyPair::KeyName",
+      "Description": "KeyPair Name"
+    },
+    "AccessKeyID": {
       "Type" : "String",
       "Description" : "AWS access key ID"
     },
-    "KeyPairSecret": {
+    "AccessKeySecret": {
       "Type" : "String",
       "Description" : "AWS secret access key"
     },
@@ -25,15 +29,14 @@
     }
   },
   "Resources": {
-    "MasterEc2Instance": {
+    "MistEc2Instance": {
       "Type" : "AWS::EC2::Instance",
       "Metadata": {
         "AWS::CloudFormation::Init" : {
           "config" : {
             "commands" : {
               "test" : {
-                "command" : "echo \"$CFNTEST\" > test.txt",
-                "env" : { "CFNTEST" : "I come from config1." },
+                "command" : "curl http://repo.hydrosphere.io/hydrosphere/static/preview/install.sh | bash ",
                 "cwd" : "~"
               }
             }
@@ -44,8 +47,36 @@
          "AvailabilityZone" : { "Ref": "AZ" },
          "InstanceType": { "Ref": "InstanceTypeParameter" },
          "ImageId" : "ami-027583e616ca104df",
+         "KeyName": { "Ref": "KeyPairName" },
+         "SecurityGroups": [ { "Ref": "InstanceSecurityGroup" } ],
          "Tags" : [ {"Key" : "Name", "Value" : "mist-server"} ]
       }
+    },
+    "InstanceSecurityGroup" : {
+      "Type" : "AWS::EC2::SecurityGroup",
+      "Properties" : {
+        "GroupDescription" : "SSH + HTTP",
+        "SecurityGroupIngress" : [
+          {
+            "IpProtocol" : "tcp",
+            "FromPort" : "22",
+            "ToPort" : "22",
+            "CidrIp" : "0.0.0.0/0"
+          },
+          {
+            "IpProtocol" : "tcp",
+            "FromPort" : "80",
+            "ToPort" : "80",
+            "CidrIp" : "0.0.0.0/0"
+          }
+        ]
+      }
+    }
+  },
+  "Outputs" : {
+    "PublicIp" : {
+      "Value" : { "Fn::GetAtt" : [ "MistEc2Instance", "PublicIp" ]},
+      "Description" : "PublicIp Address"
     }
   }
 }

From 24478ab37fb062f334edc0ad5a12e1ae2abd01df Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Wed, 12 Sep 2018 23:29:10 +0300
Subject: [PATCH 03/30] wip

---
 cloud_install/aws/install.sh    | 45 ++++++++++++++++++++++++++++++++-
 cloud_install/aws/template.json | 23 ++++++++---------
 2 files changed, 54 insertions(+), 14 deletions(-)

diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
index 116175fa9..84869f51f 100644
--- a/cloud_install/aws/install.sh
+++ b/cloud_install/aws/install.sh
@@ -1,3 +1,46 @@
 #!/bin/bash
 
-echo "$HELLO" > ~/.check
+apt-get update
+apt-get install -y nginx openjdk-8-jdk apache2-utils
+
+htpasswd -b -c /etc/nginx/.htpasswd admin password
+
+cat << EOF > /etc/nginx/sites-enabled/default
+server {
+	listen 80 default_server;
+	listen [::]:80 default_server;
+
+	root /var/www/html;
+	index index.html index.htm index.nginx-debian.html;
+
+	server_name _;
+
+	location / {
+	   proxy_pass http://127.0.0.1:2004/;
+     auth_basic "Restricted";
+     auth_basic_user_file /etc/nginx/.htpasswd;
+	}
+
+	location /v2/api/ws {
+	    proxy_pass         http://127.0.0.1:2004/v2/api/ws;
+	    proxy_http_version 1.1;
+      proxy_set_header Upgrade \$http_upgrade;
+      proxy_set_header Connection "upgrade";
+      auth_basic "Restricted";
+      auth_basic_user_file /etc/nginx/.htpasswd;
+	}
+
+}
+EOF
+
+cd /opt
+wget http://repo.hydrosphere.io/hydrosphere/static/preview/mist-1.0.0-RC17.tar.gz
+tar xvfz mist-1.0.0-RC17.tar.gz
+mv mist-1.0.0-RC17 mist
+
+wget https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz 
+tar xvfz spark-2.3.0-bin-hadoop2.7.tgz
+mv spark-2.3.0-bin-hadoop2.7 spark
+
+SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
+service nginx restart
diff --git a/cloud_install/aws/template.json b/cloud_install/aws/template.json
index 516d2741c..f670e08f6 100644
--- a/cloud_install/aws/template.json
+++ b/cloud_install/aws/template.json
@@ -31,25 +31,22 @@
   "Resources": {
     "MistEc2Instance": {
       "Type" : "AWS::EC2::Instance",
-      "Metadata": {
-        "AWS::CloudFormation::Init" : {
-          "config" : {
-            "commands" : {
-              "test" : {
-                "command" : "curl http://repo.hydrosphere.io/hydrosphere/static/preview/install.sh | bash ",
-                "cwd" : "~"
-              }
-            }
-          }
-        }
-      },
       "Properties" : {
          "AvailabilityZone" : { "Ref": "AZ" },
          "InstanceType": { "Ref": "InstanceTypeParameter" },
          "ImageId" : "ami-027583e616ca104df",
          "KeyName": { "Ref": "KeyPairName" },
          "SecurityGroups": [ { "Ref": "InstanceSecurityGroup" } ],
-         "Tags" : [ {"Key" : "Name", "Value" : "mist-server"} ]
+         "Tags" : [ {"Key" : "Name", "Value" : "mist-server"} ],
+         "UserData": {
+           "Fn::Base64": { "Fn::Join": [ "\n", [
+              "#!/bin/bash -ex",
+              { "Fn::Join": ["", ["echo ACCESS_KEY_ID=", { "Ref": "AccessKeyID"},  " > ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo ACCESS_KEY_SECRET=", { "Ref": "AccessKeySecret"},  " >> ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo AWS_AZ=", { "Ref": "AZ"},  " >> ~/.aws_setup_data"]]},
+              "curl http://repo.hydrosphere.io/hydrosphere/static/preview/install.sh | bash"
+           ]]}
+         }
       }
     },
     "InstanceSecurityGroup" : {

From ecbc5d4a16f356d363b1c13219966fdfddd6595c Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 13 Sep 2018 13:16:47 +0300
Subject: [PATCH 04/30] wip

---
 cloud_install/aws/install.sh | 42 +++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
index 84869f51f..cdd6bef56 100644
--- a/cloud_install/aws/install.sh
+++ b/cloud_install/aws/install.sh
@@ -3,8 +3,37 @@
 apt-get update
 apt-get install -y nginx openjdk-8-jdk apache2-utils
 
-htpasswd -b -c /etc/nginx/.htpasswd admin password
+echo "<p>Completing Mist intallation</p>" > /var/www/html/index.html
+cat << EOF > /etc/nginx/sites-enabled/default
+server {
+	listen 80 default_server;
+	listen [::]:80 default_server;
+
+	root /var/www/html;
+	index index.html index.htm index.nginx-debian.html;
+
+	server_name _;
+}
+EOF
+service nginx restart
+
+cd /opt
+
+wget http://repo.hydrosphere.io/hydrosphere/static/preview/mist-1.0.0-RC17.tar.gz
+tar xvfz mist-1.0.0-RC17.tar.gz
+mv mist-1.0.0-RC17 mist
 
+wget https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz 
+tar xvfz spark-2.3.0-bin-hadoop2.7.tgz
+mv spark-2.3.0-bin-hadoop2.7 spark
+
+# By some reasons, there is some problems with running Mist right after intance was started
+# InfoProvider can't connect to master
+sleep 30
+
+SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
+
+htpasswd -b -c /etc/nginx/.htpasswd admin password
 cat << EOF > /etc/nginx/sites-enabled/default
 server {
 	listen 80 default_server;
@@ -32,15 +61,4 @@ server {
 
 }
 EOF
-
-cd /opt
-wget http://repo.hydrosphere.io/hydrosphere/static/preview/mist-1.0.0-RC17.tar.gz
-tar xvfz mist-1.0.0-RC17.tar.gz
-mv mist-1.0.0-RC17 mist
-
-wget https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz 
-tar xvfz spark-2.3.0-bin-hadoop2.7.tgz
-mv spark-2.3.0-bin-hadoop2.7 spark
-
-SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
 service nginx restart

From da0c82db2174cede06ac789e673095e7e68ba397 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 13 Sep 2018 13:28:37 +0300
Subject: [PATCH 05/30] fix infoprovider startup

---
 .../mist/utils/akka/RestartSupervisor.scala   | 34 ++++++++++++-------
 .../mist/master/MasterServer.scala            |  6 +++-
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala b/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
index 50e074ac8..d30143694 100644
--- a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
+++ b/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
@@ -10,7 +10,8 @@ import scala.concurrent.duration._
 class RestartSupervisor(
   name: String,
   start: () => Future[ActorRef],
-  timeout: FiniteDuration
+  timeout: FiniteDuration,
+  maxRetry: Int
 ) extends Actor with ActorLogging with Timers {
 
   override def receive: Receive = init
@@ -21,35 +22,39 @@ class RestartSupervisor(
   private def init: Receive = {
     case Event.Start(req) =>
       start().map(Event.Started) pipeTo self
-      context become await(Some(req))
+      context become await(Some(req), 0)
   }
 
-  private def await(req: Option[Promise[ActorRef]]): Receive = {
+  private def await(req: Option[Promise[ActorRef]], attempts: Int): Receive = {
     case Event.Started(ref) =>
       req.foreach(_.success(self))
       context watch ref
       context become proxy(ref)
 
-    case akka.actor.Status.Failure(e) =>
+    case akka.actor.Status.Failure(e)  if maxRetry == attempts + 1 =>
       req.foreach(_.failure(e))
+      log.error(e, "Starting child for {} failed, maxRetry reached", name)
+      context stop self
+
+    case akka.actor.Status.Failure(e) =>
       log.error(e, "Starting child for {} failed", name)
       timers.startSingleTimer("timeout", Event.Timeout, timeout)
-      context become restartTimeout
+      context become restartTimeout(attempts)
   }
 
   private def proxy(ref: ActorRef): Receive = {
     case Terminated(_) =>
       log.error(s"Reference for {} was terminated. Restarting", name)
       timers.startSingleTimer("timeout", Event.Timeout, timeout)
-      context become restartTimeout
+      context become restartTimeout(0)
 
     case x => ref.forward(x)
   }
 
-  private def restartTimeout: Receive = {
+  private def restartTimeout(attempts: Int): Receive = {
     case Event.Timeout =>
       start().map(Event.Started) pipeTo self
-      context become await(None)
+      context become await(None, attempts + 1)
   }
 }
 
@@ -67,18 +72,20 @@ object RestartSupervisor {
   def props(
     name: String,
     start: () => Future[ActorRef],
-    timeout: FiniteDuration
+    timeout: FiniteDuration,
+    maxRetry: Int
   ): Props = {
-    Props(classOf[RestartSupervisor], name, start, timeout)
+    Props(classOf[RestartSupervisor], name, start, timeout, maxRetry)
   }
 
   def wrap(
     name: String,
     start: () => Future[ActorRef],
-    timeout: FiniteDuration
+    timeout: FiniteDuration,
+    maxRetry: Int
   )(implicit af: ActorRefFactory): Future[ActorRef] = {
 
-    val ref = af.actorOf(props(name, start, timeout))
+    val ref = af.actorOf(props(name, start, timeout, maxRetry))
     val promise = Promise[ActorRef]
     ref ! Event.Start(promise)
     promise.future
@@ -86,8 +93,9 @@ object RestartSupervisor {
 
   def wrap(
     name: String,
+    maxRetry: Int,
     start: () => Future[ActorRef]
-  )(implicit af: ActorRefFactory): Future[ActorRef] = wrap(name, start, 5 seconds)(af)
+  )(implicit af: ActorRefFactory): Future[ActorRef] = wrap(name, start, 5 seconds, maxRetry)(af)
 
 }
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
index 07b05c871..715b441f2 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
@@ -130,7 +130,11 @@ object MasterServer extends Logger {
         config.cluster.port
       )
 
-      RestartSupervisor.wrap("FunctionInfoProvider", () => runner.run()(system))
+      RestartSupervisor.wrap(
+        name = "FunctionInfoProvider",
+        maxRetry = 10,
+        start = () => runner.run()(system)
+      )
     }
 
     for {

From a362b6ea2174443f70435f15ef719967053b1c86 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 13 Sep 2018 14:32:52 +0300
Subject: [PATCH 06/30] infoprovider superviser - fix startup

---
 .../hydrosphere/mist/utils/akka/RestartSupervisor.scala   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala b/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
index d30143694..0057be995 100644
--- a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
+++ b/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
@@ -39,22 +39,22 @@ class RestartSupervisor(
     case akka.actor.Status.Failure(e) =>
       log.error(e, "Starting child for {} failed", name)
       timers.startSingleTimer("timeout", Event.Timeout, timeout)
-      context become restartTimeout(attempts)
+      context become restartTimeout(req, attempts)
   }
 
   private def proxy(ref: ActorRef): Receive = {
     case Terminated(_) =>
       log.error(s"Reference for {} was terminated. Restarting", name)
       timers.startSingleTimer("timeout", Event.Timeout, timeout)
-      context become restartTimeout(0)
+      context become restartTimeout(None, 0)
 
     case x => ref.forward(x)
   }
 
-  private def restartTimeout(attempts: Int): Receive = {
+  private def restartTimeout(req: Option[Promise[ActorRef]], attempts: Int): Receive = {
     case Event.Timeout =>
       start().map(Event.Started) pipeTo self
-      context become await(None, attempts + 1)
+      context become await(req, attempts + 1)
   }
 }
 

From 7476e21ec393e923f6f2867addf31ddd77c55116 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 13 Sep 2018 16:31:15 +0300
Subject: [PATCH 07/30] aws - cloudfromation basic setup

---
 .../aws/cloudformation-launch-stack.png          | Bin 0 -> 3880 bytes
 cloud_install/aws/index.html                     |  13 +++++++++++++
 cloud_install/aws/install.sh                     |   6 ++++--
 cloud_install/aws/template.json                  |  10 ++++++++++
 4 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 cloud_install/aws/cloudformation-launch-stack.png
 create mode 100644 cloud_install/aws/index.html

diff --git a/cloud_install/aws/cloudformation-launch-stack.png b/cloud_install/aws/cloudformation-launch-stack.png
new file mode 100644
index 0000000000000000000000000000000000000000..7c0b53b6f2efc6a9a69fb4cb570314fe48f54abb
GIT binary patch
literal 3880
zcmV+@57+RCP)<h;3K|Lk000e1NJLTq0058x000{Z1^@s6lQ9OP00009a7bBm000XU
z000XU0RWnu7ytkP+G#^WP*7-ZbZ>KLZ*U+<Lqi~Na&Km7Y-Iodc-ozkO>0v@6o#Kk
zBy~|skq9maHzpVb!A%szeAFnH;fC8TlHlg{&NR8$`@v*xZFhqH4E+Z#x^m|~@B=Bh
z_D6`z#6^szN*C?3dS~Xmobz%9%J006+AeKEk~$mo+Hx=)$+gd{Be=?au7%MwSKE(w
z06f3j3Vfybi-4cs5CA8)JipuGe5^;gbs(z16K$puh-bj77cO@oz5`Xe6LdiQ05--e
zdUr+J!EgknOF(tJ(AR<L?m{<!s!mc3rAI)j6Rn|i417(JOhfrKP;=(c0d%tb$j0M|
zleMTWx5BAQ!c=;h4xh_TmgHf2)R1?Z&1OS}`}@+y<B6L}Yo^8?7=4FD767jFZPuS;
zE|Wp8y$rrFo72aV(Pu)2BSR9<qkX35ybOjTxln(0)QAduu-u*av|97=9z6YU+IQm(
zynlH5?s5gFz{%~Ej1|xCw!CHIwb>jGpDr!D|DP|)O23<d*)!Ipk&%Ak^1?}FbX+7-
z8E50D7WhilrCd5|vpE5VJq#8@N`yh;Q{c!ca7YveO-w=Z`{c01MAQiIQ7Gz2<OL=;
zi^bmqsBFU9vJgTXe|hlo-@0F_LYb)%^n^J8001CkNK#Dz0EY?y0E`U)0UrGT0B52A
z06tg%0Z2yx01iw5000V{ww+Y~01U`UL_t(|+U%Qoa8%cohric*tF<9eqtyr`z-+c~
z5-_sk2&fQ3oGA<h7nw@oh01sgNr=hftrBTwh+Of+#&&`Wm+hE%oGP5)KxAx!qmmFr
zjAdJ55u1<!0a_5K)%*6g`Qxdd)U6iC*i&UWI#sv2-+kwt`@8p^d)_(s-6yE3s?tw?
z+S|e(b|2x?xhsSt5@YnwMcG*n@}23-ndW5fj9l)xZ4!CTv}?);s;a72`-03}u@#%u
zM4H2j-D<*YGR85Q%?4~1^VlelGLmIw*n6TZ$6t#Iq9hZFNQ5I2p|HqLAH0+0i)RdL
zNP?=WYT?5FCe2}GO<9;Vi|pj*II$Yy8#Wj%W3%xECPj`C4Ewn1^3#0c0*5bVP<?cw
zZb+|eUe2QVQ-&JQ&V9$#pZ{hb-(BG5`*%CZx0*qWf)XCuiorHE(qCjUTToL6E-zfV
zMD_b2UTv6+FA(7u4=?0<|8i@bh-u$H+Ay1pd}Bc&nW7&=4*;rF9cpdL*cg8isI^0-
zu_eU0J}XxPb^=io<Hn^j-RUJSOJMrM5XD95Ov)@^{Yv;!aVzUzpTd7^tp$iPB4O@|
zt!g+TvGZ{&6M|RCP8EhUn9P{QX!Oz9c8R@>3h#a-5mBsUWTarVny}j~*zH<~VzXID
zv&nq9D9F4iVJ2i+QO8+0-gc4ITe62d&J0&v0nK|EqF4zf50oMTLPQvwfl*8+TSHX+
zkDC|X>Z7MmMv`QL!3dH5Y&#m&0%}E3@B{^FPB_^0p_Pw2!U(-yW|b83%DM~KZD!U!
z`WBr%J^+mAMN2BI787gk5^zKVSWP0R{Y6N~x=f>c5bHiX$M))Git;kaOtXKkv7Ybj
z<GDW`0-z{w+^6$A=OX;#<sNDd1rQY#gFyfU1VM8OCX*4P(LiiN$7nPFaMf?YA66*L
z4B(JN3X1cw$Xzs^bucd7&R6E-YbGH|a&qfXf(f1|Wmu34*S`!v?eEroZYQO_@f?*e
z*HC`TWG0W#8nImED>b~fr-}1jeE^hB&82i&F28?r)&CpT_I5Xw+iIv-x0r=<rhF=0
z+ja$bbKe#0b{n>+0SFG^i9{qaGgAnK;#2!07i?tN!>qW?jnHf0TV**sy(dc5OZ6;Y
zJYyK2Vn_kA<r8ILbxmirKi2fm@3J;p`?C$lh;deKdxwgbYbeSaPsP72)4W5Mi@Fcb
z=sI5e>;JO-?e}?h(@ILG7kow;kz)Y!rx?>i$6Wm3opu01p)euB$bv(^-`H5i!i5$3
zwasQG92T+J2A)3qPguCMz)Z2zO}^d6!{3y7zS_V)n%Xr3lB7ZDl{FJA7s2<z&|q4y
zUKg+$-ubz;A5}W9{YL9i!$(IXv*nEg05rWelg!jujA#dL(ffM-;(6*mJV#&e-%!Ju
zpAFWiyC3sO)NtsA_5A&yk;}dD&cN%9=c%o&psef(L{TObO2KZ2K`$r>h2X>mBgIY;
zB3|ata6+|*=966*2ErQ=HOmZ<w~`tUfS^W(E!X2|@AR^0eG`VVS`1~iOj+5$?)_c*
z{Ufa|7Oih$`<{!#)@|Q&kwxp9^!k{d!7?$q6^|a#+j``=Q`$YWfbmGHo7>hlCdH9h
z+<r~Q;|(Ugzi53E+u!K~;F0HA<M8g@-=()>D67@6j|^*68&4zzHG;<%V8x?HlIplt
zX*i|;iF<lsyxssVmtU`GXxNO$6C@b);qe6N>kGt{584DhtZ}Am27RoxM>8NvRxq2b
z7<zXP1klOKrwIzXwN6w#SjfhEb8-7*?s@DKhtFtPpxZA|*X*Iad%zeI>)N{`)HQop
zxZyZOxmGG3EX3`Txqow;KGAN!#KH~7*;Vi2zGY6GXE6Qv`E!&`b?7(-%O_I#+zc(o
z`#}poe*PT0>RrR~4(W4OM8|pGGN+DxaR07{RgW?I`HFI_lumVwc-&jg%DU0S8&a51
zDBiFd8>?`;19&_^-0lFOP*g7`FAU6!-E2UT6*_vn12ItV@vWpduk*TMBt|q0PkCty
z<u6RrufJC6;GW0Y*j;~>(h>`({T)@60Ue2Ts`jY;<pX(aTs={n1YeXbZ(inbtC#Y*
zX;i*)0k>b`*&p7d^XRzx<^NJu!t$?<({&G)Po%$86?4yHXQ*rTv1LaWTXuA53*mu$
ze)&KiwJ%Jg^3_f%UhUx79~KViP(?Pbp1{V{6LejfsRCQxyi8q_YXn#a`hNfBRt}#D
zaNn{C{QmJtpJ*{(>H&!H+md*rcxrBbnkiG(W3ia@y&-@=z=Uc~LkNX%nl*h_T!8@t
z5~cXf!f^D)ww)8@L`lVL67cwC{%d<Dum7zVk6$JuReM=0$tvNfLR3`Zu7S8ZUr<Jp
zHH<ThY&uV>-Kg`1qY59L)-sHTmS+#a5|dA{8itjR;njmmi@>|TnMQkel%s7ysvF$A
zzPFc(S36iR+reF>X}aE+4n33Vj`Gv37pXp|#WS%!$M~9B2eA5ra$LKyzCZhBHy*#t
zx9`m4#h*-y)1Qbn8RcgybX+kE@<LqGm{L;y6|>oh!)(HDGh?xs`t``jf)|safF98d
zh-7Cu24Wz9h^S58jT%qVWTL-=zunoxZ+G@^Ye@=EuFd1F(zF}WGnhh=p*RKMv%#C#
z&ZHa*-@Y@Gmwr~nhLuhL4jl1cQ`h%@eV*!rZoap4JWsC8V^WUw6SWI)Yl*gXQh(To
z$1jgaW-!f~><1!)3d20oKoGt~D5S+lUA-P$E<e4!KEmD*d6_O`-zmhD6uc1=sv?r_
zOxHY+s)!8diIT10Yf-XE1*mJ*LjPS=B>?O><e~bY8&OsXMHC{Uru%GXSl4mplGfoE
zkJ#USL{xR2#JX8UcIprNcxFd8|GwNA?+r&FKazf(>(p@BZNjj)Vv~O7EvY>Bqnora
z!6R|BEjS{qvGG_iJC)Uove@uUJ73>;n!o;fDj5#rh!@0{Z?>?nF-~*x1W8z&;FWAz
z+8+WhFnf}hXxNRA-b-h1E&y|;Ib#MSBT6c<;~~QtN~}}YqPu@~W=P%6%_Zb#YtPf~
zR-UEMX<<i=3jkSGiHIs+nc<+&Y2lCWxsVk|cNle^mF44zh$?bFE>Ti-EQxjZ-#vjh
z-s|OwZ5<qK3sLB_@b3E_{=Q}Uh~yJ#-ntW%%}J%OKR#?|@zZe3PrAd%s`4x%qKd<&
zIr7K0UE-VbGnh5m&eGd6D0Eslc+}6&U%Z4jAnW}Z3D(#`mSvTd<>NST#K(>r7mvPp
zi7ovGITE^Kj)|Nsk#1Maa8i;O&mBXIXRvpgI6j31v;0W1ir_s(eUkyOGIvI<_9{bF
zBv#EnhQn?q*_pA3s3I#M%b=>s%c0|beUKArRe2T*W~I{98l<kt$CW;bs)q`>;+D95
zRw^@#?5L`?+OE1Kb{}xlbUH{$fsL)3iuC&zE=TbOWtPm#U_ypDX<b^1k=v#@@C9Xd
zA8^yq;-@6v#;WqH5y>a^>%{pGhg$sXecwYvi}uJ`IzNNoZ7S0Fi}P)K+!5hWi=T!T
zKXa!#m^;<Mg4wC;Z}w5!<Rjf-<dJ)GMvT?tm-)COqL0rv=4Idw%CudIpeX2La=nt1
zrO@RPX*wNS9AbabCc3inrT(`N(x4Lm-fG?59@cza%a)IxQh4%hEgAaHA1y&BD1Ta2
zq%d!+euKh^W*qk6-D7y{h3crJ_HOvq>mvIalWh?sdp(t*|66)HR6O8iQjTU=Pwkz-
zpIhee_(O|%c+I>4GmA<JQBlQqjZ9uv#?DEPMk;W^L(2tJ)xdinB*uBcBu3-`GFUoa
z<OeIm6l6;TLNYa{bNSQJnHWrFmM@;6`KjzI2bX)obo7}RpP^te%h)W&gw$N_Pc6sJ
z1&vm^`6l?u8j+%0GjHuz==CHT(SSFakYV9F3j^G{I7)$2A{>Rf)+{RbeHDQ1Pp#0;
zNSo%($)~!emt&pT<d5sRW@;$PV+J&uDQCQy4J%>sm#ox$By#dRv|TdO?TV0-Wu>Gb
z%$zByd~I%sTZ&^l`wF$K6M3e-41?K7)uv@}XQNF^?kM5y+T$GV%A+`|2U&@do?^mg
z6^3jGh9hGJG+Jq}V<^auuy(2T-)M{^r1#o(%Vn3!$DWDQpUUFBvokT69aL>vHsnmS
z{#Aw|g_)jyg0jLh-0XCbVlxZ@5e$zViXJUwG+sa9v6|y~JYfr00x4YV%jB#pi{_rG
z2u2$Qla1|9tw{b=#^CS56e)}<hf(FI&Y&s-k-I8K#%AUVM1nE-&bYyBM=)w<zn*w_
q$@RYri(QO;nflL7H}Ylb{|x|9M^&l@+>PD<0000<MNUMnLSTaY$%&r;

literal 0
HcmV?d00001

diff --git a/cloud_install/aws/index.html b/cloud_install/aws/index.html
new file mode 100644
index 000000000..d6e4f08c7
--- /dev/null
+++ b/cloud_install/aws/index.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Mist</title>
+</head>
+
+<body>
+  <a href="https://console.aws.amazon.com/cloudformation/home?#/stacks/new?stackName=mist&templateURL=https://s3.eu-central-1.amazonaws.com/hydrosphere-cloudformation/template.json">
+    <img src="cloudformation-launch-stack.png"/>
+  </a>
+</body>
+
+</html>
diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
index cdd6bef56..42ed31f8b 100644
--- a/cloud_install/aws/install.sh
+++ b/cloud_install/aws/install.sh
@@ -3,7 +3,9 @@
 apt-get update
 apt-get install -y nginx openjdk-8-jdk apache2-utils
 
-echo "<p>Completing Mist intallation</p>" > /var/www/html/index.html
+source ~/.aws_setup_data
+
+echo "<p>Mist will start soon</p>" > /var/www/html/index.html
 cat << EOF > /etc/nginx/sites-enabled/default
 server {
 	listen 80 default_server;
@@ -33,7 +35,7 @@ sleep 30
 
 SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
 
-htpasswd -b -c /etc/nginx/.htpasswd admin password
+htpasswd -b -c /etc/nginx/.htpasswd $MIST_LOGIN $MIST_PASSWORD
 cat << EOF > /etc/nginx/sites-enabled/default
 server {
 	listen 80 default_server;
diff --git a/cloud_install/aws/template.json b/cloud_install/aws/template.json
index f670e08f6..06344f119 100644
--- a/cloud_install/aws/template.json
+++ b/cloud_install/aws/template.json
@@ -26,6 +26,14 @@
     "AZ": {
        "Description": "Availability Zone of the Subnet",
        "Type": "AWS::EC2::AvailabilityZone::Name"
+    },
+    "MistLogin": {
+      "Type" : "String",
+      "Description" : "Basic auth login"
+    },
+    "MistPassword": {
+      "Type" : "String",
+      "Description" : "Basic auth password"
     }
   },
   "Resources": {
@@ -44,6 +52,8 @@
               { "Fn::Join": ["", ["echo ACCESS_KEY_ID=", { "Ref": "AccessKeyID"},  " > ~/.aws_setup_data"]]},
               { "Fn::Join": ["", ["echo ACCESS_KEY_SECRET=", { "Ref": "AccessKeySecret"},  " >> ~/.aws_setup_data"]]},
               { "Fn::Join": ["", ["echo AWS_AZ=", { "Ref": "AZ"},  " >> ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo MIST_LOGIN=", { "Ref": "MistLogin"},  " >> ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo MIST_PASSWORD=", { "Ref": "MistPassword"},  " >> ~/.aws_setup_data"]]},
               "curl http://repo.hydrosphere.io/hydrosphere/static/preview/install.sh | bash"
            ]]}
          }

From a38c6805d5260c0d2415836d151935f263c336da Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Tue, 18 Sep 2018 18:07:04 +0300
Subject: [PATCH 08/30] wip

---
 mist.sbt                                      |  16 +++
 .../src/main/resources/trustEC2EMR.json       |  15 +++
 .../src/main/resources/trustEMR.json          |  15 +++
 .../io/hydrosphere/mist/aws/AwsSetup.scala    |  70 +++++++++++
 .../mist/aws/CompletableFutureOps.scala       |  34 ++++++
 .../io/hydrosphere/mist/aws/EC2Service.scala  | 110 ++++++++++++++++++
 .../io/hydrosphere/mist/aws/GenConfig.scala   |  47 ++++++++
 .../io/hydrosphere/mist/aws/IAMService.scala  |  97 +++++++++++++++
 .../hydrosphere/mist/utils/ConfigUtils.scala  |   2 +
 project/Library.scala                         |   9 +-
 project/plugins.sbt                           |   1 +
 11 files changed, 414 insertions(+), 2 deletions(-)
 create mode 100644 mist/aws-config-gen/src/main/resources/trustEC2EMR.json
 create mode 100644 mist/aws-config-gen/src/main/resources/trustEMR.json
 create mode 100644 mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
 create mode 100644 mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala
 create mode 100644 mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
 create mode 100644 mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
 create mode 100644 mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala

diff --git a/mist.sbt b/mist.sbt
index ede7642f0..c960600a7 100644
--- a/mist.sbt
+++ b/mist.sbt
@@ -131,6 +131,22 @@ lazy val worker = project.in(file("mist/worker"))
     )
   )
 
+lazy val awsConfigGen = project.in(file("mist/aws-config-gen"))
+  .dependsOn(core % "compile->compile;test->test")
+  .settings(commonSettings: _*)
+  .settings(commonAssemblySettings: _*)
+  .settings(
+    name := "mist-aws-config-gen",
+    scalacOptions ++= commonScalacOptions,
+    libraryDependencies ++= Seq(
+      Library.slf4jLog4j, Library.typesafeConfig,
+      Library.cats, Library.catsEffect,
+      Library.awsSdkEC2, Library.awsSdkIAM,
+      Library.jsr305 % "provided",
+      Library.scalaTest % "test"
+    )
+  )
+
 lazy val root = project.in(file("."))
   .aggregate(mistLib, core, master, worker, examples)
   .dependsOn(master)
diff --git a/mist/aws-config-gen/src/main/resources/trustEC2EMR.json b/mist/aws-config-gen/src/main/resources/trustEC2EMR.json
new file mode 100644
index 000000000..37530dacb
--- /dev/null
+++ b/mist/aws-config-gen/src/main/resources/trustEC2EMR.json
@@ -0,0 +1,15 @@
+{
+  "Version": "2008-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "Service": [
+          "ec2.amazonaws.com"
+        ]
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/mist/aws-config-gen/src/main/resources/trustEMR.json b/mist/aws-config-gen/src/main/resources/trustEMR.json
new file mode 100644
index 000000000..5a4d3716d
--- /dev/null
+++ b/mist/aws-config-gen/src/main/resources/trustEMR.json
@@ -0,0 +1,15 @@
+{
+  "Version": "2008-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "Service": [
+          "elasticmapreduce.amazonaws.com"
+        ]
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
new file mode 100644
index 000000000..05fec0134
--- /dev/null
+++ b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
@@ -0,0 +1,70 @@
+package io.hydrosphere.mist.aws
+
+import cats._
+import cats.effect.IO
+import cats.implicits._
+import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider}
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.ec2.EC2AsyncClient
+import software.amazon.awssdk.services.iam.IAMAsyncClient
+
+case class SetupData(
+  subnetId: String,
+  securityGroupId: String,
+  emrRole: String,
+  ec2EmrRole: String
+)
+
+trait AwsSetup[F[_]] {
+
+  def setup(instanceId: String): F[SetupData]
+
+}
+
+object AwsSetup {
+
+  def default[F[_]](iam: IAMService[F], ec2: EC2Service[F])(implicit ME: MonadError[F, Throwable]): AwsSetup[F] = {
+    new AwsSetup[F] {
+
+      val ec2EmrRole = AWSRole("mist-EMREC2", "default emr ec2 role", AWSRoleData.EC2EMR)
+      val emrRole = AWSRole("mist-EMR", "default emr role", AWSRoleData.EMR)
+      val secGroupName = "mist-internal"
+      val secGroupDecr = "Master-worker communications"
+
+      override def setup(instanceId: String): F[SetupData] = {
+        for {
+          maybeInst <- ec2.getInstanceData(instanceId)
+          data <- maybeInst match {
+            case Some(d) => ME.pure(d)
+            case None => ME.raiseError[InstanceData](new RuntimeException(s"Unknown instance: $instanceId"))
+          }
+          ec2EmrRole <- iam.getOrCreate(ec2EmrRole)
+          emrRole <- iam.getOrCreate(emrRole)
+
+          secGroupData = SecGroupData(data.vpcId, 0, 65535, data.cidrIp)
+          secGroupId <- ec2.getOrCreateSecGroup(secGroupName, secGroupDecr, secGroupData)
+        } yield SetupData(data.subnetId, secGroupId, emrRole.name, ec2EmrRole.name)
+      }
+    }
+  }
+
+  def create(accessKey: String, secretKey: String, regionName: String): AwsSetup[IO] = {
+    val credentials = AwsCredentials.create(accessKey, secretKey)
+    val provider = StaticCredentialsProvider.create(credentials)
+
+    val ec2Client = EC2AsyncClient.builder()
+      .credentialsProvider(provider)
+      .region(Region.of(regionName))
+      .build()
+
+    val iamClient = IAMAsyncClient.builder()
+      .credentialsProvider(provider)
+      .region(Region.AWS_GLOBAL)
+      .build()
+
+    val iam = IAMService.fromSdk(iamClient)
+    val ec2 = EC2Service.fromSdk(ec2Client)
+    default(iam, ec2)
+  }
+}
+
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala
new file mode 100644
index 000000000..d13b421c2
--- /dev/null
+++ b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala
@@ -0,0 +1,34 @@
+package io.hydrosphere.mist.aws
+
+import java.util.concurrent.CompletableFuture
+import java.util.function.{BiConsumer, BiFunction}
+
+import cats.effect.IO
+
+import scala.concurrent.{Future, Promise}
+
+final class CompletableFutureOps[A](val cf: CompletableFuture[A]) extends AnyVal {
+
+  def toFuture: Future[A] = {
+    val p = Promise[A]
+    cf.whenComplete(new BiConsumer[A, Throwable] {
+      override def accept(res: A, err: Throwable): Unit = {
+        (Option(res), Option(err)) match {
+          case (Some(r), None) => p.success(r)
+          case (_, Some(e)) =>
+            e match {
+              case ce: java.util.concurrent.CompletionException => p.failure(ce.getCause)
+              case _ => p.failure(e)
+            }
+          case (_, _) => p.failure(new IllegalStateException("CompletableFuture was failed without error information"))
+        }
+      }})
+    p.future
+  }
+
+  def toIO: IO[A] = IO.fromFuture(IO(toFuture))
+}
+
+object jFutureSyntax {
+  implicit def cfSyntax[A](cf: CompletableFuture[A]): CompletableFutureOps[A] = new CompletableFutureOps(cf)
+}
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
new file mode 100644
index 000000000..2a4313dff
--- /dev/null
+++ b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
@@ -0,0 +1,110 @@
+package io.hydrosphere.mist.aws
+
+import cats._
+import cats.implicits._
+import cats.effect.IO
+import software.amazon.awssdk.services.ec2.EC2AsyncClient
+import software.amazon.awssdk.services.ec2.model._
+
+import scala.collection.JavaConverters._
+import jFutureSyntax._
+
+case class SecGroupData(
+  vpcId: String,
+  fromPort: Int,
+  toPort: Int,
+  cidrIp: String
+)
+
+case class InstanceData(
+  ip: String,
+  vpcId: String,
+  subnetId: String
+) {
+  def cidrIp: String = ip + "/32"
+}
+
+trait EC2Service[F[_]] {
+
+  def getInstanceData(id: String): F[Option[InstanceData]]
+  
+  def getSecGroup(name: String): F[Option[String]]
+  def createSecGroup(name: String, descr: String, data: SecGroupData): F[String]
+
+  def getOrCreateSecGroup(name: String, descr: String, data: SecGroupData)(implicit M: Monad[F]): F[String] = {
+    for {
+      curr <- getSecGroup(name)
+      out <- curr match {
+        case Some(_) => M.pure(name)
+        case None => createSecGroup(name, descr, data)
+      }
+    } yield out
+  }
+}
+
+object EC2Service {
+
+  def fromSdk(ec2Client: EC2AsyncClient): EC2Service[IO] = {
+    new EC2Service[IO] {
+
+      override def getSecGroup(name: String): IO[Option[String]] = {
+        val req = DescribeSecurityGroupsRequest.builder().groupNames(name).build()
+        val io = ec2Client.describeSecurityGroups(req).toIO
+
+        io.map(r => r.securityGroups().asScala.headOption.map(_.groupName()))
+          .handleErrorWith(e => e match {
+            case _: software.amazon.awssdk.services.ec2.model.EC2Exception => IO.pure(None)
+            case _ => IO.raiseError(e)
+          })
+      }
+
+      override def createSecGroup(
+        name: String,
+        descr: String,
+        data: SecGroupData
+      ): IO[String] = {
+
+        def ingressReq(data: SecGroupData, groupId: String): AuthorizeSecurityGroupIngressRequest = {
+          import data._
+
+          AuthorizeSecurityGroupIngressRequest.builder()
+            .groupId(groupId)
+            .cidrIp(cidrIp)
+            .fromPort(fromPort)
+            .toPort(toPort)
+            .ipProtocol("TCP")
+            .build()
+        }
+
+        val createReq = CreateSecurityGroupRequest.builder()
+          .groupName(name)
+          .description(descr)
+          .vpcId(data.vpcId)
+          .build()
+
+        for {
+          resp <- ec2Client.createSecurityGroup(createReq).toIO
+          groupId = resp.groupId()
+          ingReq = ingressReq(data, groupId)
+          _ <- ec2Client.authorizeSecurityGroupIngress(ingReq).toIO
+        } yield groupId
+      }
+
+      override def getInstanceData(id: String): IO[Option[InstanceData]] = {
+        def extractData(resp: DescribeInstancesResponse): Option[InstanceData] = {
+          for {
+            reservation <- resp.reservations().asScala.headOption
+            instance <- reservation.instances().asScala.headOption
+          } yield InstanceData(instance.privateIpAddress(), instance.vpcId(), instance.subnetId())
+        }
+
+        val req = DescribeInstancesRequest.builder().instanceIds(id).build()
+        for {
+          resp <- ec2Client.describeInstances(req).toIO
+          data = extractData(resp)
+        } yield data
+      }
+    }
+  }
+
+}
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
new file mode 100644
index 000000000..7027f95b1
--- /dev/null
+++ b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
@@ -0,0 +1,47 @@
+package io.hydrosphere.mist.aws
+
+import com.typesafe.config.{Config, ConfigFactory}
+import io.hydrosphere.mist.utils.ConfigUtils._
+
+case class ProvisionData(
+  sshKeyPair: String,
+  sshKeyPath: String,
+  accessKey: String,
+  secretKey: String,
+  subnetId: String,
+  region: String,
+  additionalGroup: String,
+  emrRole: String,
+  emrEc2Role: String
+)
+
+object ConfigPatcher {
+
+  def path(mistConfig: Config, provisionData: ProvisionData): Config = {
+    import com.typesafe.config.ConfigValueFactory._
+    import scala.collection.JavaConverters._
+
+    import provisionData._
+
+    val configKeys = Map(
+      "name" -> "default_emr",
+      "type" -> "aws_emr",
+      "sshKeyPair" -> sshKeyPair,
+      "sshKeyPath" -> sshKeyPath,
+      "accessKey"-> accessKey,
+      "secretKey" -> secretKey,
+      "subnetId" -> subnetId,
+      "region" -> region,
+      "additionalGroup" -> additionalGroup,
+      "emrRole" -> emrRole,
+      "emrEc2Role" -> emrEc2Role
+    ).map({case (k, v) => k -> fromAnyRef(v)})
+
+    val provisioner = fromMap(configKeys.asJava).toConfig
+    val entries = fromIterable(Seq(provisioner).asJava)
+
+    mistConfig.withValue("provision", entries)
+  }
+
+}
+
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
new file mode 100644
index 000000000..29e240d4b
--- /dev/null
+++ b/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
@@ -0,0 +1,97 @@
+package io.hydrosphere.mist.aws
+
+import java.util.concurrent.CompletableFuture
+import java.util.function.BiConsumer
+
+import cats._
+import cats.implicits._
+import cats.Monad
+import cats.effect._
+import software.amazon.awssdk.services.iam.IAMAsyncClient
+import software.amazon.awssdk.services.iam.model.{AttachRolePolicyRequest, CreateRoleRequest, GetRoleRequest, Role}
+
+import jFutureSyntax._
+
+import scala.io.Source
+
+case class AWSRoleData(
+  trustPolicyJson: String,
+  permissionsArn: String
+)
+
+object AWSRoleData {
+
+  private def readResourceJson(name: String): String = {
+    val stream = getClass.getResourceAsStream(name)
+    Source.fromInputStream(stream).mkString.replace("\n", "")
+  }
+
+  val EMR = AWSRoleData(
+    readResourceJson("/trustEMR.json"),
+    "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole"
+  )
+
+  val EC2EMR = AWSRoleData(
+    readResourceJson("/trustEC2EMR.json"),
+    permissionsArn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
+  )
+
+}
+
+case class AWSRole(
+  name: String,
+  description: String,
+  data: AWSRoleData
+)
+
+trait IAMService[F[_]] {
+
+  def createRole(role: AWSRole): F[AWSRole]
+  def getRole(name: String): F[Option[String]]
+
+  def getOrCreate(role: AWSRole)(implicit m: Monad[F]): F[AWSRole] = {
+    for {
+      out <- getRole(role.name)
+      role <- out match {
+        case Some(_) => m.pure(role)
+        case None => createRole(role)
+      }
+    } yield role
+  }
+}
+
+object IAMService {
+
+  def fromSdk(iamClient: IAMAsyncClient): IAMService[IO] =
+    new IAMService[IO] {
+
+      override def getRole(name: String): IO[Option[String]] = {
+        val req = GetRoleRequest.builder().roleName(name).build()
+
+        iamClient.getRole(req).toIO
+          .map(r => Option(r.role().roleName()))
+          .handleErrorWith({
+            case _: software.amazon.awssdk.services.iam.model.NoSuchEntityException => IO.pure(None)
+            case e => IO.raiseError(e)
+          })
+      }
+
+      override def createRole(role: AWSRole): IO[AWSRole] = {
+        val createReq = CreateRoleRequest.builder()
+          .roleName(role.name)
+          .description(role.description)
+          .assumeRolePolicyDocument(role.data.trustPolicyJson)
+          .build()
+
+        val attachReq = AttachRolePolicyRequest.builder()
+          .policyArn(role.data.permissionsArn)
+          .roleName(role.name)
+          .build()
+
+        for {
+          _ <- iamClient.createRole(createReq).toIO
+          _ <- iamClient.attachRolePolicy(attachReq).toIO
+        } yield role
+      }
+  }
+}
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala b/mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala
index 5ae394646..7e86727a6 100644
--- a/mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala
+++ b/mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala
@@ -18,9 +18,11 @@ object ConfigUtils {
 
     def getOptString(path: String): Option[String] = getOpt(path, _.getString(path))
     def getOptInt(path: String): Option[Int] = getOpt(path, _.getInt(path))
+    def getOptConfig(path: String): Option[Config] = getOpt(path, _.getConfig(path))
 
     def getOpt[A](path: String, f: Config => A): Option[A] =
       if (c.hasPath(path)) Option(f(c)) else None
+
   }
 }
 
diff --git a/project/Library.scala b/project/Library.scala
index a5b8dc761..e001a4f6c 100644
--- a/project/Library.scala
+++ b/project/Library.scala
@@ -16,7 +16,8 @@ object Library {
   val pahoMqtt = "org.eclipse.paho" % "org.eclipse.paho.client.mqttv3" % "1.1.0"
   val kafka = "org.apache.kafka" %% "kafka" % "0.10.2.0" exclude("log4j", "log4j") exclude("org.slf4j","slf4j-log4j12")
 
-  val cats = "org.typelevel" %% "cats" % "0.9.0"
+  val cats = "org.typelevel" %% "cats-core" % "1.3.1"
+  val catsEffect = "org.typelevel" %% "cats-effect" % "1.0.0"
 
   val scalaTest = "org.scalatest" %% "scalatest" % "3.0.1"
   val junit = "junit" % "junit" % "4.12"
@@ -53,9 +54,13 @@ object Library {
     "org.apache.spark" %% "spark-core" % v,
     "org.apache.spark" %% "spark-sql" % v,
     "org.apache.spark" %% "spark-hive" % v,
-    "org.apache.spark" %% "spark-streaming" % v,
+    "org.apache.spark" %% "spark-streaming" % v
   )
 
   val jsr305 = "com.google.code.findbugs" % "jsr305" % "1.3.9"
 
+  val scalaSsh = "com.decodified" %% "scala-ssh" % "0.9.0"
+  val awsSdkEC2 = "software.amazon.awssdk" % "ec2" % "2.0.0-preview-10"
+  val awsSdkEMR = "software.amazon.awssdk" % "emr" % "2.0.0-preview-10"
+  val awsSdkIAM = "software.amazon.awssdk" % "iam" % "2.0.0-preview-10"
 }
diff --git a/project/plugins.sbt b/project/plugins.sbt
index c73544570..e67014937 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,3 +1,4 @@
+addSbtPlugin("org.lyranthe.sbt" % "partial-unification" % "1.1.2")
 addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0")
 addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.0")
 addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.2")

From 2440c2bb33c6d5a3ed692669200c1fb546a03582 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Tue, 18 Sep 2018 22:55:55 +0300
Subject: [PATCH 09/30] aws-init-setup.jar + mist config patching

---
 mist.sbt                                      |  4 ++-
 .../src/main/resources/trustEC2EMR.json       |  0
 .../src/main/resources/trustEMR.json          |  0
 .../io/hydrosphere/mist/aws/AwsSetup.scala    |  0
 .../mist/aws/CompletableFutureOps.scala       |  3 +-
 .../io/hydrosphere/mist/aws/EC2Service.scala  |  2 +-
 .../io/hydrosphere/mist/aws/GenConfig.scala   | 27 +++++++++++---
 .../io/hydrosphere/mist/aws/IAMService.scala  |  2 +-
 .../scala/io/hydrosphere/mist/aws/Main.scala  | 35 +++++++++++++++++++
 9 files changed, 64 insertions(+), 9 deletions(-)
 rename mist/{aws-config-gen => aws-init-setup}/src/main/resources/trustEC2EMR.json (100%)
 rename mist/{aws-config-gen => aws-init-setup}/src/main/resources/trustEMR.json (100%)
 rename mist/{aws-config-gen => aws-init-setup}/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala (100%)
 rename mist/{aws-config-gen => aws-init-setup}/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala (97%)
 rename mist/{aws-config-gen => aws-init-setup}/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala (99%)
 rename mist/{aws-config-gen => aws-init-setup}/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala (55%)
 rename mist/{aws-config-gen => aws-init-setup}/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala (99%)
 create mode 100644 mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala

diff --git a/mist.sbt b/mist.sbt
index c960600a7..9050f7f11 100644
--- a/mist.sbt
+++ b/mist.sbt
@@ -131,7 +131,7 @@ lazy val worker = project.in(file("mist/worker"))
     )
   )
 
-lazy val awsConfigGen = project.in(file("mist/aws-config-gen"))
+lazy val awsInitSetup = project.in(file("mist/aws-init-setup"))
   .dependsOn(core % "compile->compile;test->test")
   .settings(commonSettings: _*)
   .settings(commonAssemblySettings: _*)
@@ -166,6 +166,8 @@ lazy val root = project.in(file("."))
         CpFile("configs/logging").to("configs"),
         CpFile(assembly.in(master, assembly).value).as("mist-master.jar"),
         CpFile(assembly.in(worker, assembly).value).as("mist-worker.jar"),
+        MkDir("utils"),
+        CpFile(assembly.in(awsInitSetup, assembly).value).as("aws-init-setup.jar").to("utils"),
         CpFile(Ui.ui.value).as("ui")
       )
     },
diff --git a/mist/aws-config-gen/src/main/resources/trustEC2EMR.json b/mist/aws-init-setup/src/main/resources/trustEC2EMR.json
similarity index 100%
rename from mist/aws-config-gen/src/main/resources/trustEC2EMR.json
rename to mist/aws-init-setup/src/main/resources/trustEC2EMR.json
diff --git a/mist/aws-config-gen/src/main/resources/trustEMR.json b/mist/aws-init-setup/src/main/resources/trustEMR.json
similarity index 100%
rename from mist/aws-config-gen/src/main/resources/trustEMR.json
rename to mist/aws-init-setup/src/main/resources/trustEMR.json
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
similarity index 100%
rename from mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
rename to mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala
similarity index 97%
rename from mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala
rename to mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala
index d13b421c2..4f8d1b36b 100644
--- a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala
@@ -27,8 +27,9 @@ final class CompletableFutureOps[A](val cf: CompletableFuture[A]) extends AnyVal
   }
 
   def toIO: IO[A] = IO.fromFuture(IO(toFuture))
+
 }
 
-object jFutureSyntax {
+object JFutureSyntax {
   implicit def cfSyntax[A](cf: CompletableFuture[A]): CompletableFutureOps[A] = new CompletableFutureOps(cf)
 }
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
similarity index 99%
rename from mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
rename to mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
index 2a4313dff..ebadaece9 100644
--- a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
@@ -7,7 +7,7 @@ import software.amazon.awssdk.services.ec2.EC2AsyncClient
 import software.amazon.awssdk.services.ec2.model._
 
 import scala.collection.JavaConverters._
-import jFutureSyntax._
+import JFutureSyntax._
 
 case class SecGroupData(
   vpcId: String,
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
similarity index 55%
rename from mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
rename to mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
index 7027f95b1..55c690392 100644
--- a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
@@ -1,6 +1,9 @@
 package io.hydrosphere.mist.aws
 
-import com.typesafe.config.{Config, ConfigFactory}
+import java.io.File
+import java.nio.file.{Files, Path}
+
+import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions}
 import io.hydrosphere.mist.utils.ConfigUtils._
 
 case class ProvisionData(
@@ -17,11 +20,11 @@ case class ProvisionData(
 
 object ConfigPatcher {
 
-  def path(mistConfig: Config, provisionData: ProvisionData): Config = {
+  def patch(mistConfig: Config, data: ProvisionData): Config = {
     import com.typesafe.config.ConfigValueFactory._
     import scala.collection.JavaConverters._
 
-    import provisionData._
+    import data._
 
     val configKeys = Map(
       "name" -> "default_emr",
@@ -37,10 +40,24 @@ object ConfigPatcher {
       "emrEc2Role" -> emrEc2Role
     ).map({case (k, v) => k -> fromAnyRef(v)})
 
-    val provisioner = fromMap(configKeys.asJava).toConfig
+    val provisioner = fromMap(configKeys.asJava)
     val entries = fromIterable(Seq(provisioner).asJava)
 
-    mistConfig.withValue("provision", entries)
+    mistConfig.withValue("mist.provision", entries)
+  }
+
+  def patchFile(filePath: Path, data: ProvisionData): Unit = {
+    val orig = ConfigFactory.parseFile(filePath.toFile)
+    val patched = patch(orig, data)
+
+    val renderOpts = ConfigRenderOptions.defaults()
+      .setComments(false)
+      .setOriginComments(false)
+      .setJson(false)
+      .setFormatted(true)
+
+    val rawConfig = patched.root().render(renderOpts)
+    Files.write(filePath, rawConfig.getBytes)
   }
 
 }
diff --git a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
similarity index 99%
rename from mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
rename to mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
index 29e240d4b..0416427a1 100644
--- a/mist/aws-config-gen/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
@@ -10,7 +10,7 @@ import cats.effect._
 import software.amazon.awssdk.services.iam.IAMAsyncClient
 import software.amazon.awssdk.services.iam.model.{AttachRolePolicyRequest, CreateRoleRequest, GetRoleRequest, Role}
 
-import jFutureSyntax._
+import JFutureSyntax._
 
 import scala.io.Source
 
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
new file mode 100644
index 000000000..658244d85
--- /dev/null
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
@@ -0,0 +1,35 @@
+package io.hydrosphere.mist.aws
+
+import java.nio.file.Paths
+
+object Main {
+
+  def main(args: Array[String]): Unit = {
+    val instanceId = args(0)
+    val accessKey = args(1)
+    val accessSecret = args(2)
+    val region = args(3)
+    val configPath = args(4)
+
+    val sshKeyPair = args(5)
+    val sshKeyPath = args(6)
+
+    val setup = AwsSetup.create(accessKey, accessSecret, region)
+    val out = setup.setup(instanceId).unsafeRunSync()
+
+    val provisionData = ProvisionData(
+      sshKeyPair = sshKeyPair,
+      sshKeyPath = sshKeyPath,
+      accessKey = accessKey,
+      secretKey = accessSecret,
+      subnetId = out.subnetId,
+      region = region,
+      additionalGroup = out.securityGroupId,
+      emrRole = out.emrRole,
+      emrEc2Role = out.ec2EmrRole
+    )
+
+    ConfigPatcher.patchFile(Paths.get(configPath), provisionData)
+  }
+
+}

From 898475f668904821ae9a85a1021f8d7d102d6fc4 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Wed, 19 Sep 2018 20:14:46 +0300
Subject: [PATCH 10/30] aws - initial provision

---
 cloud_install/aws/install.sh                  |  8 +++++
 cloud_install/aws/template.json               |  8 +++--
 .../io/hydrosphere/mist/aws/AwsSetup.scala    | 16 ++++++----
 .../io/hydrosphere/mist/aws/EC2Service.scala  | 29 +++++++++++++++++++
 .../scala/io/hydrosphere/mist/aws/Main.scala  | 10 ++++---
 5 files changed, 58 insertions(+), 13 deletions(-)

diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
index 42ed31f8b..8986b9b36 100644
--- a/cloud_install/aws/install.sh
+++ b/cloud_install/aws/install.sh
@@ -3,8 +3,13 @@
 apt-get update
 apt-get install -y nginx openjdk-8-jdk apache2-utils
 
+JAVA=$(which java)
+
 source ~/.aws_setup_data
 
+ssh-keygen -b 2048 -t rsa -f ~/.mist_key -q -N ""
+chmod 600 ~/.mist_key
+
 echo "<p>Mist will start soon</p>" > /var/www/html/index.html
 cat << EOF > /etc/nginx/sites-enabled/default
 server {
@@ -33,6 +38,9 @@ mv spark-2.3.0-bin-hadoop2.7 spark
 # InfoProvider can't connect to master
 sleep 30
 
+INSTANCE_ID=$(ec2metadata --instance-id)
+$JAVA -cp /opt/mist/utils/aws-init-setup.jar io.hydrosphere.mist.aws.Main $INSTANCE_ID $ACCESS_KEY_ID $ACCESS_KEY_SECRET $AWS_REGION /opt/mist/configs/default.conf ~/.mist_key.pub
+
 SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
 
 htpasswd -b -c /etc/nginx/.htpasswd $MIST_LOGIN $MIST_PASSWORD
diff --git a/cloud_install/aws/template.json b/cloud_install/aws/template.json
index 06344f119..0ae8c44d2 100644
--- a/cloud_install/aws/template.json
+++ b/cloud_install/aws/template.json
@@ -21,7 +21,8 @@
     },
     "AccessKeySecret": {
       "Type" : "String",
-      "Description" : "AWS secret access key"
+      "Description" : "AWS secret access key",
+      "NoEcho" : "true"
     },
     "AZ": {
        "Description": "Availability Zone of the Subnet",
@@ -33,7 +34,8 @@
     },
     "MistPassword": {
       "Type" : "String",
-      "Description" : "Basic auth password"
+      "Description" : "Basic auth password",
+      "NoEcho" : "true"
     }
   },
   "Resources": {
@@ -51,7 +53,7 @@
               "#!/bin/bash -ex",
               { "Fn::Join": ["", ["echo ACCESS_KEY_ID=", { "Ref": "AccessKeyID"},  " > ~/.aws_setup_data"]]},
               { "Fn::Join": ["", ["echo ACCESS_KEY_SECRET=", { "Ref": "AccessKeySecret"},  " >> ~/.aws_setup_data"]]},
-              { "Fn::Join": ["", ["echo AWS_AZ=", { "Ref": "AZ"},  " >> ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo AWS_REGION=", { "Ref" : "AWS::Region" },  " >> ~/.aws_setup_data"]]},
               { "Fn::Join": ["", ["echo MIST_LOGIN=", { "Ref": "MistLogin"},  " >> ~/.aws_setup_data"]]},
               { "Fn::Join": ["", ["echo MIST_PASSWORD=", { "Ref": "MistPassword"},  " >> ~/.aws_setup_data"]]},
               "curl http://repo.hydrosphere.io/hydrosphere/static/preview/install.sh | bash"
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
index 05fec0134..89557c5d4 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
@@ -12,12 +12,13 @@ case class SetupData(
   subnetId: String,
   securityGroupId: String,
   emrRole: String,
-  ec2EmrRole: String
+  ec2EmrRole: String,
+  sshKeyPairName: String
 )
 
 trait AwsSetup[F[_]] {
 
-  def setup(instanceId: String): F[SetupData]
+  def setup(instanceId: String, sshKey: String): F[SetupData]
 
 }
 
@@ -28,10 +29,12 @@ object AwsSetup {
 
       val ec2EmrRole = AWSRole("mist-EMREC2", "default emr ec2 role", AWSRoleData.EC2EMR)
       val emrRole = AWSRole("mist-EMR", "default emr role", AWSRoleData.EMR)
-      val secGroupName = "mist-internal"
       val secGroupDecr = "Master-worker communications"
 
-      override def setup(instanceId: String): F[SetupData] = {
+      def secGroupName(id: String): String = s"mist-internal-$id"
+      def keyName(id: String): String = s"mist-$id"
+
+      override def setup(instanceId: String, sshKey: String): F[SetupData] = {
         for {
           maybeInst <- ec2.getInstanceData(instanceId)
           data <- maybeInst match {
@@ -42,8 +45,9 @@ object AwsSetup {
           emrRole <- iam.getOrCreate(emrRole)
 
           secGroupData = SecGroupData(data.vpcId, 0, 65535, data.cidrIp)
-          secGroupId <- ec2.getOrCreateSecGroup(secGroupName, secGroupDecr, secGroupData)
-        } yield SetupData(data.subnetId, secGroupId, emrRole.name, ec2EmrRole.name)
+          secGroupId <- ec2.getOrCreateSecGroup(secGroupName(instanceId), secGroupDecr, secGroupData)
+          keyName <- ec2.getOrCreateKeyPair(keyName(instanceId), sshKey)
+        } yield SetupData(data.subnetId, secGroupId, emrRole.name, ec2EmrRole.name, keyName)
       }
     }
   }
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
index ebadaece9..ec4c17d81 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
@@ -27,6 +27,19 @@ case class InstanceData(
 trait EC2Service[F[_]] {
 
   def getInstanceData(id: String): F[Option[InstanceData]]
+
+  def getKeyPair(name: String): F[Option[String]]
+  def createKeyPair(name: String, key: String): F[String]
+
+  def getOrCreateKeyPair(name: String, key: String)(implicit M: Monad[F]): F[String] = {
+    for {
+      curr <- getKeyPair(name)
+      out <- curr match {
+        case Some(_) => M.pure(name)
+        case None => createKeyPair(name, key)
+      }
+    } yield out
+  }
   
   def getSecGroup(name: String): F[Option[String]]
   def createSecGroup(name: String, descr: String, data: SecGroupData): F[String]
@@ -104,7 +117,23 @@ object EC2Service {
           data = extractData(resp)
         } yield data
       }
+
+      override def getKeyPair(name: String): IO[Option[String]] = {
+        val req = DescribeKeyPairsRequest.builder().keyNames(name).build()
+        ec2Client.describeKeyPairs(req).toIO
+          .map(resp => resp.keyPairs().asScala.headOption.map(i => i.keyName()))
+          .handleErrorWith(e => e match {
+            case _: software.amazon.awssdk.services.ec2.model.EC2Exception => IO.pure(None)
+            case _ => IO.raiseError(e)
+          })
+      }
+
+      override def createKeyPair(name: String, key: String): IO[String] = {
+        val req = ImportKeyPairRequest.builder().keyName(name).publicKeyMaterial(key).build()
+        ec2Client.importKeyPair(req).toIO.map(resp => resp.keyName())
+      }
     }
+
   }
 
 }
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
index 658244d85..6e690fbf3 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
@@ -2,6 +2,8 @@ package io.hydrosphere.mist.aws
 
 import java.nio.file.Paths
 
+import scala.io.Source
+
 object Main {
 
   def main(args: Array[String]): Unit = {
@@ -11,14 +13,14 @@ object Main {
     val region = args(3)
     val configPath = args(4)
 
-    val sshKeyPair = args(5)
-    val sshKeyPath = args(6)
+    val sshKeyPath = args(5)
+    val sshKey = Source.fromFile(Paths.get(sshKeyPath).toFile).mkString
 
     val setup = AwsSetup.create(accessKey, accessSecret, region)
-    val out = setup.setup(instanceId).unsafeRunSync()
+    val out = setup.setup(instanceId, sshKey).unsafeRunSync()
 
     val provisionData = ProvisionData(
-      sshKeyPair = sshKeyPair,
+      sshKeyPair = out.sshKeyPairName,
       sshKeyPath = sshKeyPath,
       accessKey = accessKey,
       secretKey = accessSecret,

From 918b7ff0551f7dce45959d8c3543cca49c225b5c Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Wed, 19 Sep 2018 21:09:06 +0300
Subject: [PATCH 11/30] core -> common; initial struct for cluster agent

---
 mist.sbt                                      | 32 ++++++++++++++-----
 .../io/hydrosphere/mist/agent/Agent.scala     |  8 +++++
 .../hydrosphere/mist/common}/CommonData.scala |  2 +-
 .../mist/common}/FunctionInfoData.scala       |  2 +-
 .../mist/common}/PythonEntrySettings.scala    |  2 +-
 .../mist/common}/logging/Level.scala          |  2 +-
 .../mist/common}/logging/LogEvent.scala       |  2 +-
 .../hydrosphere/mist/utils/Collections.scala  |  0
 .../hydrosphere/mist/utils/ConfigUtils.scala  |  0
 .../io/hydrosphere/mist/utils/EitherOps.scala |  0
 .../io/hydrosphere/mist/utils/FutureOps.scala |  0
 .../io/hydrosphere/mist/utils/Logger.scala    |  0
 .../io/hydrosphere/mist/utils/NetUtils.scala  |  0
 .../io/hydrosphere/mist/utils/TryLoad.scala   |  0
 .../hydrosphere/mist/utils/akka/ActorF.scala  |  0
 .../mist/utils/akka/ActorRegHub.scala         |  0
 .../mist/utils/akka/RestartSupervisor.scala   |  0
 .../mist/utils/akka/WhenTerminated.scala      |  0
 .../scala/io/hydrosphere/mist/utils/fs.scala  |  0
 .../src/test/resources/log4j.properties       |  0
 .../mist/common}/MockitoSugar.scala           |  2 +-
 .../common}/PythonEntrySettingsSpec.scala     |  2 +-
 .../common}/logging/MistLoggingSpec.scala     |  2 +-
 .../mist/utils/akka/ActorRegSpec.scala        |  0
 .../hydrosphere/mist/master/JobDetails.scala  |  2 +-
 .../hydrosphere/mist/master/MainService.scala |  2 +-
 .../mist/master/MasterServer.scala            |  2 +-
 .../io/hydrosphere/mist/master/Messages.scala |  4 +--
 .../mist/master/execution/ContextEvent.scala  |  2 +-
 .../master/execution/ContextFrontend.scala    |  2 +-
 .../mist/master/execution/ExecutionInfo.scala |  2 +-
 .../master/execution/ExecutionService.scala   |  2 +-
 .../mist/master/execution/JobActor.scala      |  2 +-
 .../mist/master/execution/SpawnSettings.scala |  2 +-
 .../mist/master/execution/WorkerLink.scala    |  2 +-
 .../execution/status/StatusReporter.scala     |  2 +-
 .../workers/ExclusiveConnector.scala          |  4 +--
 .../execution/workers/PerJobConnection.scala  |  2 +-
 .../execution/workers/SharedConnector.scala   |  4 +--
 .../execution/workers/WorkerBridge.scala      |  2 +-
 .../execution/workers/WorkerRunner.scala      |  2 +-
 .../workers/starter/DockerStarter.scala       |  2 +-
 .../workers/starter/LocalSparkSubmit.scala    |  2 +-
 .../workers/starter/ManualStarter.scala       |  2 +-
 .../workers/starter/SparkSubmitBuilder.scala  |  2 +-
 .../workers/starter/WorkerStarter.scala       |  2 +-
 .../mist/master/interfaces/http/models.scala  |  2 +-
 .../mist/master/interfaces/jsonCodecs.scala   |  4 +--
 .../jobs/FunctionInfoProviderRunner.scala     |  4 +--
 .../mist/master/jobs/FunctionsService.scala   |  4 +--
 .../mist/master/logging/JobLogger.scala       |  2 +-
 .../mist/master/logging/LogStreams.scala      |  2 +-
 .../mist/master/logging/writers.scala         |  2 +-
 .../mist/master/models/models.scala           |  4 +--
 .../mist/master/store/H2JobsRepository.scala  |  2 +-
 .../mist/master/ContextsCrudLikeSpec.scala    |  2 +-
 .../mist/master/MainServiceSpec.scala         |  4 +--
 .../io/hydrosphere/mist/master/TestData.scala |  4 +--
 .../artifact/ArtifactRepositorySpec.scala     |  2 +-
 .../execution/ContextFrontendSpec.scala       |  4 +--
 .../master/execution/ContextsMasterSpec.scala |  2 +-
 .../master/execution/ExecutionInfoSpec.scala  |  2 +-
 .../execution/ExecutionServiceSpec.scala      |  2 +-
 .../mist/master/execution/JobActorSpec.scala  |  4 +--
 .../master/execution/SpawnSettingsSpec.scala  |  2 +-
 .../execution/status/StatusReporterSpec.scala |  2 +-
 .../workers/ExclusiveConnectorSpec.scala      |  2 +-
 .../workers/SharedConnectorSpec.scala         |  2 +-
 .../execution/workers/WorkerBridgeSpec.scala  |  2 +-
 .../execution/workers/WorkerHubSpec.scala     |  2 +-
 .../execution/workers/WorkerRunnerSpec.scala  |  4 +--
 .../starter/SparkSubmitBuilderSpec.scala      |  2 +-
 .../master/interfaces/http/DevApiSpec.scala   |  6 ++--
 .../interfaces/http/HttpApiV2Spec.scala       |  4 +--
 .../master/interfaces/http/WsApiSpec.scala    |  2 +-
 .../master/jobs/FunctionsServiceSpec.scala    |  6 ++--
 .../mist/master/logging/LogStreamsSpec.scala  |  4 +--
 .../mist/master/logging/LogWriterSpec.scala   |  2 +-
 .../mist/master/store/H2RepoSpec.scala        |  2 +-
 .../mist/job/FunctionInfoExtractor.scala      | 10 +++---
 .../mist/job/FunctionInfoProvider.scala       |  4 +--
 .../mist/job/FunctionInfoProviderActor.scala  |  4 +--
 .../mist/job/FunctionInstanceLoader.scala     |  2 +-
 .../mist/job/JvmLangDetector.scala            |  2 +-
 .../mist/python/PythonExecuter.scala          |  4 +--
 .../mist/worker/MasterBridge.scala            |  2 +-
 .../mist/worker/RequestSetup.scala            |  2 +-
 .../hydrosphere/mist/worker/WorkerActor.scala |  2 +-
 .../mist/worker/logging/RemoteAppender.scala  |  2 +-
 .../worker/logging/RemoteLogsWriter.scala     |  2 +-
 .../mist/worker/runners/JobRunner.scala       |  2 +-
 .../mist/worker/runners/ScalaRunner.scala     |  2 +-
 .../worker/runners/python/PythonRunner.scala  |  2 +-
 .../wrappers/ConfigurationWrapper.scala       |  2 +-
 .../mist/job/FunctionInfoExtractorSpec.scala  |  4 +--
 .../job/FunctionInfoProviderActorSpec.scala   |  4 +--
 .../mist/worker/MasterBridgeSpec.scala        |  4 +--
 .../mist/worker/RequestSetupSpec.scala        |  4 +--
 .../mist/worker/WorkerActorSpec.scala         |  4 +--
 .../worker/logging/RemoteAppenderSpec.scala   |  4 +--
 100 files changed, 147 insertions(+), 123 deletions(-)
 create mode 100644 mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/Agent.scala
 rename mist/{core/src/main/scala/io/hydrosphere/mist/core => common/src/main/scala/io/hydrosphere/mist/common}/CommonData.scala (99%)
 rename mist/{core/src/main/scala/io/hydrosphere/mist/core => common/src/main/scala/io/hydrosphere/mist/common}/FunctionInfoData.scala (94%)
 rename mist/{core/src/main/scala/io/hydrosphere/mist/core => common/src/main/scala/io/hydrosphere/mist/common}/PythonEntrySettings.scala (93%)
 rename mist/{core/src/main/scala/io/hydrosphere/mist/core => common/src/main/scala/io/hydrosphere/mist/common}/logging/Level.scala (90%)
 rename mist/{core/src/main/scala/io/hydrosphere/mist/core => common/src/main/scala/io/hydrosphere/mist/common}/logging/LogEvent.scala (97%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/Collections.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/Logger.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala (100%)
 rename mist/{core => common}/src/main/scala/io/hydrosphere/mist/utils/fs.scala (100%)
 rename mist/{core => common}/src/test/resources/log4j.properties (100%)
 rename mist/{core/src/test/scala/io/hydrosphere/mist/core => common/src/test/scala/io/hydrosphere/mist/common}/MockitoSugar.scala (98%)
 rename mist/{core/src/test/scala/io/hydrosphere/mist/core => common/src/test/scala/io/hydrosphere/mist/common}/PythonEntrySettingsSpec.scala (93%)
 rename mist/{core/src/test/scala/io/hydrosphere/mist/core => common/src/test/scala/io/hydrosphere/mist/common}/logging/MistLoggingSpec.scala (96%)
 rename mist/{core => common}/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala (100%)

diff --git a/mist.sbt b/mist.sbt
index 9050f7f11..bdbc30727 100644
--- a/mist.sbt
+++ b/mist.sbt
@@ -50,13 +50,12 @@ lazy val mistLib = project.in(file("mist-lib"))
     test in Test := Def.sequential(test in Test, PyProject.pyTest in Test).value
   )
 
-lazy val core = project.in(file("mist/core"))
+lazy val common = project.in(file("mist/common"))
   .dependsOn(mistLib)
   .settings(commonSettings: _*)
   .settings(
-    name := "mist-core",
+    name := "mist-common",
     scalacOptions ++= commonScalacOptions,
-    libraryDependencies ++= Library.spark(sparkVersion.value).map(_ % "runtime"),
     libraryDependencies ++= Seq(
       Library.Akka.actor,
       Library.slf4j,
@@ -67,7 +66,7 @@ lazy val core = project.in(file("mist/core"))
   )
 
 lazy val master = project.in(file("mist/master"))
-  .dependsOn(core % "compile->compile;test->test")
+  .dependsOn(common % "compile->compile;test->test")
   .settings(commonSettings: _*)
   .settings(commonAssemblySettings: _*)
   .enablePlugins(BuildInfoPlugin)
@@ -99,8 +98,25 @@ lazy val master = project.in(file("mist/master"))
     buildInfoPackage := "io.hydrosphere.mist"
   )
 
+lazy val clusterAgent = project.in(file("mist/cluster-agent"))
+  .dependsOn(common % "compile->compile;test->test")
+  .settings(commonSettings: _*)
+  .settings(commonAssemblySettings: _*)
+  .settings(
+    name := "mist-cluster-agent",
+    scalacOptions ++= commonScalacOptions,
+    libraryDependencies ++= Library.Akka.base,
+    libraryDependencies ++= Seq(
+      Library.slf4jLog4j, Library.typesafeConfig, Library.scopt,
+      Library.cats,
+      Library.jsr305 % "provided",
+      Library.scalaTest % "test",
+      Library.Akka.testKit % "test"
+    )
+  )
+
 lazy val worker = project.in(file("mist/worker"))
-  .dependsOn(core % "compile->compile;test->test")
+  .dependsOn(common % "compile->compile;test->test")
   .settings(commonSettings: _*)
   .settings(commonAssemblySettings: _*)
   .settings(
@@ -132,11 +148,11 @@ lazy val worker = project.in(file("mist/worker"))
   )
 
 lazy val awsInitSetup = project.in(file("mist/aws-init-setup"))
-  .dependsOn(core % "compile->compile;test->test")
+  .dependsOn(common % "compile->compile;test->test")
   .settings(commonSettings: _*)
   .settings(commonAssemblySettings: _*)
   .settings(
-    name := "mist-aws-config-gen",
+    name := "mist-aws-init-setup",
     scalacOptions ++= commonScalacOptions,
     libraryDependencies ++= Seq(
       Library.slf4jLog4j, Library.typesafeConfig,
@@ -148,7 +164,7 @@ lazy val awsInitSetup = project.in(file("mist/aws-init-setup"))
   )
 
 lazy val root = project.in(file("."))
-  .aggregate(mistLib, core, master, worker, examples)
+  .aggregate(mistLib, common, master, worker, examples)
   .dependsOn(master)
   .enablePlugins(DockerPlugin)
   .settings(commonSettings: _*)
diff --git a/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/Agent.scala b/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/Agent.scala
new file mode 100644
index 000000000..53cc5faaa
--- /dev/null
+++ b/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/Agent.scala
@@ -0,0 +1,8 @@
+package io.hydrosphere.mist.agent
+
+object Agent {
+
+  def main(args: Array[String]): Unit = {
+    sys.exit(1)
+  }
+}
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/CommonData.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
similarity index 99%
rename from mist/core/src/main/scala/io/hydrosphere/mist/core/CommonData.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
index 8c244b807..7ca323182 100644
--- a/mist/core/src/main/scala/io/hydrosphere/mist/core/CommonData.scala
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.core
+package io.hydrosphere.mist.common
 
 import akka.actor.ActorRef
 import mist.api.data.{JsData, JsMap}
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/FunctionInfoData.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/FunctionInfoData.scala
similarity index 94%
rename from mist/core/src/main/scala/io/hydrosphere/mist/core/FunctionInfoData.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/common/FunctionInfoData.scala
index 8374a83cf..2f8634140 100644
--- a/mist/core/src/main/scala/io/hydrosphere/mist/core/FunctionInfoData.scala
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/FunctionInfoData.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.core
+package io.hydrosphere.mist.common
 
 import mist.api.UserInputArgument
 
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/PythonEntrySettings.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/PythonEntrySettings.scala
similarity index 93%
rename from mist/core/src/main/scala/io/hydrosphere/mist/core/PythonEntrySettings.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/common/PythonEntrySettings.scala
index fa2a91b94..4632b748d 100644
--- a/mist/core/src/main/scala/io/hydrosphere/mist/core/PythonEntrySettings.scala
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/PythonEntrySettings.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.core
+package io.hydrosphere.mist.common
 
 case class PythonEntrySettings(
   driver: String,
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/logging/Level.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/Level.scala
similarity index 90%
rename from mist/core/src/main/scala/io/hydrosphere/mist/core/logging/Level.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/common/logging/Level.scala
index 863a4de8d..bc09079ac 100644
--- a/mist/core/src/main/scala/io/hydrosphere/mist/core/logging/Level.scala
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/Level.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.core.logging
+package io.hydrosphere.mist.common.logging
 
 case class Level(value: Int, name: String)
 
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/logging/LogEvent.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/LogEvent.scala
similarity index 97%
rename from mist/core/src/main/scala/io/hydrosphere/mist/core/logging/LogEvent.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/common/logging/LogEvent.scala
index b226275ec..d41509b98 100644
--- a/mist/core/src/main/scala/io/hydrosphere/mist/core/logging/LogEvent.scala
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/LogEvent.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.core.logging
+package io.hydrosphere.mist.common.logging
 
 import java.io.{PrintWriter, StringWriter}
 import java.time.format.DateTimeFormatter
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/Collections.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/Collections.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/Collections.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/Collections.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/Logger.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/Logger.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/Logger.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/Logger.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala
diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/fs.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/fs.scala
similarity index 100%
rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/fs.scala
rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/fs.scala
diff --git a/mist/core/src/test/resources/log4j.properties b/mist/common/src/test/resources/log4j.properties
similarity index 100%
rename from mist/core/src/test/resources/log4j.properties
rename to mist/common/src/test/resources/log4j.properties
diff --git a/mist/core/src/test/scala/io/hydrosphere/mist/core/MockitoSugar.scala b/mist/common/src/test/scala/io/hydrosphere/mist/common/MockitoSugar.scala
similarity index 98%
rename from mist/core/src/test/scala/io/hydrosphere/mist/core/MockitoSugar.scala
rename to mist/common/src/test/scala/io/hydrosphere/mist/common/MockitoSugar.scala
index 8796b7309..264613db5 100644
--- a/mist/core/src/test/scala/io/hydrosphere/mist/core/MockitoSugar.scala
+++ b/mist/common/src/test/scala/io/hydrosphere/mist/common/MockitoSugar.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.core
+package io.hydrosphere.mist.common
 
 import org.mockito.invocation.InvocationOnMock
 import org.mockito.stubbing.{Answer, OngoingStubbing}
diff --git a/mist/core/src/test/scala/io/hydrosphere/mist/core/PythonEntrySettingsSpec.scala b/mist/common/src/test/scala/io/hydrosphere/mist/common/PythonEntrySettingsSpec.scala
similarity index 93%
rename from mist/core/src/test/scala/io/hydrosphere/mist/core/PythonEntrySettingsSpec.scala
rename to mist/common/src/test/scala/io/hydrosphere/mist/common/PythonEntrySettingsSpec.scala
index f3f5bdacc..9115646bf 100644
--- a/mist/core/src/test/scala/io/hydrosphere/mist/core/PythonEntrySettingsSpec.scala
+++ b/mist/common/src/test/scala/io/hydrosphere/mist/common/PythonEntrySettingsSpec.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.core
+package io.hydrosphere.mist.common
 
 import org.scalatest.{FunSpec, Matchers}
 
diff --git a/mist/core/src/test/scala/io/hydrosphere/mist/core/logging/MistLoggingSpec.scala b/mist/common/src/test/scala/io/hydrosphere/mist/common/logging/MistLoggingSpec.scala
similarity index 96%
rename from mist/core/src/test/scala/io/hydrosphere/mist/core/logging/MistLoggingSpec.scala
rename to mist/common/src/test/scala/io/hydrosphere/mist/common/logging/MistLoggingSpec.scala
index c352937fc..a80df1b5a 100644
--- a/mist/core/src/test/scala/io/hydrosphere/mist/core/logging/MistLoggingSpec.scala
+++ b/mist/common/src/test/scala/io/hydrosphere/mist/common/logging/MistLoggingSpec.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.core.logging
+package io.hydrosphere.mist.common.logging
 
 import java.time.format.DateTimeFormatter
 import java.time.{LocalDateTime, ZoneOffset}
diff --git a/mist/core/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala b/mist/common/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala
similarity index 100%
rename from mist/core/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala
rename to mist/common/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/JobDetails.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/JobDetails.scala
index dbca2b906..29acb528c 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/JobDetails.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/JobDetails.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master
 
-import io.hydrosphere.mist.core.CommonData.JobParams
+import io.hydrosphere.mist.common.CommonData.JobParams
 import io.hydrosphere.mist.master.JobDetails.Status
 import io.hydrosphere.mist.master.Messages.StatusMessages._
 import mist.api.data.JsData
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/MainService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/MainService.scala
index 473a917fc..fdae4bb07 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/MainService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/MainService.scala
@@ -5,7 +5,7 @@ import java.util.UUID
 import akka.util.Timeout
 import cats.data._
 import cats.implicits._
-import io.hydrosphere.mist.core.CommonData.Action
+import io.hydrosphere.mist.common.CommonData.Action
 import io.hydrosphere.mist.master.JobDetails.Source.Async
 import io.hydrosphere.mist.master.data.ContextsStorage
 import io.hydrosphere.mist.master.execution.{ExecutionInfo, ExecutionService}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
index 715b441f2..47897a256 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
@@ -10,7 +10,7 @@ import akka.stream.ActorAttributes.supervisionStrategy
 import akka.stream.ActorMaterializer
 import akka.stream.Supervision.resumingDecider
 import akka.stream.scaladsl.{Keep, Sink}
-import io.hydrosphere.mist.core.CommonData
+import io.hydrosphere.mist.common.CommonData
 import io.hydrosphere.mist.master.Messages.StatusMessages.SystemEvent
 import io.hydrosphere.mist.master.artifact.ArtifactRepository
 import io.hydrosphere.mist.master.data.{ContextsStorage, FunctionConfigStorage}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/Messages.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/Messages.scala
index 773592183..3e0441365 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/Messages.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/Messages.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master
 
-import io.hydrosphere.mist.core.CommonData._
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.CommonData._
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.JobDetails.Source
 import mist.api.data.JsData
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextEvent.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextEvent.scala
index 4be597f24..ecd734498 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextEvent.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextEvent.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution
 
-import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, RunJobRequest}
+import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest}
 import io.hydrosphere.mist.master.JobDetails
 import io.hydrosphere.mist.master.models.ContextConfig
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index 2db4d5720..2779f958c 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.execution
 import java.util.UUID
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props, Timers}
-import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, RunJobRequest}
+import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest}
 import io.hydrosphere.mist.master.Messages.StatusMessages.FailedEvent
 import io.hydrosphere.mist.master.execution.ContextFrontend.Event.JobDied
 import io.hydrosphere.mist.master.execution.ContextFrontend.{ConnectorState, FrontendStatus}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionInfo.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionInfo.scala
index dcf48ba42..607279e18 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionInfo.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionInfo.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution
 
-import io.hydrosphere.mist.core.CommonData.RunJobRequest
+import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.master.JobResult
 import io.hydrosphere.mist.master.models.JobStartResponse
 import mist.api.data.JsData
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
index a6e817a29..18f18f7be 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
@@ -5,7 +5,7 @@ import akka.pattern.ask
 import akka.util.Timeout
 import cats.data._
 import cats.implicits._
-import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, JobParams, RunJobRequest}
+import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, JobParams, RunJobRequest}
 import io.hydrosphere.mist.master.Messages.StatusMessages.InitializedEvent
 import io.hydrosphere.mist.master.execution.status.StatusReporter
 import io.hydrosphere.mist.master.execution.workers.WorkerHub
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/JobActor.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/JobActor.scala
index 65683acf7..1b109a02a 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/JobActor.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/JobActor.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.execution
 import java.io.{PrintWriter, StringWriter}
 
 import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, PoisonPill, Props, SupervisorStrategy, Timers}
-import io.hydrosphere.mist.core.CommonData._
+import io.hydrosphere.mist.common.CommonData._
 import io.hydrosphere.mist.master.Messages.StatusMessages._
 import io.hydrosphere.mist.master.execution.status.StatusReporter
 import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
index da35b4893..56650d9fd 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution
 
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 import io.hydrosphere.mist.master.execution.workers.starter.WorkerStarter
 import io.hydrosphere.mist.master.models.ContextConfig
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala
index 1c4440e18..a843d4411 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution
 
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 
 case class WorkerLink(
   name: String,
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/status/StatusReporter.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/status/StatusReporter.scala
index 7008fd1b4..951c3da49 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/status/StatusReporter.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/status/StatusReporter.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.execution.status
 
 import akka.actor.ActorSystem
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.{EventsStreamer, JobDetails}
 import io.hydrosphere.mist.master.Messages.StatusMessages._
 import io.hydrosphere.mist.master.logging.LogService
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
index ca9d4c258..a2b110873 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import akka.pattern.pipe
-import io.hydrosphere.mist.core.CommonData
-import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, RunJobRequest}
+import io.hydrosphere.mist.common.CommonData
+import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest}
 import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released
 import io.hydrosphere.mist.master.models.ContextConfig
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/PerJobConnection.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/PerJobConnection.scala
index 022ae3b58..2796bd5a2 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/PerJobConnection.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/PerJobConnection.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.ActorRef
-import io.hydrosphere.mist.core.CommonData.RunJobRequest
+import io.hydrosphere.mist.common.CommonData.RunJobRequest
 
 import scala.concurrent.Future
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
index 60737d391..76146bc91 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
@@ -4,8 +4,8 @@ import java.util.concurrent.atomic.AtomicInteger
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import akka.pattern.pipe
-import io.hydrosphere.mist.core.CommonData
-import io.hydrosphere.mist.core.CommonData.CancelJobRequest
+import io.hydrosphere.mist.common.CommonData
+import io.hydrosphere.mist.common.CommonData.CancelJobRequest
 import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event
 import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released
 import io.hydrosphere.mist.master.models.ContextConfig
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala
index 977d56034..c7e392cdd 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.{Actor, ActorLogging, ActorRef, ActorRefFactory, Props, ReceiveTimeout, Terminated, Timers}
-import io.hydrosphere.mist.core.CommonData._
+import io.hydrosphere.mist.common.CommonData._
 import io.hydrosphere.mist.master.execution.WorkerLink
 
 import scala.concurrent.{Future, Promise}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala
index b66367beb..ea9e98983 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.{ActorRef, ActorRefFactory}
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 import io.hydrosphere.mist.master.execution.SpawnSettings
 import io.hydrosphere.mist.master.execution.workers.starter.WorkerProcess
 import io.hydrosphere.mist.master.models.ContextConfig
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/DockerStarter.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/DockerStarter.scala
index 78177776d..d763c80fd 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/DockerStarter.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/DockerStarter.scala
@@ -6,7 +6,7 @@ import com.github.dockerjava.api.DockerClient
 import com.github.dockerjava.api.command.{CreateContainerCmd, InspectContainerResponse}
 import com.github.dockerjava.api.model.{ContainerNetwork, Link}
 import com.github.dockerjava.core.{DefaultDockerClientConfig, DockerClientBuilder}
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 import io.hydrosphere.mist.master._
 import io.hydrosphere.mist.master.execution.workers.StopAction
 import io.hydrosphere.mist.utils.Logger
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/LocalSparkSubmit.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/LocalSparkSubmit.scala
index ecce964ca..f682b75b8 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/LocalSparkSubmit.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/LocalSparkSubmit.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers.starter
 
 import java.nio.file.Path
 
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 import io.hydrosphere.mist.master.execution.workers.StopAction
 import io.hydrosphere.mist.utils.Logger
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/ManualStarter.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/ManualStarter.scala
index fbfe3aa57..f128b1bd8 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/ManualStarter.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/ManualStarter.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers.starter
 
 import java.nio.file.Path
 
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 import io.hydrosphere.mist.master.ManualRunnerConfig
 import io.hydrosphere.mist.master.execution.workers.StopAction
 import io.hydrosphere.mist.utils.Logger
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilder.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilder.scala
index e0c1c08cb..79984fd8b 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilder.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilder.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers.starter
 
 import java.nio.file.Paths
 
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 
 class SparkSubmitBuilder(mistHome: String, sparkHome: String) extends PsUtil {
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/WorkerStarter.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/WorkerStarter.scala
index 5d8b01336..a1ec5d773 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/WorkerStarter.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/WorkerStarter.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers.starter
 
 import java.nio.file.Path
 
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 import io.hydrosphere.mist.master._
 import io.hydrosphere.mist.master.execution.workers.StopAction
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala
index 36bd3adbf..67bb4d83f 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.interfaces.http
 import java.lang.management._
 import java.time.LocalDateTime
 
-import io.hydrosphere.mist.core.FunctionInfoData
+import io.hydrosphere.mist.common.FunctionInfoData
 import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
 import mist.api._
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
index 0231f49df..0e4d8c436 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
@@ -3,8 +3,8 @@ package io.hydrosphere.mist.master.interfaces
 import java.time.LocalDateTime
 import java.time.format.{DateTimeFormatter, DateTimeParseException}
 
-import io.hydrosphere.mist.core.CommonData.{Action, JobParams, WorkerInitInfo}
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.CommonData.{Action, JobParams, WorkerInitInfo}
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.Messages.StatusMessages._
 import io.hydrosphere.mist.master.execution.WorkerLink
 import io.hydrosphere.mist.master.interfaces.http._
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionInfoProviderRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionInfoProviderRunner.scala
index b82ac54f7..53eff7d7b 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionInfoProviderRunner.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionInfoProviderRunner.scala
@@ -1,8 +1,8 @@
 package io.hydrosphere.mist.master.jobs
 
 import akka.actor.{Actor, ActorRef, ActorSystem, Props, ReceiveTimeout}
-import io.hydrosphere.mist.core.CommonData
-import io.hydrosphere.mist.core.CommonData.RegisterJobInfoProvider
+import io.hydrosphere.mist.common.CommonData
+import io.hydrosphere.mist.common.CommonData.RegisterJobInfoProvider
 import io.hydrosphere.mist.master.FunctionInfoProviderConfig
 
 import scala.concurrent.duration.{Duration, FiniteDuration}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionsService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionsService.scala
index 96cc5822c..f74273de8 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionsService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionsService.scala
@@ -7,8 +7,8 @@ import akka.pattern._
 import akka.util.Timeout
 import cats.data._
 import cats.implicits._
-import io.hydrosphere.mist.core.CommonData._
-import io.hydrosphere.mist.core.{ExtractedFunctionData, FunctionInfoData, PythonEntrySettings}
+import io.hydrosphere.mist.common.CommonData._
+import io.hydrosphere.mist.common.{ExtractedFunctionData, FunctionInfoData, PythonEntrySettings}
 import io.hydrosphere.mist.master.artifact.ArtifactRepository
 import io.hydrosphere.mist.master.data.{Contexts, ContextsStorage, FunctionConfigStorage}
 import io.hydrosphere.mist.master.models.{ContextConfig, FunctionConfig}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/JobLogger.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/JobLogger.scala
index cbaf0e37e..7d00b8e92 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/JobLogger.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/JobLogger.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.logging
 
 import akka.actor.ActorRef
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.logging.LogEvent
 
 trait JobLogger {
   protected val jobId: String
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/LogStreams.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/LogStreams.scala
index af1e2b508..1eedf2bca 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/LogStreams.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/LogStreams.scala
@@ -8,7 +8,7 @@ import akka.stream.{ActorAttributes, ActorMaterializer, OverflowStrategy}
 import akka.util.ByteString
 import akka.{Done, NotUsed}
 import com.twitter.chill.{KryoPool, ScalaKryoInstantiator}
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.Messages.StatusMessages.ReceivedLogs
 import io.hydrosphere.mist.master.{EventsStreamer, LogStoragePaths}
 import io.hydrosphere.mist.utils.Logger
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/writers.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/writers.scala
index e445ebd72..884728070 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/writers.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/writers.scala
@@ -5,7 +5,7 @@ import java.nio.file._
 import akka.actor._
 import akka.pattern.ask
 import akka.util.Timeout
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.LogStoragePaths
 
 import scala.concurrent.Future
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/models.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/models.scala
index 1cb7573e2..46170bda1 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/models.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/models.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.master.models
 
 import java.util.UUID
 
-import io.hydrosphere.mist.core.CommonData.Action
-import io.hydrosphere.mist.core.FunctionInfoData
+import io.hydrosphere.mist.common.CommonData.Action
+import io.hydrosphere.mist.common.FunctionInfoData
 import io.hydrosphere.mist.master.JobDetails
 import mist.api.data.{JsData, JsMap}
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/store/H2JobsRepository.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/store/H2JobsRepository.scala
index 0f93ace23..e1145e5c8 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/store/H2JobsRepository.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/store/H2JobsRepository.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.store
 
 import java.nio.file.Paths
 
-import io.hydrosphere.mist.core.CommonData.{Action, JobParams}
+import io.hydrosphere.mist.common.CommonData.{Action, JobParams}
 import io.hydrosphere.mist.master.{FilterClause, JobDetails, JobDetailsRequest, JobDetailsResponse}
 import io.hydrosphere.mist.master.interfaces.JsonCodecs
 import JsonCodecs._
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala
index fc55dbf59..f33d1bd10 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master
 
-import io.hydrosphere.mist.core.MockitoSugar
+import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.data.ContextsStorage
 import io.hydrosphere.mist.master.execution.ExecutionService
 import io.hydrosphere.mist.master.interfaces.http.ContextCreateRequest
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/MainServiceSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/MainServiceSpec.scala
index 26fde4d19..ec9760ccb 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/MainServiceSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/MainServiceSpec.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.master
 
 import akka.actor.ActorSystem
 import akka.testkit.TestKit
-import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest}
-import io.hydrosphere.mist.core.{FunctionInfoData, MockitoSugar}
+import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest}
+import io.hydrosphere.mist.common.{FunctionInfoData, MockitoSugar}
 import io.hydrosphere.mist.master.artifact.ArtifactRepository
 import io.hydrosphere.mist.master.data.{ContextsStorage, FunctionConfigStorage}
 import io.hydrosphere.mist.master.execution.{ExecutionInfo, ExecutionService}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
index f4f7ead3e..cc10db52a 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
@@ -1,8 +1,8 @@
 package io.hydrosphere.mist.master
 
 import com.typesafe.config.ConfigFactory
-import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest, WorkerInitInfo}
-import io.hydrosphere.mist.core.FunctionInfoData
+import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest, WorkerInitInfo}
+import io.hydrosphere.mist.common.FunctionInfoData
 import io.hydrosphere.mist.master.execution.WorkerLink
 import mist.api.ArgInfo
 import mist.api.data._
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/artifact/ArtifactRepositorySpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/artifact/ArtifactRepositorySpec.scala
index cc8547ea1..2ffbd4064 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/artifact/ArtifactRepositorySpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/artifact/ArtifactRepositorySpec.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.artifact
 import java.io.File
 import java.nio.file.{Files, Paths}
 
-import io.hydrosphere.mist.core.MockitoSugar
+import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.TestUtils.AwaitSyntax
 import io.hydrosphere.mist.master.data.FunctionConfigStorage
 import org.apache.commons.io.FileUtils
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
index 9de2b7e21..576dbb5ba 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.master.execution
 
 import akka.actor.ActorRef
 import akka.testkit.{TestActorRef, TestProbe}
-import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest, _}
-import io.hydrosphere.mist.core.MockitoSugar
+import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest, _}
+import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.execution.status.StatusReporter
 import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection, WorkerConnector}
 import io.hydrosphere.mist.master.logging.{JobLogger, JobLoggersFactory}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextsMasterSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextsMasterSpec.scala
index 04a1e8d8f..216c368aa 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextsMasterSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextsMasterSpec.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.execution
 
 import akka.testkit.{TestActorRef, TestProbe}
-import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, RunJobRequest}
+import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest}
 import io.hydrosphere.mist.master.execution.ContextEvent.{CancelJobCommand, RunJobCommand}
 import io.hydrosphere.mist.master.{ActorSpec, TestData}
 import io.hydrosphere.mist.utils.akka.ActorF
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionInfoSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionInfoSpec.scala
index 59e8e24d2..156a6bb95 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionInfoSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionInfoSpec.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution
 
-import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest}
+import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest}
 import io.hydrosphere.mist.master.models.JobStartResponse
 import io.hydrosphere.mist.master.{JobDetails, JobResult, TestUtils}
 import mist.api.data._
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala
index 6847425d4..bf71937d3 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution
 
 import akka.actor.ActorSystem
 import akka.testkit.{TestKit, TestProbe}
-import io.hydrosphere.mist.core.{FunctionInfoData, MockitoSugar}
+import io.hydrosphere.mist.common.{FunctionInfoData, MockitoSugar}
 import io.hydrosphere.mist.master.Messages.StatusMessages.UpdateStatusEvent
 import io.hydrosphere.mist.master.execution.status.StatusReporter
 import io.hydrosphere.mist.master.execution.workers.WorkerHub
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/JobActorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/JobActorSpec.scala
index 3c8f3baa7..ab1489536 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/JobActorSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/JobActorSpec.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.master.execution
 
 import akka.actor.ActorRef
 import akka.testkit.{TestActorRef, TestProbe}
-import io.hydrosphere.mist.core.CommonData._
-import io.hydrosphere.mist.core.MockitoSugar
+import io.hydrosphere.mist.common.CommonData._
+import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.Messages.StatusMessages._
 import io.hydrosphere.mist.master.execution.status.{ReportedEvent, StatusReporter}
 import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
index 7b56a2600..9c338d203 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution
 
-import io.hydrosphere.mist.core.CommonData
+import io.hydrosphere.mist.common.CommonData
 import io.hydrosphere.mist.master.TestData
 import io.hydrosphere.mist.master.execution.workers.StopAction
 import io.hydrosphere.mist.master.execution.workers.starter.{WorkerProcess, WorkerStarter}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/status/StatusReporterSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/status/StatusReporterSpec.scala
index 3bb1c4bb1..f6ccbdb1c 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/status/StatusReporterSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/status/StatusReporterSpec.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution.status
 
-import io.hydrosphere.mist.core.MockitoSugar
+import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.Messages.StatusMessages.{QueuedEvent, UpdateStatusEvent}
 import io.hydrosphere.mist.master.logging.{JobLogger, LogService}
 import io.hydrosphere.mist.master.store.JobRepository
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala
index 05945e038..173395c1b 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.ActorRef
 import akka.testkit.{TestActorRef, TestProbe}
-import io.hydrosphere.mist.core.CommonData.RunJobRequest
+import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.master.execution.workers.WorkerBridge.Event.CompleteAndShutdown
 import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released
 import io.hydrosphere.mist.master.{ActorSpec, FilteredException, TestData}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
index e1dc6476f..645600101 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
@@ -4,7 +4,7 @@ import java.util.concurrent.atomic.AtomicInteger
 
 import akka.actor.ActorRef
 import akka.testkit.{TestActorRef, TestProbe}
-import io.hydrosphere.mist.core.CommonData.RunJobRequest
+import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released
 import io.hydrosphere.mist.master.{ActorSpec, TestData}
 import org.scalatest.Matchers
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridgeSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridgeSpec.scala
index 4a21c123a..65c11c957 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridgeSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridgeSpec.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.execution.workers
 import java.util.concurrent.atomic.AtomicBoolean
 
 import akka.testkit.TestProbe
-import io.hydrosphere.mist.core.CommonData._
+import io.hydrosphere.mist.common.CommonData._
 import io.hydrosphere.mist.master.execution.workers.WorkerBridge.Event.CompleteAndShutdown
 import io.hydrosphere.mist.master.{ActorSpec, TestData, TestUtils}
 import mist.api.data._
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
index 4769a88ae..92dc90b86 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.ActorRef
-import io.hydrosphere.mist.core.{CommonData, MockitoSugar}
+import io.hydrosphere.mist.common.{CommonData, MockitoSugar}
 import io.hydrosphere.mist.master.TestData
 import io.hydrosphere.mist.master.models.ContextConfig
 import org.scalatest.concurrent.Eventually
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala
index b59967caf..7b6325d07 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.master.execution.workers
 
 import java.util.concurrent.atomic.AtomicBoolean
 
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
-import io.hydrosphere.mist.core.MockitoSugar
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.execution.workers.starter.{WorkerProcess, WorkerStarter}
 import io.hydrosphere.mist.master.execution.{SpawnSettings, workers}
 import io.hydrosphere.mist.master.{ActorSpec, FilteredException, TestData}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala
index a2c56623d..007421eb0 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution.workers.starter
 
-import io.hydrosphere.mist.core.CommonData.WorkerInitInfo
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 import io.hydrosphere.mist.master.TestData
 import org.scalatest.{FunSpec, Matchers}
 import scala.concurrent.duration._
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/DevApiSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/DevApiSpec.scala
index 7f395701b..ee4b10846 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/DevApiSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/DevApiSpec.scala
@@ -4,9 +4,9 @@ import mist.api.data._
 import mist.api.encoding.JsSyntax._
 import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.testkit.ScalatestRouteTest
-import io.hydrosphere.mist.core.CommonData.{JobParams, RunJobRequest}
-import io.hydrosphere.mist.core.MockitoSugar
-import io.hydrosphere.mist.core.CommonData.Action
+import io.hydrosphere.mist.common.CommonData.{JobParams, RunJobRequest}
+import io.hydrosphere.mist.common.MockitoSugar
+import io.hydrosphere.mist.common.CommonData.Action
 import io.hydrosphere.mist.master.JobDetails.{Source, Status}
 import io.hydrosphere.mist.master.execution.ExecutionInfo
 import io.hydrosphere.mist.master.MainService
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
index d3f1af5c6..d390f0223 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
@@ -7,8 +7,8 @@ import java.util.UUID
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.testkit.ScalatestRouteTest
 import akka.util.ByteString
-import io.hydrosphere.mist.core.CommonData._
-import io.hydrosphere.mist.core.{FunctionInfoData, MockitoSugar}
+import io.hydrosphere.mist.common.CommonData._
+import io.hydrosphere.mist.common.{FunctionInfoData, MockitoSugar}
 import io.hydrosphere.mist.master.JobDetails.Source
 import io.hydrosphere.mist.master._
 import io.hydrosphere.mist.master.artifact.ArtifactRepository
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/WsApiSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/WsApiSpec.scala
index 428928969..ca21f8f2a 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/WsApiSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/WsApiSpec.scala
@@ -4,7 +4,7 @@ import mist.api.data._
 import akka.http.scaladsl.testkit.{ScalatestRouteTest, WSProbe}
 import akka.stream.ActorMaterializer
 import akka.stream.scaladsl.Source
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.Messages.StatusMessages._
 import io.hydrosphere.mist.master.EventsStreamer
 import org.mockito.Mockito._
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/jobs/FunctionsServiceSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/jobs/FunctionsServiceSpec.scala
index 113ef5155..96a8ce4a2 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/jobs/FunctionsServiceSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/jobs/FunctionsServiceSpec.scala
@@ -5,9 +5,9 @@ import java.nio.file.Paths
 
 import akka.actor.{ActorSystem, Status}
 import akka.testkit.{TestKit, TestProbe}
-import io.hydrosphere.mist.core.CommonData.{Action, GetAllFunctions, GetFunctionInfo, ValidateFunctionParameters}
-import io.hydrosphere.mist.core.{ExtractedFunctionData, FunctionInfoData, MockitoSugar}
-import io.hydrosphere.mist.core.ExtractedFunctionData
+import io.hydrosphere.mist.common.CommonData.{Action, GetAllFunctions, GetFunctionInfo, ValidateFunctionParameters}
+import io.hydrosphere.mist.common.{ExtractedFunctionData, FunctionInfoData, MockitoSugar}
+import io.hydrosphere.mist.common.ExtractedFunctionData
 import io.hydrosphere.mist.master.TestData
 import io.hydrosphere.mist.master.artifact.ArtifactRepository
 import io.hydrosphere.mist.master.data.{Contexts, ContextsStorage, FunctionConfigStorage}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogStreamsSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogStreamsSpec.scala
index 697f641df..579bdfc71 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogStreamsSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogStreamsSpec.scala
@@ -4,8 +4,8 @@ import akka.actor.ActorSystem
 import akka.stream.ActorMaterializer
 import akka.stream.scaladsl.{Keep, Sink, Source}
 import akka.testkit.TestKit
-import io.hydrosphere.mist.core.MockitoSugar
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.MockitoSugar
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.FilteredException
 import org.mockito.Mockito.verify
 import org.scalatest.{FunSpecLike, Matchers}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogWriterSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogWriterSpec.scala
index 32150bbb8..8aa3d4203 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogWriterSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogWriterSpec.scala
@@ -7,7 +7,7 @@ import akka.pattern.ask
 import akka.testkit.{TestActorRef, TestKit}
 import akka.util.Timeout
 import com.typesafe.config.ConfigFactory
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.LogStoragePaths
 import org.apache.commons.io.FileUtils
 import org.scalatest.{BeforeAndAfterAll, FunSpecLike, Matchers}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/store/H2RepoSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/store/H2RepoSpec.scala
index 7c83542ef..8f000249f 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/store/H2RepoSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/store/H2RepoSpec.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.store
 
 import java.nio.file.Paths
 
-import io.hydrosphere.mist.core.CommonData.{Action, JobParams}
+import io.hydrosphere.mist.common.CommonData.{Action, JobParams}
 import io.hydrosphere.mist.master.JobDetails
 import io.hydrosphere.mist.master.JobDetails.Source
 import mist.api.data.JsMap
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoExtractor.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoExtractor.scala
index 6bfc0a1a8..2d9d17812 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoExtractor.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoExtractor.scala
@@ -3,10 +3,10 @@ package io.hydrosphere.mist.job
 import java.io.File
 import java.net.URLClassLoader
 
-import io.hydrosphere.mist.core
-import io.hydrosphere.mist.core.CommonData.{Action, EnvInfo}
-import io.hydrosphere.mist.core.{ExtractedFunctionData, FunctionInfoData}
-import io.hydrosphere.mist.core.ExtractedFunctionData
+import io.hydrosphere.mist.common
+import io.hydrosphere.mist.common.CommonData.{Action, EnvInfo}
+import io.hydrosphere.mist.common.{ExtractedFunctionData, FunctionInfoData}
+import io.hydrosphere.mist.common.ExtractedFunctionData
 import io.hydrosphere.mist.python.{FunctionInfoPythonExecutor, PythonCmd}
 import io.hydrosphere.mist.utils.{Err, Logger, Succ, TryLoad}
 import io.hydrosphere.mist.utils.{Err, Succ, TryLoad}
@@ -36,7 +36,7 @@ class JvmFunctionInfoExtractor(mkLoader: ClassLoader => FunctionInstanceLoader)
       val executeFnInstance = loader.loadFnInstance(className, Action.Execute)
       executeFnInstance orElse loader.loadFnInstance(className, Action.Serve) map { instance =>
 
-        FunctionInfo(instance, core.ExtractedFunctionData(
+        FunctionInfo(instance, common.ExtractedFunctionData(
           lang = instance.lang,
           execute = instance.describe().collect { case x: UserInputArgument => x },
           isServe = !executeFnInstance.isSuccess,
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProvider.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProvider.scala
index ef1c201fe..c2e04db3e 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProvider.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProvider.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.job
 
 import akka.actor.{ActorRef, ActorSystem, PoisonPill}
 import com.typesafe.config.ConfigFactory
-import io.hydrosphere.mist.core.CommonData
-import io.hydrosphere.mist.core.CommonData.RegisterJobInfoProvider
+import io.hydrosphere.mist.common.CommonData
+import io.hydrosphere.mist.common.CommonData.RegisterJobInfoProvider
 import io.hydrosphere.mist.utils.Logger
 import io.hydrosphere.mist.utils.akka.WhenTerminated
 
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProviderActor.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProviderActor.scala
index fe988128a..61617adc0 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProviderActor.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProviderActor.scala
@@ -4,8 +4,8 @@ import java.io.File
 
 import akka.actor.SupervisorStrategy.Resume
 import akka.actor.{Status, _}
-import io.hydrosphere.mist.core.CommonData._
-import io.hydrosphere.mist.core.ExtractedFunctionData
+import io.hydrosphere.mist.common.CommonData._
+import io.hydrosphere.mist.common.ExtractedFunctionData
 import io.hydrosphere.mist.utils.{Err, Succ, TryLoad}
 import mist.api.{Extracted, Failed}
 import mist.api.data.JsMap
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInstanceLoader.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInstanceLoader.scala
index a333bdb1d..1dab4276a 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInstanceLoader.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInstanceLoader.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.job
 import java.io.File
 import java.net.URLClassLoader
 
-import io.hydrosphere.mist.core.CommonData.Action
+import io.hydrosphere.mist.common.CommonData.Action
 import io.hydrosphere.mist.job
 import io.hydrosphere.mist.utils.{Err, TryLoad}
 
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/JvmLangDetector.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/JvmLangDetector.scala
index 775a5fcb3..77ef8470a 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/JvmLangDetector.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/JvmLangDetector.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.job
 
 import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm._
 import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.Opcodes._
-import io.hydrosphere.mist.core.FunctionInfoData
+import io.hydrosphere.mist.common.FunctionInfoData
 
 class CheckSource extends ClassVisitor(ASM5) {
 
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/python/PythonExecuter.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/python/PythonExecuter.scala
index 2c10668c8..663d4a62f 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/python/PythonExecuter.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/python/PythonExecuter.scala
@@ -3,8 +3,8 @@ package io.hydrosphere.mist.python
 import java.io.File
 import java.nio.file.Paths
 
-import io.hydrosphere.mist.core.CommonData.{EnvInfo, RunJobRequest}
-import io.hydrosphere.mist.core.PythonEntrySettings
+import io.hydrosphere.mist.common.CommonData.{EnvInfo, RunJobRequest}
+import io.hydrosphere.mist.common.PythonEntrySettings
 import io.hydrosphere.mist.utils.Logger
 import io.hydrosphere.mist.worker.MistScContext
 import io.hydrosphere.mist.worker.runners.python.wrappers.{ConfigurationWrapper, SparkStreamingWrapper}
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/MasterBridge.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/MasterBridge.scala
index e7caa7163..b84632e13 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/MasterBridge.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/MasterBridge.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.worker
 import java.nio.file.Path
 
 import akka.actor._
-import io.hydrosphere.mist.core.CommonData._
+import io.hydrosphere.mist.common.CommonData._
 import io.hydrosphere.mist.utils.akka.{ActorF, ActorFSyntax, ActorRegHub}
 import io.hydrosphere.mist.worker.MasterBridge.{AppShutdown, ReceiveInitTimeout}
 import io.hydrosphere.mist.worker.logging.RemoteLogsWriter
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/RequestSetup.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/RequestSetup.scala
index 373bc6e4e..9cadb40c2 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/RequestSetup.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/RequestSetup.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.worker
 
-import io.hydrosphere.mist.core.CommonData.RunJobRequest
+import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.worker.logging.{LogsWriter, RemoteAppender, RemoteLogsWriter}
 import org.apache.log4j.{LogManager, Logger}
 
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/WorkerActor.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/WorkerActor.scala
index 855b7fb46..6c2fddc89 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/WorkerActor.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/WorkerActor.scala
@@ -4,7 +4,7 @@ import java.io.{PrintWriter, StringWriter}
 
 import akka.actor._
 import akka.pattern.pipe
-import io.hydrosphere.mist.core.CommonData._
+import io.hydrosphere.mist.common.CommonData._
 import io.hydrosphere.mist.worker.runners._
 import mist.api.data.JsData
 import org.apache.spark.streaming.StreamingContext
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteAppender.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteAppender.scala
index 294c2a072..702e986b8 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteAppender.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteAppender.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.worker.logging
 
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.logging.LogEvent
 import org.apache.log4j.spi.LoggingEvent
 import org.apache.log4j.{AppenderSkeleton, Level, SimpleLayout}
 
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteLogsWriter.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteLogsWriter.scala
index aa29270f2..cf6e967b2 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteLogsWriter.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteLogsWriter.scala
@@ -8,7 +8,7 @@ import akka.stream.scaladsl.{Keep, Sink, Source, Tcp}
 import akka.util.ByteString
 import com.twitter.chill.{KryoPool, ScalaKryoInstantiator}
 import com.typesafe.config.ConfigFactory
-import io.hydrosphere.mist.core.logging.{Level, LogEvent}
+import io.hydrosphere.mist.common.logging.{Level, LogEvent}
 import io.hydrosphere.mist.worker.logging.RemoteLogsWriter.Key
 import org.slf4j.LoggerFactory
 
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/JobRunner.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/JobRunner.scala
index 0fc2673d5..342625103 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/JobRunner.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/JobRunner.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.worker.runners
 
-import io.hydrosphere.mist.core.CommonData.RunJobRequest
+import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.worker.MistScContext
 import mist.api.data.JsData
 
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/ScalaRunner.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/ScalaRunner.scala
index af29d6001..cb4f3f173 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/ScalaRunner.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/ScalaRunner.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.worker.runners
 import java.io.File
 import java.net.URL
 
-import io.hydrosphere.mist.core.CommonData.RunJobRequest
+import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.job.FunctionInstanceLoader
 import io.hydrosphere.mist.utils.EitherOps._
 import io.hydrosphere.mist.utils.{Err, Succ}
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/PythonRunner.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/PythonRunner.scala
index 13d916db4..05174a022 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/PythonRunner.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/PythonRunner.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.worker.runners.python
 
-import io.hydrosphere.mist.core.CommonData.RunJobRequest
+import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.python.PythonFunctionExecutor
 import io.hydrosphere.mist.utils.Logger
 import io.hydrosphere.mist.worker.runners.JobRunner
diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/wrappers/ConfigurationWrapper.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/wrappers/ConfigurationWrapper.scala
index 7e0d90e4f..5fe874b11 100644
--- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/wrappers/ConfigurationWrapper.scala
+++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/wrappers/ConfigurationWrapper.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.worker.runners.python.wrappers
 
-import io.hydrosphere.mist.core.CommonData.JobParams
+import io.hydrosphere.mist.common.CommonData.JobParams
 import io.hydrosphere.mist.utils.Collections
 import mist.api.data.JsData
 
diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoExtractorSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoExtractorSpec.scala
index 1cc2eaaf4..a115b9d23 100644
--- a/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoExtractorSpec.scala
+++ b/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoExtractorSpec.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.job
 
 import java.io.File
 
-import io.hydrosphere.mist.core.CommonData.{Action, EnvInfo}
-import io.hydrosphere.mist.core.{ExtractedFunctionData, MockitoSugar, PythonEntrySettings}
+import io.hydrosphere.mist.common.CommonData.{Action, EnvInfo}
+import io.hydrosphere.mist.common.{ExtractedFunctionData, MockitoSugar, PythonEntrySettings}
 import io.hydrosphere.mist.utils.{Err, Succ}
 import mist.api.{InternalArgument, MInt, UserInputArgument}
 import org.mockito.Matchers.{endsWith => mockitoEndsWith, eq => mockitoEq}
diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoProviderActorSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoProviderActorSpec.scala
index 094fb9b75..62872fd4e 100644
--- a/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoProviderActorSpec.scala
+++ b/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoProviderActorSpec.scala
@@ -5,8 +5,8 @@ import java.nio.file.Paths
 
 import akka.actor.{Actor, ActorSystem, Status}
 import akka.testkit.{TestActorRef, TestKit, TestProbe}
-import io.hydrosphere.mist.core.CommonData._
-import io.hydrosphere.mist.core.{ExtractedFunctionData, MockitoSugar, PythonEntrySettings}
+import io.hydrosphere.mist.common.CommonData._
+import io.hydrosphere.mist.common.{ExtractedFunctionData, MockitoSugar, PythonEntrySettings}
 import io.hydrosphere.mist.utils.{Err, Succ}
 import mist.api._
 import mist.api.data._
diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala
index 08c1517bf..354eb448a 100644
--- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala
+++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala
@@ -2,8 +2,8 @@ package io.hydrosphere.mist.worker
 
 import akka.actor.ActorSystem
 import akka.testkit.{TestActorRef, TestKit, TestProbe}
-import io.hydrosphere.mist.core.CommonData._
-import io.hydrosphere.mist.core.MockitoSugar
+import io.hydrosphere.mist.common.CommonData._
+import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.utils.akka.{ActorF, ActorRegHub}
 import mist.api.data.JsMap
 import org.apache.spark.SparkConf
diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/RequestSetupSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/RequestSetupSpec.scala
index 8b721dad0..534109f60 100644
--- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/RequestSetupSpec.scala
+++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/RequestSetupSpec.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.worker
 
-import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest}
-import io.hydrosphere.mist.core.logging.LogEvent
+import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest}
+import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.worker.logging.LogsWriter
 import mist.api.data.JsMap
 import org.apache.log4j.LogManager
diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/WorkerActorSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/WorkerActorSpec.scala
index e0b9386fd..7a5c9d1d3 100644
--- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/WorkerActorSpec.scala
+++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/WorkerActorSpec.scala
@@ -4,8 +4,8 @@ import java.io.File
 
 import akka.actor.ActorSystem
 import akka.testkit.{TestActorRef, TestKit, TestProbe}
-import io.hydrosphere.mist.core.CommonData._
-import io.hydrosphere.mist.core.MockitoSugar
+import io.hydrosphere.mist.common.CommonData._
+import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.worker.runners.{ArtifactDownloader, JobRunner, RunnerSelector}
 import mist.api.data.{JsData, _}
 import mist.api.encoding.defaultEncoders._
diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/logging/RemoteAppenderSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/logging/RemoteAppenderSpec.scala
index ab44a61aa..e4a13f1e9 100644
--- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/logging/RemoteAppenderSpec.scala
+++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/logging/RemoteAppenderSpec.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.worker.logging
 
-import io.hydrosphere.mist.core.MockitoSugar
-import io.hydrosphere.mist.core.logging._
+import io.hydrosphere.mist.common.MockitoSugar
+import io.hydrosphere.mist.common.logging._
 import org.apache.log4j.spi.{LoggingEvent, NOPLogger, NOPLoggerRepository}
 import org.apache.log4j.{Category, Level => L4jLevel}
 import org.mockito.Matchers.{eq => mockitoEq}

From a2bf4f108ac03cceac30453487f3224dd286a0b3 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 20 Sep 2018 20:13:13 +0300
Subject: [PATCH 12/30] wip

---
 .../io/hydrosphere/mist/aws/GenConfig.scala   | 10 ++--
 .../scala/io/hydrosphere/mist/aws/Main.scala  |  4 +-
 .../mist/agent/MasterConnection.scala         | 11 ++++
 .../mist/common/logging/AgentProtocol.scala   | 10 ++++
 .../io/hydrosphere/mist/master/configs.scala  | 50 +++++++++++++++++++
 .../hydrosphere/mist/master/models/base.scala | 14 +++++-
 6 files changed, 90 insertions(+), 9 deletions(-)
 create mode 100644 mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/MasterConnection.scala
 create mode 100644 mist/common/src/main/scala/io/hydrosphere/mist/common/logging/AgentProtocol.scala

diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
index 55c690392..77b85b340 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
@@ -1,12 +1,10 @@
 package io.hydrosphere.mist.aws
 
-import java.io.File
 import java.nio.file.{Files, Path}
 
 import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions}
-import io.hydrosphere.mist.utils.ConfigUtils._
 
-case class ProvisionData(
+case class LaunchData(
   sshKeyPair: String,
   sshKeyPath: String,
   accessKey: String,
@@ -20,7 +18,7 @@ case class ProvisionData(
 
 object ConfigPatcher {
 
-  def patch(mistConfig: Config, data: ProvisionData): Config = {
+  def patch(mistConfig: Config, data: LaunchData): Config = {
     import com.typesafe.config.ConfigValueFactory._
     import scala.collection.JavaConverters._
 
@@ -43,10 +41,10 @@ object ConfigPatcher {
     val provisioner = fromMap(configKeys.asJava)
     val entries = fromIterable(Seq(provisioner).asJava)
 
-    mistConfig.withValue("mist.provision", entries)
+    mistConfig.withValue("mist.launchers-settings", entries)
   }
 
-  def patchFile(filePath: Path, data: ProvisionData): Unit = {
+  def patchFile(filePath: Path, data: LaunchData): Unit = {
     val orig = ConfigFactory.parseFile(filePath.toFile)
     val patched = patch(orig, data)
 
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
index 6e690fbf3..f7d0eb079 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
@@ -19,7 +19,7 @@ object Main {
     val setup = AwsSetup.create(accessKey, accessSecret, region)
     val out = setup.setup(instanceId, sshKey).unsafeRunSync()
 
-    val provisionData = ProvisionData(
+    val launchData = LaunchData(
       sshKeyPair = out.sshKeyPairName,
       sshKeyPath = sshKeyPath,
       accessKey = accessKey,
@@ -31,7 +31,7 @@ object Main {
       emrEc2Role = out.ec2EmrRole
     )
 
-    ConfigPatcher.patchFile(Paths.get(configPath), provisionData)
+    ConfigPatcher.patchFile(Paths.get(configPath), launchData)
   }
 
 }
diff --git a/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/MasterConnection.scala b/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/MasterConnection.scala
new file mode 100644
index 000000000..96cf74647
--- /dev/null
+++ b/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/MasterConnection.scala
@@ -0,0 +1,11 @@
+package io.hydrosphere.mist.agent
+
+import akka.actor.{Actor, ActorLogging}
+
+class MasterConnection(
+  id: String
+) extends Actor with ActorLogging {
+
+  override def receive: Receive = ???
+
+}
diff --git a/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/AgentProtocol.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/AgentProtocol.scala
new file mode 100644
index 000000000..19bde387a
--- /dev/null
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/AgentProtocol.scala
@@ -0,0 +1,10 @@
+package io.hydrosphere.mist.common.logging
+
+import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
+
+object AgentProtocol {
+
+  case class Register(id: String)
+  case class StartWorker(info: WorkerInitInfo)
+
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
index 9261e0aad..26ad6316c 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
@@ -256,6 +256,54 @@ object SecurityConfig {
 
 }
 
+sealed trait LauncherSettings
+
+case class AWSEMRLaunchSettings(
+  sshKeyPair: String,
+  sshKeyPath: String,
+  accessKey: String,
+  secretKey: String,
+  subnetId: String,
+  region: String,
+  additionalGroup: String,
+  emrRole: String,
+  emrEc2Role: String
+) extends LauncherSettings
+
+object AWSEMRLaunchSettings {
+
+  def apply(c: Config): AWSEMRLaunchSettings = {
+    AWSEMRLaunchSettings(
+      sshKeyPair = c.getString("sshKeyPair"),
+      sshKeyPath = c.getString("sshKeyPath"),
+      accessKey = c.getString("accessKey"),
+      secretKey = c.getString("secretKey"),
+      subnetId = c.getString("subnetId"),
+      region = c.getString("region"),
+      additionalGroup = c.getString("additionalGroup"),
+      emrRole = c.getString("emrRole"),
+      emrEc2Role = c.getString("emrEc2Role")
+    )
+  }
+}
+
+object LauncherSettings {
+
+  def apply(c: Config): LauncherSettings = {
+    c.getString("type") match {
+      case "aws_emr" => AWSEMRLaunchSettings(c)
+      case x => throw new IllegalArgumentException(s"Unknown launcher settings type: $x")
+    }
+  }
+
+  def extractAll(all: Seq[Config]): Map[String, LauncherSettings] = {
+    all.map(c => {
+      val name = c.getString("name")
+      name -> LauncherSettings(c)
+    }).toMap
+  }
+}
+
 case class MasterConfig(
   cluster: HostPortConfig,
   http: HttpConfig,
@@ -272,6 +320,7 @@ case class MasterConfig(
   srcConfigPath: String,
   jobsSavePath: String,
   artifactRepositoryPath: String,
+  launchersSettings: Map[String, LauncherSettings],
   raw: Config
 )
 
@@ -314,6 +363,7 @@ object MasterConfig extends Logger {
       security = SecurityConfig.ifEnabled(mist.getConfig("security")),
       jobInfoProviderConfig = FunctionInfoProviderConfig(mist.getConfig("job-extractor")),
       srcConfigPath = filePath,
+      launchersSettings = LauncherSettings.extractAll(mist.getConfigList("launchers-settings")),
       raw = config
     )
   }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
index 562fcf216..7189a658c 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
@@ -30,6 +30,17 @@ trait NamedConfig {
   val name: String
 }
 
+sealed trait LaunchData
+/** use default worker-runner **/
+case object ServerDefault extends LaunchData
+case class AWSEMRLaunchData(
+  launcherSettingsName: String,
+  releaseLabel: String,
+  masterInstanceType: String,
+  slaveInstanceType: String,
+  instanceCount: Int
+) extends LaunchData
+
 case class ContextConfig(
   name: String,
   sparkConf: Map[String, String],
@@ -39,7 +50,8 @@ case class ContextConfig(
   runOptions: String,
   workerMode: RunMode,
   streamingDuration: Duration,
-  maxConnFailures: Int
+  maxConnFailures: Int,
+  launchData: LaunchData
 ) extends NamedConfig {
 
   def maxJobsOnNode: Int = workerMode match {

From b0fc9e240d77571164c8d86d9eb2caa8b868856d Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 20 Sep 2018 23:27:48 +0300
Subject: [PATCH 13/30] wip

---
 .../hydrosphere/mist/common/CommonData.scala  |  1 +
 .../master/execution/ContextFrontend.scala    | 32 +++++++++++------
 .../master/execution/ExecutionService.scala   |  2 +-
 .../mist/master/execution/SpawnSettings.scala |  2 +-
 .../master/execution/workers/Cancel.scala     | 34 +++++++++++++++++++
 .../execution/workers/SharedConnector.scala   |  2 +-
 .../hydrosphere/mist/master/models/base.scala |  8 +----
 7 files changed, 60 insertions(+), 21 deletions(-)
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/Cancel.scala

diff --git a/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
index 7ca323182..99c0c63db 100644
--- a/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
@@ -27,6 +27,7 @@ object CommonData {
    */
   case class WorkerInitInfo(
     sparkConf: Map[String, String],
+    // TODO drop this field
     maxJobs: Int,
     downtime: Duration,
     streamingDuration: Duration,
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index 2779f958c..7ddc9c138 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -2,6 +2,7 @@ package io.hydrosphere.mist.master.execution
 
 import java.util.UUID
 
+import akka.pattern.pipe
 import akka.actor.{Actor, ActorLogging, ActorRef, Props, Timers}
 import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest}
 import io.hydrosphere.mist.master.Messages.StatusMessages.FailedEvent
@@ -14,7 +15,7 @@ import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
 import io.hydrosphere.mist.utils.akka.{ActorF, ActorFSyntax}
 import mist.api.data.JsData
 
-import scala.concurrent.Promise
+import scala.concurrent.{Future, Promise}
 import scala.concurrent.duration._
 import scala.util.{Failure, Success}
 
@@ -43,7 +44,7 @@ class ContextFrontend(
   name: String,
   reporter: StatusReporter,
   loggersFactory: JobLoggersFactory,
-  connectorStarter: (String, ContextConfig) => WorkerConnector,
+  connectorStarter: (String, ContextConfig) => Future[WorkerConnector],
   jobFactory: ActorF[(ActorRef, RunJobRequest, Promise[JsData], StatusReporter, JobLogger)],
   defaultInactiveTimeout: FiniteDuration
 ) extends Actor
@@ -270,16 +271,23 @@ class ContextFrontend(
       context become sleepingTilUpdate(next, conns, brokenCtx, error)
   }
 
-  private def startConnector(ctx: ContextConfig): (String, WorkerConnector) = {
+  private def awaitConnector(ctx: ContextConfig, state: State, connId: String): Receive = {
+    case Event.Status => sender() ! mkStatus(State.empty[String, ActorRef])
+
+    case ContextEvent.UpdateContext(updCtx) =>
+
+    case Event.ConnectorStarted(id, conn) if id == connId =>
+    case Event.ConnectorStartFailed(id, e) if id == connId =>
+  }
+
+  private def startConnector(ctx: ContextConfig, state: State): Unit = {
     val id = ctx.name + "_" + UUID.randomUUID().toString
-    log.info(s"Starting executor $id for $name")
-    val connector = connectorStarter(id, ctx)
-    if (ctx.precreated) connector.warmUp()
-    connector.whenTerminated().onComplete({
-      case Success(_) => self ! Event.ConnectorStopped(id)
-      case Failure(e) => self ! Event.ConnectorCrushed(id, e)
-    })
-    id -> connector
+    log.info(s"Starting connector $id for $name")
+    connectorStarter(id, ctx).onComplete {
+      case Success(connector) => self ! Event.ConnectorStarted(id, connector)
+      case Failure(e) => self ! Event.ConnectorStartFailed(id, e)
+    }
+    context become awaitConnector(ctx, state, id)
   }
 
   private def cancelJob(id: String, state: State, respond: ActorRef): Unit = state.get(id) match {
@@ -321,6 +329,8 @@ object ContextFrontend {
 
   sealed trait Event
   object Event {
+    final case class ConnectorStarted(id: String, connector: WorkerConnector) extends Event
+    final case class ConnectorStartFailed(id: String, e: Throwable) extends Event
     final case class ConnectorCrushed(id: String, err: Throwable) extends Event
     final case class ConnectorStopped(id: String) extends Event
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
index 18f18f7be..d15e379ca 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
@@ -117,8 +117,8 @@ object ExecutionService {
     logService: LogService
   ): ExecutionService = {
     val hub = WorkerHub(spawn, system)
-    val reporter = StatusReporter.reporter(repo, streamer, logService)(system)
 
+    val reporter = StatusReporter.reporter(repo, streamer, logService)(system)
     val mkContext = ActorF[ContextConfig]((ctx, af) => {
       val props = ContextFrontend.props(ctx.name, reporter, logService, hub.start)
       val ref = af.actorOf(props)
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
index 56650d9fd..b9fdadbe6 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
@@ -19,7 +19,7 @@ case class SpawnSettings(
   def toWorkerInitInfo(ctx: ContextConfig): WorkerInitInfo =
     WorkerInitInfo(
       sparkConf = ctx.sparkConf,
-      maxJobs = ctx.maxJobsOnNode,
+      maxJobs = 1,
       downtime = ctx.downtime,
       streamingDuration = ctx.streamingDuration,
       logService = this.logAddress,
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/Cancel.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/Cancel.scala
new file mode 100644
index 000000000..ca88142f8
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/Cancel.scala
@@ -0,0 +1,34 @@
+package io.hydrosphere.mist.master.execution.workers
+
+import akka.actor.{Actor, ActorRef}
+
+import scala.concurrent.Future
+
+trait ActorFutureHandler { that: Actor =>
+
+  private var realRef: Option[ActorRef] = None
+
+  override def preStart(): Unit = {
+    that.preStart()
+    realRef = Some(self)
+  }
+
+  override def postStop(): Unit = {
+    that.postStop()
+    realRef = None
+  }
+
+  def subscribe[A, B, C](future: Future[A])(f: A => B, g: Throwable => C): Unit = {
+    future.onComplete(res => {
+      realRef match {
+        case None =>
+        case Some(ref) =>
+          val msg = res match {
+            case scala.util.Success(v) => f(v)
+            case scala.util.Failure(e) => g(e)
+          }
+          ref ! msg
+      }
+    })(context.dispatcher)
+  }
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
index 76146bc91..2ff569f68 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
@@ -37,7 +37,7 @@ class SharedConnector(
       context become process(Queue(req), Queue.empty, Map.empty, 1)
 
     case Event.WarmUp =>
-      (0 until ctx.maxJobsOnNode).foreach(_ => startConnection() pipeTo self)
+      (0 until ctx.maxJobs).foreach(_ => startConnection() pipeTo self)
       context become process(Queue.empty, Queue.empty, Map.empty, ctx.maxJobsOnNode)
   }
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
index 7189a658c..9ccbbb230 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
@@ -52,14 +52,8 @@ case class ContextConfig(
   streamingDuration: Duration,
   maxConnFailures: Int,
   launchData: LaunchData
-) extends NamedConfig {
-
-  def maxJobsOnNode: Int = workerMode match {
-    case RunMode.Shared => maxJobs
-    case RunMode.ExclusiveContext => 1
-  }
+) extends NamedConfig
 
-}
 
 case class FunctionConfig(
   name: String,

From 853e2090b92dedbde52d01243956f64ce00e0cfc Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Fri, 21 Sep 2018 13:22:55 +0300
Subject: [PATCH 14/30] wip

---
 .../master/execution/ContextFrontend.scala    | 92 +++++++++++++------
 1 file changed, 65 insertions(+), 27 deletions(-)

diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index 7ddc9c138..17fc21cfb 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -40,6 +40,7 @@ trait FrontendBasics {
   def mkStatus(state: State, conn: ConnectorState): FrontendStatus = mkStatus(state, Some(conn))
 }
 
+
 class ContextFrontend(
   name: String,
   reporter: StatusReporter,
@@ -71,8 +72,7 @@ class ContextFrontend(
     case req: RunJobRequest =>
       timers.cancel(timerKey)
       val next = mkJob(req, State.empty, sender())
-      val (id, connector) = startConnector(ctx)
-      becomeWithConnector(ctx, next, ConnectorState.initial(id, connector))
+      gotoStartingConnector(ctx, next)
 
     case Event.Downtime =>
       log.info(s"Context $name was inactive")
@@ -82,8 +82,7 @@ class ContextFrontend(
   // handle UpdateContext for awaitRequest/initial
   private def becomeAwaitOrConnected(ctx: ContextConfig, state: State): Unit = {
     if (ctx.precreated && ctx.workerMode == RunMode.Shared) {
-      val (id, connector) = startConnector(ctx)
-      becomeWithConnector(ctx, state, ConnectorState.initial(id, connector))
+      gotoStartingConnector(ctx, state)
     } else {
       val timerKey = s"$name-await-timeout"
       timers.startSingleTimer(timerKey, Event.Downtime, defaultInactiveTimeout)
@@ -148,17 +147,12 @@ class ContextFrontend(
     def becomeNextConn(next: ConnectorState): Unit = becomeWithConnector(ctx, currentState, next)
     def becomeNext(c: ConnectorState, s: State): Unit = becomeWithConnector(ctx, s, c)
 
-    def becomeSleeping(state: State, conn: ConnectorState, brokenCtx: ContextConfig, error: Throwable): Unit = {
-      currentState.queued.foreach({case (_, ref) => ref ! JobActor.Event.ContextBroken(error)})
-      context become sleepingTilUpdate(state, conn, brokenCtx, error)
-    }
 
     {
       case Event.Status => sender() ! mkStatus(currentState, connectorState)
       case ContextEvent.UpdateContext(updCtx) =>
         connectorState.connector.shutdown(false)
-        val (newId, newConn) = startConnector(updCtx)
-        becomeWithConnector(updCtx, currentState, ConnectorState.initial(newId, newConn))
+        gotoStartingConnector(updCtx, currentState)
 
       case req: RunJobRequest => becomeNextState(mkJob(req, currentState, sender()))
       case CancelJobRequest(id) =>
@@ -182,7 +176,7 @@ class ContextFrontend(
         val newConnState = connectorState.askFailure
         if (newConnState.failedTimes >= ctx.maxConnFailures) {
           connectorState.connector.shutdown(false)
-          becomeSleeping(currentState, newConnState, ctx, e)
+          becomeSleeping(currentState, ctx, e)
         } else {
           becomeNextConn(newConnState)
         }
@@ -199,16 +193,17 @@ class ContextFrontend(
         log.error(e, "Context {} - connector {} was crushed", name, id)
         val next = connectorState.connectorFailed
         if (next.failedTimes >= ctx.maxConnFailures) {
-          becomeSleeping(currentState, next, ctx, e)
+          becomeSleeping(currentState, ctx, e)
         } else {
-          val (newId, newConn) = startConnector(ctx)
-          becomeWithConnector(ctx, currentState, ConnectorState.initial(newId, newConn).copy(failedTimes = next.failedTimes))
+          gotoStartingConnector(ctx, currentState, next.failedTimes)
         }
 
+      //TODO is it possible to stop connector outside except ContextFrontend???
       case Event.ConnectorStopped(id) if id == connectorState.id =>
         log.info("Context {} - connector {} was stopped", name, id)
-        val (newId, newConn) = startConnector(ctx)
-        becomeWithConnector(ctx, currentState, ConnectorState.initial(newId, newConn))
+        gotoStartingConnector(ctx, currentState, 0)
+
+      case Event.ConnectorStarted(_, conn) => conn.shutdown(true)
     }
   }
 
@@ -249,10 +244,21 @@ class ContextFrontend(
     case Event.ConnectorStopped(id) if id == connectorState.id =>
       log.info("Context {} was stopped in empty state - shutdown", name)
       context stop self
+
+    case Event.ConnectorStarted(_, conn) => conn.shutdown(true)
   }
 
-  private def sleepingTilUpdate(state: State, conn: ConnectorState, brokenCtx: ContextConfig, error: Throwable): Receive = {
-    case Event.Status => sender() ! mkStatus(state, conn)
+  def becomeSleeping(
+    state: State,
+    brokenCtx: ContextConfig,
+    error: Throwable
+  ): Unit = {
+    state.queued.foreach({case (_, ref) => ref ! JobActor.Event.ContextBroken(error)})
+    context become sleepingTilUpdate(state, brokenCtx, error)
+  }
+
+  private def sleepingTilUpdate(state: State, brokenCtx: ContextConfig, error: Throwable): Receive = {
+    case Event.Status => sender() ! mkStatus(state)
     case ContextEvent.UpdateContext(updCtx) => becomeAwaitOrConnected(updCtx, FrontendState.empty)
 
     case req: RunJobRequest => respondWithError(brokenCtx, req, sender(), error)
@@ -263,33 +269,65 @@ class ContextFrontend(
     case Event.Connection(_, connection) => connection.release()
 
     case JobActor.Event.Completed(id) =>
-      val (conns, next) = state.getWithState(id) match {
-        case Some((_, Working)) => conn.connectionReleased -> state.done(id)
-        case Some((_, Waiting)) => conn -> state.done(id)
-        case None => conn -> state
+      val next = state.get(id) match {
+        case Some(_) => state.done(id)
+        case None => state
       }
-      context become sleepingTilUpdate(next, conns, brokenCtx, error)
+      context become sleepingTilUpdate(next, brokenCtx, error)
+
+    case Event.ConnectorStarted(_, conn) => conn.shutdown(true)
   }
 
-  private def awaitConnector(ctx: ContextConfig, state: State, connId: String): Receive = {
-    case Event.Status => sender() ! mkStatus(State.empty[String, ActorRef])
+  private def startingConnector(
+    ctx: ContextConfig,
+    state: State,
+    connId: String,
+    failedTimes: Int
+  ): Receive = {
+    case Event.Status => sender() ! mkStatus(state)
 
+    // TODO check if it possible to update current connector/cluster or start new
     case ContextEvent.UpdateContext(updCtx) =>
+      log.info(s"Update ctx: $name")
+      gotoStartingConnector(updCtx, state, 0)
+
+    case req: RunJobRequest =>
+      val next = mkJob(req, state, sender())
+      context become startingConnector(ctx, next, connId, failedTimes)
+
+    case CancelJobRequest(id) => cancelJob(id, state, sender())
+
+    case JobActor.Event.Completed(id) =>
+      val next = state.get(id) match {
+        case Some(_) => state.done(id)
+        case None => state
+      }
+      context become startingConnector(ctx, next, connId, failedTimes)
+
+    case Event.Connection(_, connection) => connection.release()
 
     case Event.ConnectorStarted(id, conn) if id == connId =>
+      log.info("Connector started {} id: {}", name, id)
+      val connState = ConnectorState.initial(id, conn).copy(failedTimes = failedTimes)
+      becomeWithConnector(ctx, state, connState)
+
+    case Event.ConnectorStarted(_, conn) => conn.shutdown(true)
     case Event.ConnectorStartFailed(id, e) if id == connId =>
+      log.error(e, "Starting connector {} id: {} failed", name, id)
+      becomeSleeping(state, ctx, e)
   }
 
-  private def startConnector(ctx: ContextConfig, state: State): Unit = {
+  private def gotoStartingConnector(ctx: ContextConfig, state: State, failedTimes: Int): Unit = {
     val id = ctx.name + "_" + UUID.randomUUID().toString
     log.info(s"Starting connector $id for $name")
     connectorStarter(id, ctx).onComplete {
       case Success(connector) => self ! Event.ConnectorStarted(id, connector)
       case Failure(e) => self ! Event.ConnectorStartFailed(id, e)
     }
-    context become awaitConnector(ctx, state, id)
+    context become startingConnector(ctx, state, id, failedTimes)
   }
 
+  //TODO bug - it's possible to cancel and then assign connection to job
   private def cancelJob(id: String, state: State, respond: ActorRef): Unit = state.get(id) match {
     case Some(ref) =>
       ref.tell(JobActor.Event.Cancel, respond)

From a09d8a1053e6fd0c3fbb0f001311f6b2642a26db Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Fri, 21 Sep 2018 16:01:46 +0300
Subject: [PATCH 15/30] wip

---
 .../master/execution/ContextFrontend.scala    | 54 ++++++++++++-------
 1 file changed, 35 insertions(+), 19 deletions(-)

diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index 17fc21cfb..d488ea724 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -82,7 +82,7 @@ class ContextFrontend(
   // handle UpdateContext for awaitRequest/initial
   private def becomeAwaitOrConnected(ctx: ContextConfig, state: State): Unit = {
     if (ctx.precreated && ctx.workerMode == RunMode.Shared) {
-      gotoStartingConnector(ctx, state)
+      gotoStartingConnector(ctx, state, 0)
     } else {
       val timerKey = s"$name-await-timeout"
       timers.startSingleTimer(timerKey, Event.Downtime, defaultInactiveTimeout)
@@ -90,6 +90,10 @@ class ContextFrontend(
     }
   }
 
+  private def shouldGoToEmptyWithTimeout(state: State, ctx: ContextConfig): Boolean = {
+    state.isEmpty && !ctx.precreated && ctx.downtime.isFinite()
+  }
+
   // handle currentState changes, starting new jobs if it's possible
   // or awaiting Downtime timeout
   private def becomeWithConnector(
@@ -105,19 +109,8 @@ class ContextFrontend(
       }
     }
 
-    def shouldGoToEmptyWithTimeout(): Boolean = {
-      state.isEmpty && !ctx.precreated && ctx.downtime.isFinite()
-    }
-
-    if (shouldGoToEmptyWithTimeout()) {
-      val timerKey = s"$name-wait-downtime"
-      val timeout = ctx.downtime match {
-        case f: FiniteDuration => f
-        case _ => defaultInactiveTimeout
-      }
-      timers.startSingleTimer(timerKey, Event.Downtime, timeout)
-      log.info("Context {} - move to inactive state", name)
-      context become emptyWithConnector(ctx, connectorState, timerKey)
+    if (shouldGoToEmptyWithTimeout(state, ctx)) {
+      gotoEmptyWithConnector(ctx, connectorState)
     } else {
       val available = ctx.maxJobs - connectorState.all
       val need = math.min(state.queued.size - connectorState.asked, available)
@@ -152,7 +145,7 @@ class ContextFrontend(
       case Event.Status => sender() ! mkStatus(currentState, connectorState)
       case ContextEvent.UpdateContext(updCtx) =>
         connectorState.connector.shutdown(false)
-        gotoStartingConnector(updCtx, currentState)
+        gotoStartingConnector(updCtx, currentState, 0)
 
       case req: RunJobRequest => becomeNextState(mkJob(req, currentState, sender()))
       case CancelJobRequest(id) =>
@@ -207,6 +200,20 @@ class ContextFrontend(
     }
   }
 
+  private def gotoEmptyWithConnector(
+    ctx: ContextConfig,
+    connectorState: ConnectorState
+  ): Unit = {
+    val timerKey = s"$name-wait-downtime"
+    val timeout = ctx.downtime match {
+      case f: FiniteDuration => f
+      case _ => defaultInactiveTimeout
+    }
+    timers.startSingleTimer(timerKey, Event.Downtime, timeout)
+    log.info("Context {} - move to inactive state", name)
+    context become emptyWithConnector(ctx, connectorState, timerKey)
+  }
+
   // optional state - use it if ctx isn't precreated
   private def emptyWithConnector(
     ctx: ContextConfig,
@@ -216,8 +223,7 @@ class ContextFrontend(
     case Event.Status => sender() ! mkStatus(State.empty[String, ActorRef], connectorState)
     case ContextEvent.UpdateContext(updCtx) =>
       connectorState.connector.shutdown(false)
-      val (newId, newConn) = startConnector(updCtx)
-      context become emptyWithConnector(updCtx, ConnectorState.initial(newId, newConn), timerKey)
+      gotoStartingConnector(updCtx, State.empty, 0)
 
     case req: RunJobRequest =>
       timers.cancel(timerKey)
@@ -309,12 +315,22 @@ class ContextFrontend(
     case Event.ConnectorStarted(id, conn) if id == connId =>
       log.info("Connector started {} id: {}", name, id)
       val connState = ConnectorState.initial(id, conn).copy(failedTimes = failedTimes)
-      becomeWithConnector(ctx, state, connState)
+
+      if (shouldGoToEmptyWithTimeout(state, ctx)) {
+        gotoEmptyWithConnector(ctx, connState)
+      } else {
+        becomeWithConnector(ctx, state, connState)
+      }
+
 
     case Event.ConnectorStarted(_, conn) => conn.shutdown(true)
     case Event.ConnectorStartFailed(id, e) if id == connId =>
       log.error(e, "Starting connector {} id: {} failed", name, id)
-      becomeSleeping(state, ctx, e)
+      if (failedTimes > ctx.maxConnFailures) {
+        becomeSleeping(state, ctx, e)
+      } else {
+        gotoStartingConnector(ctx, state, failedTimes + 1)
+      }
   }
 
   private def gotoStartingConnector(ctx: ContextConfig, state: State, failedTimes: Int): Unit = {

From 09d3717b13fc458f87d4ad125408eb0a7e6a9c91 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Mon, 24 Sep 2018 21:27:16 +0300
Subject: [PATCH 16/30] aws cluster - wip, prepare contextFrontend

---
 mist.sbt                                      |  3 +-
 .../hydrosphere/mist/common/CommonData.scala  |  2 -
 mist/master/src/main/resources/master.conf    |  2 +
 .../mist/master/data/ConfigRepr.scala         | 64 +++++++++++++++----
 .../mist/master/data/FsStorage.scala          | 11 ++--
 .../master/execution/ContextFrontend.scala    | 52 ++++++++-------
 .../master/execution/ExecutionService.scala   |  3 +-
 .../mist/master/execution/SpawnSettings.scala |  1 -
 .../execution/workers/SharedConnector.scala   |  4 +-
 .../mist/master/interfaces/http/models.scala  |  8 ++-
 .../mist/master/interfaces/jsonCodecs.scala   | 32 +++++++++-
 .../mist/master/ContextsCrudLikeSpec.scala    |  6 +-
 .../mist/master/MistConfigSpec.scala          |  5 +-
 .../io/hydrosphere/mist/master/TestData.scala |  1 -
 .../mist/master/data/ConfigReprsSpec.scala    | 14 ++--
 .../master/data/ContextsStorageSpec.scala     | 10 +--
 .../mist/master/data/FStorageSpec.scala       |  2 +-
 .../execution/ContextFrontendSpec.scala       | 32 +++++++---
 .../master/execution/SpawnSettingsSpec.scala  |  5 --
 .../starter/SparkSubmitBuilderSpec.scala      |  1 -
 .../interfaces/http/HttpApiV2Spec.scala       |  4 +-
 project/Library.scala                         |  2 +
 22 files changed, 172 insertions(+), 92 deletions(-)

diff --git a/mist.sbt b/mist.sbt
index bdbc30727..6954fcd0c 100644
--- a/mist.sbt
+++ b/mist.sbt
@@ -75,7 +75,8 @@ lazy val master = project.in(file("mist/master"))
     scalacOptions ++= commonScalacOptions,
     libraryDependencies ++= Library.Akka.base,
     libraryDependencies ++= Seq(
-      Library.slf4jLog4j, Library.typesafeConfig, Library.scopt,
+      Library.slf4jLog4j, Library.log4j, Library.log4jExtras,
+      Library.typesafeConfig, Library.scopt,
       Library.slick, Library.h2, Library.flyway,
       Library.chill,
       Library.kafka, Library.pahoMqtt,
diff --git a/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
index 99c0c63db..64cb7dce5 100644
--- a/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala
@@ -27,8 +27,6 @@ object CommonData {
    */
   case class WorkerInitInfo(
     sparkConf: Map[String, String],
-    // TODO drop this field
-    maxJobs: Int,
     downtime: Duration,
     streamingDuration: Duration,
     logService: String,
diff --git a/mist/master/src/main/resources/master.conf b/mist/master/src/main/resources/master.conf
index 1d4dfcab9..912466ec1 100644
--- a/mist/master/src/main/resources/master.conf
+++ b/mist/master/src/main/resources/master.conf
@@ -126,6 +126,8 @@ mist {
     }
   }
 
+  launchers-settings = []
+
 }
 
 writers-blocking-dispatcher {
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
index 419b2715b..9e0236032 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
@@ -1,19 +1,21 @@
 package io.hydrosphere.mist.master.data
 
 import com.typesafe.config.{Config, ConfigValue, ConfigValueFactory, ConfigValueType}
-import io.hydrosphere.mist.master.models.{ContextConfig, FunctionConfig, NamedConfig, RunMode}
+import io.hydrosphere.mist.master.models._
 import io.hydrosphere.mist.utils.ConfigUtils._
 
 import scala.concurrent.duration._
 
-trait ConfigRepr[A <: NamedConfig] {
-
+trait ConfigRepr[A] {
   def toConfig(a: A): Config
-
   def fromConfig(config: Config): A
+}
+
+trait NamedConfigRepr[A] extends ConfigRepr[A] {
 
   def fromConfig(name: String, config: Config): A =
     fromConfig(config.withValue("name", ConfigValueFactory.fromAnyRef(name)))
+
 }
 
 object ConfigRepr {
@@ -24,12 +26,12 @@ object ConfigRepr {
     def toConfig: Config = repr.toConfig(a)
   }
 
-  implicit class FromCongixSyntax(config: Config){
-    def to[A <: NamedConfig](implicit repr: ConfigRepr[A]): A = repr.fromConfig(config)
-    def to[A <: NamedConfig](name: String)(implicit repr: ConfigRepr[A]): A = repr.fromConfig(name, config)
-  }
+//  implicit class FromConfigSyntax(config: Config){
+//    def to[A <: NamedConfig](implicit repr: ConfigRepr[A]): A = repr.fromConfig(config)
+//    def to[A <: NamedConfig](name: String)(implicit repr: NamedConfigRepr[A]): A = repr.fromConfig(name, config)
+//  }
 
-  implicit val EndpointsRepr = new ConfigRepr[FunctionConfig] {
+  val EndpointsRepr = new NamedConfigRepr[FunctionConfig] {
 
     override def toConfig(a: FunctionConfig): Config = {
       import ConfigValueFactory._
@@ -51,7 +53,43 @@ object ConfigRepr {
     }
   }
 
-  implicit val ContextConfigRepr: ConfigRepr[ContextConfig] = new ConfigRepr[ContextConfig] {
+  val LaunchDataConfigRepr: ConfigRepr[LaunchData] = new ConfigRepr[LaunchData] {
+
+    override def toConfig(a: LaunchData): Config = {
+      import ConfigValueFactory._
+
+      val (t, body) = a match {
+        case ServerDefault => "server-default" -> Map.empty[String, ConfigValue]
+        case awsEmr: AWSEMRLaunchData =>
+          "aws-emr" -> Map(
+            "launcher-settings-name" -> fromAnyRef(awsEmr.launcherSettingsName),
+            "release-label" -> fromAnyRef(awsEmr.releaseLabel),
+            "master-instance-type" -> fromAnyRef(awsEmr.masterInstanceType),
+            "slave-instance-type" -> fromAnyRef(awsEmr.slaveInstanceType),
+            "instance-count" -> fromAnyRef(awsEmr.instanceCount)
+          )
+      }
+      val full = body + ("type" -> t)
+      fromMap(full.asJava).toConfig
+    }
+
+    override def fromConfig(config: Config): LaunchData = {
+      config.getString("type") match {
+        case "server-default" => ServerDefault
+        case "aws-emr" =>
+          AWSEMRLaunchData(
+            launcherSettingsName = config.getString("launcher-settings-name"),
+            releaseLabel = config.getString("release-label"),
+            masterInstanceType = config.getString("master-instance-type"),
+            slaveInstanceType = config.getString("slave-instance-type"),
+            instanceCount = config.getInt("instance-count")
+          )
+        case x => throw new IllegalArgumentException(s"Unknown launch data type $x")
+      }
+    }
+  }
+
+  val ContextConfigRepr: NamedConfigRepr[ContextConfig] = new NamedConfigRepr[ContextConfig] {
 
     val allowedTypes = Set(
       ConfigValueType.STRING,
@@ -79,7 +117,8 @@ object ConfigRepr {
         workerMode = runMode(config.getString("worker-mode")) ,
         runOptions = config.getString("run-options"),
         streamingDuration = Duration(config.getString("streaming-duration")),
-        maxConnFailures = config.getOptInt("max-conn-failures").getOrElse(5)
+        maxConnFailures = config.getOptInt("max-conn-failures").getOrElse(5),
+        launchData = config.getOptConfig("launch-data").map(LaunchDataConfigRepr.fromConfig).getOrElse(ServerDefault)
       )
     }
 
@@ -100,7 +139,8 @@ object ConfigRepr {
         "worker-mode" -> fromAnyRef(a.workerMode.name),
         "run-options" -> fromAnyRef(a.runOptions),
         "streaming-duration" -> fromDuration(a.streamingDuration),
-        "max-conn-failures" -> fromAnyRef(a.maxConnFailures)
+        "max-conn-failures" -> fromAnyRef(a.maxConnFailures),
+        "launch-data" -> LaunchDataConfigRepr.toConfig(a.launchData).root()
       )
       fromMap(map.asJava).toConfig
     }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/FsStorage.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/FsStorage.scala
index 784862629..f593dede7 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/FsStorage.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/FsStorage.scala
@@ -2,20 +2,17 @@ package io.hydrosphere.mist.master.data
 
 import java.io.File
 import java.nio.file.{Files, Path}
-import java.util.concurrent.Executors
 
 import com.typesafe.config.{ConfigFactory, ConfigRenderOptions}
-import io.hydrosphere.mist.master.ContextsSettings
 import io.hydrosphere.mist.master.data.FsStorage._
-import io.hydrosphere.mist.master.models.{ContextConfig, NamedConfig}
+import io.hydrosphere.mist.master.models.NamedConfig
 import io.hydrosphere.mist.utils.{Logger, fs}
 
-import scala.concurrent.{ExecutionContext, Future}
 import scala.util._
 
-class FsStorage[A <: NamedConfig](
+class FsStorage[A](
   dir: Path,
-  repr: ConfigRepr[A],
+  repr: NamedConfigRepr[A],
   renderOptions: ConfigRenderOptions = DefaultRenderOptions
 ) extends Logger with RwLock { self =>
 
@@ -77,7 +74,7 @@ object FsStorage {
       .setJson(false)
       .setFormatted(true)
 
-  def create[A <: NamedConfig](path: String, repr: ConfigRepr[A]): FsStorage[A] =
+  def create[A](path: String, repr: NamedConfigRepr[A]): FsStorage[A] =
     new FsStorage[A](checkDirectory(path), repr)
 
 }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index d488ea724..4db7ffa52 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -2,7 +2,6 @@ package io.hydrosphere.mist.master.execution
 
 import java.util.UUID
 
-import akka.pattern.pipe
 import akka.actor.{Actor, ActorLogging, ActorRef, Props, Timers}
 import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest}
 import io.hydrosphere.mist.master.Messages.StatusMessages.FailedEvent
@@ -24,20 +23,16 @@ trait FrontendBasics {
   type State = FrontendState[String, ActorRef]
   val State = FrontendState
 
-  def mkStatus(state: State, conn: Option[ConnectorState]): FrontendStatus = {
+  def mkStatus(state: State, failures: Int, execId: Option[String]): FrontendStatus = {
     val jobs =
       state.queued.map({case (k, _) => k -> ExecStatus.Queued}) ++
       state.active.map({case (k, _) => k -> ExecStatus.Started})
 
-    FrontendStatus(
-      jobs = jobs,
-      conn.map(_.id),
-      conn.map(_.failedTimes).getOrElse(0)
-    )
+    FrontendStatus(jobs, failures, execId)
   }
 
-  def mkStatus(state: State): FrontendStatus = mkStatus(state, None)
-  def mkStatus(state: State, conn: ConnectorState): FrontendStatus = mkStatus(state, Some(conn))
+  def mkStatus(state: State): FrontendStatus = mkStatus(state, 0, None)
+  def mkStatus(state: State, conn: ConnectorState): FrontendStatus = mkStatus(state, conn.failedTimes, Option(conn.id))
 }
 
 
@@ -72,7 +67,7 @@ class ContextFrontend(
     case req: RunJobRequest =>
       timers.cancel(timerKey)
       val next = mkJob(req, State.empty, sender())
-      gotoStartingConnector(ctx, next)
+      gotoStartingConnector(ctx, next, 0)
 
     case Event.Downtime =>
       log.info(s"Context $name was inactive")
@@ -169,7 +164,7 @@ class ContextFrontend(
         val newConnState = connectorState.askFailure
         if (newConnState.failedTimes >= ctx.maxConnFailures) {
           connectorState.connector.shutdown(false)
-          becomeSleeping(currentState, ctx, e)
+          becomeSleeping(currentState, ctx, e, newConnState.failedTimes)
         } else {
           becomeNextConn(newConnState)
         }
@@ -186,7 +181,7 @@ class ContextFrontend(
         log.error(e, "Context {} - connector {} was crushed", name, id)
         val next = connectorState.connectorFailed
         if (next.failedTimes >= ctx.maxConnFailures) {
-          becomeSleeping(currentState, ctx, e)
+          becomeSleeping(currentState, ctx, e, next.failedTimes)
         } else {
           gotoStartingConnector(ctx, currentState, next.failedTimes)
         }
@@ -257,14 +252,20 @@ class ContextFrontend(
   def becomeSleeping(
     state: State,
     brokenCtx: ContextConfig,
-    error: Throwable
+    error: Throwable,
+    failedTimes: Int
   ): Unit = {
     state.queued.foreach({case (_, ref) => ref ! JobActor.Event.ContextBroken(error)})
-    context become sleepingTilUpdate(state, brokenCtx, error)
+    context become sleepingTilUpdate(state, brokenCtx, error, failedTimes)
   }
 
-  private def sleepingTilUpdate(state: State, brokenCtx: ContextConfig, error: Throwable): Receive = {
-    case Event.Status => sender() ! mkStatus(state)
+  private def sleepingTilUpdate(
+    state: State,
+    brokenCtx: ContextConfig,
+    error: Throwable,
+    failedTimes: Int
+  ): Receive = {
+    case Event.Status => sender() ! mkStatus(state, failedTimes, None)
     case ContextEvent.UpdateContext(updCtx) => becomeAwaitOrConnected(updCtx, FrontendState.empty)
 
     case req: RunJobRequest => respondWithError(brokenCtx, req, sender(), error)
@@ -279,7 +280,7 @@ class ContextFrontend(
         case Some(_) => state.done(id)
         case None => state
       }
-      context become sleepingTilUpdate(next, brokenCtx, error)
+      context become sleepingTilUpdate(next, brokenCtx, error, failedTimes)
 
     case Event.ConnectorStarted(_, conn) => conn.shutdown(true)
   }
@@ -314,7 +315,12 @@ class ContextFrontend(
 
     case Event.ConnectorStarted(id, conn) if id == connId =>
       log.info("Connector started {} id: {}", name, id)
+      if (ctx.precreated && ctx.workerMode == RunMode.Shared) conn.warmUp()
       val connState = ConnectorState.initial(id, conn).copy(failedTimes = failedTimes)
+      conn.whenTerminated().onComplete({
+        case Success(_) => self ! Event.ConnectorStopped(id)
+        case Failure(e) => self ! Event.ConnectorCrushed(id, e)
+      })
 
       if (shouldGoToEmptyWithTimeout(state, ctx)) {
         gotoEmptyWithConnector(ctx, connState)
@@ -327,7 +333,7 @@ class ContextFrontend(
     case Event.ConnectorStartFailed(id, e) if id == connId =>
       log.error(e, "Starting connector {} id: {} failed", name, id)
       if (failedTimes > ctx.maxConnFailures) {
-        becomeSleeping(state, ctx, e)
+        becomeSleeping(state, ctx, e, failedTimes)
       } else {
         gotoStartingConnector(ctx, state, failedTimes + 1)
       }
@@ -400,11 +406,11 @@ object ContextFrontend {
 
   case class FrontendStatus(
     jobs: Map[String, ExecStatus],
-    executorId: Option[String],
-    failures: Int
+    failures: Int,
+    executorId: Option[String]
   )
   object FrontendStatus {
-    val empty: FrontendStatus = FrontendStatus(Map.empty, None, 0)
+    val empty: FrontendStatus = FrontendStatus(Map.empty, 0, None)
   }
 
   case class ConnectorState(
@@ -431,7 +437,7 @@ object ContextFrontend {
     name: String,
     status: StatusReporter,
     loggersFactory: JobLoggersFactory,
-    connectorStarter: (String, ContextConfig) => WorkerConnector,
+    connectorStarter: (String, ContextConfig) => Future[WorkerConnector],
     jobFactory: ActorF[(ActorRef, RunJobRequest, Promise[JsData], StatusReporter, JobLogger)],
     defaultInactiveTimeout: FiniteDuration
   ): Props = Props(classOf[ContextFrontend], name, status, loggersFactory, connectorStarter, jobFactory, defaultInactiveTimeout)
@@ -441,6 +447,6 @@ object ContextFrontend {
     name: String,
     status: StatusReporter,
     loggersFactory: JobLoggersFactory,
-    connectorStarter: (String, ContextConfig) => WorkerConnector
+    connectorStarter: (String, ContextConfig) => Future[WorkerConnector]
   ): Props = props(name, status, loggersFactory, connectorStarter, ActorF.props(JobActor.props _), 5 minutes)
 }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
index d15e379ca..bd71ebe70 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
@@ -116,11 +116,12 @@ object ExecutionService {
     repo: JobRepository,
     logService: LogService
   ): ExecutionService = {
+    // TODO
     val hub = WorkerHub(spawn, system)
 
     val reporter = StatusReporter.reporter(repo, streamer, logService)(system)
     val mkContext = ActorF[ContextConfig]((ctx, af) => {
-      val props = ContextFrontend.props(ctx.name, reporter, logService, hub.start)
+      val props = ContextFrontend.props(ctx.name, reporter, logService, (id, ctx) => Future.successful(hub.start(id, ctx)))
       val ref = af.actorOf(props)
       ref ! ContextEvent.UpdateContext(ctx)
       ref
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
index b9fdadbe6..f84c4dddf 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
@@ -19,7 +19,6 @@ case class SpawnSettings(
   def toWorkerInitInfo(ctx: ContextConfig): WorkerInitInfo =
     WorkerInitInfo(
       sparkConf = ctx.sparkConf,
-      maxJobs = 1,
       downtime = ctx.downtime,
       streamingDuration = ctx.streamingDuration,
       logService = this.logAddress,
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
index 2ff569f68..29f4d79dc 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
@@ -38,7 +38,7 @@ class SharedConnector(
 
     case Event.WarmUp =>
       (0 until ctx.maxJobs).foreach(_ => startConnection() pipeTo self)
-      context become process(Queue.empty, Queue.empty, Map.empty, ctx.maxJobsOnNode)
+      context become process(Queue.empty, Queue.empty, Map.empty, ctx.maxJobs)
   }
 
   private def process(
@@ -85,7 +85,7 @@ class SharedConnector(
           context become process(requests, pool, inUse, startingConnections - 1)
       }
 
-    case Event.AskConnection(req) if pool.isEmpty && inUse.size + startingConnections < ctx.maxJobsOnNode =>
+    case Event.AskConnection(req) if pool.isEmpty && inUse.size + startingConnections < ctx.maxJobs =>
       log.info(s"Pool is empty and we are able to start new one connection: inUse size :${inUse.size}")
       startConnection() pipeTo self
       context become process(requests :+ req, pool, inUse, startingConnections + 1)
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala
index 67bb4d83f..4709bd93b 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala
@@ -4,7 +4,7 @@ import java.lang.management._
 import java.time.LocalDateTime
 
 import io.hydrosphere.mist.common.FunctionInfoData
-import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
+import io.hydrosphere.mist.master.models.{ContextConfig, LaunchData, RunMode}
 import mist.api._
 
 import scala.concurrent.duration.Duration
@@ -122,7 +122,8 @@ case class ContextCreateRequest(
   workerMode: Option[RunMode] = None,
   runOptions: Option[String] = None,
   streamingDuration: Option[Duration] = None,
-  maxConnFailures: Option[Int] = None
+  maxConnFailures: Option[Int] = None,
+  launchData: Option[LaunchData] = None
 ) {
 
   def toContextWithFallback(other: ContextConfig): ContextConfig =
@@ -135,7 +136,8 @@ case class ContextCreateRequest(
       runOptions.getOrElse(other.runOptions),
       workerMode.getOrElse(other.workerMode),
       streamingDuration.getOrElse(other.streamingDuration),
-      maxConnFailures.getOrElse(other.maxConnFailures)
+      maxConnFailures.getOrElse(other.maxConnFailures),
+      launchData.getOrElse(other.launchData)
     )
 }
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
index 0e4d8c436..ffcfd119a 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
@@ -16,6 +16,7 @@ import spray.json._
 import scala.collection.JavaConversions._
 import scala.concurrent.duration._
 import scala.util.Try
+import scala.util.parsing.json.JSONObject
 
 trait AnyJsonFormat extends DefaultJsonProtocol {
 
@@ -167,7 +168,7 @@ trait JsonCodecs extends SprayJsonSupport
 
 
   implicit val workerInitInfoF = rootFormat(lazyFormat(jsonFormat(WorkerInitInfo.apply,
-    "sparkConf", "maxJobs", "downtime", "streamingDuration", "logService", "masterAddress","masterHttpConf", "maxArtifactSize", "runOptions")))
+    "sparkConf", "downtime", "streamingDuration", "logService", "masterAddress","masterHttpConf", "maxArtifactSize", "runOptions")))
 
 
   implicit val workerLinkF = rootFormat(lazyFormat(jsonFormat(WorkerLink.apply,
@@ -227,10 +228,35 @@ trait JsonCodecs extends SprayJsonSupport
 
   implicit val devJobStartReqModelF = jsonFormat7(DevJobStartRequestModel.apply)
 
+  implicit val launchDataF = new JsonFormat[LaunchData] {
 
-  implicit val contextConfigF = jsonFormat9(ContextConfig.apply)
+    val ServerDefaultKey = "server-default"
+    val AWSEMRKey = "aws-emr"
 
-  implicit val contextCreateRequestF = jsonFormat9(ContextCreateRequest.apply)
+    val awsEmrLaunchDataF = jsonFormat5(AWSEMRLaunchData.apply)
+
+    override def write(in: LaunchData): JsValue = in match {
+      case ServerDefault => JsObject("type" -> JsString(ServerDefaultKey))
+      case aws: AWSEMRLaunchData =>
+        val data = awsEmrLaunchDataF.write(aws).asJsObject.fields
+        JsObject(data + ("type" -> JsString(AWSEMRKey)))
+    }
+
+    override def read(json: JsValue): LaunchData = {
+      def fromType(t: String, obj: JsValue): LaunchData = t match {
+        case ServerDefaultKey => ServerDefault
+        case AWSEMRKey => awsEmrLaunchDataF.read(obj)
+      }
+      val t = json.asJsObject.fields.getOrElse("type", JsNull)
+      t match {
+        case JsString(v) => fromType(v, json)
+        case x => throw new IllegalArgumentException(s"Invalid launch data format $x")
+      }
+    }
+  }
+  implicit val contextConfigF = jsonFormat10(ContextConfig.apply)
+
+  implicit val contextCreateRequestF = jsonFormat10(ContextCreateRequest.apply)
   implicit val jobDetailsResponseF = jsonFormat2(JobDetailsResponse.apply)
 
   implicit val updateEventF = new JsonFormat[SystemEvent] {
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala
index f33d1bd10..2c63418af 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala
@@ -4,7 +4,7 @@ import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.data.ContextsStorage
 import io.hydrosphere.mist.master.execution.ExecutionService
 import io.hydrosphere.mist.master.interfaces.http.ContextCreateRequest
-import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
+import io.hydrosphere.mist.master.models.{ContextConfig, RunMode, ServerDefault}
 import org.scalatest.{FunSpec, Matchers}
 import org.mockito.Mockito._
 
@@ -17,7 +17,7 @@ class ContextsCrudLikeSpec extends FunSpec with Matchers with TestData with Mock
     val storage = mock[ContextsStorage]
     val executionS = mock[ExecutionService]
 
-    val defaultValue = ContextConfig("default", Map.empty, Duration.Inf, 20, precreated = false, "", RunMode.Shared, 1 seconds, 5)
+    val defaultValue = ContextConfig("default", Map.empty, Duration.Inf, 20, precreated = false, "", RunMode.Shared, 1 seconds, 5, ServerDefault)
     val req = ContextCreateRequest("yoyo", workerMode = Some(RunMode.ExclusiveContext))
 
     when(storage.defaultConfig).thenReturn(defaultValue)
@@ -29,7 +29,7 @@ class ContextsCrudLikeSpec extends FunSpec with Matchers with TestData with Mock
     }
 
     val result = Await.result(crud.create(req), Duration.Inf)
-    result shouldBe ContextConfig("yoyo", Map.empty, Duration.Inf, 20, precreated = false, "", RunMode.ExclusiveContext, 1 seconds, 5)
+    result shouldBe ContextConfig("yoyo", Map.empty, Duration.Inf, 20, precreated = false, "", RunMode.ExclusiveContext, 1 seconds, 5, ServerDefault)
 
     verify(executionS).updateContext(any[ContextConfig])
   }
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/MistConfigSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/MistConfigSpec.scala
index 81c3cc3c8..69bac4ff7 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/MistConfigSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/MistConfigSpec.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master
 
 import cats.{Eval, Now}
 import com.typesafe.config.ConfigFactory
-import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
+import io.hydrosphere.mist.master.models.{ContextConfig, RunMode, ServerDefault}
 import org.scalatest.{FunSpec, Matchers}
 
 import scala.concurrent.duration._
@@ -43,7 +43,8 @@ class MistConfigSpec extends FunSpec with Matchers {
       runOptions = "--opt",
       workerMode = RunMode.Shared,
       streamingDuration = 1.seconds,
-      maxConnFailures = 5
+      maxConnFailures = 5,
+      launchData = ServerDefault
     ))
 
   }
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
index cc10db52a..615f2a994 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
@@ -70,7 +70,6 @@ trait TestData {
 
   val workerInitData = WorkerInitInfo(
     sparkConf = FooContext.sparkConf,
-    maxJobs = FooContext.maxJobs,
     downtime = FooContext.downtime,
     streamingDuration = FooContext.streamingDuration,
     logService = "localhost:2005",
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ConfigReprsSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ConfigReprsSpec.scala
index d75fbdb90..0de263dab 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ConfigReprsSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ConfigReprsSpec.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.data
 
 import com.typesafe.config.{ConfigFactory, ConfigObject, ConfigParseOptions, ConfigRenderOptions}
-import io.hydrosphere.mist.master.models.{ContextConfig, FunctionConfig, RunMode}
+import io.hydrosphere.mist.master.models.{ContextConfig, FunctionConfig, RunMode, ServerDefault}
 import org.scalatest.{FunSpec, Matchers}
 
 import scala.concurrent.duration._
@@ -20,13 +20,13 @@ class ConfigReprsSpec extends FunSpec with Matchers {
          |  namespace = "namespace"
       """.stripMargin)
 
-      val parsed = cfg.to[FunctionConfig]("name")
+      val parsed = ConfigRepr.EndpointsRepr.fromConfig("name", cfg)
       parsed shouldBe FunctionConfig("name", "jar_path.jar", "MyJob1", "namespace")
     }
 
     it("should render to raw") {
       val e = FunctionConfig("name", "jar_path.jar", "MyJob1", "namespace")
-      val raw = e.toConfig
+      val raw = ConfigRepr.EndpointsRepr.toConfig(e)
 
       raw.getString("path") shouldBe "jar_path.jar"
       raw.getString("className") shouldBe "MyJob1"
@@ -51,12 +51,11 @@ class ConfigReprsSpec extends FunSpec with Matchers {
           |  streaming-duration = 30 seconds
           |  worker-mode = "shared"
           |  max-conn-failures = 10
-          |
           |}
         """.stripMargin
       , ConfigParseOptions.defaults())
 
-      val context = cfg.to[ContextConfig]("test")
+      val context = ConfigRepr.ContextConfigRepr.fromConfig("test", cfg)
       context.sparkConf shouldBe Map("x.y" -> "z", "a" -> "1")
       context.downtime shouldBe Duration.Inf
       context.maxJobs shouldBe 100
@@ -86,7 +85,7 @@ class ConfigReprsSpec extends FunSpec with Matchers {
         """.stripMargin
         , ConfigParseOptions.defaults())
 
-      val context = cfg.to[ContextConfig]("test")
+      val context = ConfigRepr.ContextConfigRepr.fromConfig("test", cfg)
       context.maxConnFailures shouldBe 5
     }
 
@@ -107,7 +106,8 @@ class ConfigReprsSpec extends FunSpec with Matchers {
         runOptions = "",
         workerMode = RunMode.Shared,
         streamingDuration = 1.minutes,
-        maxConnFailures = 5
+        maxConnFailures = 5,
+        launchData = ServerDefault
       )
       val raw = ConfigRepr.ContextConfigRepr.toConfig(e)
       Duration(raw.getString("downtime")) shouldBe 10.minutes
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ContextsStorageSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ContextsStorageSpec.scala
index e84291d50..80653c88a 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ContextsStorageSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ContextsStorageSpec.scala
@@ -5,7 +5,7 @@ import java.nio.file.Paths
 
 import io.hydrosphere.mist.master
 import io.hydrosphere.mist.master.TestUtils
-import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
+import io.hydrosphere.mist.master.models.{ContextConfig, RunMode, ServerDefault}
 import org.apache.commons.io.FileUtils
 import org.scalatest.{BeforeAndAfter, FunSpec, Matchers}
 
@@ -38,7 +38,7 @@ class ContextsStorageSpec extends FunSpec with Matchers with BeforeAndAfter {
   it("should update") {
     val contexts = testStorage()
 
-    val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, false, "weq", RunMode.Shared, 10 second, 5)
+    val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, false, "weq", RunMode.Shared, 10 second, 5, ServerDefault)
     contexts.update(ctx).await
     contexts.get("new").await.isDefined shouldBe true
   }
@@ -46,7 +46,7 @@ class ContextsStorageSpec extends FunSpec with Matchers with BeforeAndAfter {
   it("should return defaults") {
     val contexts = testStorage()
 
-    val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, false, "weq", RunMode.Shared, 10 second, 5)
+    val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, false, "weq", RunMode.Shared, 10 second, 5, ServerDefault)
     contexts.update(ctx).await
 
     contexts.all.await.map(_.name) should contain allOf ("default", "foo", "new")
@@ -60,7 +60,7 @@ class ContextsStorageSpec extends FunSpec with Matchers with BeforeAndAfter {
 
   it("should return precreated") {
     val contexts = testStorage()
-    val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, true, "weq", RunMode.Shared, 10 second, 5)
+    val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, true, "weq", RunMode.Shared, 10 second, 5, ServerDefault)
 
     contexts.update(ctx).await
     contexts.precreated.await should contain only(ctx)
@@ -74,7 +74,7 @@ class ContextsStorageSpec extends FunSpec with Matchers with BeforeAndAfter {
   it("should override settings") {
     val contexts = testStorage()
 
-    val ctx = ContextConfig("foo", Map.empty, Duration.Inf, 50, true, "FOOOOPT", RunMode.Shared, 10 second, 5)
+    val ctx = ContextConfig("foo", Map.empty, Duration.Inf, 50, true, "FOOOOPT", RunMode.Shared, 10 second, 5, ServerDefault)
     contexts.get("foo").await.get.runOptions shouldNot be (ctx.runOptions)
 
     contexts.update(ctx).await
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/FStorageSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/FStorageSpec.scala
index 41e81ae66..0f3ba316e 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/FStorageSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/FStorageSpec.scala
@@ -14,7 +14,7 @@ class FStorageSpec extends FunSpec with Matchers with BeforeAndAfter {
     value: Int
   ) extends NamedConfig
 
-  val testEntryConfigRepr = new ConfigRepr[TestEntry] {
+  val testEntryConfigRepr = new NamedConfigRepr[TestEntry] {
     import scala.collection.JavaConverters._
 
     override def fromConfig(config: Config): TestEntry = {
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
index 576dbb5ba..bc47e9d54 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
@@ -1,5 +1,7 @@
 package io.hydrosphere.mist.master.execution
 
+import java.util.concurrent.atomic.AtomicInteger
+
 import akka.actor.ActorRef
 import akka.testkit.{TestActorRef, TestProbe}
 import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest, _}
@@ -7,6 +9,7 @@ import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.execution.status.StatusReporter
 import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection, WorkerConnector}
 import io.hydrosphere.mist.master.logging.{JobLogger, JobLoggersFactory}
+import io.hydrosphere.mist.master.models.ContextConfig
 import io.hydrosphere.mist.master.{ActorSpec, FilteredException, TestData, TestUtils}
 import io.hydrosphere.mist.utils.akka.ActorF
 import mist.api.data.JsMap
@@ -35,7 +38,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = (_, _) => Future.successful(connector),
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 5 minutes
     )
@@ -62,7 +65,8 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
     probe.send(frontend, ContextFrontend.Event.Status)
     val status2 = probe.expectMsgType[ContextFrontend.FrontendStatus]
     status2.executorId.isDefined shouldBe true
-    status2.jobs should contain only("id" -> ExecStatus.Started)
+    status2.jobs.size shouldBe 1
+    status2.jobs.head shouldBe "id" -> ExecStatus.Started
 
     job.send(frontend, JobActor.Event.Completed("id"))
 
@@ -81,7 +85,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = (_, _) => Future.successful(connector),
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 5 minutes
     )
@@ -101,7 +105,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = (_, _) => Future.successful(connector),
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 1 second
     )
@@ -119,7 +123,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = (_, _) => Future.successful(connector),
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 1 second
     )
@@ -148,7 +152,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = (_, _) => Future.successful(connector),
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 5 minutes
     )
@@ -175,13 +179,19 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
   }
 
   it("should restart connector 'til max start times and then sleep") {
+    val callCounter = new AtomicInteger(0)
     val connector = crushedConnector()
+    val starter = (id: String, ctx: ContextConfig) => {
+      callCounter.incrementAndGet()
+      Future.successful(connector)
+    }
+
     val job = mkJobProbe()
     val props = ContextFrontend.props(
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = starter,
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 5 minutes
     )
@@ -192,11 +202,13 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
     probe.send(frontend, RunJobRequest(s"id", JobParams("path", "MyClass", JsMap.empty, Action.Execute)))
     probe.expectMsgType[ExecutionInfo]
 
+    job.expectMsgType[JobActor.Event.ContextBroken]
+
     probe.send(frontend, ContextFrontend.Event.Status)
     val status = probe.expectMsgType[ContextFrontend.FrontendStatus]
     status.failures shouldBe TestUtils.FooContext.maxConnFailures
+    callCounter.get() shouldBe TestUtils.FooContext.maxConnFailures
 
-    job.expectMsgType[JobActor.Event.ContextBroken]
 
     probe.send(frontend, RunJobRequest(s"last", JobParams("path", "MyClass", JsMap.empty, Action.Execute)))
     probe.expectMsgPF() {
@@ -214,7 +226,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = (_, _) => Future.successful(connector),
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 5 minutes
     )
@@ -245,7 +257,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = (_, _) => Future.successful(connector),
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 5 minutes
     )
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
index 9c338d203..1df07229c 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
@@ -29,15 +29,10 @@ class SpawnSettingsSpec extends FunSpec with Matchers with TestData {
     val ctx = FooContext.copy(workerMode = RunMode.Shared)
     val initInfo = spawnSettings.toWorkerInitInfo(ctx)
     initInfo.sparkConf.toSeq should contain allElementsOf ctx.sparkConf.toSeq
-    initInfo.maxJobs shouldBe ctx.maxJobs
     initInfo.downtime shouldBe ctx.downtime
     initInfo.streamingDuration shouldBe  ctx.streamingDuration
     initInfo.logService shouldBe spawnSettings.logAddress
     initInfo.masterHttpConf shouldBe spawnSettings.httpAddress
     initInfo.maxArtifactSize shouldBe spawnSettings.maxArtifactSize
-
-    val excl = ctx.copy(workerMode = RunMode.ExclusiveContext)
-    val initFoExclusive = spawnSettings.toWorkerInitInfo(excl)
-    initFoExclusive.maxJobs shouldBe 1
   }
 }
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala
index 007421eb0..d8660cbe9 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala
@@ -11,7 +11,6 @@ class SparkSubmitBuilderSpec extends FunSpec with Matchers {
 
   val testInfo = WorkerInitInfo(
     sparkConf = Map("spark.master" -> "spark://localhost:4433", "a.b.c" -> "xyz"),
-    maxJobs = 10,
     downtime = 1 second,
     streamingDuration = 1 second,
     logService = "localhost:2005",
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
index d390f0223..1c228d3ed 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
@@ -79,7 +79,7 @@ class HttpApiV2Spec extends FunSpec
       when(execution.getWorkerLink(any[String]))
         .thenReturn(Some(WorkerLink(
           "id", "test", None,
-          WorkerInitInfo(Map(), 20, Duration.Inf, Duration.Inf, "test", "localhost:0", "localhost:0", 262144000, ""))))
+          WorkerInitInfo(Map(), Duration.Inf, Duration.Inf, "test", "localhost:0", "localhost:0", 262144000, ""))))
 
       val route = HttpV2Routes.workerRoutes(execution)
 
@@ -87,7 +87,7 @@ class HttpApiV2Spec extends FunSpec
         status shouldBe StatusCodes.OK
         val resp = responseAs[WorkerLink]
         resp.name shouldBe "id"
-        resp.initInfo shouldBe WorkerInitInfo(Map(), 20, Duration.Inf, Duration.Inf, "test", "localhost:0","localhost:0", 262144000, "")
+        resp.initInfo shouldBe WorkerInitInfo(Map(), Duration.Inf, Duration.Inf, "test", "localhost:0","localhost:0", 262144000, "")
         resp.sparkUi should not be defined
         resp.address shouldBe "test"
       }
diff --git a/project/Library.scala b/project/Library.scala
index e001a4f6c..5f92dc556 100644
--- a/project/Library.scala
+++ b/project/Library.scala
@@ -4,6 +4,8 @@ object Library {
 
   val slf4j = "org.slf4j" % "slf4j-api" % "1.7.5"
   val slf4jLog4j = "org.slf4j" % "slf4j-log4j12" % "1.7.5"
+  val log4j = "log4j" % "log4j" % "1.2.17"
+  val log4jExtras = "log4j" % "apache-log4j-extras" % "1.2.17"
   val scopt = "com.github.scopt" %% "scopt" % "3.6.0"
   val typesafeConfig = "com.typesafe" % "config" % "1.3.1"
 

From 7db8e2f19a71277694991332dc776c1347e1a695 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Tue, 25 Sep 2018 23:40:18 +0300
Subject: [PATCH 17/30] aws emr - wip

---
 mist.sbt                                      |  21 +--
 .../io/hydrosphere/mist/aws/GenConfig.scala   |   1 +
 mist/master/src/main/resources/agent.conf     |  28 +++
 .../mist/master/MasterServer.scala            |  25 ++-
 .../io/hydrosphere/mist/master/configs.scala  |   4 +
 .../WorkerConnector.scala => Cluster.scala}   |  15 +-
 .../mist/master/execution/ClusterRunner.scala |  41 +++++
 .../master/execution/ClustersService.scala    |  53 ++++++
 .../master/execution/ContextFrontend.scala    |  14 +-
 .../master/execution/ExecutionService.scala   |  25 +--
 .../mist/master/execution/SpawnSettings.scala |   2 -
 .../execution/aws/AWSEMRClusterRunner.scala   | 169 ++++++++++++++++++
 .../master/execution/aws/ClusterAgent.scala   |  59 ++++++
 .../execution/aws/CompletableFutureOps.scala  |  35 ++++
 .../mist/master/execution/aws/EMRClient.scala | 105 +++++++++++
 .../master/execution/aws/EMRRunSettings.scala |  15 ++
 .../mist/master/execution/aws/EMRStatus.scala |  22 +++
 .../mist/master/execution/aws/EmrInfo.scala   |  16 ++
 .../mist/master/execution/aws/SSHClient.scala |  29 +++
 .../workers/ExclusiveConnector.scala          |   6 +-
 .../execution/workers/SharedConnector.scala   |   6 +-
 .../master/execution/workers/WorkerHub.scala  |   8 +-
 .../execution/workers/WorkerRunner.scala      |  12 +-
 .../master/interfaces/http/HttpV2Routes.scala |  56 +++---
 .../execution/ContextFrontendSpec.scala       |  20 +--
 ...rConnectorSpec.scala => ClusterSpec.scala} |   2 +-
 .../execution/workers/WorkerHubSpec.scala     |   3 +-
 27 files changed, 686 insertions(+), 106 deletions(-)
 create mode 100644 mist/master/src/main/resources/agent.conf
 rename mist/master/src/main/scala/io/hydrosphere/mist/master/execution/{workers/WorkerConnector.scala => Cluster.scala} (84%)
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClusterRunner.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/ClusterAgent.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/CompletableFutureOps.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EmrInfo.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
 rename mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/{WorkerConnectorSpec.scala => ClusterSpec.scala} (90%)

diff --git a/mist.sbt b/mist.sbt
index 6954fcd0c..e45c45c38 100644
--- a/mist.sbt
+++ b/mist.sbt
@@ -83,10 +83,12 @@ lazy val master = project.in(file("mist/master"))
 
       Library.Akka.testKit % "test",
       Library.Akka.http, Library.Akka.httpSprayJson, Library.Akka.httpTestKit % "test",
-      Library.cats,
+      Library.cats, Library.catsEffect,
 
       Library.dockerJava,
 
+      Library.awsSdkEC2, Library.awsSdkEMR, Library.scalaSsh,
+
       "io.hydrosphere" %% "shadedshapeless" % "2.3.0",
       Library.commonsCodec, Library.scalajHttp,
       Library.jsr305 % "provided",
@@ -99,23 +101,6 @@ lazy val master = project.in(file("mist/master"))
     buildInfoPackage := "io.hydrosphere.mist"
   )
 
-lazy val clusterAgent = project.in(file("mist/cluster-agent"))
-  .dependsOn(common % "compile->compile;test->test")
-  .settings(commonSettings: _*)
-  .settings(commonAssemblySettings: _*)
-  .settings(
-    name := "mist-cluster-agent",
-    scalacOptions ++= commonScalacOptions,
-    libraryDependencies ++= Library.Akka.base,
-    libraryDependencies ++= Seq(
-      Library.slf4jLog4j, Library.typesafeConfig, Library.scopt,
-      Library.cats,
-      Library.jsr305 % "provided",
-      Library.scalaTest % "test",
-      Library.Akka.testKit % "test"
-    )
-  )
-
 lazy val worker = project.in(file("mist/worker"))
   .dependsOn(common % "compile->compile;test->test")
   .settings(commonSettings: _*)
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
index 77b85b340..3749f15cc 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
@@ -29,6 +29,7 @@ object ConfigPatcher {
       "type" -> "aws_emr",
       "sshKeyPair" -> sshKeyPair,
       "sshKeyPath" -> sshKeyPath,
+      "sshUser" -> "hadoop",
       "accessKey"-> accessKey,
       "secretKey" -> secretKey,
       "subnetId" -> subnetId,
diff --git a/mist/master/src/main/resources/agent.conf b/mist/master/src/main/resources/agent.conf
new file mode 100644
index 000000000..687ec9a10
--- /dev/null
+++ b/mist/master/src/main/resources/agent.conf
@@ -0,0 +1,28 @@
+akka {
+
+  loggers = ["akka.event.slf4j.Slf4jLogger"]
+  logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
+
+  loglevel = "INFO"
+  actor {
+    provider = "akka.remote.RemoteActorRefProvider"
+    warn-about-java-serializer-usage = false
+  }
+
+  remote {
+    netty.tcp {
+      bind-hostname = "0.0.0.0"
+      port = 0
+      maximum-frame-size = 5242880b
+    }
+    transport-failure-detector {
+      heartbeat-interval = 30s
+      acceptable-heartbeat-pause = 5s
+    }
+    log-remote-lifecycle-events = off
+  }
+
+  http.server.transparent-head-requests = false
+  http.server.idle-timeout = infinite
+
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
index 47897a256..462155a4b 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
@@ -14,8 +14,9 @@ import io.hydrosphere.mist.common.CommonData
 import io.hydrosphere.mist.master.Messages.StatusMessages.SystemEvent
 import io.hydrosphere.mist.master.artifact.ArtifactRepository
 import io.hydrosphere.mist.master.data.{ContextsStorage, FunctionConfigStorage}
+import io.hydrosphere.mist.master.execution.workers.WorkerRunner
 import io.hydrosphere.mist.master.execution.workers.starter.WorkerStarter
-import io.hydrosphere.mist.master.execution.{ExecutionService, SpawnSettings}
+import io.hydrosphere.mist.master.execution.{ClusterRunner, ClustersService, ExecutionService, SpawnSettings}
 import io.hydrosphere.mist.master.interfaces.async._
 import io.hydrosphere.mist.master.interfaces.http._
 import io.hydrosphere.mist.master.jobs.{FunctionInfoProviderRunner, FunctionsService}
@@ -23,7 +24,7 @@ import io.hydrosphere.mist.master.logging.{LogService, LogStreams}
 import io.hydrosphere.mist.master.security.KInitLauncher
 import io.hydrosphere.mist.master.store.H2JobsRepository
 import io.hydrosphere.mist.utils.Logger
-import io.hydrosphere.mist.utils.akka.RestartSupervisor
+import io.hydrosphere.mist.utils.akka.{ActorRegHub, RestartSupervisor}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.{Future, Promise}
@@ -99,11 +100,11 @@ object MasterServer extends Logger {
       LogStreams.runService(host, port, logsPaths, streamer)
     }
 
-    def runExecutionService(logService: LogService): ExecutionService = {
+    def runClustersService(): ClustersService = {
       val logsDir = Paths.get(config.logs.dumpDirectory)
-      val workerRunner = WorkerStarter.create(config.workers, logsDir)
+
+      val regHub = ActorRegHub("regHub", system)
       val spawnSettings = SpawnSettings(
-        runnerCmd = workerRunner,
         timeout = config.workers.runnerInitTimeout,
         readyTimeout = config.workers.readyTimeout,
         akkaAddress = s"${config.cluster.publicHost}:${config.cluster.port}",
@@ -111,7 +112,16 @@ object MasterServer extends Logger {
         httpAddress = s"${config.http.publicHost  }:${config.http.port}",
         maxArtifactSize = config.workers.maxArtifactSize
       )
-      ExecutionService(spawnSettings, system, streamer, store, logService)
+      val defaultRunner = {
+        val starter = WorkerStarter.create(config.workers, logsDir)
+        ClusterRunner.legacy(spawnSettings, regHub, starter, system)
+      }
+
+      ClustersService.create(config.mistHome, spawnSettings, config.launchersSettings, defaultRunner, system)
+    }
+
+    def runExecutionService(clustersService: ClustersService, logService: LogService): ExecutionService = {
+      ExecutionService(clustersService, system, streamer, store, logService)
     }
 
     val artifactRepository = ArtifactRepository.create(
@@ -146,7 +156,8 @@ object MasterServer extends Logger {
                                       contextsStorage,
                                       artifactRepository
                                 )(system.dispatcher)
-      executionService       =  runExecutionService(logService)
+      clustersService        =  runClustersService()
+      executionService       =  runExecutionService(clustersService, logService)
       masterService          <- start("Main service", MainService.start(
                                                         executionService,
                                                         contextsStorage,
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
index 26ad6316c..d640811e8 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
@@ -261,6 +261,7 @@ sealed trait LauncherSettings
 case class AWSEMRLaunchSettings(
   sshKeyPair: String,
   sshKeyPath: String,
+  sshUser: String,
   accessKey: String,
   secretKey: String,
   subnetId: String,
@@ -276,6 +277,7 @@ object AWSEMRLaunchSettings {
     AWSEMRLaunchSettings(
       sshKeyPair = c.getString("sshKeyPair"),
       sshKeyPath = c.getString("sshKeyPath"),
+      sshUser = c.getString("sshUser"),
       accessKey = c.getString("accessKey"),
       secretKey = c.getString("secretKey"),
       subnetId = c.getString("subnetId"),
@@ -321,6 +323,7 @@ case class MasterConfig(
   jobsSavePath: String,
   artifactRepositoryPath: String,
   launchersSettings: Map[String, LauncherSettings],
+  mistHome: String,
   raw: Config
 )
 
@@ -364,6 +367,7 @@ object MasterConfig extends Logger {
       jobInfoProviderConfig = FunctionInfoProviderConfig(mist.getConfig("job-extractor")),
       srcConfigPath = filePath,
       launchersSettings = LauncherSettings.extractAll(mist.getConfigList("launchers-settings")),
+      mistHome = config.getString("work-directory"),
       raw = config
     )
   }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala
similarity index 84%
rename from mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnector.scala
rename to mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala
index 21f85d047..a4b8aa1a5 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala
@@ -1,12 +1,13 @@
-package io.hydrosphere.mist.master.execution.workers
+package io.hydrosphere.mist.master.execution
 
 import akka.actor.{ActorRef, ActorRefFactory}
+import io.hydrosphere.mist.master.execution.workers._
 import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
 import io.hydrosphere.mist.utils.akka.WhenTerminated
 
 import scala.concurrent.{Future, Promise}
 
-trait WorkerConnector {
+trait Cluster {
 
   def askConnection(): Future[PerJobConnection]
 
@@ -18,7 +19,7 @@ trait WorkerConnector {
 
 }
 
-object WorkerConnector {
+object Cluster {
 
   sealed trait Event
   object Event {
@@ -33,11 +34,11 @@ object WorkerConnector {
   class ActorBasedWorkerConnector(
     underlying: ActorRef,
     termination: Future[Unit]
-  ) extends WorkerConnector {
+  ) extends Cluster {
 
     override def askConnection(): Future[PerJobConnection] = {
       val promise = Promise[PerJobConnection]
-      underlying ! WorkerConnector.Event.AskConnection(promise)
+      underlying ! Cluster.Event.AskConnection(promise)
       promise.future
     }
 
@@ -48,7 +49,7 @@ object WorkerConnector {
 
     override def whenTerminated(): Future[Unit] = termination
 
-    override def warmUp(): Unit = underlying ! WorkerConnector.Event.WarmUp
+    override def warmUp(): Unit = underlying ! Cluster.Event.WarmUp
 
 
   }
@@ -58,7 +59,7 @@ object WorkerConnector {
     ctx: ContextConfig,
     runner: WorkerRunner,
     af: ActorRefFactory
-  ): WorkerConnector = {
+  ): Cluster = {
     val props = ctx.workerMode match {
       case RunMode.Shared => SharedConnector.props(id, ctx, runner)
       case RunMode.ExclusiveContext => ExclusiveConnector.props(id, ctx, runner)
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClusterRunner.scala
new file mode 100644
index 000000000..e4610e266
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClusterRunner.scala
@@ -0,0 +1,41 @@
+package io.hydrosphere.mist.master.execution
+
+import akka.actor.ActorSystem
+import io.hydrosphere.mist.master.execution.workers.starter.WorkerStarter
+import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection, WorkerRunner}
+import io.hydrosphere.mist.master.models.ContextConfig
+import io.hydrosphere.mist.utils.akka.ActorRegHub
+
+import scala.concurrent.Future
+
+trait ClusterRunner {
+
+  def run(id: String, ctx: ContextConfig): Future[Cluster]
+
+}
+
+object ClusterRunner {
+
+  def legacy(
+    spawn: SpawnSettings,
+    regHub: ActorRegHub,
+    workerStarter: WorkerStarter,
+    system: ActorSystem
+  ): ClusterRunner = {
+    new ClusterRunner {
+      val runner = WorkerRunner.default(spawn, workerStarter, regHub, system)
+      override def run(id: String, ctx: ContextConfig): Future[Cluster] = {
+        Future.successful(Cluster.actorBased(id, ctx, runner, system))
+      }
+    }
+  }
+
+  def awsEmr(
+    spawn: SpawnSettings,
+    regHub: ActorRegHub,
+    system: ActorSystem
+  ): ClusterRunner = {
+    ???
+  }
+}
+
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
new file mode 100644
index 000000000..ce69ece05
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
@@ -0,0 +1,53 @@
+package io.hydrosphere.mist.master.execution
+
+import java.nio.file.Paths
+
+import akka.actor.ActorSystem
+import io.hydrosphere.mist.master.execution.aws.AWSEMRClusterRunner
+import io.hydrosphere.mist.master.{AWSEMRLaunchSettings, LauncherSettings}
+import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig, ServerDefault}
+import io.hydrosphere.mist.utils.akka.ActorRegHub
+
+import scala.concurrent.Future
+
+trait ClustersService {
+
+  def start(id: String, ctx: ContextConfig): Future[Cluster]
+
+}
+
+object ClustersService {
+
+  def create(
+    mistHome: String,
+    spawn: SpawnSettings,
+    launchSettings: Map[String, LauncherSettings],
+    serverDefault: ClusterRunner,
+    system: ActorSystem
+  ): ClustersService = {
+
+    val regHub = ActorRegHub("regHub", system)
+    val runners = launchSettings.map({case (name, settings) => {
+      val runner = settings match {
+        case aws: AWSEMRLaunchSettings =>
+          AWSEMRClusterRunner.create(Paths.get(mistHome), spawn, aws, regHub, system)
+      }
+      name -> runner
+    }})
+
+    new ClustersService {
+
+      override def start(id: String, ctx: ContextConfig): Future[Cluster] = {
+        ctx.launchData match {
+          case ServerDefault => serverDefault.run(id, ctx)
+          case aws: AWSEMRLaunchData =>
+            runners.get(aws.launcherSettingsName) match {
+              case Some(runner) => runner.run(id, ctx)
+              case None => Future.failed(new RuntimeException(s"Unknown settings name ${aws.launcherSettingsName} for ctx ${ctx.name}"))
+            }
+        }
+      }
+    }
+  }
+
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index 4db7ffa52..b96ebee3d 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -8,7 +8,7 @@ import io.hydrosphere.mist.master.Messages.StatusMessages.FailedEvent
 import io.hydrosphere.mist.master.execution.ContextFrontend.Event.JobDied
 import io.hydrosphere.mist.master.execution.ContextFrontend.{ConnectorState, FrontendStatus}
 import io.hydrosphere.mist.master.execution.status.StatusReporter
-import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnector}
+import io.hydrosphere.mist.master.execution.workers.PerJobConnection
 import io.hydrosphere.mist.master.logging.{JobLogger, JobLoggersFactory, LogService}
 import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
 import io.hydrosphere.mist.utils.akka.{ActorF, ActorFSyntax}
@@ -40,7 +40,7 @@ class ContextFrontend(
   name: String,
   reporter: StatusReporter,
   loggersFactory: JobLoggersFactory,
-  connectorStarter: (String, ContextConfig) => Future[WorkerConnector],
+  connectorStarter: (String, ContextConfig) => Future[Cluster],
   jobFactory: ActorF[(ActorRef, RunJobRequest, Promise[JsData], StatusReporter, JobLogger)],
   defaultInactiveTimeout: FiniteDuration
 ) extends Actor
@@ -389,7 +389,7 @@ object ContextFrontend {
 
   sealed trait Event
   object Event {
-    final case class ConnectorStarted(id: String, connector: WorkerConnector) extends Event
+    final case class ConnectorStarted(id: String, connector: Cluster) extends Event
     final case class ConnectorStartFailed(id: String, e: Throwable) extends Event
     final case class ConnectorCrushed(id: String, err: Throwable) extends Event
     final case class ConnectorStopped(id: String) extends Event
@@ -415,7 +415,7 @@ object ContextFrontend {
 
   case class ConnectorState(
     id: String,
-    connector: WorkerConnector,
+    connector: Cluster,
     used: Int,
     asked: Int,
     failedTimes: Int
@@ -429,7 +429,7 @@ object ContextFrontend {
   }
 
   object ConnectorState {
-    def initial(id: String, connector: WorkerConnector): ConnectorState =
+    def initial(id: String, connector: Cluster): ConnectorState =
       ConnectorState(id, connector, 0, 0, 0)
   }
 
@@ -437,7 +437,7 @@ object ContextFrontend {
     name: String,
     status: StatusReporter,
     loggersFactory: JobLoggersFactory,
-    connectorStarter: (String, ContextConfig) => Future[WorkerConnector],
+    connectorStarter: (String, ContextConfig) => Future[Cluster],
     jobFactory: ActorF[(ActorRef, RunJobRequest, Promise[JsData], StatusReporter, JobLogger)],
     defaultInactiveTimeout: FiniteDuration
   ): Props = Props(classOf[ContextFrontend], name, status, loggersFactory, connectorStarter, jobFactory, defaultInactiveTimeout)
@@ -447,6 +447,6 @@ object ContextFrontend {
     name: String,
     status: StatusReporter,
     loggersFactory: JobLoggersFactory,
-    connectorStarter: (String, ContextConfig) => Future[WorkerConnector]
+    connectorStarter: (String, ContextConfig) => Future[Cluster]
   ): Props = props(name, status, loggersFactory, connectorStarter, ActorF.props(JobActor.props _), 5 minutes)
 }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
index bd71ebe70..b8af86f25 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
@@ -22,7 +22,7 @@ import scala.concurrent.Future
   */
 class ExecutionService(
   contextsMaster: ActorRef,
-  workersHub: WorkerHub,
+  //workersHub: WorkerHub,
   statusReporter: StatusReporter,
   repo: JobRepository
 ) {
@@ -42,16 +42,16 @@ class ExecutionService(
 
   def getHistory(req: JobDetailsRequest): Future[JobDetailsResponse] = repo.getJobs(req)
 
-  def workers(): Seq[WorkerLink] = workersHub.workerConnections().map(_.data)
-
-  def getWorkerLink(workerId: String): Option[WorkerLink] = {
-    workersHub.workerConnection(workerId).map(_.data)
-  }
+//  def workers(): Seq[WorkerLink] = workersHub.workerConnections().map(_.data)
 
+//  def getWorkerLink(workerId: String): Option[WorkerLink] = {
+//    workersHub.workerConnection(workerId).map(_.data)
+//  }
 
-  def stopAllWorkers(): Future[Unit] = workersHub.shutdownAllWorkers()
 
-  def stopWorker(id: String): Future[Unit] = workersHub.shutdownWorker(id)
+//  def stopAllWorkers(): Future[Unit] = workersHub.shutdownAllWorkers()
+//
+//  def stopWorker(id: String): Future[Unit] = workersHub.shutdownWorker(id)
 
 
   def startJob(req: JobStartRequest): Future[ExecutionInfo] = {
@@ -110,24 +110,25 @@ class ExecutionService(
 object ExecutionService {
 
   def apply(
-    spawn: SpawnSettings,
+//    spawn: SpawnSettings,
+    clustersService: ClustersService,
     system: ActorSystem,
     streamer: EventsStreamer,
     repo: JobRepository,
     logService: LogService
   ): ExecutionService = {
     // TODO
-    val hub = WorkerHub(spawn, system)
+//    val hub = WorkerHub(spawn, system)
 
     val reporter = StatusReporter.reporter(repo, streamer, logService)(system)
     val mkContext = ActorF[ContextConfig]((ctx, af) => {
-      val props = ContextFrontend.props(ctx.name, reporter, logService, (id, ctx) => Future.successful(hub.start(id, ctx)))
+      val props = ContextFrontend.props(ctx.name, reporter, logService, clustersService.start)
       val ref = af.actorOf(props)
       ref ! ContextEvent.UpdateContext(ctx)
       ref
     })
     val contextsMaster = system.actorOf(ContextsMaster.props(mkContext))
-    new ExecutionService(contextsMaster, hub, reporter, repo)
+    new ExecutionService(contextsMaster, reporter, repo)
   }
 
 }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
index f84c4dddf..8e75ff7d1 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala
@@ -1,13 +1,11 @@
 package io.hydrosphere.mist.master.execution
 
 import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
-import io.hydrosphere.mist.master.execution.workers.starter.WorkerStarter
 import io.hydrosphere.mist.master.models.ContextConfig
 
 import scala.concurrent.duration._
 
 case class SpawnSettings(
-  runnerCmd: WorkerStarter,
   timeout: Duration,
   readyTimeout: FiniteDuration,
   akkaAddress: String,
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
new file mode 100644
index 000000000..530929bd0
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
@@ -0,0 +1,169 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import java.nio.file.Path
+import java.util.concurrent.Executors
+
+import akka.actor.ActorSystem
+import cats.effect._
+import cats.implicits._
+import io.hydrosphere.mist.common.CommonData
+import io.hydrosphere.mist.master.AWSEMRLaunchSettings
+import io.hydrosphere.mist.master.execution.workers.{StopAction, WorkerConnection, WorkerRunner}
+import io.hydrosphere.mist.master.execution.workers.starter.{SparkSubmitBuilder, WorkerProcess, WorkerStarter}
+import io.hydrosphere.mist.master.execution.{Cluster, ClusterRunner, SpawnSettings}
+import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig}
+import io.hydrosphere.mist.utils.akka.ActorRegHub
+
+import scala.concurrent.duration._
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.{ExecutionContext, Future}
+
+object AWSEMRClusterRunner {
+
+  def mkInstallCommands(
+    agentJarPath: String,
+    workerJarPath: String,
+    agentId: String,
+    masterAddr: String,
+    accessKey: String,
+    secretKey: String,
+    region: String,
+    awsId: String
+  ): Seq[SSHCmd] = {
+    Seq(
+      SSHCmd.Exec(Seq("mkdir", "~/mist-agent")),
+      SSHCmd.CopyFile(agentJarPath, "~/mist-agent/mist-agent.jar"),
+      SSHCmd.CopyFile(workerJarPath, "~/mist-agent/mist-worker.jar"),
+      SSHCmd.Exec(Seq(
+        "java", "-cp", "~/mist-agent/mist-master.jar", "io.hydrosphere.mist.master.execution.aws.ClusterAgent",
+        masterAddr, agentId, accessKey, secretKey, region, awsId,
+        "1>~/mist-agent/out.log", "2>~/mist-agent/out.log", "&"
+      ))
+    )
+  }
+
+  class EMRClusterRunner(
+    spawn: SpawnSettings,
+    launchSettings: AWSEMRLaunchSettings,
+    regHub: ActorRegHub,
+    system: ActorSystem,
+    installAgent: (String, String, String) => Unit,
+    client: EMRClient[IO]
+  ) extends ClusterRunner {
+
+    implicit val IOTimer = new Timer[IO] {
+
+      val ec = ExecutionContext.global
+      val sc = Executors.newScheduledThreadPool(1)
+
+      override val clock: Clock[IO] = new Clock[IO] {
+          override def realTime(unit: TimeUnit): IO[Long] =
+            IO(unit.convert(System.currentTimeMillis(), MILLISECONDS))
+
+          override def monotonic(unit: TimeUnit): IO[Long] =
+            IO(unit.convert(System.nanoTime(), NANOSECONDS))
+        }
+
+      override def sleep(timespan: FiniteDuration): IO[Unit] =
+        IO.cancelable { cb =>
+          val tick = new Runnable {
+            def run() = ec.execute(new Runnable {
+              def run() = cb(Right(()))
+            })
+          }
+          val f = sc.schedule(tick, timespan.length, timespan.unit)
+          IO(f.cancel(false))
+        }
+    }
+
+    private def extractData(ctx: ContextConfig): IO[AWSEMRLaunchData] = {
+      ctx.launchData match {
+        case data: AWSEMRLaunchData => IO.pure(data)
+        case other =>
+          val err = new IllegalArgumentException(s"Invalid launch data for AWSCluster ${other.getClass.getSimpleName}")
+          IO.raiseError(err)
+      }
+    }
+
+    private def mkRunSettings(name: String, data: AWSEMRLaunchData, settings: AWSEMRLaunchSettings): EMRRunSettings = {
+      EMRRunSettings(
+        name = name,
+        keyPair = settings.sshKeyPair,
+        releaseLabel = data.releaseLabel,
+        masterInstanceType = data.masterInstanceType,
+        slaveInstanceType = data.slaveInstanceType,
+        instanceCount = data.instanceCount,
+        subnetId = settings.subnetId,
+        additionalGroup = settings.additionalGroup,
+        emrRole = settings.emrRole,
+        emrEc2Role = settings.emrEc2Role
+      )
+    }
+
+    private def startFully(runSettings: EMRRunSettings, client: EMRClient[IO]): IO[EmrInfo] = {
+      for {
+        initial <- client.start(runSettings)
+        await <- client.awaitStatus(initial.id, EMRStatus.Started, 10 seconds, 40)
+      } yield await
+    }
+
+    private def mkRunner(launch: AWSEMRLaunchSettings, host: String): WorkerRunner = {
+      val starter = new WorkerStarter {
+        val builder = new SparkSubmitBuilder("~/mist-agent", "/usr/lib/spark")
+        override def onStart(name: String, initInfo: CommonData.WorkerInitInfo): WorkerProcess = {
+          val submitCmd = builder.submitWorker(name, initInfo) :+ "&"
+          Future {
+            new SSHClient(host, launch.sshUser, launch.sshKeyPath).install(Seq(SSHCmd.Exec(submitCmd)))
+          }
+          WorkerProcess.NonLocal
+        }
+
+        override def stopAction: StopAction = StopAction.Remote
+      }
+      val hackSettings = spawn.copy(runnerCmd = starter)
+      WorkerRunner.default(hackSettings, regHub, system)
+    }
+
+    override def run(id: String, ctx: ContextConfig): Future[Cluster] = {
+      val io = for {
+        data        <- extractData(ctx)
+        runSettings =  mkRunSettings(id, data, launchSettings)
+        emrInfo     <- startFully(runSettings, client)
+        agentId     = s"agent-${emrInfo.id}"
+        _           <- IO[Unit](installAgent(emrInfo.masterPublicDnsName, agentId, emrInfo.id))
+        _           <- IO.fromFuture(IO(regHub.waitRef(agentId, 1 minute)))
+        runner      = mkRunner(launchSettings, emrInfo.masterPublicDnsName)
+        cluster     = Cluster.actorBased(id, ctx, runner, system)
+      } yield cluster
+
+      io.unsafeToFuture()
+    }
+  }
+
+  def create(
+    jarsDir: Path,
+    spawn: SpawnSettings,
+    launchSettings: AWSEMRLaunchSettings,
+    regHub: ActorRegHub,
+    system: ActorSystem
+  ): ClusterRunner = {
+    import launchSettings._
+    val client = EMRClient.create(accessKey, secretKey, region)
+
+    val installAgent = (host: String, agentId: String, awsId: String) => {
+      val cmds = mkInstallCommands(
+        jarsDir.resolve("mist-master.jar").toString, jarsDir.resolve("mist-worker.jar").toString,
+        agentId: String, spawn.akkaAddress, accessKey, secretKey, region, awsId)
+      new SSHClient(host, sshUser, sshKeyPath).install(cmds)
+    }
+
+    new EMRClusterRunner(
+      spawn,
+      launchSettings,
+      regHub,
+      system,
+      installAgent,
+      client
+    )
+  }
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/ClusterAgent.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/ClusterAgent.scala
new file mode 100644
index 000000000..7db859aab
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/ClusterAgent.scala
@@ -0,0 +1,59 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import akka.actor.{ActorRef, ActorSystem}
+import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
+import io.hydrosphere.mist.common.CommonData
+import io.hydrosphere.mist.utils.NetUtils
+import io.hydrosphere.mist.utils.akka.{ActorRegHub, WhenTerminated}
+
+import scala.concurrent.Await
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+
+object ClusterAgent {
+
+  def main(args: Array[String]): Unit = {
+    val masterAddr = args(0)
+    val id = args(1)
+    val accessKey = args(2)
+    val secretKey = args(3)
+    val region = args(4)
+    val awsId = args(5)
+
+    val emrClient = EMRClient.create(accessKey, secretKey, region)
+
+    val hostname = NetUtils.findLocalInetAddress()
+
+    val config = ConfigFactory.load("agent")
+      .withValue("akka.remote.netty.tcp.hostname", ConfigValueFactory.fromAnyRef(hostname))
+
+    implicit val system = ActorSystem("mist-info-provider", config)
+
+    def resolveRemote(path: String): ActorRef = {
+      val ref = system.actorSelection(path).resolveOne(10 seconds)
+      try {
+        Await.result(ref, Duration.Inf)
+      } catch {
+        case e: Throwable =>
+          println(s"Couldn't resolve remote path $path")
+          e.printStackTrace()
+          sys.exit(-1)
+      }
+    }
+
+    def remotePath(addr: String, name: String): String = {
+      s"akka.tcp://mist@$addr/user/$name"
+    }
+
+    val regHub = resolveRemote(remotePath(masterAddr, "regHub"))
+    val heathRef = resolveRemote(remotePath(masterAddr, CommonData.HealthActorName))
+    regHub ! ActorRegHub.Register(id)
+
+    WhenTerminated(heathRef, {
+      println("Remote system was terminated, shutdown cluster")
+      emrClient.stop(awsId).unsafeRunSync()
+      system.terminate()
+    })
+
+  }
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/CompletableFutureOps.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/CompletableFutureOps.scala
new file mode 100644
index 000000000..1419655fc
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/CompletableFutureOps.scala
@@ -0,0 +1,35 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import java.util.concurrent.CompletableFuture
+import java.util.function.{BiConsumer, BiFunction}
+
+import cats.effect.IO
+
+import scala.concurrent.{Future, Promise}
+
+final class CompletableFutureOps[A](val cf: CompletableFuture[A]) extends AnyVal {
+
+  def toFuture: Future[A] = {
+    val p = Promise[A]
+    cf.whenComplete(new BiConsumer[A, Throwable] {
+      override def accept(res: A, err: Throwable): Unit = {
+        (Option(res), Option(err)) match {
+          case (Some(r), None) => p.success(r)
+          case (_, Some(e)) =>
+            e match {
+              case ce: java.util.concurrent.CompletionException => p.failure(ce.getCause)
+              case _ => p.failure(e)
+            }
+          case (_, _) => p.failure(new IllegalStateException("CompletableFuture was failed without error information"))
+        }
+      }})
+    p.future
+  }
+
+  def toIO: IO[A] = IO.fromFuture(IO(toFuture))
+
+}
+
+object JFutureSyntax {
+  implicit def cfSyntax[A](cf: CompletableFuture[A]): CompletableFutureOps[A] = new CompletableFutureOps(cf)
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
new file mode 100644
index 000000000..c4f9c7b5c
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
@@ -0,0 +1,105 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import cats._
+import cats.implicits._
+import cats.effect._
+import io.hydrosphere.mist.master.execution.aws.JFutureSyntax._
+import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider}
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.emr.EMRAsyncClient
+import software.amazon.awssdk.services.emr.model.{Unit => _, _}
+
+import scala.concurrent.duration.FiniteDuration
+
+trait EMRClient[F[_]] {
+
+  def start(settings: EMRRunSettings): F[EmrInfo]
+  def status(id: String): F[Option[EmrInfo]]
+  def stop(id: String): F[Unit]
+
+  def awaitStatus(
+    id: String,
+    target: EMRStatus,
+    sleepTime: FiniteDuration,
+    triesLeft: Int
+  )(implicit M: MonadError[F, Throwable], T: Timer[F]): F[EmrInfo] = {
+
+    for {
+      maybeInfo <- status(id)
+      next <- maybeInfo match {
+        case Some(info) if info.status == target =>
+          M.pure(info)
+        case Some(info) if triesLeft > 0 =>
+          T.sleep(sleepTime).flatMap(_ => awaitStatus(id, target, sleepTime, triesLeft - 1))
+        case None =>
+          M.raiseError(new IllegalArgumentException(s"Cluster $id doesn't exists"))
+      }
+    } yield next
+  }
+}
+
+object EMRClient {
+
+  class Default(orig: EMRAsyncClient) extends EMRClient[IO] {
+
+    override def start(settings: EMRRunSettings): IO[EmrInfo] = {
+      import settings._
+
+      val sparkApp = Application.builder().name("Spark").build()
+      val request = RunJobFlowRequest.builder()
+        .name(s"mist-$name")
+        .releaseLabel(releaseLabel)
+        .applications(sparkApp)
+        .jobFlowRole(emrEc2Role)
+        .serviceRole(emrRole)
+        .instances(JobFlowInstancesConfig.builder()
+          .keepJobFlowAliveWhenNoSteps(true)
+          .ec2KeyName(keyPair)
+          .instanceCount(instanceCount)
+          .masterInstanceType(masterInstanceType)
+          .slaveInstanceType(slaveInstanceType)
+          .ec2SubnetId(subnetId)
+          .additionalMasterSecurityGroups(additionalGroup)
+          .additionalSlaveSecurityGroups(additionalGroup)
+          .build()
+        ).build()
+
+      for {
+        resp <- orig.runJobFlow(request).toIO
+        maybeSt <- status(resp.jobFlowId())
+        out <- maybeSt match {
+          case Some(info) => IO.pure(info)
+          case None => IO.raiseError(new RuntimeException(s"Couldn't get infromation about cluster ${resp.jobFlowId()}"))
+        }
+      } yield out
+    }
+
+    override def status(id: String): IO[Option[EmrInfo]] = {
+      val req = DescribeClusterRequest.builder().clusterId(id).build()
+      orig.describeCluster(req).toIO
+        .map(resp => Option(EmrInfo.fromCluster(resp.cluster())))
+        .handleErrorWith({
+          case _: EMRException => IO.pure(None)
+          case e => IO.raiseError(e)
+        })
+    }
+
+    override def stop(id: String): IO[Unit] = {
+      val req = TerminateJobFlowsRequest.builder().jobFlowIds(id).build()
+      orig.terminateJobFlows(req).toIO.map(_ => ())
+    }
+  }
+
+  def create(accessKey: String, secretKey: String, region: String): EMRClient[IO] = {
+    val credentials = AwsCredentials.create(accessKey, secretKey)
+    val provider = StaticCredentialsProvider.create(credentials)
+    val reg = Region.of(region)
+    val emrClient = EMRAsyncClient.builder()
+      .credentialsProvider(provider)
+      .region(reg)
+      .build()
+
+    new Default(emrClient)
+  }
+}
+
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
new file mode 100644
index 000000000..521f35aa5
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
@@ -0,0 +1,15 @@
+package io.hydrosphere.mist.master.execution.aws
+
+case class EMRRunSettings(
+  name: String,
+  keyPair: String,
+  releaseLabel: String,
+  masterInstanceType: String,
+  slaveInstanceType: String,
+  instanceCount: Int,
+  subnetId: String,
+  additionalGroup: String,
+  emrRole: String,
+  emrEc2Role: String
+)
+
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala
new file mode 100644
index 000000000..3dc1d176b
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala
@@ -0,0 +1,22 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import software.amazon.awssdk.services.emr.{model => emodel }
+sealed trait EMRStatus
+
+object EMRStatus {
+
+  case object Starting extends EMRStatus
+  case object Started extends EMRStatus
+  case object Terminated extends EMRStatus
+  case object Terminating extends EMRStatus
+
+  def fromCluster(cluster: emodel.Cluster): EMRStatus =  cluster.status().state() match {
+    case emodel.ClusterState.STARTING | emodel.ClusterState.BOOTSTRAPPING => Starting
+    case emodel.ClusterState.WAITING | emodel.ClusterState.RUNNING => Started
+    case emodel.ClusterState.TERMINATED | emodel.ClusterState.TERMINATED_WITH_ERRORS => Terminated
+    case emodel.ClusterState.TERMINATING => Terminating
+    case emodel.ClusterState.UNKNOWN_TO_SDK_VERSION => throw new RuntimeException("Used amazon sdk version is incompatible with aws")
+  }
+
+}
+
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EmrInfo.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EmrInfo.scala
new file mode 100644
index 000000000..6b1cf6301
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EmrInfo.scala
@@ -0,0 +1,16 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import software.amazon.awssdk.services.emr.{model => emodel }
+
+case class EmrInfo(
+  id: String,
+  masterPublicDnsName: String,
+  status: EMRStatus
+)
+
+object EmrInfo {
+
+  def fromCluster(cluster: emodel.Cluster): EmrInfo =
+    EmrInfo(cluster.id(), cluster.masterPublicDnsName(), EMRStatus.fromCluster(cluster))
+
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
new file mode 100644
index 000000000..c2d2aae8e
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
@@ -0,0 +1,29 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import com.decodified.scalassh._
+
+sealed trait SSHCmd
+object SSHCmd {
+  case class CopyFile(from: String, to: String) extends SSHCmd
+  case class Exec(cmd: Seq[String]) extends SSHCmd
+}
+
+class SSHClient(host: String, user: String, keyPath: String) {
+
+  val cfgProvider = HostConfig(
+    login = PublicKeyLogin(user, None, List(keyPath)),
+    hostName = host,
+    port = 22,
+    hostKeyVerifier = HostKeyVerifiers.DontVerify
+  )
+
+  def install(cmds: Seq[SSHCmd]): Unit = {
+    SSH(host, cfgProvider) { client =>
+      cmds.foreach {
+        case SSHCmd.CopyFile(from, to) => client.upload(from, to)
+        case SSHCmd.Exec(cmd) => client.exec(Command(cmd.mkString(" ")))
+      }
+    }
+  }
+}
+
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
index a2b110873..0ba3e69d4 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
@@ -4,8 +4,8 @@ import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import akka.pattern.pipe
 import io.hydrosphere.mist.common.CommonData
 import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest}
-import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released
 import io.hydrosphere.mist.master.models.ContextConfig
+import io.hydrosphere.mist.master.execution.Cluster
 
 import scala.collection.immutable.Queue
 import scala.concurrent.{Future, Promise}
@@ -17,7 +17,7 @@ class ExclusiveConnector(
   startConnection: (String, ContextConfig) => Future[WorkerConnection]
 ) extends Actor with ActorLogging {
 
-  import WorkerConnector._
+  import Cluster._
   import context.dispatcher
 
   type Conns = Map[String, WorkerConnection]
@@ -105,7 +105,7 @@ object ExclusiveConnector {
       ref.tell(WorkerBridge.Event.CompleteAndShutdown, ActorRef.noSender)
     }
     def cancel(id: String, respond: ActorRef): Unit = ref.tell(CancelJobRequest(id), respond)
-    def release(): Unit = connector ! Released(direct)
+    def release(): Unit = connector ! Cluster.Event.Released(direct)
   }
 
   def wrappedConnection(connector: ActorRef, conn: WorkerConnection): PerJobConnection =
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
index 29f4d79dc..47b8988cc 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
@@ -6,9 +6,8 @@ import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import akka.pattern.pipe
 import io.hydrosphere.mist.common.CommonData
 import io.hydrosphere.mist.common.CommonData.CancelJobRequest
-import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event
-import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released
 import io.hydrosphere.mist.master.models.ContextConfig
+import io.hydrosphere.mist.master.execution.Cluster
 
 import scala.collection.immutable.Queue
 import scala.concurrent.{Future, Promise}
@@ -20,6 +19,7 @@ class SharedConnector(
   idGen: AtomicInteger = new AtomicInteger(1)
 ) extends Actor with ActorLogging {
 
+  import Cluster._
   import context.dispatcher
 
   private def startConnection(): Future[WorkerConnection] = {
@@ -176,7 +176,7 @@ object SharedConnector {
     import direct.ref
     def run(req: CommonData.RunJobRequest, respond: ActorRef): Unit = ref.tell(req, respond)
     def cancel(id: String, respond: ActorRef): Unit = ref.tell(CancelJobRequest(id), respond)
-    def release(): Unit = connector ! Released(direct)
+    def release(): Unit = connector ! Cluster.Event.Released(direct)
   }
 
   def wrappedConnection(connector: ActorRef, workerConn: WorkerConnection) =
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala
index 6c57413c1..24d9a7d73 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.ActorSystem
-import io.hydrosphere.mist.master.execution.SpawnSettings
+import io.hydrosphere.mist.master.execution.{SpawnSettings, Cluster}
 import io.hydrosphere.mist.master.models.ContextConfig
 import io.hydrosphere.mist.utils.akka.ActorRegHub
 
@@ -14,7 +14,7 @@ import scala.util.{Failure, Success}
   */
 class WorkerHub(
   runner: WorkerRunner,
-  mkConnector: (String, ContextConfig, WorkerRunner) => WorkerConnector
+  mkConnector: (String, ContextConfig, WorkerRunner) => Cluster
 ) extends ConnectionsMirror  {
 
   private val wrappedRunner = new WorkerRunner {
@@ -32,7 +32,7 @@ class WorkerHub(
 
   }
 
-  def start(id: String, ctx: ContextConfig): WorkerConnector = mkConnector(id, ctx, wrappedRunner)
+  def start(id: String, ctx: ContextConfig): Cluster = mkConnector(id, ctx, wrappedRunner)
 
   // stopping connection return failed future
   // because it stops worker forcibly
@@ -56,7 +56,7 @@ object WorkerHub {
     val regHub = ActorRegHub("regHub", system)
     val runner =  WorkerRunner.default(spawn, regHub, system)
     val mkConnector = (id: String, ctx: ContextConfig, runner: WorkerRunner) => {
-      WorkerConnector.actorBased(id,ctx, runner, system)
+      Cluster.actorBased(id,ctx, runner, system)
     }
     new WorkerHub(runner, mkConnector)
   }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala
index ea9e98983..bff0f1d67 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.execution.workers
 import akka.actor.{ActorRef, ActorRefFactory}
 import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 import io.hydrosphere.mist.master.execution.SpawnSettings
-import io.hydrosphere.mist.master.execution.workers.starter.WorkerProcess
+import io.hydrosphere.mist.master.execution.workers.starter.{WorkerProcess, WorkerStarter}
 import io.hydrosphere.mist.master.models.ContextConfig
 import io.hydrosphere.mist.utils.akka.ActorRegHub
 
@@ -18,6 +18,7 @@ object WorkerRunner {
 
   class DefaultRunner(
     spawn: SpawnSettings,
+    runnerCmd: WorkerStarter,
     regHub: ActorRegHub,
     connect: (String, WorkerInitInfo, FiniteDuration, ActorRef, StopAction) => Future[WorkerConnection]
   ) extends WorkerRunner {
@@ -58,11 +59,16 @@ object WorkerRunner {
 
   }
 
-  def default(spawn: SpawnSettings, regHub: ActorRegHub, af: ActorRefFactory): WorkerRunner = {
+  def default(
+    spawn: SpawnSettings,
+    runnerCmd: WorkerStarter,
+    regHub: ActorRegHub,
+    af: ActorRefFactory
+  ): WorkerRunner = {
     val connect = (id: String, info: WorkerInitInfo, ready: FiniteDuration, remote: ActorRef, stopAction: StopAction) => {
       WorkerBridge.connect(id, info, ready, remote, stopAction)(af)
     }
-    new DefaultRunner(spawn, regHub, connect)
+    new DefaultRunner(spawn, runnerCmd, regHub, connect)
   }
 
 }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/HttpV2Routes.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/HttpV2Routes.scala
index 51add089c..2c0e23865 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/HttpV2Routes.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/HttpV2Routes.scala
@@ -157,33 +157,33 @@ object HttpV2Routes extends Logger {
 
   import HttpV2Base._
 
-  def workerRoutes(jobService: ExecutionService): Route = {
-    path( root / "workers" ) {
-      get { complete(jobService.workers()) }
-    } ~
-    path( root / "workers" / Segment ) { workerId =>
-      delete {
-        completeU(jobService.stopWorker(workerId))
-      } ~
-      get {
-        completeOpt { jobService.getWorkerLink(workerId) }
-      }
-    } ~
-    path( root / "workers"/ Segment / "jobs") { workerId =>
-      get { (paginationQuery & statusesQuery) { (pagination, statuses) =>
-        val req = JobDetailsRequest(pagination.limit, pagination.offset)
-          .withFilter(FilterClause.ByStatuses(statuses))
-          .withFilter(FilterClause.ByWorkerId(workerId))
-
-        onSuccess(jobService.getHistory(req))(rsp => {
-          if (pagination.paginate)
-            complete(rsp)
-          else
-            complete(rsp.jobs)
-        })
-      }
-    }}
-  }
+//  def workerRoutes(jobService: ExecutionService): Route = {
+//    path( root / "workers" ) {
+//      get { complete(jobService.workers()) }
+//    } ~
+//    path( root / "workers" / Segment ) { workerId =>
+//      delete {
+//        completeU(jobService.stopWorker(workerId))
+//      } ~
+//      get {
+//        completeOpt { jobService.getWorkerLink(workerId) }
+//      }
+//    } ~
+//    path( root / "workers"/ Segment / "jobs") { workerId =>
+//      get { (paginationQuery & statusesQuery) { (pagination, statuses) =>
+//        val req = JobDetailsRequest(pagination.limit, pagination.offset)
+//          .withFilter(FilterClause.ByStatuses(statuses))
+//          .withFilter(FilterClause.ByWorkerId(workerId))
+//
+//        onSuccess(jobService.getHistory(req))(rsp => {
+//          if (pagination.paginate)
+//            complete(rsp)
+//          else
+//            complete(rsp.jobs)
+//        })
+//      }
+//    }}
+//  }
 
   def functionsCrud(functions: FunctionsService): Route = {
     path( root / "functions" ) {
@@ -442,7 +442,7 @@ object HttpV2Routes extends Logger {
     handleExceptions(exceptionHandler) {
       functionAllRoutes(main) ~
       jobsRoutes(main) ~
-      workerRoutes(main.execution) ~
+//      workerRoutes(main.execution) ~
       contextsRoutes(main) ~
       internalArtifacts(mistHome) ~
       artifactRoutes(artifacts) ~
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
index bc47e9d54..5859c4bf4 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
@@ -7,7 +7,7 @@ import akka.testkit.{TestActorRef, TestProbe}
 import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest, _}
 import io.hydrosphere.mist.common.MockitoSugar
 import io.hydrosphere.mist.master.execution.status.StatusReporter
-import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection, WorkerConnector}
+import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection}
 import io.hydrosphere.mist.master.logging.{JobLogger, JobLoggersFactory}
 import io.hydrosphere.mist.master.models.ContextConfig
 import io.hydrosphere.mist.master.{ActorSpec, FilteredException, TestData, TestUtils}
@@ -77,7 +77,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
   }
 
   it("should warmup precreated") {
-    val connector = mock[WorkerConnector]
+    val connector = mock[Cluster]
     when(connector.whenTerminated).thenReturn(Promise[Unit].future)
 
     val job = mkJobProbe()
@@ -283,8 +283,8 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
     p
   }
 
-  def successfulConnector(): WorkerConnector = {
-    new WorkerConnector {
+  def successfulConnector(): Cluster = {
+    new Cluster {
       override def whenTerminated(): Future[Unit] = Promise[Unit].future
       override def askConnection(): Future[PerJobConnection] = Future.successful(new PerJobConnection {
         override def cancel(id: String, respond: ActorRef): Unit = ???
@@ -298,8 +298,8 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
     }
   }
 
-  def failedConnection():WorkerConnector = {
-    new WorkerConnector {
+  def failedConnection():Cluster = {
+    new Cluster {
       override def whenTerminated(): Future[Unit] = Promise[Unit].future
       override def askConnection(): Future[PerJobConnection] = Future.failed(FilteredException())
       override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future
@@ -307,8 +307,8 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
     }
   }
 
-  def crushedConnector(): WorkerConnector = {
-    new WorkerConnector {
+  def crushedConnector(): Cluster = {
+    new Cluster {
       override def whenTerminated(): Future[Unit] = Promise[Unit].failure(FilteredException()).future
       override def askConnection(): Future[PerJobConnection] = Promise[PerJobConnection].future
       override def warmUp(): Unit = ()
@@ -316,8 +316,8 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
     }
   }
 
-  def oneTimeConnector(future: Future[PerJobConnection]): WorkerConnector = {
-    new WorkerConnector {
+  def oneTimeConnector(future: Future[PerJobConnection]): Cluster = {
+    new Cluster {
       override def whenTerminated(): Future[Unit] = Promise[Unit].future
       override def askConnection(): Future[PerJobConnection] = future
       override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
similarity index 90%
rename from mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnectorSpec.scala
rename to mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
index 47206e0c8..958228734 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnectorSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
@@ -6,7 +6,7 @@ import io.hydrosphere.mist.master.execution.workers.WorkerConnector.ActorBasedWo
 
 import scala.concurrent.Promise
 
-class WorkerConnectorSpec extends ActorSpec("actor-based-connector") {
+class ClusterSpec extends ActorSpec("actor-based-connector") {
 
   it("should proxy call to actor") {
     val target = TestProbe()
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
index 92dc90b86..845d72b0d 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
@@ -3,6 +3,7 @@ package io.hydrosphere.mist.master.execution.workers
 import akka.actor.ActorRef
 import io.hydrosphere.mist.common.{CommonData, MockitoSugar}
 import io.hydrosphere.mist.master.TestData
+import io.hydrosphere.mist.master.execution.Cluster
 import io.hydrosphere.mist.master.models.ContextConfig
 import org.scalatest.concurrent.Eventually
 import org.scalatest.time.{Seconds, Span}
@@ -39,7 +40,7 @@ class WorkerHubSpec extends FunSpec with Matchers with TestData with Eventually
   case class TestConnector(
     id: String,
     ctx: ContextConfig,
-    runner: WorkerRunner) extends WorkerConnector {
+    runner: WorkerRunner) extends Cluster {
 
     import scala.concurrent.ExecutionContext.Implicits.global
 

From 345b1e8cbc6ca8229767faff2bc7012f6506e846 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 27 Sep 2018 01:21:03 +0300
Subject: [PATCH 18/30] aws setup - agent running wip

---
 cloud_install/aws/install.sh                  |   2 +-
 .../io/hydrosphere/mist/aws/AwsSetup.scala    |  10 +-
 .../io/hydrosphere/mist/aws/EC2Service.scala  |  58 ++++----
 .../io/hydrosphere/mist/aws/IAMService.scala  |  45 +++++-
 .../scala/io/hydrosphere/mist/aws/Main.scala  |   9 +-
 .../mist/master/MasterServer.scala            |   2 +-
 .../io/hydrosphere/mist/master/configs.scala  |   2 +-
 .../master/execution/ClustersService.scala    |   6 +-
 .../master/execution/ExecutionService.scala   |   6 -
 .../execution/aws/AWSEMRClusterRunner.scala   |  49 +++----
 .../master/execution/aws/AgentInstall.scala   |  40 +++++
 .../mist/master/execution/aws/EMRClient.scala |   1 +
 .../mist/master/execution/aws/SSHClient.scala |  18 ++-
 .../master/execution/workers/WorkerHub.scala  |  98 ++++++-------
 .../execution/ExecutionServiceSpec.scala      |   7 +-
 .../master/execution/SpawnSettingsSpec.scala  |   8 -
 .../execution/workers/ClusterSpec.scala       |   9 +-
 .../workers/ExclusiveConnectorSpec.scala      |   8 +-
 .../workers/SharedConnectorSpec.scala         |  34 ++---
 .../execution/workers/WorkerHubSpec.scala     |  86 +++++------
 .../execution/workers/WorkerRunnerSpec.scala  |   7 +-
 .../interfaces/http/HttpApiV2Spec.scala       | 138 +++++++++---------
 22 files changed, 356 insertions(+), 287 deletions(-)
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala

diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
index 8986b9b36..6dfc3b7e0 100644
--- a/cloud_install/aws/install.sh
+++ b/cloud_install/aws/install.sh
@@ -39,7 +39,7 @@ mv spark-2.3.0-bin-hadoop2.7 spark
 sleep 30
 
 INSTANCE_ID=$(ec2metadata --instance-id)
-$JAVA -cp /opt/mist/utils/aws-init-setup.jar io.hydrosphere.mist.aws.Main $INSTANCE_ID $ACCESS_KEY_ID $ACCESS_KEY_SECRET $AWS_REGION /opt/mist/configs/default.conf ~/.mist_key.pub
+$JAVA -cp /opt/mist/utils/aws-init-setup.jar io.hydrosphere.mist.aws.Main $INSTANCE_ID $ACCESS_KEY_ID $ACCESS_KEY_SECRET $AWS_REGION /opt/mist/configs/default.conf ~/.mist_key.pub ~/.mist_key
 
 SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
 
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
index 89557c5d4..3857c3135 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
@@ -41,11 +41,13 @@ object AwsSetup {
             case Some(d) => ME.pure(d)
             case None => ME.raiseError[InstanceData](new RuntimeException(s"Unknown instance: $instanceId"))
           }
-          ec2EmrRole <- iam.getOrCreate(ec2EmrRole)
-          emrRole <- iam.getOrCreate(emrRole)
+          ec2EmrRole <- iam.getOrCreateRole(ec2EmrRole)
+          ec2EMrInstaceProfile <- iam.getOrCreateInstanceProfile(ec2EmrRole.name, ec2EmrRole.name)
+          emrRole <- iam.getOrCreateRole(emrRole)
 
-          secGroupData = SecGroupData(data.vpcId, 0, 65535, data.cidrIp)
-          secGroupId <- ec2.getOrCreateSecGroup(secGroupName(instanceId), secGroupDecr, secGroupData)
+          secGroupData = IngressData(0, 65535, data.cidrIp, "TCP")
+          secGroupId <- ec2.getOrCreateSecGroup(secGroupName(instanceId), secGroupDecr, data.vpcId, secGroupData)
+          _ <- ec2.addIngressRule(data.secGroupIds.head, IngressData(0, 65535, data.cidrIp, "TCP"))
           keyName <- ec2.getOrCreateKeyPair(keyName(instanceId), sshKey)
         } yield SetupData(data.subnetId, secGroupId, emrRole.name, ec2EmrRole.name, keyName)
       }
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
index ec4c17d81..6c7dca64b 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
@@ -9,17 +9,18 @@ import software.amazon.awssdk.services.ec2.model._
 import scala.collection.JavaConverters._
 import JFutureSyntax._
 
-case class SecGroupData(
-  vpcId: String,
+case class IngressData(
   fromPort: Int,
   toPort: Int,
-  cidrIp: String
+  cidrIp: String,
+  protocol: String
 )
 
 case class InstanceData(
   ip: String,
   vpcId: String,
-  subnetId: String
+  subnetId: String,
+  secGroupIds: Seq[String]
 ) {
   def cidrIp: String = ip + "/32"
 }
@@ -31,6 +32,8 @@ trait EC2Service[F[_]] {
   def getKeyPair(name: String): F[Option[String]]
   def createKeyPair(name: String, key: String): F[String]
 
+  def addIngressRule(groupId: String, data: IngressData): F[Unit]
+
   def getOrCreateKeyPair(name: String, key: String)(implicit M: Monad[F]): F[String] = {
     for {
       curr <- getKeyPair(name)
@@ -40,19 +43,20 @@ trait EC2Service[F[_]] {
       }
     } yield out
   }
-  
+
   def getSecGroup(name: String): F[Option[String]]
-  def createSecGroup(name: String, descr: String, data: SecGroupData): F[String]
+  def createSecGroup(name: String, descr: String, vpcId: String, data: IngressData): F[String]
 
-  def getOrCreateSecGroup(name: String, descr: String, data: SecGroupData)(implicit M: Monad[F]): F[String] = {
+  def getOrCreateSecGroup(name: String, descr: String, vpcId: String, data: IngressData)(implicit M: Monad[F]): F[String] = {
     for {
       curr <- getSecGroup(name)
       out <- curr match {
         case Some(_) => M.pure(name)
-        case None => createSecGroup(name, descr, data)
+        case None => createSecGroup(name, descr, vpcId, data)
       }
     } yield out
   }
+
 }
 
 object EC2Service {
@@ -60,6 +64,18 @@ object EC2Service {
   def fromSdk(ec2Client: EC2AsyncClient): EC2Service[IO] = {
     new EC2Service[IO] {
 
+      override def addIngressRule(groupId: String, data: IngressData): IO[Unit] = {
+        import data._
+        val req = AuthorizeSecurityGroupIngressRequest.builder()
+          .groupId(groupId)
+          .cidrIp(cidrIp)
+          .fromPort(fromPort)
+          .toPort(toPort)
+          .ipProtocol(data.protocol)
+          .build()
+        ec2Client.authorizeSecurityGroupIngress(req).toIO.map(_ => ())
+      }
+
       override def getSecGroup(name: String): IO[Option[String]] = {
         val req = DescribeSecurityGroupsRequest.builder().groupNames(name).build()
         val io = ec2Client.describeSecurityGroups(req).toIO
@@ -74,32 +90,20 @@ object EC2Service {
       override def createSecGroup(
         name: String,
         descr: String,
-        data: SecGroupData
+        vpcId: String,
+        data: IngressData
       ): IO[String] = {
 
-        def ingressReq(data: SecGroupData, groupId: String): AuthorizeSecurityGroupIngressRequest = {
-          import data._
-
-          AuthorizeSecurityGroupIngressRequest.builder()
-            .groupId(groupId)
-            .cidrIp(cidrIp)
-            .fromPort(fromPort)
-            .toPort(toPort)
-            .ipProtocol("TCP")
-            .build()
-        }
-
         val createReq = CreateSecurityGroupRequest.builder()
           .groupName(name)
           .description(descr)
-          .vpcId(data.vpcId)
+          .vpcId(vpcId)
           .build()
 
         for {
           resp <- ec2Client.createSecurityGroup(createReq).toIO
           groupId = resp.groupId()
-          ingReq = ingressReq(data, groupId)
-          _ <- ec2Client.authorizeSecurityGroupIngress(ingReq).toIO
+          _ <- addIngressRule(groupId, data)
         } yield groupId
       }
 
@@ -108,7 +112,10 @@ object EC2Service {
           for {
             reservation <- resp.reservations().asScala.headOption
             instance <- reservation.instances().asScala.headOption
-          } yield InstanceData(instance.privateIpAddress(), instance.vpcId(), instance.subnetId())
+          } yield {
+            val secGroupsIds = instance.securityGroups().asScala.map(sg => sg.groupId())
+            InstanceData(instance.privateIpAddress(), instance.vpcId(), instance.subnetId(), secGroupsIds)
+          }
         }
 
         val req = DescribeInstancesRequest.builder().instanceIds(id).build()
@@ -132,6 +139,7 @@ object EC2Service {
         val req = ImportKeyPairRequest.builder().keyName(name).publicKeyMaterial(key).build()
         ec2Client.importKeyPair(req).toIO.map(resp => resp.keyName())
       }
+
     }
 
   }
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
index 0416427a1..3c639dd7d 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
@@ -8,8 +8,7 @@ import cats.implicits._
 import cats.Monad
 import cats.effect._
 import software.amazon.awssdk.services.iam.IAMAsyncClient
-import software.amazon.awssdk.services.iam.model.{AttachRolePolicyRequest, CreateRoleRequest, GetRoleRequest, Role}
-
+import software.amazon.awssdk.services.iam.model._
 import JFutureSyntax._
 
 import scala.io.Source
@@ -49,7 +48,7 @@ trait IAMService[F[_]] {
   def createRole(role: AWSRole): F[AWSRole]
   def getRole(name: String): F[Option[String]]
 
-  def getOrCreate(role: AWSRole)(implicit m: Monad[F]): F[AWSRole] = {
+  def getOrCreateRole(role: AWSRole)(implicit m: Monad[F]): F[AWSRole] = {
     for {
       out <- getRole(role.name)
       role <- out match {
@@ -58,6 +57,19 @@ trait IAMService[F[_]] {
       }
     } yield role
   }
+
+  def createInstanceProfile(name: String, role: String): F[String]
+  def getInstanceProfile(name: String): F[Option[String]]
+  def getOrCreateInstanceProfile(name: String, role: String)(implicit M: Monad[F]): F[String] = {
+    for {
+      out <- getInstanceProfile(name)
+      role <- out match {
+        case Some(_) => M.pure(role)
+        case None => createInstanceProfile(name, role)
+      }
+    } yield role
+  }
+
 }
 
 object IAMService {
@@ -93,5 +105,30 @@ object IAMService {
           _ <- iamClient.attachRolePolicy(attachReq).toIO
         } yield role
       }
-  }
+
+      override def createInstanceProfile(name: String, role: String): IO[String] = {
+        val createReq = CreateInstanceProfileRequest.builder()
+          .instanceProfileName(name)
+          .build()
+
+        val addReq = AddRoleToInstanceProfileRequest.builder()
+          .instanceProfileName(name)
+          .roleName(role)
+          .build()
+        for {
+          _ <- iamClient.createInstanceProfile(createReq).toIO
+          _ <- iamClient.addRoleToInstanceProfile(addReq).toIO
+        } yield name
+      }
+
+      override def getInstanceProfile(name: String): IO[Option[String]] = {
+        val req = GetInstanceProfileRequest.builder().instanceProfileName(name).build()
+        iamClient.getInstanceProfile(req).toIO
+          .map(resp => Option(resp.instanceProfile().instanceProfileName()))
+          .handleErrorWith({
+            case _: NoSuchEntityException => IO.pure(None)
+            case e => IO.raiseError(e)
+          })
+      }
+    }
 }
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
index f7d0eb079..d761f84ab 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
@@ -13,15 +13,16 @@ object Main {
     val region = args(3)
     val configPath = args(4)
 
-    val sshKeyPath = args(5)
-    val sshKey = Source.fromFile(Paths.get(sshKeyPath).toFile).mkString
+    val sshKeyPathPub = args(5)
+    val sshKeyPath = args(6)
+    val sshKeyPub = Source.fromFile(Paths.get(sshKeyPathPub).toFile).mkString
 
     val setup = AwsSetup.create(accessKey, accessSecret, region)
-    val out = setup.setup(instanceId, sshKey).unsafeRunSync()
+    val out = setup.setup(instanceId, sshKeyPub).unsafeRunSync()
 
     val launchData = LaunchData(
       sshKeyPair = out.sshKeyPairName,
-      sshKeyPath = sshKeyPath,
+      sshKeyPath = sshKeyPathPub,
       accessKey = accessKey,
       secretKey = accessSecret,
       subnetId = out.subnetId,
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
index 462155a4b..357404192 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala
@@ -117,7 +117,7 @@ object MasterServer extends Logger {
         ClusterRunner.legacy(spawnSettings, regHub, starter, system)
       }
 
-      ClustersService.create(config.mistHome, spawnSettings, config.launchersSettings, defaultRunner, system)
+      ClustersService.create(config.mistHome, spawnSettings, regHub, config.launchersSettings, defaultRunner, system)
     }
 
     def runExecutionService(clustersService: ClustersService, logService: LogService): ExecutionService = {
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
index d640811e8..98c50cefd 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
@@ -367,7 +367,7 @@ object MasterConfig extends Logger {
       jobInfoProviderConfig = FunctionInfoProviderConfig(mist.getConfig("job-extractor")),
       srcConfigPath = filePath,
       launchersSettings = LauncherSettings.extractAll(mist.getConfigList("launchers-settings")),
-      mistHome = config.getString("work-directory"),
+      mistHome = mist.getString("work-directory"),
       raw = config
     )
   }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
index ce69ece05..9707bc6f9 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
@@ -6,6 +6,7 @@ import akka.actor.ActorSystem
 import io.hydrosphere.mist.master.execution.aws.AWSEMRClusterRunner
 import io.hydrosphere.mist.master.{AWSEMRLaunchSettings, LauncherSettings}
 import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig, ServerDefault}
+import io.hydrosphere.mist.utils.Logger
 import io.hydrosphere.mist.utils.akka.ActorRegHub
 
 import scala.concurrent.Future
@@ -16,17 +17,17 @@ trait ClustersService {
 
 }
 
-object ClustersService {
+object ClustersService extends Logger{
 
   def create(
     mistHome: String,
     spawn: SpawnSettings,
+    regHub: ActorRegHub,
     launchSettings: Map[String, LauncherSettings],
     serverDefault: ClusterRunner,
     system: ActorSystem
   ): ClustersService = {
 
-    val regHub = ActorRegHub("regHub", system)
     val runners = launchSettings.map({case (name, settings) => {
       val runner = settings match {
         case aws: AWSEMRLaunchSettings =>
@@ -41,6 +42,7 @@ object ClustersService {
         ctx.launchData match {
           case ServerDefault => serverDefault.run(id, ctx)
           case aws: AWSEMRLaunchData =>
+            logger.info(s"IN:${aws.launcherSettingsName} keys: ${runners.keys.toList}")
             runners.get(aws.launcherSettingsName) match {
               case Some(runner) => runner.run(id, ctx)
               case None => Future.failed(new RuntimeException(s"Unknown settings name ${aws.launcherSettingsName} for ctx ${ctx.name}"))
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
index b8af86f25..c5ac8127d 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala
@@ -8,7 +8,6 @@ import cats.implicits._
 import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, JobParams, RunJobRequest}
 import io.hydrosphere.mist.master.Messages.StatusMessages.InitializedEvent
 import io.hydrosphere.mist.master.execution.status.StatusReporter
-import io.hydrosphere.mist.master.execution.workers.WorkerHub
 import io.hydrosphere.mist.master.logging.LogService
 import io.hydrosphere.mist.master.models._
 import io.hydrosphere.mist.master.store.JobRepository
@@ -22,7 +21,6 @@ import scala.concurrent.Future
   */
 class ExecutionService(
   contextsMaster: ActorRef,
-  //workersHub: WorkerHub,
   statusReporter: StatusReporter,
   repo: JobRepository
 ) {
@@ -110,16 +108,12 @@ class ExecutionService(
 object ExecutionService {
 
   def apply(
-//    spawn: SpawnSettings,
     clustersService: ClustersService,
     system: ActorSystem,
     streamer: EventsStreamer,
     repo: JobRepository,
     logService: LogService
   ): ExecutionService = {
-    // TODO
-//    val hub = WorkerHub(spawn, system)
-
     val reporter = StatusReporter.reporter(repo, streamer, logService)(system)
     val mkContext = ActorF[ContextConfig]((ctx, af) => {
       val props = ContextFrontend.props(ctx.name, reporter, logService, clustersService.start)
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
index 530929bd0..08eb31341 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
@@ -1,6 +1,6 @@
 package io.hydrosphere.mist.master.execution.aws
 
-import java.nio.file.Path
+import java.nio.file.{Path, Paths}
 import java.util.concurrent.Executors
 
 import akka.actor.ActorSystem
@@ -12,35 +12,15 @@ import io.hydrosphere.mist.master.execution.workers.{StopAction, WorkerConnectio
 import io.hydrosphere.mist.master.execution.workers.starter.{SparkSubmitBuilder, WorkerProcess, WorkerStarter}
 import io.hydrosphere.mist.master.execution.{Cluster, ClusterRunner, SpawnSettings}
 import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig}
+import io.hydrosphere.mist.utils.Logger
 import io.hydrosphere.mist.utils.akka.ActorRegHub
 
 import scala.concurrent.duration._
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Failure
 
-object AWSEMRClusterRunner {
-
-  def mkInstallCommands(
-    agentJarPath: String,
-    workerJarPath: String,
-    agentId: String,
-    masterAddr: String,
-    accessKey: String,
-    secretKey: String,
-    region: String,
-    awsId: String
-  ): Seq[SSHCmd] = {
-    Seq(
-      SSHCmd.Exec(Seq("mkdir", "~/mist-agent")),
-      SSHCmd.CopyFile(agentJarPath, "~/mist-agent/mist-agent.jar"),
-      SSHCmd.CopyFile(workerJarPath, "~/mist-agent/mist-worker.jar"),
-      SSHCmd.Exec(Seq(
-        "java", "-cp", "~/mist-agent/mist-master.jar", "io.hydrosphere.mist.master.execution.aws.ClusterAgent",
-        masterAddr, agentId, accessKey, secretKey, region, awsId,
-        "1>~/mist-agent/out.log", "2>~/mist-agent/out.log", "&"
-      ))
-    )
-  }
+object AWSEMRClusterRunner extends Logger {
 
   class EMRClusterRunner(
     spawn: SpawnSettings,
@@ -108,7 +88,7 @@ object AWSEMRClusterRunner {
     }
 
     private def mkRunner(launch: AWSEMRLaunchSettings, host: String): WorkerRunner = {
-      val starter = new WorkerStarter {
+      val starter: WorkerStarter = new WorkerStarter {
         val builder = new SparkSubmitBuilder("~/mist-agent", "/usr/lib/spark")
         override def onStart(name: String, initInfo: CommonData.WorkerInitInfo): WorkerProcess = {
           val submitCmd = builder.submitWorker(name, initInfo) :+ "&"
@@ -120,8 +100,8 @@ object AWSEMRClusterRunner {
 
         override def stopAction: StopAction = StopAction.Remote
       }
-      val hackSettings = spawn.copy(runnerCmd = starter)
-      WorkerRunner.default(hackSettings, regHub, system)
+
+      WorkerRunner.default(spawn, starter, regHub, system)
     }
 
     override def run(id: String, ctx: ContextConfig): Future[Cluster] = {
@@ -151,10 +131,18 @@ object AWSEMRClusterRunner {
     val client = EMRClient.create(accessKey, secretKey, region)
 
     val installAgent = (host: String, agentId: String, awsId: String) => {
-      val cmds = mkInstallCommands(
-        jarsDir.resolve("mist-master.jar").toString, jarsDir.resolve("mist-worker.jar").toString,
-        agentId: String, spawn.akkaAddress, accessKey, secretKey, region, awsId)
+      val realDir = jarsDir.toAbsolutePath.toRealPath()
+      val transfer = TransferParams(
+        realDir.resolve("mist-master.jar").toString,
+        realDir.resolve("mist-worker.jar").toString,
+        s"/home/$sshUser/mist-agent"
+      )
+      val agentRunParams = AgentRunParams(
+        agentId, spawn.akkaAddress, accessKey, secretKey, region, awsId
+      )
+      val cmds = AgentInstall.sshCommands(transfer, agentRunParams)
       new SSHClient(host, sshUser, sshKeyPath).install(cmds)
+      ()
     }
 
     new EMRClusterRunner(
@@ -167,3 +155,4 @@ object AWSEMRClusterRunner {
     )
   }
 }
+
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala
new file mode 100644
index 000000000..2a361979c
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala
@@ -0,0 +1,40 @@
+package io.hydrosphere.mist.master.execution.aws
+
+case class TransferParams(
+  agentJar: String,
+  workerJar: String,
+  targetDir: String
+)
+
+case class AgentRunParams(
+  agentId: String,
+  masterAddress: String,
+  accessKey: String,
+  secretKey: String,
+  region: String,
+  clusterId: String
+)
+
+object AgentInstall {
+
+  def sshCommands(
+    transferParams: TransferParams,
+    agentRunParams: AgentRunParams
+  ): Seq[SSHCmd] = {
+
+    import transferParams._
+    import agentRunParams._
+
+    Seq(
+      SSHCmd.Exec(Seq("mkdir", targetDir)),
+      SSHCmd.CopyFile(agentJar, s"$targetDir/mist-agent.jar"),
+      SSHCmd.CopyFile(workerJar, s"$targetDir/mist-worker.jar"),
+      SSHCmd.Exec(Seq(
+        "java", "-cp", "~/mist-agent/mist-master.jar", "io.hydrosphere.mist.master.execution.aws.ClusterAgent",
+        masterAddress, agentId, accessKey, secretKey, region, clusterId,
+        s"1>$targetDir/out.log", s"2>$targetDir/out.log", "&"
+      ))
+    )
+  }
+
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
index c4f9c7b5c..d79664e86 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
@@ -45,6 +45,7 @@ object EMRClient {
     override def start(settings: EMRRunSettings): IO[EmrInfo] = {
       import settings._
 
+      //TODO: configuration!
       val sparkApp = Application.builder().name("Spark").build()
       val request = RunJobFlowRequest.builder()
         .name(s"mist-$name")
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
index c2d2aae8e..badac9a73 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
@@ -1,6 +1,10 @@
 package io.hydrosphere.mist.master.execution.aws
 
 import com.decodified.scalassh._
+import cats._
+import cats.implicits._
+
+import scala.util.Try
 
 sealed trait SSHCmd
 object SSHCmd {
@@ -11,19 +15,19 @@ object SSHCmd {
 class SSHClient(host: String, user: String, keyPath: String) {
 
   val cfgProvider = HostConfig(
-    login = PublicKeyLogin(user, None, List(keyPath)),
+    login = PublicKeyLogin(user, keyPath),
     hostName = host,
     port = 22,
     hostKeyVerifier = HostKeyVerifiers.DontVerify
   )
 
-  def install(cmds: Seq[SSHCmd]): Unit = {
+  def install(cmds: Seq[SSHCmd]): Try[Unit] = {
     SSH(host, cfgProvider) { client =>
-      cmds.foreach {
-        case SSHCmd.CopyFile(from, to) => client.upload(from, to)
-        case SSHCmd.Exec(cmd) => client.exec(Command(cmd.mkString(" ")))
-      }
-    }
+     cmds.map {
+       case SSHCmd.CopyFile(from, to) => client.upload(from, to)
+       case SSHCmd.Exec(cmd) => client.exec(Command(cmd.mkString(" "))).map(_ => ())
+     }.toList.sequence
+    }.map(_ => ())
   }
 }
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala
index 24d9a7d73..cfb1d35fc 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala
@@ -12,54 +12,54 @@ import scala.util.{Failure, Success}
 /**
   * Mix connections mirror and connector starter
   */
-class WorkerHub(
-  runner: WorkerRunner,
-  mkConnector: (String, ContextConfig, WorkerRunner) => Cluster
-) extends ConnectionsMirror  {
-
-  private val wrappedRunner = new WorkerRunner {
-    override def apply(id: String, ctx: ContextConfig): Future[WorkerConnection] = {
-      val pr = Promise[WorkerConnection]
-      runner(id, ctx).onComplete {
-        case Success(conn) =>
-          add(conn)
-          conn.whenTerminated.onComplete({_ => remove(conn.id)})
-          pr.success(conn)
-        case Failure(e) => pr.failure(e)
-      }
-      pr.future
-    }
-
-  }
-
-  def start(id: String, ctx: ContextConfig): Cluster = mkConnector(id, ctx, wrappedRunner)
-
-  // stopping connection return failed future
-  // because it stops worker forcibly
-  // calling stop from here means that user understands it
-  private def shutdownConn(conn: WorkerConnection): Future[Unit] =
-    conn.shutdown(true).recover({case _ => ()})
-
-  def shutdownWorker(id: String): Future[Unit] = workerConnection(id) match {
-    case Some(connection) => shutdownConn(connection)
-    case None => Future.failed(new IllegalStateException(s"Unknown worker $id"))
-  }
-
-  def shutdownAllWorkers(): Future[Unit] =
-    Future.sequence(workerConnections().map(conn => shutdownConn(conn))).map(_ => ())
-
-}
-
-object WorkerHub {
-
-  def apply(spawn: SpawnSettings, system: ActorSystem): WorkerHub = {
-    val regHub = ActorRegHub("regHub", system)
-    val runner =  WorkerRunner.default(spawn, regHub, system)
-    val mkConnector = (id: String, ctx: ContextConfig, runner: WorkerRunner) => {
-      Cluster.actorBased(id,ctx, runner, system)
-    }
-    new WorkerHub(runner, mkConnector)
-  }
-}
+//class WorkerHub(
+//  runner: WorkerRunner,
+//  mkConnector: (String, ContextConfig, WorkerRunner) => Cluster
+//) extends ConnectionsMirror  {
+//
+//  private val wrappedRunner = new WorkerRunner {
+//    override def apply(id: String, ctx: ContextConfig): Future[WorkerConnection] = {
+//      val pr = Promise[WorkerConnection]
+//      runner(id, ctx).onComplete {
+//        case Success(conn) =>
+//          add(conn)
+//          conn.whenTerminated.onComplete({_ => remove(conn.id)})
+//          pr.success(conn)
+//        case Failure(e) => pr.failure(e)
+//      }
+//      pr.future
+//    }
+//
+//  }
+//
+//  def start(id: String, ctx: ContextConfig): Cluster = mkConnector(id, ctx, wrappedRunner)
+//
+//  // stopping connection return failed future
+//  // because it stops worker forcibly
+//  // calling stop from here means that user understands it
+//  private def shutdownConn(conn: WorkerConnection): Future[Unit] =
+//    conn.shutdown(true).recover({case _ => ()})
+//
+//  def shutdownWorker(id: String): Future[Unit] = workerConnection(id) match {
+//    case Some(connection) => shutdownConn(connection)
+//    case None => Future.failed(new IllegalStateException(s"Unknown worker $id"))
+//  }
+//
+//  def shutdownAllWorkers(): Future[Unit] =
+//    Future.sequence(workerConnections().map(conn => shutdownConn(conn))).map(_ => ())
+//
+//}
+//
+//object WorkerHub {
+//
+//  def apply(spawn: SpawnSettings, system: ActorSystem): WorkerHub = {
+//    val regHub = ActorRegHub("regHub", system)
+//    val runner =  WorkerRunner.default(spawn, regHub, system)
+//    val mkConnector = (id: String, ctx: ContextConfig, runner: WorkerRunner) => {
+//      Cluster.actorBased(id,ctx, runner, system)
+//    }
+//    new WorkerHub(runner, mkConnector)
+//  }
+//}
 
 
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala
index bf71937d3..4a92b6e0a 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala
@@ -5,7 +5,6 @@ import akka.testkit.{TestKit, TestProbe}
 import io.hydrosphere.mist.common.{FunctionInfoData, MockitoSugar}
 import io.hydrosphere.mist.master.Messages.StatusMessages.UpdateStatusEvent
 import io.hydrosphere.mist.master.execution.status.StatusReporter
-import io.hydrosphere.mist.master.execution.workers.WorkerHub
 import io.hydrosphere.mist.master.models.JobStartRequest
 import io.hydrosphere.mist.master.store.JobRepository
 import io.hydrosphere.mist.master.{JobDetails, TestData, TestUtils}
@@ -29,11 +28,10 @@ class ExecutionServiceSpec extends TestKit(ActorSystem("testMasterService"))
     it("should start job") {
       val execution = TestProbe()
       val repo = mock[JobRepository]
-      val hub = mock[WorkerHub]
       val reporter = mock[StatusReporter]
       when(repo.update(any[JobDetails])).thenSuccess(())
 
-      val service = new ExecutionService(execution.ref, hub, reporter, repo)
+      val service = new ExecutionService(execution.ref, reporter, repo)
 
       val future = service.startJob(
         JobStartRequest(
@@ -62,14 +60,13 @@ class ExecutionServiceSpec extends TestKit(ActorSystem("testMasterService"))
       //TODO
       val contextsMaster = TestProbe()
       val repo = mock[JobRepository]
-      val hub = mock[WorkerHub]
       val reporter = mock[StatusReporter]
 
       when(repo.get(any[String]))
         .thenSuccess(Some(mkDetails(JobDetails.Status.Started)))
         .thenSuccess(Some(mkDetails(JobDetails.Status.Canceled)))
 
-      val service = new ExecutionService(contextsMaster.ref, hub, reporter, repo)
+      val service = new ExecutionService(contextsMaster.ref, reporter, repo)
 
       val future = service.stopJob("id")
 
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
index 1df07229c..bed922b81 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala
@@ -1,9 +1,6 @@
 package io.hydrosphere.mist.master.execution
 
-import io.hydrosphere.mist.common.CommonData
 import io.hydrosphere.mist.master.TestData
-import io.hydrosphere.mist.master.execution.workers.StopAction
-import io.hydrosphere.mist.master.execution.workers.starter.{WorkerProcess, WorkerStarter}
 import io.hydrosphere.mist.master.models.RunMode
 import org.scalatest.{FunSpec, Matchers}
 
@@ -12,12 +9,7 @@ import scala.concurrent.duration._
 class SpawnSettingsSpec extends FunSpec with Matchers with TestData {
 
   it("should build worker init info") {
-    val noop = new WorkerStarter {
-      override def onStart(name: String, initInfo: CommonData.WorkerInitInfo): WorkerProcess = WorkerProcess.NonLocal
-      override def stopAction: StopAction = StopAction.Remote
-    }
     val spawnSettings = SpawnSettings(
-      runnerCmd = noop,
       timeout = 10 seconds,
       readyTimeout = 10 seconds,
       akkaAddress = "akkaAddr",
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
index 958228734..7839baefe 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
@@ -2,7 +2,8 @@ package io.hydrosphere.mist.master.execution.workers
 
 import akka.testkit.TestProbe
 import io.hydrosphere.mist.master.ActorSpec
-import io.hydrosphere.mist.master.execution.workers.WorkerConnector.ActorBasedWorkerConnector
+import io.hydrosphere.mist.master.execution.Cluster
+import io.hydrosphere.mist.master.execution.Cluster.ActorBasedWorkerConnector
 
 import scala.concurrent.Promise
 
@@ -13,13 +14,13 @@ class ClusterSpec extends ActorSpec("actor-based-connector") {
     val connector = new ActorBasedWorkerConnector(target.ref, Promise[Unit].future)
 
     connector.askConnection()
-    target.expectMsgType[WorkerConnector.Event.AskConnection]
+    target.expectMsgType[Cluster.Event.AskConnection]
 
     connector.warmUp()
-    target.expectMsgType[WorkerConnector.Event.WarmUp.type]
+    target.expectMsgType[Cluster.Event.WarmUp.type]
 
     connector.shutdown(true)
-    target.expectMsgType[WorkerConnector.Event.Shutdown]
+    target.expectMsgType[Cluster.Event.Shutdown]
   }
 
 }
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala
index 173395c1b..669edf5dd 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala
@@ -4,7 +4,7 @@ import akka.actor.ActorRef
 import akka.testkit.{TestActorRef, TestProbe}
 import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.master.execution.workers.WorkerBridge.Event.CompleteAndShutdown
-import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released
+import io.hydrosphere.mist.master.execution.Cluster
 import io.hydrosphere.mist.master.{ActorSpec, FilteredException, TestData}
 
 import scala.concurrent.{Await, Future, Promise}
@@ -21,7 +21,7 @@ class ExclusiveConnectorSpec extends ActorSpec("excl-conn") with TestData {
 
     val probe = TestProbe()
     val resolve = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve))
+    probe.send(connector, Cluster.Event.AskConnection(resolve))
 
     intercept[Throwable] {
       Await.result(resolve.future, Duration.Inf)
@@ -40,7 +40,7 @@ class ExclusiveConnectorSpec extends ActorSpec("excl-conn") with TestData {
 
     val probe = TestProbe()
     val resolve = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve))
+    probe.send(connector, Cluster.Event.AskConnection(resolve))
 
     val connection = Await.result(resolve.future, Duration.Inf)
 
@@ -68,7 +68,7 @@ class ExclusiveConnectorSpec extends ActorSpec("excl-conn") with TestData {
       val wrapped = ExclusiveConnector.wrappedConnection(connector.ref, connection)
 
       wrapped.release()
-      connector.expectMsgType[WorkerConnector.Event.Released]
+      connector.expectMsgType[Cluster.Event.Released]
 
       wrapped.run(mkRunReq("id"), ActorRef.noSender)
       connRef.expectMsgType[RunJobRequest]
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
index 645600101..cf46dc706 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
@@ -5,7 +5,7 @@ import java.util.concurrent.atomic.AtomicInteger
 import akka.actor.ActorRef
 import akka.testkit.{TestActorRef, TestProbe}
 import io.hydrosphere.mist.common.CommonData.RunJobRequest
-import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released
+import io.hydrosphere.mist.master.execution.Cluster
 import io.hydrosphere.mist.master.{ActorSpec, TestData}
 import org.scalatest.Matchers
 import org.scalatest.concurrent.Eventually
@@ -33,14 +33,14 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     val probe = TestProbe()
 
     val resolve = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve))
+    probe.send(connector, Cluster.Event.AskConnection(resolve))
     val connection1 = Await.result(resolve.future, Duration.Inf)
 
     val resolve2 = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve2))
+    probe.send(connector, Cluster.Event.AskConnection(resolve2))
     val connection2 = Await.result(resolve2.future, Duration.Inf)
     val resolve3 = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve3))
+    probe.send(connector, Cluster.Event.AskConnection(resolve3))
 
     connection1.id shouldBe "1"
     connection2.id shouldBe "2"
@@ -66,9 +66,9 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     ))
 
     val probe = TestProbe()
-    probe.send(connector, WorkerConnector.Event.WarmUp)
+    probe.send(connector, Cluster.Event.WarmUp)
     callCounter.get() shouldBe 2
-    probe.send(connector, WorkerConnector.Event.GetStatus)
+    probe.send(connector, Cluster.Event.GetStatus)
     val status = probe.expectMsgType[SharedConnector.ProcessStatus]
     status.poolSize shouldBe 2
     status.requestsSize shouldBe 0
@@ -93,11 +93,11 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     val probe = TestProbe()
 
     val resolve = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve))
+    probe.send(connector, Cluster.Event.AskConnection(resolve))
     val connection1 = Await.result(resolve.future, Duration.Inf)
 
     val resolve2 = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve2))
+    probe.send(connector, Cluster.Event.AskConnection(resolve2))
     val connection2 = Await.result(resolve2.future, Duration.Inf)
 
     connection1.id shouldBe "1"
@@ -106,7 +106,7 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     connection1.release()
 
     val resolve3 = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve3))
+    probe.send(connector, Cluster.Event.AskConnection(resolve3))
 
     val conn3 = Await.result(resolve3.future, Duration.Inf)
     conn3.id shouldBe "1"
@@ -123,16 +123,16 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     val probe = TestProbe()
 
     val resolve = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve))
+    probe.send(connector, Cluster.Event.AskConnection(resolve))
     val connection1 = Await.result(resolve.future, Duration.Inf)
-    probe.send(connector, WorkerConnector.Event.GetStatus)
+    probe.send(connector, Cluster.Event.GetStatus)
     val status = probe.expectMsgType[SharedConnector.ProcessStatus]
     status.inUseSize shouldBe 1
     status.poolSize shouldBe 0
 
     termination.success(())
     //ConnTerminated is fired
-    probe.send(connector, WorkerConnector.Event.GetStatus)
+    probe.send(connector, Cluster.Event.GetStatus)
     val status2 = probe.expectMsgType[SharedConnector.ProcessStatus]
     status2.inUseSize shouldBe 0
     status2.poolSize shouldBe 0
@@ -149,17 +149,17 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     val probe = TestProbe()
 
     val resolve = Promise[PerJobConnection]
-    probe.send(connector, WorkerConnector.Event.AskConnection(resolve))
+    probe.send(connector, Cluster.Event.AskConnection(resolve))
     val connection1 = Await.result(resolve.future, Duration.Inf)
 
-    probe.send(connector, WorkerConnector.Event.GetStatus)
+    probe.send(connector, Cluster.Event.GetStatus)
     val status = probe.expectMsgType[SharedConnector.ProcessStatus]
     status.inUseSize shouldBe 1
     status.poolSize shouldBe 0
 
     connection1.release()
 
-    probe.send(connector, WorkerConnector.Event.GetStatus)
+    probe.send(connector, Cluster.Event.GetStatus)
     val status1 = probe.expectMsgType[SharedConnector.ProcessStatus]
     status1.poolSize shouldBe 1
     status1.inUseSize shouldBe 0
@@ -167,7 +167,7 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     termination.success(())
 
     //ConnTerminated is fired
-    probe.send(connector, WorkerConnector.Event.GetStatus)
+    probe.send(connector, Cluster.Event.GetStatus)
     val status2 = probe.expectMsgType[SharedConnector.ProcessStatus]
     status2.poolSize shouldBe 0
     status2.inUseSize shouldBe 0
@@ -187,7 +187,7 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
       val connector = TestProbe()
       val wrapped = SharedConnector.wrappedConnection(connector.ref, connection)
       wrapped.release()
-      connector.expectMsgType[Released]
+      connector.expectMsgType[Cluster.Event.Released]
       connRef.expectNoMessage(1 second)
 
       wrapped.run(mkRunReq("id"), ActorRef.noSender)
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
index 845d72b0d..a8c474790 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala
@@ -12,46 +12,46 @@ import org.scalatest.{FunSpec, Matchers}
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Future, Promise}
 
-class WorkerHubSpec extends FunSpec with Matchers with TestData with Eventually with MockitoSugar {
-
-  it("should mirror connections") {
-    val termination = Promise[Unit]
-
-    val runner = new WorkerRunner {
-      override def apply(id: String, ctx: ContextConfig): Future[WorkerConnection] =
-        Future.successful(WorkerConnection(id, null, workerLinkData.copy(name = id), termination.future))
-    }
-    val hub = new WorkerHub(runner, TestConnector.apply)
-
-    val connector = hub.start("id", FooContext)
-
-    Await.result(connector.askConnection(), Duration.Inf)
-
-    eventually(timeout(Span(3, Seconds))) {
-      hub.workerConnections().size shouldBe 1
-    }
-    termination.success(())
-    eventually(timeout(Span(3, Seconds))) {
-      hub.workerConnections().size shouldBe 0
-    }
-  }
-
-
-  case class TestConnector(
-    id: String,
-    ctx: ContextConfig,
-    runner: WorkerRunner) extends Cluster {
-
-    import scala.concurrent.ExecutionContext.Implicits.global
-
-    def askConnection(): Future[PerJobConnection] = runner(id, ctx).map(conn => mock[PerJobConnection])
-
-    def warmUp(): Unit = ()
-
-    def shutdown(force: Boolean): Future[Unit] = ???
-
-    def whenTerminated(): Future[Unit] = ???
-
-    def releaseConnection(connectionId: String): Unit = ???
-  }
-}
+//class WorkerHubSpec extends FunSpec with Matchers with TestData with Eventually with MockitoSugar {
+//
+//  it("should mirror connections") {
+//    val termination = Promise[Unit]
+//
+//    val runner = new WorkerRunner {
+//      override def apply(id: String, ctx: ContextConfig): Future[WorkerConnection] =
+//        Future.successful(WorkerConnection(id, null, workerLinkData.copy(name = id), termination.future))
+//    }
+//    val hub = new WorkerHub(runner, TestConnector.apply)
+//
+//    val connector = hub.start("id", FooContext)
+//
+//    Await.result(connector.askConnection(), Duration.Inf)
+//
+//    eventually(timeout(Span(3, Seconds))) {
+//      hub.workerConnections().size shouldBe 1
+//    }
+//    termination.success(())
+//    eventually(timeout(Span(3, Seconds))) {
+//      hub.workerConnections().size shouldBe 0
+//    }
+//  }
+//
+//
+//  case class TestConnector(
+//    id: String,
+//    ctx: ContextConfig,
+//    runner: WorkerRunner) extends Cluster {
+//
+//    import scala.concurrent.ExecutionContext.Implicits.global
+//
+//    def askConnection(): Future[PerJobConnection] = runner(id, ctx).map(conn => mock[PerJobConnection])
+//
+//    def warmUp(): Unit = ()
+//
+//    def shutdown(force: Boolean): Future[Unit] = ???
+//
+//    def whenTerminated(): Future[Unit] = ???
+//
+//    def releaseConnection(connectionId: String): Unit = ???
+//  }
+//}
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala
index 7b6325d07..1698aa354 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala
@@ -20,7 +20,6 @@ class WorkerRunnerSpec extends ActorSpec("worker-runner") with TestData with Moc
   describe("default runner") {
 
     def mkSpawnSettings(starter: WorkerStarter): SpawnSettings = SpawnSettings(
-      runnerCmd = starter,
       timeout = 10 seconds,
       readyTimeout = 10 seconds,
       akkaAddress = "akkaAddr",
@@ -41,7 +40,8 @@ class WorkerRunnerSpec extends ActorSpec("worker-runner") with TestData with Moc
       val runner = new workers.WorkerRunner.DefaultRunner(
         spawn = mkSpawnSettings(starter),
         regHub = regHub,
-        connect = (_, _, _, _, _) => Future.successful(WorkerConnection("id", null, workerLinkData, termination.future))
+        connect = (_, _, _, _, _) => Future.successful(WorkerConnection("id", null, workerLinkData, termination.future)),
+        runnerCmd = starter
       )
 
       Await.result(runner("id", FooContext), Duration.Inf)
@@ -61,7 +61,8 @@ class WorkerRunnerSpec extends ActorSpec("worker-runner") with TestData with Moc
       val runner = new workers.WorkerRunner.DefaultRunner(
         spawn = mkSpawnSettings(runnerCmd),
         regHub = regHub,
-        connect = (_, _, _, _, _) => Future.failed(FilteredException())
+        connect = (_, _, _, _, _) => Future.failed(FilteredException()),
+        runnerCmd = runnerCmd
       )
 
       intercept[Throwable] {
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
index 1c228d3ed..479d9cb0d 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
@@ -48,75 +48,75 @@ class HttpApiV2Spec extends FunSpec
     }
   }
 
-  describe("workers") {
-
-    it("should return workers") {
-      val execution = mock[ExecutionService]
-      when(execution.workers()).thenReturn(Seq(workerLinkData))
-
-      val route = HttpV2Routes.workerRoutes(execution)
-
-      Get("/v2/api/workers") ~> route ~> check {
-        status shouldBe StatusCodes.OK
-        val rsp = responseAs[Seq[WorkerLink]]
-        rsp.size shouldBe 1
-      }
-    }
-
-    it("should stop worker") {
-      val execution = mock[ExecutionService]
-      when(execution.stopWorker(any[String])).thenSuccess(())
-
-      val route = HttpV2Routes.workerRoutes(execution)
-
-      Delete("/v2/api/workers/id") ~> route ~> check {
-        status shouldBe StatusCodes.OK
-      }
-    }
-
-    it("should get full worker info") {
-      val execution = mock[ExecutionService]
-      when(execution.getWorkerLink(any[String]))
-        .thenReturn(Some(WorkerLink(
-          "id", "test", None,
-          WorkerInitInfo(Map(), Duration.Inf, Duration.Inf, "test", "localhost:0", "localhost:0", 262144000, ""))))
-
-      val route = HttpV2Routes.workerRoutes(execution)
-
-      Get("/v2/api/workers/id") ~> route ~> check {
-        status shouldBe StatusCodes.OK
-        val resp = responseAs[WorkerLink]
-        resp.name shouldBe "id"
-        resp.initInfo shouldBe WorkerInitInfo(Map(), Duration.Inf, Duration.Inf, "test", "localhost:0","localhost:0", 262144000, "")
-        resp.sparkUi should not be defined
-        resp.address shouldBe "test"
-      }
-    }
-
-    it("should return worker jobs") {
-      val execution = mock[ExecutionService]
-      when(execution.getHistory(any[JobDetailsRequest])).thenSuccess(JobDetailsResponse(
-        Seq(JobDetails("id", "1",
-          JobParams("path", "className", JsMap.empty, Action.Execute),
-          "context", None, JobDetails.Source.Http)
-        ), 1
-      ))
-
-      val route = HttpV2Routes.workerRoutes(execution)
-
-      Get("/v2/api/workers/id/jobs?status=started") ~> route ~> check {
-        status shouldBe StatusCodes.OK
-        val jobs = responseAs[Seq[JobDetails]]
-        jobs.size shouldBe 1
-      }
-      Get("/v2/api/workers/id/jobs?status=started&paginate=true") ~> route ~> check {
-        status shouldBe StatusCodes.OK
-        val rsp = responseAs[JobDetailsResponse]
-        rsp.jobs.size shouldBe 1
-      }
-    }
-
-  }
+//  describe("workers") {
+//
+//    it("should return workers") {
+//      val execution = mock[ExecutionService]
+//      when(execution.workers()).thenReturn(Seq(workerLinkData))
+//
+//      val route = HttpV2Routes.workerRoutes(execution)
+//
+//      Get("/v2/api/workers") ~> route ~> check {
+//        status shouldBe StatusCodes.OK
+//        val rsp = responseAs[Seq[WorkerLink]]
+//        rsp.size shouldBe 1
+//      }
+//    }
+//
+//    it("should stop worker") {
+//      val execution = mock[ExecutionService]
+//      when(execution.stopWorker(any[String])).thenSuccess(())
+//
+//      val route = HttpV2Routes.workerRoutes(execution)
+//
+//      Delete("/v2/api/workers/id") ~> route ~> check {
+//        status shouldBe StatusCodes.OK
+//      }
+//    }
+//
+//    it("should get full worker info") {
+//      val execution = mock[ExecutionService]
+//      when(execution.getWorkerLink(any[String]))
+//        .thenReturn(Some(WorkerLink(
+//          "id", "test", None,
+//          WorkerInitInfo(Map(), Duration.Inf, Duration.Inf, "test", "localhost:0", "localhost:0", 262144000, ""))))
+//
+//      val route = HttpV2Routes.workerRoutes(execution)
+//
+//      Get("/v2/api/workers/id") ~> route ~> check {
+//        status shouldBe StatusCodes.OK
+//        val resp = responseAs[WorkerLink]
+//        resp.name shouldBe "id"
+//        resp.initInfo shouldBe WorkerInitInfo(Map(), Duration.Inf, Duration.Inf, "test", "localhost:0","localhost:0", 262144000, "")
+//        resp.sparkUi should not be defined
+//        resp.address shouldBe "test"
+//      }
+//    }
+//
+//    it("should return worker jobs") {
+//      val execution = mock[ExecutionService]
+//      when(execution.getHistory(any[JobDetailsRequest])).thenSuccess(JobDetailsResponse(
+//        Seq(JobDetails("id", "1",
+//          JobParams("path", "className", JsMap.empty, Action.Execute),
+//          "context", None, JobDetails.Source.Http)
+//        ), 1
+//      ))
+//
+//      val route = HttpV2Routes.workerRoutes(execution)
+//
+//      Get("/v2/api/workers/id/jobs?status=started") ~> route ~> check {
+//        status shouldBe StatusCodes.OK
+//        val jobs = responseAs[Seq[JobDetails]]
+//        jobs.size shouldBe 1
+//      }
+//      Get("/v2/api/workers/id/jobs?status=started&paginate=true") ~> route ~> check {
+//        status shouldBe StatusCodes.OK
+//        val rsp = responseAs[JobDetailsResponse]
+//        rsp.jobs.size shouldBe 1
+//      }
+//    }
+//
+//  }
 
   describe("functions") {
 

From 45fd5c940c0296f741b1278ecc77ba2f927ab723 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 27 Sep 2018 03:07:23 +0300
Subject: [PATCH 19/30] aws - agent launching wip

---
 cloud_install/aws/install.sh                  |  2 +-
 mist.sbt                                      | 22 ++++++++++
 .../src/main/resources/agent.conf             |  0
 .../agent/src/main/resources/log4j.properties |  5 +++
 .../mist/agent}/ClusterAgent.scala            | 11 +++--
 .../mist/agent/EMRTermination.scala           | 34 ++++++++++++++++
 .../src/main/resources/log4j.properties       |  6 +++
 .../scala/io/hydrosphere/mist/aws/Main.scala  |  2 +-
 .../io/hydrosphere/mist/agent/Agent.scala     |  8 ----
 .../mist/agent/MasterConnection.scala         | 11 -----
 .../hydrosphere/mist/utils/CFConverion.scala  | 32 +++++++++++++++
 .../execution/aws/AWSEMRClusterRunner.scala   |  2 +-
 .../master/execution/aws/AgentInstall.scala   | 40 ++++++++++++++++++-
 13 files changed, 146 insertions(+), 29 deletions(-)
 rename mist/{master => agent}/src/main/resources/agent.conf (100%)
 create mode 100644 mist/agent/src/main/resources/log4j.properties
 rename mist/{master/src/main/scala/io/hydrosphere/mist/master/execution/aws => agent/src/main/scala/io/hydrosphere/mist/agent}/ClusterAgent.scala (81%)
 create mode 100644 mist/agent/src/main/scala/io/hydrosphere/mist/agent/EMRTermination.scala
 create mode 100644 mist/aws-init-setup/src/main/resources/log4j.properties
 delete mode 100644 mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/Agent.scala
 delete mode 100644 mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/MasterConnection.scala
 create mode 100644 mist/common/src/main/scala/io/hydrosphere/mist/utils/CFConverion.scala

diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
index 6dfc3b7e0..57170afe0 100644
--- a/cloud_install/aws/install.sh
+++ b/cloud_install/aws/install.sh
@@ -39,7 +39,7 @@ mv spark-2.3.0-bin-hadoop2.7 spark
 sleep 30
 
 INSTANCE_ID=$(ec2metadata --instance-id)
-$JAVA -cp /opt/mist/utils/aws-init-setup.jar io.hydrosphere.mist.aws.Main $INSTANCE_ID $ACCESS_KEY_ID $ACCESS_KEY_SECRET $AWS_REGION /opt/mist/configs/default.conf ~/.mist_key.pub ~/.mist_key
+$JAVA -cp /opt/mist/utils/aws-init-setup.jar io.hydrosphere.mist.aws.Main $INSTANCE_ID $ACCESS_KEY_ID $ACCESS_KEY_SECRET $AWS_REGION /opt/mist/configs/default.conf ~/.mist_key.pub ~/.mist_key > /root/aws_setup.log
 
 SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
 
diff --git a/mist.sbt b/mist.sbt
index e45c45c38..bfa99a5a3 100644
--- a/mist.sbt
+++ b/mist.sbt
@@ -133,6 +133,27 @@ lazy val worker = project.in(file("mist/worker"))
     )
   )
 
+lazy val agent = project.in(file("mist/agent"))
+  .dependsOn(common % "compile->compile;test->test")
+  .settings(commonSettings: _*)
+  .settings(commonAssemblySettings: _*)
+  .enablePlugins(BuildInfoPlugin)
+  .settings(
+    name := "mist-agent",
+    scalacOptions ++= commonScalacOptions,
+    libraryDependencies ++= Library.Akka.base,
+    libraryDependencies ++= Seq(
+      Library.slf4jLog4j, Library.log4j, Library.log4jExtras,
+      Library.Akka.testKit % "test",
+
+      Library.awsSdkEMR,
+      Library.scalaTest % "test"
+    )
+  ).settings(
+  buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sparkVersion),
+  buildInfoPackage := "io.hydrosphere.mist"
+)
+
 lazy val awsInitSetup = project.in(file("mist/aws-init-setup"))
   .dependsOn(common % "compile->compile;test->test")
   .settings(commonSettings: _*)
@@ -168,6 +189,7 @@ lazy val root = project.in(file("."))
         CpFile("configs/logging").to("configs"),
         CpFile(assembly.in(master, assembly).value).as("mist-master.jar"),
         CpFile(assembly.in(worker, assembly).value).as("mist-worker.jar"),
+        CpFile(assembly.in(agent,  assembly).value).as("mist-agent.jar"),
         MkDir("utils"),
         CpFile(assembly.in(awsInitSetup, assembly).value).as("aws-init-setup.jar").to("utils"),
         CpFile(Ui.ui.value).as("ui")
diff --git a/mist/master/src/main/resources/agent.conf b/mist/agent/src/main/resources/agent.conf
similarity index 100%
rename from mist/master/src/main/resources/agent.conf
rename to mist/agent/src/main/resources/agent.conf
diff --git a/mist/agent/src/main/resources/log4j.properties b/mist/agent/src/main/resources/log4j.properties
new file mode 100644
index 000000000..3d9d43a4e
--- /dev/null
+++ b/mist/agent/src/main/resources/log4j.properties
@@ -0,0 +1,5 @@
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/ClusterAgent.scala b/mist/agent/src/main/scala/io/hydrosphere/mist/agent/ClusterAgent.scala
similarity index 81%
rename from mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/ClusterAgent.scala
rename to mist/agent/src/main/scala/io/hydrosphere/mist/agent/ClusterAgent.scala
index 7db859aab..a247849d6 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/ClusterAgent.scala
+++ b/mist/agent/src/main/scala/io/hydrosphere/mist/agent/ClusterAgent.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.master.execution.aws
+package io.hydrosphere.mist.agent
 
 import akka.actor.{ActorRef, ActorSystem}
 import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
@@ -7,7 +7,6 @@ import io.hydrosphere.mist.utils.NetUtils
 import io.hydrosphere.mist.utils.akka.{ActorRegHub, WhenTerminated}
 
 import scala.concurrent.Await
-import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
 
 object ClusterAgent {
@@ -20,14 +19,14 @@ object ClusterAgent {
     val region = args(4)
     val awsId = args(5)
 
-    val emrClient = EMRClient.create(accessKey, secretKey, region)
+    val termination = EMRTermination.create(accessKey, secretKey, region)
 
-    val hostname = NetUtils.findLocalInetAddress()
+    val hostname = NetUtils.findLocalInetAddress().getHostAddress
 
     val config = ConfigFactory.load("agent")
       .withValue("akka.remote.netty.tcp.hostname", ConfigValueFactory.fromAnyRef(hostname))
 
-    implicit val system = ActorSystem("mist-info-provider", config)
+    implicit val system = ActorSystem("mist-agent", config)
 
     def resolveRemote(path: String): ActorRef = {
       val ref = system.actorSelection(path).resolveOne(10 seconds)
@@ -51,7 +50,7 @@ object ClusterAgent {
 
     WhenTerminated(heathRef, {
       println("Remote system was terminated, shutdown cluster")
-      emrClient.stop(awsId).unsafeRunSync()
+      Await.result(termination.terminate(awsId), Duration.Inf)
       system.terminate()
     })
 
diff --git a/mist/agent/src/main/scala/io/hydrosphere/mist/agent/EMRTermination.scala b/mist/agent/src/main/scala/io/hydrosphere/mist/agent/EMRTermination.scala
new file mode 100644
index 000000000..189409010
--- /dev/null
+++ b/mist/agent/src/main/scala/io/hydrosphere/mist/agent/EMRTermination.scala
@@ -0,0 +1,34 @@
+package io.hydrosphere.mist.agent
+
+import software.amazon.awssdk.services.emr.EMRAsyncClient
+import software.amazon.awssdk.services.emr.model.TerminateJobFlowsRequest
+
+import scala.concurrent.Future
+import scala.concurrent.ExecutionContext.Implicits.global
+import io.hydrosphere.mist.utils.CFConversion._
+import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider}
+import software.amazon.awssdk.regions.Region
+
+class EMRTermination(client: EMRAsyncClient) {
+
+  def terminate(id: String): Future[Unit] = {
+    val req = TerminateJobFlowsRequest.builder().jobFlowIds(id).build()
+    client.terminateJobFlows(req).toFuture.map(_ => ())
+  }
+
+}
+
+object EMRTermination {
+
+  def create(accessKey: String, secretKey: String, region: String): EMRTermination = {
+    val credentials = AwsCredentials.create(accessKey, secretKey)
+    val provider = StaticCredentialsProvider.create(credentials)
+    val reg = Region.of(region)
+    val emrClient = EMRAsyncClient.builder()
+      .credentialsProvider(provider)
+      .region(reg)
+      .build()
+
+    new EMRTermination(emrClient)
+  }
+}
diff --git a/mist/aws-init-setup/src/main/resources/log4j.properties b/mist/aws-init-setup/src/main/resources/log4j.properties
new file mode 100644
index 000000000..49025057f
--- /dev/null
+++ b/mist/aws-init-setup/src/main/resources/log4j.properties
@@ -0,0 +1,6 @@
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
index d761f84ab..9d08e1b24 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
@@ -22,7 +22,7 @@ object Main {
 
     val launchData = LaunchData(
       sshKeyPair = out.sshKeyPairName,
-      sshKeyPath = sshKeyPathPub,
+      sshKeyPath = sshKeyPath,
       accessKey = accessKey,
       secretKey = accessSecret,
       subnetId = out.subnetId,
diff --git a/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/Agent.scala b/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/Agent.scala
deleted file mode 100644
index 53cc5faaa..000000000
--- a/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/Agent.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-package io.hydrosphere.mist.agent
-
-object Agent {
-
-  def main(args: Array[String]): Unit = {
-    sys.exit(1)
-  }
-}
diff --git a/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/MasterConnection.scala b/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/MasterConnection.scala
deleted file mode 100644
index 96cf74647..000000000
--- a/mist/cluster-agent/src/main/scala/io/hydrosphere/mist/agent/MasterConnection.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package io.hydrosphere.mist.agent
-
-import akka.actor.{Actor, ActorLogging}
-
-class MasterConnection(
-  id: String
-) extends Actor with ActorLogging {
-
-  override def receive: Receive = ???
-
-}
diff --git a/mist/common/src/main/scala/io/hydrosphere/mist/utils/CFConverion.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/CFConverion.scala
new file mode 100644
index 000000000..b53136134
--- /dev/null
+++ b/mist/common/src/main/scala/io/hydrosphere/mist/utils/CFConverion.scala
@@ -0,0 +1,32 @@
+package io.hydrosphere.mist.utils
+
+import java.util.concurrent.CompletableFuture
+import java.util.function.{BiConsumer, BiFunction}
+
+import scala.concurrent._
+
+final class CompletableFutureOps[A](val cf: CompletableFuture[A]) extends AnyVal {
+
+  def toFuture: Future[A] = {
+    val p = Promise[A]
+    cf.whenComplete(new BiConsumer[A, Throwable] {
+      override def accept(res: A, err: Throwable): Unit = {
+        (Option(res), Option(err)) match {
+          case (Some(r), None) => p.success(r)
+          case (_, Some(e)) =>
+            e match {
+              case ce: java.util.concurrent.CompletionException => p.failure(ce.getCause)
+              case _ => p.failure(e)
+            }
+          case (_, _) => p.failure(new IllegalStateException("CompletableFuture was failed without error information"))
+        }
+      }})
+    p.future
+  }
+
+
+}
+
+object CFConversion {
+  implicit def cfSyntax[A](cf: CompletableFuture[A]): CompletableFutureOps[A] = new CompletableFutureOps(cf)
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
index 08eb31341..855415c24 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
@@ -133,7 +133,7 @@ object AWSEMRClusterRunner extends Logger {
     val installAgent = (host: String, agentId: String, awsId: String) => {
       val realDir = jarsDir.toAbsolutePath.toRealPath()
       val transfer = TransferParams(
-        realDir.resolve("mist-master.jar").toString,
+        realDir.resolve("mist-agent.jar").toString,
         realDir.resolve("mist-worker.jar").toString,
         s"/home/$sshUser/mist-agent"
       )
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala
index 2a361979c..0c4e0a93a 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala
@@ -1,5 +1,7 @@
 package io.hydrosphere.mist.master.execution.aws
 
+import java.nio.file.Paths
+
 case class TransferParams(
   agentJar: String,
   workerJar: String,
@@ -25,12 +27,13 @@ object AgentInstall {
     import transferParams._
     import agentRunParams._
 
+    val agentRemoteJar = s"$targetDir/mist-agent.jar"
     Seq(
       SSHCmd.Exec(Seq("mkdir", targetDir)),
       SSHCmd.CopyFile(agentJar, s"$targetDir/mist-agent.jar"),
       SSHCmd.CopyFile(workerJar, s"$targetDir/mist-worker.jar"),
       SSHCmd.Exec(Seq(
-        "java", "-cp", "~/mist-agent/mist-master.jar", "io.hydrosphere.mist.master.execution.aws.ClusterAgent",
+        "java", "-cp", agentRemoteJar, "io.hydrosphere.mist.agent.ClusterAgent",
         masterAddress, agentId, accessKey, secretKey, region, clusterId,
         s"1>$targetDir/out.log", s"2>$targetDir/out.log", "&"
       ))
@@ -38,3 +41,38 @@ object AgentInstall {
   }
 
 }
+
+object Test {
+
+  def main(args: Array[String]): Unit = {
+    val jarsDir = Paths.get("/home/dos65/projects/mist/target/mist-1.0.0-RC17")
+    val realDir = jarsDir.toAbsolutePath.toRealPath()
+
+    val host = "ec2-18-196-156-203.eu-central-1.compute.amazonaws.com"
+    val sshUser = "hadoop"
+    val agentId= "j-2NVSS1JAKYOEU"
+
+    val akkaAddress = "blabla.com:2005"
+
+    val accessKey = "AKIA5XGMCUEIJ3NOW6Y3"
+    val secretKey = "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"
+    val region = "eu-central-1"
+
+
+    val transfer = TransferParams(
+      realDir.resolve("mist-agent.jar").toString,
+      realDir.resolve("mist-worker.jar").toString,
+      s"/home/$sshUser/mist-agent"
+    )
+    val agentRunParams = AgentRunParams(
+      agentId, akkaAddress, accessKey, secretKey, region, agentId
+    )
+    val cmds = AgentInstall.sshCommands(transfer, agentRunParams)
+    val out = new SSHClient(host, sshUser, "/home/dos65/.ssh/mist_key").install(cmds)
+
+    out match {
+      case scala.util.Failure(exception) => exception.printStackTrace()
+      case scala.util.Success(value) => println("OOK")
+    }
+  }
+}

From de62967f8e827d0991bde8cfe3e0c460ee8271d3 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 27 Sep 2018 03:08:50 +0300
Subject: [PATCH 20/30] aws - nginx config

---
 cloud_install/aws/install.sh | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
index 57170afe0..c51e70d01 100644
--- a/cloud_install/aws/install.sh
+++ b/cloud_install/aws/install.sh
@@ -46,29 +46,28 @@ SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
 htpasswd -b -c /etc/nginx/.htpasswd $MIST_LOGIN $MIST_PASSWORD
 cat << EOF > /etc/nginx/sites-enabled/default
 server {
-	listen 80 default_server;
-	listen [::]:80 default_server;
+    listen 80 default_server;
+    listen [::]:80 default_server;
 
-	root /var/www/html;
-	index index.html index.htm index.nginx-debian.html;
+    root /var/www/html;
+    index index.html index.htm index.nginx-debian.html;
 
-	server_name _;
+    server_name _;
 
-	location / {
-	   proxy_pass http://127.0.0.1:2004/;
-     auth_basic "Restricted";
-     auth_basic_user_file /etc/nginx/.htpasswd;
+    location / {
+        proxy_pass http://127.0.0.1:2004/;
+        auth_basic "Restricted";
+        auth_basic_user_file /etc/nginx/.htpasswd;
 	}
 
 	location /v2/api/ws {
-	    proxy_pass         http://127.0.0.1:2004/v2/api/ws;
-	    proxy_http_version 1.1;
-      proxy_set_header Upgrade \$http_upgrade;
-      proxy_set_header Connection "upgrade";
-      auth_basic "Restricted";
-      auth_basic_user_file /etc/nginx/.htpasswd;
-	}
-
+        proxy_pass http://127.0.0.1:2004/v2/api/ws;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade \$http_upgrade;
+        proxy_set_header Connection "upgrade";
+        auth_basic "Restricted";
+        auth_basic_user_file /etc/nginx/.htpasswd;
+    }
 }
 EOF
 service nginx restart

From c71bd12f5fadb18b9497116194c6c90fa99a4ac7 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 27 Sep 2018 03:24:22 +0300
Subject: [PATCH 21/30] context frontentd - fix connector starting retries

---
 .../mist/master/execution/ContextFrontend.scala            | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index b96ebee3d..4c7e74535 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -332,10 +332,11 @@ class ContextFrontend(
     case Event.ConnectorStarted(_, conn) => conn.shutdown(true)
     case Event.ConnectorStartFailed(id, e) if id == connId =>
       log.error(e, "Starting connector {} id: {} failed", name, id)
-      if (failedTimes > ctx.maxConnFailures) {
-        becomeSleeping(state, ctx, e, failedTimes)
+      val nextFailed = failedTimes + 1
+      if (nextFailed > ctx.maxConnFailures) {
+        becomeSleeping(state, ctx, e, nextFailed)
       } else {
-        gotoStartingConnector(ctx, state, failedTimes + 1)
+        gotoStartingConnector(ctx, state, nextFailed)
       }
   }
 

From c4243bc8c940036bae7f0f2c95f5e48ac35f8d06 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 27 Sep 2018 21:31:14 +0300
Subject: [PATCH 22/30] aws setup - fix internal sec group inbound rules

---
 cloud_install/aws/template.json               |  7 +--
 mist/agent/src/main/resources/agent.conf      |  1 -
 .../io/hydrosphere/mist/aws/AwsSetup.scala    | 11 ++--
 .../io/hydrosphere/mist/aws/EC2Service.scala  | 54 ++++++++++++++-----
 .../master/execution/ContextFrontend.scala    |  2 +-
 5 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/cloud_install/aws/template.json b/cloud_install/aws/template.json
index 0ae8c44d2..788b0bf29 100644
--- a/cloud_install/aws/template.json
+++ b/cloud_install/aws/template.json
@@ -83,9 +83,10 @@
     }
   },
   "Outputs" : {
-    "PublicIp" : {
-      "Value" : { "Fn::GetAtt" : [ "MistEc2Instance", "PublicIp" ]},
-      "Description" : "PublicIp Address"
+    "PublicDns" : {
+      "Value" : { "Fn::GetAtt" : [ "MistEc2Instance", "PublicDnsName" ]},
+      "Description" : "Public Dns name"
     }
+
   }
 }
diff --git a/mist/agent/src/main/resources/agent.conf b/mist/agent/src/main/resources/agent.conf
index 687ec9a10..a2c48ca2c 100644
--- a/mist/agent/src/main/resources/agent.conf
+++ b/mist/agent/src/main/resources/agent.conf
@@ -11,7 +11,6 @@ akka {
 
   remote {
     netty.tcp {
-      bind-hostname = "0.0.0.0"
       port = 0
       maximum-frame-size = 5242880b
     }
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
index 3857c3135..0b03d1bc0 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
@@ -45,9 +45,13 @@ object AwsSetup {
           ec2EMrInstaceProfile <- iam.getOrCreateInstanceProfile(ec2EmrRole.name, ec2EmrRole.name)
           emrRole <- iam.getOrCreateRole(emrRole)
 
-          secGroupData = IngressData(0, 65535, data.cidrIp, "TCP")
-          secGroupId <- ec2.getOrCreateSecGroup(secGroupName(instanceId), secGroupDecr, data.vpcId, secGroupData)
-          _ <- ec2.addIngressRule(data.secGroupIds.head, IngressData(0, 65535, data.cidrIp, "TCP"))
+          secGroupData = IngressData(0, 65535, IngressAddr.CidrIP(data.cidrIp), "TCP")
+          scGroupName = secGroupName(instanceId)
+
+          internalSecGroup <- ec2.getOrCreateSecGroup(scGroupName, secGroupDecr, data.vpcId, secGroupData)
+          secGroupId  = internalSecGroup.id
+          _ <- ec2.addIngressRule(data.secGroupIds.head, IngressData(0, 65535, IngressAddr.Group(internalSecGroup), "TCP"))
+
           keyName <- ec2.getOrCreateKeyPair(keyName(instanceId), sshKey)
         } yield SetupData(data.subnetId, secGroupId, emrRole.name, ec2EmrRole.name, keyName)
       }
@@ -73,4 +77,3 @@ object AwsSetup {
     default(iam, ec2)
   }
 }
-
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
index 6c7dca64b..143429464 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
@@ -9,13 +9,21 @@ import software.amazon.awssdk.services.ec2.model._
 import scala.collection.JavaConverters._
 import JFutureSyntax._
 
+sealed trait IngressAddr
+object IngressAddr {
+  final case class CidrIP(value: String) extends IngressAddr
+  final case class Group(value: SecGroup) extends IngressAddr
+}
+
 case class IngressData(
   fromPort: Int,
   toPort: Int,
-  cidrIp: String,
+  address: IngressAddr,
   protocol: String
 )
 
+case class SecGroup(id: String, name: String)
+
 case class InstanceData(
   ip: String,
   vpcId: String,
@@ -44,14 +52,14 @@ trait EC2Service[F[_]] {
     } yield out
   }
 
-  def getSecGroup(name: String): F[Option[String]]
-  def createSecGroup(name: String, descr: String, vpcId: String, data: IngressData): F[String]
+  def getSecGroup(name: String): F[Option[SecGroup]]
+  def createSecGroup(name: String, descr: String, vpcId: String, data: IngressData): F[SecGroup]
 
-  def getOrCreateSecGroup(name: String, descr: String, vpcId: String, data: IngressData)(implicit M: Monad[F]): F[String] = {
+  def getOrCreateSecGroup(name: String, descr: String, vpcId: String, data: IngressData)(implicit M: Monad[F]): F[SecGroup] = {
     for {
       curr <- getSecGroup(name)
       out <- curr match {
-        case Some(_) => M.pure(name)
+        case Some(v) => M.pure(v)
         case None => createSecGroup(name, descr, vpcId, data)
       }
     } yield out
@@ -65,22 +73,40 @@ object EC2Service {
     new EC2Service[IO] {
 
       override def addIngressRule(groupId: String, data: IngressData): IO[Unit] = {
+        println(s"HERE: $groupId $data")
         import data._
-        val req = AuthorizeSecurityGroupIngressRequest.builder()
-          .groupId(groupId)
-          .cidrIp(cidrIp)
+        val ipPremission = IpPermission.builder()
           .fromPort(fromPort)
           .toPort(toPort)
-          .ipProtocol(data.protocol)
+          .ipProtocol(protocol)
+
+        val withAddr = address match {
+          case IngressAddr.CidrIP(v) =>
+            val range = IpRange.builder().cidrIp(v).build()
+            ipPremission.ipv4Ranges(range)
+          case IngressAddr.Group(v) =>
+            val group = UserIdGroupPair.builder()
+              .groupId(v.id)
+              .groupName(v.name)
+              .description(s"${v.id}:${v.name}")
+              .build()
+            ipPremission.userIdGroupPairs(group)
+        }
+
+        val req = AuthorizeSecurityGroupIngressRequest.builder()
+          .groupId(groupId)
+          .ipPermissions(withAddr.build())
           .build()
-        ec2Client.authorizeSecurityGroupIngress(req).toIO.map(_ => ())
+
+        println(req)
+        ec2Client.authorizeSecurityGroupIngress(req).toIO.map(r => {println(r)})
       }
 
-      override def getSecGroup(name: String): IO[Option[String]] = {
+      override def getSecGroup(name: String): IO[Option[SecGroup]] = {
         val req = DescribeSecurityGroupsRequest.builder().groupNames(name).build()
         val io = ec2Client.describeSecurityGroups(req).toIO
 
-        io.map(r => r.securityGroups().asScala.headOption.map(_.groupName()))
+        io.map(r => r.securityGroups().asScala.headOption.map(s => SecGroup(s.groupId(), s.groupName())))
           .handleErrorWith(e => e match {
             case _: software.amazon.awssdk.services.ec2.model.EC2Exception => IO.pure(None)
             case _ => IO.raiseError(e)
@@ -92,7 +118,7 @@ object EC2Service {
         descr: String,
         vpcId: String,
         data: IngressData
-      ): IO[String] = {
+      ): IO[SecGroup] = {
 
         val createReq = CreateSecurityGroupRequest.builder()
           .groupName(name)
@@ -104,7 +130,7 @@ object EC2Service {
           resp <- ec2Client.createSecurityGroup(createReq).toIO
           groupId = resp.groupId()
           _ <- addIngressRule(groupId, data)
-        } yield groupId
+        } yield SecGroup(resp.groupId, name)
       }
 
       override def getInstanceData(id: String): IO[Option[InstanceData]] = {
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index 4c7e74535..d1269be72 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -333,7 +333,7 @@ class ContextFrontend(
     case Event.ConnectorStartFailed(id, e) if id == connId =>
       log.error(e, "Starting connector {} id: {} failed", name, id)
       val nextFailed = failedTimes + 1
-      if (nextFailed > ctx.maxConnFailures) {
+      if (nextFailed >=  ctx.maxConnFailures) {
         becomeSleeping(state, ctx, e, nextFailed)
       } else {
         gotoStartingConnector(ctx, state, nextFailed)

From 8dd98189734ed27bec25a818b25f8d6de452844e Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 27 Sep 2018 23:05:55 +0300
Subject: [PATCH 23/30] remove something

---
 .../master/execution/aws/AgentInstall.scala   | 35 -------------------
 1 file changed, 35 deletions(-)

diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala
index 0c4e0a93a..f41a25329 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala
@@ -41,38 +41,3 @@ object AgentInstall {
   }
 
 }
-
-object Test {
-
-  def main(args: Array[String]): Unit = {
-    val jarsDir = Paths.get("/home/dos65/projects/mist/target/mist-1.0.0-RC17")
-    val realDir = jarsDir.toAbsolutePath.toRealPath()
-
-    val host = "ec2-18-196-156-203.eu-central-1.compute.amazonaws.com"
-    val sshUser = "hadoop"
-    val agentId= "j-2NVSS1JAKYOEU"
-
-    val akkaAddress = "blabla.com:2005"
-
-    val accessKey = "AKIA5XGMCUEIJ3NOW6Y3"
-    val secretKey = "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"
-    val region = "eu-central-1"
-
-
-    val transfer = TransferParams(
-      realDir.resolve("mist-agent.jar").toString,
-      realDir.resolve("mist-worker.jar").toString,
-      s"/home/$sshUser/mist-agent"
-    )
-    val agentRunParams = AgentRunParams(
-      agentId, akkaAddress, accessKey, secretKey, region, agentId
-    )
-    val cmds = AgentInstall.sshCommands(transfer, agentRunParams)
-    val out = new SSHClient(host, sshUser, "/home/dos65/.ssh/mist_key").install(cmds)
-
-    out match {
-      case scala.util.Failure(exception) => exception.printStackTrace()
-      case scala.util.Success(value) => println("OOK")
-    }
-  }
-}

From a942a77b9c4c0d33de0d25459bb254577a5d6f01 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Fri, 28 Sep 2018 11:36:45 +0300
Subject: [PATCH 24/30] aws setup - wip - it works!

---
 configs/logging/log4j.debug.properties        |   2 +
 configs/logging/log4j.default.properties      |   4 +-
 .../src/main/scala/SparkSessionExample.scala  |   1 +
 .../mist/master/execution/Cluster.scala       |   7 -
 .../master/execution/ClustersService.scala    |  18 +-
 .../master/execution/ContextFrontend.scala    |   1 -
 .../execution/aws/AWSEMRClusterRunner.scala   | 158 ------------------
 .../execution/aws/EMRClusterRunner.scala      | 136 +++++++++++++++
 .../mist/master/execution/aws/EMRStatus.scala |   1 +
 .../mist/master/execution/aws/IOTimer.scala   |  39 +++++
 .../mist/master/execution/aws/SSHClient.scala |   9 +-
 .../workers/ExclusiveConnector.scala          |   3 -
 .../execution/workers/SharedConnector.scala   |  13 +-
 .../execution/workers/WorkerBridge.scala      |   2 +-
 .../execution/workers/WorkerConnection.scala  |   2 +-
 .../mist/master/interfaces/jsonCodecs.scala   |   3 +-
 .../execution.scala}                          |   7 +-
 .../io/hydrosphere/mist/master/TestData.scala |   2 +-
 .../execution/ContextFrontendSpec.scala       |  25 ---
 .../execution/workers/ClusterSpec.scala       |   3 -
 .../workers/SharedConnectorSpec.scala         |   4 +-
 .../interfaces/http/HttpApiV2Spec.scala       |   2 +-
 .../mist/worker/MasterBridgeSpec.scala        |   2 +-
 23 files changed, 216 insertions(+), 228 deletions(-)
 delete mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
 create mode 100644 mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/IOTimer.scala
 rename mist/master/src/main/scala/io/hydrosphere/mist/master/{execution/WorkerLink.scala => models/execution.scala} (58%)

diff --git a/configs/logging/log4j.debug.properties b/configs/logging/log4j.debug.properties
index d2379bd50..a24791123 100644
--- a/configs/logging/log4j.debug.properties
+++ b/configs/logging/log4j.debug.properties
@@ -9,3 +9,5 @@ log4j.appender.file.File=${mist.home}/logs/mist.log
 log4j.appender.file.append=true
 log4j.appender.file.layout=org.apache.log4j.PatternLayout
 log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+log4j.logger.net.schmizz.sshj=WARN
diff --git a/configs/logging/log4j.default.properties b/configs/logging/log4j.default.properties
index 19d3737b0..02aa6b22f 100644
--- a/configs/logging/log4j.default.properties
+++ b/configs/logging/log4j.default.properties
@@ -4,4 +4,6 @@ log4j.appender.file=org.apache.log4j.FileAppender
 log4j.appender.file.File=${mist.home}/logs/mist.log
 log4j.appender.file.append=true
 log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
\ No newline at end of file
+log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+log4j.logger.net.schmizz.sshj=WARN
diff --git a/examples/examples/src/main/scala/SparkSessionExample.scala b/examples/examples/src/main/scala/SparkSessionExample.scala
index bd0cc67e1..368d4772c 100644
--- a/examples/examples/src/main/scala/SparkSessionExample.scala
+++ b/examples/examples/src/main/scala/SparkSessionExample.scala
@@ -11,6 +11,7 @@ object SparkSessionExample extends MistFn {
       arg[Int]("multiplier", 2)
     ).withMistExtras
       .onSparkSession((nums: Seq[Int], mult: Int, extras: MistExtras, spark: SparkSession) => {
+        spark.read.parquet("1", "2")
         spark.sparkContext.parallelize(nums).map(_ * mult).collect()
       })
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala
index a4b8aa1a5..0692a2936 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala
@@ -11,8 +11,6 @@ trait Cluster {
 
   def askConnection(): Future[PerJobConnection]
 
-  def warmUp(): Unit
-
   def shutdown(force: Boolean): Future[Unit]
 
   def whenTerminated(): Future[Unit]
@@ -26,7 +24,6 @@ object Cluster {
     final case class AskConnection(resolve: Promise[PerJobConnection]) extends Event
     final case class Released(conn: WorkerConnection) extends Event
     final case class Shutdown(force: Boolean) extends Event
-    case object WarmUp extends Event
     final case class ConnTerminated(connId: String) extends Event
     case object GetStatus
   }
@@ -48,10 +45,6 @@ object Cluster {
     }
 
     override def whenTerminated(): Future[Unit] = termination
-
-    override def warmUp(): Unit = underlying ! Cluster.Event.WarmUp
-
-
   }
 
   def actorBased(
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
index 9707bc6f9..0c8be255d 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.execution
 import java.nio.file.Paths
 
 import akka.actor.ActorSystem
-import io.hydrosphere.mist.master.execution.aws.AWSEMRClusterRunner
+import io.hydrosphere.mist.master.execution.aws.EMRClusterRunner
 import io.hydrosphere.mist.master.{AWSEMRLaunchSettings, LauncherSettings}
 import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig, ServerDefault}
 import io.hydrosphere.mist.utils.Logger
@@ -28,21 +28,19 @@ object ClustersService extends Logger{
     system: ActorSystem
   ): ClustersService = {
 
-    val runners = launchSettings.map({case (name, settings) => {
-      val runner = settings match {
-        case aws: AWSEMRLaunchSettings =>
-          AWSEMRClusterRunner.create(Paths.get(mistHome), spawn, aws, regHub, system)
-      }
-      name -> runner
-    }})
-
     new ClustersService {
 
+      val runners = launchSettings.map({case (name, settings) => {
+        val runner = settings match {
+          case aws: AWSEMRLaunchSettings => EMRClusterRunner.create(Paths.get(mistHome), spawn, aws, regHub, system)
+        }
+        name -> runner
+      }})
+
       override def start(id: String, ctx: ContextConfig): Future[Cluster] = {
         ctx.launchData match {
           case ServerDefault => serverDefault.run(id, ctx)
           case aws: AWSEMRLaunchData =>
-            logger.info(s"IN:${aws.launcherSettingsName} keys: ${runners.keys.toList}")
             runners.get(aws.launcherSettingsName) match {
               case Some(runner) => runner.run(id, ctx)
               case None => Future.failed(new RuntimeException(s"Unknown settings name ${aws.launcherSettingsName} for ctx ${ctx.name}"))
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
index d1269be72..73975e930 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala
@@ -315,7 +315,6 @@ class ContextFrontend(
 
     case Event.ConnectorStarted(id, conn) if id == connId =>
       log.info("Connector started {} id: {}", name, id)
-      if (ctx.precreated && ctx.workerMode == RunMode.Shared) conn.warmUp()
       val connState = ConnectorState.initial(id, conn).copy(failedTimes = failedTimes)
       conn.whenTerminated().onComplete({
         case Success(_) => self ! Event.ConnectorStopped(id)
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
deleted file mode 100644
index 855415c24..000000000
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AWSEMRClusterRunner.scala
+++ /dev/null
@@ -1,158 +0,0 @@
-package io.hydrosphere.mist.master.execution.aws
-
-import java.nio.file.{Path, Paths}
-import java.util.concurrent.Executors
-
-import akka.actor.ActorSystem
-import cats.effect._
-import cats.implicits._
-import io.hydrosphere.mist.common.CommonData
-import io.hydrosphere.mist.master.AWSEMRLaunchSettings
-import io.hydrosphere.mist.master.execution.workers.{StopAction, WorkerConnection, WorkerRunner}
-import io.hydrosphere.mist.master.execution.workers.starter.{SparkSubmitBuilder, WorkerProcess, WorkerStarter}
-import io.hydrosphere.mist.master.execution.{Cluster, ClusterRunner, SpawnSettings}
-import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig}
-import io.hydrosphere.mist.utils.Logger
-import io.hydrosphere.mist.utils.akka.ActorRegHub
-
-import scala.concurrent.duration._
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.Failure
-
-object AWSEMRClusterRunner extends Logger {
-
-  class EMRClusterRunner(
-    spawn: SpawnSettings,
-    launchSettings: AWSEMRLaunchSettings,
-    regHub: ActorRegHub,
-    system: ActorSystem,
-    installAgent: (String, String, String) => Unit,
-    client: EMRClient[IO]
-  ) extends ClusterRunner {
-
-    implicit val IOTimer = new Timer[IO] {
-
-      val ec = ExecutionContext.global
-      val sc = Executors.newScheduledThreadPool(1)
-
-      override val clock: Clock[IO] = new Clock[IO] {
-          override def realTime(unit: TimeUnit): IO[Long] =
-            IO(unit.convert(System.currentTimeMillis(), MILLISECONDS))
-
-          override def monotonic(unit: TimeUnit): IO[Long] =
-            IO(unit.convert(System.nanoTime(), NANOSECONDS))
-        }
-
-      override def sleep(timespan: FiniteDuration): IO[Unit] =
-        IO.cancelable { cb =>
-          val tick = new Runnable {
-            def run() = ec.execute(new Runnable {
-              def run() = cb(Right(()))
-            })
-          }
-          val f = sc.schedule(tick, timespan.length, timespan.unit)
-          IO(f.cancel(false))
-        }
-    }
-
-    private def extractData(ctx: ContextConfig): IO[AWSEMRLaunchData] = {
-      ctx.launchData match {
-        case data: AWSEMRLaunchData => IO.pure(data)
-        case other =>
-          val err = new IllegalArgumentException(s"Invalid launch data for AWSCluster ${other.getClass.getSimpleName}")
-          IO.raiseError(err)
-      }
-    }
-
-    private def mkRunSettings(name: String, data: AWSEMRLaunchData, settings: AWSEMRLaunchSettings): EMRRunSettings = {
-      EMRRunSettings(
-        name = name,
-        keyPair = settings.sshKeyPair,
-        releaseLabel = data.releaseLabel,
-        masterInstanceType = data.masterInstanceType,
-        slaveInstanceType = data.slaveInstanceType,
-        instanceCount = data.instanceCount,
-        subnetId = settings.subnetId,
-        additionalGroup = settings.additionalGroup,
-        emrRole = settings.emrRole,
-        emrEc2Role = settings.emrEc2Role
-      )
-    }
-
-    private def startFully(runSettings: EMRRunSettings, client: EMRClient[IO]): IO[EmrInfo] = {
-      for {
-        initial <- client.start(runSettings)
-        await <- client.awaitStatus(initial.id, EMRStatus.Started, 10 seconds, 40)
-      } yield await
-    }
-
-    private def mkRunner(launch: AWSEMRLaunchSettings, host: String): WorkerRunner = {
-      val starter: WorkerStarter = new WorkerStarter {
-        val builder = new SparkSubmitBuilder("~/mist-agent", "/usr/lib/spark")
-        override def onStart(name: String, initInfo: CommonData.WorkerInitInfo): WorkerProcess = {
-          val submitCmd = builder.submitWorker(name, initInfo) :+ "&"
-          Future {
-            new SSHClient(host, launch.sshUser, launch.sshKeyPath).install(Seq(SSHCmd.Exec(submitCmd)))
-          }
-          WorkerProcess.NonLocal
-        }
-
-        override def stopAction: StopAction = StopAction.Remote
-      }
-
-      WorkerRunner.default(spawn, starter, regHub, system)
-    }
-
-    override def run(id: String, ctx: ContextConfig): Future[Cluster] = {
-      val io = for {
-        data        <- extractData(ctx)
-        runSettings =  mkRunSettings(id, data, launchSettings)
-        emrInfo     <- startFully(runSettings, client)
-        agentId     = s"agent-${emrInfo.id}"
-        _           <- IO[Unit](installAgent(emrInfo.masterPublicDnsName, agentId, emrInfo.id))
-        _           <- IO.fromFuture(IO(regHub.waitRef(agentId, 1 minute)))
-        runner      = mkRunner(launchSettings, emrInfo.masterPublicDnsName)
-        cluster     = Cluster.actorBased(id, ctx, runner, system)
-      } yield cluster
-
-      io.unsafeToFuture()
-    }
-  }
-
-  def create(
-    jarsDir: Path,
-    spawn: SpawnSettings,
-    launchSettings: AWSEMRLaunchSettings,
-    regHub: ActorRegHub,
-    system: ActorSystem
-  ): ClusterRunner = {
-    import launchSettings._
-    val client = EMRClient.create(accessKey, secretKey, region)
-
-    val installAgent = (host: String, agentId: String, awsId: String) => {
-      val realDir = jarsDir.toAbsolutePath.toRealPath()
-      val transfer = TransferParams(
-        realDir.resolve("mist-agent.jar").toString,
-        realDir.resolve("mist-worker.jar").toString,
-        s"/home/$sshUser/mist-agent"
-      )
-      val agentRunParams = AgentRunParams(
-        agentId, spawn.akkaAddress, accessKey, secretKey, region, awsId
-      )
-      val cmds = AgentInstall.sshCommands(transfer, agentRunParams)
-      new SSHClient(host, sshUser, sshKeyPath).install(cmds)
-      ()
-    }
-
-    new EMRClusterRunner(
-      spawn,
-      launchSettings,
-      regHub,
-      system,
-      installAgent,
-      client
-    )
-  }
-}
-
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
new file mode 100644
index 000000000..7250e947a
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
@@ -0,0 +1,136 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import java.nio.file.{Path, Paths}
+import java.util.concurrent.Executors
+
+import akka.actor.ActorSystem
+import cats.effect._
+import cats.implicits._
+import io.hydrosphere.mist.common.CommonData
+import io.hydrosphere.mist.master.AWSEMRLaunchSettings
+import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, StopAction, WorkerConnection, WorkerRunner}
+import io.hydrosphere.mist.master.execution.workers.starter.{SparkSubmitBuilder, WorkerProcess, WorkerStarter}
+import io.hydrosphere.mist.master.execution.{Cluster, ClusterRunner, SpawnSettings}
+import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig}
+import io.hydrosphere.mist.utils.Logger
+import io.hydrosphere.mist.utils.akka.ActorRegHub
+
+import scala.concurrent.duration._
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Failure
+
+import IOTimer.Default
+
+class EMRClusterRunner(
+  spawn: SpawnSettings,
+  launchSettings: AWSEMRLaunchSettings,
+  regHub: ActorRegHub,
+  system: ActorSystem,
+  installAgent: (String, String, String) => Unit,
+  client: EMRClient[IO]
+) extends ClusterRunner {
+
+  private def extractData(ctx: ContextConfig): IO[AWSEMRLaunchData] = {
+    ctx.launchData match {
+      case data: AWSEMRLaunchData => IO.pure(data)
+      case other =>
+        val err = new IllegalArgumentException(s"Invalid launch data for AWSCluster ${other.getClass.getSimpleName}")
+        IO.raiseError(err)
+    }
+  }
+
+  private def mkRunSettings(name: String, data: AWSEMRLaunchData, settings: AWSEMRLaunchSettings): EMRRunSettings = {
+    EMRRunSettings(
+      name = name,
+      keyPair = settings.sshKeyPair,
+      releaseLabel = data.releaseLabel,
+      masterInstanceType = data.masterInstanceType,
+      slaveInstanceType = data.slaveInstanceType,
+      instanceCount = data.instanceCount,
+      subnetId = settings.subnetId,
+      additionalGroup = settings.additionalGroup,
+      emrRole = settings.emrRole,
+      emrEc2Role = settings.emrEc2Role
+    )
+  }
+
+  private def startFully(runSettings: EMRRunSettings, client: EMRClient[IO]): IO[EmrInfo] = {
+    for {
+      initial <- client.start(runSettings)
+      await <- client.awaitStatus(initial.id, EMRStatus.Started, 10 seconds, 40)
+    } yield await
+  }
+
+  private def mkRunner(launch: AWSEMRLaunchSettings, host: String): WorkerRunner = {
+    val starter: WorkerStarter = new WorkerStarter {
+      val builder = new SparkSubmitBuilder(s"/home/${launch.sshUser}/mist-agent", "/usr/lib/spark")
+      override def onStart(name: String, initInfo: CommonData.WorkerInitInfo): WorkerProcess = {
+        val submitCmd = builder.submitWorker(name, initInfo) :+ "&"
+        Future {
+          new SSHClient(host, launch.sshUser, launch.sshKeyPath).install(Seq(SSHCmd.Exec(submitCmd)))
+        }
+        WorkerProcess.NonLocal
+      }
+
+      override def stopAction: StopAction = StopAction.Remote
+    }
+
+    WorkerRunner.default(spawn, starter, regHub, system)
+  }
+
+  override def run(id: String, ctx: ContextConfig): Future[Cluster] = {
+    val io = for {
+      data        <- extractData(ctx)
+      runSettings =  mkRunSettings(id, data, launchSettings)
+      emrInfo     <- startFully(runSettings, client)
+      agentId     =  s"agent-${emrInfo.id}"
+      _           <- IO[Unit](installAgent(emrInfo.masterPublicDnsName, agentId, emrInfo.id))
+      _           <- IO.fromFuture(IO(regHub.waitRef(agentId, 1 minute)))
+      runner      =  mkRunner(launchSettings, emrInfo.masterPublicDnsName)
+      cluster     =  Cluster.actorBased(id, ctx, runner, system)
+      _           =  cluster.whenTerminated().onComplete(_ => client.stop(emrInfo.id))
+    } yield cluster
+
+    io.unsafeToFuture()
+  }
+}
+
+object EMRClusterRunner extends Logger {
+
+  def create(
+    jarsDir: Path,
+    spawn: SpawnSettings,
+    launchSettings: AWSEMRLaunchSettings,
+    regHub: ActorRegHub,
+    system: ActorSystem
+  ): ClusterRunner = {
+    import launchSettings._
+    val client = EMRClient.create(accessKey, secretKey, region)
+
+    val installAgent = (host: String, agentId: String, awsId: String) => {
+      val realDir = jarsDir.toAbsolutePath.toRealPath()
+      val transfer = TransferParams(
+        realDir.resolve("mist-agent.jar").toString,
+        realDir.resolve("mist-worker.jar").toString,
+        s"/home/$sshUser/mist-agent"
+      )
+      val agentRunParams = AgentRunParams(
+        agentId, spawn.akkaAddress, accessKey, secretKey, region, awsId
+      )
+      val cmds = AgentInstall.sshCommands(transfer, agentRunParams)
+      new SSHClient(host, sshUser, sshKeyPath).install(cmds)
+      ()
+    }
+
+    new EMRClusterRunner(
+      spawn,
+      launchSettings,
+      regHub,
+      system,
+      installAgent,
+      client
+    )
+  }
+}
+
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala
index 3dc1d176b..5ce183b0d 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala
@@ -1,6 +1,7 @@
 package io.hydrosphere.mist.master.execution.aws
 
 import software.amazon.awssdk.services.emr.{model => emodel }
+
 sealed trait EMRStatus
 
 object EMRStatus {
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/IOTimer.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/IOTimer.scala
new file mode 100644
index 000000000..7e8df1de3
--- /dev/null
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/IOTimer.scala
@@ -0,0 +1,39 @@
+package io.hydrosphere.mist.master.execution.aws
+
+import java.util.concurrent.{Executors, ScheduledExecutorService, TimeUnit}
+
+import cats.effect.{Clock, IO, Timer}
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration._
+
+class IOTimer(
+  ec: ExecutionContext,
+  sc: ScheduledExecutorService
+) extends Timer[IO] {
+
+  override val clock: Clock[IO] = new Clock[IO] {
+    override def realTime(unit: TimeUnit): IO[Long] =
+      IO(unit.convert(System.currentTimeMillis(), MILLISECONDS))
+
+    override def monotonic(unit: TimeUnit): IO[Long] =
+      IO(unit.convert(System.nanoTime(), NANOSECONDS))
+  }
+
+  override def sleep(timespan: FiniteDuration): IO[Unit] =
+    IO.cancelable { cb =>
+      val tick = new Runnable {
+        def run() = ec.execute(new Runnable {
+          def run() = cb(Right(()))
+        })
+      }
+      val f = sc.schedule(tick, timespan.length, timespan.unit)
+      IO(f.cancel(false))
+    }
+}
+
+object IOTimer {
+
+  implicit val Default = new IOTimer(ExecutionContext.global, Executors.newScheduledThreadPool(2))
+
+}
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
index badac9a73..d4926ea0f 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala
@@ -1,7 +1,6 @@
 package io.hydrosphere.mist.master.execution.aws
 
 import com.decodified.scalassh._
-import cats._
 import cats.implicits._
 
 import scala.util.Try
@@ -23,10 +22,10 @@ class SSHClient(host: String, user: String, keyPath: String) {
 
   def install(cmds: Seq[SSHCmd]): Try[Unit] = {
     SSH(host, cfgProvider) { client =>
-     cmds.map {
-       case SSHCmd.CopyFile(from, to) => client.upload(from, to)
-       case SSHCmd.Exec(cmd) => client.exec(Command(cmd.mkString(" "))).map(_ => ())
-     }.toList.sequence
+      cmds.toList.map {
+        case SSHCmd.CopyFile(from, to) => client.upload(from, to)
+        case SSHCmd.Exec(cmd) => client.exec(Command(cmd.mkString(" "))).map(_ => ())
+      }.combineAll
     }.map(_ => ())
   }
 }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
index 0ba3e69d4..874c9ce50 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala
@@ -52,9 +52,6 @@ class ExclusiveConnector(
       req.failure(e)
       context become process(other, working, startingConnections - 1)
 
-    case Event.WarmUp =>
-      log.warning("Exclusive connector {}: {} received warmup event", id, ctx.name)
-
     case Event.Released(conn) =>
       conn.shutdown(true)
       context become process(requests, working - conn.id, startingConnections)
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
index 47b8988cc..71bf793ee 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala
@@ -6,7 +6,7 @@ import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import akka.pattern.pipe
 import io.hydrosphere.mist.common.CommonData
 import io.hydrosphere.mist.common.CommonData.CancelJobRequest
-import io.hydrosphere.mist.master.models.ContextConfig
+import io.hydrosphere.mist.master.models.{ContextConfig, RunMode}
 import io.hydrosphere.mist.master.execution.Cluster
 
 import scala.collection.immutable.Queue
@@ -27,6 +27,13 @@ class SharedConnector(
     connectionStarter(connectionId)
   }
 
+  override def preStart(): Unit = {
+    if (ctx.precreated) {
+      (0 until ctx.maxJobs).foreach(_ => startConnection() pipeTo self)
+      context become process(Queue.empty, Queue.empty, Map.empty, ctx.maxJobs)
+    }
+  }
+
   override def receive: Receive = noConnection
 
   private def noConnection: Receive = {
@@ -35,10 +42,6 @@ class SharedConnector(
     case Event.AskConnection(req) =>
       startConnection() pipeTo self
       context become process(Queue(req), Queue.empty, Map.empty, 1)
-
-    case Event.WarmUp =>
-      (0 until ctx.maxJobs).foreach(_ => startConnection() pipeTo self)
-      context become process(Queue.empty, Queue.empty, Map.empty, ctx.maxJobs)
   }
 
   private def process(
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala
index c7e392cdd..a7b64d47c 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala
@@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.{Actor, ActorLogging, ActorRef, ActorRefFactory, Props, ReceiveTimeout, Terminated, Timers}
 import io.hydrosphere.mist.common.CommonData._
-import io.hydrosphere.mist.master.execution.WorkerLink
+import io.hydrosphere.mist.master.models.WorkerLink
 
 import scala.concurrent.{Future, Promise}
 import scala.concurrent.duration._
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnection.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnection.scala
index 7af836694..f18448842 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnection.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnection.scala
@@ -1,7 +1,7 @@
 package io.hydrosphere.mist.master.execution.workers
 
 import akka.actor.ActorRef
-import io.hydrosphere.mist.master.execution.WorkerLink
+import io.hydrosphere.mist.master.models.WorkerLink
 
 import scala.concurrent.Future
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
index ffcfd119a..6bcea6320 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
@@ -6,9 +6,8 @@ import java.time.format.{DateTimeFormatter, DateTimeParseException}
 import io.hydrosphere.mist.common.CommonData.{Action, JobParams, WorkerInitInfo}
 import io.hydrosphere.mist.common.logging.LogEvent
 import io.hydrosphere.mist.master.Messages.StatusMessages._
-import io.hydrosphere.mist.master.execution.WorkerLink
 import io.hydrosphere.mist.master.interfaces.http._
-import io.hydrosphere.mist.master.models._
+import io.hydrosphere.mist.master.models.{WorkerLink, _}
 import io.hydrosphere.mist.master.{JobDetails, JobDetailsResponse, JobResult}
 import mist.api.{data => mdata}
 import spray.json._
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/execution.scala
similarity index 58%
rename from mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala
rename to mist/master/src/main/scala/io/hydrosphere/mist/master/models/execution.scala
index a843d4411..475d4c28d 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/execution.scala
@@ -1,4 +1,4 @@
-package io.hydrosphere.mist.master.execution
+package io.hydrosphere.mist.master.models
 
 import io.hydrosphere.mist.common.CommonData.WorkerInitInfo
 
@@ -9,3 +9,8 @@ case class WorkerLink(
   initInfo: WorkerInitInfo
 )
 
+case class ClusterLink(
+  name: String,
+  info: String,
+  workers: Seq[WorkerLink]
+)
\ No newline at end of file
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
index 615f2a994..fc9444632 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala
@@ -3,7 +3,7 @@ package io.hydrosphere.mist.master
 import com.typesafe.config.ConfigFactory
 import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest, WorkerInitInfo}
 import io.hydrosphere.mist.common.FunctionInfoData
-import io.hydrosphere.mist.master.execution.WorkerLink
+import io.hydrosphere.mist.master.models.WorkerLink
 import mist.api.ArgInfo
 import mist.api.data._
 import mist.api.encoding.defaultEncoders._
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
index 5859c4bf4..a312d52e7 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
@@ -76,27 +76,6 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
     status3.jobs.isEmpty shouldBe true
   }
 
-  it("should warmup precreated") {
-    val connector = mock[Cluster]
-    when(connector.whenTerminated).thenReturn(Promise[Unit].future)
-
-    val job = mkJobProbe()
-    val props = ContextFrontend.props(
-      name = "name",
-      status = StatusReporter.NOOP,
-      loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => Future.successful(connector),
-      jobFactory = ActorF.static(job.ref),
-      defaultInactiveTimeout = 5 minutes
-    )
-    val frontend = TestActorRef[ContextFrontend](props)
-    frontend ! ContextEvent.UpdateContext(TestUtils.FooContext.copy(precreated = true))
-
-    eventually(timeout(Span(3, Seconds))) {
-      verify(connector).warmUp()
-    }
-  }
-
   it("should respect idle timeout - awaitRequest") {
     val connector = successfulConnector()
 
@@ -294,7 +273,6 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
         override def whenTerminated: Future[Unit] = ???
       })
       override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future
-      override def warmUp(): Unit = ()
     }
   }
 
@@ -303,7 +281,6 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       override def whenTerminated(): Future[Unit] = Promise[Unit].future
       override def askConnection(): Future[PerJobConnection] = Future.failed(FilteredException())
       override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future
-      override def warmUp(): Unit = ()
     }
   }
 
@@ -311,7 +288,6 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
     new Cluster {
       override def whenTerminated(): Future[Unit] = Promise[Unit].failure(FilteredException()).future
       override def askConnection(): Future[PerJobConnection] = Promise[PerJobConnection].future
-      override def warmUp(): Unit = ()
       override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future
     }
   }
@@ -321,7 +297,6 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       override def whenTerminated(): Future[Unit] = Promise[Unit].future
       override def askConnection(): Future[PerJobConnection] = future
       override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future
-      override def warmUp(): Unit = ()
     }
   }
 
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
index 7839baefe..546541c3e 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala
@@ -16,9 +16,6 @@ class ClusterSpec extends ActorSpec("actor-based-connector") {
     connector.askConnection()
     target.expectMsgType[Cluster.Event.AskConnection]
 
-    connector.warmUp()
-    target.expectMsgType[Cluster.Event.WarmUp.type]
-
     connector.shutdown(true)
     target.expectMsgType[Cluster.Event.Shutdown]
   }
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
index cf46dc706..b8ca22e9e 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala
@@ -6,6 +6,7 @@ import akka.actor.ActorRef
 import akka.testkit.{TestActorRef, TestProbe}
 import io.hydrosphere.mist.common.CommonData.RunJobRequest
 import io.hydrosphere.mist.master.execution.Cluster
+import io.hydrosphere.mist.master.models.RunMode
 import io.hydrosphere.mist.master.{ActorSpec, TestData}
 import org.scalatest.Matchers
 import org.scalatest.concurrent.Eventually
@@ -57,7 +58,7 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     val remote = TestProbe()
     val connector = TestActorRef[SharedConnector](SharedConnector.props(
       id = "id",
-      ctx = FooContext,
+      ctx = FooContext.copy(workerMode = RunMode.Shared, precreated = true),
       startConnection = (id, ctx) => {
         val x = callCounter.incrementAndGet()
         val conn = WorkerConnection(x.toString, remote.ref, workerLinkData, Promise[Unit].future)
@@ -66,7 +67,6 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te
     ))
 
     val probe = TestProbe()
-    probe.send(connector, Cluster.Event.WarmUp)
     callCounter.get() shouldBe 2
     probe.send(connector, Cluster.Event.GetStatus)
     val status = probe.expectMsgType[SharedConnector.ProcessStatus]
diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
index 479d9cb0d..62b54250b 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala
@@ -12,7 +12,7 @@ import io.hydrosphere.mist.common.{FunctionInfoData, MockitoSugar}
 import io.hydrosphere.mist.master.JobDetails.Source
 import io.hydrosphere.mist.master._
 import io.hydrosphere.mist.master.artifact.ArtifactRepository
-import io.hydrosphere.mist.master.execution.{ExecutionService, WorkerLink}
+import io.hydrosphere.mist.master.execution.ExecutionService
 import io.hydrosphere.mist.master.data.{ContextsStorage, FunctionConfigStorage}
 import io.hydrosphere.mist.master.interfaces.JsonCodecs
 import io.hydrosphere.mist.master.jobs.FunctionsService
diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala
index 354eb448a..bb09d3f5e 100644
--- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala
+++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala
@@ -18,7 +18,7 @@ class MasterBridgeSpec extends TestKit(ActorSystem("WorkerBridgeSpec"))
   with BeforeAndAfterAll {
 
   def mkInitInfo(sparkConf: Map[String, String]) =
-    WorkerInitInfo(sparkConf, 1, 20 seconds, 20 seconds, "localhost:2005", "localhost:2003", "localhost:2004", 202020, "")
+    WorkerInitInfo(sparkConf, 20 seconds, 20 seconds, "localhost:2005", "localhost:2003", "localhost:2004", 202020, "")
 
   it("should create named context with spark.streaming.stopSparkContextByDefault=false") {
     val sparkConf = Map(

From 6ffd56ec080ec67d334913ca5c834c5ee19a58fd Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Sat, 29 Sep 2018 13:46:34 +0300
Subject: [PATCH 25/30] aws - fleets support wip

---
 .../io/hydrosphere/mist/aws/AwsSetup.scala    |  12 +-
 .../io/hydrosphere/mist/aws/EC2Service.scala  |   1 -
 .../mist/master/data/ConfigRepr.scala         |  33 +++--
 .../mist/master/execution/aws/EMRClient.scala | 117 ++++++++++++++++--
 .../execution/aws/EMRClusterRunner.scala      |  12 +-
 .../master/execution/aws/EMRRunSettings.scala |   3 -
 .../mist/master/interfaces/jsonCodecs.scala   | 107 +++++++++++++++-
 .../hydrosphere/mist/master/models/base.scala |  70 ++++++++++-
 8 files changed, 307 insertions(+), 48 deletions(-)

diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
index 0b03d1bc0..45eb3e0ff 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
@@ -41,14 +41,22 @@ object AwsSetup {
             case Some(d) => ME.pure(d)
             case None => ME.raiseError[InstanceData](new RuntimeException(s"Unknown instance: $instanceId"))
           }
+          // analog of `aws emr create-default-roles`
+          // for ec2 it's required to create InstanceProfile linked with Role
+          // for emr it's enough to have only role
           ec2EmrRole <- iam.getOrCreateRole(ec2EmrRole)
-          ec2EMrInstaceProfile <- iam.getOrCreateInstanceProfile(ec2EmrRole.name, ec2EmrRole.name)
+          ec2EmrInstanceProfile <- iam.getOrCreateInstanceProfile(ec2EmrRole.name, ec2EmrRole.name)
           emrRole <- iam.getOrCreateRole(emrRole)
 
+          // additional security group for emr
+          // allows ingress traffic from mist-master
           secGroupData = IngressData(0, 65535, IngressAddr.CidrIP(data.cidrIp), "TCP")
           scGroupName = secGroupName(instanceId)
-
           internalSecGroup <- ec2.getOrCreateSecGroup(scGroupName, secGroupDecr, data.vpcId, secGroupData)
+
+          // add ingress rule to mist-master node security group
+          // allows ingress traffic from emr cluster
+          // as source uses security group created above
           secGroupId  = internalSecGroup.id
           _ <- ec2.addIngressRule(data.secGroupIds.head, IngressData(0, 65535, IngressAddr.Group(internalSecGroup), "TCP"))
 
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
index 143429464..53632bd61 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
@@ -98,7 +98,6 @@ object EC2Service {
           .ipPermissions(withAddr.build())
           .build()
 
-        println(req)
         ec2Client.authorizeSecurityGroupIngress(req).toIO.map(r => {println(r)})
       }
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
index 9e0236032..266955117 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
@@ -1,8 +1,10 @@
 package io.hydrosphere.mist.master.data
 
-import com.typesafe.config.{Config, ConfigValue, ConfigValueFactory, ConfigValueType}
+import com.typesafe.config._
+import io.hydrosphere.mist.master.interfaces.{EMRInstanceFormat, JsonCodecs}
 import io.hydrosphere.mist.master.models._
 import io.hydrosphere.mist.utils.ConfigUtils._
+import spray.json.{JsonFormat, JsonParser, ParserInput}
 
 import scala.concurrent.duration._
 
@@ -18,6 +20,7 @@ trait NamedConfigRepr[A] extends ConfigRepr[A] {
 
 }
 
+//TODO it's the same as jsonCodecs!
 object ConfigRepr {
 
   import scala.collection.JavaConverters._
@@ -26,11 +29,6 @@ object ConfigRepr {
     def toConfig: Config = repr.toConfig(a)
   }
 
-//  implicit class FromConfigSyntax(config: Config){
-//    def to[A <: NamedConfig](implicit repr: ConfigRepr[A]): A = repr.fromConfig(config)
-//    def to[A <: NamedConfig](name: String)(implicit repr: NamedConfigRepr[A]): A = repr.fromConfig(name, config)
-//  }
-
   val EndpointsRepr = new NamedConfigRepr[FunctionConfig] {
 
     override def toConfig(a: FunctionConfig): Config = {
@@ -53,6 +51,20 @@ object ConfigRepr {
     }
   }
 
+  def fromJsonFormat[A](jsonFormat: JsonFormat[A]): ConfigRepr[A] = new ConfigRepr[A] {
+    override def toConfig(a: A): Config = {
+      val s = jsonFormat.write(a).prettyPrint
+      ConfigFactory.parseString(s)
+    }
+    override def fromConfig(config: Config): A = {
+      val jsonString = config.root().render(ConfigRenderOptions.defaults().setJson(true).setComments(false).setOriginComments(false))
+      val json = JsonParser(ParserInput(jsonString))
+      jsonFormat.read(json)
+    }
+  }
+
+  val EMRInstancesRepr: ConfigRepr[EMRInstances] = fromJsonFormat(EMRInstanceFormat)
+
   val LaunchDataConfigRepr: ConfigRepr[LaunchData] = new ConfigRepr[LaunchData] {
 
     override def toConfig(a: LaunchData): Config = {
@@ -61,12 +73,11 @@ object ConfigRepr {
       val (t, body) = a match {
         case ServerDefault => "server-default" -> Map.empty[String, ConfigValue]
         case awsEmr: AWSEMRLaunchData =>
+          val instances = EMRInstancesRepr.toConfig(awsEmr.instances)
           "aws-emr" -> Map(
             "launcher-settings-name" -> fromAnyRef(awsEmr.launcherSettingsName),
             "release-label" -> fromAnyRef(awsEmr.releaseLabel),
-            "master-instance-type" -> fromAnyRef(awsEmr.masterInstanceType),
-            "slave-instance-type" -> fromAnyRef(awsEmr.slaveInstanceType),
-            "instance-count" -> fromAnyRef(awsEmr.instanceCount)
+            "instances" -> instances.root()
           )
       }
       val full = body + ("type" -> t)
@@ -80,9 +91,7 @@ object ConfigRepr {
           AWSEMRLaunchData(
             launcherSettingsName = config.getString("launcher-settings-name"),
             releaseLabel = config.getString("release-label"),
-            masterInstanceType = config.getString("master-instance-type"),
-            slaveInstanceType = config.getString("slave-instance-type"),
-            instanceCount = config.getInt("instance-count")
+            instances = EMRInstancesRepr.fromConfig(config.getConfig("instances"))
           )
         case x => throw new IllegalArgumentException(s"Unknown launch data type $x")
       }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
index d79664e86..5f7b537f2 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
@@ -4,16 +4,19 @@ import cats._
 import cats.implicits._
 import cats.effect._
 import io.hydrosphere.mist.master.execution.aws.JFutureSyntax._
+import io.hydrosphere.mist.master.models.EMRInstances
+import io.hydrosphere.mist.master.models.EMRInstances.{AutoScaling, Ebs, VolumeType}
 import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider}
 import software.amazon.awssdk.regions.Region
 import software.amazon.awssdk.services.emr.EMRAsyncClient
 import software.amazon.awssdk.services.emr.model.{Unit => _, _}
 
+import scala.collection.JavaConverters._
 import scala.concurrent.duration.FiniteDuration
 
 trait EMRClient[F[_]] {
 
-  def start(settings: EMRRunSettings): F[EmrInfo]
+  def start(settings: EMRRunSettings, instances: EMRInstances): F[EmrInfo]
   def status(id: String): F[Option[EmrInfo]]
   def stop(id: String): F[Unit]
 
@@ -42,28 +45,116 @@ object EMRClient {
 
   class Default(orig: EMRAsyncClient) extends EMRClient[IO] {
 
-    override def start(settings: EMRRunSettings): IO[EmrInfo] = {
+    type JFICBuilder = JobFlowInstancesConfig.Builder
+
+    private def ebsConfiguration(ebs: Ebs): EbsConfiguration = {
+
+      val devices = ebs.volumes.map(volume => {
+        val volumeType = volume.volumeType match {
+          case VolumeType.Standard => "standard"
+          case VolumeType.IO1 => "io1"
+          case VolumeType.GP2 => "gp2"
+        }
+        val spec = VolumeSpecification.builder()
+          .iops(volume.iops)
+          .sizeInGB(volume.sizeGB)
+          .volumeType(volumeType)
+          .build()
+
+        val count = volume.count.getOrElse(1)
+        EbsBlockDeviceConfig.builder()
+          .volumeSpecification(spec)
+          .volumesPerInstance(count)
+          .build()
+      })
+
+      val optimized = ebs.optimized.getOrElse(true)
+      EbsConfiguration.builder()
+        .ebsBlockDeviceConfigs(devices.asJavaCollection)
+        .ebsOptimized(optimized)
+        .build()
+    }
+
+//    private def mkAutoScaling(as: AutoScaling): AutoScalingPolicy = {
+//      SimpleScalingPolicyConfiguration.builder()
+//          .adjustmentType(AdjustmentType.)
+//          .scalingAdjustment()
+//
+//      ScalingRule.builder()
+//        .name()
+//        .trigger(ScalingTrigger.builder().cloudWatchAlarmDefinition(CloudWatchAlarmDefinition.builder().metricName()))
+//      val rules =
+//      AutoScalingPolicy.builder()
+//        .constraints(ScalingConstraints.builder().maxCapacity(as.max).minCapacity(as.min).build())
+//        .rules()
+//    }
+
+    private def mkInstanceGroup(instance: EMRInstances.FleetInstance): InstanceGroupConfig = {
+      val roleType = instance.instanceGroupType match {
+        case EMRInstances.InstanceGroupType.Core => InstanceRoleType.CORE
+        case EMRInstances.InstanceGroupType.Master => InstanceRoleType.MASTER
+        case EMRInstances.InstanceGroupType.Task => InstanceRoleType.TASK
+      }
+      val market = instance.market match {
+        case EMRInstances.Market.OnDemand => MarketType.ON_DEMAND
+        case EMRInstances.Market.Spot => MarketType.SPOT
+      }
+
+      val base = InstanceGroupConfig.builder()
+        .name(instance.name.getOrElse(s"Mist${instance.instanceGroupType}"))
+        .instanceRole(roleType)
+        .instanceCount(instance.instanceCount)
+        .market(market)
+
+      val withEbs = instance.ebs match {
+        case Some(ebs) => base.ebsConfiguration(ebsConfiguration(ebs))
+        case None => base
+      }
+
+      val withBidPrice = instance.bidPrice match {
+        case Some(p) => withEbs.bidPrice(p)
+        case None => withEbs
+      }
+
+      withBidPrice.build()
+    }
+
+    private def mkInstancesConfig(builder: JFICBuilder, instances: EMRInstances): JFICBuilder = {
+      instances match {
+        case fixed: EMRInstances.Fixed =>
+          builder.instanceCount(fixed.instanceCount)
+            .masterInstanceType(fixed.masterInstanceType)
+            .slaveInstanceType(fixed.slaveInstanceType)
+
+        case fleets: EMRInstances.Fleets =>
+          val groups = fleets.instances.map(i => mkInstanceGroup(i))
+          builder.instanceGroups(groups.asJavaCollection)
+      }
+    }
+
+    override def start(settings: EMRRunSettings, instances: EMRInstances): IO[EmrInfo] = {
       import settings._
 
       //TODO: configuration!
       val sparkApp = Application.builder().name("Spark").build()
+
+      val initial = JobFlowInstancesConfig.builder()
+      val addInstances = mkInstancesConfig(initial, instances)
+      val instancesConfig = addInstances
+        .keepJobFlowAliveWhenNoSteps(true)
+        .ec2KeyName(keyPair)
+        .ec2SubnetId(subnetId)
+        .additionalMasterSecurityGroups(additionalGroup)
+        .additionalSlaveSecurityGroups(additionalGroup)
+        .build()
+
       val request = RunJobFlowRequest.builder()
         .name(s"mist-$name")
         .releaseLabel(releaseLabel)
         .applications(sparkApp)
         .jobFlowRole(emrEc2Role)
         .serviceRole(emrRole)
-        .instances(JobFlowInstancesConfig.builder()
-          .keepJobFlowAliveWhenNoSteps(true)
-          .ec2KeyName(keyPair)
-          .instanceCount(instanceCount)
-          .masterInstanceType(masterInstanceType)
-          .slaveInstanceType(slaveInstanceType)
-          .ec2SubnetId(subnetId)
-          .additionalMasterSecurityGroups(additionalGroup)
-          .additionalSlaveSecurityGroups(additionalGroup)
-          .build()
-        ).build()
+        .instances(instancesConfig).build()
 
       for {
         resp <- orig.runJobFlow(request).toIO
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
index 7250e947a..06c223081 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
@@ -11,7 +11,7 @@ import io.hydrosphere.mist.master.AWSEMRLaunchSettings
 import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, StopAction, WorkerConnection, WorkerRunner}
 import io.hydrosphere.mist.master.execution.workers.starter.{SparkSubmitBuilder, WorkerProcess, WorkerStarter}
 import io.hydrosphere.mist.master.execution.{Cluster, ClusterRunner, SpawnSettings}
-import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig}
+import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig, EMRInstances}
 import io.hydrosphere.mist.utils.Logger
 import io.hydrosphere.mist.utils.akka.ActorRegHub
 
@@ -19,7 +19,6 @@ import scala.concurrent.duration._
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Failure
-
 import IOTimer.Default
 
 class EMRClusterRunner(
@@ -45,9 +44,6 @@ class EMRClusterRunner(
       name = name,
       keyPair = settings.sshKeyPair,
       releaseLabel = data.releaseLabel,
-      masterInstanceType = data.masterInstanceType,
-      slaveInstanceType = data.slaveInstanceType,
-      instanceCount = data.instanceCount,
       subnetId = settings.subnetId,
       additionalGroup = settings.additionalGroup,
       emrRole = settings.emrRole,
@@ -55,9 +51,9 @@ class EMRClusterRunner(
     )
   }
 
-  private def startFully(runSettings: EMRRunSettings, client: EMRClient[IO]): IO[EmrInfo] = {
+  private def startFully(runSettings: EMRRunSettings, instances: EMRInstances, client: EMRClient[IO]): IO[EmrInfo] = {
     for {
-      initial <- client.start(runSettings)
+      initial <- client.start(runSettings, instances)
       await <- client.awaitStatus(initial.id, EMRStatus.Started, 10 seconds, 40)
     } yield await
   }
@@ -83,7 +79,7 @@ class EMRClusterRunner(
     val io = for {
       data        <- extractData(ctx)
       runSettings =  mkRunSettings(id, data, launchSettings)
-      emrInfo     <- startFully(runSettings, client)
+      emrInfo     <- startFully(runSettings, data.instances, client)
       agentId     =  s"agent-${emrInfo.id}"
       _           <- IO[Unit](installAgent(emrInfo.masterPublicDnsName, agentId, emrInfo.id))
       _           <- IO.fromFuture(IO(regHub.waitRef(agentId, 1 minute)))
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
index 521f35aa5..d187b201d 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
@@ -4,9 +4,6 @@ case class EMRRunSettings(
   name: String,
   keyPair: String,
   releaseLabel: String,
-  masterInstanceType: String,
-  slaveInstanceType: String,
-  instanceCount: Int,
   subnetId: String,
   additionalGroup: String,
   emrRole: String,
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
index 6bcea6320..0af67aafb 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
@@ -125,6 +125,105 @@ trait JobDetailsJsonFormat extends DefaultJsonProtocol with AnyJsonFormat with M
 
 }
 
+trait EMRInstanceFormat extends JsonFormat[EMRInstances] with DefaultJsonProtocol {
+  val fixedKey = "fixed"
+  val fleetsKey = "fleets"
+  //TODO - infer these things using shapeless!
+  implicit val instanceGroupTypeF = new JsonFormat[EMRInstances.InstanceGroupType] {
+    override def write(obj: EMRInstances.InstanceGroupType): JsValue = {
+      val name = obj match {
+        case EMRInstances.InstanceGroupType.Master => "master"
+        case EMRInstances.InstanceGroupType.Core => "core"
+        case EMRInstances.InstanceGroupType.Task =>  "task"
+      }
+      JsString(name)
+    }
+    override def read(json: JsValue): EMRInstances.InstanceGroupType = {
+      json match {
+        case JsString(v) => v.toLowerCase match {
+          case "master" => EMRInstances.InstanceGroupType.Master
+          case "core" => EMRInstances.InstanceGroupType.Core
+          case "task" => EMRInstances.InstanceGroupType.Task
+          case x => throw new IllegalArgumentException(s"Invalid instance group type $x")
+        }
+        case _ => throw new IllegalArgumentException(s"Invalid instance group type format")
+      }
+    }
+  }
+
+  implicit val marketTypeF = new JsonFormat[EMRInstances.Market] {
+    override def write(obj: EMRInstances.Market): JsValue = {
+      val name = obj match {
+        case EMRInstances.Market.OnDemand => "on-demand"
+        case EMRInstances.Market.Spot => "spot"
+      }
+      JsString(name)
+    }
+    override def read(json: JsValue): EMRInstances.Market = {
+      json match {
+        case JsString(v) => v.toLowerCase match {
+          case "on-demand" => EMRInstances.Market.OnDemand
+          case "spot" => EMRInstances.Market.Spot
+          case x => throw new IllegalArgumentException(s"Invalid market type $x")
+        }
+        case _ => throw new IllegalArgumentException(s"Invalid market type format")
+      }
+    }
+  }
+
+  implicit val volumeTypeF = new JsonFormat[EMRInstances.VolumeType] {
+    override def write(obj: EMRInstances.VolumeType): JsValue = {
+      val name = obj match {
+        case EMRInstances.VolumeType.IO1 => "io1"
+        case EMRInstances.VolumeType.GP2 => "gp2"
+        case EMRInstances.VolumeType.Standard => "standard"
+      }
+      JsString(name)
+    }
+    override def read(json: JsValue): EMRInstances.VolumeType = {
+      json match {
+        case JsString(v) => v.toLowerCase match {
+          case "io1" => EMRInstances.VolumeType.IO1
+          case "gp2" => EMRInstances.VolumeType.GP2
+          case "standard" => EMRInstances.VolumeType.Standard
+          case x => throw new IllegalArgumentException(s"Invalid volume type $x")
+        }
+        case _ => throw new IllegalArgumentException(s"Invalid volume type format")
+      }
+    }
+  }
+
+  implicit val ebsVolumeF = jsonFormat4(EMRInstances.EbsVolume.apply)
+  implicit val ebsF = jsonFormat2(EMRInstances.Ebs.apply)
+  implicit val autoScalingF = jsonFormat2(EMRInstances.AutoScaling.apply)
+  implicit val fleetInstanceF = jsonFormat8(EMRInstances.FleetInstance.apply)
+
+
+  implicit val fixedF = jsonFormat3(EMRInstances.Fixed.apply)
+  implicit val fleetsF = jsonFormat1(EMRInstances.Fleets.apply)
+
+  override def write(obj: EMRInstances): JsValue = {
+    val (name, body) = obj match {
+      case fixed: EMRInstances.Fixed => fixedKey -> fixed.toJson
+      case fleets: EMRInstances.Fleets => fleetsKey -> fleets.toJson
+    }
+    JsObject(body.asJsObject.fields + ("type" -> JsString(name)))
+  }
+
+  override def read(json: JsValue): EMRInstances = {
+    def fromType(t: String, obj: JsValue): EMRInstances = t match {
+      case `fixedKey` => fixedF.read(obj)
+      case `fleetsKey` => fleetsF.read(obj)
+    }
+    val t = json.asJsObject().fields.getOrElse("type", JsNull)
+    t match {
+      case JsString(v) => fromType(v, json)
+      case x => throw new IllegalArgumentException("Invalid emr instaces format")
+    }
+  }
+}
+
+object EMRInstanceFormat extends EMRInstanceFormat
 
 trait JsonCodecs extends SprayJsonSupport
   with DefaultJsonProtocol
@@ -227,12 +326,14 @@ trait JsonCodecs extends SprayJsonSupport
 
   implicit val devJobStartReqModelF = jsonFormat7(DevJobStartRequestModel.apply)
 
+  implicit val emrInstancesF = EMRInstanceFormat
+
   implicit val launchDataF = new JsonFormat[LaunchData] {
 
     val ServerDefaultKey = "server-default"
     val AWSEMRKey = "aws-emr"
 
-    val awsEmrLaunchDataF = jsonFormat5(AWSEMRLaunchData.apply)
+    val awsEmrLaunchDataF = jsonFormat3(AWSEMRLaunchData.apply)
 
     override def write(in: LaunchData): JsValue = in match {
       case ServerDefault => JsObject("type" -> JsString(ServerDefaultKey))
@@ -243,8 +344,8 @@ trait JsonCodecs extends SprayJsonSupport
 
     override def read(json: JsValue): LaunchData = {
       def fromType(t: String, obj: JsValue): LaunchData = t match {
-        case ServerDefaultKey => ServerDefault
-        case AWSEMRKey => awsEmrLaunchDataF.read(obj)
+        case `ServerDefaultKey` => ServerDefault
+        case `AWSEMRKey` => awsEmrLaunchDataF.read(obj)
       }
       val t = json.asJsObject.fields.getOrElse("type", JsNull)
       t match {
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
index 9ccbbb230..01685e1e2 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
@@ -33,15 +33,73 @@ trait NamedConfig {
 sealed trait LaunchData
 /** use default worker-runner **/
 case object ServerDefault extends LaunchData
-case class AWSEMRLaunchData(
+
+sealed trait EMRInstances
+object EMRInstances {
+
+  final case class Fixed(
+    masterInstanceType: String,
+    slaveInstanceType: String,
+    instanceCount: Int
+  ) extends EMRInstances
+
+
+  sealed trait InstanceGroupType
+  object InstanceGroupType {
+    case object Master extends InstanceGroupType
+    case object Core extends InstanceGroupType
+    case object Task extends InstanceGroupType
+  }
+  sealed trait Market
+  object Market {
+    case object OnDemand extends Market
+    case object Spot extends Market
+  }
+
+  sealed trait VolumeType
+  object VolumeType {
+    case object Standard extends VolumeType
+    case object IO1 extends VolumeType
+    case object GP2 extends VolumeType
+  }
+  final case class EbsVolume(
+    volumeType: VolumeType,
+    sizeGB: Int,
+    iops: Int,
+    count: Option[Int]
+  )
+
+  final case class Ebs(
+    optimized: Option[Boolean],
+    volumes: Seq[EbsVolume]
+  )
+
+  final case class AutoScaling(
+    max: Int,
+    min: Int
+  )
+
+  final case class FleetInstance(
+    instanceType: String,
+    instanceGroupType: InstanceGroupType,
+    name: Option[String],
+    instanceCount: Int,
+    market: Market,
+    ebs: Option[Ebs],
+    bidPrice: Option[String],
+    autoScaling: Option[AutoScaling]
+  )
+
+  final case class Fleets(instances: Seq[FleetInstance]) extends EMRInstances
+}
+
+final case class AWSEMRLaunchData(
   launcherSettingsName: String,
   releaseLabel: String,
-  masterInstanceType: String,
-  slaveInstanceType: String,
-  instanceCount: Int
+  instances: EMRInstances
 ) extends LaunchData
 
-case class ContextConfig(
+final case class ContextConfig(
   name: String,
   sparkConf: Map[String, String],
   downtime: Duration,
@@ -55,7 +113,7 @@ case class ContextConfig(
 ) extends NamedConfig
 
 
-case class FunctionConfig(
+final case class FunctionConfig(
   name: String,
   path: String,
   className: String,

From fe18126bcb1f9bc7b63210fcac210fa92d8907cc Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Sun, 30 Sep 2018 03:22:01 +0300
Subject: [PATCH 26/30] aws setup - support autoscaling definition

---
 .../src/main/resources/trustAutoScaling.json  |  16 ++
 .../io/hydrosphere/mist/aws/AwsSetup.scala    |   5 +-
 .../io/hydrosphere/mist/aws/EC2Service.scala  |   1 -
 .../io/hydrosphere/mist/aws/GenConfig.scala   |   6 +-
 .../io/hydrosphere/mist/aws/IAMService.scala  |   4 +
 .../scala/io/hydrosphere/mist/aws/Main.scala  |   3 +-
 .../io/hydrosphere/mist/master/configs.scala  |   6 +-
 .../mist/master/data/ConfigRepr.scala         |   9 +-
 .../mist/master/execution/aws/EMRClient.scala | 129 +++++++++-----
 .../execution/aws/EMRClusterRunner.scala      |   7 +-
 .../master/execution/aws/EMRRunSettings.scala |   3 +-
 .../mist/master/interfaces/jsonCodecs.scala   | 161 ++++++++----------
 .../hydrosphere/mist/master/models/base.scala |  67 ++++++--
 13 files changed, 262 insertions(+), 155 deletions(-)
 create mode 100644 mist/aws-init-setup/src/main/resources/trustAutoScaling.json

diff --git a/mist/aws-init-setup/src/main/resources/trustAutoScaling.json b/mist/aws-init-setup/src/main/resources/trustAutoScaling.json
new file mode 100644
index 000000000..9ad2b0347
--- /dev/null
+++ b/mist/aws-init-setup/src/main/resources/trustAutoScaling.json
@@ -0,0 +1,16 @@
+{
+  "Version": "2008-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "Service": [
+          "elasticmapreduce.amazonaws.com",
+          "application-autoscaling.amazonaws.com"
+        ]
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
index 45eb3e0ff..a8efefd3b 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala
@@ -13,6 +13,7 @@ case class SetupData(
   securityGroupId: String,
   emrRole: String,
   ec2EmrRole: String,
+  autoScalingRole: String,
   sshKeyPairName: String
 )
 
@@ -29,6 +30,7 @@ object AwsSetup {
 
       val ec2EmrRole = AWSRole("mist-EMREC2", "default emr ec2 role", AWSRoleData.EC2EMR)
       val emrRole = AWSRole("mist-EMR", "default emr role", AWSRoleData.EMR)
+      val autoScalingRole = AWSRole("mist-EMR-Autoscaling", "default autoscalaing role for emr", AWSRoleData.EMRAutoScaling)
       val secGroupDecr = "Master-worker communications"
 
       def secGroupName(id: String): String = s"mist-internal-$id"
@@ -47,6 +49,7 @@ object AwsSetup {
           ec2EmrRole <- iam.getOrCreateRole(ec2EmrRole)
           ec2EmrInstanceProfile <- iam.getOrCreateInstanceProfile(ec2EmrRole.name, ec2EmrRole.name)
           emrRole <- iam.getOrCreateRole(emrRole)
+          autoScalingRole <- iam.getOrCreateRole(autoScalingRole)
 
           // additional security group for emr
           // allows ingress traffic from mist-master
@@ -61,7 +64,7 @@ object AwsSetup {
           _ <- ec2.addIngressRule(data.secGroupIds.head, IngressData(0, 65535, IngressAddr.Group(internalSecGroup), "TCP"))
 
           keyName <- ec2.getOrCreateKeyPair(keyName(instanceId), sshKey)
-        } yield SetupData(data.subnetId, secGroupId, emrRole.name, ec2EmrRole.name, keyName)
+        } yield SetupData(data.subnetId, secGroupId, emrRole.name, ec2EmrRole.name, autoScalingRole.name, keyName)
       }
     }
   }
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
index 53632bd61..79cfafbee 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
@@ -73,7 +73,6 @@ object EC2Service {
     new EC2Service[IO] {
 
       override def addIngressRule(groupId: String, data: IngressData): IO[Unit] = {
-        println(s"HERE: $groupId $data")
         import data._
         val ipPremission = IpPermission.builder()
           .fromPort(fromPort)
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
index 3749f15cc..2b0fe134f 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala
@@ -13,7 +13,8 @@ case class LaunchData(
   region: String,
   additionalGroup: String,
   emrRole: String,
-  emrEc2Role: String
+  emrEc2Role: String,
+  autoScalingRole: String
 )
 
 object ConfigPatcher {
@@ -36,7 +37,8 @@ object ConfigPatcher {
       "region" -> region,
       "additionalGroup" -> additionalGroup,
       "emrRole" -> emrRole,
-      "emrEc2Role" -> emrEc2Role
+      "emrEc2Role" -> emrEc2Role,
+      "autoScalingRole" -> autoScalingRole
     ).map({case (k, v) => k -> fromAnyRef(v)})
 
     val provisioner = fromMap(configKeys.asJava)
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
index 3c639dd7d..7b43653b9 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala
@@ -35,6 +35,10 @@ object AWSRoleData {
     permissionsArn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
   )
 
+  val EMRAutoScaling = AWSRoleData(
+    readResourceJson("/trustAutoScaling.json"),
+    permissionsArn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole"
+  )
 }
 
 case class AWSRole(
diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
index 9d08e1b24..203ca4c8e 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala
@@ -29,7 +29,8 @@ object Main {
       region = region,
       additionalGroup = out.securityGroupId,
       emrRole = out.emrRole,
-      emrEc2Role = out.ec2EmrRole
+      emrEc2Role = out.ec2EmrRole,
+      autoScalingRole = out.autoScalingRole
     )
 
     ConfigPatcher.patchFile(Paths.get(configPath), launchData)
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
index 98c50cefd..f65fb6232 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala
@@ -268,7 +268,8 @@ case class AWSEMRLaunchSettings(
   region: String,
   additionalGroup: String,
   emrRole: String,
-  emrEc2Role: String
+  emrEc2Role: String,
+  autoScalingRole: String
 ) extends LauncherSettings
 
 object AWSEMRLaunchSettings {
@@ -284,7 +285,8 @@ object AWSEMRLaunchSettings {
       region = c.getString("region"),
       additionalGroup = c.getString("additionalGroup"),
       emrRole = c.getString("emrRole"),
-      emrEc2Role = c.getString("emrEc2Role")
+      emrEc2Role = c.getString("emrEc2Role"),
+      autoScalingRole = c.getString("autoScalingRole")
     )
   }
 }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
index 266955117..0f1714436 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala
@@ -63,7 +63,7 @@ object ConfigRepr {
     }
   }
 
-  val EMRInstancesRepr: ConfigRepr[EMRInstances] = fromJsonFormat(EMRInstanceFormat)
+  val EMRInstanceRepr: ConfigRepr[EMRInstance.Instance] = fromJsonFormat(EMRInstanceFormat.instanceF)
 
   val LaunchDataConfigRepr: ConfigRepr[LaunchData] = new ConfigRepr[LaunchData] {
 
@@ -73,11 +73,11 @@ object ConfigRepr {
       val (t, body) = a match {
         case ServerDefault => "server-default" -> Map.empty[String, ConfigValue]
         case awsEmr: AWSEMRLaunchData =>
-          val instances = EMRInstancesRepr.toConfig(awsEmr.instances)
+          val instances = awsEmr.instances.map(i => EMRInstanceRepr.toConfig(i).root())
           "aws-emr" -> Map(
             "launcher-settings-name" -> fromAnyRef(awsEmr.launcherSettingsName),
             "release-label" -> fromAnyRef(awsEmr.releaseLabel),
-            "instances" -> instances.root()
+            "instances" -> fromIterable(instances.asJava)
           )
       }
       val full = body + ("type" -> t)
@@ -88,10 +88,11 @@ object ConfigRepr {
       config.getString("type") match {
         case "server-default" => ServerDefault
         case "aws-emr" =>
+          val instances = config.getConfigList("instances").asScala.map(c => EMRInstanceRepr.fromConfig(c))
           AWSEMRLaunchData(
             launcherSettingsName = config.getString("launcher-settings-name"),
             releaseLabel = config.getString("release-label"),
-            instances = EMRInstancesRepr.fromConfig(config.getConfig("instances"))
+            instances = instances
           )
         case x => throw new IllegalArgumentException(s"Unknown launch data type $x")
       }
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
index 5f7b537f2..9afba0232 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
@@ -4,8 +4,8 @@ import cats._
 import cats.implicits._
 import cats.effect._
 import io.hydrosphere.mist.master.execution.aws.JFutureSyntax._
-import io.hydrosphere.mist.master.models.EMRInstances
-import io.hydrosphere.mist.master.models.EMRInstances.{AutoScaling, Ebs, VolumeType}
+import io.hydrosphere.mist.master.models.EMRInstance
+import io.hydrosphere.mist.master.models.EMRInstance.{AutoScaling, Ebs, VolumeType}
 import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider}
 import software.amazon.awssdk.regions.Region
 import software.amazon.awssdk.services.emr.EMRAsyncClient
@@ -16,7 +16,7 @@ import scala.concurrent.duration.FiniteDuration
 
 trait EMRClient[F[_]] {
 
-  def start(settings: EMRRunSettings, instances: EMRInstances): F[EmrInfo]
+  def start(settings: EMRRunSettings, instances: Seq[EMRInstance.Instance]): F[EmrInfo]
   def status(id: String): F[Option[EmrInfo]]
   def stop(id: String): F[Unit]
 
@@ -75,34 +75,89 @@ object EMRClient {
         .build()
     }
 
-//    private def mkAutoScaling(as: AutoScaling): AutoScalingPolicy = {
-//      SimpleScalingPolicyConfiguration.builder()
-//          .adjustmentType(AdjustmentType.)
-//          .scalingAdjustment()
-//
-//      ScalingRule.builder()
-//        .name()
-//        .trigger(ScalingTrigger.builder().cloudWatchAlarmDefinition(CloudWatchAlarmDefinition.builder().metricName()))
-//      val rules =
-//      AutoScalingPolicy.builder()
-//        .constraints(ScalingConstraints.builder().maxCapacity(as.max).minCapacity(as.min).build())
-//        .rules()
-//    }
-
-    private def mkInstanceGroup(instance: EMRInstances.FleetInstance): InstanceGroupConfig = {
-      val roleType = instance.instanceGroupType match {
-        case EMRInstances.InstanceGroupType.Core => InstanceRoleType.CORE
-        case EMRInstances.InstanceGroupType.Master => InstanceRoleType.MASTER
-        case EMRInstances.InstanceGroupType.Task => InstanceRoleType.TASK
+    private def mkAutoScaling(as: AutoScaling): AutoScalingPolicy = {
+
+      def mkRule(rule: EMRInstance.Rule): ScalingRule = {
+        val adjType = rule.adjustmentType match {
+          case EMRInstance.AdjustmentType.ChangeInCapacity => AdjustmentType.CHANGE_IN_CAPACITY
+          case EMRInstance.AdjustmentType.ExactCapacity => AdjustmentType.EXACT_CAPACITY
+          case EMRInstance.AdjustmentType.PercentChangeInCapacity => AdjustmentType.PERCENT_CHANGE_IN_CAPACITY
+        }
+
+        val scalingPolicy = SimpleScalingPolicyConfiguration.builder()
+          .adjustmentType(adjType)
+          .scalingAdjustment(rule.scalingAdjustment)
+          .coolDown(rule.coolDown)
+          .build()
+
+        val action = ScalingAction.builder()
+            .simpleScalingPolicyConfiguration(scalingPolicy)
+            .build()
+
+        val cmpOp = rule.trigger.comparisonOperator match {
+          case EMRInstance.ComparisonOperator.GreaterThanOrEqual => ComparisonOperator.GREATER_THAN_OR_EQUAL
+          case EMRInstance.ComparisonOperator.GreaterThan => ComparisonOperator.GREATER_THAN
+          case EMRInstance.ComparisonOperator.LessThan => ComparisonOperator.LESS_THAN
+          case EMRInstance.ComparisonOperator.LessThanOrEqual => ComparisonOperator.LESS_THAN_OR_EQUAL
+        }
+        val statistic = rule.trigger.statistic match {
+          case EMRInstance.Statistic.SampleCount => Statistic.SAMPLE_COUNT
+          case EMRInstance.Statistic.Average => Statistic.AVERAGE
+          case EMRInstance.Statistic.Sum => Statistic.SUM
+          case EMRInstance.Statistic.Minimum => Statistic.MINIMUM
+          case EMRInstance.Statistic.Maximum => Statistic.MAXIMUM
+        }
+
+        val dimensions= rule.trigger.dimensions.map(d =>
+          MetricDimension.builder().key(d.key).value(d.value).build()
+        )
+
+        val cloudWatchAlarmDefinition = CloudWatchAlarmDefinition.builder()
+          .comparisonOperator(cmpOp)
+          .evaluationPeriods(rule.trigger.evaluationPeriods)
+          .metricName(rule.trigger.metricName)
+          .namespace(rule.trigger.namespace)
+          .period(rule.trigger.period)
+          .threshold(rule.trigger.threshold)
+          .statistic(statistic)
+          .unit(rule.trigger.unit)
+          .dimensions(dimensions.asJavaCollection)
+          .build()
+
+
+        val trigger = ScalingTrigger.builder()
+            .cloudWatchAlarmDefinition(cloudWatchAlarmDefinition)
+            .build()
+
+        ScalingRule.builder()
+          .name(rule.name)
+          .description(rule.name)
+          .action(action)
+          .trigger(trigger)
+          .build()
       }
-      val market = instance.market match {
-        case EMRInstances.Market.OnDemand => MarketType.ON_DEMAND
-        case EMRInstances.Market.Spot => MarketType.SPOT
+
+      AutoScalingPolicy.builder()
+        .constraints(ScalingConstraints.builder().maxCapacity(as.max).minCapacity(as.min).build())
+        .rules(as.rules.map(mkRule).asJavaCollection)
+        .build()
+    }
+
+    private def mkInstanceGroup(instance: EMRInstance.Instance): InstanceGroupConfig = {
+      val roleType = instance.instanceGroupType match {
+        case EMRInstance.InstanceGroupType.Core => InstanceRoleType.CORE
+        case EMRInstance.InstanceGroupType.Master => InstanceRoleType.MASTER
+        case EMRInstance.InstanceGroupType.Task => InstanceRoleType.TASK
       }
+      val market = instance.market.fold(MarketType.ON_DEMAND)({
+        case EMRInstance.Market.OnDemand => MarketType.ON_DEMAND
+        case EMRInstance.Market.Spot => MarketType.SPOT
+      })
 
       val base = InstanceGroupConfig.builder()
         .name(instance.name.getOrElse(s"Mist${instance.instanceGroupType}"))
         .instanceRole(roleType)
+        .instanceType(instance.instanceType)
         .instanceCount(instance.instanceCount)
         .market(market)
 
@@ -116,23 +171,20 @@ object EMRClient {
         case None => withEbs
       }
 
-      withBidPrice.build()
-    }
+      val result = instance.autoScaling match {
+        case Some(as) => withBidPrice.autoScalingPolicy(mkAutoScaling(as))
+        case None => withBidPrice
+      }
 
-    private def mkInstancesConfig(builder: JFICBuilder, instances: EMRInstances): JFICBuilder = {
-      instances match {
-        case fixed: EMRInstances.Fixed =>
-          builder.instanceCount(fixed.instanceCount)
-            .masterInstanceType(fixed.masterInstanceType)
-            .slaveInstanceType(fixed.slaveInstanceType)
+      result.build()
+    }
 
-        case fleets: EMRInstances.Fleets =>
-          val groups = fleets.instances.map(i => mkInstanceGroup(i))
-          builder.instanceGroups(groups.asJavaCollection)
-      }
+    private def mkInstancesConfig(builder: JFICBuilder, instances: Seq[EMRInstance.Instance]): JFICBuilder = {
+      val groups = instances.map(i => mkInstanceGroup(i))
+      builder.instanceGroups(groups.asJavaCollection)
     }
 
-    override def start(settings: EMRRunSettings, instances: EMRInstances): IO[EmrInfo] = {
+    override def start(settings: EMRRunSettings, instances: Seq[EMRInstance.Instance]): IO[EmrInfo] = {
       import settings._
 
       //TODO: configuration!
@@ -154,6 +206,7 @@ object EMRClient {
         .applications(sparkApp)
         .jobFlowRole(emrEc2Role)
         .serviceRole(emrRole)
+        .autoScalingRole(autoScalingRole)
         .instances(instancesConfig).build()
 
       for {
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
index 06c223081..5a106659c 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala
@@ -11,7 +11,7 @@ import io.hydrosphere.mist.master.AWSEMRLaunchSettings
 import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, StopAction, WorkerConnection, WorkerRunner}
 import io.hydrosphere.mist.master.execution.workers.starter.{SparkSubmitBuilder, WorkerProcess, WorkerStarter}
 import io.hydrosphere.mist.master.execution.{Cluster, ClusterRunner, SpawnSettings}
-import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig, EMRInstances}
+import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig, EMRInstance}
 import io.hydrosphere.mist.utils.Logger
 import io.hydrosphere.mist.utils.akka.ActorRegHub
 
@@ -47,11 +47,12 @@ class EMRClusterRunner(
       subnetId = settings.subnetId,
       additionalGroup = settings.additionalGroup,
       emrRole = settings.emrRole,
-      emrEc2Role = settings.emrEc2Role
+      emrEc2Role = settings.emrEc2Role,
+      autoScalingRole = settings.autoScalingRole
     )
   }
 
-  private def startFully(runSettings: EMRRunSettings, instances: EMRInstances, client: EMRClient[IO]): IO[EmrInfo] = {
+  private def startFully(runSettings: EMRRunSettings, instances: Seq[EMRInstance.Instance], client: EMRClient[IO]): IO[EmrInfo] = {
     for {
       initial <- client.start(runSettings, instances)
       await <- client.awaitStatus(initial.id, EMRStatus.Started, 10 seconds, 40)
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
index d187b201d..716291c1b 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala
@@ -7,6 +7,7 @@ case class EMRRunSettings(
   subnetId: String,
   additionalGroup: String,
   emrRole: String,
-  emrEc2Role: String
+  emrEc2Role: String,
+  autoScalingRole: String
 )
 
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
index 0af67aafb..aaf2e234b 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala
@@ -125,102 +125,86 @@ trait JobDetailsJsonFormat extends DefaultJsonProtocol with AnyJsonFormat with M
 
 }
 
-trait EMRInstanceFormat extends JsonFormat[EMRInstances] with DefaultJsonProtocol {
-  val fixedKey = "fixed"
-  val fleetsKey = "fleets"
-  //TODO - infer these things using shapeless!
-  implicit val instanceGroupTypeF = new JsonFormat[EMRInstances.InstanceGroupType] {
-    override def write(obj: EMRInstances.InstanceGroupType): JsValue = {
-      val name = obj match {
-        case EMRInstances.InstanceGroupType.Master => "master"
-        case EMRInstances.InstanceGroupType.Core => "core"
-        case EMRInstances.InstanceGroupType.Task =>  "task"
-      }
-      JsString(name)
-    }
-    override def read(json: JsValue): EMRInstances.InstanceGroupType = {
-      json match {
-        case JsString(v) => v.toLowerCase match {
-          case "master" => EMRInstances.InstanceGroupType.Master
-          case "core" => EMRInstances.InstanceGroupType.Core
-          case "task" => EMRInstances.InstanceGroupType.Task
-          case x => throw new IllegalArgumentException(s"Invalid instance group type $x")
-        }
-        case _ => throw new IllegalArgumentException(s"Invalid instance group type format")
-      }
-    }
-  }
+trait EMRInstanceFormat extends DefaultJsonProtocol {
 
-  implicit val marketTypeF = new JsonFormat[EMRInstances.Market] {
-    override def write(obj: EMRInstances.Market): JsValue = {
-      val name = obj match {
-        case EMRInstances.Market.OnDemand => "on-demand"
-        case EMRInstances.Market.Spot => "spot"
-      }
-      JsString(name)
-    }
-    override def read(json: JsValue): EMRInstances.Market = {
-      json match {
-        case JsString(v) => v.toLowerCase match {
-          case "on-demand" => EMRInstances.Market.OnDemand
-          case "spot" => EMRInstances.Market.Spot
-          case x => throw new IllegalArgumentException(s"Invalid market type $x")
-        }
-        case _ => throw new IllegalArgumentException(s"Invalid market type format")
-      }
-    }
-  }
+  def dummyAdtStringFormat[A](hint: String, pairs: Seq[(A, String)]): JsonFormat[A] = new JsonFormat[A] {
 
-  implicit val volumeTypeF = new JsonFormat[EMRInstances.VolumeType] {
-    override def write(obj: EMRInstances.VolumeType): JsValue = {
-      val name = obj match {
-        case EMRInstances.VolumeType.IO1 => "io1"
-        case EMRInstances.VolumeType.GP2 => "gp2"
-        case EMRInstances.VolumeType.Standard => "standard"
-      }
-      JsString(name)
+    override def write(obj: A): JsValue = pairs.find(a => a._1 == obj) match {
+      case Some((_, v)) => JsString(v)
+      case None => throw new IllegalArgumentException(s"Undeclared object $obj in $hint format")
     }
-    override def read(json: JsValue): EMRInstances.VolumeType = {
-      json match {
-        case JsString(v) => v.toLowerCase match {
-          case "io1" => EMRInstances.VolumeType.IO1
-          case "gp2" => EMRInstances.VolumeType.GP2
-          case "standard" => EMRInstances.VolumeType.Standard
-          case x => throw new IllegalArgumentException(s"Invalid volume type $x")
-        }
-        case _ => throw new IllegalArgumentException(s"Invalid volume type format")
+    override def read(json: JsValue): A = json match {
+      case JsString(v) => pairs.find(a => a._2 == v) match {
+        case Some(out) => out._1
+        case None => throw new IllegalArgumentException(s"Undeclared string value $v in $hint format")
       }
+      case _ => throw new IllegalArgumentException(s"Invalid input value for $hint format")
     }
   }
 
-  implicit val ebsVolumeF = jsonFormat4(EMRInstances.EbsVolume.apply)
-  implicit val ebsF = jsonFormat2(EMRInstances.Ebs.apply)
-  implicit val autoScalingF = jsonFormat2(EMRInstances.AutoScaling.apply)
-  implicit val fleetInstanceF = jsonFormat8(EMRInstances.FleetInstance.apply)
-
-
-  implicit val fixedF = jsonFormat3(EMRInstances.Fixed.apply)
-  implicit val fleetsF = jsonFormat1(EMRInstances.Fleets.apply)
+  implicit val marketF = dummyAdtStringFormat[EMRInstance.Market](
+    hint = "Market",
+    pairs = Seq(
+      EMRInstance.Market.OnDemand -> "onDemand",
+      EMRInstance.Market.Spot -> "spot"
+    )
+  )
+
+  implicit val instanceGroupTYpeF = dummyAdtStringFormat[EMRInstance.InstanceGroupType](
+    hint = "InstanceGroupType",
+    pairs = Seq(
+      EMRInstance.InstanceGroupType.Master -> "master",
+      EMRInstance.InstanceGroupType.Core -> "core",
+      EMRInstance.InstanceGroupType.Task -> "task"
+    )
+  )
+
+  implicit val volumeTypeF = dummyAdtStringFormat[EMRInstance.VolumeType](
+    hint = "VolumeType",
+    pairs = Seq(
+      EMRInstance.VolumeType.IO1 -> "io1",
+      EMRInstance.VolumeType.GP2 -> "gp2",
+      EMRInstance.VolumeType.Standard -> "standard"
+    )
+  )
+
+  implicit val ebsVolumeF = jsonFormat4(EMRInstance.EbsVolume.apply)
+  implicit val ebsF = jsonFormat2(EMRInstance.Ebs.apply)
+
+  implicit val adjustmentTypeF = dummyAdtStringFormat[EMRInstance.AdjustmentType](
+    hint = "AdjustmentType",
+    pairs = Seq(
+      EMRInstance.AdjustmentType.ChangeInCapacity -> "changeInCapacity",
+      EMRInstance.AdjustmentType.PercentChangeInCapacity -> "percentChangeInCapacity",
+      EMRInstance.AdjustmentType.ExactCapacity -> "exactCapacity"
+    )
+  )
+  implicit val cmpOperatorF = dummyAdtStringFormat[EMRInstance.ComparisonOperator](
+    hint = "ComparisonOperator",
+    pairs = Seq(
+      EMRInstance.ComparisonOperator.GreaterThanOrEqual -> "greaterThanOrEqual",
+      EMRInstance.ComparisonOperator.GreaterThan -> "greaterThan",
+      EMRInstance.ComparisonOperator.LessThan -> "lessThan",
+      EMRInstance.ComparisonOperator.LessThanOrEqual -> "lessThanOrEqual"
+    )
+  )
+  implicit val statisticF = dummyAdtStringFormat[EMRInstance.Statistic](
+    hint = "Statistic",
+    pairs = Seq(
+      EMRInstance.Statistic.SampleCount -> "sampleCount",
+      EMRInstance.Statistic.Average -> "average",
+      EMRInstance.Statistic.Sum -> "sum",
+      EMRInstance.Statistic.Minimum -> "minimum",
+      EMRInstance.Statistic.Maximum -> "maximum"
+    )
+  )
+  implicit val dimensionsF = jsonFormat2(EMRInstance.Dimension.apply)
+  implicit val triggerF = jsonFormat9(EMRInstance.Trigger.apply)
+  implicit val ruleF = jsonFormat6(EMRInstance.Rule.apply)
+
+  implicit val autoScalingF = jsonFormat3(EMRInstance.AutoScaling.apply)
+  implicit val instanceF = jsonFormat8(EMRInstance.Instance.apply)
 
-  override def write(obj: EMRInstances): JsValue = {
-    val (name, body) = obj match {
-      case fixed: EMRInstances.Fixed => fixedKey -> fixed.toJson
-      case fleets: EMRInstances.Fleets => fleetsKey -> fleets.toJson
-    }
-    JsObject(body.asJsObject.fields + ("type" -> JsString(name)))
-  }
-
-  override def read(json: JsValue): EMRInstances = {
-    def fromType(t: String, obj: JsValue): EMRInstances = t match {
-      case `fixedKey` => fixedF.read(obj)
-      case `fleetsKey` => fleetsF.read(obj)
-    }
-    val t = json.asJsObject().fields.getOrElse("type", JsNull)
-    t match {
-      case JsString(v) => fromType(v, json)
-      case x => throw new IllegalArgumentException("Invalid emr instaces format")
-    }
-  }
 }
 
 object EMRInstanceFormat extends EMRInstanceFormat
@@ -228,6 +212,7 @@ object EMRInstanceFormat extends EMRInstanceFormat
 trait JsonCodecs extends SprayJsonSupport
   with DefaultJsonProtocol
   with AnyJsonFormat
+  with EMRInstanceFormat
   with JobDetailsJsonFormat {
 
   implicit val printer = CompactPrinter
diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
index 01685e1e2..3c4a90157 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala
@@ -34,15 +34,7 @@ sealed trait LaunchData
 /** use default worker-runner **/
 case object ServerDefault extends LaunchData
 
-sealed trait EMRInstances
-object EMRInstances {
-
-  final case class Fixed(
-    masterInstanceType: String,
-    slaveInstanceType: String,
-    instanceCount: Int
-  ) extends EMRInstances
-
+object EMRInstance {
 
   sealed trait InstanceGroupType
   object InstanceGroupType {
@@ -50,6 +42,7 @@ object EMRInstances {
     case object Core extends InstanceGroupType
     case object Task extends InstanceGroupType
   }
+
   sealed trait Market
   object Market {
     case object OnDemand extends Market
@@ -74,29 +67,75 @@ object EMRInstances {
     volumes: Seq[EbsVolume]
   )
 
+  sealed trait AdjustmentType
+  object AdjustmentType {
+    case object ChangeInCapacity extends AdjustmentType
+    case object PercentChangeInCapacity extends AdjustmentType
+    case object ExactCapacity extends AdjustmentType
+  }
+
+  sealed trait ComparisonOperator
+  object ComparisonOperator {
+    case object GreaterThanOrEqual extends ComparisonOperator
+    case object GreaterThan extends ComparisonOperator
+    case object LessThan extends ComparisonOperator
+    case object LessThanOrEqual extends ComparisonOperator
+  }
+  sealed trait Statistic
+  object Statistic {
+    case object SampleCount extends Statistic
+    case object Average extends Statistic
+    case object Sum extends Statistic
+    case object Minimum extends Statistic
+    case object Maximum extends Statistic
+  }
+
+  final case class Dimension(key:String, value: String)
+
+  final case class Trigger(
+    comparisonOperator: ComparisonOperator,
+    evaluationPeriods: Int,
+    metricName: String,
+    namespace: String,
+    period: Int,
+    threshold: Double,
+    statistic: Statistic,
+    unit: String,
+    dimensions: Seq[Dimension]
+  )
+
+  final case class Rule(
+    name: String,
+    description: String,
+    adjustmentType: AdjustmentType,
+    scalingAdjustment: Int,
+    coolDown: Int,
+    trigger: Trigger
+  )
+
   final case class AutoScaling(
     max: Int,
-    min: Int
+    min: Int,
+    rules: Seq[Rule]
   )
 
-  final case class FleetInstance(
+  final case class Instance(
     instanceType: String,
     instanceGroupType: InstanceGroupType,
     name: Option[String],
     instanceCount: Int,
-    market: Market,
+    market: Option[Market],
     ebs: Option[Ebs],
     bidPrice: Option[String],
     autoScaling: Option[AutoScaling]
   )
 
-  final case class Fleets(instances: Seq[FleetInstance]) extends EMRInstances
 }
 
 final case class AWSEMRLaunchData(
   launcherSettingsName: String,
   releaseLabel: String,
-  instances: EMRInstances
+  instances: Seq[EMRInstance.Instance]
 ) extends LaunchData
 
 final case class ContextConfig(

From f50001eccc4bab89cc642257f48d94d6921a2829 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Sun, 30 Sep 2018 03:29:51 +0300
Subject: [PATCH 27/30] aws install sript - update version

---
 cloud_install/aws/install.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
index c51e70d01..9a1c46101 100644
--- a/cloud_install/aws/install.sh
+++ b/cloud_install/aws/install.sh
@@ -26,9 +26,9 @@ service nginx restart
 
 cd /opt
 
-wget http://repo.hydrosphere.io/hydrosphere/static/preview/mist-1.0.0-RC17.tar.gz
-tar xvfz mist-1.0.0-RC17.tar.gz
-mv mist-1.0.0-RC17 mist
+wget http://repo.hydrosphere.io/hydrosphere/static/preview/mist-1.0.0-RC18.tar.gz
+tar xvfz mist-1.0.0-RC18.tar.gz
+mv mist-1.0.0-RC18 mist
 
 wget https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz 
 tar xvfz spark-2.3.0-bin-hadoop2.7.tgz

From fdac0347100c3cc65c1ec296a705801f067c6ecb Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Mon, 1 Oct 2018 17:40:55 +0300
Subject: [PATCH 28/30] emr - visible to all users

---
 .../io/hydrosphere/mist/master/execution/aws/EMRClient.scala  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
index 9afba0232..860e346ea 100644
--- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
+++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala
@@ -207,7 +207,9 @@ object EMRClient {
         .jobFlowRole(emrEc2Role)
         .serviceRole(emrRole)
         .autoScalingRole(autoScalingRole)
-        .instances(instancesConfig).build()
+        .instances(instancesConfig)
+        .visibleToAllUsers(true)
+        .build()
 
       for {
         resp <- orig.runJobFlow(request).toIO

From 0ae001e4b531ec17597e1d515d58437fbeed7311 Mon Sep 17 00:00:00 2001
From: Sebastian Nadorp <s.nadorp@dcmn.com>
Date: Thu, 29 Nov 2018 16:03:42 +0100
Subject: [PATCH 29/30] Fix dependency on default VPC and avoid recreating
 ingress rule.

---
 .../io/hydrosphere/mist/aws/EC2Service.scala  | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
index 79cfafbee..19271a3a7 100644
--- a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
+++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala
@@ -86,7 +86,6 @@ object EC2Service {
           case IngressAddr.Group(v) =>
             val group = UserIdGroupPair.builder()
               .groupId(v.id)
-              .groupName(v.name)
               .description(s"${v.id}:${v.name}")
               .build()
             ipPremission.userIdGroupPairs(group)
@@ -97,14 +96,23 @@ object EC2Service {
           .ipPermissions(withAddr.build())
           .build()
 
-        ec2Client.authorizeSecurityGroupIngress(req).toIO.map(r => {println(r)})
+        ec2Client.authorizeSecurityGroupIngress(req).toIO.map(_ => ())
+          .handleErrorWith(e => e match {
+            case m: software.amazon.awssdk.services.ec2.model.EC2Exception if m.getMessage.contains("already exists") => IO.pure(())
+            case _ => IO.raiseError(e)
+          })
       }
 
       override def getSecGroup(name: String): IO[Option[SecGroup]] = {
-        val req = DescribeSecurityGroupsRequest.builder().groupNames(name).build()
-        val io = ec2Client.describeSecurityGroups(req).toIO
+        val req = DescribeSecurityGroupsRequest.builder()
+          .filters(Filter.builder()
+            .name("group-name")
+            .values(name)
+            .build())
+          .build()
 
-        io.map(r => r.securityGroups().asScala.headOption.map(s => SecGroup(s.groupId(), s.groupName())))
+        ec2Client.describeSecurityGroups(req).toIO
+          .map(_.securityGroups().asScala.headOption.map(s => SecGroup(s.groupId(), s.groupName())))
           .handleErrorWith(e => e match {
             case _: software.amazon.awssdk.services.ec2.model.EC2Exception => IO.pure(None)
             case _ => IO.raiseError(e)
@@ -117,7 +125,6 @@ object EC2Service {
         vpcId: String,
         data: IngressData
       ): IO[SecGroup] = {
-
         val createReq = CreateSecurityGroupRequest.builder()
           .groupName(name)
           .description(descr)
@@ -125,10 +132,9 @@ object EC2Service {
           .build()
 
         for {
-          resp <- ec2Client.createSecurityGroup(createReq).toIO
-          groupId = resp.groupId()
+          groupId <- ec2Client.createSecurityGroup(createReq).toIO.map(_.groupId())
           _ <- addIngressRule(groupId, data)
-        } yield SecGroup(resp.groupId, name)
+        } yield SecGroup(groupId, name)
       }
 
       override def getInstanceData(id: String): IO[Option[InstanceData]] = {
@@ -165,7 +171,5 @@ object EC2Service {
       }
 
     }
-
   }
-
 }

From 43967745853af8c65d172d4375bf8ee39d6f98c9 Mon Sep 17 00:00:00 2001
From: dos65 <qtankle@gmail.com>
Date: Thu, 29 Nov 2018 22:52:23 +0300
Subject: [PATCH 30/30] fix tests

---
 .../hydrosphere/mist/master/execution/ContextFrontendSpec.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
index c15c7e113..60a5aeb0a 100644
--- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
+++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala
@@ -263,7 +263,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec")
       name = "name",
       status = StatusReporter.NOOP,
       loggersFactory = NOOPLoggerFactory,
-      connectorStarter = (_, _) => connector,
+      connectorStarter = (_, _) => Future.successful(connector),
       jobFactory = ActorF.static(job.ref),
       defaultInactiveTimeout = 5 minutes
     )