Hydrospheredata · dos65 · Sep 11, 2018 · Sep 11, 2018 · Sep 12, 2018 · Sep 13, 2018
diff --git a/cloud_install/aws/cloudformation-launch-stack.png b/cloud_install/aws/cloudformation-launch-stack.png
diff --git a/cloud_install/aws/index.html b/cloud_install/aws/index.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>Mist</title>
+</head>
+
+<body>
+  <a href="https://console.aws.amazon.com/cloudformation/home?#/stacks/new?stackName=mist&templateURL=https://s3.eu-central-1.amazonaws.com/hydrosphere-cloudformation/template.json">
+    <img src="cloudformation-launch-stack.png"/>
+  </a>
+</body>
+
+</html>
diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+apt-get update
+apt-get install -y nginx openjdk-8-jdk apache2-utils
+
+JAVA=$(which java)
+
+source ~/.aws_setup_data
+
+ssh-keygen -b 2048 -t rsa -f ~/.mist_key -q -N ""
+chmod 600 ~/.mist_key
+
+echo "<p>Mist will start soon</p>" > /var/www/html/index.html
+cat << EOF > /etc/nginx/sites-enabled/default
+server {
+	listen 80 default_server;
+	listen [::]:80 default_server;
+
+	root /var/www/html;
+	index index.html index.htm index.nginx-debian.html;
+
+	server_name _;
+}
+EOF
+service nginx restart
+
+cd /opt
+
+wget http://repo.hydrosphere.io/hydrosphere/static/preview/mist-1.0.0-RC18.tar.gz
+tar xvfz mist-1.0.0-RC18.tar.gz
+mv mist-1.0.0-RC18 mist
+
+wget https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz 
+tar xvfz spark-2.3.0-bin-hadoop2.7.tgz
+mv spark-2.3.0-bin-hadoop2.7 spark
+
+# By some reasons, there is some problems with running Mist right after intance was started
+# InfoProvider can't connect to master
+sleep 30
+
+INSTANCE_ID=$(ec2metadata --instance-id)
+$JAVA -cp /opt/mist/utils/aws-init-setup.jar io.hydrosphere.mist.aws.Main $INSTANCE_ID $ACCESS_KEY_ID $ACCESS_KEY_SECRET $AWS_REGION /opt/mist/configs/default.conf ~/.mist_key.pub ~/.mist_key > /root/aws_setup.log
+
+SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start
+
+htpasswd -b -c /etc/nginx/.htpasswd $MIST_LOGIN $MIST_PASSWORD
+cat << EOF > /etc/nginx/sites-enabled/default
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    root /var/www/html;
+    index index.html index.htm index.nginx-debian.html;
+
+    server_name _;
+
+    location / {
+        proxy_pass http://127.0.0.1:2004/;
+        auth_basic "Restricted";
+        auth_basic_user_file /etc/nginx/.htpasswd;
+	}
+
+	location /v2/api/ws {
+        proxy_pass http://127.0.0.1:2004/v2/api/ws;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade \$http_upgrade;
+        proxy_set_header Connection "upgrade";
+        auth_basic "Restricted";
+        auth_basic_user_file /etc/nginx/.htpasswd;
+    }
+}
+EOF
+service nginx restart
diff --git a/cloud_install/aws/template.json b/cloud_install/aws/template.json
@@ -0,0 +1,92 @@
+{
+  "AWSTemplateFormatVersion" : "2010-09-09",
+  "Description" : "Hydrosphere/mist setup template",
+  "Metadata" : {
+    "Instances" : {"Description" : "Information about the instances"},
+    "Databases" : {"Description" : "Information about the databases"}
+  },
+  "Parameters" : {
+    "InstanceTypeParameter" : {
+      "Type" : "String",
+      "Default" : "m4.large",
+      "Description" : "Enter t2.micro, m1.small, m1.large ..."
+    },
+    "KeyPairName": {
+      "Type": "AWS::EC2::KeyPair::KeyName",
+      "Description": "KeyPair Name"
+    },
+    "AccessKeyID": {
+      "Type" : "String",
+      "Description" : "AWS access key ID"
+    },
+    "AccessKeySecret": {
+      "Type" : "String",
+      "Description" : "AWS secret access key",
+      "NoEcho" : "true"
+    },
+    "AZ": {
+       "Description": "Availability Zone of the Subnet",
+       "Type": "AWS::EC2::AvailabilityZone::Name"
+    },
+    "MistLogin": {
+      "Type" : "String",
+      "Description" : "Basic auth login"
+    },
+    "MistPassword": {
+      "Type" : "String",
+      "Description" : "Basic auth password",
+      "NoEcho" : "true"
+    }
+  },
+  "Resources": {
+    "MistEc2Instance": {
+      "Type" : "AWS::EC2::Instance",
+      "Properties" : {
+         "AvailabilityZone" : { "Ref": "AZ" },
+         "InstanceType": { "Ref": "InstanceTypeParameter" },
+         "ImageId" : "ami-027583e616ca104df",
+         "KeyName": { "Ref": "KeyPairName" },
+         "SecurityGroups": [ { "Ref": "InstanceSecurityGroup" } ],
+         "Tags" : [ {"Key" : "Name", "Value" : "mist-server"} ],
+         "UserData": {
+           "Fn::Base64": { "Fn::Join": [ "\n", [
+              "#!/bin/bash -ex",
+              { "Fn::Join": ["", ["echo ACCESS_KEY_ID=", { "Ref": "AccessKeyID"},  " > ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo ACCESS_KEY_SECRET=", { "Ref": "AccessKeySecret"},  " >> ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo AWS_REGION=", { "Ref" : "AWS::Region" },  " >> ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo MIST_LOGIN=", { "Ref": "MistLogin"},  " >> ~/.aws_setup_data"]]},
+              { "Fn::Join": ["", ["echo MIST_PASSWORD=", { "Ref": "MistPassword"},  " >> ~/.aws_setup_data"]]},
+              "curl http://repo.hydrosphere.io/hydrosphere/static/preview/install.sh | bash"
+           ]]}
+         }
+      }
+    },
+    "InstanceSecurityGroup" : {
+      "Type" : "AWS::EC2::SecurityGroup",
+      "Properties" : {
+        "GroupDescription" : "SSH + HTTP",
+        "SecurityGroupIngress" : [
+          {
+            "IpProtocol" : "tcp",
+            "FromPort" : "22",
+            "ToPort" : "22",
+            "CidrIp" : "0.0.0.0/0"
+          },
+          {
+            "IpProtocol" : "tcp",
+            "FromPort" : "80",
+            "ToPort" : "80",
+            "CidrIp" : "0.0.0.0/0"
+          }
+        ]
+      }
+    }
+  },
+  "Outputs" : {
+    "PublicDns" : {
+      "Value" : { "Fn::GetAtt" : [ "MistEc2Instance", "PublicDnsName" ]},
+      "Description" : "Public Dns name"
+    }
+
+  }
+}
diff --git a/configs/logging/log4j.debug.properties b/configs/logging/log4j.debug.properties
@@ -9,3 +9,5 @@ log4j.appender.file.File=${mist.home}/logs/mist.log
 log4j.appender.file.append=true
 log4j.appender.file.layout=org.apache.log4j.PatternLayout
 log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+log4j.logger.net.schmizz.sshj=WARN
diff --git a/configs/logging/log4j.default.properties b/configs/logging/log4j.default.properties
@@ -4,4 +4,6 @@ log4j.appender.file=org.apache.log4j.FileAppender
 log4j.appender.file.File=${mist.home}/logs/mist.log
 log4j.appender.file.append=true
 log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+log4j.logger.net.schmizz.sshj=WARN
diff --git a/examples/examples/src/main/scala/SparkSessionExample.scala b/examples/examples/src/main/scala/SparkSessionExample.scala
@@ -11,6 +11,7 @@ object SparkSessionExample extends MistFn {
       arg[Int]("multiplier", 2)
     ).withMistExtras
       .onSparkSession((nums: Seq[Int], mult: Int, extras: MistExtras, spark: SparkSession) => {
+        spark.read.parquet("1", "2")
         spark.sparkContext.parallelize(nums).map(_ * mult).collect()
       })
 

diff --git a/mist.sbt b/mist.sbt
@@ -50,13 +50,12 @@ lazy val mistLib = project.in(file("mist-lib"))
     test in Test := Def.sequential(test in Test, PyProject.pyTest in Test).value
   )
 
-lazy val core = project.in(file("mist/core"))
+lazy val common = project.in(file("mist/common"))
   .dependsOn(mistLib)
   .settings(commonSettings: _*)
   .settings(
-    name := "mist-core",
+    name := "mist-common",
     scalacOptions ++= commonScalacOptions,
-    libraryDependencies ++= Library.spark(sparkVersion.value).map(_ % "runtime"),
     libraryDependencies ++= Seq(
       Library.Akka.actor,
       Library.slf4j,
@@ -67,7 +66,7 @@ lazy val core = project.in(file("mist/core"))
   )
 
 lazy val master = project.in(file("mist/master"))
-  .dependsOn(core % "compile->compile;test->test")
+  .dependsOn(common % "compile->compile;test->test")
   .settings(commonSettings: _*)
   .settings(commonAssemblySettings: _*)
   .enablePlugins(BuildInfoPlugin)
@@ -76,17 +75,20 @@ lazy val master = project.in(file("mist/master"))
     scalacOptions ++= commonScalacOptions,
     libraryDependencies ++= Library.Akka.base,
     libraryDependencies ++= Seq(
-      Library.slf4jLog4j, Library.typesafeConfig, Library.scopt,
+      Library.slf4jLog4j, Library.log4j, Library.log4jExtras,
+      Library.typesafeConfig, Library.scopt,
       Library.slick, Library.h2, Library.flyway,
       Library.chill,
       Library.kafka, Library.pahoMqtt,
 
       Library.Akka.testKit % "test",
       Library.Akka.http, Library.Akka.httpSprayJson, Library.Akka.httpTestKit % "test",
-      Library.cats,
+      Library.cats, Library.catsEffect,
 
       Library.dockerJava,
 
+      Library.awsSdkEC2, Library.awsSdkEMR, Library.scalaSsh,
+
       "io.hydrosphere" %% "shadedshapeless" % "2.3.0",
       Library.commonsCodec, Library.scalajHttp,
       Library.jsr305 % "provided",
@@ -100,7 +102,7 @@ lazy val master = project.in(file("mist/master"))
   )
 
 lazy val worker = project.in(file("mist/worker"))
-  .dependsOn(core % "compile->compile;test->test")
+  .dependsOn(common % "compile->compile;test->test")
   .settings(commonSettings: _*)
   .settings(commonAssemblySettings: _*)
   .settings(
@@ -131,8 +133,45 @@ lazy val worker = project.in(file("mist/worker"))
     )
   )
 
+lazy val agent = project.in(file("mist/agent"))
+  .dependsOn(common % "compile->compile;test->test")
+  .settings(commonSettings: _*)
+  .settings(commonAssemblySettings: _*)
+  .enablePlugins(BuildInfoPlugin)
+  .settings(
+    name := "mist-agent",
+    scalacOptions ++= commonScalacOptions,
+    libraryDependencies ++= Library.Akka.base,
+    libraryDependencies ++= Seq(
+      Library.slf4jLog4j, Library.log4j, Library.log4jExtras,
+      Library.Akka.testKit % "test",
+
+      Library.awsSdkEMR,
+      Library.scalaTest % "test"
+    )
+  ).settings(
+  buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sparkVersion),
+  buildInfoPackage := "io.hydrosphere.mist"
+)
+
+lazy val awsInitSetup = project.in(file("mist/aws-init-setup"))
+  .dependsOn(common % "compile->compile;test->test")
+  .settings(commonSettings: _*)
+  .settings(commonAssemblySettings: _*)
+  .settings(
+    name := "mist-aws-init-setup",
+    scalacOptions ++= commonScalacOptions,
+    libraryDependencies ++= Seq(
+      Library.slf4jLog4j, Library.typesafeConfig,
+      Library.cats, Library.catsEffect,
+      Library.awsSdkEC2, Library.awsSdkIAM,
+      Library.jsr305 % "provided",
+      Library.scalaTest % "test"
+    )
+  )
+
 lazy val root = project.in(file("."))
-  .aggregate(mistLib, core, master, worker, examples)
+  .aggregate(mistLib, common, master, worker, examples)
   .dependsOn(master)
   .enablePlugins(DockerPlugin)
   .settings(commonSettings: _*)
@@ -150,6 +189,9 @@ lazy val root = project.in(file("."))
         CpFile("configs/logging").to("configs"),
         CpFile(assembly.in(master, assembly).value).as("mist-master.jar"),
         CpFile(assembly.in(worker, assembly).value).as("mist-worker.jar"),
+        CpFile(assembly.in(agent,  assembly).value).as("mist-agent.jar"),
+        MkDir("utils"),
+        CpFile(assembly.in(awsInitSetup, assembly).value).as("aws-init-setup.jar").to("utils"),
         CpFile(Ui.ui.value).as("ui")
       )
     },

diff --git a/mist/agent/src/main/resources/agent.conf b/mist/agent/src/main/resources/agent.conf
@@ -0,0 +1,27 @@
+akka {
+
+  loggers = ["akka.event.slf4j.Slf4jLogger"]
+  logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
+
+  loglevel = "INFO"
+  actor {
+    provider = "akka.remote.RemoteActorRefProvider"
+    warn-about-java-serializer-usage = false
+  }
+
+  remote {
+    netty.tcp {
+      port = 0
+      maximum-frame-size = 5242880b
+    }
+    transport-failure-detector {
+      heartbeat-interval = 30s
+      acceptable-heartbeat-pause = 5s
+    }
+    log-remote-lifecycle-events = off
+  }
+
+  http.server.transparent-head-requests = false
+  http.server.idle-timeout = infinite
+
+}
diff --git a/mist/agent/src/main/resources/log4j.properties b/mist/agent/src/main/resources/log4j.properties
@@ -0,0 +1,5 @@
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n