From 93466827cc75685eb2176017bff675690a1cd704 Mon Sep 17 00:00:00 2001
From: Fabio Buso <fabio@logicalclocks.com>
Date: Tue, 19 Sep 2023 12:41:21 +0200
Subject: [PATCH 01/14] Release version: 3.4.0

---
 Berksfile | 46 +++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/Berksfile b/Berksfile
index 366486c69..16989ed40 100644
--- a/Berksfile
+++ b/Berksfile
@@ -3,29 +3,29 @@ source 'https://supermarket.chef.io'
 
 metadata
 
-cookbook 'conda', github: "logicalclocks/conda-chef", branch: "master"
-cookbook 'kagent', github: "logicalclocks/kagent-chef", branch: "master"
-cookbook 'hops', github: "logicalclocks/hops-hadoop-chef", branch: "master"
-cookbook 'ndb', github: "logicalclocks/ndb-chef", branch: "master"
-cookbook 'hadoop_spark', github: "logicalclocks/spark-chef", branch: "master"
-cookbook 'flink', github: "logicalclocks/flink-chef", branch: "master"
-cookbook 'livy', github: "logicalclocks/livy-chef", branch: "master"
-cookbook 'epipe', github: "logicalclocks/epipe-chef", branch: "master"
-cookbook 'tensorflow', github: "logicalclocks/tensorflow-chef", branch: "master"
-cookbook 'kzookeeper', github: "logicalclocks/kzookeeper", branch: "master"
-cookbook 'kkafka', github: "logicalclocks/kafka-cookbook", branch: "master"
-cookbook 'elastic', github: "logicalclocks/elasticsearch-chef", branch: "master"
-cookbook 'hopslog', github: "logicalclocks/hopslog-chef", branch: "master"
-cookbook 'hopsmonitor', github: "logicalclocks/hopsmonitor-chef", branch: "master"
-cookbook 'hops_airflow', github: "logicalclocks/airflow-chef", branch: "master"
-cookbook 'hive2', github: "logicalclocks/hive-chef", branch: "master"
-cookbook 'consul', github: "logicalclocks/consul-chef", branch: "master"
-cookbook 'kube-hops', github: "logicalclocks/kube-hops-chef", branch: "master"
-cookbook 'cloud', github: "logicalclocks/cloud-chef", branch: "master"
-cookbook 'onlinefs', github: "logicalclocks/onlinefs-chef", branch: "master"
-cookbook 'flyingduck', github: "logicalclocks/flyingduck-chef", branch: "master"
+cookbook 'conda', github: "logicalclocks/conda-chef", branch: "3.4"
+cookbook 'kagent', github: "logicalclocks/kagent-chef", branch: "3.4"
+cookbook 'hops', github: "logicalclocks/hops-hadoop-chef", branch: "3.4"
+cookbook 'ndb', github: "logicalclocks/ndb-chef", branch: "3.4"
+cookbook 'hadoop_spark', github: "logicalclocks/spark-chef", branch: "3.4"
+cookbook 'flink', github: "logicalclocks/flink-chef", branch: "3.4"
+cookbook 'livy', github: "logicalclocks/livy-chef", branch: "3.4"
+cookbook 'epipe', github: "logicalclocks/epipe-chef", branch: "3.4"
+cookbook 'tensorflow', github: "logicalclocks/tensorflow-chef", branch: "3.4"
+cookbook 'kzookeeper', github: "logicalclocks/kzookeeper", branch: "3.4"
+cookbook 'kkafka', github: "logicalclocks/kafka-cookbook", branch: "3.4"
+cookbook 'elastic', github: "logicalclocks/elasticsearch-chef", branch: "3.4"
+cookbook 'hopslog', github: "logicalclocks/hopslog-chef", branch: "3.4"
+cookbook 'hopsmonitor', github: "logicalclocks/hopsmonitor-chef", branch: "3.4"
+cookbook 'hops_airflow', github: "logicalclocks/airflow-chef", branch: "3.4"
+cookbook 'hive2', github: "logicalclocks/hive-chef", branch: "3.4"
+cookbook 'consul', github: "logicalclocks/consul-chef", branch: "3.4"
+cookbook 'kube-hops', github: "logicalclocks/kube-hops-chef", branch: "3.4"
+cookbook 'cloud', github: "logicalclocks/cloud-chef", branch: "3.4"
+cookbook 'onlinefs', github: "logicalclocks/onlinefs-chef", branch: "3.4"
+cookbook 'flyingduck', github: "logicalclocks/flyingduck-chef", branch: "3.4"
 
-cookbook 'ulimit', github: "logicalclocks/chef-ulimit", branch: "master"
-cookbook 'glassfish', github: "logicalclocks/chef-glassfish", branch: "master"
+cookbook 'ulimit', github: "logicalclocks/chef-ulimit", branch: "3.4"
+cookbook 'glassfish', github: "logicalclocks/chef-glassfish", branch: "3.4"
 cookbook 'java', github: "logicalclocks/java", branch: "v7.0.0-1"
 cookbook 'packagecloud', '= 1.0.1'

From 00c1394f020949bee944758a437f241cc14281d0 Mon Sep 17 00:00:00 2001
From: Fabio Buso <buso.fabio@gmail.com>
Date: Tue, 19 Sep 2023 14:35:14 +0200
Subject: [PATCH 02/14] [HWORKS-749] Remove chef-ulimit dependency from
 hopsworks-chef Berksfile (#1045)

---
 Berksfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Berksfile b/Berksfile
index 16989ed40..229ca7a9c 100644
--- a/Berksfile
+++ b/Berksfile
@@ -25,7 +25,6 @@ cookbook 'cloud', github: "logicalclocks/cloud-chef", branch: "3.4"
 cookbook 'onlinefs', github: "logicalclocks/onlinefs-chef", branch: "3.4"
 cookbook 'flyingduck', github: "logicalclocks/flyingduck-chef", branch: "3.4"
 
-cookbook 'ulimit', github: "logicalclocks/chef-ulimit", branch: "3.4"
 cookbook 'glassfish', github: "logicalclocks/chef-glassfish", branch: "3.4"
 cookbook 'java', github: "logicalclocks/java", branch: "v7.0.0-1"
 cookbook 'packagecloud', '= 1.0.1'

From 2c32d118c99c3f5b9a259c6cae187f8900eab697 Mon Sep 17 00:00:00 2001
From: Alex Ormenisan <ormenisan.adrian@gmail.com>
Date: Thu, 21 Sep 2023 14:16:08 +0300
Subject: [PATCH 03/14] [HWORKS-547][APPEND] reindex if required to (#1049)

---
 Karamelfile        |  5 +++-
 metadata.rb        |  1 +
 recipes/reindex.rb | 63 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 68 insertions(+), 1 deletion(-)
 create mode 100644 recipes/reindex.rb

diff --git a/Karamelfile b/Karamelfile
index 125fd5060..f298811bc 100755
--- a/Karamelfile
+++ b/Karamelfile
@@ -38,4 +38,7 @@ dependencies:
   - recipe: hopsworks::loadbalancer
     global:
       - hopsworks::config_node
-    
+  - recipe: hopsworks::reindex
+    global:
+      - hopsworks::default
+      - elastic::default
diff --git a/metadata.rb b/metadata.rb
index 2521d519e..eebe01bbf 100644
--- a/metadata.rb
+++ b/metadata.rb
@@ -51,6 +51,7 @@
 recipe  "hopsworks::migrate", "Call expat to migrate between Hopsworks versions"
 
 recipe  "hopsworks::purge", "Deletes glassfish installation."
+recipe  "hopsworks::reindex", "Reindex the featurestore search index"
 #######################################################################################
 # Required Attributes
 #######################################################################################
diff --git a/recipes/reindex.rb b/recipes/reindex.rb
new file mode 100644
index 000000000..239dfed2c
--- /dev/null
+++ b/recipes/reindex.rb
@@ -0,0 +1,63 @@
+ruby_block 'featurestore reindex' do
+  block do
+    require 'net/https'
+    require 'http-cookie'
+    require 'json'
+    require 'securerandom'
+
+    hopsworks_fqdn = consul_helper.get_service_fqdn("hopsworks.glassfish")
+    _, hopsworks_port = consul_helper.get_service("glassfish", ["http", "hopsworks"])
+    if hopsworks_port.nil? || hopsworks_fqdn.nil?
+      raise "Could not get Hopsworks faqdn/port from local Consul agent. Verify Hopsworks is running with service name: glassfish and tags: [http, hopsworks]"
+    end
+
+    hopsworks_endpoint = "https://#{hopsworks_fqdn}:#{hopsworks_port}"
+    url = URI.parse("#{hopsworks_endpoint}/hopsworks-api/api/auth/service")
+    reindex_url = URI.parse("#{hopsworks_endpoint}/hopsworks-api/api/admin/search/featurestore/reindex")
+
+    params =  {
+      :email => node["kagent"]["dashboard"]["user"] ,
+      :password => node["kagent"]["dashboard"]["password"]
+    }
+
+    http = Net::HTTP.new(url.host, url.port)
+    http.read_timeout = 120
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+    jar = ::HTTP::CookieJar.new
+
+    http.start do |connection|
+
+      request = Net::HTTP::Post.new(url)
+      request.set_form_data(params, '&')
+      response = connection.request(request)
+
+      if( response.is_a?( Net::HTTPSuccess ) )
+          # your request was successful
+          puts "Hopsworks Admin login successful: -> #{response.body}"
+
+          response.get_fields('Set-Cookie').each do |value|
+            jar.parse(value, url)
+          end
+
+          request = Net::HTTP::Post.new(reindex_url)
+          request['Content-Type'] = "application/json"
+          request['Cookie'] = ::HTTP::Cookie.cookie_value(jar.cookies(reindex_url))
+          request['Authorization'] = response['Authorization']
+          response = connection.request(request)
+
+          if ( response.is_a? (Net::HTTPSuccess))
+            puts "Hopsworks Reindex - triggered"
+          else
+            puts response.body
+            raise "Hopsworks Reindex - failed: #{response.uri}"
+          end
+      else
+          puts response.body
+          raise "Hopsworks Reindex - failed"
+      end
+    end
+  end
+  only_if { node['elastic']['featurestore']['reindex'] == "true" }
+end
\ No newline at end of file

From 56c9fae6a5a0542312e169fbec13b5adde87adcf Mon Sep 17 00:00:00 2001
From: Fabio Buso <buso.fabio@gmail.com>
Date: Mon, 25 Sep 2023 10:16:35 +0200
Subject: [PATCH 04/14] [HWORKS-757] das_node and config_node recipes don't
 start glassfish-instance systemd unit (#1052) (#1053)

---
 providers/worker.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/providers/worker.rb b/providers/worker.rb
index cba11a648..f1625e093 100644
--- a/providers/worker.rb
+++ b/providers/worker.rb
@@ -37,8 +37,12 @@
               start_domain_timeout: systemd_start_timeout,
               stop_domain_timeout: systemd_stop_timeout,
               authbind: new_resource.requires_authbind)
-    notifies :start, "service[#{service_name}]", :delayed
   end
+
+  kagent_config service_name do
+    action :systemd_reload
+  end
+
   if node['services']['enabled'].casecmp?("true")
     service service_name do
       supports start: true, restart: true, stop: true, status: true

From 901e3c13848290d79d9cae7a50b4f269b82705e1 Mon Sep 17 00:00:00 2001
From: Mahmoud Ismail <mahmoud@hopsworks.ai>
Date: Tue, 26 Sep 2023 16:55:10 +0200
Subject: [PATCH 05/14] [FSTORE-1010][Append] Encapsulate update statement with
 SQL_SAFE_UPDATES (#1057)

---
 files/default/sql/ddl/updates/3.4.0.sql | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/files/default/sql/ddl/updates/3.4.0.sql b/files/default/sql/ddl/updates/3.4.0.sql
index 3b79065c8..d01da2672 100644
--- a/files/default/sql/ddl/updates/3.4.0.sql
+++ b/files/default/sql/ddl/updates/3.4.0.sql
@@ -177,9 +177,10 @@ WHERE REGEXP_SUBSTR(`subject`, "^([0-9]+)_([0-9]+)_(.+)_([0-9]+)(_onlinefs|$)");
 UPDATE `hopsworks`.`feature_group` `fg`
     JOIN `hopsworks`.`feature_store` `fs` ON `fg`.`feature_store_id` = `fs`.`id`
 SET `fg`.`topic_name` = CONCAT(fs.project_id, "_", fg.id, "_", fg.name, "_", fg.version, IF(fg.online_enabled , "_onlinefs", ""));
-SET SQL_SAFE_UPDATES = 1;
 
 -- FSTORE-1010: Don't leave orphaned subjects when deleting online enabled fg
 UPDATE `hopsworks`.`project_topics`
 SET `subject_id` = NULL
-    WHERE REGEXP_SUBSTR(`topic_name`, "^([0-9]+)_([0-9]+)_(.+)_([0-9]+)(_onlinefs|$)");
\ No newline at end of file
+    WHERE REGEXP_SUBSTR(`topic_name`, "^([0-9]+)_([0-9]+)_(.+)_([0-9]+)(_onlinefs|$)");
+
+SET SQL_SAFE_UPDATES = 1;
\ No newline at end of file

From 3e329f792c67bd933c26fda19d3ed066366eed6d Mon Sep 17 00:00:00 2001
From: Fabio Buso <buso.fabio@gmail.com>
Date: Wed, 27 Sep 2023 21:01:27 +0200
Subject: [PATCH 06/14] =?UTF-8?q?[HWORKS-762]=20Don't=20try=20to=20undeplo?=
 =?UTF-8?q?y=20hopsworks-*=20current=5Fversion=20if=20it'=E2=80=A6=20(#105?=
 =?UTF-8?q?8)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [HWORKS-762] Don't try to undeploy hopsworks-* current_version if it'… (#1055)

* [HWORKS-762] Don't try to undeploy hopsworks-* current_version if it's not deployed

* Fix
---
 recipes/default.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/recipes/default.rb b/recipes/default.rb
index c8dc19275..788b1776d 100644
--- a/recipes/default.rb
+++ b/recipes/default.rb
@@ -374,6 +374,7 @@
     keep_state true
     enabled true
     secure true
+    only_if "#{asadmin_cmd} list-applications --type ejb domain | grep -w \"hopsworks-ear:#{node['hopsworks']['current_version']}\""
   end
 
   glassfish_deployable "hopsworks" do
@@ -391,6 +392,7 @@
     retries 1
     keep_state true
     enabled true
+    only_if "#{asadmin_cmd} list-applications --type web domain | grep -w \"hopsworks-web:#{node['hopsworks']['current_version']}\""
   end
 
   glassfish_deployable "hopsworks-ca" do
@@ -408,6 +410,7 @@
     retries 1
     keep_state true
     enabled true
+    only_if "#{asadmin_cmd} list-applications --type ejb domain | grep -w \"hopsworks-ca:#{node['hopsworks']['current_version']}\""
   end
 end  
 

From 55edf998ff9441691f821d7995c7e97c28efed9e Mon Sep 17 00:00:00 2001
From: Antonis Kouzoupis <antonios@logicalclocks.com>
Date: Thu, 26 Oct 2023 10:13:10 +0200
Subject: [PATCH 07/14] [HWORKS-816] Insert multiregion configuration in
 Hopsworks settings (#1065)

---
 templates/default/sql/dml/3.4.0.sql.erb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/templates/default/sql/dml/3.4.0.sql.erb b/templates/default/sql/dml/3.4.0.sql.erb
index e96f31ebc..7b2d04974 100644
--- a/templates/default/sql/dml/3.4.0.sql.erb
+++ b/templates/default/sql/dml/3.4.0.sql.erb
@@ -28,3 +28,8 @@ REPLACE INTO `hopsworks`.`variables`(`id`, `value`, `visibility`) VALUES ("featu
 REPLACE INTO `hopsworks`.`variables`(`id`, `value`, `visibility`) VALUES ("featurestore_db_admin_pwd", "<%= node['featurestore']['password'] %>", 0);
 
 REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("ldap_groups_search_filter", "<%= node['ldap']['groups_search_filter'] %>");
+
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_enabled", "<%= node['hops']['alive-watchdog']['enabled'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_interval", "<%= node['hops']['alive-watchdog']['interval'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_url", "<%= node['hops']['alive-watchdog']['http-poll']['url'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_region", "<%= node['hops']['alive-watchdog']['json-poll']['dc-id'] %>");
\ No newline at end of file

From 643808e6f39f00fd1252c3e6a451a9ff83d77a6c Mon Sep 17 00:00:00 2001
From: Antonis Kouzoupis <antonios@logicalclocks.com>
Date: Fri, 27 Oct 2023 16:37:12 +0200
Subject: [PATCH 08/14] [HWORKS-815] Remove requires directive so judge service
 will not be stopped when stopping docker. Instead it will be killed and
 restarted (#1067)

---
 templates/default/judge/judge.service.erb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/templates/default/judge/judge.service.erb b/templates/default/judge/judge.service.erb
index 746d15c5a..d3e5792c4 100644
--- a/templates/default/judge/judge.service.erb
+++ b/templates/default/judge/judge.service.erb
@@ -1,7 +1,6 @@
 [Unit]
 Description = Judge service advertising which region is active
 After = network-online.target docker.service
-Requires = docker.service
 
 [Service]
 ExecStart = /usr/bin/docker run --rm --name %n \

From 0226116c3f3f87ee3f0baef4dbc9dd1f7eba5389 Mon Sep 17 00:00:00 2001
From: Antonis Kouzoupis <antonios@logicalclocks.com>
Date: Thu, 2 Nov 2023 10:46:22 +0100
Subject: [PATCH 09/14] [HWORKS-826] Docker build should push to both region
 registries (#1069) (#1071)

[HWORKS-826] Remove old code

Co-authored-by: Fabio Buso <buso.fabio@gmail.com>
---
 templates/default/dockerImage.sh.erb | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/templates/default/dockerImage.sh.erb b/templates/default/dockerImage.sh.erb
index c10f920d8..9019690c2 100644
--- a/templates/default/dockerImage.sh.erb
+++ b/templates/default/dockerImage.sh.erb
@@ -25,12 +25,18 @@ if [ "$1" == "create" ] ; then
     fi
 
     # Remove single quotes from args
-    DOCKER_BUILD_OPTS=$(echo $4 | xargs)
+    # Use the last argument as building opts
+    DOCKER_BUILD_OPTS=$(echo ${@: -1}  | xargs)
 
     DOCKER_BUILDKIT=1 docker build $DOCKER_BUILD_OPTS -f $2 --tag $3 ./
 
     docker push $3
 
+    if ! [[ "$4" == "" ]] ; then
+        docker tag $3 $4
+        docker push $4
+    fi
+
 <% if node['install']['managed_docker_registry'] == "false" -%>
 elif [ "$1" == "delete" ] ; then
     # Remove tags from docker registry

From d4ad939edb963c68714d51df20365c8ffd5a3e63 Mon Sep 17 00:00:00 2001
From: Gibson Chikafa <gibbschikafa@gmail.com>
Date: Wed, 15 Nov 2023 13:58:26 +0200
Subject: [PATCH 10/14] Rebase on 3.4

---
 attributes/default.rb                         |  32 +++-
 .../hopsworks_templates/config_template.json  |   1 +
 .../rstudio_logging_config_template           |   9 +
 .../rstudio_rserver_config_template           |   5 +
 .../rstudio_rsession_config_template          |   2 +
 .../hopsworks_templates/sparklyr_config.yml   |  10 ++
 .../default/sql/ddl/3.4.0__initial_tables.sql |  43 ++---
 recipes/install.rb                            |  38 +++-
 templates/default/rstudio-kill.sh.erb         |  26 +++
 templates/default/rstudio-launch.sh.erb       | 130 ++++++++++++++
 .../default/rstudio-project-cleanup.sh.erb    |  30 ++++
 templates/default/rstudio.sh.erb              | 164 ++++++++++++++++++
 templates/default/sql/dml/3.4.0.sql.erb       |  15 +-
 13 files changed, 475 insertions(+), 30 deletions(-)
 create mode 100644 files/default/hopsworks_templates/rstudio_logging_config_template
 create mode 100644 files/default/hopsworks_templates/rstudio_rserver_config_template
 create mode 100644 files/default/hopsworks_templates/rstudio_rsession_config_template
 create mode 100644 files/default/hopsworks_templates/sparklyr_config.yml
 create mode 100644 templates/default/rstudio-kill.sh.erb
 create mode 100644 templates/default/rstudio-launch.sh.erb
 create mode 100644 templates/default/rstudio-project-cleanup.sh.erb
 create mode 100644 templates/default/rstudio.sh.erb

diff --git a/attributes/default.rb b/attributes/default.rb
index c7f1a20db..ac4ec0a08 100644
--- a/attributes/default.rb
+++ b/attributes/default.rb
@@ -521,4 +521,34 @@
 default['judge']['port']      = "1111"
 default['judge']['home']      = "#{node['install']['dir']}/judge"
 default['judge']['etc']       = "#{node['judge']['home']}/etc"
-default['judge']['logs']      = "#{node['judge']['home']}/logs"
\ No newline at end of file
+default['judge']['logs']      = "#{node['judge']['home']}/logs"
+
+
+default['hopsworks']['rstudio_dir'] = node['hopsworks']['dir'] + "/rstudio"
+#rstudio configuration variables
+default["hopsworks"]['rstudio_host']                 = "localhost"
+default["hopsworks"]['rstudio_origin_scheme']        = "https"
+default["hopsworks"]["rstudio_www_address"]          = "0.0.0.0"
+default["hopsworks"]["rstudio_session_timeout_minutes"] = 360
+default["hopsworks"]["rstudio_logging_level"]        = "info"
+default["hopsworks"]["rstudio_logger_type"]          = "file"
+default["hopsworks"]["rstudio_log_file_max_size"]    = 512
+default["hopsworks"]["rstudio_default_cran_repo"]    = "https://cloud.r-project.org/"
+
+default['rstudio']['base_dir']                       = node['install']['dir'].empty? ? node['hopsworks']['dir'] + "/rstudio" : node['install']['dir'] + "/rstudio"
+default['rstudio']['shutdown_timer_interval']          = "30m"
+
+# CRAN
+default['rstudio']['cran']['mirror'] = 'http://cran.rstudio.com/'
+
+# APT configuration for Ubuntu or Debian installs.
+case node["platform"].downcase
+when "ubuntu"
+  default['rstudio']['apt']['key'] = 'E084DAB9'
+  default['rstudio']['apt']['keyserver'] = 'keyserver.ubuntu.com'
+  default['rstudio']['apt']['uri'] = 'http://cran.stat.ucla.edu/bin/linux/ubuntu'
+when "debian"
+  default['rstudio']['apt']['key'] = '381BA480'
+  default['rstudio']['apt']['keyserver'] = 'subkeys.pgp.net'
+  default['rstudio']['apt']['uri'] = 'http://cran.stat.ucla.edu/bin/linux/debian'
+end
\ No newline at end of file
diff --git a/files/default/hopsworks_templates/config_template.json b/files/default/hopsworks_templates/config_template.json
index 19fca1939..21a73e4ed 100644
--- a/files/default/hopsworks_templates/config_template.json
+++ b/files/default/hopsworks_templates/config_template.json
@@ -50,6 +50,7 @@
     "executorCores": ${conf.executorCores?c},
     "executorMemory": "${conf.executorMemory}",
     "proxyUser": "${conf.hdfsUser}",
+    "name": "${conf.livySessionName}",
     "queue": "${conf.yarnQueue}",
     "conf": {
 ${conf.sparkConfiguration}
diff --git a/files/default/hopsworks_templates/rstudio_logging_config_template b/files/default/hopsworks_templates/rstudio_logging_config_template
new file mode 100644
index 000000000..d89a89a82
--- /dev/null
+++ b/files/default/hopsworks_templates/rstudio_logging_config_template
@@ -0,0 +1,9 @@
+[*]
+log-level=warn
+logger-type=syslog
+
+[@rserver]
+log-level=${conf.logLevel}
+logger-type=${conf.loggerType}
+max-size-mb=${conf.maxSizeMb}
+log-file-include-pid=${conf.includePid}
\ No newline at end of file
diff --git a/files/default/hopsworks_templates/rstudio_rserver_config_template b/files/default/hopsworks_templates/rstudio_rserver_config_template
new file mode 100644
index 000000000..e4c6a7856
--- /dev/null
+++ b/files/default/hopsworks_templates/rstudio_rserver_config_template
@@ -0,0 +1,5 @@
+rsession-which-r=${conf.versionPath}
+www-address=${conf.ipAddress}
+www-port=${conf.port}
+www-root-path=${conf.rootPath}
+server-user=${conf.serverUser}
\ No newline at end of file
diff --git a/files/default/hopsworks_templates/rstudio_rsession_config_template b/files/default/hopsworks_templates/rstudio_rsession_config_template
new file mode 100644
index 000000000..42cc2e1d4
--- /dev/null
+++ b/files/default/hopsworks_templates/rstudio_rsession_config_template
@@ -0,0 +1,2 @@
+session-timeout-minutes=${conf.sessionTimeoutMinutes}
+r-cran-repos=${conf.cranRepo}
\ No newline at end of file
diff --git a/files/default/hopsworks_templates/sparklyr_config.yml b/files/default/hopsworks_templates/sparklyr_config.yml
new file mode 100644
index 000000000..2d3b4e3d7
--- /dev/null
+++ b/files/default/hopsworks_templates/sparklyr_config.yml
@@ -0,0 +1,10 @@
+default:
+  livy.driverCores: ${conf.driverCores}
+  livy.driverMemory: "${conf.driverMemory}"
+  livy.numExecutors: ${conf.numExecutors}
+  livy.executorCores: ${conf.executorCores}
+  livy.executorMemory: "${conf.executorMemory}"
+  livy.proxyUser: "${conf.proxyUser}"
+  livy.queue: "${conf.yarnQueue}"
+  livy.name: "${conf.livyAppName}"
+${conf.sparkConfiguration}
\ No newline at end of file
diff --git a/files/default/sql/ddl/3.4.0__initial_tables.sql b/files/default/sql/ddl/3.4.0__initial_tables.sql
index d1a5fd182..b8c3c6933 100644
--- a/files/default/sql/ddl/3.4.0__initial_tables.sql
+++ b/files/default/sql/ddl/3.4.0__initial_tables.sql
@@ -1084,17 +1084,21 @@ CREATE TABLE `rstudio_interpreter` (
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `rstudio_project` (
-                                   `port` int(11) NOT NULL,
+                                   `port` int NOT NULL,
+                                   `hdfs_user_id` int NOT NULL,
                                    `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                                   `expires` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
                                    `last_accessed` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-                                   `host_ip` varchar(255) COLLATE latin1_general_cs NOT NULL,
-                                   `token` varchar(255) COLLATE latin1_general_cs NOT NULL,
-                                   `secret` varchar(64) COLLATE latin1_general_cs NOT NULL,
-                                   `pid` bigint(20) NOT NULL,
-                                   `project_id` int(11) NOT NULL,
+                                   `secret` varchar(64) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                   `pid` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                   `project_id` int NOT NULL,
+                                   `login_password` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
+                                   `login_username` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
                                    PRIMARY KEY (`port`),
+                                   KEY `hdfs_user_idx` (`hdfs_user_id`),
                                    KEY `project_id` (`project_id`),
-                                   CONSTRAINT `FK_284_578` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
+                                   CONSTRAINT `FK_103_577` FOREIGN KEY (`hdfs_user_id`) REFERENCES `hops`.`hdfs_users` (`id`) ON DELETE CASCADE,
+                                   CONSTRAINT `FK_284_578` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE
 ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
@@ -1107,32 +1111,17 @@ CREATE TABLE `rstudio_project` (
 CREATE TABLE `rstudio_settings` (
                                     `project_id` int(11) NOT NULL,
                                     `team_member` varchar(150) COLLATE latin1_general_cs NOT NULL,
-                                    `num_tf_ps` int(11) DEFAULT '1',
-                                    `num_tf_gpus` int(11) DEFAULT '0',
-                                    `num_mpi_np` int(11) DEFAULT '1',
-                                    `appmaster_cores` int(11) DEFAULT '1',
-                                    `appmaster_memory` int(11) DEFAULT '1024',
-                                    `num_executors` int(11) DEFAULT '1',
-                                    `num_executor_cores` int(11) DEFAULT '1',
-                                    `executor_memory` int(11) DEFAULT '1024',
-                                    `dynamic_initial_executors` int(11) DEFAULT '1',
-                                    `dynamic_min_executors` int(11) DEFAULT '1',
-                                    `dynamic_max_executors` int(11) DEFAULT '1',
                                     `secret` varchar(255) COLLATE latin1_general_cs NOT NULL,
-                                    `log_level` varchar(32) COLLATE latin1_general_cs DEFAULT 'INFO',
-                                    `mode` varchar(32) COLLATE latin1_general_cs NOT NULL,
-                                    `umask` varchar(32) COLLATE latin1_general_cs DEFAULT '022',
                                     `advanced` tinyint(1) DEFAULT '0',
-                                    `archives` varchar(1500) COLLATE latin1_general_cs DEFAULT '',
-                                    `jars` varchar(1500) COLLATE latin1_general_cs DEFAULT '',
-                                    `files` varchar(1500) COLLATE latin1_general_cs DEFAULT '',
-                                    `py_files` varchar(1500) COLLATE latin1_general_cs DEFAULT '',
-                                    `spark_params` varchar(6500) COLLATE latin1_general_cs DEFAULT '',
                                     `shutdown_level` int(11) NOT NULL DEFAULT '6',
+                                    `base_dir` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+                                    `job_config` varchar(11000) COLLATE latin1_general_cs DEFAULT NULL,
+                                    `docker_config` varchar(1000) COLLATE latin1_general_cs DEFAULT NULL,
                                     PRIMARY KEY (`project_id`,`team_member`),
                                     KEY `team_member` (`team_member`),
                                     KEY `secret_idx` (`secret`),
-                                    CONSTRAINT `RS_FK_USERS` FOREIGN KEY (`team_member`) REFERENCES `users` (`email`) ON DELETE CASCADE ON UPDATE NO ACTION,
+                                    CONSTRAINT `RS_FK_USERS` FOREIGN KEY (`team_member`) REFERENCES `users` (`email`) ON DELETE CASCADE ON UPDATE NO
+                                        ACTION,
                                     CONSTRAINT `RS_FK_PROJS` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
 ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
 /*!40101 SET character_set_client = @saved_cs_client */;
diff --git a/recipes/install.rb b/recipes/install.rb
index 5300c275b..a63636b66 100644
--- a/recipes/install.rb
+++ b/recipes/install.rb
@@ -147,6 +147,14 @@
   action :create
 end
 
+#update permissions of base_dir for rstudio to 770
+directory node['rstudio']['base_dir']  do
+  owner node['hops']['yarnapp']['user']
+  group node['hops']['group']
+  mode "770"
+  action :create
+end
+
 directory node['hopsworks']['dir']  do
   owner node['hopsworks']['user']
   group node['hopsworks']['group']
@@ -641,6 +649,23 @@
   not_if       { node['install']['kubernetes'].casecmp("true") == 0 }
 end
 
+kagent_sudoers "rstudio" do
+  user          node['glassfish']['user']
+  group         "root"
+  script_name   "rstudio.sh"
+  template      "rstudio.sh.erb"
+  run_as        "ALL" # run this as root - inside we change to different users
+end
+
+kagent_sudoers "rstudio-project-cleanup" do
+  user          node['glassfish']['user']
+  group         "root"
+  script_name   "rstudio-project-cleanup.sh"
+  template      "rstudio-project-cleanup.sh.erb"
+  run_as        "ALL"
+  not_if       { node['install']['kubernetes'].casecmp("true") == 0 }
+end
+
 kagent_sudoers "convert-ipython-notebook" do 
   user          node['glassfish']['user']
   group         "root"
@@ -739,7 +764,7 @@
 end
 
 ["tensorboard-launch.sh", "tensorboard-cleanup.sh", "condasearch.sh", "list_environment.sh", "jupyter-kill.sh",
-"tfserving-kill.sh", "sklearn_serving-launch.sh", "sklearn_serving-kill.sh", "git-container-kill.sh"].each do |script|
+"tfserving-kill.sh", "sklearn_serving-launch.sh", "sklearn_serving-kill.sh", "git-container-kill.sh", "rstudio-kill.sh"].each do |script|
   template "#{theDomain}/bin/#{script}" do
     source "#{script}.erb"
     owner node['glassfish']['user']
@@ -760,6 +785,17 @@
             })
 end
 
+template "#{theDomain}/bin/rstudio-launch.sh" do
+  source "rstudio-launch.sh.erb"
+  owner node['glassfish']['user']
+  group node['glassfish']['group']
+  mode "500"
+  action :create
+  variables({
+              :namenode_fdqn => namenode_fdqn,
+            })
+end
+
 template "#{theDomain}/bin/git-container-launch.sh" do
   source "git-container-launch.sh.erb"
   owner node['glassfish']['user']
diff --git a/templates/default/rstudio-kill.sh.erb b/templates/default/rstudio-kill.sh.erb
new file mode 100644
index 000000000..b2e5747c1
--- /dev/null
+++ b/templates/default/rstudio-kill.sh.erb
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+help() {
+    echo ""
+    echo "usage: $0 CONTAINER_ID PROJECT_USER_NAME"
+    echo ""
+    exit 1
+}
+
+
+if [ $# -ne 2 ]; then
+  help
+fi
+
+if [ "$2" != "" ]; then
+    PROJECT_USER_NAME=$2
+    CONTAINER_NAME=${PROJECT_USER_NAME}__rstudio
+
+    echo "Killing input container_name: $CONTAINER_NAME"
+    docker rm -f "$CONTAINER_NAME" > /dev/null 2>&1
+    exit $?
+fi
+
+echo "Killing input container_id: $1"
+docker rm -f "$1" > /dev/null 2>&1
+exit $?
\ No newline at end of file
diff --git a/templates/default/rstudio-launch.sh.erb b/templates/default/rstudio-launch.sh.erb
new file mode 100644
index 000000000..a7e1fde6d
--- /dev/null
+++ b/templates/default/rstudio-launch.sh.erb
@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+WAIT_START=60
+RSTUDIO_HOME=$1
+HADOOP_HOME=$2
+HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
+HADOOP_USERNAME=$3
+PORT=$4
+SECRET_DIR=$5
+CERTS_DIR=$6
+IMAGE=$7
+LOGFILE=${RSTUDIO_HOME}/logs/$8
+PROJECT_NAME=$9
+LIVY_IP=${10}
+LIVY_PORT=${11}
+HADOOP_BASE_DIR=${12}
+SERVER_PASSWORD=${13}
+HADOOP_VERSION=${14}
+SPARK_VERSION=${15}
+HADOOP_CLIENT_ENV_OPTS='-D fs.permissions.umask-mode=0002'
+CONTAINER_NAME=${HADOOP_USERNAME}__rstudio
+PID_FILE=${RSTUDIO_HOME}/run/rstudio.pid
+SPARK_CONF_DIR=/srv/hops/spark/conf
+FLINK_CONF_DIR=/srv/hops/flink/conf
+NOT_FOUND=127
+SPARKLYR_CONFIG_FILE=${RSTUDIO_HOME}/conf/config.yml
+NAMENODE_IP=<%= @namenode_fdqn %>
+NAMENODE_PORT=<%= node['hops']['nn']['port'] %>
+CLIENT_CERTIFICATES_BUNDLE=$CERTS_DIR/certificate_bundle.pem
+ROOT_CA_BUNDLE=$CERTS_DIR/root_ca.pem
+CLIENT_KEY=$CERTS_DIR/private_key.pem
+
+
+help() {
+    echo ""
+    echo "usage: $0 RSTUDIO_HOME HADOOP_HOME HADOOP_USERNAME PORT SECRET_DIR CERTS_DIR IMAGE LOGFILE PROJECT_NAME LIVY_IP LIVY_PORT HADOOP_BASE_DIR SERVER_PASSWORD HADOOP_VERSION SPARK_VERSION"
+    echo ""
+    exit 1
+}
+
+function kill_named {
+    CID=$(docker container list -a | grep $CONTAINER_NAME | grep -v grep | awk '{print $1}')
+    if [ "$CID" != "" ] ; then
+        docker rm -f "$CID" > /dev/null 2>&1
+        res=$?
+    else
+        res=$NOT_FOUND
+    fi
+    return "$res"
+}
+
+if [ $# -ne 15 ]; then
+  help
+fi
+
+#check if the folders exist
+cd "$RSTUDIO_HOME" || exit
+cd "$SECRET_DIR" || exit
+
+kill_named
+
+if [ -f "$PID_FILE" ] ; then
+    rm $PID_FILE
+fi
+
+touch $LOGFILE
+if [ $? -ne 0 ] ; then
+  echo "Error: could not create the log file for rstudio server"
+  exit 1
+fi
+chmod 766 $LOGFILE
+
+docker run --rm -d --cap-add SYS_ADMIN --device /dev/fuse --security-opt apparmor:unconfined --name $CONTAINER_NAME --cidfile=$PID_FILE\
+       --network=host \
+       --init \
+       -e "RSTUDIO_PATH=$RSTUDIO_HOME" \
+       -e "RSTUDIO_DATA_DIR=$RSTUDIO_HOME" \
+       -e "PDIR=$SECRET_DIR" \
+       -e "RSTUDIO_CONFIG_DIR=${RSTUDIO_HOME}/conf" \
+       -e "RSTUDIO_RUNTIME_DIR=${RSTUDIO_HOME}/run" \
+       -e "HADOOP_HDFS_HOME=${HADOOP_HOME}" \
+       -e "HADOOP_CONF_DIR=${HADOOP_CONF_DIR}" \
+       -e "HADOOP_CLIENT_OPTS='-Dfs.permissions.umask-mode=0002'" \
+       -e "MATERIAL_DIRECTORY=$CERTS_DIR" \
+       -e "HADOOP_USERNAME=$HADOOP_USERNAME" \
+       -e "HADOOP_HOME=${HADOOP_HOME}" \
+       -e "LOGFILE=${LOGFILE}" \
+       -e "RSTUDIO_PORT=${PORT}" \
+       -e "PROJECT_NAME=${PROJECT_NAME}" \
+       -e "LIVY_IP=${LIVY_IP}" \
+       -e "LIVY_PORT=${LIVY_PORT}" \
+       -e "HADOOP_BASE_DIR=${HADOOP_BASE_DIR}" \
+       -e "HADOOP_CLIENT_ENV_OPTS=${HADOOP_CLIENT_ENV_OPTS}" \
+       -e "SPARKLYR_CONFIG_FILE=${SPARKLYR_CONFIG_FILE}" \
+       -e "SERVER_PASSWORD=${SERVER_PASSWORD}" \
+       -e "HADOOP_VERSION=${HADOOP_VERSION}" \
+       -e "NAMENODE_IP=${NAMENODE_IP}" \
+       -e "NAMENODE_PORT=${NAMENODE_PORT}" \
+       -e "SPARK_VERSION=${SPARK_VERSION}" \
+       -e "CLIENT_CERTIFICATES_BUNDLE=${CLIENT_CERTIFICATES_BUNDLE}" \
+       -e "ROOT_CA_BUNDLE=${ROOT_CA_BUNDLE}" \
+       -e "CLIENT_KEY=${CLIENT_KEY}" \
+       -v $RSTUDIO_HOME:$RSTUDIO_HOME:rw\
+       -v $SECRET_DIR:$SECRET_DIR:rw\
+       -v ${HADOOP_CONF_DIR}:${HADOOP_CONF_DIR}:ro \
+       -v ${SPARK_CONF_DIR}:${SPARK_CONF_DIR}:ro \
+       -v $LOGFILE:"/var/log/rstudio-server/rserver.log":rw\
+       -u="yarnapp" \
+       -w="$SECRET_DIR" \
+       $IMAGE & \
+
+# Wait for rstudio to start
+timeout=0
+while [ $timeout -lt $WAIT_START ] ; do
+  docker logs $(cat $PID_FILE) | grep "...done"
+        if [ $? -eq 0 ] ; then
+          break
+        fi
+  echo -n "."
+  timeout=$((timeout + 1))
+done
+echo ""
+
+# If the timeout was exceeded, kill rstudio
+if [ "$timeout" -eq $WAIT_START ] ; then
+ kill_named
+fi
+
+
+exit $?
\ No newline at end of file
diff --git a/templates/default/rstudio-project-cleanup.sh.erb b/templates/default/rstudio-project-cleanup.sh.erb
new file mode 100644
index 000000000..41bad3f01
--- /dev/null
+++ b/templates/default/rstudio-project-cleanup.sh.erb
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# This script allows Hopsworks to cleanup local directories for RStudio servers.
+# There will be 1 RStudio server per ProjectUser.
+# This script can be run by hopsworks (running as user 'glassfish') as a sudo command as 'root'
+# The script should run as 'root' as the rstudio user does not have read privileges on the base directory for a user's
+# project and cannot do a recursive delete on the directory.
+#
+
+help() {
+    echo ""
+    echo "usage: $0 project"
+    echo "e.g., "
+    exit 1
+}
+
+if [ $# -ne 1 ]; then
+    help
+fi
+
+base="<%= node['rstudio']['base_dir'] %>/Projects/$1"
+# Sanity checks for injection attacks
+if [ ! -d "$base" ] ; then
+    echo "Invalid RSTUDIO_HOME directory: $base"
+    exit 1
+fi
+
+rm -rf $base
+
+exit $?
\ No newline at end of file
diff --git a/templates/default/rstudio.sh.erb b/templates/default/rstudio.sh.erb
new file mode 100644
index 000000000..0fd8beef1
--- /dev/null
+++ b/templates/default/rstudio.sh.erb
@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+
+# This script allows Hopsworks to start/kill RStudio .
+# There will be 1 rstudio-server per ProjectUser.
+# This script can be run by hopsworks (running as user 'glassfish') as a sudo command whereupon
+# it changes user to 'rstudio' to run the command as user 'rstudio'.
+#
+
+help() {
+    echo ""
+    echo "usage: $0 [start rstudio_home hadoop_home hadoop_username port secret_dir certs_dir image_name logfile project_name livy_address livy_port base_dir server_password hadoop_version spark_version] | [kill rstudio_home container_id project_user_name]"
+    echo ""
+    exit 1
+}
+
+
+DOMAINS_DIR=/srv/hops/domains
+RSTUDIO_USER=yarnapp
+RSTUDIO_GROUP=hadoop
+HOPSWORKS_USER=glassfish
+VALID_IMAGE_NAME='^([a-z0-9]+(-[a-z0-9]+)*.)*[a-z0-9]+(:[0-9]*)?(/([a-zA-Z0-9-]*))?/([-:._a-zA-Z0-9]{0,62}[-:.a-zA-Z0-9]$)'
+
+if [ "$1" == "kill" ] ; then
+
+    if [ $# -ne 4 ]; then
+        help
+    fi
+
+    # Don't kill the pid from this script, as it is run with 'sudo' privileges. Only do it as user 'rstudio'
+    sudo ${DOMAINS_DIR}/domain1/bin/rstudio-kill.sh $3 $4
+
+    # Remove all the directories in the home rstudio folder for this project-user.
+    if [ "$2" != "" ] ; then
+        # Try and make sure some silly directory is not deleted, that this
+        if [[ $2 = *"hops/rstudio/Projects"* ]]; then
+            rm -rf "${2}"/../*
+        fi
+    fi
+
+elif [ "$1" == "start" ] ; then
+
+    if [ $# -ne 16 ]; then
+        help
+    fi
+
+    # Sanity checks for injection attacks
+    if [ ! -d "$2" ] ; then
+        echo "Invalid RSTUDIO_HOME directory: $2"
+        exit 1
+    fi
+    if [ ! -d "$3" ] ; then
+        echo "Invalid HADOOP_HOME directory: $3"
+        exit 2
+    fi
+
+    if [ ! -d "${7}" ] ; then
+        echo "Invalid certificates directory: $9"
+        exit 3
+    fi
+
+    re='^[0-9]+$'
+    if ! [[ $5 =~ $re ]] ; then
+      echo "error: Not a number" >&2
+      help
+    fi
+
+    if ! [[ ${8} =~ $VALID_IMAGE_NAME ]] ; then
+       echo "error: Not a valid image name ${8}" >&2
+       help
+    fi
+
+    pid=$(ps -ef | grep -E "rstudio-server" | grep "port=$5" | awk '{print $2}')
+    if [ "$pid" != "" ] ; then
+        echo "There is already an rstudio server using this port"
+        exit 1
+    fi
+
+    mkdir -p "$6"
+    if [ $? -ne 0 ] ; then
+        echo "Error: could not create private_dir: $6"
+        exit 1
+    fi
+    chmod 770 "$6"
+    chown "${RSTUDIO_USER}":"${RSTUDIO_GROUP}" "$6"
+
+    # Make the group of the files, the rstudio group so that they can write to the files
+    chown -R "${HOPSWORKS_USER}":"${RSTUDIO_GROUP}" "$2/"..
+    if [ $? -ne 0 ] ; then
+        echo "Error: could not change ownership of config_dir for RStudio: $2"
+        exit 1
+    fi
+    chmod 0730 "$2/"..
+    chown -R "${RSTUDIO_USER}":"${RSTUDIO_GROUP}" "$2"
+    chmod -R 770 "$2"
+
+    # Launch RStudio server
+    ${DOMAINS_DIR}/domain1/bin/rstudio-launch.sh $2 $3 $4 $5 $6 $7 $8 $9 ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19}
+
+# list
+elif [ "$1" == "list" ] ; then
+    # This command will output line-separated PIDs for all running rstudio server instances into the tmp file, from
+    # where it is read by Hopsworks
+    docker container list -a | grep "__rstudio" |  grep -v 'grep' | awk '{print $1}' > /tmp/rstudioServerPids.pids
+elif [ "$1" == "generate_pems" ]; then
+    if [ $# -ne 3 ]; then
+        help
+    fi
+
+    CERTS_DIR=$2
+    HADOOP_USERNAME=$3
+    TSTORE_FILE=$CERTS_DIR/${HADOOP_USERNAME}__tstore.jks
+    KSTORE_FILE=$CERTS_DIR/${HADOOP_USERNAME}__kstore.jks
+    KEY_FILE=$CERTS_DIR/${HADOOP_USERNAME}__cert.key
+
+
+    PEM_CERTIFICATE_BUNDLE="certificate_bundle.pem"
+    PEM_ROOT_CA="root_ca.pem"
+    PEM_PRIVATE_KEY="private_key.pem"
+
+    KEY=$( cat ${KEY_FILE} )
+
+    #1. generate pem certificates bundle from the keystore.jks file
+    printf "$KEY\n$KEY\n$KEY\n" | keytool -importkeystore -srckeystore $KSTORE_FILE -destkeystore $CERTS_DIR/${HADOOP_USERNAME}__keystore.p12 -deststoretype PKCS12
+    echo $KEY | keytool -deststoretype PKCS12 -keystore $CERTS_DIR/${HADOOP_USERNAME}__keystore.p12 -list
+    echo $KEY | openssl pkcs12 -nokeys -in $CERTS_DIR/${HADOOP_USERNAME}__keystore.p12 -out $CERTS_DIR/$PEM_CERTIFICATE_BUNDLE
+
+    #2. generate root ca pem from the tstore.jks file file
+    printf "$KEY\n$KEY\n$KEY\n" | keytool -importkeystore -srckeystore $TSTORE_FILE -destkeystore $CERTS_DIR/${HADOOP_USERNAME}__tstore.p12 -deststoretype PKCS12
+    echo $KEY | keytool -deststoretype PKCS12 -keystore $CERTS_DIR/${HADOOP_USERNAME}__tstore.p12 -list
+    echo $KEY | openssl pkcs12 -nokeys -in $CERTS_DIR/${HADOOP_USERNAME}__tstore.p12 -out $CERTS_DIR/$PEM_ROOT_CA
+
+    #3 extract private key from the keystore
+    echo $KEY | openssl pkcs12 -info -in $CERTS_DIR/${HADOOP_USERNAME}__keystore.p12 -nodes -nocerts > $CERTS_DIR/$PEM_PRIVATE_KEY
+
+    #4. verify that files have been generated
+    CERTIFICATES_BUNDLE=$CERTS_DIR/$PEM_CERTIFICATE_BUNDLE
+    if [ ! -f ${CERTIFICATES_BUNDLE} ]; then
+      echo "Failed to generate certificates bundle for project"
+      exit 4
+    fi
+    ROOT_CA=$CERTS_DIR/$PEM_ROOT_CA
+    if [ ! -f ${ROOT_CA} ]; then
+      echo "Failed to generate root ca for project"
+      exit 5
+    fi
+    PRIVATE_KEY=$CERTS_DIR/$PEM_PRIVATE_KEY
+    if [ ! -f ${PRIVATE_KEY} ]; then
+      echo "Failed to generate private key for project"
+      exit 6
+    fi
+
+    chmod 644 $ROOT_CA
+    chmod 644 $CERTIFICATES_BUNDLE
+    chmod 644 $PRIVATE_KEY
+
+    chown glassfish:glassfish $ROOT_CA
+    chown glassfish:glassfish $CERTIFICATES_BUNDLE
+    chown glassfish:glassfish $PRIVATE_KEY
+
+    rm $CERTS_DIR/${HADOOP_USERNAME}__keystore.p12
+    rm $CERTS_DIR/${HADOOP_USERNAME}__tstore.p12
+else
+  help
+fi
diff --git a/templates/default/sql/dml/3.4.0.sql.erb b/templates/default/sql/dml/3.4.0.sql.erb
index 7b2d04974..75d0c141b 100644
--- a/templates/default/sql/dml/3.4.0.sql.erb
+++ b/templates/default/sql/dml/3.4.0.sql.erb
@@ -32,4 +32,17 @@ REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("ldap_groups_search_
 REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_enabled", "<%= node['hops']['alive-watchdog']['enabled'] %>");
 REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_interval", "<%= node['hops']['alive-watchdog']['interval'] %>");
 REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_url", "<%= node['hops']['alive-watchdog']['http-poll']['url'] %>");
-REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_region", "<%= node['hops']['alive-watchdog']['json-poll']['dc-id'] %>");
\ No newline at end of file
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("multiregion_watchdog_region", "<%= node['hops']['alive-watchdog']['json-poll']['dc-id'] %>");
+
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_host", "<%= node['hopsworks']['rstudio_host'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_origin_scheme", "<%= node['hopsworks']['rstudio_origin_scheme'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_www_address", "<%= node['hopsworks']['rstudio_www_address'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_session_timeout_minutes", "<%= node['hopsworks']['rstudio_session_timeout_minutes'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_logging_level", "<%= node['hopsworks']['rstudio_logging_level'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_logger_type", "<%= node['hopsworks']['rstudio_logger_type'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_log_file_max_size", "<%= node['hopsworks']['rstudio_log_file_max_size'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_dir", "<%= node['hopsworks']['rstudio_dir'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_user", "<%= node['hops']['yarnapp']['user'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_group", "<%= node['hops']['group'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_cran_repo", "<%= node['hopsworks']['rstudio_cran_repo'] %>");
+REPLACE INTO `hopsworks`.`variables`(`id`, `value`) VALUES ("rstudio_shutdown_timer_interval", "<%= node['rstudio']['shutdown_timer_interval'] %>");
\ No newline at end of file

From 904bfe963528c4ad98f510bd8d9671cd143c111e Mon Sep 17 00:00:00 2001
From: Gibson Chikafa <gibbschikafa@gmail.com>
Date: Thu, 16 Nov 2023 10:41:07 +0200
Subject: [PATCH 11/14] Image build stuff

---
 .../rstudio_dockerfile_template               |  6 +++
 .../default/sql/ddl/3.4.0__initial_tables.sql | 20 ++++++++++
 files/default/sql/ddl/updates/3.4.0.sql       |  4 +-
 recipes/install.rb                            |  3 +-
 templates/default/rstudio.sh.erb              | 14 +++++++
 templates/default/rstudio_image_build.sh.erb  | 40 +++++++++++++++++++
 6 files changed, 85 insertions(+), 2 deletions(-)
 create mode 100644 files/default/hopsworks_templates/rstudio_dockerfile_template
 create mode 100644 templates/default/rstudio_image_build.sh.erb

diff --git a/files/default/hopsworks_templates/rstudio_dockerfile_template b/files/default/hopsworks_templates/rstudio_dockerfile_template
new file mode 100644
index 000000000..731643da3
--- /dev/null
+++ b/files/default/hopsworks_templates/rstudio_dockerfile_template
@@ -0,0 +1,6 @@
+FROM ${conf.baseImage}
+
+COPY ${conf.buildScript} ./
+RUN chmod +x ${conf.buildScript}
+RUN ./${conf.buildScript}
+RUN rm ${conf.buildScript}
\ No newline at end of file
diff --git a/files/default/sql/ddl/3.4.0__initial_tables.sql b/files/default/sql/ddl/3.4.0__initial_tables.sql
index b8c3c6933..15d343883 100644
--- a/files/default/sql/ddl/3.4.0__initial_tables.sql
+++ b/files/default/sql/ddl/3.4.0__initial_tables.sql
@@ -824,6 +824,7 @@ CREATE TABLE `project` (
                            `topic_name` VARCHAR(255) DEFAULT NULL,
                            `python_env_id` int(11) DEFAULT NULL,
                            `creation_status` tinyint(1) NOT NULL DEFAULT '0',
+                           `rstudio_docker_image` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
                            PRIMARY KEY (`id`),
                            UNIQUE KEY `projectname` (`projectname`),
                            KEY `user_idx` (`username`),
@@ -1129,6 +1130,25 @@ CREATE TABLE `rstudio_settings` (
 /*!40101 SET character_set_client = utf8 */;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
+CREATE TABLE `rstudio_environment_build` (
+                                             `id` int NOT NULL AUTO_INCREMENT,
+                                             `build_script` varchar(1000) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                             `user` int NOT NULL,
+                                             `project` int NOT NULL,
+                                             `build_start` bigint DEFAULT NULL,
+                                             `build_finish` bigint DEFAULT NULL,
+                                             `build_result` varchar(128) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                             `secret` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                             `logFile` varchar(1000) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
+                                             `build_name` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
+                                             `description` varchar(1000) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
+                                             PRIMARY KEY (`id`),
+                                             KEY `user_fk` (`user`),
+                                             KEY `rstudio_env_build_project_fk` (`project`),
+                                             CONSTRAINT `rstudio_env_build_project_fk` FOREIGN KEY (`project`) REFERENCES `project` (`id`) ON DELETE CASCADE,
+                                             CONSTRAINT `rstudio_env_build_usr_fkc` FOREIGN KEY (`user`) REFERENCES `users` (`uid`) ON DELETE CASCADE
+) ENGINE=ndbcluster AUTO_INCREMENT=5154 DEFAULT CHARSET=latin1
+
 --
 -- Table structure for table `serving`
 --
diff --git a/files/default/sql/ddl/updates/3.4.0.sql b/files/default/sql/ddl/updates/3.4.0.sql
index d01da2672..de0002bae 100644
--- a/files/default/sql/ddl/updates/3.4.0.sql
+++ b/files/default/sql/ddl/updates/3.4.0.sql
@@ -183,4 +183,6 @@ UPDATE `hopsworks`.`project_topics`
 SET `subject_id` = NULL
     WHERE REGEXP_SUBSTR(`topic_name`, "^([0-9]+)_([0-9]+)_(.+)_([0-9]+)(_onlinefs|$)");
 
-SET SQL_SAFE_UPDATES = 1;
\ No newline at end of file
+SET SQL_SAFE_UPDATES = 1;
+
+ALTER TABLE `hopsworks`.`project` ADD COLUMN `rstudio_docker_image` VARCHAR(255) COLLATE latin1_general_cs DEFAULT NULL;
\ No newline at end of file
diff --git a/recipes/install.rb b/recipes/install.rb
index a63636b66..09450f090 100644
--- a/recipes/install.rb
+++ b/recipes/install.rb
@@ -764,7 +764,8 @@
 end
 
 ["tensorboard-launch.sh", "tensorboard-cleanup.sh", "condasearch.sh", "list_environment.sh", "jupyter-kill.sh",
-"tfserving-kill.sh", "sklearn_serving-launch.sh", "sklearn_serving-kill.sh", "git-container-kill.sh", "rstudio-kill.sh"].each do |script|
+"tfserving-kill.sh", "sklearn_serving-launch.sh", "sklearn_serving-kill.sh", "git-container-kill.sh", "rstudio-kill.sh",
+ "rstudio_image_build.sh"].each do |script|
   template "#{theDomain}/bin/#{script}" do
     source "#{script}.erb"
     owner node['glassfish']['user']
diff --git a/templates/default/rstudio.sh.erb b/templates/default/rstudio.sh.erb
index 0fd8beef1..69a157602 100644
--- a/templates/default/rstudio.sh.erb
+++ b/templates/default/rstudio.sh.erb
@@ -159,6 +159,20 @@ elif [ "$1" == "generate_pems" ]; then
 
     rm $CERTS_DIR/${HADOOP_USERNAME}__keystore.p12
     rm $CERTS_DIR/${HADOOP_USERNAME}__tstore.p12
+elif [ "$1" == "build" ] ; then
+    ${DOMAINS_DIR}/domain1/bin/rstudio_image_build.sh "build" $2 $3 $4 $5 &
+    echo $! > build.pid
+elif [ "$1" == "cancel_build" ] ; then
+    kill -9 $(cat build.pid)
+elif [ "$1" == "remove_image" ] ; then
+    REGISTRY_URL=$2
+    TAG=$3
+    docker rmi -f $REGISTRY_URL/rstudio:$TAG >> delete_docker_image_log.log
+    DIGEST=$(curl -g -k -sSL -I \
+    -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+    "https://${REGISTRY_URL}/v2/rstudio/manifests/${TAG}" | awk '$1 == "docker-content-digest:" { print $2 }' | tr -d $'\r')
+    echo $DIGEST >> delete_docker_image_log.log
+    curl -g -k -v -sSL -X DELETE "https://${REGISTRY_URL}/v2/rstudio/manifests/${DIGEST}" >> delete_docker_image_log.log
 else
   help
 fi
diff --git a/templates/default/rstudio_image_build.sh.erb b/templates/default/rstudio_image_build.sh.erb
new file mode 100644
index 000000000..5161d55a5
--- /dev/null
+++ b/templates/default/rstudio_image_build.sh.erb
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+LOG_FILE=$2
+PROJECT_ID=$3
+BUILD_ID=$4
+IMAGE_TAG=$5
+GLASSFISH_IP=glassfish.service.consul
+GLASSFISH_PORT=8182
+TOKEN=$(cat token.jwt)
+
+
+send_response_hopsworks() {
+  NEW_STATUS=$1
+  curl -k -X PUT https://$GLASSFISH_IP:$GLASSFISH_PORT/hopsworks-api/api/project/$PROJECT_ID/rstudio/environment/build/$BUILD_ID/state?status=$NEW_STATUS -H "Accept: application/json" -H "Authorization: Bearer $TOKEN" >> $LOG_FILE
+}
+
+
+echo "Starting build..." > $LOG_FILE
+
+docker build --progress=plain --tag $IMAGE_TAG ./  >> $LOG_FILE
+
+if [ $? -ne 0 ] ; then
+   echo "Docker build failed"
+   send_response_hopsworks "FAILED"
+   exit 1
+fi
+
+echo "Pushing images to registry..." >> $LOG_FILE
+
+docker push $IMAGE_TAG >> $LOG_FILE
+
+if [ $? -ne 0 ] ; then
+   echo "Failed to push to registry"
+   send_response_hopsworks "FAILED"
+   exit 2
+fi
+
+#cat $LOG_FILE | sed 's/\x1B[@A-Z\\\]^_]\|\x1B\[[0-9:;<=>?]*[-!"#$%&'"'"'()*+,.\/]*[][\\@A-Z^_`a-z{|}~]//g' > $LOG_FILE
+
+send_response_hopsworks "SUCCESS"
\ No newline at end of file

From 71c68a68dc675b79b16fdc9ca56cc28db404276b Mon Sep 17 00:00:00 2001
From: Gibson Chikafa <gibbschikafa@gmail.com>
Date: Thu, 16 Nov 2023 11:20:30 +0200
Subject: [PATCH 12/14] Updates

---
 files/default/sql/ddl/updates/3.4.0.sql | 47 ++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/files/default/sql/ddl/updates/3.4.0.sql b/files/default/sql/ddl/updates/3.4.0.sql
index de0002bae..56979a350 100644
--- a/files/default/sql/ddl/updates/3.4.0.sql
+++ b/files/default/sql/ddl/updates/3.4.0.sql
@@ -185,4 +185,49 @@ SET `subject_id` = NULL
 
 SET SQL_SAFE_UPDATES = 1;
 
-ALTER TABLE `hopsworks`.`project` ADD COLUMN `rstudio_docker_image` VARCHAR(255) COLLATE latin1_general_cs DEFAULT NULL;
\ No newline at end of file
+
+-- Rstudio updates
+ALTER TABLE `hopsworks`.`rstudio_project` DROP COLUMN `host_ip`,
+    DROP COLUMN `token`;
+
+ALTER TABLE `hopsworks`.`rstudio_project` ADD COLUMN `login_password` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL;
+ALTER TABLE `hopsworks`.`rstudio_project` ADD COLUMN `login_username` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL;
+
+ALTER TABLE `hopsworks`.`rstudio_project` MODIFY COLUMN `pid` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL;
+
+ALTER TABLE `hopsworks`.`rstudio_settings`
+    DROP COLUMN num_tf_ps, DROP COLUMN num_tf_gpus,
+    DROP COLUMN num_mpi_np, DROP COLUMN appmaster_cores,
+    DROP COLUMN appmaster_memory, DROP COLUMN num_executors,
+    DROP COLUMN num_executor_cores, DROP COLUMN executor_memory,
+    DROP COLUMN dynamic_initial_executors, DROP COLUMN dynamic_min_executors,
+    DROP COLUMN dynamic_max_executors, DROP COLUMN log_level,
+    DROP COLUMN mode, DROP COLUMN umask,
+    DROP COLUMN archives, DROP COLUMN jars,
+    DROP COLUMN files, DROP COLUMN py_files,
+    DROP COLUMN spark_params;
+
+ALTER TABLE `hopsworks`.`rstudio_settings` ADD COLUMN  `base_dir` varchar(255) COLLATE latin1_general_cs DEFAULT NULL;
+ALTER TABLE `hopsworks`.`rstudio_settings` ADD COLUMN  `job_config` varchar(11000) COLLATE latin1_general_cs DEFAULT NULL;
+ALTER TABLE `hopsworks`.`rstudio_settings` ADD COLUMN  `docker_config` varchar(1000) COLLATE latin1_general_cs DEFAULT NULL;
+
+ALTER TABLE `hopsworks`.`project` ADD COLUMN `rstudio_docker_image` VARCHAR(255) COLLATE latin1_general_cs DEFAULT NULL;
+
+CREATE TABLE `rstudio_environment_build` (
+                                             `id` int NOT NULL AUTO_INCREMENT,
+                                             `build_script` varchar(1000) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                             `user` int NOT NULL,
+                                             `project` int NOT NULL,
+                                             `build_start` bigint DEFAULT NULL,
+                                             `build_finish` bigint DEFAULT NULL,
+                                             `build_result` varchar(128) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                             `secret` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                             `logFile` varchar(1000) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
+                                             `build_name` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
+                                             `description` varchar(1000) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
+                                             PRIMARY KEY (`id`),
+                                             KEY `user_fk` (`user`),
+                                             KEY `rstudio_env_build_project_fk` (`project`),
+                                             CONSTRAINT `rstudio_env_build_project_fk` FOREIGN KEY (`project`) REFERENCES `project` (`id`) ON DELETE CASCADE,
+                                             CONSTRAINT `rstudio_env_build_usr_fkc` FOREIGN KEY (`user`) REFERENCES `users` (`uid`) ON DELETE CASCADE
+) ENGINE=ndbcluster AUTO_INCREMENT=5154 DEFAULT CHARSET=latin1
\ No newline at end of file

From d451415d35d462d403c70dc6861dea98df12086d Mon Sep 17 00:00:00 2001
From: Gibson Chikafa <gibbschikafa@gmail.com>
Date: Thu, 16 Nov 2023 11:22:12 +0200
Subject: [PATCH 13/14] Fix

---
 files/default/sql/ddl/3.4.0__initial_tables.sql | 2 +-
 files/default/sql/ddl/updates/3.4.0.sql         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/files/default/sql/ddl/3.4.0__initial_tables.sql b/files/default/sql/ddl/3.4.0__initial_tables.sql
index 15d343883..064a96f11 100644
--- a/files/default/sql/ddl/3.4.0__initial_tables.sql
+++ b/files/default/sql/ddl/3.4.0__initial_tables.sql
@@ -1147,7 +1147,7 @@ CREATE TABLE `rstudio_environment_build` (
                                              KEY `rstudio_env_build_project_fk` (`project`),
                                              CONSTRAINT `rstudio_env_build_project_fk` FOREIGN KEY (`project`) REFERENCES `project` (`id`) ON DELETE CASCADE,
                                              CONSTRAINT `rstudio_env_build_usr_fkc` FOREIGN KEY (`user`) REFERENCES `users` (`uid`) ON DELETE CASCADE
-) ENGINE=ndbcluster AUTO_INCREMENT=5154 DEFAULT CHARSET=latin1
+) ENGINE=ndbcluster AUTO_INCREMENT=5154 DEFAULT CHARSET=latin1;
 
 --
 -- Table structure for table `serving`
diff --git a/files/default/sql/ddl/updates/3.4.0.sql b/files/default/sql/ddl/updates/3.4.0.sql
index 56979a350..d2a0e882b 100644
--- a/files/default/sql/ddl/updates/3.4.0.sql
+++ b/files/default/sql/ddl/updates/3.4.0.sql
@@ -230,4 +230,4 @@ CREATE TABLE `rstudio_environment_build` (
                                              KEY `rstudio_env_build_project_fk` (`project`),
                                              CONSTRAINT `rstudio_env_build_project_fk` FOREIGN KEY (`project`) REFERENCES `project` (`id`) ON DELETE CASCADE,
                                              CONSTRAINT `rstudio_env_build_usr_fkc` FOREIGN KEY (`user`) REFERENCES `users` (`uid`) ON DELETE CASCADE
-) ENGINE=ndbcluster AUTO_INCREMENT=5154 DEFAULT CHARSET=latin1
\ No newline at end of file
+) ENGINE=ndbcluster AUTO_INCREMENT=5154 DEFAULT CHARSET=latin1;
\ No newline at end of file

From 5ef9086064a165c9047f3f58e40d924141963b97 Mon Sep 17 00:00:00 2001
From: Gibson Chikafa <gibbschikafa@gmail.com>
Date: Thu, 16 Nov 2023 16:24:59 +0200
Subject: [PATCH 14/14] Fix docker build output

---
 templates/default/rstudio_image_build.sh.erb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/default/rstudio_image_build.sh.erb b/templates/default/rstudio_image_build.sh.erb
index 5161d55a5..427a64850 100644
--- a/templates/default/rstudio_image_build.sh.erb
+++ b/templates/default/rstudio_image_build.sh.erb
@@ -17,7 +17,7 @@ send_response_hopsworks() {
 
 echo "Starting build..." > $LOG_FILE
 
-docker build --progress=plain --tag $IMAGE_TAG ./  >> $LOG_FILE
+docker build --progress=plain --tag $IMAGE_TAG . &>> $LOG_FILE
 
 if [ $? -ne 0 ] ; then
    echo "Docker build failed"