RStudio

logicalclocks · Nov 16, 2022 · 1b025ca · 1b025ca
1 parent 82787c9
commit 1b025ca
Show file tree

Hide file tree

Showing 15 changed files with 519 additions and 33 deletions.
diff --git a/attributes/default.rb b/attributes/default.rb
@@ -446,4 +446,34 @@
 
 default['hops']['cadvisor']['dir']                                    = "#{node['hops']['dir']}/cadvisor"
 default['hops']['cadvisor']['download-url']                           = "#{node['download_url']}/docker/cadvisor"
-default['hops']['cadvisor']['port']                                   = "4194"
+default['hops']['cadvisor']['port']                                   = "4194"
+
+
+default['hopsworks']['rstudio_dir'] = node['hopsworks']['dir'] + "/rstudio"
+#rstudio configuration variables
+default["hopsworks"]['rstudio_host']                 = "localhost"
+default["hopsworks"]['rstudio_origin_scheme']        = "https"
+default["hopsworks"]["rstudio_www_address"]          = "0.0.0.0"
+default["hopsworks"]["rstudio_session_timeout_minutes"] = 360
+default["hopsworks"]["rstudio_logging_level"]        = "info"
+default["hopsworks"]["rstudio_logger_type"]          = "file"
+default["hopsworks"]["rstudio_log_file_max_size"]    = 512
+default["hopsworks"]["rstudio_default_cran_repo"]    = "https://cloud.r-project.org/"
+
+default['rstudio']['base_dir']                       = node['install']['dir'].empty? ? node['hopsworks']['dir'] + "/rstudio" : node['install']['dir'] + "/rstudio"
+default['rstudio']['shutdown_timer_interval']          = "30m"
+
+# CRAN
+default['rstudio']['cran']['mirror'] = 'http://cran.rstudio.com/'
+
+# APT configuration for Ubuntu or Debian installs.
+case node["platform"].downcase
+when "ubuntu"
+  default['rstudio']['apt']['key'] = 'E084DAB9'
+  default['rstudio']['apt']['keyserver'] = 'keyserver.ubuntu.com'
+  default['rstudio']['apt']['uri'] = 'http://cran.stat.ucla.edu/bin/linux/ubuntu'
+when "debian"
+  default['rstudio']['apt']['key'] = '381BA480'
+  default['rstudio']['apt']['keyserver'] = 'subkeys.pgp.net'
+  default['rstudio']['apt']['uri'] = 'http://cran.stat.ucla.edu/bin/linux/debian'
+end
diff --git a/files/default/hopsworks_templates/config_template.json b/files/default/hopsworks_templates/config_template.json
@@ -50,6 +50,7 @@
     "executorCores": ${conf.executorCores?c},
     "executorMemory": "${conf.executorMemory}",
     "proxyUser": "${conf.hdfsUser}",
+    "name": "${conf.livySessionName}",
     "queue": "${conf.yarnQueue}",
     "conf": {
 ${conf.sparkConfiguration}

diff --git a/files/default/hopsworks_templates/rstudio_logging_config_template b/files/default/hopsworks_templates/rstudio_logging_config_template
@@ -0,0 +1,9 @@
+[*]
+log-level=warn
+logger-type=syslog
+
+[@rserver]
+log-level=${conf.logLevel}
+logger-type=${conf.loggerType}
+max-size-mb=${conf.maxSizeMb}
+log-file-include-pid=${conf.includePid}
diff --git a/files/default/hopsworks_templates/rstudio_rserver_config_template b/files/default/hopsworks_templates/rstudio_rserver_config_template
@@ -0,0 +1,5 @@
+rsession-which-r=${conf.versionPath}
+www-address=${conf.ipAddress}
+www-port=${conf.port}
+www-root-path=${conf.rootPath}
+server-user=${conf.serverUser}
diff --git a/files/default/hopsworks_templates/rstudio_rsession_config_template b/files/default/hopsworks_templates/rstudio_rsession_config_template
@@ -0,0 +1,2 @@
+session-timeout-minutes=${conf.sessionTimeoutMinutes}
+r-cran-repos=${conf.cranRepo}
diff --git a/files/default/hopsworks_templates/sparklyr_config.yml b/files/default/hopsworks_templates/sparklyr_config.yml
@@ -0,0 +1,10 @@
+default:
+  livy.driverCores: ${conf.driverCores}
+  livy.driverMemory: "${conf.driverMemory}"
+  livy.numExecutors: ${conf.numExecutors}
+  livy.executorCores: ${conf.executorCores}
+  livy.executorMemory: "${conf.executorMemory}"
+  livy.proxyUser: "${conf.proxyUser}"
+  livy.queue: "${conf.yarnQueue}"
+  livy.name: "${conf.livyAppName}"
+${conf.sparkConfiguration}
diff --git a/files/default/sql/ddl/3.1.0__initial_tables.sql b/files/default/sql/ddl/3.1.0__initial_tables.sql
@@ -1047,20 +1047,21 @@ CREATE TABLE `rstudio_interpreter` (
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `rstudio_project` (
-                                   `port` int(11) NOT NULL,
-                                   `hdfs_user_id` int(11) NOT NULL,
+                                   `port` int NOT NULL,
+                                   `hdfs_user_id` int NOT NULL,
                                    `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                                   `expires` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
                                    `last_accessed` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-                                   `host_ip` varchar(255) COLLATE latin1_general_cs NOT NULL,
-                                   `token` varchar(255) COLLATE latin1_general_cs NOT NULL,
-                                   `secret` varchar(64) COLLATE latin1_general_cs NOT NULL,
-                                   `pid` bigint(20) NOT NULL,
-                                   `project_id` int(11) NOT NULL,
+                                   `secret` varchar(64) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                   `pid` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs NOT NULL,
+                                   `project_id` int NOT NULL,
+                                   `login_password` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
+                                   `login_username` varchar(255) CHARACTER SET latin1 COLLATE latin1_general_cs DEFAULT NULL,
                                    PRIMARY KEY (`port`),
                                    KEY `hdfs_user_idx` (`hdfs_user_id`),
                                    KEY `project_id` (`project_id`),
-                                   CONSTRAINT `FK_103_577` FOREIGN KEY (`hdfs_user_id`) REFERENCES `hops`.`hdfs_users` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION,
-                                   CONSTRAINT `FK_284_578` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
+                                   CONSTRAINT `FK_103_577` FOREIGN KEY (`hdfs_user_id`) REFERENCES `hops`.`hdfs_users` (`id`) ON DELETE CASCADE,
+                                   CONSTRAINT `FK_284_578` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE
 ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
@@ -1073,34 +1074,19 @@ CREATE TABLE `rstudio_project` (
 CREATE TABLE `rstudio_settings` (
                                     `project_id` int(11) NOT NULL,
                                     `team_member` varchar(150) COLLATE latin1_general_cs NOT NULL,
-                                    `num_tf_ps` int(11) DEFAULT '1',
-                                    `num_tf_gpus` int(11) DEFAULT '0',
-                                    `num_mpi_np` int(11) DEFAULT '1',
-                                    `appmaster_cores` int(11) DEFAULT '1',
-                                    `appmaster_memory` int(11) DEFAULT '1024',
-                                    `num_executors` int(11) DEFAULT '1',
-                                    `num_executor_cores` int(11) DEFAULT '1',
-                                    `executor_memory` int(11) DEFAULT '1024',
-                                    `dynamic_initial_executors` int(11) DEFAULT '1',
-                                    `dynamic_min_executors` int(11) DEFAULT '1',
-                                    `dynamic_max_executors` int(11) DEFAULT '1',
                                     `secret` varchar(255) COLLATE latin1_general_cs NOT NULL,
-                                    `log_level` varchar(32) COLLATE latin1_general_cs DEFAULT 'INFO',
-                                    `mode` varchar(32) COLLATE latin1_general_cs NOT NULL,
-                                    `umask` varchar(32) COLLATE latin1_general_cs DEFAULT '022',
                                     `advanced` tinyint(1) DEFAULT '0',
-                                    `archives` varchar(1500) COLLATE latin1_general_cs DEFAULT '',
-                                    `jars` varchar(1500) COLLATE latin1_general_cs DEFAULT '',
-                                    `files` varchar(1500) COLLATE latin1_general_cs DEFAULT '',
-                                    `py_files` varchar(1500) COLLATE latin1_general_cs DEFAULT '',
-                                    `spark_params` varchar(6500) COLLATE latin1_general_cs DEFAULT '',
                                     `shutdown_level` int(11) NOT NULL DEFAULT '6',
+                                    `base_dir` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+                                    `job_config` varchar(11000) COLLATE latin1_general_cs DEFAULT NULL,
+                                    `docker_config` varchar(1000) COLLATE latin1_general_cs DEFAULT NULL,
                                     PRIMARY KEY (`project_id`,`team_member`),
                                     KEY `team_member` (`team_member`),
                                     KEY `secret_idx` (`secret`),
-                                    CONSTRAINT `RS_FK_USERS` FOREIGN KEY (`team_member`) REFERENCES `users` (`email`) ON DELETE CASCADE ON UPDATE NO ACTION,
+                                    CONSTRAINT `RS_FK_USERS` FOREIGN KEY (`team_member`) REFERENCES `users` (`email`) ON DELETE CASCADE ON UPDATE NO
+                                        ACTION,
                                     CONSTRAINT `RS_FK_PROJS` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
-) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+                                ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 /*!40101 SET character_set_client = utf8 */;

diff --git a/files/default/sql/ddl/updates/3.1.0.sql b/files/default/sql/ddl/updates/3.1.0.sql
@@ -123,5 +123,27 @@ ALTER TABLE `hopsworks`.`feature_store_activity` ADD CONSTRAINT `fs_act_validati
 ALTER TABLE `hopsworks`.`feature_store_activity` ADD COLUMN `expectation_suite_id` Int(11) NULL;
 ALTER TABLE `hopsworks`.`feature_store_activity` ADD CONSTRAINT `fs_act_expectationsuite_fk` FOREIGN KEY (`expectation_suite_id`) REFERENCES `expectation_suite` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION;
 
+ALTER TABLE `hopsworks`.`project` ADD COLUMN `creation_status` TINYINT(1) NOT NULL DEFAULT '0';
 
-ALTER TABLE `hopsworks`.`project` ADD COLUMN `creation_status` TINYINT(1) NOT NULL DEFAULT '0';
+ALTER TABLE `hopsworks`.`rstudio_settings` DROP `num_tf_ps`, DROP `num_tf_gpus`, DROP `num_mpi_np`,
+DROP `appmaster_cores`, DROP `appmaster_memory`, DROP `num_executors`, DROP `num_executor_cores`,
+    DROP `executor_memory`, DROP `dynamic_initial_executors`,DROP `dynamic_min_executors`, DROP `dynamic_max_executors`,
+    DROP `log_level`, DROP `mode`, DROP `umask`, DROP `archives`, DROP `jars`, DROP `files`,DROP `py_files`, DROP `spark_params`;
+
+ALTER TABLE `hopsworks`.`rstudio_project` DROP `host_ip`, DROP `token`;
+
+ALTER TABLE `hopsworks`.`rstudio_project` ADD COLUMN `expires` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP;
+
+ALTER TABLE `hopsworks`.`rstudio_project` ADD COLUMN `login_username` varchar(255) COLLATE latin1_general_cs DEFAULT
+    NULL;
+
+ALTER TABLE `hopsworks`.`rstudio_project` ADD COLUMN `login_password` varchar(255) COLLATE latin1_general_cs DEFAULT
+    NULL;
+
+ALTER TABLE `hopsworks`.`rstudio_project` MODIFY COLUMN `pid`  varchar(255) COLLATE latin1_general_cs NOT NULL;
+
+ALTER TABLE `hopsworks`.`rstudio_settings` ADD COLUMN `job_config` varchar(11000) COLLATE latin1_general_cs DEFAULT
+    NULL;
+
+ALTER TABLE `hopsworks`.`rstudio_settings` ADD COLUMN `docker_config` varchar(11000) COLLATE latin1_general_cs DEFAULT
+    NULL;
diff --git a/recipes/install.rb b/recipes/install.rb
@@ -125,6 +125,14 @@
   not_if { node['install']['external_users'].casecmp("true") == 0 }
 end
 
+#update permissions of base_dir for rstudio to 770
+directory node['rstudio']['base_dir']  do
+  owner node['hops']['yarnapp']['user']
+  group node['hops']['group']
+  mode "770"
+  action :create
+end
+
 #update permissions of base_dir to 770
 directory node['jupyter']['base_dir']  do
   owner node['hops']['yarnapp']['user']
@@ -578,6 +586,14 @@
   not_if       { node['install']['kubernetes'].casecmp("true") == 0 }
 end
 
+kagent_sudoers "rstudio" do
+  user          node['glassfish']['user']
+  group         "root"
+  script_name   "rstudio.sh"
+  template      "rstudio.sh.erb"
+  run_as        "ALL" # run this as root - inside we change to different users
+end
+
 kagent_sudoers "convert-ipython-notebook" do 
   user          node['glassfish']['user']
   group         "root"
@@ -665,9 +681,19 @@
   action :create
 end
 
+kagent_sudoers "rstudio-project-cleanup" do
+  user          node['glassfish']['user']
+  group         "root"
+  script_name   "rstudio-project-cleanup.sh"
+  template      "rstudio-project-cleanup.sh.erb"
+  run_as        "ALL"
+  not_if       { node['install']['kubernetes'].casecmp("true") == 0 }
+end
+
 ["zip-hdfs-files.sh", "zip-background.sh", "unzip-background.sh",  "tensorboard-launch.sh",
  "tensorboard-cleanup.sh", "condasearch.sh", "list_environment.sh", "jupyter-kill.sh",
- "jupyter-launch.sh", "tfserving-kill.sh", "sklearn_serving-launch.sh", "sklearn_serving-kill.sh", "git-container-kill.sh"].each do |script|
+ "jupyter-launch.sh", "tfserving-kill.sh", "sklearn_serving-launch.sh", "sklearn_serving-kill.sh", "git-container-kill.sh",  "rstudio-kill.sh",
+ "rstudio-launch.sh"].each do |script|
   template "#{theDomain}/bin/#{script}" do
     source "#{script}.erb"
     owner node['glassfish']['user']
@@ -677,6 +703,25 @@
   end
 end
 
+#update permissions of base_dir to 770
+directory node["rstudio"]["base_dir"]  do
+  owner node["rstudio"]["user"]
+  group node["rstudio"]["group"]
+  mode "770"
+  action :create
+end
+
+template "#{theDomain}/bin/rstudio-launch.sh" do
+  source "rstudio-launch.sh.erb"
+  owner node['glassfish']['user']
+  group node['glassfish']['group']
+  mode "500"
+  action :create
+  variables({
+              :namenode_fdqn => namenode_fdqn,
+            })
+end
+
 template "#{theDomain}/bin/git-container-launch.sh" do
   source "git-container-launch.sh.erb"
   owner node['glassfish']['user']

diff --git a/templates/default/rstudio-kill.sh.erb b/templates/default/rstudio-kill.sh.erb
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+help() {
+    echo ""
+    echo "usage: $0 CONTAINER_ID PROJECT_USER_NAME"
+    echo ""
+    exit 1
+}
+
+
+if [ $# -ne 2 ]; then
+  help
+fi
+
+if [ "$2" != "" ]; then
+    PROJECT_USER_NAME=$2
+    CONTAINER_NAME=${PROJECT_USER_NAME}__rstudio
+
+    echo "Killing input container_name: $CONTAINER_NAME"
+    docker rm -f "$CONTAINER_NAME" > /dev/null 2>&1
+    exit $?
+fi
+
+echo "Killing input container_id: $1"
+docker rm -f "$1" > /dev/null 2>&1
+exit $?