[WIP] [SPARK-1146] Vagrant support for Spark

vagrant/.gitignore

@@ -0,0 +1,5 @@
+.downloads
+.DS_STORE
+*.box
+.vagrant
+.bashrc

vagrant/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Binh Nguyen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vagrant/README.md

@@ -0,0 +1,19 @@
+Vagrant template with shell provision for Ubuntu Precise clusers with JDK 7.
+===========
+
+## Installation
+1. Install [vagrant](//www.vagrantup.com). *Tested with version 1.4.3*
+2. Install [virtualbox](//www.virtualbox.org). *Tested with version 4.2.16 but should work with any 4.+*
+3. Checkout this repo `git clone [email protected]:ngbinh/vagrant_jdk.git`
+4. Change to the directory `cd vagrant_jdk`
+5. Bring up the nodes `vagrant up`
+6. Wait a while, then make sure the nodes are up: `vagrant status`. You should see three nodes named `spark-master`, `spark-worker-1` and `spark-worker-2` running. 
+7. Access the nodes with user `spark-user` and password `spark`.
+
+Note that it will take a while to download Ubuntu Precise image at the first run. Subsequent runs should not have to re-download. 
+
+### Customization
+1. You can change the number of nodes, their basic parameters by modifying `Vagrantfile`.
+2. You can change the JDK version by modifying `scripts/setup.sh`
+
+

vagrant/Vagrantfile

@@ -0,0 +1,65 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Using Vagrant shell scripts
+# https://github.com/StanAngeloff/vagrant-shell-scripts
+require File.join(File.dirname(__FILE__), 'scripts/vagrant-shell-scripts/vagrant')
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+Vagrant.require_version ">= 1.4.2"
+
+# the domain of all the nodes
+DOMAIN = 'local'
+
+# Ubuntu 12.04.4 LTS (Precise Pangolin) 64 bit
+BOX = 'precise64'
+BOX_URL = 'http://files.vagrantup.com/precise64.box'
+
+# define all the nodes here.
+# :host is the id in Vagrant of the node. It will also be its hostname
+# :ip the ip address of the node
+# :cpu the number of core for the node
+# :ram the amount of RAM allocated to the node in MBytes.
+NODES = [
+  { :host => 'spark-master', :ip => '192.168.2.10', :cpu => 1, :ram => '3072' },
+  { :host => 'spark-worker-1', :ip => '192.168.2.20', :cpu => 1, :ram => '2048' },
+  { :host => 'spark-worker-2', :ip => '192.168.2.21', :cpu => 1, :ram => '2048' }
+]
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  NODES.each do | node |
+    config.vm.define node[:host] do | node_config |
+      node_config.vm.box = BOX
+      node_config.vm.box_url = BOX_URL
+
+      node_config.vm.hostname = node[:host] + "." + DOMAIN
+      node_config.vm.network :private_network, ip: node[:ip]
+
+      # by default, the current folder is shared as /vagrant in the nodes
+      # you can also share an outside folder here
+      # node_config.vm.synced_folder "shared", "/shared"
+
+      memory = node[:ram] ? node[:ram] : 1024
+      cpu = node[:cpu] ? node[:cpu] : 1
+
+      node_config.vm.provider :virtualbox do | vbox |
+        vbox.gui = false
+        vbox.customize ['modifyvm', :id, '--memory', memory.to_s]
+        vbox.customize ['modifyvm', :id, '--cpus', cpu.to_s]
+
+        # fix the connection slow
+        # https://github.com/mitchellh/vagrant/issues/1807
+        vbox.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+        vbox.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
+      end
+
+      node_config.vm.provision :shell do |shell|
+        vagrant_shell_scripts_configure(
+        shell,
+        File.dirname(__FILE__),
+        'scripts/setup.sh')
+      end
+    end
+  end
+end

vagrant/copy-keys.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# make sure all the nodes are up
+# vagrant up 1>/dev/null 2>&1 
+
+source vagrant_nodes.sh
+
+sshpass -p "$VAGRANT_SPARK_PASSWORD" ssh-copy-id "$VAGRANT_SPARK_USER"@"$VAGRANT_SPARK_MASTER" 1>/dev/null 2>&1 
+sshpass -p "$VAGRANT_SPARK_PASSWORD" ssh-copy-id "$VAGRANT_SPARK_USER"@"$VAGRANT_SPARK_WORKER_1" 1>/dev/null 2>&1 
+sshpass -p "$VAGRANT_SPARK_PASSWORD" ssh-copy-id "$VAGRANT_SPARK_USER"@"$VAGRANT_SPARK_WORKER_2" 1>/dev/null 2>&1 
+

vagrant/scripts/setup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+<%= import 'scripts/vagrant-shell-scripts/ubuntu.sh' %>
+<%= import 'scripts/vagrant-shell-scripts/ubuntu-extras.sh' %>
+
+# }}}
+
+# Use Google Public DNS for resolving domain names.
+# The default is host-only DNS which may not be installed.
+# nameservers-local-purge
+# nameservers-append '8.8.8.8'
+# nameservers-append '8.8.4.4'
+
+# Update packages cache.
+apt-packages-update
+
+# Install VM packages.
+apt-packages-install \
+  rsync \
+  telnet \
+  wget \
+  git \
+  curl\
+  whois
+
+download_dir="/vagrant/.downloads/oracle-jdk-download"
+mkdir -p "$download_dir" && oracle-jdk-install "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" "http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz" "$download_dir"
+
+# create user spark if necessary
+if [ getent passwd spark-user > /dev/null 2>&1 ]; then
+	echo "user spark-user exists. Skipping create user"
+else
+	echo "creating (sudo) user spark-user"
+    sudo useradd -m -G `groups vagrant | cut -d" " -f4- | sed 's/ /,/g'` -s/bin/bash -p `mkpasswd spark` spark-user
+    echo "sucess"
+fi
+# done. Create user

vagrant/scripts/vagrant-shell-scripts/.gitignore

@@ -0,0 +1,2 @@
+.bundle
+.vagrant

vagrant/scripts/vagrant-shell-scripts/.lvimrc

@@ -0,0 +1,13 @@
+let s:roundup_command = 'bin/roundup'
+
+let VimuxResetSequence = "q C-c C-u"
+
+augroup runTestOnSave
+  au!
+  au BufWritePost *-test.sh call VimuxRunCommand(s:roundup_command . ' ' . expand('%:p:~:.'))
+augroup END
+
+augroup reloadLocalConfiguration
+  au!
+  au BufWritePost .lvimrc source %
+augroup END

vagrant/scripts/vagrant-shell-scripts/LICENSE.md

@@ -0,0 +1,22 @@
+The MIT License
+===============
+
+> Copyright (c) 2011 Stan Angeloff
+>
+> Permission is hereby granted, free of charge, to any person obtaining a copy
+> of this software and associated documentation files (the "Software"), to deal
+> in the Software without restriction, including without limitation the rights
+> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+> copies of the Software, and to permit persons to whom the Software is
+> furnished to do so, subject to the following conditions:
+>
+> The above copyright notice and this permission notice shall be included in
+> all copies or substantial portions of the Software.
+>
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+> THE SOFTWARE.