Publish mini to bigger instance types (closes #173)

.travis.yml

@@ -1,5 +1,6 @@
 ---
 sudo: required
+dist: trusty
 language: go
 
 go:

Packerfile.json

@@ -5,9 +5,9 @@
       "ami_groups": [
         "all"
       ],
-      "ami_name": "snowplow-mini-{{user `version`}}-{{ timestamp }}-hvm-ebs-amd64",
+      "ami_name": "snowplow-mini-{{user `aws_version`}}-{{user `sp_mini_size`}}-{{ timestamp }}-hvm-ebs-amd64",
       "ami_regions": "us-east-2,us-west-1,us-west-2,ca-central-1,eu-west-1,eu-central-1,eu-west-2,ap-southeast-1,ap-southeast-2,ap-northeast-2,ap-northeast-1,ap-south-1,sa-east-1",
-      "instance_type": "t2.medium",
+      "instance_type": "{{user `aws_instance_type`}}",
       "region": "us-east-1",
       "source_ami": "ami-58167327",
       "ssh_username": "ubuntu",
@@ -20,8 +20,8 @@
     {
       "type": "googlecompute",
       "image_description": "Snowplow Mini - The Snowplow Pipeline in a box",
-      "image_name": "snowplow-mini-{{user `version`}}-{{timestamp}}",
-      "machine_type": "n1-standard-2",
+      "image_name": "snowplow-mini-{{user `gcp_version`}}-{{user `sp_mini_size`}}-{{timestamp}}",
+      "machine_type": "{{user `gcp_machine_type`}}",
       "account_file": "account.json",
       "project_id": "engineering-sandbox",
       "source_image_family": "ubuntu-1404-lts",
@@ -37,6 +37,10 @@
     }
   ],
   "variables": {
-    "version": "0.5.0"
+    "aws_version": "0.5.0",
+    "gcp_version": "0-5-0",
+    "gcp_machine_type": "{{env `GCP_MACHINE_TYPE`}}",
+    "aws_instance_type": "{{env `AWS_INSTANCE_TYPE`}}",
+    "sp_mini_size": "{{env `SP_MINI_SIZE`}}"
   }
 }

Vagrantfile

@@ -32,10 +32,16 @@ Vagrant.configure("2") do |config|
   # Requires Vagrant 1.7.0+
   config.push.define "publish", strategy: "local-exec" do |push|
     push.inline = <<-SCRIPT
-      # comment/uncomment below to enable/disable pushing to AWS
-      # vagrant/push.bash aws
-      # comment/uncomment below to enable/disable pushing to GCP
-      vagrant/push.bash gcp
+      # comment/uncomment below to enable/disable pushing to AWS/GCP
+      # or configure instance type large/xlarge/xxlarge
+
+      # vagrant/push.bash aws large
+      # vagrant/push.bash aws xlarge
+      # vagrant/push.bash aws xxlarge
+
+      # vagrant/push.bash gcp large
+      # vagrant/push.bash gcp xlarge
+      # vagrant/push.bash gcp xxlarge
     SCRIPT
   end
 

provisioning/roles/docker/files/.env

@@ -0,0 +1,2 @@
+ES_JVM_SIZE=4g
+SP_JVM_SIZE=512m

provisioning/roles/docker/files/docker-compose.yml

@@ -8,7 +8,7 @@ services:
     environment:
       # Swapping needs to be disabled for performance and node stability
       - "bootstrap.memory_lock=true"
-      - ES_JAVA_OPTS=-Xms4g -Xmx4g
+      - ES_JAVA_OPTS=-Xms${ES_JVM_SIZE} -Xmx${ES_JVM_SIZE}
     volumes:
       - /home/ubuntu/snowplow/elasticsearch/data:/usr/share/elasticsearch/data
       - /home/ubuntu/snowplow/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
@@ -53,7 +53,7 @@ services:
         max-size: "1M"
         max-file: "10"
     environment:
-      - "SP_JAVA_OPTS=-Xmx512m"
+      - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}"
 
   elasticsearch-loader-bad:
     image: snowplow-docker-registry.bintray.io/snowplow/elasticsearch-loader:0.10.1
@@ -69,7 +69,7 @@ services:
         max-size: "1M"
         max-file: "10"
     environment:
-      - "SP_JAVA_OPTS=-Xmx512m"
+      - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}"
 
   nsqlookupd:
     image: nsqio/nsq:v1.0.0-compat
@@ -123,7 +123,7 @@ services:
         max-size: "1M"
         max-file: "10"
     environment:
-      - "SP_JAVA_OPTS=-Xmx512m"
+      - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}"
 
   stream-enrich:
     image: snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.1
@@ -144,7 +144,7 @@ services:
         max-size: "1M"
         max-file: "10"
     environment:
-      - "SP_JAVA_OPTS=-Xmx512m"
+      - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}"
 
   postgres:
     image: postgres:9.5

provisioning/roles/docker/tasks/main.yml

@@ -18,10 +18,11 @@
   become: yes
   shell: chmod +x /usr/local/bin/docker-compose
 
-- name: Copy docker-compose.yml
+- name: Copy docker-compose.yml and its environment file
   copy: src={{ item.src }} dest={{ item.dest }} owner=ubuntu group=ubuntu mode=0644
   with_items:
     - { src: '../files/docker-compose.yml', dest: '/home/ubuntu/snowplow/' }
+    - { src: '../files/.env', dest: '/home/ubuntu/snowplow/' }
 
 - name: Create & set permissions of elasticsearch data directory
   become: yes

vagrant/push.bash

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -e
 
+########## FUNCTIONS #################
 # Similar to Perl die
 function die() {
 	echo "$@" 1>&2 ; exit 1;
@@ -38,41 +39,113 @@ function cd_root() {
 	dir="$( cd -P "$( dirname "${source}" )/.." && pwd )"
 	cd ${dir}
 }
+######################################
 
 cd_root
 
 # Precondition for running
 running=0 && is_running "running"
 [ ${running} -eq 1 ] || die "Vagrant guest must be running to push"
 
-if [ -z "$1" ]
+###### input validation ######
+if [ "$#" -ne 2 ]
 then
-	echo "Specify a platform! Aborted."
-	echo "Usage: push.bash <platform>"
-	echo "Supported platforms are 'aws', 'gcp'."
+	echo "Inappropriate number of arguments! Aborted."
+	echo "Usage: push.bash <platform> <size>"
 	exit 1
 else
-	if [ "$1" == "aws" ]
+	SP_MINI_PLATFORM="$1"
+	SP_MINI_SIZE="$2"
+
+	if [ "$SP_MINI_PLATFORM" == "aws" -o "$SP_MINI_PLATFORM" == "gcp" ]
 	then
-		# Can't pass args thru vagrant push so have to prompt
-		read -e -p "Please enter your AWS_ACCESS_KEY_ID: " aws_access_key_id
-		read -e -p "Please enter your AWS_SECRET_ACCESS_KEY: " aws_secret_access_key
+		echo "Platform recognized!"
+	else
+		echo "Unrecognized platform! Aborted."
+		echo "Supported platforms; aws, gcp."
+		exit 1
+	fi
 
-		# Build AMI
-		cmd="export AWS_ACCESS_KEY_ID=$aws_access_key_id && \
-			export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key && \
-			cd /vagrant && \
-			packer build -only=amazon-ebs Packerfile.json"
-	elif [ "$1" == "gcp" ]
+	if [ "$SP_MINI_SIZE" == "large" ]
 	then
-		echo "GCP uses account.json file to authenticate."
-		echo "Make sure account.json and Packerfile.json are in same directory!"
-		cmd="cd /vagrant && \
-			packer build -only=googlecompute Packerfile.json"
+		# prepare env vars for docker-compose
+		# to be used as -Xmx jvm option for Elasticsearch & Snowplow apps
+		echo -n > provisioning/roles/docker/files/.env
+		echo "ES_JVM_SIZE=4g" >> provisioning/roles/docker/files/.env
+		echo "SP_JVM_SIZE=512m" >> provisioning/roles/docker/files/.env
+		# prepare env var for packer
+		# to be used to determine which instance type to use
+		if [ "$SP_MINI_PLATFORM" == "aws" ]
+		then
+			platform_cmd="export AWS_INSTANCE_TYPE=t2.large"
+		elif [ "$SP_MINI_PLATFORM" == "gcp" ]
+		then
+			platform_cmd="export GCP_MACHINE_TYPE=n1-standard-2"
+		fi
+	elif [ "$SP_MINI_SIZE" == "xlarge" ]
+	then
+		# prepare env vars for docker-compose
+		# to be used as -Xmx jvm option for Elasticsearch & Snowplow apps
+		echo -n > provisioning/roles/docker/files/.env
+		echo "ES_JVM_SIZE=8g" >> provisioning/roles/docker/files/.env
+		echo "SP_JVM_SIZE=1536m" >> provisioning/roles/docker/files/.env
+		# prepare env var for packer
+		# to be used to determine which instance type to use
+		if [ "$SP_MINI_PLATFORM" == "aws" ]
+		then
+			platform_cmd="export AWS_INSTANCE_TYPE=t2.xlarge"
+		elif [ "$SP_MINI_PLATFORM" == "gcp" ]
+		then
+			platform_cmd="export GCP_MACHINE_TYPE=n1-standard-4"
+		fi
+	elif [ "$SP_MINI_SIZE" == "xxlarge" ]
+	then
+		# prepare env vars for docker-compose
+		# to be used as -Xmx jvm option for Elasticsearch & Snowplow apps
+		echo -n > provisioning/roles/docker/files/.env
+		echo "ES_JVM_SIZE=16g" >> provisioning/roles/docker/files/.env
+		echo "SP_JVM_SIZE=3g" >> provisioning/roles/docker/files/.env
+		# prepare env var for packer
+		# to be used to determine which instance type to use
+		if [ "$SP_MINI_PLATFORM" == "aws" ]
+		then
+			platform_cmd="export AWS_INSTANCE_TYPE=t2.2xlarge"
+		elif [ "$SP_MINI_PLATFORM" == "gcp" ]
+		then
+			platform_cmd="export GCP_MACHINE_TYPE=n1-standard-8"
+		fi
 	else
-		echo "Unrecognized platform. Aborted."
+		echo "Unrecognized size! Aborted."
+		echo "Available sizes; large, xlarge and xxlarge."
 		exit 1
 	fi
-	vagrant ssh -c "${cmd}"
-	exit 0
 fi
+###############################################
+
+if [ "$SP_MINI_PLATFORM" == "aws" ]
+then
+	# Can't pass args through vagrant push so have to prompt
+	read -e -p "Please enter your AWS_ACCESS_KEY_ID: " aws_access_key_id
+	read -e -p "Please enter your AWS_SECRET_ACCESS_KEY: " aws_secret_access_key
+
+	# Build AMI
+	cmd="$platform_cmd && \
+		export SP_MINI_SIZE=$SP_MINI_SIZE && \
+		export AWS_ACCESS_KEY_ID=$aws_access_key_id && \
+		export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key && \
+		cd /vagrant && \
+		packer build -only=amazon-ebs Packerfile.json"
+elif [ "$SP_MINI_PLATFORM" == "gcp" ]
+then
+	echo "GCP uses account.json file to authenticate."
+	echo "Make sure account.json and Packerfile.json are in same directory!"
+	cmd="$platform_cmd && \
+		export SP_MINI_SIZE=$SP_MINI_SIZE && \
+		cd /vagrant && \
+		packer build -only=googlecompute Packerfile.json"
+else
+	echo "Unrecognized platform. Aborted."
+	exit 1
+fi
+
+vagrant ssh -c "${cmd}"