From c395b979f52eb9754316163347e4c21d837425dc Mon Sep 17 00:00:00 2001 From: Oguzhan Unlu Date: Thu, 12 Jul 2018 19:22:21 +0300 Subject: [PATCH] Publish mini to bigger instance types (closes #173) --- .travis.yml | 1 + Packerfile.json | 14 ++- Vagrantfile | 14 ++- provisioning/roles/docker/files/.env | 2 + .../roles/docker/files/docker-compose.yml | 10 +- provisioning/roles/docker/tasks/main.yml | 3 +- vagrant/push.bash | 115 ++++++++++++++---- 7 files changed, 123 insertions(+), 36 deletions(-) create mode 100755 provisioning/roles/docker/files/.env diff --git a/.travis.yml b/.travis.yml index 38b2a3fe..71757f5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ --- sudo: required +dist: trusty language: go go: diff --git a/Packerfile.json b/Packerfile.json index 2650d88d..b0b824ac 100644 --- a/Packerfile.json +++ b/Packerfile.json @@ -5,9 +5,9 @@ "ami_groups": [ "all" ], - "ami_name": "snowplow-mini-{{user `version`}}-{{ timestamp }}-hvm-ebs-amd64", + "ami_name": "snowplow-mini-{{user `aws_version`}}-{{user `sp_mini_size`}}-{{ timestamp }}-hvm-ebs-amd64", "ami_regions": "us-east-2,us-west-1,us-west-2,ca-central-1,eu-west-1,eu-central-1,eu-west-2,ap-southeast-1,ap-southeast-2,ap-northeast-2,ap-northeast-1,ap-south-1,sa-east-1", - "instance_type": "t2.medium", + "instance_type": "{{user `aws_instance_type`}}", "region": "us-east-1", "source_ami": "ami-58167327", "ssh_username": "ubuntu", @@ -20,8 +20,8 @@ { "type": "googlecompute", "image_description": "Snowplow Mini - The Snowplow Pipeline in a box", - "image_name": "snowplow-mini-{{user `version`}}-{{timestamp}}", - "machine_type": "n1-standard-2", + "image_name": "snowplow-mini-{{user `gcp_version`}}-{{user `sp_mini_size`}}-{{timestamp}}", + "machine_type": "{{user `gcp_machine_type`}}", "account_file": "account.json", "project_id": "engineering-sandbox", "source_image_family": "ubuntu-1404-lts", @@ -37,6 +37,10 @@ } ], "variables": { - "version": "0.5.0" + "aws_version": "0.5.0", + "gcp_version": "0-5-0", + "gcp_machine_type": "{{env `GCP_MACHINE_TYPE`}}", + "aws_instance_type": "{{env `AWS_INSTANCE_TYPE`}}", + "sp_mini_size": "{{env `SP_MINI_SIZE`}}" } } diff --git a/Vagrantfile b/Vagrantfile index e348e923..8b726478 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -32,10 +32,16 @@ Vagrant.configure("2") do |config| # Requires Vagrant 1.7.0+ config.push.define "publish", strategy: "local-exec" do |push| push.inline = <<-SCRIPT - # comment/uncomment below to enable/disable pushing to AWS - # vagrant/push.bash aws - # comment/uncomment below to enable/disable pushing to GCP - vagrant/push.bash gcp + # comment/uncomment below to enable/disable pushing to AWS/GCP + # or configure instance type large/xlarge/xxlarge + + # vagrant/push.bash aws large + # vagrant/push.bash aws xlarge + # vagrant/push.bash aws xxlarge + + # vagrant/push.bash gcp large + # vagrant/push.bash gcp xlarge + # vagrant/push.bash gcp xxlarge SCRIPT end diff --git a/provisioning/roles/docker/files/.env b/provisioning/roles/docker/files/.env new file mode 100755 index 00000000..268f9274 --- /dev/null +++ b/provisioning/roles/docker/files/.env @@ -0,0 +1,2 @@ +ES_JVM_SIZE=4g +SP_JVM_SIZE=512m diff --git a/provisioning/roles/docker/files/docker-compose.yml b/provisioning/roles/docker/files/docker-compose.yml index 6f8996b2..5fcf58b2 100644 --- a/provisioning/roles/docker/files/docker-compose.yml +++ b/provisioning/roles/docker/files/docker-compose.yml @@ -8,7 +8,7 @@ services: environment: # Swapping needs to be disabled for performance and node stability - "bootstrap.memory_lock=true" - - ES_JAVA_OPTS=-Xms4g -Xmx4g + - ES_JAVA_OPTS=-Xms${ES_JVM_SIZE} -Xmx${ES_JVM_SIZE} volumes: - /home/ubuntu/snowplow/elasticsearch/data:/usr/share/elasticsearch/data - /home/ubuntu/snowplow/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml @@ -53,7 +53,7 @@ services: max-size: "1M" max-file: "10" environment: - - "SP_JAVA_OPTS=-Xmx512m" + - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}" elasticsearch-loader-bad: image: snowplow-docker-registry.bintray.io/snowplow/elasticsearch-loader:0.10.1 @@ -69,7 +69,7 @@ services: max-size: "1M" max-file: "10" environment: - - "SP_JAVA_OPTS=-Xmx512m" + - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}" nsqlookupd: image: nsqio/nsq:v1.0.0-compat @@ -123,7 +123,7 @@ services: max-size: "1M" max-file: "10" environment: - - "SP_JAVA_OPTS=-Xmx512m" + - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}" stream-enrich: image: snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.1 @@ -144,7 +144,7 @@ services: max-size: "1M" max-file: "10" environment: - - "SP_JAVA_OPTS=-Xmx512m" + - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}" postgres: image: postgres:9.5 diff --git a/provisioning/roles/docker/tasks/main.yml b/provisioning/roles/docker/tasks/main.yml index 44b83dcd..0e512ece 100644 --- a/provisioning/roles/docker/tasks/main.yml +++ b/provisioning/roles/docker/tasks/main.yml @@ -18,10 +18,11 @@ become: yes shell: chmod +x /usr/local/bin/docker-compose -- name: Copy docker-compose.yml +- name: Copy docker-compose.yml and its environment file copy: src={{ item.src }} dest={{ item.dest }} owner=ubuntu group=ubuntu mode=0644 with_items: - { src: '../files/docker-compose.yml', dest: '/home/ubuntu/snowplow/' } + - { src: '../files/.env', dest: '/home/ubuntu/snowplow/' } - name: Create & set permissions of elasticsearch data directory become: yes diff --git a/vagrant/push.bash b/vagrant/push.bash index 4a313c4a..39266ee7 100755 --- a/vagrant/push.bash +++ b/vagrant/push.bash @@ -1,6 +1,7 @@ #!/bin/bash set -e +########## FUNCTIONS ################# # Similar to Perl die function die() { echo "$@" 1>&2 ; exit 1; @@ -38,6 +39,7 @@ function cd_root() { dir="$( cd -P "$( dirname "${source}" )/.." && pwd )" cd ${dir} } +###################################### cd_root @@ -45,34 +47,105 @@ cd_root running=0 && is_running "running" [ ${running} -eq 1 ] || die "Vagrant guest must be running to push" -if [ -z "$1" ] +###### input validation ###### +if [ "$#" -ne 2 ] then - echo "Specify a platform! Aborted." - echo "Usage: push.bash " - echo "Supported platforms are 'aws', 'gcp'." + echo "Inappropriate number of arguments! Aborted." + echo "Usage: push.bash " exit 1 else - if [ "$1" == "aws" ] + SP_MINI_PLATFORM="$1" + SP_MINI_SIZE="$2" + + if [ "$SP_MINI_PLATFORM" == "aws" -o "$SP_MINI_PLATFORM" == "gcp" ] then - # Can't pass args thru vagrant push so have to prompt - read -e -p "Please enter your AWS_ACCESS_KEY_ID: " aws_access_key_id - read -e -p "Please enter your AWS_SECRET_ACCESS_KEY: " aws_secret_access_key + echo "Platform recognized!" + else + echo "Unrecognized platform! Aborted." + echo "Supported platforms; aws, gcp." + exit 1 + fi - # Build AMI - cmd="export AWS_ACCESS_KEY_ID=$aws_access_key_id && \ - export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key && \ - cd /vagrant && \ - packer build -only=amazon-ebs Packerfile.json" - elif [ "$1" == "gcp" ] + if [ "$SP_MINI_SIZE" == "large" ] then - echo "GCP uses account.json file to authenticate." - echo "Make sure account.json and Packerfile.json are in same directory!" - cmd="cd /vagrant && \ - packer build -only=googlecompute Packerfile.json" + # prepare env vars for docker-compose + # to be used as -Xmx jvm option for Elasticsearch & Snowplow apps + echo -n > provisioning/roles/docker/files/.env + echo "ES_JVM_SIZE=4g" >> provisioning/roles/docker/files/.env + echo "SP_JVM_SIZE=512m" >> provisioning/roles/docker/files/.env + # prepare env var for packer + # to be used to determine which instance type to use + if [ "$SP_MINI_PLATFORM" == "aws" ] + then + platform_cmd="export AWS_INSTANCE_TYPE=t2.large" + elif [ "$SP_MINI_PLATFORM" == "gcp" ] + then + platform_cmd="export GCP_MACHINE_TYPE=n1-standard-2" + fi + elif [ "$SP_MINI_SIZE" == "xlarge" ] + then + # prepare env vars for docker-compose + # to be used as -Xmx jvm option for Elasticsearch & Snowplow apps + echo -n > provisioning/roles/docker/files/.env + echo "ES_JVM_SIZE=8g" >> provisioning/roles/docker/files/.env + echo "SP_JVM_SIZE=1536m" >> provisioning/roles/docker/files/.env + # prepare env var for packer + # to be used to determine which instance type to use + if [ "$SP_MINI_PLATFORM" == "aws" ] + then + platform_cmd="export AWS_INSTANCE_TYPE=t2.xlarge" + elif [ "$SP_MINI_PLATFORM" == "gcp" ] + then + platform_cmd="export GCP_MACHINE_TYPE=n1-standard-4" + fi + elif [ "$SP_MINI_SIZE" == "xxlarge" ] + then + # prepare env vars for docker-compose + # to be used as -Xmx jvm option for Elasticsearch & Snowplow apps + echo -n > provisioning/roles/docker/files/.env + echo "ES_JVM_SIZE=16g" >> provisioning/roles/docker/files/.env + echo "SP_JVM_SIZE=3g" >> provisioning/roles/docker/files/.env + # prepare env var for packer + # to be used to determine which instance type to use + if [ "$SP_MINI_PLATFORM" == "aws" ] + then + platform_cmd="export AWS_INSTANCE_TYPE=t2.2xlarge" + elif [ "$SP_MINI_PLATFORM" == "gcp" ] + then + platform_cmd="export GCP_MACHINE_TYPE=n1-standard-8" + fi else - echo "Unrecognized platform. Aborted." + echo "Unrecognized size! Aborted." + echo "Available sizes; large, xlarge and xxlarge." exit 1 fi - vagrant ssh -c "${cmd}" - exit 0 fi +############################################### + +if [ "$SP_MINI_PLATFORM" == "aws" ] +then + # Can't pass args through vagrant push so have to prompt + read -e -p "Please enter your AWS_ACCESS_KEY_ID: " aws_access_key_id + read -e -p "Please enter your AWS_SECRET_ACCESS_KEY: " aws_secret_access_key + + # Build AMI + cmd="$platform_cmd && \ + export SP_MINI_SIZE=$SP_MINI_SIZE && \ + export AWS_ACCESS_KEY_ID=$aws_access_key_id && \ + export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key && \ + cd /vagrant && \ + packer build -only=amazon-ebs Packerfile.json" +elif [ "$SP_MINI_PLATFORM" == "gcp" ] +then + echo "GCP uses account.json file to authenticate." + echo "Make sure account.json and Packerfile.json are in same directory!" + cmd="$platform_cmd && \ + export SP_MINI_SIZE=$SP_MINI_SIZE && \ + cd /vagrant && \ + packer build -only=googlecompute Packerfile.json" +else + echo "Unrecognized platform. Aborted." + exit 1 +fi + +vagrant ssh -c "${cmd}"