Update build process to build Caddy from source (closes #132)

.travis.yml

@@ -1,11 +1,13 @@
 ---
 sudo: required
-language: bash
+language: go
+
+go:
+  - 1.8
 
 services:
   - postgresql
 
-
 before_install:
   - sudo apt-get update -qq
 
@@ -14,7 +16,7 @@ install:
   - sudo pip install ansible
 
 before_script:
-  - ansible-playbook -i provisioning/inventory provisioning/with_building_ui.yml --connection=local --sudo
+  - ansible-playbook -i provisioning/inventory provisioning/with_building_ui_and_go_projects.yml --connection=local --sudo
 
 script:
   - ./integration/integration_test.sh

Packerfile.json

@@ -24,7 +24,7 @@
   "provisioners": [
     {
       "type": "ansible",
-      "playbook_file": "provisioning/without_building_ui.yml"
+      "playbook_file": "provisioning/without_building_ui_and_go_projects.yml"
     }
   ]
 }

provisioning/resources/control-plane/.gitignore

@@ -0,0 +1 @@
+snowplow-mini-control-plane-api

provisioning/resources/init/caddy_init

@@ -9,9 +9,9 @@
 # Description:       Enable service provided by daemon.
 ### END INIT INFO
 
-dir="/usr/local/bin"
+dir="/home/ubuntu/snowplow/bin"
 
-cmd="caddy -conf /home/ubuntu/snowplow/configs/Caddyfile"
+cmd="$dir/caddy -conf /home/ubuntu/snowplow/configs/Caddyfile"
 user=""
 
 name="caddy_server"

provisioning/roles/ansible-go/README.md

@@ -0,0 +1,37 @@
+Go
+========
+
+Ansible role that installs [Go](https://golang.org/). The latest stable release that has been compiled for x86 64-bit Linux systems is installed by default, and different platforms and versions are supported by modifying the role variables.
+
+Role Variables
+--------------
+
+All of these variables are optional and should only be changed if you need to install a different version of Go (e.g. if you are installing on FreeBSD, or if you need to use an earlier release).
+
+`go_tarball`: The tarball that you want to install. A list of options can be found on the [Go Downloads page](https://golang.org/dl/). The default is the official x86 64-bit Linux tarball for the latest stable release.
+
+`go_tarball_checksum`: This variable specifies the algorithm and checksum for the tarball that you want to install (e.g. `sha1:c7d78ba4df574b5f9a9bb5d17505f40c4d89b81c` or `sha256:a96cce8ce43a9bf9b2a4c7d470bc7ee0cb00410da815980681c8353218dcf146`). The default is the SHA256 checksum of the official x86 64-bit tarball for the latest stable release. Checksums can be found on the [Go Download Page](https://golang.org/dl/).
+
+`go_version_target`: The string that the `go version` command is expected to return (e.g. "go version go1.2.1 linux/amd64"). This variable is used to control whether or not the Go tarball should be extracted, thereby upgrading (or downgrading) a previously installed copy. If the installed version already matches the target, the extraction step is skipped.
+
+`go_download_location`: The full download URL. This variable simply appends the `go_tarball` variable onto the Go Download URL. This should not need to be modified.
+
+`set_go_path`: Whether or not to set the GOPATH for all users. The default is `true`.
+
+License
+-------
+
+The MIT License (MIT)
+
+Copyright (c) 2013-2016 Joshua Lund
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Author Information
+------------------
+
+You can find me on [Twitter](https://twitter.com/joshualund), and on [GitHub](https://github.com/jlund/). I also occasionally blog at [MissingM](https://missingm.co).

provisioning/roles/ansible-go/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+go_tarball: "go1.8.1.linux-amd64.tar.gz"
+go_tarball_checksum: "sha256:a579ab19d5237e263254f1eac5352efcf1d70b9dacadb6d6bb12b0911ede8994"
+go_version_target: "go version go1.8.1 linux/amd64"
+set_go_path: true

provisioning/roles/ansible-go/files/go-bin.sh

@@ -0,0 +1 @@
+export PATH=$PATH:/usr/local/go/bin

provisioning/roles/ansible-go/files/go-path.sh

@@ -0,0 +1,2 @@
+export GOPATH=$HOME/go
+export PATH=$GOPATH/bin:$PATH

provisioning/roles/ansible-go/meta/main.yml

@@ -0,0 +1,40 @@
+---
+galaxy_info:
+  author: "Joshua Lund"
+  description: "Ansible role that installs Go (https://golang.org/). The latest stable release that has been compiled for x86 64-bit Linux systems is installed by default, and different platforms and versions are supported by modifying the role variables."
+  license: MIT
+  min_ansible_version: 2.0
+  platforms:
+    - name: EL
+      versions:
+        - all
+    - name: GenericUNIX
+      versions:
+        - all
+    - name: Fedora
+      versions:
+        - all
+    - name: opensuse
+      versions:
+        - all
+    - name: GenericBSD
+      versions:
+        - all
+    - name: FreeBSD
+      versions:
+        - all
+    - name: Ubuntu
+      versions:
+        - all
+    - name: SLES
+      versions:
+        - all
+    - name: GenericLinux
+      versions:
+        - all
+    - name: Debian
+      versions:
+        - all
+  categories:
+    - development
+dependencies: []

provisioning/roles/ansible-go/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+- name: Download the Go tarball
+  become: yes
+  get_url:
+    url: "{{ go_download_location }}"
+    dest: /usr/local/src/{{ go_tarball }}
+    checksum: "{{ go_tarball_checksum }}"
+
+- name: Register the current Go version (if any)
+  become: yes
+  command: /usr/local/go/bin/go version
+  ignore_errors: yes
+  register: go_version
+  changed_when: false
+
+- name: Remove old installation of Go
+  become: yes
+  file:
+    path: /usr/local/go
+    state: absent
+  when: go_version|failed or go_version.stdout != go_version_target
+
+- name: Extract the Go tarball if Go is not yet installed or not the desired version
+  become: yes
+  unarchive:
+    src: /usr/local/src/{{ go_tarball }}
+    dest: /usr/local
+    copy: no
+  when: go_version|failed or go_version.stdout != go_version_target
+
+- name: Add the Go bin directory to the PATH environment variable for all users
+  become: yes
+  copy:
+    src: go-bin.sh
+    dest: /etc/profile.d
+
+- name: Set GOPATH for all users
+  become: yes
+  copy:
+    src: go-path.sh
+    dest: /etc/profile.d
+  when: set_go_path

provisioning/roles/ansible-go/vars/main.yml

@@ -0,0 +1,2 @@
+---
+go_download_location: "https://storage.googleapis.com/golang/{{ go_tarball }}"

provisioning/roles/sp_mini_3_build_go_projects/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+- include_vars: ../../common_vars.yml
+
+- name: Set variables
+  set_fact:
+    control_plane_dir: '{{playbook_dir}}/resources/control-plane'
+    go_bin: '/usr/local/go/bin'
+    go_path: "{{ lookup('env', 'HOME') }}/go"
+
+- name: Install Git
+  become: yes
+  apt:
+    name: git
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+
+- name: Get toml parser library for Golang
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "go get github.com/BurntSushi/toml"
+
+- name: Get psql drive for Golang
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "go get gopkg.in/pg.v5"
+
+- name: Build Control Plane API
+  become: yes
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "go build -o {{control_plane_dir}}/snowplow-mini-control-plane-api {{control_plane_dir}}/*.go"
+
+- name: Copy Control API to executables dir
+  become: yes
+  synchronize:
+    src: "{{control_plane_dir}}/snowplow-mini-control-plane-api"
+    dest: "{{executables_dir}}"
+
+- name: Get Caddy source code
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "go get -u github.com/mholt/caddy"
+
+- name: Get Caddyserver builds
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "go get -u github.com/caddyserver/builds"
+
+- name: Build Caddy
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "cd $GOPATH/src/github.com/mholt/caddy/caddy; go run build.go -goos=linux -goarch=amd64"
+
+- name: Copy Caddy executable to executables dir
+  become: yes
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  synchronize:
+    src: "/$GOPATH/src/github.com/mholt/caddy/caddy/caddy"
+    dest: "{{executables_dir}}"

provisioning/roles/sp_mini_3_setup_apps/tasks/main.yml → provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml

@@ -1,14 +1,16 @@
 ---
 - include_vars: ../../common_vars.yml
 
-- name: Set version variables
+- name: Set variables
   set_fact:
-    kinesis_package: 'snowplow_kinesis_r85_metamorphosis.zip'
-    es_loader_package: 'snowplow_elasticsearch_loader_http_0.10.0_rc1.zip'
+    stream_collector_package: 'snowplow_scala_stream_collector_0.11.0.zip'
+    stream_enrich_package: 'snowplow_stream_enrich_0.12.0.zip'
+    es_loader_package: 'snowplow_elasticsearch_loader_http_0.10.1.zip'
     iglu_server_package: 'iglu_server_0.2.0.zip'
     kibana_v: '4.0.1'
     nsq_package: 'nsq-1.0.0-compat.linux-amd64.go1.8.tar.gz'
     nsq_bin_dir: 'nsq-1.0.0-compat.linux-amd64.go1.8/bin'
+    control_plane_dir: '{{playbook_dir}}/resources/control-plane'
 
 - name: Install unzip
   become: yes
@@ -41,7 +43,7 @@
 - name: Copy Control API to executables dir
   become: yes
   synchronize:
-    src: "{{playbook_dir}}/resources/control-plane/snowplow-mini-control-plane-api"
+    src: "{{executables_dir}}/snowplow-mini-control-plane-api"
     dest: "{{executables_dir}}"
 
 - name: Copy VERSION file to /home/ubuntu/snowplow for Control API
@@ -50,30 +52,37 @@
      src: "{{playbook_dir}}/../VERSION"
      dest: "{{main_dir}}"
 
-#- name: Check Kinesis Packages
-#  stat:
-#    path: "{{staging_dir}}/{{kinesis_package}}"
-#  register: check_kinesis_packages_result
-
-#- name: Download Kinesis Packages
-#  get_url:
-#    url: "http://dl.bintray.com/snowplow/snowplow-generic/{{kinesis_package}}"
-#    dest: "{{staging_dir}}"
-#  when: check_kinesis_packages_result.stat.exists == False
-#  register: kinesis_packages_downloaded
-
-#- name: Unzip downloaded Kinesis Packages
-#  shell: "unzip {{staging_dir}}/{{kinesis_package}} -d {{executables_dir}}"
-#  when: kinesis_packages_downloaded|changed
-
-# this section will be removed after new version of the snowplow-apps
-# is placed on the bintray
-- name: Copy snowplow-apps to the executables dir
-  become: yes
-  block:
-    - synchronize:
-        src: "{{playbook_dir}}/resources/snowplow-apps/"
-        dest: "{{executables_dir}}"
+- name: Check Stream Collector
+  stat:
+    path: "{{staging_dir}}/{{stream_collector_package}}"
+  register: check_stream_collector_result
+
+- name: Download Stream Collector
+  get_url:
+    url: "http://dl.bintray.com/snowplow/snowplow-generic/{{stream_collector_package}}"
+    dest: "{{staging_dir}}"
+  when: check_stream_collector_result.stat.exists == False
+  register: stream_collector_downloaded
+
+- name: Unzip downloaded Stream Collector
+  shell: "unzip {{staging_dir}}/{{stream_collector_package}} -d {{executables_dir}}"
+  when: stream_collector_downloaded|changed
+
+- name: Check Stream Enrich
+  stat:
+    path: "{{staging_dir}}/{{stream_enrich_package}}"
+  register: check_stream_enrich_result
+
+- name: Download Stream Enrich
+  get_url:
+    url: "http://dl.bintray.com/snowplow/snowplow-generic/{{stream_enrich_package}}"
+    dest: "{{staging_dir}}"
+  when: check_stream_enrich_result.stat.exists == False
+  register: stream_enrich_downloaded
+
+- name: Unzip downloaded Stream Enrich
+  shell: "unzip {{staging_dir}}/{{stream_enrich_package}} -d {{executables_dir}}"
+  when: stream_enrich_downloaded|changed
 
 - name: Check Elasticsearch Loader
   stat:
@@ -147,7 +156,7 @@
 
 - name: Check Kibana
   stat:
-    path: "{{staging_dir}}/{{kinesis_package}}"
+    path: "{{staging_dir}}/kibana-{{kibana_v}}-linux-x64.zip"
   register: check_kibana_result
 
 - name: Download Kibana
@@ -161,17 +170,24 @@
   become: yes
   shell: "unzip {{staging_dir}}/kibana-{{kibana_v}}-linux-x64.zip -d /opt/"
   when: kibana_downloaded|changed
+  register: kibana_unzipped
 
 - name: Symlink for kibana
   become: yes
   file:
     src: "/opt/kibana-{{kibana_v}}-linux-x64"
     dest: "/opt/kibana"
     state: link
+  when: kibana_unzipped|changed
 
-- name: Install Caddy
+- name: Copy Caddy executable to executables dir
   become: yes
-  shell: "curl https://getcaddy.com | bash"
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  synchronize:
+    src: "{{executables_dir}}/caddy"
+    dest: "{{executables_dir}}"
 
 - name: Set owner of the main directory
   become: yes