Update build process to build Caddy from source (closes #132)

.travis.yml

@@ -1,6 +1,9 @@
 ---
 sudo: required
-language: bash
+language: go
+
+go:
+  - 1.8
 
 services:
   - postgresql
@@ -14,7 +17,7 @@ install:
   - sudo pip install ansible
 
 before_script:
-  - ansible-playbook -i provisioning/inventory provisioning/with_building_ui.yml --connection=local --sudo
+  - ansible-playbook -i provisioning/inventory provisioning/with_building_ui_and_go_projects.yml --connection=local --sudo
 
 script:
   - ./integration/integration_test.sh

Packerfile.json

@@ -24,7 +24,7 @@
   "provisioners": [
     {
       "type": "ansible",
-      "playbook_file": "provisioning/without_building_ui.yml"
+      "playbook_file": "provisioning/without_building_ui_and_go_projects.yml"
     }
   ]
 }

provisioning/resources/control-plane/.gitignore

@@ -0,0 +1 @@
+snowplow-mini-control-plane-api

provisioning/resources/init/caddy_init

@@ -9,9 +9,9 @@
 # Description:       Enable service provided by daemon.
 ### END INIT INFO
 
-dir="/usr/local/bin"
+dir="/home/ubuntu/snowplow/bin"
 
-cmd="caddy -conf /home/ubuntu/snowplow/configs/Caddyfile"
+cmd="./caddy -conf /home/ubuntu/snowplow/configs/Caddyfile"
 user=""
 
 name="caddy_server"
@@ -31,6 +31,7 @@ case "$1" in
         echo "Already started"
     else
         echo "Starting $name"
+        cd $dir
         if [ -z "$user" ]; then
             sudo $cmd &
         else

provisioning/roles/ansible-go/README.md

@@ -0,0 +1,37 @@
+Go
+========
+
+Ansible role that installs [Go](https://golang.org/). The latest stable release that has been compiled for x86 64-bit Linux systems is installed by default, and different platforms and versions are supported by modifying the role variables.
+
+Role Variables
+--------------
+
+All of these variables are optional and should only be changed if you need to install a different version of Go (e.g. if you are installing on FreeBSD, or if you need to use an earlier release).
+
+`go_tarball`: The tarball that you want to install. A list of options can be found on the [Go Downloads page](https://golang.org/dl/). The default is the official x86 64-bit Linux tarball for the latest stable release.
+
+`go_tarball_checksum`: This variable specifies the algorithm and checksum for the tarball that you want to install (e.g. `sha1:c7d78ba4df574b5f9a9bb5d17505f40c4d89b81c` or `sha256:a96cce8ce43a9bf9b2a4c7d470bc7ee0cb00410da815980681c8353218dcf146`). The default is the SHA256 checksum of the official x86 64-bit tarball for the latest stable release. Checksums can be found on the [Go Download Page](https://golang.org/dl/).
+
+`go_version_target`: The string that the `go version` command is expected to return (e.g. "go version go1.2.1 linux/amd64"). This variable is used to control whether or not the Go tarball should be extracted, thereby upgrading (or downgrading) a previously installed copy. If the installed version already matches the target, the extraction step is skipped.
+
+`go_download_location`: The full download URL. This variable simply appends the `go_tarball` variable onto the Go Download URL. This should not need to be modified.
+
+`set_go_path`: Whether or not to set the GOPATH for all users. The default is `true`.
+
+License
+-------
+
+The MIT License (MIT)
+
+Copyright (c) 2013-2016 Joshua Lund
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Author Information
+------------------
+
+You can find me on [Twitter](https://twitter.com/joshualund), and on [GitHub](https://github.com/jlund/). I also occasionally blog at [MissingM](https://missingm.co).

provisioning/roles/ansible-go/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+go_tarball: "go1.8.1.linux-amd64.tar.gz"
+go_tarball_checksum: "sha256:a579ab19d5237e263254f1eac5352efcf1d70b9dacadb6d6bb12b0911ede8994"
+go_version_target: "go version go1.8.1 linux/amd64"
+set_go_path: true

provisioning/roles/ansible-go/files/go-bin.sh

@@ -0,0 +1 @@
+export PATH=$PATH:/usr/local/go/bin

provisioning/roles/ansible-go/files/go-path.sh

@@ -0,0 +1,2 @@
+export GOPATH=$HOME/go
+export PATH=$GOPATH/bin:$PATH

provisioning/roles/ansible-go/meta/main.yml

@@ -0,0 +1,40 @@
+---
+galaxy_info:
+  author: "Joshua Lund"
+  description: "Ansible role that installs Go (https://golang.org/). The latest stable release that has been compiled for x86 64-bit Linux systems is installed by default, and different platforms and versions are supported by modifying the role variables."
+  license: MIT
+  min_ansible_version: 2.0
+  platforms:
+    - name: EL
+      versions:
+        - all
+    - name: GenericUNIX
+      versions:
+        - all
+    - name: Fedora
+      versions:
+        - all
+    - name: opensuse
+      versions:
+        - all
+    - name: GenericBSD
+      versions:
+        - all
+    - name: FreeBSD
+      versions:
+        - all
+    - name: Ubuntu
+      versions:
+        - all
+    - name: SLES
+      versions:
+        - all
+    - name: GenericLinux
+      versions:
+        - all
+    - name: Debian
+      versions:
+        - all
+  categories:
+    - development
+dependencies: []

provisioning/roles/ansible-go/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+- name: Download the Go tarball
+  become: yes
+  get_url:
+    url: "{{ go_download_location }}"
+    dest: /usr/local/src/{{ go_tarball }}
+    checksum: "{{ go_tarball_checksum }}"
+
+- name: Register the current Go version (if any)
+  become: yes
+  command: /usr/local/go/bin/go version
+  ignore_errors: yes
+  register: go_version
+  changed_when: false
+
+- name: Remove old installation of Go
+  become: yes
+  file:
+    path: /usr/local/go
+    state: absent
+  when: go_version|failed or go_version.stdout != go_version_target
+
+- name: Extract the Go tarball if Go is not yet installed or not the desired version
+  become: yes
+  unarchive:
+    src: /usr/local/src/{{ go_tarball }}
+    dest: /usr/local
+    copy: no
+  when: go_version|failed or go_version.stdout != go_version_target
+
+- name: Add the Go bin directory to the PATH environment variable for all users
+  become: yes
+  copy:
+    src: go-bin.sh
+    dest: /etc/profile.d
+
+- name: Set GOPATH for all users
+  become: yes
+  copy:
+    src: go-path.sh
+    dest: /etc/profile.d
+  when: set_go_path

provisioning/roles/ansible-go/vars/main.yml

@@ -0,0 +1,2 @@
+---
+go_download_location: "https://storage.googleapis.com/golang/{{ go_tarball }}"

provisioning/roles/sp_mini_3_build_go_projects/tasks/main.yml

@@ -0,0 +1,56 @@
+---
+- include_vars: ../../common_vars.yml
+
+- name: Set variables
+  set_fact:
+    control_plane_dir: '{{playbook_dir}}/resources/control-plane'
+    go_bin: '/usr/local/go/bin'
+    go_path: "{{ lookup('env', 'HOME') }}/go"
+
+- name: Install Git
+  become: yes
+  apt:
+    name: git
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+
+- name: Build Control Plane API
+  become: yes
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "go build -o {{control_plane_dir}}/snowplow-mini-control-plane-api {{control_plane_dir}}/snowplow-mini-control-plane-api.go {{control_plane_dir}}/utils.go"
+
+- name: Copy Control API to executables dir
+  become: yes
+  synchronize:
+    src: "{{control_plane_dir}}/snowplow-mini-control-plane-api"
+    dest: "{{executables_dir}}"
+
+- name: Get Caddy source code
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "go get -u github.com/mholt/caddy"
+
+- name: Get Caddyserver builds
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "go get -u github.com/caddyserver/builds"
+
+- name: Build Caddy
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  shell: "cd $GOPATH/src/github.com/mholt/caddy/caddy; go run build.go -goos=linux -goarch=amd64"
+
+- name: Copy Caddy executable to executables dir
+  become: yes
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  synchronize:
+    src: "/$GOPATH/src/github.com/mholt/caddy/caddy/caddy"
+    dest: "{{executables_dir}}"

provisioning/roles/sp_mini_3_setup_apps/tasks/main.yml → provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 - include_vars: ../../common_vars.yml
 
-- name: Set version variables
+- name: Set variables
   set_fact:
     kinesis_package: 'snowplow_kinesis_r85_metamorphosis.zip'
     es_loader_package: 'snowplow_elasticsearch_loader_http_0.10.0_rc1.zip'
@@ -10,6 +10,7 @@
     elasticsearch_v: '5.5.2'
     nsq_package: 'nsq-1.0.0-compat.linux-amd64.go1.8.tar.gz'
     nsq_bin_dir: 'nsq-1.0.0-compat.linux-amd64.go1.8/bin'
+    control_plane_dir: '{{playbook_dir}}/resources/control-plane'
 
 - name: Install unzip
   become: yes
@@ -42,13 +43,13 @@
 - name: Copy Control API to executables dir
   become: yes
   synchronize:
-    src: "{{playbook_dir}}/resources/control-plane/snowplow-mini-control-plane-api"
+    src: "{{executables_dir}}/snowplow-mini-control-plane-api"
     dest: "{{executables_dir}}"
 
 - name: Copy Control API scripts to the scripts' directory
   become: yes
   synchronize:
-    src: "{{playbook_dir}}/resources/control-plane/scripts/"
+    src: "{{control_plane_dir}}/scripts/"
     dest: "{{scripts_dir}}/snowplow-mini-control-plane-api-scripts"
 
 - name: Copy VERSION file to /home/ubuntu/snowplow for Control API
@@ -140,14 +141,6 @@
     name: iglu
     owner: snowplow
 
-- name: Install Git for getting Elasticsearch Head plugin
-  become: yes
-  apt:
-    name: git
-    state: present
-    update_cache: yes
-    cache_valid_time: 3600
-
 - name: Download Elasticsearch
   become: yes
   shell: "wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.2.deb -P {{staging_dir}}"
@@ -160,6 +153,14 @@
   become: yes
   shell: "cat {{es_dir}}/elasticsearch.yml >> /etc/elasticsearch/elasticsearch.yml"
 
+- name: Install Git for downloading ES Head Plugin
+  become: yes
+  apt:
+    name: git
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+
 - name: Download Elasticsearch Head Plugin
   become: yes
   shell: "git clone https://github.com/mobz/elasticsearch-head.git {{es_dir}}/elasticsearch-head"
@@ -176,9 +177,14 @@
   become: yes
   shell: "cat {{es_dir}}/kibana.yml >> /etc/kibana/kibana.yml"
 
-- name: Install Caddy
+- name: Copy Caddy executable to executables dir
   become: yes
-  shell: "curl https://getcaddy.com | bash"
+  environment:
+    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
+    GOPATH: "{{ go_path }}"
+  synchronize:
+    src: "{{executables_dir}}/caddy"
+    dest: "{{executables_dir}}"
 
 - name: Set owner of the main directory
   become: yes

provisioning/with_building_ui_and_go_projects.yml

@@ -0,0 +1,21 @@
+---
+# This playbook is used for provisioning of the vagrant machine.
+# All tools for compilation of the Control Plane API, Caddy Server
+# and UI is setup to the vagrant machine in here.
+# Additionally, Snowplow Mini environment is made ready to use locally.
+- name: apply all scripts
+  hosts: all
+
+  roles:
+    - ansible-go
+    - nodejs
+    - typescript
+    - packer
+    - sp_mini_1_create_dirs_and_pipes
+    - sp_mini_2_install_postgresl
+    - sp_mini_3_build_go_projects
+    - sp_mini_4_setup_apps
+    - sp_mini_5_build_ui
+    - sp_mini_6_copy_ui_folders
+    - sp_mini_7_setup_init
+    - sp_mini_8_configure

provisioning/without_building_ui_and_go_projects.yml

@@ -0,0 +1,14 @@
+---
+# This playbook is used for provisioning of the AMI
+# UI and executables file are made ready in the vagrant environment
+# They are only copied into the AMI here
+- name: apply all scripts
+  hosts: all
+
+  roles:
+    - sp_mini_1_create_dirs_and_pipes
+    - sp_mini_2_install_postgresl
+    - sp_mini_4_setup_apps
+    - sp_mini_6_copy_ui_folders
+    - sp_mini_7_setup_init
+    - sp_mini_8_configure

vagrant/up.bash

@@ -19,5 +19,5 @@ sudo pip install ansible
 
 vagrant_dir=/vagrant/vagrant
 cd $vagrant_dir/..
-ansible-playbook -i provisioning/inventory provisioning/with_building_ui.yml --connection=local --sudo
+ansible-playbook -i provisioning/inventory provisioning/with_building_ui_and_go_projects.yml --connection=local --sudo