diff --git a/.travis.yml b/.travis.yml index 9cd3a060..1eb913bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ --- sudo: required -language: bash +language: go + +go: + - 1.8 services: - postgresql @@ -14,7 +17,7 @@ install: - sudo pip install ansible before_script: - - ansible-playbook -i provisioning/inventory provisioning/with_building_ui.yml --connection=local --sudo + - ansible-playbook -i provisioning/inventory provisioning/with_building_ui_and_go_projects.yml --connection=local --sudo script: - ./integration/integration_test.sh diff --git a/Packerfile.json b/Packerfile.json index d104b61f..c623fd3e 100644 --- a/Packerfile.json +++ b/Packerfile.json @@ -24,7 +24,7 @@ "provisioners": [ { "type": "ansible", - "playbook_file": "provisioning/without_building_ui.yml" + "playbook_file": "provisioning/without_building_ui_and_go_projects.yml" } ] } diff --git a/provisioning/resources/control-plane/.gitignore b/provisioning/resources/control-plane/.gitignore new file mode 100644 index 00000000..8dd83ad1 --- /dev/null +++ b/provisioning/resources/control-plane/.gitignore @@ -0,0 +1 @@ +snowplow-mini-control-plane-api diff --git a/provisioning/resources/init/caddy_init b/provisioning/resources/init/caddy_init index 589dd30d..ec76ff36 100755 --- a/provisioning/resources/init/caddy_init +++ b/provisioning/resources/init/caddy_init @@ -9,9 +9,9 @@ # Description: Enable service provided by daemon. ### END INIT INFO -dir="/usr/local/bin" +dir="/home/ubuntu/snowplow/bin" -cmd="caddy -conf /home/ubuntu/snowplow/configs/Caddyfile" +cmd="$dir/caddy -conf /home/ubuntu/snowplow/configs/Caddyfile" user="" name="caddy_server" diff --git a/provisioning/roles/ansible-go/README.md b/provisioning/roles/ansible-go/README.md new file mode 100644 index 00000000..dbf59f9e --- /dev/null +++ b/provisioning/roles/ansible-go/README.md @@ -0,0 +1,37 @@ +Go +======== + +Ansible role that installs [Go](https://golang.org/). The latest stable release that has been compiled for x86 64-bit Linux systems is installed by default, and different platforms and versions are supported by modifying the role variables. + +Role Variables +-------------- + +All of these variables are optional and should only be changed if you need to install a different version of Go (e.g. if you are installing on FreeBSD, or if you need to use an earlier release). + +`go_tarball`: The tarball that you want to install. A list of options can be found on the [Go Downloads page](https://golang.org/dl/). The default is the official x86 64-bit Linux tarball for the latest stable release. + +`go_tarball_checksum`: This variable specifies the algorithm and checksum for the tarball that you want to install (e.g. `sha1:c7d78ba4df574b5f9a9bb5d17505f40c4d89b81c` or `sha256:a96cce8ce43a9bf9b2a4c7d470bc7ee0cb00410da815980681c8353218dcf146`). The default is the SHA256 checksum of the official x86 64-bit tarball for the latest stable release. Checksums can be found on the [Go Download Page](https://golang.org/dl/). + +`go_version_target`: The string that the `go version` command is expected to return (e.g. "go version go1.2.1 linux/amd64"). This variable is used to control whether or not the Go tarball should be extracted, thereby upgrading (or downgrading) a previously installed copy. If the installed version already matches the target, the extraction step is skipped. + +`go_download_location`: The full download URL. This variable simply appends the `go_tarball` variable onto the Go Download URL. This should not need to be modified. + +`set_go_path`: Whether or not to set the GOPATH for all users. The default is `true`. + +License +------- + +The MIT License (MIT) + +Copyright (c) 2013-2016 Joshua Lund + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Author Information +------------------ + +You can find me on [Twitter](https://twitter.com/joshualund), and on [GitHub](https://github.com/jlund/). I also occasionally blog at [MissingM](https://missingm.co). diff --git a/provisioning/roles/ansible-go/defaults/main.yml b/provisioning/roles/ansible-go/defaults/main.yml new file mode 100644 index 00000000..e559479d --- /dev/null +++ b/provisioning/roles/ansible-go/defaults/main.yml @@ -0,0 +1,5 @@ +--- +go_tarball: "go1.8.1.linux-amd64.tar.gz" +go_tarball_checksum: "sha256:a579ab19d5237e263254f1eac5352efcf1d70b9dacadb6d6bb12b0911ede8994" +go_version_target: "go version go1.8.1 linux/amd64" +set_go_path: true diff --git a/provisioning/roles/ansible-go/files/go-bin.sh b/provisioning/roles/ansible-go/files/go-bin.sh new file mode 100644 index 00000000..680c12af --- /dev/null +++ b/provisioning/roles/ansible-go/files/go-bin.sh @@ -0,0 +1 @@ +export PATH=$PATH:/usr/local/go/bin diff --git a/provisioning/roles/ansible-go/files/go-path.sh b/provisioning/roles/ansible-go/files/go-path.sh new file mode 100644 index 00000000..5f4ffac8 --- /dev/null +++ b/provisioning/roles/ansible-go/files/go-path.sh @@ -0,0 +1,2 @@ +export GOPATH=$HOME/go +export PATH=$GOPATH/bin:$PATH diff --git a/provisioning/roles/ansible-go/meta/main.yml b/provisioning/roles/ansible-go/meta/main.yml new file mode 100644 index 00000000..55c97f94 --- /dev/null +++ b/provisioning/roles/ansible-go/meta/main.yml @@ -0,0 +1,40 @@ +--- +galaxy_info: + author: "Joshua Lund" + description: "Ansible role that installs Go (https://golang.org/). The latest stable release that has been compiled for x86 64-bit Linux systems is installed by default, and different platforms and versions are supported by modifying the role variables." + license: MIT + min_ansible_version: 2.0 + platforms: + - name: EL + versions: + - all + - name: GenericUNIX + versions: + - all + - name: Fedora + versions: + - all + - name: opensuse + versions: + - all + - name: GenericBSD + versions: + - all + - name: FreeBSD + versions: + - all + - name: Ubuntu + versions: + - all + - name: SLES + versions: + - all + - name: GenericLinux + versions: + - all + - name: Debian + versions: + - all + categories: + - development +dependencies: [] diff --git a/provisioning/roles/ansible-go/tasks/main.yml b/provisioning/roles/ansible-go/tasks/main.yml new file mode 100644 index 00000000..114be643 --- /dev/null +++ b/provisioning/roles/ansible-go/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Download the Go tarball + become: yes + get_url: + url: "{{ go_download_location }}" + dest: /usr/local/src/{{ go_tarball }} + checksum: "{{ go_tarball_checksum }}" + +- name: Register the current Go version (if any) + become: yes + command: /usr/local/go/bin/go version + ignore_errors: yes + register: go_version + changed_when: false + +- name: Remove old installation of Go + become: yes + file: + path: /usr/local/go + state: absent + when: go_version|failed or go_version.stdout != go_version_target + +- name: Extract the Go tarball if Go is not yet installed or not the desired version + become: yes + unarchive: + src: /usr/local/src/{{ go_tarball }} + dest: /usr/local + copy: no + when: go_version|failed or go_version.stdout != go_version_target + +- name: Add the Go bin directory to the PATH environment variable for all users + become: yes + copy: + src: go-bin.sh + dest: /etc/profile.d + +- name: Set GOPATH for all users + become: yes + copy: + src: go-path.sh + dest: /etc/profile.d + when: set_go_path diff --git a/provisioning/roles/ansible-go/vars/main.yml b/provisioning/roles/ansible-go/vars/main.yml new file mode 100644 index 00000000..7618a612 --- /dev/null +++ b/provisioning/roles/ansible-go/vars/main.yml @@ -0,0 +1,2 @@ +--- +go_download_location: "https://storage.googleapis.com/golang/{{ go_tarball }}" diff --git a/provisioning/roles/sp_mini_3_build_go_projects/tasks/main.yml b/provisioning/roles/sp_mini_3_build_go_projects/tasks/main.yml new file mode 100644 index 00000000..d72af213 --- /dev/null +++ b/provisioning/roles/sp_mini_3_build_go_projects/tasks/main.yml @@ -0,0 +1,68 @@ +--- +- include_vars: ../../common_vars.yml + +- name: Set variables + set_fact: + control_plane_dir: '{{playbook_dir}}/resources/control-plane' + go_bin: '/usr/local/go/bin' + go_path: "{{ lookup('env', 'HOME') }}/go" + +- name: Install Git + become: yes + apt: + name: git + state: present + update_cache: yes + cache_valid_time: 3600 + +- name: Get toml parser library for Golang + environment: + PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}" + GOPATH: "{{ go_path }}" + shell: "go get github.com/BurntSushi/toml" + +- name: Get psql drive for Golang + environment: + PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}" + GOPATH: "{{ go_path }}" + shell: "go get gopkg.in/pg.v5" + +- name: Build Control Plane API + become: yes + environment: + PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}" + GOPATH: "{{ go_path }}" + shell: "go build -o {{control_plane_dir}}/snowplow-mini-control-plane-api {{control_plane_dir}}/*.go" + +- name: Copy Control API to executables dir + become: yes + synchronize: + src: "{{control_plane_dir}}/snowplow-mini-control-plane-api" + dest: "{{executables_dir}}" + +- name: Get Caddy source code + environment: + PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}" + GOPATH: "{{ go_path }}" + shell: "go get -u github.com/mholt/caddy" + +- name: Get Caddyserver builds + environment: + PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}" + GOPATH: "{{ go_path }}" + shell: "go get -u github.com/caddyserver/builds" + +- name: Build Caddy + environment: + PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}" + GOPATH: "{{ go_path }}" + shell: "cd $GOPATH/src/github.com/mholt/caddy/caddy; go run build.go -goos=linux -goarch=amd64" + +- name: Copy Caddy executable to executables dir + become: yes + environment: + PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}" + GOPATH: "{{ go_path }}" + synchronize: + src: "/$GOPATH/src/github.com/mholt/caddy/caddy/caddy" + dest: "{{executables_dir}}" diff --git a/provisioning/roles/sp_mini_3_setup_apps/tasks/main.yml b/provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml similarity index 93% rename from provisioning/roles/sp_mini_3_setup_apps/tasks/main.yml rename to provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml index ad043b78..58462d64 100644 --- a/provisioning/roles/sp_mini_3_setup_apps/tasks/main.yml +++ b/provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml @@ -1,7 +1,7 @@ --- - include_vars: ../../common_vars.yml -- name: Set version variables +- name: Set variables set_fact: kinesis_package: 'snowplow_kinesis_r85_metamorphosis.zip' es_loader_package: 'snowplow_elasticsearch_loader_http_0.10.0_rc1.zip' @@ -9,6 +9,7 @@ kibana_v: '4.0.1' nsq_package: 'nsq-1.0.0-compat.linux-amd64.go1.8.tar.gz' nsq_bin_dir: 'nsq-1.0.0-compat.linux-amd64.go1.8/bin' + control_plane_dir: '{{playbook_dir}}/resources/control-plane' - name: Install unzip become: yes @@ -41,7 +42,7 @@ - name: Copy Control API to executables dir become: yes synchronize: - src: "{{playbook_dir}}/resources/control-plane/snowplow-mini-control-plane-api" + src: "{{executables_dir}}/snowplow-mini-control-plane-api" dest: "{{executables_dir}}" - name: Copy VERSION file to /home/ubuntu/snowplow for Control API @@ -169,9 +170,14 @@ dest: "/opt/kibana" state: link -- name: Install Caddy +- name: Copy Caddy executable to executables dir become: yes - shell: "curl https://getcaddy.com | bash" + environment: + PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}" + GOPATH: "{{ go_path }}" + synchronize: + src: "{{executables_dir}}/caddy" + dest: "{{executables_dir}}" - name: Set owner of the main directory become: yes diff --git a/provisioning/roles/sp_mini_4_build_ui/tasks/main.yml b/provisioning/roles/sp_mini_5_build_ui/tasks/main.yml similarity index 100% rename from provisioning/roles/sp_mini_4_build_ui/tasks/main.yml rename to provisioning/roles/sp_mini_5_build_ui/tasks/main.yml diff --git a/provisioning/roles/sp_mini_5_copy_ui_folders/tasks/main.yml b/provisioning/roles/sp_mini_6_copy_ui_folders/tasks/main.yml similarity index 100% rename from provisioning/roles/sp_mini_5_copy_ui_folders/tasks/main.yml rename to provisioning/roles/sp_mini_6_copy_ui_folders/tasks/main.yml diff --git a/provisioning/roles/sp_mini_6_setup_init/tasks/main.yml b/provisioning/roles/sp_mini_7_setup_init/tasks/main.yml similarity index 100% rename from provisioning/roles/sp_mini_6_setup_init/tasks/main.yml rename to provisioning/roles/sp_mini_7_setup_init/tasks/main.yml diff --git a/provisioning/roles/sp_mini_7_configure/tasks/main.yml b/provisioning/roles/sp_mini_8_configure/tasks/main.yml similarity index 93% rename from provisioning/roles/sp_mini_7_configure/tasks/main.yml rename to provisioning/roles/sp_mini_8_configure/tasks/main.yml index b3a4dd28..eed8e5a9 100644 --- a/provisioning/roles/sp_mini_7_configure/tasks/main.yml +++ b/provisioning/roles/sp_mini_8_configure/tasks/main.yml @@ -3,8 +3,8 @@ - name: Starting Elasticsearch become: yes - service: - name: elasticsearch + service: + name: elasticsearch state: started register: ElasticsearchStarted @@ -19,17 +19,17 @@ - name: curl put bad-mapping.json shell: "curl -XPUT 'http://localhost:9200/bad' -d @{{es_dir}}/bad-mapping.json" -- name: Starting nsqd +- name: Starting nsqd become: yes - service: + service: name: nsqd_init state: started register: NsqdStarted - name: Starting nsqlookupd become: yes - service: - name: nsqlookupd_init + service: + name: nsqlookupd_init state: started register: NsqlookupdStarted - name: Wait the other services to start @@ -63,4 +63,3 @@ - name: Create new topic for BadEnrichedEvents shell: "curl -X POST http://127.0.0.1:4151/topic/create?topic=BadEnrichedEvents" - \ No newline at end of file diff --git a/provisioning/with_building_ui.yml b/provisioning/with_building_ui.yml deleted file mode 100644 index 77ed28e6..00000000 --- a/provisioning/with_building_ui.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: apply all scripts - hosts: all - - roles: - - nodejs - - typescript - - packer - - sp_mini_1_create_dirs_and_pipes - - sp_mini_2_install_postgresl - - sp_mini_3_setup_apps - - sp_mini_4_build_ui - - sp_mini_5_copy_ui_folders - - sp_mini_6_setup_init - - sp_mini_7_configure diff --git a/provisioning/with_building_ui_and_go_projects.yml b/provisioning/with_building_ui_and_go_projects.yml new file mode 100644 index 00000000..578955ec --- /dev/null +++ b/provisioning/with_building_ui_and_go_projects.yml @@ -0,0 +1,21 @@ +--- +# This playbook is used for provisioning of the vagrant machine. +# All tools for compilation of the Control Plane API, Caddy Server +# and UI is setup to the vagrant machine in here. +# Additionally, Snowplow Mini environment is made ready to use locally. +- name: apply all scripts + hosts: all + + roles: + - ansible-go + - nodejs + - typescript + - packer + - sp_mini_1_create_dirs_and_pipes + - sp_mini_2_install_postgresl + - sp_mini_3_build_go_projects + - sp_mini_4_setup_apps + - sp_mini_5_build_ui + - sp_mini_6_copy_ui_folders + - sp_mini_7_setup_init + - sp_mini_8_configure diff --git a/provisioning/without_building_ui.yml b/provisioning/without_building_ui.yml deleted file mode 100644 index 2dd03e35..00000000 --- a/provisioning/without_building_ui.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: apply all scripts - hosts: all - - roles: - - sp_mini_1_create_dirs_and_pipes - - sp_mini_2_install_postgresl - - sp_mini_3_setup_apps - - sp_mini_5_copy_ui_folders - - sp_mini_6_setup_init - - sp_mini_7_configure diff --git a/provisioning/without_building_ui_and_go_projects.yml b/provisioning/without_building_ui_and_go_projects.yml new file mode 100644 index 00000000..6a407211 --- /dev/null +++ b/provisioning/without_building_ui_and_go_projects.yml @@ -0,0 +1,14 @@ +--- +# This playbook is used for provisioning of the AMI +# UI and executables file are made ready in the vagrant environment +# They are only copied into the AMI here +- name: apply all scripts + hosts: all + + roles: + - sp_mini_1_create_dirs_and_pipes + - sp_mini_2_install_postgresl + - sp_mini_4_setup_apps + - sp_mini_6_copy_ui_folders + - sp_mini_7_setup_init + - sp_mini_8_configure diff --git a/vagrant/up.bash b/vagrant/up.bash index 66835d9f..8b956e0b 100755 --- a/vagrant/up.bash +++ b/vagrant/up.bash @@ -19,5 +19,5 @@ sudo pip install ansible vagrant_dir=/vagrant/vagrant cd $vagrant_dir/.. -ansible-playbook -i provisioning/inventory provisioning/with_building_ui.yml --connection=local --sudo +ansible-playbook -i provisioning/inventory provisioning/with_building_ui_and_go_projects.yml --connection=local --sudo