Merge branch 'v0.3.0'

.travis.yml

@@ -1,21 +1,20 @@
+---
 sudo: required
 language: bash
 
 services:
   - postgresql
 
-## Mimics the Packerfile.json steps
+
+before_install:
+  - sudo apt-get update -qq
+
+install:
+  - sudo pip install setuptools
+  - sudo pip install ansible
+
 before_script:
-  - sudo adduser ubuntu --disabled-password --gecos ""
-  - sudo passwd -d ubuntu
-  - sudo ./scripts/1_setup_dirs_pipes.sh
-  - sudo cp -rf ./resources/elasticsearch /home/ubuntu/snowplow
-  - sudo ./scripts/3_setup_apps.sh
-  - sudo cp -rf ./resources/configs /home/ubuntu/snowplow
-  - sudo cp -rf ./scripts /home/ubuntu/snowplow
-  - sudo cp -rf ./resources/init /home/ubuntu/snowplow
-  - sudo ./scripts/5_setup_init.sh
-  - sudo ./scripts/6_configure.sh
+  - ansible-playbook -i provisioning/inventory provisioning/with_building_ui.yml --connection=local --sudo
 
 script:
   - ./integration/integration_test.sh

CHANGELOG

@@ -1,3 +1,17 @@
+Version 0.3.0 (2017-08-30)
+--------------------------
+Get username and password for basic authentication via user_data.sh (#107)
+Add domain name to Caddyfile for TLS via user_data.sh (#105)
+Add button in UI to restart all services (#59)
+Deploy Caddy so that each app is on an authenticated sub-path (#12)
+Authenticate Iglu schema registry access from Stream Enrich (#92)
+Update to Packer 0.10.0+ and convert shell scripts to ansible playbooks (#52)
+Add API call to Control Plane to clear schema cache without resetting the box (#82)
+Have Snowplow Mini run the basic enrichments as standard (#64)
+Enable third-party cookie set by collector (#93)
+Remove legacy iglu_server_upload.sh script (#58)
+Upgrade Snowplow services to R85 (#81)
+
 Version 0.2.2 (2017-03-21)
 --------------------------
 Update Packerfile to deploy to all available regions (#75)

Packerfile.json

@@ -1,92 +1,30 @@
 {
+   "variables": {
+     "version": "0.3.0"
+   },
+
   "builders": [
     {
       "type": "amazon-ebs",
       "region": "us-east-1",
       "source_ami": "ami-05dddc6f",
       "instance_type": "t2.small",
       "ssh_username": "ubuntu",
-      "ami_name": "snowplow-mini-0.2.2-{{ timestamp }}-hvm-ebs-amd64",
+      "ami_name": "snowplow-mini-{{user `version`}}-{{ timestamp }}-hvm-ebs-amd64",
       "ami_groups": [ "all" ],
       "ami_regions": "us-east-2,us-west-1,us-west-2,ca-central-1,eu-west-1,eu-central-1,eu-west-2,ap-southeast-1,ap-southeast-2,ap-northeast-2,ap-northeast-1,ap-south-1,sa-east-1",
       "ami_description": "Snowplow Mini - The Snowplow Pipeline in a box",
       "tags": {
-        "OS_Version": "Ubuntu-12.04",
-        "Release": "0.2.2"
+        "OS_Version": "Ubuntu-14.04",
+        "Release": "{{user `version`}}"
       }
     }
   ],
 
   "provisioners": [
     {
-      "type": "shell",
-      "script": "scripts/1_setup_dirs_pipes.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "file",
-      "source": "resources/elasticsearch",
-      "destination": "/home/ubuntu/snowplow"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/2_setup_postgres.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/3_setup_apps.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "file",
-      "source": "resources/configs",
-      "destination": "/home/ubuntu/snowplow"
-    },
-    {
-      "type": "file",
-      "source": "scripts",
-      "destination": "/home/ubuntu/snowplow"
-    },
-    {
-      "type": "file",
-      "source": "resources/init",
-      "destination": "/home/ubuntu/snowplow"
-    },
-    {
-      "type": "file",
-      "source": "ui/index.html",
-      "destination": "/home/ubuntu/snowplow/ui/index.html"
-    },
-    {
-      "type": "file",
-      "source": "ui/dist/snowplow-mini.js",
-      "destination": "/home/ubuntu/snowplow/ui/dist/snowplow-mini.js"
-    },
-    {
-      "type": "file",
-      "source": "ui/assets",
-      "destination": "/home/ubuntu/snowplow/ui"
-    },
-    {
-      "type": "file",
-      "source": "ui/node_modules",
-      "destination": "/home/ubuntu/snowplow/ui"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/4_setup_ui.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/5_setup_init.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/6_configure.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
+      "type": "ansible",
+      "playbook_file": "provisioning/without_building_ui.yml"
     }
   ]
 }

README.md

@@ -23,7 +23,7 @@ An easily-deployable, single instance version of Snowplow that serves three use
 Snowplow-Mini runs several distinct applications on the same box which are all linked by named pipes.  In a production deployment each instance could be an Autoscaling Group and each named pipe would be a distinct Kinesis Stream.
 
 * Scala Stream Collector:
-  - Starts server listening on port `8080` which events can be sent to.
+  - Starts server listening on `http://< sp mini public ip>/` which events can be sent to.
   - Sends "good" events to the `raw-events-pipe`
   - Sends "bad" events to the `bad-1-pipe`
 * Stream Enrich
@@ -38,9 +38,9 @@ Snowplow-Mini runs several distinct applications on the same box which are all l
   - Reads events in from the `bad-1-pipe`
   - Sends the events to the "bad" index of the cluster
 
-These events can then be viewed at port `5601` in Kibana.
+These events can then be viewed from `http://< sp mini public ip>/kibana`.
 
-![](https://raw.githubusercontent.com/snowplow/snowplow-mini/master/resources/topology/snowplow-mini-topology.jpg)
+![](https://raw.githubusercontent.com/snowplow/snowplow-mini/master/utils/topology/snowplow-mini-topology.jpg)
 
 ## Roadmap
 
@@ -53,7 +53,7 @@ These events can then be viewed at port `5601` in Kibana.
 
 ## Copyright and license
 
-Snowplow Mini is copyright 2016 Snowplow Analytics Ltd.
+Snowplow Mini is copyright 2016-2017 Snowplow Analytics Ltd.
 
 Licensed under the **[Apache License, Version 2.0][license]** (the "License");
 you may not use this software except in compliance with the License.
@@ -69,7 +69,7 @@ limitations under the License.
 [travis]: https://travis-ci.org/snowplow/snowplow-mini
 [travis-image]: https://travis-ci.org/snowplow/snowplow-mini.svg?branch=master
 
-[release-image]: http://img.shields.io/badge/release-0.2.2-blue.svg?style=flat
+[release-image]: http://img.shields.io/badge/release-0.3.0-blue.svg?style=flat
 [releases]: https://github.com/snowplow/snowplow-mini/releases
 
 [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat

VERSION

@@ -1 +1 @@
-0.2.2
+0.3.0

Vagrantfile

@@ -8,11 +8,13 @@ Vagrant.configure("2") do |config|
   config.vm.network :private_network, ip: '192.168.50.50' # Uncomment to use NFS
   config.vm.synced_folder '.', '/vagrant', nfs: true # Uncomment to use NFS
 
-  config.vm.network "forwarded_port", guest: 80, host: 3000
+  config.vm.network "forwarded_port", guest: 2000, host: 2000
+  config.vm.network "forwarded_port", guest: 3000, host: 3000
   config.vm.network "forwarded_port", guest: 8080, host: 8080
   config.vm.network "forwarded_port", guest: 9200, host: 9200
   config.vm.network "forwarded_port", guest: 5601, host: 5601
   config.vm.network "forwarded_port", guest: 8081, host: 8081
+  config.vm.network "forwarded_port", guest: 10000, host: 10000
 
   config.vm.provider :virtualbox do |vb|
     vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s

integration/integration_test.sh

@@ -2,10 +2,10 @@
 
 sudo service elasticsearch start
 sudo service iglu_server_0.2.0 start
-sudo service snowplow_stream_collector_0.6.0 start
-sudo service snowplow_stream_enrich_0.7.0 start
-sudo service snowplow_elasticsearch_sink_good_0.5.0 start
-sudo service snowplow_elasticsearch_sink_bad_0.5.0 start
+sudo service snowplow_stream_collector_0.9.0 start
+sudo service snowplow_stream_enrich_0.10.0 start
+sudo service snowplow_elasticsearch_sink_good_0.8.0 start
+sudo service snowplow_elasticsearch_sink_bad_0.8.0 start
 sudo service kibana4_init start
 sudo service nginx start
 sleep 15
@@ -27,8 +27,32 @@ echo "Event Counts:"
 echo " - Good: ${good_count}"
 echo " - Bad: ${bad_count}"
 
+stream_enrich_pid_file=/var/run/snowplow_stream_enrich_0.10.0.pid
+stream_collector_pid_file=/var/run/snowplow_stream_collector_0.9.0.pid
+sink_bad_pid_file=/var/run/snowplow_elasticsearch_sink_bad_0.8.0-2x.pid
+sink_good_pid_file=/var/run/snowplow_elasticsearch_sink_good_0.8.0-2x.pid
+
+
+stream_enrich_pid_old="$(cat "${stream_enrich_pid_file}")"
+stream_collector_pid_old="$(cat "${stream_collector_pid_file}")"
+sink_bad_pid_old="$(cat "${sink_bad_pid_file}")"
+sink_good_pid_old="$(cat "${sink_good_pid_file}")"
+
+req_result=$(curl --silent -XPUT 'http://localhost:10000/restart-services')
+
+stream_enrich_pid_new="$(cat "${stream_enrich_pid_file}")"
+stream_collector_pid_new="$(cat "${stream_collector_pid_file}")"
+sink_bad_pid_new="$(cat "${sink_bad_pid_file}")"
+sink_good_pid_new="$(cat "${sink_good_pid_file}")"
+
 # Bad Count is 11 due to bad logging
-if [[ "${good_count}" -eq "10" ]] && [[ "${bad_count}" -eq "11" ]]; then
+if [[ "${good_count}" -eq "10" ]] && [[ "${bad_count}" -eq "11" ]] && 
+   [[ "${req_result}" == "OK" ]] &&
+   [[ "${stream_enrich_pid_old}" -ne "${stream_enrich_pid_new}" ]] &&
+   [[ "${stream_collector_pid_old}" -ne "${stream_collector_pid_new}" ]] &&
+   [[ "${sink_bad_pid_old}" -ne "${sink_bad_pid_new}" ]] &&
+   [[ "${sink_good_pid_old}" -ne "${sink_good_pid_new}" ]]; then
+
   exit 0
 else
   exit 1

vagrant/ansible.hosts → provisioning/inventory

@@ -1,2 +1,4 @@
 [vagrant]
 127.0.0.1:2222
+[localhost]
+127.0.0.1:22

provisioning/resources/configs/Caddyfile

@@ -0,0 +1,36 @@
+localhost:2000 {
+  basicauth "USERNAME_PLACEHOLDER" PASSWORD_PLACEHOLDER {
+    /home
+    /kibana
+    /elasticsearch
+    /control-plane
+    /_plugin
+  }
+  redir /home /home/
+  redir /kibana /kibana/
+  redir /iglu-server /iglu-server/
+
+  proxy / localhost:8080
+
+  proxy /home localhost:3000 {
+    without /home
+  }
+
+  proxy /kibana localhost:5601 {
+    without /kibana
+  }
+
+  proxy /iglu-server localhost:8081 {
+    without /iglu-server
+  }
+  proxy /api localhost:8081
+
+  proxy /elasticsearch localhost:9200 {
+    without /elasticsearch
+  }
+  proxy /_plugin localhost:9200
+
+  proxy /control-plane localhost:10000 {
+    without /control-plane
+  }
+}

provisioning/resources/configs/enrichments/campaign_attribution.json

@@ -0,0 +1,18 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/campaign_attribution/jsonschema/1-0-1",
+  "data": {
+    "name": "campaign_attribution",
+    "vendor": "com.snowplowanalytics.snowplow",
+    "enabled": true,
+    "parameters": {
+      "mapping": "static",
+      "fields": {
+        "mktMedium": ["utm_medium"],
+        "mktSource": ["utm_source"],
+        "mktTerm": ["utm_term"],
+        "mktContent": ["utm_content"],
+        "mktCampaign": ["utm_campaign"]
+      }
+    }
+  }
+}

provisioning/resources/configs/enrichments/event_fingerprint_config.json

@@ -0,0 +1,12 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/event_fingerprint_config/jsonschema/1-0-0",
+  "data": {
+    "name": "event_fingerprint_config",
+    "vendor": "com.snowplowanalytics.snowplow",
+    "enabled": true,
+    "parameters": {
+      "excludeParameters": ["eid", "stm"],
+      "hashAlgorithm": "MD5"
+    }
+  }
+}

provisioning/resources/configs/enrichments/ip_lookups.json

@@ -0,0 +1,14 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/ip_lookups/jsonschema/1-0-0",
+  "data": {
+    "name": "ip_lookups",
+    "vendor": "com.snowplowanalytics.snowplow",
+    "enabled": true,
+    "parameters": {
+      "geo": {
+        "database": "GeoLiteCity.dat",
+        "uri": "http://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/maxmind"
+      }
+    }
+  }
+}

provisioning/resources/configs/enrichments/referer_parser.json

@@ -0,0 +1,12 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/referer_parser/jsonschema/1-0-0",
+  "data": {
+    "name": "referer_parser",
+    "vendor": "com.snowplowanalytics.snowplow",
+    "enabled": true,
+    "parameters": {
+      "internalDomains": [
+      ]
+    }
+  }
+}

provisioning/resources/configs/enrichments/ua_parser_config.json

@@ -0,0 +1,9 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/ua_parser_config/jsonschema/1-0-0",
+  "data": {
+    "vendor": "com.snowplowanalytics.snowplow",
+    "name": "ua_parser_config",
+    "enabled": true,
+    "parameters": {}
+  }
+}

provisioning/resources/configs/enrichments/user_agent_utils_config.json

@@ -0,0 +1,9 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/user_agent_utils_config/jsonschema/1-0-0",
+  "data": {
+    "vendor": "com.snowplowanalytics.snowplow",
+    "name": "user_agent_utils_config",
+    "enabled": true,
+    "parameters": {}
+  }
+}