Merge branch 'v0.6.0'

.travis.yml

@@ -1,5 +1,6 @@
 ---
 sudo: required
+dist: trusty
 language: go
 
 go:
@@ -20,3 +21,8 @@ before_script:
 
 script:
   - ./integration/integration_test.sh
+
+env:
+  global:
+  - secure: CP4KymCt5vhWEOGBYw+mBKxfd34PoGOybo5QdiGJ0tj5l6Tn8utPn/MSOMZLdlxdqUnPstnhsBNpGybnBSQIKC8TSZqMqgebw59Qq75bk2gNQ5NSv0iOjija9oKvBLaKgyXwwzL2o4JWnFTuFsHGyGLjSHoKy3sptfKIih3vwa0ey4SevElUtqpwT32AjCYW0L+iMP+kWRUEna6vdR4BTLrl8/UXAQRccQ8gmcpecfhqJL6sYgmK4oqgbhyMn8dvWVMIFsUjJ76rvJ5KvcGceNqJ07Yb8qaLKx/OBDn7I8GbRHrnxgLXF/7gU40jG+nOeYUTfdflMDmJTplk+Sgi2WxumpDdLuEJPge6kZMQGcZ5rHPS3dAXJ8APVn/SirHSVpjApdYkkoL7DXBFYPvIfpgKfqj2ofwKLoY4OZwOPU6+XCLQJkxxIszTPbyXaBofeB+CpLTIcU0nGn6NXAGT6ONDSlIKVNXotlhz9robtOeJ723tFgmUoTSvHHwXAal/3LBVjx2qnTg7pW3/9wcMaI+uaETdIcv3DBRpr8SfcObfYCMU7HOck3gfIIbn6YB8MbY3fceeZ8UPw9R9UyIvTVISOJW+SCCJoybEqxGnNhNXRTbn/anCS9dN1zesVCPZMl70bJyQNBkTW+sTusOASrfkYKvDiU/qkzUSx/Qlz0s=
+  - secure: gvyo8HuSuIEiSIZH7A2B0Nz6HJx04lTIV4QYx+bpEcCWailbgqSjDBRVwb6IdsQBsZsy79924Go3j4s0qtAOXGOLhHkmEE+bmLzT7Ejey9ENkQm5iAkKb2clYTKiDa94tB/Rqk+yW669HILttB/mcOAj+LFhBKb3jJJS0n0hDfYKsbleWSAVMyPw+V7SEQBc+9VTmJcq0YYYHS1Ie4g92QXxA33nLyGcOhCY7SS+gAqbMuQDhGa+Y0jh8Tz4YMVCd0tFLjsIB818DYE2YgRZerHUC8G7+kyIylV3/Nr1tEb/i6F8Ii6ET7GnzCnMJ0pj0vp/hsIsZBJeZUyUJEwtBx9798OeBWPrpuJtesTuNVO3DJmahrdIwe+fWOt5wFJFgZwvpmu12px+R8yTYkCl2oWSf+CNJi5fhb68m3D3p5JAqRNvvuKQE+zdX1pMCTmLWIjl8SOFN5ptP4kpDStziBsKyH9YQ8KUvR24kcyWO4u2pkRNgkI+SZTl83U/zPPKh/R1jGjm+TC9rlGCyj+OJIfl2hiLJKgBQtdSkJK0dz5ZQx9ZcGDPW88oeLUqJGRHQsPUVZB/cutrIwg2WIG4nBdCHhGG7t1u/hCr27aOTlJfy/3uA9cdkZCoVccwiwMcydtBmEgcYxjGbE378fXtBSpqPvuRs5r2dEpAbmcOmOU=

CHANGELOG

@@ -1,3 +1,19 @@
+Version 0.6.0 (2018-07-02)
+--------------------------
+Bump Stream Enrich to 0.18.0 (#174)
+Publish mini to bigger instance types (#173)
+Add publication of image for Google Compute Engine (#36)
+Add support for external Postgres for Iglu Server (#154)
+Assemble the apps using Docker Compose (#23)
+Extend copyright to 2018 (#164)
+Add gitter badge (#157)
+Install NTP (#167)
+Add ami-factory credentials to .travis.yml (#143)
+Fix typo in README (#160)
+Expose nsqadmin UI (#144)
+Update example user_data.sh (#148)
+Ensure Stream Enrich resolver uses cacheTtl of one (#137)
+
 Version 0.5.0 (2018-05-29)
 --------------------------
 Bump Packer to 1.2.3 (#163)

Packerfile.json

@@ -5,9 +5,9 @@
       "ami_groups": [
         "all"
       ],
-      "ami_name": "snowplow-mini-{{user `version`}}-{{ timestamp }}-hvm-ebs-amd64",
+      "ami_name": "snowplow-mini-{{user `aws_version`}}-{{user `sp_mini_size`}}-{{ timestamp }}-hvm-ebs-amd64",
       "ami_regions": "us-east-2,us-west-1,us-west-2,ca-central-1,eu-west-1,eu-central-1,eu-west-2,ap-southeast-1,ap-southeast-2,ap-northeast-2,ap-northeast-1,ap-south-1,sa-east-1",
-      "instance_type": "t2.medium",
+      "instance_type": "{{user `aws_instance_type`}}",
       "region": "us-east-1",
       "source_ami": "ami-58167327",
       "ssh_username": "ubuntu",
@@ -16,6 +16,17 @@
         "Release": "{{user `version`}}"
       },
       "type": "amazon-ebs"
+    },
+    {
+      "type": "googlecompute",
+      "image_description": "Snowplow Mini - The Snowplow Pipeline in a box",
+      "image_name": "snowplow-mini-{{user `gcp_version`}}-{{user `sp_mini_size`}}-{{timestamp}}",
+      "machine_type": "{{user `gcp_machine_type`}}",
+      "account_file": "account.json",
+      "project_id": "snowplow-images",
+      "source_image_family": "ubuntu-1404-lts",
+      "ssh_username": "ubuntu",
+      "zone": "us-central1-a"
     }
   ],
   "post-processors": [],
@@ -26,6 +37,10 @@
     }
   ],
   "variables": {
-    "version": "0.5.0"
+    "aws_version": "0.6.0",
+    "gcp_version": "0-6-0",
+    "gcp_machine_type": "{{env `GCP_MACHINE_TYPE`}}",
+    "aws_instance_type": "{{env `AWS_INSTANCE_TYPE`}}",
+    "sp_mini_size": "{{env `SP_MINI_SIZE`}}"
   }
 }

README.md

@@ -1,10 +1,13 @@
 # Snowplow-Mini
 
-[![Build Status][travis-image]][travis] [![Release][release-image]][releases] [![License][license-image]][license]
+[![Join the chat at https://gitter.im/snowplow/snowplow-mini](https://badges.gitter.im/snowplow/snowplow-mini.svg)](https://gitter.im/snowplow/snowplow-mini?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Build Status][travis-image]][travis]
+[![Release][release-image]][releases]
+[![License][license-image]][license]
 
 An easily-deployable, single instance version of Snowplow that serves three use cases:
 
-1. Gives a Snowplow consumer (e.g. an analyst / data team / marketing team) a way to quickly understand what Snowplow "does" i.e. what you put it at one end and take out of the other
+1. Gives a Snowplow consumer (e.g. an analyst / data team / marketing team) a way to quickly understand what Snowplow "does" i.e. what you put in at one end and take out of the other
 2. Gives developers new to Snowplow an easy way to start with Snowplow and understand how the different pieces fit together
 3. Gives people running Snowplow a quick way to debug tracker updates (because they can)
 
@@ -70,7 +73,7 @@ limitations under the License.
 [travis]: https://travis-ci.org/snowplow/snowplow-mini
 [travis-image]: https://travis-ci.org/snowplow/snowplow-mini.svg?branch=master
 
-[release-image]: http://img.shields.io/badge/release-0.5.0-blue.svg?style=flat
+[release-image]: http://img.shields.io/badge/release-0.6.0-blue.svg?style=flat
 [releases]: https://github.com/snowplow/snowplow-mini/releases
 
 [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat

VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

Vagrantfile

@@ -10,6 +10,7 @@ Vagrant.configure("2") do |config|
 
   config.vm.network "forwarded_port", guest: 80, host: 2000
   config.vm.network "forwarded_port", guest: 3000, host: 3000
+  config.vm.network "forwarded_port", guest: 4171, host: 4171
   config.vm.network "forwarded_port", guest: 8080, host: 8080
   config.vm.network "forwarded_port", guest: 9200, host: 9200
   config.vm.network "forwarded_port", guest: 5601, host: 5601
@@ -20,8 +21,8 @@ Vagrant.configure("2") do |config|
     vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
     vb.customize [ "guestproperty", "set", :id, "--timesync-threshold", 10000 ]
-    vb.memory = 4096
-    vb.cpus = 1
+    vb.memory = 8192
+    vb.cpus = 2
   end
 
   config.vm.provision :shell do |sh|
@@ -30,7 +31,18 @@ Vagrant.configure("2") do |config|
 
   # Requires Vagrant 1.7.0+
   config.push.define "publish", strategy: "local-exec" do |push|
-    push.script = "vagrant/push.bash"
+    push.inline = <<-SCRIPT
+      # comment/uncomment below to enable/disable pushing to AWS/GCP
+      # or configure instance type large/xlarge/xxlarge
+
+      # vagrant/push.bash aws large
+      # vagrant/push.bash aws xlarge
+      # vagrant/push.bash aws xxlarge
+
+      # vagrant/push.bash gcp large
+      # vagrant/push.bash gcp xlarge
+      # vagrant/push.bash gcp xxlarge
+    SCRIPT
   end
 
 end

integration/integration_test.sh

@@ -1,22 +1,13 @@
 #!/bin/bash
 
-sudo service elasticsearch start
-sudo service iglu_server_0.2.0 start
-sudo service snowplow_stream_collector start
-sudo service snowplow_stream_enrich start
-sudo service snowplow_elasticsearch_loader_good start
-sudo service snowplow_elasticsearch_loader_bad start
-sudo service kibana4_init start
-sleep 15
-
 # Send good and bad events
 COUNTER=0
 while [  $COUNTER -lt 10 ]; do
   curl http://localhost:8080/i?e=pv
   curl http://localhost:8080/i
   let COUNTER=COUNTER+1
 done
-sleep 60
+sleep 90
 
 # Assertions
 good_count="$(curl --silent -XGET 'http://localhost:9200/good/good/_count' | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["count"]')"

provisioning/resources/configs/Caddyfile

@@ -4,12 +4,14 @@
     /home
     /kibana
     /elasticsearch
+    /nsqadmin
     /control-plane
-    /_plugin
   }
   redir /home /home/
-  redir /kibana /kibana/
   redir /iglu-server /iglu-server/
+  redir /kibana /kibana/
+  redir /nsqadmin /nsqadmin/
+  redir /elasticsearch /elasticsearch/
 
   proxy / localhost:8080
 
@@ -20,16 +22,29 @@
   proxy /kibana localhost:5601 {
     without /kibana
   }
+  proxy /app/kibana localhost:5601
+  proxy /app/timelion localhost:5601
+  proxy /bundles localhost:5601
+  proxy /plugins localhost:5601
+  proxy /ui localhost:5601
+  proxy /api localhost:5601
 
   proxy /iglu-server localhost:8081 {
     without /iglu-server
   }
-  proxy /api localhost:8081
+  proxy /api-docs localhost:8081
+
+  proxy /nsqadmin localhost:4171 {
+    without /nsqadmin
+  }
+  proxy /static localhost:4171
+  proxy /api/counter localhost:4171
+  proxy /api/nodes localhost:4171
+  proxy /api/topics localhost:4171
 
   proxy /elasticsearch localhost:9200 {
     without /elasticsearch
   }
-  proxy /_plugin localhost:9200
 
   proxy /control-plane localhost:10000 {
     without /control-plane

provisioning/resources/configs/control-plane-api.toml

@@ -12,17 +12,18 @@ config = "/home/ubuntu/snowplow/configs" #directory which all the configs are in
 [config_file_names]
 caddy = "Caddyfile"
 iglu_resolver = "iglu-resolver.json"
+iglu_server = "iglu-server.conf"
 
 [init_scripts]
-stream_collector = "snowplow_stream_collector"
-stream_enrich = "snowplow_stream_enrich"
-es_loader_good = "snowplow_elasticsearch_loader_good"
-es_loader_bad = "snowplow_elasticsearch_loader_bad"
-iglu = "iglu_server_0.2.0"
+stream_collector = "scala-stream-collector"
+stream_enrich = "stream-enrich"
+es_loader_good = "elasticsearch-loader-good"
+es_loader_bad = "elasticsearch-loader-bad"
+iglu = "iglu-server"
 caddy = "caddy_init"
 
 [PSQL]
 user = "snowplow"
 password = "snowplow"
 database = "iglu"
-adddress = "127.0.0.1:5432"
+address = "127.0.0.1:5433"

provisioning/resources/configs/enrichments/pii_enrichment_config.json

@@ -1,32 +1,24 @@
 {
-  "schema": "iglu:com.snowplowanalytics.snowplow.enrichments/pii_enrichment_config/jsonschema/1-0-0",
+  "schema": "iglu:com.snowplowanalytics.snowplow.enrichments/pii_enrichment_config/jsonschema/2-0-0",
   "data": {
     "vendor": "com.snowplowanalytics.snowplow.enrichments",
     "name": "pii_enrichment_config",
-    "enabled": false,
+    "emitEvent": true,
+    "enabled": true,
     "parameters": {
       "pii": [
-        {
-          "pojo": {
-            "field": "user_id"
-          }
-        },
-        {
-          "pojo": {
-            "field": "user_fingerprint"
-          }
-        },
         {
           "json": {
             "field": "unstruct_event",
-            "schemaCriterion": "iglu:com.mailchimp/subscribe/jsonschema/1-0-*",
-            "jsonPath": "$.data.['email', 'ip_opt']"
+            "schemaCriterion": "iglu:com.snowplowanalytics.snowplow.enrichments/pii_enrichment_config/jsonschema/2-*-*",
+            "jsonPath": "$.data.['vendor']"
           }
         }
       ],
       "strategy": {
         "pseudonymize": {
-          "hashFunction": "SHA-256"
+          "hashFunction": "SHA-1",
+          "salt": "pepper123"
         }
       }
     }

provisioning/resources/configs/iglu-resolver.json

@@ -1,7 +1,8 @@
 {
-  "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-1",
+  "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-2",
   "data": {
     "cacheSize": 500,
+    "cacheTtl": 1,
     "repositories": [
       {
         "name": "Iglu Central",
@@ -23,7 +24,7 @@
         ],
         "connection": {
           "http": {
-            "uri": "http://localhost:8081/api",
+            "uri": "http://iglu-server:8081/api",
             "apikey": "PLACEHOLDER"
           }
         }

provisioning/resources/configs/iglu-server.conf

@@ -1,4 +1,4 @@
-# Copyright (c) 2014 Snowplow Analytics Ltd. All rights reserved.
+# Copyright (c) 2014-2018 Snowplow Analytics Ltd. All rights reserved.
 #
 # This program is licensed to you under the Apache License Version 2.0, and
 # you may not use this file except in compliance with the Apache License
@@ -15,15 +15,19 @@
 # the Iglu repository server.
 
 # 'repo-server' contains configuration options for the repo-server.
+# interface on which the server will be running
+# baseURL is address of deployment, "<ip>:<port>/<deployment_path>" address used for baseURL of Swagger UI
+# port on which the server will be running
 repo-server {
   interface = "0.0.0.0"
+  baseURL = "0.0.0.0/iglu-server"
   port = 8081
 }
 
 # 'postgres' contains configuration options for the postgre instance the server
 # is using
 postgres {
-  host = "localhost"
+  host = "postgres"
   port = 5432
   dbname = "iglu"
   username = "snowplow"
@@ -32,14 +36,16 @@ postgres {
 }
 
 akka {
+  loggers = ["akka.event.slf4j.Slf4jLogger"]
   loglevel = INFO
   log-dead-letters = off
+  stdout-loglevel = "DEBUG"
+  logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
 }
 
-# spray-can is the HTTP server the Iglu repository server is built on.
-spray.can {
+akka.http {
   server {
-    request-timeout = 10s
+    request-timeout = 10 seconds
     remote-address-header = on
     parsing.uri-parsing-mode = relaxed
   }