Merge branch 'v0.5.0'

snowplow · May 29, 2018 · ad6cecf · ad6cecf
2 parents ded5124 + 9df1f29
commit ad6cecf
Show file tree

Hide file tree

Showing 13 changed files with 117 additions and 103 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,11 @@
+Version 0.5.0 (2018-05-29)
+--------------------------
+Bump Packer to 1.2.3 (#163)
+Bump Stream Enrich to 0.16.1 (#161)
+Add log rotation for internal services to prevent out of disk issues (#151)
+Update base Ubuntu AMI being used (#147)
+Make build process of Caddy deterministic (#158)
+
 Version 0.4.0 (2017-12-21)
 --------------------------
 Add Control Plane API call to write domain name to Caddyfile for automatic TLS (#118)

diff --git a/Packerfile.json b/Packerfile.json
@@ -1,30 +1,31 @@
 {
-   "variables": {
-     "version": "0.4.0"
-   },
-
   "builders": [
     {
-      "type": "amazon-ebs",
-      "region": "us-east-1",
-      "source_ami": "ami-05dddc6f",
-      "instance_type": "t2.medium",
-      "ssh_username": "ubuntu",
+      "ami_description": "Snowplow Mini - The Snowplow Pipeline in a box",
+      "ami_groups": [
+        "all"
+      ],
       "ami_name": "snowplow-mini-{{user `version`}}-{{ timestamp }}-hvm-ebs-amd64",
-      "ami_groups": [ "all" ],
       "ami_regions": "us-east-2,us-west-1,us-west-2,ca-central-1,eu-west-1,eu-central-1,eu-west-2,ap-southeast-1,ap-southeast-2,ap-northeast-2,ap-northeast-1,ap-south-1,sa-east-1",
-      "ami_description": "Snowplow Mini - The Snowplow Pipeline in a box",
+      "instance_type": "t2.medium",
+      "region": "us-east-1",
+      "source_ami": "ami-58167327",
+      "ssh_username": "ubuntu",
       "tags": {
         "OS_Version": "Ubuntu-14.04",
         "Release": "{{user `version`}}"
-      }
+      },
+      "type": "amazon-ebs"
     }
   ],
-
+  "post-processors": [],
   "provisioners": [
     {
-      "type": "ansible",
-      "playbook_file": "provisioning/without_building_ui_and_go_projects.yml"
+      "playbook_file": "provisioning/without_building_ui_and_go_projects.yml",
+      "type": "ansible"
     }
-  ]
+  ],
+  "variables": {
+    "version": "0.5.0"
+  }
 }
diff --git a/README.md b/README.md
@@ -54,7 +54,7 @@ These events can then be viewed in Kibana at `http://< sp mini public ip>/kibana
 
 ## Copyright and license
 
-Snowplow Mini is copyright 2016-2017 Snowplow Analytics Ltd.
+Snowplow Mini is copyright 2016-2018 Snowplow Analytics Ltd.
 
 Licensed under the **[Apache License, Version 2.0][license]** (the "License");
 you may not use this software except in compliance with the License.
@@ -70,7 +70,7 @@ limitations under the License.
 [travis]: https://travis-ci.org/snowplow/snowplow-mini
 [travis-image]: https://travis-ci.org/snowplow/snowplow-mini.svg?branch=master
 
-[release-image]: http://img.shields.io/badge/release-0.4.0-blue.svg?style=flat
+[release-image]: http://img.shields.io/badge/release-0.5.0-blue.svg?style=flat
 [releases]: https://github.com/snowplow/snowplow-mini/releases
 
 [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.4.0
+0.5.0
diff --git a/provisioning/resources/configs/enrichments/ip_lookups.json b/provisioning/resources/configs/enrichments/ip_lookups.json
@@ -1,13 +1,13 @@
 {
-  "schema": "iglu:com.snowplowanalytics.snowplow/ip_lookups/jsonschema/1-0-0",
+  "schema": "iglu:com.snowplowanalytics.snowplow/ip_lookups/jsonschema/2-0-0",
   "data": {
     "name": "ip_lookups",
     "vendor": "com.snowplowanalytics.snowplow",
     "enabled": true,
     "parameters": {
       "geo": {
-        "database": "GeoLiteCity.dat",
-        "uri": "http://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/maxmind"
+        "database": "GeoLite2-City.mmdb",
+        "uri": "http://snowplow-hosted-assets.s3.amazonaws.com/third-party/maxmind"
       }
     }
   }

diff --git a/provisioning/resources/configs/enrichments/pii_enrichment_config.json b/provisioning/resources/configs/enrichments/pii_enrichment_config.json
@@ -0,0 +1,34 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow.enrichments/pii_enrichment_config/jsonschema/1-0-0",
+  "data": {
+    "vendor": "com.snowplowanalytics.snowplow.enrichments",
+    "name": "pii_enrichment_config",
+    "enabled": false,
+    "parameters": {
+      "pii": [
+        {
+          "pojo": {
+            "field": "user_id"
+          }
+        },
+        {
+          "pojo": {
+            "field": "user_fingerprint"
+          }
+        },
+        {
+          "json": {
+            "field": "unstruct_event",
+            "schemaCriterion": "iglu:com.mailchimp/subscribe/jsonschema/1-0-*",
+            "jsonPath": "$.data.['email', 'ip_opt']"
+          }
+        }
+      ],
+      "strategy": {
+        "pseudonymize": {
+          "hashFunction": "SHA-256"
+        }
+      }
+    }
+  }
+}
diff --git a/provisioning/resources/configs/snowplow-stream-enrich.hocon b/provisioning/resources/configs/snowplow-stream-enrich.hocon
@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2017 Snowplow Analytics Ltd. All rights reserved.
+# Copyright (c) 2013-2018 Snowplow Analytics Ltd. All rights reserved.
 #
 # This program is licensed to you under the Apache License Version 2.0, and
 # you may not use this file except in compliance with the Apache License
@@ -15,33 +15,13 @@
 # configuration options for Stream Enrich.
 
 enrich {
-  # Sources currently supported are:
-  # 'kinesis' for reading Thrift-serialized records from a Kinesis stream
-  # 'kafka' for reading Thrift-serialized records from a Kafka topic
-  # 'stdin' for reading Base64-encoded Thrift-serialized records from stdin
-  # 'nsq' for reading Base64-encoded Thrift-serialized records from NSQ
-  source = nsq
-
-  # Sinks currently supported are:
-  # 'kinesis' for writing enriched events to one Kinesis stream and invalid events to another.
-  # 'kafka' for writing enriched events to one Kafka topic and invalid events to another.
-  # 'stdouterr' for writing enriched events to stdout and invalid events to stderr.
-  #    Using "sbt assembly" and "java -jar" is recommended to disable sbt logging.
-  # 'nsq' for writing enriched events to one NSQ topic and invalid events to another.
-  sink = nsq
-
-  # AWS credentials
-  # If both are set to 'default', use the default AWS credentials provider chain.
-  # If both are set to 'iam', use AWS IAM Roles to provision credentials.
-  # If both are set to 'env', use environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
-  aws {
-    accessKey = ""
-    secretKey = ""
-  }
 
   streams {
-    # Stream/topic where the raw events to be enriched are located
-    in.raw = RawEvents
+
+    in {
+      # Stream/topic where the raw events to be enriched are located
+      raw = RawEvents
+    }
 
     out {
       # Stream/topic where the events that were successfully enriched will end up
@@ -53,73 +33,33 @@ enrich {
       # Possible partition keys are: event_id, event_fingerprint, domain_userid, network_userid,
       # user_ipaddress, domain_sessionid, user_fingerprint.
       # Refer to https://github.com/snowplow/snowplow/wiki/canonical-event-model to know what the
-      # possible parittion keys correspond to.
+      # possible partition keys correspond to.
       # Otherwise, the partition key will be a random UUID.
+      # Note: Nsq does not make use of partition key.
       partitionKey = ""
     }
 
-    kinesis {
-      # Region where the streams are located
-      region = ""
-
-      # Maximum number of records to get from Kinesis per call to GetRecords
-      maxRecords = 10000
-
-      # LATEST: most recent data.
-      # TRIM_HORIZON: oldest available data.
-      # Note: This only effects the first run of this application
-      # on a stream.
-      initialPosition = TRIM_HORIZON
-
-      # Minimum and maximum backoff periods, in milliseconds
-      backoffPolicy {
-        minBackoff = 50
-        maxBackoff = 500
-      }
-    }
-
-    # Kafka configuration
-    kafka {
-      brokers = ""
-
-      # Number of retries to perform before giving up on sending a record
-      retries = 0
-    }
+    sourceSink {
+      enabled = nsq
 
-    # config for nsq
-    nsq {
-      # Channel name for raw event source
+      # Channel name for nsq source
       rawChannel = StreamEnrichChannel
-
       # Host name for nsqd
       host = "127.0.0.1"
-
-      # TCP port for nsqd
+      # TCP port for nsqd, 4150 by default
       port = 4150
-
       # Host name for lookupd
       lookupHost = "127.0.0.1"
-
-      # Port for nsqlookupd
+      # HTTP port for nsqlookupd, 4161 by default
       lookupPort = 4161
     }
 
-    # After enrichment, events are accumulated in a buffer before being sent to Kinesis/Kafka.
-    # NOTE: Buffering is not supported by NSQ.
-    # The buffer is emptied whenever:
-    # - the number of stored records reaches recordLimit or
-    # - the combined size of the stored records reaches byteLimit or
-    # - the time in milliseconds since it was last emptied exceeds timeLimit when
-    #   a new event enters the buffer
     buffer {
       byteLimit = 4500000
-      recordLimit = 500 # Not supported by Kafka; will be ignored
+      recordLimit = 500
       timeLimit = 5000
     }
 
-    # Used for a DynamoDB table to maintain stream state.
-    # Used as the Kafka consumer group ID.
-    # You can set it automatically using: "SnowplowEnrich-$\\{enrich.streams.in.raw\\}"
     appName = ""
   }
 }
diff --git a/provisioning/resources/init/snowplow_stream_enrich b/provisioning/resources/init/snowplow_stream_enrich
@@ -10,7 +10,7 @@
 ### END INIT INFO
 
 dir="/home/ubuntu/snowplow/bin/"
-cmd="java -jar snowplow-stream-enrich-0.12.0.jar --config /home/ubuntu/snowplow/configs/snowplow-stream-enrich.hocon --resolver file:/home/ubuntu/snowplow/configs/iglu-resolver.json --enrichments file:/home/ubuntu/snowplow/configs/enrichments"
+cmd="java -jar snowplow-stream-enrich-nsq-0.16.1.jar --config /home/ubuntu/snowplow/configs/snowplow-stream-enrich.hocon --resolver file:/home/ubuntu/snowplow/configs/iglu-resolver.json --enrichments file:/home/ubuntu/snowplow/configs/enrichments"
 user="ubuntu"
 
 name="snowplow_stream_enrich"

diff --git a/provisioning/resources/ui/js/components/Overview.tsx b/provisioning/resources/ui/js/components/Overview.tsx
@@ -50,7 +50,7 @@ export class Overview extends React.Component<{}, {}> {
         <h3>The software stack installed: </h3>
         <ul>
           <li>Snowplow Stream Collector 0.11.0</li>
-          <li>Snowplow Stream Enrich 0.12.0</li>
+          <li>Snowplow Stream Enrich NSQ 0.16.1</li>
           <li>Snowplow Elasticsearch Sink 0.10.1</li>
           <li>Snowplow Iglu Server 0.2.0</li>
           <li>NSQ 1.0.0</li>

diff --git a/provisioning/roles/packer/defaults/main.yml b/provisioning/roles/packer/defaults/main.yml
@@ -1,2 +1,2 @@
 ---
-packer_version: "0.12.3"
+packer_version: "1.2.3"
diff --git a/provisioning/roles/sp_mini_1_create_dirs/tasks/main.yml b/provisioning/roles/sp_mini_1_create_dirs/tasks/main.yml
@@ -10,6 +10,24 @@
   shell: 'adduser ubuntu --disabled-password --gecos "" ;
           passwd -d ubuntu'
 
+- name: Insert logrotate configuration for Snowplow Services
+  become: yes
+  copy:
+    dest: "/etc/logrotate.d/snowplow-apps"
+    content: |
+      /var/log/snowplow*.log /var/log/snowplow*.err /var/log/nsq*.log  /var/log/nsq*.err /var/log/iglu_server*.log /var/log/iglu_server*.err {
+          hourly
+          rotate 3
+          missingok
+          notifempty
+          copytruncate
+      }
+    mode: 0644
+
+- name: Change logrotate cron to hourly
+  become: yes
+  shell: 'mv /etc/cron.daily/logrotate /etc/cron.hourly && service cron restart'
+
 - name: creating directories
   file: path={{item}} state=directory
   with_items:

diff --git a/provisioning/roles/sp_mini_3_build_go_projects/tasks/main.yml b/provisioning/roles/sp_mini_3_build_go_projects/tasks/main.yml
@@ -40,11 +40,24 @@
     src: "{{control_plane_dir}}/snowplow-mini-control-plane-api"
     dest: "{{executables_dir}}"
 
-- name: Get Caddy source code
+- name: Create caddy source dir
+  become: yes
+  environment:
+    GOPATH: "{{ go_path }}"
+  file:
+    path: $GOPATH/src/github.com/mholt
+    state: directory
+
+- git:
+    repo: 'https://github.com/mholt/caddy.git'
+    dest: /usr/local/src/caddy
+    version: v0.10.10
+
+- name: Locate caddy
+  become: yes
   environment:
-    PATH: "{{ lookup('env','PATH') }}:{{ go_bin }}"
     GOPATH: "{{ go_path }}"
-  shell: "go get -u github.com/mholt/caddy"
+  command: mv /usr/local/src/caddy $GOPATH/src/github.com/mholt/
 
 - name: Get Caddyserver builds
   environment:

diff --git a/provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml b/provisioning/roles/sp_mini_4_setup_apps/tasks/main.yml
@@ -4,7 +4,7 @@
 - name: Set variables
   set_fact:
     stream_collector_package: 'snowplow_scala_stream_collector_0.11.0.zip'
-    stream_enrich_package: 'snowplow_stream_enrich_0.12.0.zip'
+    stream_enrich_package: 'snowplow_stream_enrich_nsq_0.16.1.zip'
     es_loader_package: 'snowplow_elasticsearch_loader_http_0.10.1.zip'
     iglu_server_package: 'iglu_server_0.2.0.zip'
     kibana_v: '4.0.1'