Bring up an Iglu server inside Snowplow Mini (closes #21)

resources/configs/iglu-resolver.json

@@ -16,14 +16,14 @@
         }
       },
       {
-        "name": "Iglu snowplow mini usre",
+        "name": "Iglu Server",
         "priority": 0,
         "vendorPrefixes": [
-          "com.snowplow-mini-user"
+          "com.snowplowanalytics"
         ],
         "connection": {
           "http": {
-            "uri": "https://s3.amazonaws.com/bucket-name-here"
+            "uri": "http://localhost:8081/api"
           }
         }
       }

resources/configs/iglu-server.conf

@@ -0,0 +1,46 @@
+# Copyright (c) 2014 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0, and
+# you may not use this file except in compliance with the Apache License
+# Version 2.0.  You may obtain a copy of the Apache License Version 2.0 at
+# http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the Apache License Version 2.0 is distributed on an "AS
+# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the Apache License Version 2.0 for the specific language
+# governing permissions and limitations there under.
+
+# This file (application.conf) contains all necessary configuration options for
+# the Iglu repository server.
+
+# 'repo-server' contains configuration options for the repo-server.
+repo-server {
+  interface = "0.0.0.0"
+  port = 8081
+}
+
+# 'postgres' contains configuration options for the postgre instance the server
+# is using
+postgres {
+  host = "localhost"
+  port = 5432
+  dbname = "iglu"
+  username = "snowplow"
+  password = "snowplow"
+  driver = "org.postgresql.Driver"
+}
+
+akka {
+  loglevel = INFO
+  log-dead-letters = off
+}
+
+# spray-can is the HTTP server the Iglu repository server is built on.
+spray.can {
+  server {
+    request-timeout = 10s
+    remote-address-header = on
+    parsing.uri-parsing-mode = relaxed
+  }
+}

resources/init/iglu_server_0.2.0

@@ -0,0 +1,98 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start daemon at boot time
+# Description:       Enable service provided by daemon.
+### END INIT INFO
+
+dir="/home/ubuntu/snowplow/bin/"
+cmd="java -Dconfig.file=/home/ubuntu/snowplow/configs/iglu-server.conf -jar iglu-server-0.2.0.jar com.snowplowanalytics.iglu.server.Boot"
+user="ubuntu"
+
+name=`basename $0`
+pid_file="/var/run/$name.pid"
+stdout_log="/var/log/$name.log"
+stderr_log="/var/log/$name.err"
+
+get_pid() {
+    cat "$pid_file"
+}
+
+is_running() {
+    [ -f "$pid_file" ] && ps `get_pid` > /dev/null 2>&1
+}
+
+case "$1" in
+    start)
+    if is_running; then
+        echo "Already started"
+    else
+        echo "Starting $name"
+        cd "$dir"
+        if [ -z "$user" ]; then
+            sudo $cmd >> "$stdout_log" 2>> "$stderr_log" &
+        else
+            sudo -u "$user" $cmd >> "$stdout_log" 2>> "$stderr_log" &
+        fi
+        echo $! > "$pid_file"
+        if ! is_running; then
+            echo "Unable to start, see $stdout_log and $stderr_log"
+            exit 1
+        fi
+    fi
+    ;;
+    stop)
+    if is_running; then
+        echo -n "Stopping $name.."
+        kill `get_pid`
+        for i in {1..10}
+        do
+            if ! is_running; then
+                break
+            fi
+
+            echo -n "."
+            sleep 1
+        done
+        echo
+
+        if is_running; then
+            echo "Not stopped; may still be shutting down or shutdown may have failed"
+            exit 1
+        else
+            echo "Stopped"
+            if [ -f "$pid_file" ]; then
+                rm "$pid_file"
+            fi
+        fi
+    else
+        echo "Not running"
+    fi
+    ;;
+    restart)
+    $0 stop
+    if is_running; then
+        echo "Unable to stop, will not attempt to start"
+        exit 1
+    fi
+    $0 start
+    ;;
+    status)
+    if is_running; then
+        echo "Running"
+    else
+        echo "Stopped"
+        exit 1
+    fi
+    ;;
+    *)
+    echo "Usage: $0 {start|stop|restart|status}"
+    exit 1
+    ;;
+esac
+
+exit 0

resources/init/snowplow_stream_enrich_0.7.0

@@ -10,7 +10,7 @@
 ### END INIT INFO
 
 dir="/home/ubuntu/snowplow/bin/"
-cmd="./snowplow-stream-enrich-0.7.0 --config /home/ubuntu/snowplow/configs/snowplow-stream-enrich.hocon --resolver file:/home/ubuntu/snowplow/configs/default-iglu-resolver.json"
+cmd="./snowplow-stream-enrich-0.7.0 --config /home/ubuntu/snowplow/configs/snowplow-stream-enrich.hocon --resolver file:/home/ubuntu/snowplow/configs/iglu-resolver.json"
 user="ubuntu"
 
 name=`basename $0`

scripts/provision.sh

@@ -16,6 +16,7 @@ es_dir=$main_dir/elasticsearch
 
 # Packagaes
 kinesis_package=snowplow_kinesis_r78_great_hornbill.zip
+iglu_server_package=iglu_server_0.2.0.zip
 kibana_v=4.0.1
 
 ##################
@@ -34,6 +35,24 @@ sudo apt-get install oracle-java7-installer -y
 wget http://dl.bintray.com/snowplow/snowplow-generic/${kinesis_package} -P $staging_dir
 unzip $staging_dir/${kinesis_package} -d $executables_dir
 
+#######################
+# Install Iglu Server #
+#######################
+
+wget http://bintray.com/artifact/download/snowplow/snowplow-generic/${iglu_server_package} -P $staging_dir
+unzip $staging_dir/${iglu_server_package} -d $executables_dir
+
+######################
+# Install PostgreSQL #
+######################
+
+sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" >> /etc/apt/sources.list.d/pgdg.list'
+wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key add -
+sudo apt-get update
+sudo apt-get install postgresql postgresql-contrib -y
+sudo -u postgres psql -c "create user snowplow createdb password 'snowplow';"
+sudo -u postgres psql -c "create database iglu owner snowplow;"
+
 #########################
 # Install Elasticsearch #
 #########################

scripts/setup_init.sh

@@ -32,3 +32,11 @@ sudo update-rc.d snowplow_elasticsearch_sink_good_0.5.0 defaults
 sudo cp $init_dir/snowplow_elasticsearch_sink_bad_0.5.0 /etc/init.d
 sudo chmod 0755 /etc/init.d/snowplow_elasticsearch_sink_bad_0.5.0
 sudo update-rc.d snowplow_elasticsearch_sink_bad_0.5.0 defaults
+
+sudo cp $init_dir/iglu_server_0.2.0 /etc/init.d
+sudo chmod 0755 /etc/init.d/iglu_server_0.2.0
+sudo update-rc.d iglu_server_0.2.0 defaults
+
+## Start iglu_server to ensure apikeys table gets created
+sudo service iglu_server_0.2.0 start
+sleep 60

scripts/user_data.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# NOTE: Use `uuidgen` to create new `uid`
+iglu_server_super_uid="980ae3ab-3aba-4ffe-a3c2-3b2e24e2ffce"
+
+# DO NOT ALTER BELOW #
+
+export PGPASSWORD=snowplow
+iglu_server_setup="INSERT INTO apikeys (uid, vendor_prefix, permission, createdat) VALUES ('${iglu_server_super_uid}','*','super',current_timestamp);"
+psql --host=localhost --port=5432 --username=snowplow --dbname=iglu -c "${iglu_server_setup}"