[WIP] Assemble the apps using Docker Compose (closes #23)

provisioning/resources/configs/snowplow-stream-collector.hocon

@@ -29,6 +29,17 @@ collector {
     CP = "NOI DSP COR NID PSA OUR IND COM NAV STA"
   }
 
+  # Cross domain policy configuration.
+  # If "enabled" is set to "false", the collector will respond with a 404 to the /crossdomain.xml
+  # route.
+  crossDomain {
+    enabled = false
+    # Domain that is granted access, *.acme.com will match http://acme.com and http://sub.acme.com
+    domain = "*"
+    # Whether to only grant access to HTTPS or both HTTPS and HTTP sources
+    secure = true
+  }
+
   # The collector returns a cookie to clients for user identification
   # with the following domain and expiration.
   cookie {
@@ -54,6 +65,15 @@ collector {
     fallbackNetworkUserId = "00000000-0000-4000-A000-000000000000"
   }
 
+  # When enabled, the redirect url passed via the `u` query parameter is scanned for a placeholder
+  # token. All instances of that token are replaced withe the network ID. If the placeholder isn't
+  # specified, the default value is `${SP_NUID}`.
+  redirectMacro {
+    enabled = false
+    # Optional custom placeholder token (defaults to the literal `${SP_NUID}`)
+    placeholder = "[TOKEN]"
+  }
+
   streams {
     # Events which have successfully been collected will be stored in the good stream/topic
     good = RawEvents

provisioning/roles/docker/files/docker-compose.yml

@@ -0,0 +1,106 @@
+version: "3"
+
+services:
+  elasticsearch-head-kibana:
+    build: es/
+    container_name: elasticsearch-head-kibana
+    restart: on-failure
+    ports:
+      - "9200:9200"
+      - "5601:5601"
+
+  elasticsearch-loader-good:
+    image: snowplow-docker-registry.bintray.io/snowplow/elasticsearch-loader:0.10.1
+    container_name: elasticsearch-loader-good
+    command: [ "--config", "/snowplow/config/snowplow-es-loader-good.hocon" ]
+    depends_on:
+      - elasticsearch-head-kibana
+    volumes:
+      - /home/ubuntu/snowplow/configs:/snowplow/config
+    logging:
+      options:
+        max-size: "1M"
+        max-file: "10"
+    environment:
+      - "SP_JAVA_OPTS=-Xms512m -Xmx512m"
+
+  elasticsearch-loader-bad:
+    image: snowplow-docker-registry.bintray.io/snowplow/elasticsearch-loader:0.10.1
+    container_name: elasticsearch-loader-bad
+    command: [ "--config", "/snowplow/config/snowplow-es-loader-bad.hocon" ]
+    depends_on:
+      - elasticsearch-head-kibana
+    volumes:
+      - /home/ubuntu/snowplow/configs:/snowplow/config
+    logging:
+      options:
+        max-size: "1M"
+        max-file: "10"
+    environment:
+      - "SP_JAVA_OPTS=-Xms512m -Xmx512m"
+
+  nsqlookupd:
+    image: nsqio/nsq:v1.0.0-compat
+    container_name: nsqlookupd
+    command: /nsqlookupd
+    ports:
+      - "4160:4160"
+      - "4161:4161"
+
+  nsqd:
+    image: nsqio/nsq:v1.0.0-compat
+    container_name: nsqd
+    command: /nsqd --lookupd-tcp-address=nsqlookupd:4160 --data-path=/home/ubuntu/snowplow/nsq-data
+    volumes:
+      - /home/ubuntu/snowplow/nsq-data:/home/ubuntu/snowplow/nsq-data
+    depends_on:
+      - nsqlookupd
+    ports:
+      - "4150:4150"
+      - "4151:4151"
+
+  nsqadmin:
+    image: nsqio/nsq:v1.0.0-compat
+    container_name: nsqadmin
+    command: /nsqadmin --lookupd-http-address=nsqlookupd:4161
+    depends_on:
+      - nsqlookupd
+    ports:
+      - "4171:4171"
+
+  scala-stream-collector:
+    image: snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:0.13.0
+    container_name: scala-stream-collector-nsq
+    command: [ "--config", "/snowplow/config/snowplow-stream-collector.hocon" ]
+    depends_on:
+      - nsqd
+    ports:
+      - "8080:8080"
+    volumes:
+      - /home/ubuntu/snowplow/configs:/snowplow/config
+    logging:
+      options:
+        max-size: "1M"
+        max-file: "10"
+    environment:
+      - "SP_JAVA_OPTS=-Xms512m -Xmx512m"
+
+  stream-enrich:
+    image: snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.0
+    container_name: stream-enrich-nsq
+    command: [
+      "--config", "/snowplow/config/snowplow-stream-enrich.hocon",
+      "--resolver", "file:/snowplow/config/iglu-resolver.json",
+      "--enrichments", "file:/snowplow/config/enrichments",
+      "--force-ip-lookups-download"
+    ]
+    depends_on:
+      - scala-stream-collector
+    volumes:
+      - /home/ubuntu/snowplow/configs:/snowplow/config
+    logging:
+      options:
+        max-size: "1M"
+        max-file: "10"
+    environment:
+      - "SP_JAVA_OPTS=-Xms512m -Xmx512m"

provisioning/roles/docker/files/es/Dockerfile

@@ -0,0 +1,42 @@
+FROM ubuntu:14.04
+
+# The versions of apps in elasticsearch stack
+ENV ELASTICSEARCH_VERSION="1.7.5"
+ENV KIBANA_VERSION="4.0.1"
+
+# The names of the archives to be downloaded
+ENV ES_ARCHIVE="elasticsearch-${ELASTICSEARCH_VERSION}.deb"
+ENV KIBANA_ARCHIVE="kibana-${KIBANA_VERSION}-linux-x64.zip"
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y curl wget unzip --no-install-recommends software-properties-common && \
+    add-apt-repository -y ppa:webupd8team/java && \
+    apt-get update && \
+    (echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections) && \
+    apt-get install --no-install-recommends -y oracle-java8-installer && \
+    rm -rf /var/cache/oracle-jdk8-installer && \
+    echo "networkaddress.cache.ttl=60" >> /usr/lib/jvm/java-8-oracle/jre/lib/security/java.security && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
+
+# Install Elasticsearch, Elasticsearch Head Plugin & Kibana
+RUN mkdir -p /tmp/build && \
+    cd /tmp/build && \
+    wget -q https://download.elastic.co/elasticsearch/elasticsearch/${ES_ARCHIVE} && \
+    sudo dpkg -i $ES_ARCHIVE && \
+    /usr/share/elasticsearch/bin/plugin --install mobz/elasticsearch-head && \
+    wget -q https://download.elasticsearch.org/kibana/kibana/${KIBANA_ARCHIVE} && \
+    unzip $KIBANA_ARCHIVE -d /opt/ && \
+    ln -s /opt/kibana-${KIBANA_VERSION}-linux-x64 /opt/kibana
+
+COPY config/* /usr/share/elasticsearch/config/
+
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+
+ENV PATH /usr/share/elasticsearch/bin:/opt/kibana/bin:$PATH
+
+EXPOSE 9200 5601
+
+ENTRYPOINT [ "docker-entrypoint.sh" ]

provisioning/roles/docker/files/es/config/bad-mapping.json

@@ -0,0 +1,49 @@
+{
+    "settings": {
+        "analysis": {
+            "analyzer": {
+                "default": {
+                    "type": "keyword"
+                }
+            }
+        },
+        "index" : {
+            "number_of_replicas" : "0",
+            "number_of_shards" : "2"
+        }
+    },
+    "mappings": {
+        "bad": {
+            "_timestamp" : {
+                "enabled" : "yes",
+                "path" : "failure_tstamp"
+            },
+            "_ttl": {
+              "enabled": true,
+              "default": "604800000"
+            },
+            "properties": {
+                "errors": {
+                    "properties": {
+                        "message" : {
+                            "type": "string",
+                            "analyzer": "standard"
+                        },
+                        "level" : {
+                            "type": "string",
+                            "analyzer": "standard"
+                        }
+                    }
+                },
+                "failure_tstamp": {
+                    "type": "date",
+                    "format": "dateOptionalTime"
+                },
+                "line": {
+                    "type": "string",
+                    "analyzer": "standard"
+                }
+            }
+        }
+    }
+}