[WIP] Assemble the apps using Docker Compose (closes #23)

snowplow · Jun 5, 2018 · 47ff51f · 47ff51f
1 parent 175c784
commit 47ff51f
Show file tree

Hide file tree

Showing 5 changed files with 166 additions and 30 deletions.
diff --git a/provisioning/resources/configs/snowplow-stream-collector.hocon b/provisioning/resources/configs/snowplow-stream-collector.hocon
@@ -29,6 +29,17 @@ collector {
     CP = "NOI DSP COR NID PSA OUR IND COM NAV STA"
   }
 
+  # Cross domain policy configuration.
+  # If "enabled" is set to "false", the collector will respond with a 404 to the /crossdomain.xml
+  # route.
+  crossDomain {
+    enabled = false
+    # Domain that is granted access, *.acme.com will match http://acme.com and http://sub.acme.com
+    domain = "*"
+    # Whether to only grant access to HTTPS or both HTTPS and HTTP sources
+    secure = true
+  }
+
   # The collector returns a cookie to clients for user identification
   # with the following domain and expiration.
   cookie {
@@ -54,6 +65,15 @@ collector {
     fallbackNetworkUserId = "00000000-0000-4000-A000-000000000000"
   }
 
+  # When enabled, the redirect url passed via the `u` query parameter is scanned for a placeholder
+  # token. All instances of that token are replaced withe the network ID. If the placeholder isn't
+  # specified, the default value is `${SP_NUID}`.
+  redirectMacro {
+    enabled = false
+    # Optional custom placeholder token (defaults to the literal `${SP_NUID}`)
+    placeholder = "[TOKEN]"
+  }
+
   streams {
     # Events which have successfully been collected will be stored in the good stream/topic
     good = RawEvents

diff --git a/provisioning/roles/docker/files/docker-compose.yml b/provisioning/roles/docker/files/docker-compose.yml
@@ -0,0 +1,65 @@
+version: "3"
+
+services:
+  nsqlookupd:
+    image: nsqio/nsq
+    command: /nsqlookupd
+    ports:
+      - "4160:4160"
+      - "4161:4161"
+
+  nsqd:
+    image: nsqio/nsq
+    command: /nsqd --lookupd-tcp-address=nsqlookupd:4160 --data-path=/home/ubuntu/snowplow/nsq-data
+    volumes:
+      - /home/ubuntu/snowplow/nsq-data:/home/ubuntu/snowplow/nsq-data
+    depends_on:
+      - nsqlookupd
+    ports:
+      - "4150:4150"
+      - "4151:4151"
+
+  nsqadmin:
+    image: nsqio/nsq
+    command: /nsqadmin --lookupd-http-address=nsqlookupd:4161
+    depends_on:
+      - nsqlookupd
+    ports:
+      - "4171:4171"
+
+  scala-stream-collector:
+    image: snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector-nsq:0.13.0
+    command: [ "--config", "/snowplow/config/snowplow-stream-collector.hocon" ]
+    depends_on:
+      - nsqd
+    ports:
+      - "8080:8080"
+    volumes:
+      - /home/ubuntu/snowplow/configs:/snowplow/config
+    logging:
+      options:
+        max-size: "1M"
+        max-file: "10"
+    environment:
+      - "SP_JAVA_OPTS=-Xms512m -Xmx512m"
+
+  stream-enrich:
+    image: snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.0
+    command: [
+      "--config", "/snowplow/config/snowplow-stream-enrich.hocon",
+      "--resolver", "file:/snowplow/config/iglu-resolver.json",
+      "--enrichments", "file:/snowplow/config/enrichments",
+      "--force-ip-lookups-download"
+    ]
+    depends_on:
+      - scala-stream-collector
+    volumes:
+      - /home/ubuntu/snowplow/configs:/snowplow/config
+    logging:
+      options:
+        max-size: "1M"
+        max-file: "10"
+    environment:
+      - "SP_JAVA_OPTS=-Xms512m -Xmx512m"
+
+
diff --git a/provisioning/roles/docker/tasks/main.yml b/provisioning/roles/docker/tasks/main.yml
@@ -0,0 +1,65 @@
+---
+- name: Setup the docker repository and install docker
+  sudo: yes
+  shell: |
+    apt-get update
+    apt-get install apt-transport-https ca-certificates curl software-properties-common --yes
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    apt-get update && apt-get install docker-ce --yes
+
+- name: Download docker-compose
+  sudo: yes
+  shell: curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
+
+- name: Apply executable permissions to the docker-compose binary
+  sudo: yes
+  shell: chmod +x /usr/local/bin/docker-compose
+
+# - name: Pull required images
+#   sudo: yes
+#   shell: docker pull "{{ item }}"
+#   with_items:
+#     - nsqio/nsq:v1.0.0-compat
+#     - postgres:9.5
+#     - elasticsearch:1.7.5
+#     - kibana:4.0.2
+#     - snowplow-docker-registry.bintray.io/snowplow/scala-stream-collector:0.11.0
+#     - snowplow-docker-registry.bintray.io/snowplow/stream-enrich-nsq:0.16.1
+#     - snowplow-docker-registry.bintray.io/snowplow/elasticsearch-loader:0.10.1
+#     - snowplow-docker-registry.bintray.io/snowplow/iglu-server:0.3.0
+
+# - name: Initialize a docker swarm
+#   sudo: yes
+#   shell: docker swarm init --advertise-addr 172.17.0.1
+
+- copy:
+    src: ../files/docker-compose.yml
+    dest: /home/ubuntu/snowplow/
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Ensure file permissions
+  become: yes
+  shell: chown ubuntu:ubuntu -R /home/ubuntu/snowplow && chmod 755 -R /home/ubuntu/snowplow
+
+- name: Deploy snowplow mini
+  become: yes
+  shell: cd /home/ubuntu/snowplow/ && docker-compose up -d
+
+- name: Create new topic for RawEvents
+  become: yes
+  shell: "curl -X POST http://127.0.0.1:4151/topic/create?topic=RawEvents"
+
+- name: Create new topic for BadEvents
+  become: yes
+  shell: "curl -X POST http://127.0.0.1:4151/topic/create?topic=BadEvents"
+
+- name: Create new topic for EnrichedEvents
+  become: yes
+  shell: "curl -X POST http://127.0.0.1:4151/topic/create?topic=EnrichedEvents"
+
+- name: Create new topic for BadEnrichedEvents
+  become: yes
+  shell: "curl -X POST http://127.0.0.1:4151/topic/create?topic=BadEnrichedEvents"
diff --git a/provisioning/roles/sp_mini_1_create_dirs/tasks/main.yml b/provisioning/roles/sp_mini_1_create_dirs/tasks/main.yml
@@ -10,26 +10,8 @@
   shell: 'adduser ubuntu --disabled-password --gecos "" ;
           passwd -d ubuntu'
 
-- name: Insert logrotate configuration for Snowplow Services
-  become: yes
-  copy:
-    dest: "/etc/logrotate.d/snowplow-apps"
-    content: |
-      /var/log/snowplow*.log /var/log/snowplow*.err /var/log/nsq*.log  /var/log/nsq*.err /var/log/iglu_server*.log /var/log/iglu_server*.err {
-          hourly
-          rotate 3
-          missingok
-          notifempty
-          copytruncate
-      }
-    mode: 0644
-
-- name: Change logrotate cron to hourly
-  become: yes
-  shell: 'mv /etc/cron.daily/logrotate /etc/cron.hourly && service cron restart'
-
 - name: creating directories
-  file: path={{item}} state=directory
+  file: path={{item}} state=directory mode=0755
   with_items:
     - "{{configs_dir}}"
     - "{{staging_dir}}"
@@ -48,13 +30,16 @@
         src: "{{playbook_dir}}/resources/elasticsearch"
         dest: "{{main_dir}}"
         recursive: yes
+        archive: no
 
     - synchronize:
         src: "{{playbook_dir}}/resources/configs"
         dest: "{{main_dir}}"
         recursive: yes
+        archive: no
 
     - synchronize:
         src: "{{playbook_dir}}/resources/init"
         dest: "{{main_dir}}"
         recursive: yes
+        archive: no
diff --git a/provisioning/with_building_ui_and_go_projects.yml b/provisioning/with_building_ui_and_go_projects.yml
@@ -7,15 +7,16 @@
   hosts: all
 
   roles:
-    - ansible-go
-    - nodejs
-    - typescript
-    - packer
+    # - ansible-go
+    # - nodejs
+    # - typescript
+    # - packer
     - sp_mini_1_create_dirs
-    - sp_mini_2_install_postgresl
-    - sp_mini_3_build_go_projects
-    - sp_mini_4_setup_apps
-    - sp_mini_5_build_ui
-    - sp_mini_6_copy_ui_folders
-    - sp_mini_7_setup_init
-    - sp_mini_8_configure
+    - docker
+    # - sp_mini_2_install_postgresl
+    # - sp_mini_3_build_go_projects
+    # - sp_mini_4_setup_apps
+    # - sp_mini_5_build_ui
+    # - sp_mini_6_copy_ui_folders
+    # - sp_mini_7_setup_init
+    # - sp_mini_8_configure