Update to Packer 0.10.0+ and convert shell scripts to ansible playboo…

…ks (closes #52)

.travis.yml

@@ -1,21 +1,20 @@
+---
 sudo: required
 language: bash
 
 services:
   - postgresql
 
-## Mimics the Packerfile.json steps
+
+before_install:
+  - sudo apt-get update -qq
+
+install:
+  - sudo pip install setuptools
+  - sudo pip install ansible
+
 before_script:
-  - sudo adduser ubuntu --disabled-password --gecos ""
-  - sudo passwd -d ubuntu
-  - sudo ./scripts/1_setup_dirs_pipes.sh
-  - sudo cp -rf ./resources/elasticsearch /home/ubuntu/snowplow
-  - sudo ./scripts/3_setup_apps.sh
-  - sudo cp -rf ./resources/configs /home/ubuntu/snowplow
-  - sudo cp -rf ./scripts /home/ubuntu/snowplow
-  - sudo cp -rf ./resources/init /home/ubuntu/snowplow
-  - sudo ./scripts/5_setup_init.sh
-  - sudo ./scripts/6_configure.sh
+  - ansible-playbook -i provisioning/inventory provisioning/with_building_ui.yml --connection=local --sudo
 
 script:
   - ./integration/integration_test.sh

Packerfile.json

@@ -1,92 +1,30 @@
 {
+   "variables": {
+     "version": "0.3.0-RC"
+   },
+
   "builders": [
     {
       "type": "amazon-ebs",
       "region": "us-east-1",
       "source_ami": "ami-05dddc6f",
       "instance_type": "t2.small",
       "ssh_username": "ubuntu",
-      "ami_name": "snowplow-mini-0.2.2-{{ timestamp }}-hvm-ebs-amd64",
+      "ami_name": "snowplow-mini-{{user `version`}}-{{ timestamp }}-hvm-ebs-amd64",
       "ami_groups": [ "all" ],
       "ami_regions": "us-east-2,us-west-1,us-west-2,ca-central-1,eu-west-1,eu-central-1,eu-west-2,ap-southeast-1,ap-southeast-2,ap-northeast-2,ap-northeast-1,ap-south-1,sa-east-1",
       "ami_description": "Snowplow Mini - The Snowplow Pipeline in a box",
       "tags": {
-        "OS_Version": "Ubuntu-12.04",
-        "Release": "0.2.2"
+        "OS_Version": "Ubuntu-14.04",
+        "Release": "{{user `version`}}"
       }
     }
   ],
 
   "provisioners": [
     {
-      "type": "shell",
-      "script": "scripts/1_setup_dirs_pipes.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "file",
-      "source": "resources/elasticsearch",
-      "destination": "/home/ubuntu/snowplow"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/2_setup_postgres.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/3_setup_apps.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "file",
-      "source": "resources/configs",
-      "destination": "/home/ubuntu/snowplow"
-    },
-    {
-      "type": "file",
-      "source": "scripts",
-      "destination": "/home/ubuntu/snowplow"
-    },
-    {
-      "type": "file",
-      "source": "resources/init",
-      "destination": "/home/ubuntu/snowplow"
-    },
-    {
-      "type": "file",
-      "source": "ui/index.html",
-      "destination": "/home/ubuntu/snowplow/ui/index.html"
-    },
-    {
-      "type": "file",
-      "source": "ui/dist/snowplow-mini.js",
-      "destination": "/home/ubuntu/snowplow/ui/dist/snowplow-mini.js"
-    },
-    {
-      "type": "file",
-      "source": "ui/assets",
-      "destination": "/home/ubuntu/snowplow/ui"
-    },
-    {
-      "type": "file",
-      "source": "ui/node_modules",
-      "destination": "/home/ubuntu/snowplow/ui"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/4_setup_ui.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/5_setup_init.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
-    },
-    {
-      "type": "shell",
-      "script": "scripts/6_configure.sh",
-      "execute_command": "chmod +x {{ .Path }}; sh '{{ .Path }}'"
+      "type": "ansible",
+      "playbook_file": "provisioning/without_building_ui.yml"
     }
   ]
 }

Vagrantfile

@@ -8,7 +8,8 @@ Vagrant.configure("2") do |config|
   config.vm.network :private_network, ip: '192.168.50.50' # Uncomment to use NFS
   config.vm.synced_folder '.', '/vagrant', nfs: true # Uncomment to use NFS
 
-  config.vm.network "forwarded_port", guest: 80, host: 3000
+  config.vm.network "forwarded_port", guest: 2000, host: 2000
+  config.vm.network "forwarded_port", guest: 3000, host: 3000
   config.vm.network "forwarded_port", guest: 8080, host: 8080
   config.vm.network "forwarded_port", guest: 9200, host: 9200
   config.vm.network "forwarded_port", guest: 5601, host: 5601
@@ -19,10 +20,10 @@ Vagrant.configure("2") do |config|
     vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
     vb.customize [ "guestproperty", "set", :id, "--timesync-threshold", 10000 ]
-    vb.memory = 4096 
+    vb.memory = 4096
     vb.cpus = 1
   end
-
+  
   config.vm.provision :shell do |sh|
     sh.path = "vagrant/up.bash"
   end
@@ -32,4 +33,4 @@ Vagrant.configure("2") do |config|
     push.script = "vagrant/push.bash"
   end
 
-end
+end

vagrant/ansible.hosts → provisioning/inventory

@@ -1,2 +1,4 @@
 [vagrant]
 127.0.0.1:2222
+[localhost]
+127.0.0.1:22

provisioning/resources/configs/enrichments/campaign_attribution.json

@@ -0,0 +1,18 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/campaign_attribution/jsonschema/1-0-1",
+  "data": {
+    "name": "campaign_attribution",
+    "vendor": "com.snowplowanalytics.snowplow",
+    "enabled": true,
+    "parameters": {
+      "mapping": "static",
+      "fields": {
+        "mktMedium": ["utm_medium"],
+        "mktSource": ["utm_source"],
+        "mktTerm": ["utm_term"],
+        "mktContent": ["utm_content"],
+        "mktCampaign": ["utm_campaign"]
+      }
+    }
+  }
+}

provisioning/resources/configs/enrichments/event_fingerprint_config.json

@@ -0,0 +1,12 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/event_fingerprint_config/jsonschema/1-0-0",
+  "data": {
+    "name": "event_fingerprint_config",
+    "vendor": "com.snowplowanalytics.snowplow",
+    "enabled": true,
+    "parameters": {
+      "excludeParameters": ["eid", "stm"],
+      "hashAlgorithm": "MD5"
+    }
+  }
+}

provisioning/resources/configs/enrichments/ip_lookups.json

@@ -0,0 +1,14 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/ip_lookups/jsonschema/1-0-0",
+  "data": {
+    "name": "ip_lookups",
+    "vendor": "com.snowplowanalytics.snowplow",
+    "enabled": true,
+    "parameters": {
+      "geo": {
+        "database": "GeoLiteCity.dat",
+        "uri": "http://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/maxmind"
+      }
+    }
+  }
+}

provisioning/resources/configs/enrichments/referer_parser.json

@@ -0,0 +1,12 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/referer_parser/jsonschema/1-0-0",
+  "data": {
+    "name": "referer_parser",
+    "vendor": "com.snowplowanalytics.snowplow",
+    "enabled": true,
+    "parameters": {
+      "internalDomains": [
+      ]
+    }
+  }
+}

provisioning/resources/configs/enrichments/ua_parser_config.json

@@ -0,0 +1,9 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/ua_parser_config/jsonschema/1-0-0",
+  "data": {
+    "vendor": "com.snowplowanalytics.snowplow",
+    "name": "ua_parser_config",
+    "enabled": true,
+    "parameters": {}
+  }
+}

provisioning/resources/configs/enrichments/user_agent_utils_config.json

@@ -0,0 +1,9 @@
+{
+  "schema": "iglu:com.snowplowanalytics.snowplow/user_agent_utils_config/jsonschema/1-0-0",
+  "data": {
+    "vendor": "com.snowplowanalytics.snowplow",
+    "name": "user_agent_utils_config",
+    "enabled": true,
+    "parameters": {}
+  }
+}

resources/configs/iglu-resolver.json → provisioning/resources/configs/iglu-resolver.json

@@ -1,32 +1,32 @@
-{
-  "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-0",
-  "data": {
-    "cacheSize": 500,
-    "repositories": [
-      {
-        "name": "Iglu Central",
-        "priority": 1,
-        "vendorPrefixes": [
-          "com.snowplowanalytics"
-        ],
-        "connection": {
-          "http": {
-            "uri": "http://iglucentral.com"
-          }
-        }
-      },
-      {
-        "name": "Iglu Server",
-        "priority": 0,
-        "vendorPrefixes": [
-          "com.snowplowanalytics"
-        ],
-        "connection": {
-          "http": {
-            "uri": "http://localhost:8081/api"
-          }
-        }
-      }
-    ]
-  }
-}
+{
+  "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-0",
+  "data": {
+    "cacheSize": 500,
+    "repositories": [
+      {
+        "name": "Iglu Central",
+        "priority": 1,
+        "vendorPrefixes": [
+          "com.snowplowanalytics"
+        ],
+        "connection": {
+          "http": {
+            "uri": "http://iglucentral.com"
+          }
+        }
+      },
+      {
+        "name": "Iglu Server",
+        "priority": 0,
+        "vendorPrefixes": [
+          "com.snowplowanalytics"
+        ],
+        "connection": {
+          "http": {
+            "uri": "http://localhost:8081/api"
+          }
+        }
+      }
+    ]
+  }
+}

resources/configs/snowplow-elasticsearch-sink-bad.hocon → provisioning/resources/configs/snowplow-elasticsearch-sink-bad.hocon

@@ -94,7 +94,6 @@ sink {
     }
   }
 
-
   # Events are accumulated in a buffer before being sent to Elasticsearch.
   # The buffer is emptied whenever:
   # - the combined size of the stored records exceeds byte-limit or
@@ -105,4 +104,4 @@ sink {
     record-limit: 10000
     time-limit: 60000
   }
-}
+}

resources/configs/snowplow-mini.conf → provisioning/resources/configs/snowplow-mini.conf

@@ -1,5 +1,5 @@
 server {
-    listen 80;
+    listen 3000;
     root /home/ubuntu/snowplow/ui;
     server_name \$hostname;
     access_log /var/log/nginx/snowplow-mini.access.log;

ui/index.html → provisioning/resources/ui/index.html

@@ -35,6 +35,7 @@ <h1 id="text">SNOWPLOW MINI</h1>
       });
       window.snowplow('setUserId', 'SnowplowMini-User');
       window.snowplow('enableActivityTracking', 10, 10);
+      window.snowplow('trackPageView');
     </script>
     <!-- Snowplow stops plowing -->
 

ui/js/components/JSTracker.tsx → provisioning/resources/ui/js/components/JSTracker.tsx

@@ -43,6 +43,7 @@ export class JSTracker extends React.Component<{}, {}> {
   private trackPageView(): void {
     alert("Tracking this page view")
     window['snowplow']('trackPageView', 'Example events');
+    console.log("hey")
   }
 
   private playMix(): void {

provisioning/roles/base/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+- name: Update apt cache
+  apt: update-cache=yes cache_valid_time=86400
+  sudo: yes
+
+- name: install basic packages
+  sudo: yes
+  apt: pkg={{ item }} state=latest
+  tags: [packages]
+  with_items:
+    - curl
+    - unzip
+
+- name: Add Google DNS nameserver
+  lineinfile:
+    dest="/etc/resolvconf/resolv.conf.d/base"
+    line="nameserver 8.8.8.8"
+  sudo: yes
+
+- name: Update nameserver list
+  command: "resolvconf -u"
+  sudo: yes

provisioning/roles/common_vars.yml

@@ -0,0 +1,13 @@
+---
+main_dir:             "/home/ubuntu/snowplow"
+configs_dir:          "{{main_dir}}/configs"
+staging_dir:          "{{main_dir}}/staging"
+executables_dir:      "{{main_dir}}/bin"
+unix_pipes_dir:       "{{main_dir}}/pipes"
+es_dir:               "{{main_dir}}/elasticsearch"
+scripts_dir:          "{{main_dir}}/scripts"
+init_dir:             "{{main_dir}}/init"
+ui_dir:               "{{main_dir}}/ui/dist"
+ansible_cache_dir:    "/var/cache/ansible" 
+software_install_dir: "/usr/local/share"
+user_binaries:        "/usr/local/bin"

provisioning/roles/nodejs/.gitignore

@@ -0,0 +1,2 @@
+*.retry
+tests/test.sh