Add Control Plane API call to put your own custom enrichments in Snow…

…plow Mini (closes #66)

Vagrantfile

@@ -8,7 +8,7 @@ Vagrant.configure("2") do |config|
   config.vm.network :private_network, ip: '192.168.50.50' # Uncomment to use NFS
   config.vm.synced_folder '.', '/vagrant', nfs: true # Uncomment to use NFS
 
-  config.vm.network "forwarded_port", guest: 2000, host: 2000
+  config.vm.network "forwarded_port", guest: 80, host: 2000
   config.vm.network "forwarded_port", guest: 3000, host: 3000
   config.vm.network "forwarded_port", guest: 8080, host: 8080
   config.vm.network "forwarded_port", guest: 9200, host: 9200

integration/integration_test.sh

@@ -27,32 +27,18 @@ echo "Event Counts:"
 echo " - Good: ${good_count}"
 echo " - Bad: ${bad_count}"
 
-stream_enrich_pid_file=/var/run/snowplow_stream_enrich_0.10.0.pid
-stream_collector_pid_file=/var/run/snowplow_stream_collector_0.9.0.pid
-sink_bad_pid_file=/var/run/snowplow_elasticsearch_sink_bad_0.8.0-2x.pid
-sink_good_pid_file=/var/run/snowplow_elasticsearch_sink_good_0.8.0-2x.pid
-
-
-stream_enrich_pid_old="$(cat "${stream_enrich_pid_file}")"
-stream_collector_pid_old="$(cat "${stream_collector_pid_file}")"
-sink_bad_pid_old="$(cat "${sink_bad_pid_file}")"
-sink_good_pid_old="$(cat "${sink_good_pid_file}")"
-
-req_result=$(curl --silent -XPUT 'http://localhost:10000/restart-services')
-
-stream_enrich_pid_new="$(cat "${stream_enrich_pid_file}")"
-stream_collector_pid_new="$(cat "${stream_collector_pid_file}")"
-sink_bad_pid_new="$(cat "${sink_bad_pid_file}")"
-sink_good_pid_new="$(cat "${sink_good_pid_file}")"
+control_plane_dir="/home/ubuntu/snowplow/control-plane"
+#copy control plane directory to /home/ubuntu/snowplow for testing
+sudo cp -r provisioning/resources/control-plane $control_plane_dir
+test_dir="$control_plane_dir/test"
+$test_dir/test.sh $test_dir
+#remove after testing is done
+sudo rm -rf $control_plane_dir
+control_plane_test_res=$?
 
 # Bad Count is 11 due to bad logging
-if [[ "${good_count}" -eq "10" ]] && [[ "${bad_count}" -eq "11" ]] && 
-   [[ "${req_result}" == "OK" ]] &&
-   [[ "${stream_enrich_pid_old}" -ne "${stream_enrich_pid_new}" ]] &&
-   [[ "${stream_collector_pid_old}" -ne "${stream_collector_pid_new}" ]] &&
-   [[ "${sink_bad_pid_old}" -ne "${sink_bad_pid_new}" ]] &&
-   [[ "${sink_good_pid_old}" -ne "${sink_good_pid_new}" ]]; then
-
+if [[ "${good_count}" -eq "10" ]] && [[ "${bad_count}" -eq "11" ]] &&
+   [[ "${control_plane_test_res}" -eq "0" ]]; then
   exit 0
 else
   exit 1

provisioning/resources/configs/Caddyfile

@@ -1,4 +1,5 @@
-localhost:2000 {
+*:80 {
+  tls off
   basicauth "USERNAME_PLACEHOLDER" PASSWORD_PLACEHOLDER {
     /home
     /kibana

provisioning/resources/control-plane/snowplow-mini-control-plane-api.go

@@ -20,34 +20,81 @@
 package main
 
 import (
-    "github.com/emicklei/go-restful"
-    "io"
-    "net/http"
-    "log"
-    "os/exec"
-    "flag"
+	"io"
+	"net/http"
+	"log"
+	"os/exec"
+	"os"
+	"flag"
 )
 
 //global variable for script's path
 var scriptsPath string
+var enrichmentsPath string
+var configPath string
 
 func main() {
-    scriptsPathFlag := flag.String("scriptsPath", "", "path for control-plane-api scripts")
-    flag.Parse()
-    scriptsPath = *scriptsPathFlag
-
-    ws := new(restful.WebService)
-    ws.Route(ws.PUT("/restart-services").To(restartSPServices))
-    restful.Add(ws)
-    log.Fatal(http.ListenAndServe(":10000", nil))
+	scriptsPathFlag := flag.String("scriptsPath", "", "path for control-plane-api scripts")
+	enrichmentsPathFlag := flag.String("enrichmentsPath", "", "path for enrichment files")
+	configPathFlag := flag.String("configPath", "", "path for config files")
+	flag.Parse()
+	scriptsPath = *scriptsPathFlag
+	enrichmentsPath = *enrichmentsPathFlag
+	configPath = *configPathFlag
+
+	http.HandleFunc("/restart-services", restartSPServices)
+	http.HandleFunc("/upload-enrichments", uploadEnrichments)
+	log.Fatal(http.ListenAndServe(":10000", nil))
+}
+
+func callRestartSPServicesScript() (string, error){
+	shellScriptCommand := []string{scriptsPath + "/" +  "restart_SP_services.sh"}
+	cmd := exec.Command("/bin/bash", shellScriptCommand...)
+	err := cmd.Run()
+	if err != nil {
+		return "ERR", err
+	}
+	return "OK", err
+}
+
+func restartSPServices(resp http.ResponseWriter, req *http.Request) {
+	if (req.Method == "PUT") {
+		_, err := callRestartSPServicesScript()
+		if err != nil {
+			http.Error(resp, err.Error(), 400)
+			return
+		} else {
+			resp.WriteHeader(http.StatusOK)
+			io.WriteString(resp, "OK")
+		}
+	}
 }
 
-func restartSPServices(req *restful.Request, resp *restful.Response) {
-    cmd := exec.Command("/bin/sh", scriptsPath + "/" +  "restart_SP_services.sh")
-    err := cmd.Run()
-    if err != nil {
-        io.WriteString(resp, "ERR")
-    } else { 
-    	io.WriteString(resp, "OK")
-    }
+func uploadEnrichments(resp http.ResponseWriter, req *http.Request) {
+	if req.Method == "POST" {
+		req.ParseMultipartForm(32 << 20)
+		file, handler, err := req.FormFile("enrichmentjson")
+		if err != nil {
+			http.Error(resp, err.Error(), 400)
+			return
+		}
+		defer file.Close()
+		f, err := os.OpenFile(enrichmentsPath + "/" + handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
+		if err != nil {
+				http.Error(resp, err.Error(), 400)
+				return
+		}
+		defer f.Close()
+		io.Copy(f, file)
+		//restart SP services to get action the enrichments 
+		_, err = callRestartSPServicesScript()
+		resp.WriteHeader(http.StatusOK)
+		if err != nil {
+			http.Error(resp, err.Error(), 400)
+			return
+		}
+		resp.WriteHeader(http.StatusOK)
+		io.WriteString(resp, "uploaded successfully")
+		return
+	}
 }

provisioning/resources/control-plane/test/test.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+testDir=$1
+testEnv="$testDir/testEnv"
+testInit="$testDir/testInit"
+
+sudo cp $testInit/snowplow_mini_control_plane_api_test_init /etc/init.d/snowplow_mini_control_plane_api
+sudo /etc/init.d/snowplow_mini_control_plane_api restart $testDir 
+sleep 2
+
+## restart SP services test
+stream_enrich_pid_file=/var/run/snowplow_stream_enrich_0.10.0.pid
+stream_collector_pid_file=/var/run/snowplow_stream_collector_0.9.0.pid
+sink_bad_pid_file=/var/run/snowplow_elasticsearch_sink_bad_0.8.0-2x.pid
+sink_good_pid_file=/var/run/snowplow_elasticsearch_sink_good_0.8.0-2x.pid
+
+
+stream_enrich_pid_old="$(cat "${stream_enrich_pid_file}")"
+stream_collector_pid_old="$(cat "${stream_collector_pid_file}")"
+sink_bad_pid_old="$(cat "${sink_bad_pid_file}")"
+sink_good_pid_old="$(cat "${sink_good_pid_file}")"
+
+req_result=$(curl -s -o /dev/null -w "%{http_code}" -XPUT http://localhost:10000/restart-services)
+
+stream_enrich_pid_new="$(cat "${stream_enrich_pid_file}")"
+stream_collector_pid_new="$(cat "${stream_collector_pid_file}")"
+sink_bad_pid_new="$(cat "${sink_bad_pid_file}")"
+sink_good_pid_new="$(cat "${sink_good_pid_file}")"
+
+if [[ "${req_result}" -eq 200 ]] &&
+   [[ "${stream_enrich_pid_old}" -ne "${stream_enrich_pid_new}" ]] &&
+   [[ "${stream_collector_pid_old}" -ne "${stream_collector_pid_new}" ]] &&
+   [[ "${sink_bad_pid_old}" -ne "${sink_bad_pid_new}" ]] &&
+   [[ "${sink_good_pid_old}" -ne "${sink_good_pid_new}" ]]; then
+
+  echo "Restarting SP services is working correctly."
+else
+  echo "Restarting SP services is not working correctly."
+  exit 1
+fi
+
+
+## upload enrichment test
+upload_enrichments_result=$(curl -s -o /dev/null -w "%{http_code}"  --header "Content-Type: multipart/form-data" -F "enrichmentjson=@$testEnv/orgEnrichments/enrich.json" localhost:10000/upload-enrichments)
+enrichment_diff=$(diff $testEnv/testEnrichments/enrich.json $testEnv/orgEnrichments/enrich.json)
+sleep 2
+
+if [[ "${upload_enrichments_result}" -eq 200 ]] && [[ "${enrichment_diff}" == "" ]];then
+  echo "Uploading enrichment is working correctly."
+else
+  echo "Uploading enrichment is not working correctly."
+  exit 1
+fi
+
+sudo cp $testInit/snowplow_mini_control_plane_api_original_init /etc/init.d/snowplow_mini_control_plane_api
+sudo /etc/init.d/snowplow_mini_control_plane_api restart 
+
+exit 0

provisioning/resources/control-plane/test/testEnv/expectedConfig/Caddyfile_add_domain_name_tls_off

@@ -0,0 +1,37 @@
+*:80 {
+  tls off
+  basicauth "USERNAME_PLACEHOLDER" PASSWORD_PLACEHOLDER {
+    /home
+    /kibana
+    /elasticsearch
+    /control-plane
+    /_plugin
+  }
+  redir /home /home/
+  redir /kibana /kibana/
+  redir /iglu-server /iglu-server/
+
+  proxy / localhost:8080
+
+  proxy /home localhost:3000 {
+    without /home
+  }
+
+  proxy /kibana localhost:5601 {
+    without /kibana
+  }
+
+  proxy /iglu-server localhost:8081 {
+    without /iglu-server
+  }
+  proxy /api localhost:8081
+
+  proxy /elasticsearch localhost:9200 {
+    without /elasticsearch
+  }
+  proxy /_plugin localhost:9200
+
+  proxy /control-plane localhost:10000 {
+    without /control-plane
+  }
+}

provisioning/resources/control-plane/test/testEnv/expectedConfig/Caddyfile_add_domain_name_tls_on

@@ -0,0 +1,37 @@
+test.com *:80 {
+  tls [email protected]
+  basicauth "USERNAME_PLACEHOLDER" PASSWORD_PLACEHOLDER {
+    /home
+    /kibana
+    /elasticsearch
+    /control-plane
+    /_plugin
+  }
+  redir /home /home/
+  redir /kibana /kibana/
+  redir /iglu-server /iglu-server/
+
+  proxy / localhost:8080
+
+  proxy /home localhost:3000 {
+    without /home
+  }
+
+  proxy /kibana localhost:5601 {
+    without /kibana
+  }
+
+  proxy /iglu-server localhost:8081 {
+    without /iglu-server
+  }
+  proxy /api localhost:8081
+
+  proxy /elasticsearch localhost:9200 {
+    without /elasticsearch
+  }
+  proxy /_plugin localhost:9200
+
+  proxy /control-plane localhost:10000 {
+    without /control-plane
+  }
+}

provisioning/resources/control-plane/test/testEnv/expectedConfig/Caddyfile_change_username_password

@@ -0,0 +1,37 @@
+*:80 {
+  tls off
+  basicauth "test_username" "test_password" {
+    /home
+    /kibana
+    /elasticsearch
+    /control-plane
+    /_plugin
+  }
+  redir /home /home/
+  redir /kibana /kibana/
+  redir /iglu-server /iglu-server/
+
+  proxy / localhost:8080
+
+  proxy /home localhost:3000 {
+    without /home
+  }
+
+  proxy /kibana localhost:5601 {
+    without /kibana
+  }
+
+  proxy /iglu-server localhost:8081 {
+    without /iglu-server
+  }
+  proxy /api localhost:8081
+
+  proxy /elasticsearch localhost:9200 {
+    without /elasticsearch
+  }
+  proxy /_plugin localhost:9200
+
+  proxy /control-plane localhost:10000 {
+    without /control-plane
+  }
+}

provisioning/resources/control-plane/test/testEnv/expectedConfig/iglu-resolver-add-super-uuid.json

@@ -0,0 +1,33 @@
+{
+  "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-1",
+  "data": {
+    "cacheSize": 500,
+    "repositories": [
+      {
+        "name": "Iglu Central",
+        "priority": 1,
+        "vendorPrefixes": [
+          "com.snowplowanalytics"
+        ],
+        "connection": {
+          "http": {
+            "uri": "http://iglucentral.com"
+          }
+        }
+      },
+      {
+        "name": "Iglu Server",
+        "priority": 0,
+        "vendorPrefixes": [
+          "com.snowplowanalytics"
+        ],
+        "connection": {
+          "http": {
+            "uri": "http://localhost:8081/api",
+            "apikey": "04577adf-6dce-49d7-8cbb-0ffdf83304de"
+          }
+        }
+      }
+    ]
+  }
+}