Add Control Plane API call to add external Iglu schema registry (closes

#62)

provisioning/resources/control-plane/scripts/add_external_iglu_server.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+iglu_server_uri=$1
+iglu_server_api_key=$2
+config_dir=$3
+control_api_scripts_dir=$4
+
+iglu_resolver_config_dir="$config_dir/iglu-resolver.json"
+template_iglu_server="$control_api_scripts_dir/template_iglu_server"
+
+##first change uri and apikey in the template_iglu_server file
+sudo sed -i 's/\(.*"uri":\)\(.*\)/\1 "'$iglu_server_uri'",/' $template_iglu_server
+sudo sed -i 's/\(.*"apikey":\)\(.*\)/\1 "'$iglu_server_api_key'"/' $template_iglu_server
+##secondly write content in the template_iglu_server to iglu_resolver.json
+sudo sed -i -E '/.*"repositories":.*/r '$template_iglu_server'' $iglu_resolver_config_dir

provisioning/resources/control-plane/scripts/template_iglu_server

@@ -0,0 +1,13 @@
+      {
+        "name": "Iglu Server",
+        "priority": 0,
+        "vendorPrefixes": [
+          "com.snowplowanalytics"
+        ],
+        "connection": {
+          "http": {
+            "uri": "PLACEHOLDER",
+            "apikey": "PLACEHOLDER"
+          }
+        }
+      },

provisioning/resources/control-plane/snowplow-mini-control-plane-api.go

@@ -30,6 +30,7 @@ import (
 
 // script file names
 var restartServicesScript = "restart_SP_services.sh"
+var addExternalIgluServerScript = "add_external_iglu_server.sh"
 
 // global variables for paths from flags
 var scriptsPath string
@@ -47,6 +48,8 @@ func main() {
 
   http.HandleFunc("/restart-services", restartSPServices)
   http.HandleFunc("/upload-enrichments", uploadEnrichments)
+  http.HandleFunc("/add-external-iglu-server", addExternalIgluServer)
+  http.HandleFunc("/check-url", checkUrl)
   log.Fatal(http.ListenAndServe(":10000", nil))
 }
 
@@ -99,3 +102,48 @@ func uploadEnrichments(resp http.ResponseWriter, req *http.Request) {
     return
   }
 }
+
+func addExternalIgluServer(resp http.ResponseWriter, req *http.Request) {
+  if req.Method == "POST" {
+    req.ParseForm()
+    if len(req.Form["iglu_server_uri"]) == 0 {
+      http.Error(resp, "parameter iglu_server_uri is not given", 400)
+      return
+    }
+    if len(req.Form["iglu_server_apikey"]) == 0 {
+      http.Error(resp, "parameter iglu_server_apikey is not given", 400)
+      return
+    }
+    igluServerUri := req.Form["iglu_server_uri"][0]
+    igluServerApikey := req.Form["iglu_server_apikey"][0]
+
+    if !isUrlReachable(igluServerUri) {
+      http.Error(resp, "Given URL is not reachable", 400)
+      return
+    }
+    if !isValidUuid(igluServerApikey) {
+      http.Error(resp, "Given apikey is not valid UUID.", 400)
+      return
+    }
+
+    shellScriptCommand := []string{scriptsPath + "/" + addExternalIgluServerScript,
+                                    igluServerUri,
+                                    igluServerApikey,
+                                    configPath,
+                                    scriptsPath}
+    cmd := exec.Command("/bin/bash", shellScriptCommand...)
+    err := cmd.Run()
+    if err != nil {
+      http.Error(resp, err.Error(), 400)
+      return
+    }
+    //restart SP services to get action the external iglu server
+    _, err = callRestartSPServicesScript()
+    if err != nil {
+      http.Error(resp, err.Error(), 400)
+      return
+    }
+    resp.WriteHeader(http.StatusOK)
+    io.WriteString(resp, "added successfully")
+  }
+}

provisioning/resources/control-plane/test/test_rest_api.sh

@@ -70,6 +70,53 @@ else
   exit 1
 fi
 
+## add external iglu server test: success
+sudo cp $testEnv/orgConfig/iglu-resolver.json $testConfigDir/.
+external_test_uuid=$(uuidgen)
+iglu_server_uri="example.com"
+add_external_iglu_server_result=$(curl -s -o /dev/null -w "%{http_code}" -d "iglu_server_uri=$iglu_server_uri&iglu_server_apikey=$external_test_uuid" localhost:10000/add-external-iglu-server)
+sleep 2
+
+uuid_grep_res=$(cat $testConfigDir/iglu-resolver.json | grep $external_test_uuid)
+iglu_server_uri_grep_res=$(cat $testConfigDir/iglu-resolver.json | grep $iglu_server_uri)
+
+if [[ "${add_external_iglu_server_result}" -eq 200 ]] &&
+   [[ "${uuid_grep_res}" != "" ]] &&
+   [[ "${iglu_server_uri_grep_res}" != "" ]] ;then
+  echo "Adding external Iglu Server success test returned as expected."
+else
+  echo "Adding external Iglu Server success test did not return as expected."
+  exit 1
+fi
+
+## add external iglu server test: invalid UUID format fail
+sudo cp $testEnv/orgConfig/iglu-resolver.json $testConfigDir/.
+external_test_uuid="123"
+iglu_server_uri="example.com"
+add_external_iglu_server_result=$(curl -s -o /dev/null -w "%{http_code}" -d "iglu_server_uri=$iglu_server_uri&iglu_server_apikey=$external_test_uuid" localhost:10000/add-external-iglu-server)
+sleep 2
+
+if [[ "${add_external_iglu_server_result}" -eq 400 ]];then
+  echo "Adding external Iglu Server invalid UUID format test returned as expected."
+else
+  echo "Adding external Iglu Server invalid UUID format test did not return as expected."
+  exit 1
+fi
+
+## add external iglu server test: down Iglu URL fail
+sudo cp $testEnv/orgConfig/iglu-resolver.json $testConfigDir/.
+external_test_uuid="123"
+iglu_server_uri="downiglu"
+add_external_iglu_server_result=$(curl -s -o /dev/null -w "%{http_code}" -d "iglu_server_uri=$iglu_server_uri&iglu_server_apikey=$external_test_uuid" localhost:10000/add-external-iglu-server)
+sleep 2
+
+if [[ "${add_external_iglu_server_result}" -eq 400 ]];then
+  echo "Adding external Iglu Server down Iglu URL test returned as expected."
+else
+  echo "Adding external Iglu Server down Iglu URL test did not return as expected."
+  exit 1
+fi
+
 sudo cp $testInit/snowplow_mini_control_plane_api_original_init /etc/init.d/snowplow_mini_control_plane_api
 sudo /etc/init.d/snowplow_mini_control_plane_api restart
 

provisioning/resources/control-plane/test/test_scripts.sh

@@ -52,6 +52,23 @@ else
   exit 1
 fi
 
+## add external iglu server test
+sudo cp $testEnv/orgConfig/iglu-resolver.json $testConfigDir/.
+external_test_uuid=$(uuidgen)
+iglu_server_uri="testigluserveruri.com"
+
+sudo $scripts/$addExternalIgluServerScript $iglu_server_uri $external_test_uuid $testConfigDir $scripts >> /dev/null
+res=$?
+
+written_apikey=$(diff $testConfigDir/iglu-resolver.json $testEnv/expectedConfig/iglu-resolver-external-iglu.json | grep $external_test_uuid)
+
+if [[ "${res}" -eq 0 ]] && [[ "${written_apikey}" != "" ]];then
+  echo "Adding external Iglu Server script is working correctly."
+else
+  echo "Adding external Iglu Server script is not working correctly."
+  exit 1
+fi
+
 #remove test control plane directory after testing is done
 sudo rm -rf $testControlPlaneDir
 

provisioning/resources/control-plane/utils.go

@@ -25,6 +25,10 @@ import (
   "net"
   "os/exec"
   "context"
+  "regexp"
+  "errors"
+  "time"
+  "io/ioutil"
 )
 
 // restarts services
@@ -43,3 +47,85 @@ func isJSON(jsonString string) bool {
   var js map[string]interface{}
   return json.Unmarshal([]byte(jsonString), &js) == nil
 }
+
+// check if given URL is reachable
+func isUrlReachable(url string) bool {
+  _, err := http.Get("http://" + url)
+  if err != nil {
+    return false
+  }
+  return true
+}
+
+// check whether given UUID follows the correct format
+func isValidUuid(uuid string) bool {
+    r := regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$")
+    return r.MatchString(uuid)
+}
+
+// returns public IP if the host machine is EC2 instance
+func getPublicEC2IP() (string, error) {
+  // URL of the instance meta service of AWS EC2
+  var urlForCheckingPubIP = "http://169.254.169.254/latest/meta-data/public-ipv4"
+  var netClient = &http.Client{
+    Timeout: time.Second * 5,
+  }
+
+  resp, err := netClient.Get(urlForCheckingPubIP)
+  if err != nil {
+    return "", err
+  }
+
+  defer resp.Body.Close()
+  body, err := ioutil.ReadAll(resp.Body)
+
+  return string(body), nil
+}
+
+// get IP addresses of the given domain name
+func getDomainNameIP(domainName string) ([]string, error) {
+  var (
+    ipAddresses []string
+    ctx context.Context
+    cancel context.CancelFunc
+  )
+
+  ctx, cancel = context.WithCancel(context.Background())
+  defer cancel()
+
+  addrs, err := net.DefaultResolver.LookupIPAddr(ctx, domainName)
+  if err != nil {
+    return nil, err
+  }
+
+  for _, ipnet := range addrs {
+    if ipnet.IP.To4() != nil {
+      ipAddresses = append(ipAddresses, ipnet.IP.String())
+    }
+  }
+
+  return ipAddresses, nil
+}
+
+// return whether given domain name resolves to the host IP or not
+func checkHostDomainName(domainName string) error{
+  // if host machine is ec2 instance,
+  // public IP must be got from instance meta service of AWS EC2
+  hostIPAddress, err := getPublicEC2IP()
+  if err != nil {
+    return errors.New("Error while getting host ip addresses")
+  }
+
+  domainIPAdresses, err := getDomainNameIP(domainName)
+  if err != nil {
+    return errors.New("Error while getting ip addresses of domain")
+  }
+
+  for _, domainIP := range domainIPAdresses {
+    if domainIP == hostIPAddress {
+      return nil
+    }
+  }
+
+  return errors.New("Given domain name does not redirect to host")
+}

provisioning/resources/ui/js/components/ControlPlane.tsx

@@ -21,6 +21,7 @@ import ReactDOM = require("react-dom");
 import axios from 'axios';
 import RestartServicesSection from "./ControlPlaneComponents/RestartServices";
 import UploadEnrichmentsForm from "./ControlPlaneComponents/UploadEnrichments";
+import AddExternalIgluServerForm from "./ControlPlaneComponents/AddExternalIgluServer";
 
 export class ControlPlane extends React.Component<{}, {}> {
 
@@ -29,7 +30,8 @@ export class ControlPlane extends React.Component<{}, {}> {
       <div className="tab-content">
         <p>The buttons below can be used to interact with the internal systems of Snowplow Mini:</p>
         <RestartServicesSection />
-        <UploadEnrichmentsForm /> 
+        <UploadEnrichmentsForm />
+        <AddExternalIgluServerForm />
       </div>
     );
   }

provisioning/resources/ui/js/components/ControlPlaneComponents/AddExternalIgluServer.tsx

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2016-2017 Snowplow Analytics Ltd. All rights reserved.
+ *
+ * This program is licensed to you under the Apache License Version 2.0,
+ * and you may not use this file except in compliance with the Apache License Version 2.0.
+ * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the Apache License Version 2.0 is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+ */
+
+/// <reference path="../../../typings/node/node.d.ts" />
+/// <reference path="../../../typings/react/react.d.ts" />
+/// <reference path="../../../typings/react/react-dom.d.ts" />
+/// <reference path="../.././Interfaces.d.ts"/>
+
+import React = require('react');
+import ReactDOM = require("react-dom");
+import axios from 'axios';
+
+export default React.createClass({
+  getInitialState () {
+    return {
+      iglu_server_uri: '',
+      iglu_server_apikey: '',
+      disabled: false
+    };
+  },
+
+  handleChange(evt) {
+    if (evt.target.name == 'iglu_server_uri'){
+      this.setState({
+        iglu_server_uri: evt.target.value
+      });
+    }
+    else if (evt.target.name == 'iglu_server_apikey'){
+      this.setState({
+        iglu_server_apikey: evt.target.value
+      });
+    }
+  },
+
+  sendFormData()  {
+    var _this = this
+
+    var igluServerUri = this.state.iglu_server_uri
+    var igluServerApikey = this.state.iglu_server_apikey
+
+    function setInitState() {
+      _this.setState({
+        iglu_server_uri: "",
+        iglu_server_apikey: "",
+        disabled: false
+      });
+    }
+
+    _this.setState({
+      disabled: true
+    });
+    var params = new URLSearchParams();
+    params.append('iglu_server_uri', _this.state.iglu_server_uri)
+    params.append('iglu_server_apikey', _this.state.iglu_server_apikey)
+
+    axios.defaults.headers.post['Content-Type'] = 'application/x-www-form-urlencoded'
+    axios.post('/control-plane/add-external-iglu-server', params, {})
+    .then(function (response) {
+      setInitState()
+      alert('Uploaded successfully');
+    })
+    .catch(function (error) {
+      setInitState()
+      alert('Error: ' + error.response.data);
+    });
+  },
+
+  handleSubmit(event) {
+    alert('Please wait...');
+    event.preventDefault();
+    this.sendFormData();
+  },
+
+  render() {
+    return (
+      <div className="tab-content">
+        <h4>Add external Iglu Server: </h4>
+        <form action="" onSubmit={this.handleSubmit}>
+          <div className="form-group">
+            <label htmlFor="iglu_server_uri">Iglu Server URI: </label>
+            <input className="form-control" name="iglu_server_uri" ref="iglu_server_uri" required type="text" onChange={this.handleChange} value={this.state.iglu_server_uri} />
+          </div>
+          <div className="form-group">
+            <label htmlFor="iglu_server_apikey">Iglu Server ApiKey: </label>
+            <input className="form-control" name="iglu_server_apikey" ref="iglu_server_apikey" required type="text" onChange={this.handleChange} value={this.state.iglu_server_apikey}/>
+          </div>
+          <div className="form-group">
+            <button className="btn btn-primary" type="submit" disabled={this.state.disabled}>Add External Iglu Server</button>
+          </div>
+        </form>
+      </div>
+    );
+  }
+});