Fetched API key from env file (razor-network#1048)

* Added dot env file utils

* Fetched API key from .env file when required

* updated dependencies

* Changed API key regex

* Added name field in customJob struct

* Replaced content-type field with headers in dataSourceURLStruct

* Updated post request creation

* picked API Regex from constants.go

* Removed unwanted field from header

* returned error as well from GetkeywordAndAPIKey

* replaces regexp.Match() with strings.Contains()

* support for headers in GET requests

* Refactored GetDataFromAPI

* Added processRequest retry constants

README.md

@@ -873,6 +873,10 @@ Shown below is an example of how your `assets.json` file should be -
 							"body": {},
 							"content-type": ""
 						},
+						 "name:" "eth_lunarCrush",
+						 "selector": "[`data`][`0`][`price`]",
+                         "power": 3,
+                         "weight": 2
 					]
 				}
 			}

core/constants.go

@@ -47,6 +47,15 @@ var AssetsDataFile = "assets.json"
 var ConfigFile = "razor.yaml"
 var LogFileDirectory = "logs"
 var DefaultPathName = ".razor"
+var DotENVFileName = ".env"
 
 //BlockNumberInterval is the interval in seconds after which blockNumber needs to be calculated again
 var BlockNumberInterval = 5
+
+//APIKeyRegex will be used as a regular expression to be matched in job Urls
+var APIKeyRegex = `$`
+
+// Following are the constants which defines retry attempts and retry delay if there is an error in processing request
+
+var ProcessRequestRetryAttempts uint = 2
+var ProcessRequestRetryDelay = 2

core/types/assets.go

@@ -51,14 +51,15 @@ type AssignedAsset struct {
 
 type CustomJob struct {
 	URL      string `json:"URL"`
+	Name     string `json:"name"`
 	Selector string `json:"selector"`
 	Power    int8   `json:"power"`
 	Weight   uint8  `json:"weight"`
 }
 
 type DataSourceURL struct {
-	Type        string                 `json:"type"`
-	URL         string                 `json:"url"`
-	Body        map[string]interface{} `json:"body"`
-	ContentType string                 `json:"content-type"`
+	Type   string                 `json:"type"`
+	URL    string                 `json:"url"`
+	Body   map[string]interface{} `json:"body"`
+	Header map[string]string      `json:"header"`
 }

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/joho/godotenv v1.4.0 // indirect
 	github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a // indirect

go.sum

@@ -344,6 +344,8 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1C
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=

path/path.go

@@ -113,3 +113,13 @@ func (PathUtils) GetDisputeDataFileName(address string) (string, error) {
 	}
 	return filepath.Join(dataFileDir, address+core.DisputeDataFile), nil
 }
+
+//This function returns the job file path
+func (PathUtils) GetDotENVFilePath() (string, error) {
+	razorPath, err := PathUtilsInterface.GetDefaultPath()
+	if err != nil {
+		return "", err
+	}
+	filePath := filepath.Join(razorPath, core.DotENVFileName)
+	return filePath, nil
+}

path/pathUtils.go

@@ -20,6 +20,7 @@ type PathInterface interface {
 	GetCommitDataFileName(address string) (string, error)
 	GetProposeDataFileName(address string) (string, error)
 	GetDisputeDataFileName(address string) (string, error)
+	GetDotENVFilePath() (string, error)
 }
 
 type OSInterface interface {

utils/api.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"io"
 	"net/http"
 	"razor/cache"
 	"razor/core"
+	"strings"
 	"time"
 
-	"io/ioutil"
 	"razor/core/types"
 
 	"github.com/PaesslerAG/jsonpath"
@@ -23,62 +24,51 @@ func (*UtilsStruct) GetDataFromAPI(dataSourceURLStruct types.DataSourceURL, loca
 	}
 	cachedData, cachedErr := localCache.Read(dataSourceURLStruct.URL)
 	if cachedErr != nil {
-		var body []byte
-		if dataSourceURLStruct.Type == "GET" {
+		var response []byte
+		switch dataSourceURLStruct.Type {
+		case "GET":
 			err := retry.Do(
 				func() error {
-					response, err := client.Get(dataSourceURLStruct.URL)
-					if err != nil {
-						return err
-					}
-					defer response.Body.Close()
-					if response.StatusCode != 200 {
-						log.Errorf("API: %s responded with status code %d", dataSourceURLStruct.URL, response.StatusCode)
-						return errors.New("unable to reach API")
-					}
-					body, err = IOInterface.ReadAll(response.Body)
+					responseBody, err := ProcessRequest(client, dataSourceURLStruct, nil)
 					if err != nil {
+						log.Error("Error in processing GET request: ", err)
 						return err
 					}
+					response = responseBody
 					return nil
-				}, retry.Attempts(2), retry.Delay(time.Second*2))
+				}, retry.Attempts(core.ProcessRequestRetryAttempts), retry.Delay(time.Second*time.Duration(core.ProcessRequestRetryDelay)))
 			if err != nil {
 				return nil, err
 			}
-		}
-		if dataSourceURLStruct.Type == "POST" {
+		case "POST":
 			postBody, err := json.Marshal(dataSourceURLStruct.Body)
 			if err != nil {
 				log.Errorf("Error in marshalling body of a POST request URL %s: %v", dataSourceURLStruct.URL, err)
+				return nil, err
 			}
-			responseBody := bytes.NewBuffer(postBody)
+			requestBody := bytes.NewBuffer(postBody)
 			err = retry.Do(
 				func() error {
-					response, err := client.Post(dataSourceURLStruct.URL, dataSourceURLStruct.ContentType, responseBody)
-					if err != nil {
-						log.Errorf("Error sending POST request URL %s: %v", dataSourceURLStruct.URL, err)
-						return err
-					}
-					defer response.Body.Close()
-					if response.StatusCode != 200 {
-						log.Errorf("URL: %s responded with status code %d", dataSourceURLStruct.URL, response.StatusCode)
-						return errors.New("unable to reach API")
-					}
-					body, err = ioutil.ReadAll(response.Body)
+					responseBody, err := ProcessRequest(client, dataSourceURLStruct, requestBody)
 					if err != nil {
+						log.Error("Error in processing POST request: ", err)
 						return err
 					}
+					response = responseBody
 					return nil
-				}, retry.Attempts(2), retry.Delay(time.Second*2))
+				}, retry.Attempts(core.ProcessRequestRetryAttempts), retry.Delay(time.Second*time.Duration(core.ProcessRequestRetryDelay)))
 			if err != nil {
 				return nil, err
 			}
+		default:
+			return nil, errors.New("invalid request type")
 		}
+
 		dataToCache := cache.Data{
-			Result: body,
+			Result: response,
 		}
 		localCache.Update(dataToCache, dataSourceURLStruct.URL, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix())
-		return body, nil
+		return response, nil
 	}
 	log.Debugf("Getting Data for URL %s from local cache...", dataSourceURLStruct.URL)
 	return cachedData.Result, nil
@@ -105,3 +95,47 @@ func (*UtilsStruct) GetDataFromXHTML(dataSourceURLStruct types.DataSourceURL, se
 	}
 	return priceData, nil
 }
+
+func AddHeaderToRequest(request *http.Request, headerMap map[string]string) (*http.Request, error) {
+	for key, value := range headerMap {
+		// If core.APIKeyRegex = `$` and if value starts with '$' then we need to fetch the respective value from env file
+		if strings.HasPrefix(value, core.APIKeyRegex) {
+			_, APIKey, err := GetKeyWordAndAPIKeyFromENVFile(value)
+			if err != nil {
+				log.Error("Error in getting value from env file: ", err)
+				return nil, err
+			}
+			value = APIKey
+		}
+		log.Debugf("Adding key: %s, value: %s pair to header", key, value)
+		request.Header.Add(key, value)
+	}
+	return request, nil
+}
+
+func ProcessRequest(client http.Client, dataSourceURLStruct types.DataSourceURL, requestBody io.Reader) ([]byte, error) {
+	request, err := http.NewRequest(dataSourceURLStruct.Type, dataSourceURLStruct.URL, requestBody)
+	if err != nil {
+		return nil, err
+	}
+	requestWithHeader, err := AddHeaderToRequest(request, dataSourceURLStruct.Header)
+	if err != nil {
+		log.Errorf("Error in adding header to %s request: %v", dataSourceURLStruct.Type, err)
+		return nil, err
+	}
+	response, err := client.Do(requestWithHeader)
+	if err != nil {
+		log.Errorf("Error sending %s request URL %s: %v", dataSourceURLStruct.Type, dataSourceURLStruct.URL, err)
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != 200 {
+		log.Errorf("API: %s responded with status code %d", dataSourceURLStruct.URL, response.StatusCode)
+		return nil, errors.New("unable to reach API")
+	}
+	responseBody, err := io.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+	return responseBody, nil
+}