Merge pull request #223 from jmpsec/redis-logs-tls

Using redis for logs

Vagrantfile

@@ -15,7 +15,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.provision "shell" do |s|
     s.path = "deploy/provision.sh"
     s.args = [
-      "--nginx", "--postgres", "--enroll", "--all-hostname",
+      "--nginx", "--postgres", "--redis", "--enroll", "--all-hostname",
       IP_ADDRESS, "--password", "admin"
     ]
     privileged = false

admin/handlers/handlers.go

@@ -2,9 +2,9 @@ package handlers
 
 import (
 	"github.com/jmpsec/osctrl/admin/sessions"
+	"github.com/jmpsec/osctrl/cache"
 	"github.com/jmpsec/osctrl/carves"
 	"github.com/jmpsec/osctrl/environments"
-	"github.com/jmpsec/osctrl/logging"
 	"github.com/jmpsec/osctrl/metrics"
 	"github.com/jmpsec/osctrl/nodes"
 	"github.com/jmpsec/osctrl/queries"
@@ -44,7 +44,7 @@ type HandlersAdmin struct {
 	Carves          *carves.Carves
 	Settings        *settings.Settings
 	Metrics         *metrics.Metrics
-	LoggerDB        *logging.LoggerDB
+	RedisCache      *cache.RedisManager
 	Sessions        *sessions.SessionManager
 	ServiceVersion  string
 	TemplatesFolder string
@@ -108,9 +108,9 @@ func WithMetrics(metrics *metrics.Metrics) HandlersOption {
 	}
 }
 
-func WithLoggerDB(logger *logging.LoggerDB) HandlersOption {
+func WithCache(rds *cache.RedisManager) HandlersOption {
 	return func(h *HandlersAdmin) {
-		h.LoggerDB = logger
+		h.RedisCache = rds
 	}
 }
 

admin/handlers/json-logs.go

@@ -1,22 +1,24 @@
 package handlers
 
 import (
+	"encoding/json"
 	"log"
 	"net/http"
 	"strconv"
 
 	"github.com/gorilla/mux"
 	"github.com/jmpsec/osctrl/admin/sessions"
 	"github.com/jmpsec/osctrl/settings"
+	"github.com/jmpsec/osctrl/types"
 	"github.com/jmpsec/osctrl/users"
 	"github.com/jmpsec/osctrl/utils"
 )
 
 // Define log types to be used
 var (
 	LogTypes = map[string]bool{
-		"result": true,
-		"status": true,
+		types.ResultLog: true,
+		types.StatusLog: true,
 	}
 )
 
@@ -122,8 +124,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request)
 	}
 	// Get logs
 	logJSON := []LogJSON{}
-	if logType == "status" && h.LoggerDB != nil {
-		statusLogs, err := h.LoggerDB.StatusLogs(UUID, env.Name, secondsBack)
+	if logType == types.StatusLog && h.RedisCache != nil {
+		statusLogs, err := h.RedisCache.StatusLogs(UUID, env.Name, secondsBack)
 		if err != nil {
 			log.Printf("error getting logs %v", err)
 			h.Inc(metricJSONErr)
@@ -132,8 +134,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request)
 		// Prepare data to be returned
 		for _, s := range statusLogs {
 			_c := CreationTimes{
-				Display:   utils.PastFutureTimes(s.CreatedAt),
-				Timestamp: utils.TimeTimestamp(s.CreatedAt),
+				Display:   utils.PastFutureTimesEpoch(int64(s.UnixTime)),
+				Timestamp: strconv.Itoa(int(s.UnixTime)),
 			}
 			_l := LogJSON{
 				Created: _c,
@@ -142,8 +144,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request)
 			}
 			logJSON = append(logJSON, _l)
 		}
-	} else if logType == "result" && h.LoggerDB != nil {
-		resultLogs, err := h.LoggerDB.ResultLogs(UUID, env.Name, secondsBack)
+	} else if logType == types.ResultLog && h.RedisCache != nil {
+		resultLogs, err := h.RedisCache.ResultLogs(UUID, env.Name, secondsBack)
 		if err != nil {
 			log.Printf("error getting logs %v", err)
 			h.Inc(metricJSONErr)
@@ -153,8 +155,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request)
 		for _, r := range resultLogs {
 			_l := LogJSON{
 				Created: CreationTimes{
-					Display:   utils.PastFutureTimes(r.CreatedAt),
-					Timestamp: utils.TimeTimestamp(r.CreatedAt),
+					Display:   utils.PastFutureTimesEpoch(int64(r.UnixTime)),
+					Timestamp: strconv.Itoa(int(r.UnixTime)),
 				},
 				First:  r.Name,
 				Second: string(r.Columns),
@@ -191,10 +193,11 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ
 		h.Inc(metricJSONErr)
 		return
 	}
-	// Get logs
+	// Iterate through targets to get logs
 	queryLogJSON := []QueryLogJSON{}
-	if h.LoggerDB != nil {
-		queryLogs, err := h.LoggerDB.QueryLogs(name)
+	// Get logs
+	if h.RedisCache != nil {
+		queryLogs, err := h.RedisCache.QueryLogs(name)
 		if err != nil {
 			log.Printf("error getting logs %v", err)
 			h.Inc(metricJSONErr)
@@ -203,22 +206,28 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ
 		// Prepare data to be returned
 		for _, q := range queryLogs {
 			// Get target node
-			node, err := h.Nodes.GetByUUID(q.UUID)
+			node, err := h.Nodes.GetByUUID(q.HostIdentifier)
 			if err != nil {
-				node.UUID = q.UUID
+				node.UUID = q.HostIdentifier
 				node.Localname = ""
 			}
 			_c := CreationTimes{
-				Display:   utils.PastFutureTimes(q.CreatedAt),
-				Timestamp: utils.TimeTimestamp(q.CreatedAt),
+				Display:   utils.PastFutureTimesEpoch(int64(q.UnixTime)),
+				Timestamp: utils.PastFutureTimesEpoch(int64(q.UnixTime)),
+			}
+			qData, err := json.Marshal(q.QueryData)
+			if err != nil {
+				log.Printf("error serializing logs %v", err)
+				h.Inc(metricJSONErr)
+				continue
 			}
 			_l := QueryLogJSON{
 				Created: _c,
 				Target: QueryTargetNode{
 					UUID: node.UUID,
 					Name: node.Localname,
 				},
-				Data: string(q.Data),
+				Data: string(qData),
 			}
 			queryLogJSON = append(queryLogJSON, _l)
 		}

admin/handlers/templates.go

@@ -920,8 +920,6 @@ func (h *HandlersAdmin) NodeHandler(w http.ResponseWriter, r *http.Request) {
 	funcMap := template.FuncMap{
 		"pastFutureTimes": utils.PastFutureTimes,
 		"jsonRawIndent":   jsonRawIndent,
-		"statusLogsLink":  h.statusLogsLink,
-		"resultLogsLink":  h.resultLogsLink,
 	}
 	// Prepare template
 	tempateFiles := NewTemplateFiles(h.TemplatesFolder, "node.html").filepaths

admin/handlers/utils.go

@@ -20,6 +20,12 @@ import (
 	"github.com/jmpsec/osctrl/utils"
 )
 
+const (
+	QueryLink   string = "/query/logs/{{NAME}}"
+	StatusLink  string = "#status-logs"
+	ResultsLink string = "#result-logs"
+)
+
 // Helper to handle admin error responses
 func adminErrorResponse(w http.ResponseWriter, msg string, code int, err error) {
 	log.Printf("%s: %v", msg, err)
@@ -187,17 +193,7 @@ func toJSONConfigurationService(values []settings.SettingValue) types.JSONConfig
 
 // Helper to generate a link to results for on-demand queries
 func (h *HandlersAdmin) queryResultLink(name string) string {
-	return strings.Replace(h.Settings.QueryResultLink(), "{{NAME}}", removeBackslash(name), 1)
-}
-
-// Helper to generate a link to results for status logs
-func (h *HandlersAdmin) statusLogsLink(uuid string) string {
-	return strings.Replace(h.Settings.StatusLogsLink(), "{{UUID}}", removeBackslash(uuid), 1)
-}
-
-// Helper to generate a link to results for result logs
-func (h *HandlersAdmin) resultLogsLink(uuid string) string {
-	return strings.Replace(h.Settings.ResultLogsLink(), "{{UUID}}", removeBackslash(uuid), 1)
+	return strings.Replace(QueryLink, "{{NAME}}", removeBackslash(name), 1)
 }
 
 // Helper to convert the list of all TLS environments with the ones with permissions for a user