diff --git a/Vagrantfile b/Vagrantfile index f22057aa..94d15bf3 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,7 +15,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision "shell" do |s| s.path = "deploy/provision.sh" s.args = [ - "--nginx", "--postgres", "--enroll", "--all-hostname", + "--nginx", "--postgres", "--redis", "--enroll", "--all-hostname", IP_ADDRESS, "--password", "admin" ] privileged = false diff --git a/admin/handlers/handlers.go b/admin/handlers/handlers.go index 1e0570c7..bdb76bb0 100644 --- a/admin/handlers/handlers.go +++ b/admin/handlers/handlers.go @@ -2,9 +2,9 @@ package handlers import ( "github.com/jmpsec/osctrl/admin/sessions" + "github.com/jmpsec/osctrl/cache" "github.com/jmpsec/osctrl/carves" "github.com/jmpsec/osctrl/environments" - "github.com/jmpsec/osctrl/logging" "github.com/jmpsec/osctrl/metrics" "github.com/jmpsec/osctrl/nodes" "github.com/jmpsec/osctrl/queries" @@ -44,7 +44,7 @@ type HandlersAdmin struct { Carves *carves.Carves Settings *settings.Settings Metrics *metrics.Metrics - LoggerDB *logging.LoggerDB + RedisCache *cache.RedisManager Sessions *sessions.SessionManager ServiceVersion string TemplatesFolder string @@ -108,9 +108,9 @@ func WithMetrics(metrics *metrics.Metrics) HandlersOption { } } -func WithLoggerDB(logger *logging.LoggerDB) HandlersOption { +func WithCache(rds *cache.RedisManager) HandlersOption { return func(h *HandlersAdmin) { - h.LoggerDB = logger + h.RedisCache = rds } } diff --git a/admin/handlers/json-logs.go b/admin/handlers/json-logs.go index dd43b916..7eab3ad5 100644 --- a/admin/handlers/json-logs.go +++ b/admin/handlers/json-logs.go @@ -1,6 +1,7 @@ package handlers import ( + "encoding/json" "log" "net/http" "strconv" @@ -8,6 +9,7 @@ import ( "github.com/gorilla/mux" "github.com/jmpsec/osctrl/admin/sessions" "github.com/jmpsec/osctrl/settings" + "github.com/jmpsec/osctrl/types" "github.com/jmpsec/osctrl/users" "github.com/jmpsec/osctrl/utils" ) @@ -15,8 +17,8 @@ import ( // Define log types to be used var ( LogTypes = map[string]bool{ - "result": true, - "status": true, + types.ResultLog: true, + types.StatusLog: true, } ) @@ -122,8 +124,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request) } // Get logs logJSON := []LogJSON{} - if logType == "status" && h.LoggerDB != nil { - statusLogs, err := h.LoggerDB.StatusLogs(UUID, env.Name, secondsBack) + if logType == types.StatusLog && h.RedisCache != nil { + statusLogs, err := h.RedisCache.StatusLogs(UUID, env.Name, secondsBack) if err != nil { log.Printf("error getting logs %v", err) h.Inc(metricJSONErr) @@ -132,8 +134,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request) // Prepare data to be returned for _, s := range statusLogs { _c := CreationTimes{ - Display: utils.PastFutureTimes(s.CreatedAt), - Timestamp: utils.TimeTimestamp(s.CreatedAt), + Display: utils.PastFutureTimesEpoch(int64(s.UnixTime)), + Timestamp: strconv.Itoa(int(s.UnixTime)), } _l := LogJSON{ Created: _c, @@ -142,8 +144,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request) } logJSON = append(logJSON, _l) } - } else if logType == "result" && h.LoggerDB != nil { - resultLogs, err := h.LoggerDB.ResultLogs(UUID, env.Name, secondsBack) + } else if logType == types.ResultLog && h.RedisCache != nil { + resultLogs, err := h.RedisCache.ResultLogs(UUID, env.Name, secondsBack) if err != nil { log.Printf("error getting logs %v", err) h.Inc(metricJSONErr) @@ -153,8 +155,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request) for _, r := range resultLogs { _l := LogJSON{ Created: CreationTimes{ - Display: utils.PastFutureTimes(r.CreatedAt), - Timestamp: utils.TimeTimestamp(r.CreatedAt), + Display: utils.PastFutureTimesEpoch(int64(r.UnixTime)), + Timestamp: strconv.Itoa(int(r.UnixTime)), }, First: r.Name, Second: string(r.Columns), @@ -191,10 +193,11 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ h.Inc(metricJSONErr) return } - // Get logs + // Iterate through targets to get logs queryLogJSON := []QueryLogJSON{} - if h.LoggerDB != nil { - queryLogs, err := h.LoggerDB.QueryLogs(name) + // Get logs + if h.RedisCache != nil { + queryLogs, err := h.RedisCache.QueryLogs(name) if err != nil { log.Printf("error getting logs %v", err) h.Inc(metricJSONErr) @@ -203,14 +206,20 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ // Prepare data to be returned for _, q := range queryLogs { // Get target node - node, err := h.Nodes.GetByUUID(q.UUID) + node, err := h.Nodes.GetByUUID(q.HostIdentifier) if err != nil { - node.UUID = q.UUID + node.UUID = q.HostIdentifier node.Localname = "" } _c := CreationTimes{ - Display: utils.PastFutureTimes(q.CreatedAt), - Timestamp: utils.TimeTimestamp(q.CreatedAt), + Display: utils.PastFutureTimesEpoch(int64(q.UnixTime)), + Timestamp: utils.PastFutureTimesEpoch(int64(q.UnixTime)), + } + qData, err := json.Marshal(q.QueryData) + if err != nil { + log.Printf("error serializing logs %v", err) + h.Inc(metricJSONErr) + continue } _l := QueryLogJSON{ Created: _c, @@ -218,7 +227,7 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ UUID: node.UUID, Name: node.Localname, }, - Data: string(q.Data), + Data: string(qData), } queryLogJSON = append(queryLogJSON, _l) } diff --git a/admin/handlers/templates.go b/admin/handlers/templates.go index 76623168..95f65e7a 100644 --- a/admin/handlers/templates.go +++ b/admin/handlers/templates.go @@ -920,8 +920,6 @@ func (h *HandlersAdmin) NodeHandler(w http.ResponseWriter, r *http.Request) { funcMap := template.FuncMap{ "pastFutureTimes": utils.PastFutureTimes, "jsonRawIndent": jsonRawIndent, - "statusLogsLink": h.statusLogsLink, - "resultLogsLink": h.resultLogsLink, } // Prepare template tempateFiles := NewTemplateFiles(h.TemplatesFolder, "node.html").filepaths diff --git a/admin/handlers/utils.go b/admin/handlers/utils.go index 8c3d849f..b0266764 100644 --- a/admin/handlers/utils.go +++ b/admin/handlers/utils.go @@ -20,6 +20,12 @@ import ( "github.com/jmpsec/osctrl/utils" ) +const ( + QueryLink string = "/query/logs/{{NAME}}" + StatusLink string = "#status-logs" + ResultsLink string = "#result-logs" +) + // Helper to handle admin error responses func adminErrorResponse(w http.ResponseWriter, msg string, code int, err error) { log.Printf("%s: %v", msg, err) @@ -187,17 +193,7 @@ func toJSONConfigurationService(values []settings.SettingValue) types.JSONConfig // Helper to generate a link to results for on-demand queries func (h *HandlersAdmin) queryResultLink(name string) string { - return strings.Replace(h.Settings.QueryResultLink(), "{{NAME}}", removeBackslash(name), 1) -} - -// Helper to generate a link to results for status logs -func (h *HandlersAdmin) statusLogsLink(uuid string) string { - return strings.Replace(h.Settings.StatusLogsLink(), "{{UUID}}", removeBackslash(uuid), 1) -} - -// Helper to generate a link to results for result logs -func (h *HandlersAdmin) resultLogsLink(uuid string) string { - return strings.Replace(h.Settings.ResultLogsLink(), "{{UUID}}", removeBackslash(uuid), 1) + return strings.Replace(QueryLink, "{{NAME}}", removeBackslash(name), 1) } // Helper to convert the list of all TLS environments with the ones with permissions for a user diff --git a/admin/main.go b/admin/main.go index 72a25e6c..7c597873 100644 --- a/admin/main.go +++ b/admin/main.go @@ -14,9 +14,9 @@ import ( "github.com/jmpsec/osctrl/admin/handlers" "github.com/jmpsec/osctrl/admin/sessions" "github.com/jmpsec/osctrl/backend" + "github.com/jmpsec/osctrl/cache" "github.com/jmpsec/osctrl/carves" "github.com/jmpsec/osctrl/environments" - "github.com/jmpsec/osctrl/logging" "github.com/jmpsec/osctrl/metrics" "github.com/jmpsec/osctrl/nodes" "github.com/jmpsec/osctrl/queries" @@ -69,6 +69,8 @@ const ( defConfigurationFile string = "config/" + settings.ServiceAdmin + ".json" // Default DB configuration file defDBConfigurationFile string = "config/db.json" + // Default redis configuration file + defRedisConfigurationFile string = "config/redis.json" // Default Logger configuration file defLoggerConfigurationFile string = "config/logger.json" // Default TLS certificate file @@ -87,8 +89,6 @@ const ( defaultRefresh int = 300 // Default hours to classify nodes as inactive defaultInactive int = -72 - // Hourly interval to cleanup logs - hourlyInterval int = 60 ) // osquery @@ -109,7 +109,9 @@ var ( err error adminConfig types.JSONConfigurationService dbConfig backend.JSONConfigurationDB + redisConfig cache.JSONConfigurationRedis db *backend.DBManager + redis *cache.RedisManager settingsmgr *settings.Settings nodesmgr *nodes.NodeManager queriesmgr *queries.Queries @@ -124,15 +126,16 @@ var ( osqueryTables []types.OsqueryTable adminMetrics *metrics.Metrics handlersAdmin *handlers.HandlersAdmin - loggerDB *logging.LoggerDB ) // Variables for flags var ( configFlag bool - configFile string dbFlag bool + redisFlag bool + serviceConfigFile string dbConfigFile string + redisConfigFile string tlsServer bool tlsCertFile string tlsKeyFile string @@ -221,7 +224,7 @@ func init() { Value: defConfigurationFile, Usage: "Load service configuration from `FILE`", EnvVars: []string{"SERVICE_CONFIG_FILE"}, - Destination: &configFile, + Destination: &serviceConfigFile, }, &cli.StringFlag{ Name: "listener", @@ -263,6 +266,71 @@ func init() { EnvVars: []string{"SERVICE_LOGGER"}, Destination: &adminConfig.Logger, }, + &cli.BoolFlag{ + Name: "redis", + Aliases: []string{"r"}, + Value: false, + Usage: "Provide redis configuration via JSON file", + EnvVars: []string{"REDIS_CONFIG"}, + Destination: &redisFlag, + }, + &cli.StringFlag{ + Name: "redis-file", + Aliases: []string{"R"}, + Value: defRedisConfigurationFile, + Usage: "Load redis configuration from `FILE`", + EnvVars: []string{"REDIS_CONFIG_FILE"}, + Destination: &redisConfigFile, + }, + &cli.StringFlag{ + Name: "redis-host", + Value: "127.0.0.1", + Usage: "Redis host to be connected to", + EnvVars: []string{"REDIS_HOST"}, + Destination: &redisConfig.Host, + }, + &cli.StringFlag{ + Name: "redis-port", + Value: "6379", + Usage: "Redis port to be connected to", + EnvVars: []string{"REDIS_PORT"}, + Destination: &redisConfig.Port, + }, + &cli.StringFlag{ + Name: "redis-pass", + Value: "redis", + Usage: "Password to be used for redis", + EnvVars: []string{"REDIS_PASS"}, + Destination: &redisConfig.Password, + }, + &cli.IntFlag{ + Name: "redis-db", + Value: 0, + Usage: "Redis database to be selected after connecting", + EnvVars: []string{"REDIS_DB"}, + Destination: &redisConfig.DB, + }, + &cli.IntFlag{ + Name: "redis-status-exp", + Value: cache.StatusExpiration, + Usage: "Redis expiration in hours for status logs", + EnvVars: []string{"REDIS_STATUS_EXP"}, + Destination: &redisConfig.StatusExpirationHours, + }, + &cli.IntFlag{ + Name: "redis-result-exp", + Value: cache.ResultExpiration, + Usage: "Redis expiration in hours for result logs", + EnvVars: []string{"REDIS_RESULT_EXP"}, + Destination: &redisConfig.ResultExpirationHours, + }, + &cli.IntFlag{ + Name: "redis-query-exp", + Value: cache.QueryExpiration, + Usage: "Redis expiration in hours for query logs", + EnvVars: []string{"REDIS_QUERY_EXP"}, + Destination: &redisConfig.QueryExpirationHours, + }, &cli.BoolFlag{ Name: "db", Aliases: []string{"d"}, @@ -458,6 +526,12 @@ func osctrlAdminService() { log.Println("Backend NOT ready! waiting...") time.Sleep(backendWait) } + log.Println("Initializing cache...") + redis, err = cache.CreateRedisManager(redisConfig) + if err != nil { + log.Fatalf("Failed to connect to redis - %v", err) + } + log.Println("Connection to cache successful!") log.Println("Initialize users") adminUsers = users.CreateUserManager(db.Conn, &jwtConfig) log.Println("Initialize tags") @@ -483,17 +557,7 @@ func osctrlAdminService() { if err != nil { log.Fatalf("Error loading metrics - %v", err) } - // TODO Initialize DB logger regardless of settings - // This is temporary until we have logs stored in Redis - if adminConfig.Logger == settings.LoggingDB { - loggerDB, err = logging.CreateLoggerDBFile(loggerFile) - if err != nil { - loggerDB, err = logging.CreateLoggerDBConfig(dbConfig) - if err != nil { - log.Fatalf("Error creating db logger - %v", err) - } - } - } + // Start SAML Middleware if we are using SAML if adminConfig.Auth == settings.AuthSAML { if settingsmgr.DebugService(settings.ServiceAdmin) { @@ -532,43 +596,6 @@ func osctrlAdminService() { } }() - // Cleaning up status/result/query logs - go func() { - for { - _e, err := envs.All() - if err != nil { - log.Printf("error getting environments when cleaning up logs - %v", err) - } - for _, e := range _e { - if settingsmgr.CleanStatusLogs() { - if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Cleaning up status logs") - } - if err := loggerDB.CleanStatusLogs(e.Name, settingsmgr.CleanStatusInterval()); err != nil { - log.Printf("error cleaning up status logs - %v", err) - } - } - if settingsmgr.CleanResultLogs() { - if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Cleaning up result logs") - } - if err := loggerDB.CleanResultLogs(e.Name, settingsmgr.CleanResultInterval()); err != nil { - log.Printf("error cleaning up result logs - %v", err) - } - } - } - if settingsmgr.CleanQueryLogs() { - if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Cleaning up query logs") - } - if err := loggerDB.CleanQueryLogs(settingsmgr.CleanQueryEntries()); err != nil { - log.Printf("error cleaning up query logs - %v", err) - } - } - time.Sleep(time.Duration(hourlyInterval) * time.Second) - } - }() - // Initialize Admin handlers before router handlersAdmin = handlers.CreateHandlersAdmin( handlers.WithDB(db.Conn), @@ -580,7 +607,7 @@ func osctrlAdminService() { handlers.WithCarves(carvesmgr), handlers.WithSettings(settingsmgr), handlers.WithMetrics(adminMetrics), - handlers.WithLoggerDB(loggerDB), + handlers.WithCache(redis), handlers.WithSessions(sessionsmgr), handlers.WithVersion(serviceVersion), handlers.WithTemplates(templatesFolder), @@ -737,14 +764,21 @@ func osctrlAdminService() { // Action to run when no flags are provided to run checks and prepare data func cliAction(c *cli.Context) error { - // Load configuration if external service JSON config file is used + // Load configuration if external JSON config file is used if configFlag { - adminConfig, err = loadConfiguration(configFile, settings.ServiceAdmin) + adminConfig, err = loadConfiguration(serviceConfigFile, settings.ServiceAdmin) + if err != nil { + return fmt.Errorf("Failed to load service configuration %s - %s", serviceConfigFile, err) + } + } + // Load redis configuration if external JSON config file is used + if redisFlag { + redisConfig, err = cache.LoadConfiguration(redisConfigFile, cache.RedisKey) if err != nil { - return fmt.Errorf("Failed to load service configuration %s - %s", configFile, err) + return fmt.Errorf("Failed to load redis configuration - %v", err) } } - // Load DB configuration if external db JSON config file is used + // Load DB configuration if external JSON config file is used if dbFlag { dbConfig, err = backend.LoadConfiguration(dbConfigFile, backend.DBKey) if err != nil { diff --git a/admin/settings.go b/admin/settings.go index e2202683..69f7a2e9 100644 --- a/admin/settings.go +++ b/admin/settings.go @@ -36,29 +36,6 @@ func loadingMetrics(mgr *settings.Settings) (*metrics.Metrics, error) { return nil, nil } -// Function to load the logging settings -func loadingLoggingSettings(mgr *settings.Settings) error { - // Check if logging settings for query results link is ready - if !mgr.IsValue(settings.ServiceAdmin, settings.QueryResultLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.QueryResultLink, settings.QueryLink); err != nil { - return fmt.Errorf("Failed to add %s to settings: %v", settings.QueryResultLink, err) - } - } - // Check if logging settings for status logs link is ready - if !mgr.IsValue(settings.ServiceAdmin, settings.StatusLogsLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.StatusLogsLink, settings.StatusLink); err != nil { - return fmt.Errorf("Failed to add %s to settings: %v", settings.DebugHTTP, err) - } - } - // Check if logging settings for result logs link is ready - if !mgr.IsValue(settings.ServiceAdmin, settings.ResultLogsLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.ResultLogsLink, settings.ResultsLink); err != nil { - return fmt.Errorf("Failed to add %s to settings: %v", settings.DebugHTTP, err) - } - } - return err -} - // Function to load all settings for the service func loadingSettings(mgr *settings.Settings) error { // Check if service settings for debug service is ready @@ -101,9 +78,6 @@ func loadingSettings(mgr *settings.Settings) error { return fmt.Errorf("Failed to add %s to settings: %v", settings.NodeDashboard, err) } } - if err := loadingLoggingSettings(mgr); err != nil { - return fmt.Errorf("Failed to load logging settings: %v", err) - } // Write JSON config to settings if err := mgr.SetAllJSON(settings.ServiceAdmin, adminConfig.Listener, adminConfig.Port, adminConfig.Host, adminConfig.Auth, adminConfig.Logger); err != nil { return fmt.Errorf("Failed to add JSON values to configuration: %v", err) diff --git a/admin/templates/node.html b/admin/templates/node.html index 03f3e12f..74ff671f 100644 --- a/admin/templates/node.html +++ b/admin/templates/node.html @@ -362,7 +362,7 @@