diff --git a/Vagrantfile b/Vagrantfile index f22057aa..94d15bf3 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,7 +15,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provision "shell" do |s| s.path = "deploy/provision.sh" s.args = [ - "--nginx", "--postgres", "--enroll", "--all-hostname", + "--nginx", "--postgres", "--redis", "--enroll", "--all-hostname", IP_ADDRESS, "--password", "admin" ] privileged = false diff --git a/admin/handlers/handlers.go b/admin/handlers/handlers.go index 1e0570c7..bdb76bb0 100644 --- a/admin/handlers/handlers.go +++ b/admin/handlers/handlers.go @@ -2,9 +2,9 @@ package handlers import ( "github.com/jmpsec/osctrl/admin/sessions" + "github.com/jmpsec/osctrl/cache" "github.com/jmpsec/osctrl/carves" "github.com/jmpsec/osctrl/environments" - "github.com/jmpsec/osctrl/logging" "github.com/jmpsec/osctrl/metrics" "github.com/jmpsec/osctrl/nodes" "github.com/jmpsec/osctrl/queries" @@ -44,7 +44,7 @@ type HandlersAdmin struct { Carves *carves.Carves Settings *settings.Settings Metrics *metrics.Metrics - LoggerDB *logging.LoggerDB + RedisCache *cache.RedisManager Sessions *sessions.SessionManager ServiceVersion string TemplatesFolder string @@ -108,9 +108,9 @@ func WithMetrics(metrics *metrics.Metrics) HandlersOption { } } -func WithLoggerDB(logger *logging.LoggerDB) HandlersOption { +func WithCache(rds *cache.RedisManager) HandlersOption { return func(h *HandlersAdmin) { - h.LoggerDB = logger + h.RedisCache = rds } } diff --git a/admin/handlers/json-logs.go b/admin/handlers/json-logs.go index dd43b916..7eab3ad5 100644 --- a/admin/handlers/json-logs.go +++ b/admin/handlers/json-logs.go @@ -1,6 +1,7 @@ package handlers import ( + "encoding/json" "log" "net/http" "strconv" @@ -8,6 +9,7 @@ import ( "github.com/gorilla/mux" "github.com/jmpsec/osctrl/admin/sessions" "github.com/jmpsec/osctrl/settings" + "github.com/jmpsec/osctrl/types" "github.com/jmpsec/osctrl/users" "github.com/jmpsec/osctrl/utils" ) @@ -15,8 +17,8 @@ import ( // Define log types to be used var ( LogTypes = map[string]bool{ - "result": true, - "status": true, + types.ResultLog: true, + types.StatusLog: true, } ) @@ -122,8 +124,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request) } // Get logs logJSON := []LogJSON{} - if logType == "status" && h.LoggerDB != nil { - statusLogs, err := h.LoggerDB.StatusLogs(UUID, env.Name, secondsBack) + if logType == types.StatusLog && h.RedisCache != nil { + statusLogs, err := h.RedisCache.StatusLogs(UUID, env.Name, secondsBack) if err != nil { log.Printf("error getting logs %v", err) h.Inc(metricJSONErr) @@ -132,8 +134,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request) // Prepare data to be returned for _, s := range statusLogs { _c := CreationTimes{ - Display: utils.PastFutureTimes(s.CreatedAt), - Timestamp: utils.TimeTimestamp(s.CreatedAt), + Display: utils.PastFutureTimesEpoch(int64(s.UnixTime)), + Timestamp: strconv.Itoa(int(s.UnixTime)), } _l := LogJSON{ Created: _c, @@ -142,8 +144,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request) } logJSON = append(logJSON, _l) } - } else if logType == "result" && h.LoggerDB != nil { - resultLogs, err := h.LoggerDB.ResultLogs(UUID, env.Name, secondsBack) + } else if logType == types.ResultLog && h.RedisCache != nil { + resultLogs, err := h.RedisCache.ResultLogs(UUID, env.Name, secondsBack) if err != nil { log.Printf("error getting logs %v", err) h.Inc(metricJSONErr) @@ -153,8 +155,8 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request) for _, r := range resultLogs { _l := LogJSON{ Created: CreationTimes{ - Display: utils.PastFutureTimes(r.CreatedAt), - Timestamp: utils.TimeTimestamp(r.CreatedAt), + Display: utils.PastFutureTimesEpoch(int64(r.UnixTime)), + Timestamp: strconv.Itoa(int(r.UnixTime)), }, First: r.Name, Second: string(r.Columns), @@ -191,10 +193,11 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ h.Inc(metricJSONErr) return } - // Get logs + // Iterate through targets to get logs queryLogJSON := []QueryLogJSON{} - if h.LoggerDB != nil { - queryLogs, err := h.LoggerDB.QueryLogs(name) + // Get logs + if h.RedisCache != nil { + queryLogs, err := h.RedisCache.QueryLogs(name) if err != nil { log.Printf("error getting logs %v", err) h.Inc(metricJSONErr) @@ -203,14 +206,20 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ // Prepare data to be returned for _, q := range queryLogs { // Get target node - node, err := h.Nodes.GetByUUID(q.UUID) + node, err := h.Nodes.GetByUUID(q.HostIdentifier) if err != nil { - node.UUID = q.UUID + node.UUID = q.HostIdentifier node.Localname = "" } _c := CreationTimes{ - Display: utils.PastFutureTimes(q.CreatedAt), - Timestamp: utils.TimeTimestamp(q.CreatedAt), + Display: utils.PastFutureTimesEpoch(int64(q.UnixTime)), + Timestamp: utils.PastFutureTimesEpoch(int64(q.UnixTime)), + } + qData, err := json.Marshal(q.QueryData) + if err != nil { + log.Printf("error serializing logs %v", err) + h.Inc(metricJSONErr) + continue } _l := QueryLogJSON{ Created: _c, @@ -218,7 +227,7 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ UUID: node.UUID, Name: node.Localname, }, - Data: string(q.Data), + Data: string(qData), } queryLogJSON = append(queryLogJSON, _l) } diff --git a/admin/handlers/templates.go b/admin/handlers/templates.go index 76623168..95f65e7a 100644 --- a/admin/handlers/templates.go +++ b/admin/handlers/templates.go @@ -920,8 +920,6 @@ func (h *HandlersAdmin) NodeHandler(w http.ResponseWriter, r *http.Request) { funcMap := template.FuncMap{ "pastFutureTimes": utils.PastFutureTimes, "jsonRawIndent": jsonRawIndent, - "statusLogsLink": h.statusLogsLink, - "resultLogsLink": h.resultLogsLink, } // Prepare template tempateFiles := NewTemplateFiles(h.TemplatesFolder, "node.html").filepaths diff --git a/admin/handlers/utils.go b/admin/handlers/utils.go index 8c3d849f..b0266764 100644 --- a/admin/handlers/utils.go +++ b/admin/handlers/utils.go @@ -20,6 +20,12 @@ import ( "github.com/jmpsec/osctrl/utils" ) +const ( + QueryLink string = "/query/logs/{{NAME}}" + StatusLink string = "#status-logs" + ResultsLink string = "#result-logs" +) + // Helper to handle admin error responses func adminErrorResponse(w http.ResponseWriter, msg string, code int, err error) { log.Printf("%s: %v", msg, err) @@ -187,17 +193,7 @@ func toJSONConfigurationService(values []settings.SettingValue) types.JSONConfig // Helper to generate a link to results for on-demand queries func (h *HandlersAdmin) queryResultLink(name string) string { - return strings.Replace(h.Settings.QueryResultLink(), "{{NAME}}", removeBackslash(name), 1) -} - -// Helper to generate a link to results for status logs -func (h *HandlersAdmin) statusLogsLink(uuid string) string { - return strings.Replace(h.Settings.StatusLogsLink(), "{{UUID}}", removeBackslash(uuid), 1) -} - -// Helper to generate a link to results for result logs -func (h *HandlersAdmin) resultLogsLink(uuid string) string { - return strings.Replace(h.Settings.ResultLogsLink(), "{{UUID}}", removeBackslash(uuid), 1) + return strings.Replace(QueryLink, "{{NAME}}", removeBackslash(name), 1) } // Helper to convert the list of all TLS environments with the ones with permissions for a user diff --git a/admin/main.go b/admin/main.go index 72a25e6c..7c597873 100644 --- a/admin/main.go +++ b/admin/main.go @@ -14,9 +14,9 @@ import ( "github.com/jmpsec/osctrl/admin/handlers" "github.com/jmpsec/osctrl/admin/sessions" "github.com/jmpsec/osctrl/backend" + "github.com/jmpsec/osctrl/cache" "github.com/jmpsec/osctrl/carves" "github.com/jmpsec/osctrl/environments" - "github.com/jmpsec/osctrl/logging" "github.com/jmpsec/osctrl/metrics" "github.com/jmpsec/osctrl/nodes" "github.com/jmpsec/osctrl/queries" @@ -69,6 +69,8 @@ const ( defConfigurationFile string = "config/" + settings.ServiceAdmin + ".json" // Default DB configuration file defDBConfigurationFile string = "config/db.json" + // Default redis configuration file + defRedisConfigurationFile string = "config/redis.json" // Default Logger configuration file defLoggerConfigurationFile string = "config/logger.json" // Default TLS certificate file @@ -87,8 +89,6 @@ const ( defaultRefresh int = 300 // Default hours to classify nodes as inactive defaultInactive int = -72 - // Hourly interval to cleanup logs - hourlyInterval int = 60 ) // osquery @@ -109,7 +109,9 @@ var ( err error adminConfig types.JSONConfigurationService dbConfig backend.JSONConfigurationDB + redisConfig cache.JSONConfigurationRedis db *backend.DBManager + redis *cache.RedisManager settingsmgr *settings.Settings nodesmgr *nodes.NodeManager queriesmgr *queries.Queries @@ -124,15 +126,16 @@ var ( osqueryTables []types.OsqueryTable adminMetrics *metrics.Metrics handlersAdmin *handlers.HandlersAdmin - loggerDB *logging.LoggerDB ) // Variables for flags var ( configFlag bool - configFile string dbFlag bool + redisFlag bool + serviceConfigFile string dbConfigFile string + redisConfigFile string tlsServer bool tlsCertFile string tlsKeyFile string @@ -221,7 +224,7 @@ func init() { Value: defConfigurationFile, Usage: "Load service configuration from `FILE`", EnvVars: []string{"SERVICE_CONFIG_FILE"}, - Destination: &configFile, + Destination: &serviceConfigFile, }, &cli.StringFlag{ Name: "listener", @@ -263,6 +266,71 @@ func init() { EnvVars: []string{"SERVICE_LOGGER"}, Destination: &adminConfig.Logger, }, + &cli.BoolFlag{ + Name: "redis", + Aliases: []string{"r"}, + Value: false, + Usage: "Provide redis configuration via JSON file", + EnvVars: []string{"REDIS_CONFIG"}, + Destination: &redisFlag, + }, + &cli.StringFlag{ + Name: "redis-file", + Aliases: []string{"R"}, + Value: defRedisConfigurationFile, + Usage: "Load redis configuration from `FILE`", + EnvVars: []string{"REDIS_CONFIG_FILE"}, + Destination: &redisConfigFile, + }, + &cli.StringFlag{ + Name: "redis-host", + Value: "127.0.0.1", + Usage: "Redis host to be connected to", + EnvVars: []string{"REDIS_HOST"}, + Destination: &redisConfig.Host, + }, + &cli.StringFlag{ + Name: "redis-port", + Value: "6379", + Usage: "Redis port to be connected to", + EnvVars: []string{"REDIS_PORT"}, + Destination: &redisConfig.Port, + }, + &cli.StringFlag{ + Name: "redis-pass", + Value: "redis", + Usage: "Password to be used for redis", + EnvVars: []string{"REDIS_PASS"}, + Destination: &redisConfig.Password, + }, + &cli.IntFlag{ + Name: "redis-db", + Value: 0, + Usage: "Redis database to be selected after connecting", + EnvVars: []string{"REDIS_DB"}, + Destination: &redisConfig.DB, + }, + &cli.IntFlag{ + Name: "redis-status-exp", + Value: cache.StatusExpiration, + Usage: "Redis expiration in hours for status logs", + EnvVars: []string{"REDIS_STATUS_EXP"}, + Destination: &redisConfig.StatusExpirationHours, + }, + &cli.IntFlag{ + Name: "redis-result-exp", + Value: cache.ResultExpiration, + Usage: "Redis expiration in hours for result logs", + EnvVars: []string{"REDIS_RESULT_EXP"}, + Destination: &redisConfig.ResultExpirationHours, + }, + &cli.IntFlag{ + Name: "redis-query-exp", + Value: cache.QueryExpiration, + Usage: "Redis expiration in hours for query logs", + EnvVars: []string{"REDIS_QUERY_EXP"}, + Destination: &redisConfig.QueryExpirationHours, + }, &cli.BoolFlag{ Name: "db", Aliases: []string{"d"}, @@ -458,6 +526,12 @@ func osctrlAdminService() { log.Println("Backend NOT ready! waiting...") time.Sleep(backendWait) } + log.Println("Initializing cache...") + redis, err = cache.CreateRedisManager(redisConfig) + if err != nil { + log.Fatalf("Failed to connect to redis - %v", err) + } + log.Println("Connection to cache successful!") log.Println("Initialize users") adminUsers = users.CreateUserManager(db.Conn, &jwtConfig) log.Println("Initialize tags") @@ -483,17 +557,7 @@ func osctrlAdminService() { if err != nil { log.Fatalf("Error loading metrics - %v", err) } - // TODO Initialize DB logger regardless of settings - // This is temporary until we have logs stored in Redis - if adminConfig.Logger == settings.LoggingDB { - loggerDB, err = logging.CreateLoggerDBFile(loggerFile) - if err != nil { - loggerDB, err = logging.CreateLoggerDBConfig(dbConfig) - if err != nil { - log.Fatalf("Error creating db logger - %v", err) - } - } - } + // Start SAML Middleware if we are using SAML if adminConfig.Auth == settings.AuthSAML { if settingsmgr.DebugService(settings.ServiceAdmin) { @@ -532,43 +596,6 @@ func osctrlAdminService() { } }() - // Cleaning up status/result/query logs - go func() { - for { - _e, err := envs.All() - if err != nil { - log.Printf("error getting environments when cleaning up logs - %v", err) - } - for _, e := range _e { - if settingsmgr.CleanStatusLogs() { - if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Cleaning up status logs") - } - if err := loggerDB.CleanStatusLogs(e.Name, settingsmgr.CleanStatusInterval()); err != nil { - log.Printf("error cleaning up status logs - %v", err) - } - } - if settingsmgr.CleanResultLogs() { - if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Cleaning up result logs") - } - if err := loggerDB.CleanResultLogs(e.Name, settingsmgr.CleanResultInterval()); err != nil { - log.Printf("error cleaning up result logs - %v", err) - } - } - } - if settingsmgr.CleanQueryLogs() { - if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Cleaning up query logs") - } - if err := loggerDB.CleanQueryLogs(settingsmgr.CleanQueryEntries()); err != nil { - log.Printf("error cleaning up query logs - %v", err) - } - } - time.Sleep(time.Duration(hourlyInterval) * time.Second) - } - }() - // Initialize Admin handlers before router handlersAdmin = handlers.CreateHandlersAdmin( handlers.WithDB(db.Conn), @@ -580,7 +607,7 @@ func osctrlAdminService() { handlers.WithCarves(carvesmgr), handlers.WithSettings(settingsmgr), handlers.WithMetrics(adminMetrics), - handlers.WithLoggerDB(loggerDB), + handlers.WithCache(redis), handlers.WithSessions(sessionsmgr), handlers.WithVersion(serviceVersion), handlers.WithTemplates(templatesFolder), @@ -737,14 +764,21 @@ func osctrlAdminService() { // Action to run when no flags are provided to run checks and prepare data func cliAction(c *cli.Context) error { - // Load configuration if external service JSON config file is used + // Load configuration if external JSON config file is used if configFlag { - adminConfig, err = loadConfiguration(configFile, settings.ServiceAdmin) + adminConfig, err = loadConfiguration(serviceConfigFile, settings.ServiceAdmin) + if err != nil { + return fmt.Errorf("Failed to load service configuration %s - %s", serviceConfigFile, err) + } + } + // Load redis configuration if external JSON config file is used + if redisFlag { + redisConfig, err = cache.LoadConfiguration(redisConfigFile, cache.RedisKey) if err != nil { - return fmt.Errorf("Failed to load service configuration %s - %s", configFile, err) + return fmt.Errorf("Failed to load redis configuration - %v", err) } } - // Load DB configuration if external db JSON config file is used + // Load DB configuration if external JSON config file is used if dbFlag { dbConfig, err = backend.LoadConfiguration(dbConfigFile, backend.DBKey) if err != nil { diff --git a/admin/settings.go b/admin/settings.go index e2202683..69f7a2e9 100644 --- a/admin/settings.go +++ b/admin/settings.go @@ -36,29 +36,6 @@ func loadingMetrics(mgr *settings.Settings) (*metrics.Metrics, error) { return nil, nil } -// Function to load the logging settings -func loadingLoggingSettings(mgr *settings.Settings) error { - // Check if logging settings for query results link is ready - if !mgr.IsValue(settings.ServiceAdmin, settings.QueryResultLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.QueryResultLink, settings.QueryLink); err != nil { - return fmt.Errorf("Failed to add %s to settings: %v", settings.QueryResultLink, err) - } - } - // Check if logging settings for status logs link is ready - if !mgr.IsValue(settings.ServiceAdmin, settings.StatusLogsLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.StatusLogsLink, settings.StatusLink); err != nil { - return fmt.Errorf("Failed to add %s to settings: %v", settings.DebugHTTP, err) - } - } - // Check if logging settings for result logs link is ready - if !mgr.IsValue(settings.ServiceAdmin, settings.ResultLogsLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.ResultLogsLink, settings.ResultsLink); err != nil { - return fmt.Errorf("Failed to add %s to settings: %v", settings.DebugHTTP, err) - } - } - return err -} - // Function to load all settings for the service func loadingSettings(mgr *settings.Settings) error { // Check if service settings for debug service is ready @@ -101,9 +78,6 @@ func loadingSettings(mgr *settings.Settings) error { return fmt.Errorf("Failed to add %s to settings: %v", settings.NodeDashboard, err) } } - if err := loadingLoggingSettings(mgr); err != nil { - return fmt.Errorf("Failed to load logging settings: %v", err) - } // Write JSON config to settings if err := mgr.SetAllJSON(settings.ServiceAdmin, adminConfig.Listener, adminConfig.Port, adminConfig.Host, adminConfig.Auth, adminConfig.Logger); err != nil { return fmt.Errorf("Failed to add JSON values to configuration: %v", err) diff --git a/admin/templates/node.html b/admin/templates/node.html index 03f3e12f..74ff671f 100644 --- a/admin/templates/node.html +++ b/admin/templates/node.html @@ -362,7 +362,7 @@
- + - + &2 ; exit 1 ; fi @@ -323,6 +333,11 @@ while true; do POSTGRES=true shift ;; + -R|--redis) + SHOW_USAGE=false + REDIS=true + shift + ;; -M|--metrics) SHOW_USAGE=false METRICS=true @@ -625,6 +640,21 @@ else db_user_postgresql "$_DB_NAME" "$_DB_SYSTEM_USER" "$_DB_USER" "$_DB_PASS" "$POSTGRES_PSQL" fi + # Redis - Cache + if [[ "$REDIS" == true ]]; then + REDIS_CONF="$SOURCE_PATH/deploy/redis/redis.conf" + if [[ "$DISTRO" == "ubuntu" ]]; then + package redis-server + REDIS_SERVICE="redis" + REDIS_ETC="/etc/redis/redis.conf" + elif [[ "$DISTRO" == "centos" ]]; then + package redis + REDIS_SERVICE="redis" + REDIS_ETC="/etc/redis.conf" + fi + configure_redis "$REDIS_CONF" "$REDIS_SERVICE" "$REDIS_ETC" "$_CACHE_PASS" + fi + # Metrics - InfluxDB + Telegraf + Grafana if [[ "$METRICS" == true ]]; then if [[ "$DISTRO" == "ubuntu" ]]; then @@ -643,6 +673,9 @@ else # Generate DB configuration file for services configuration_db "$SOURCE_PATH/deploy/config/$DB_TEMPLATE" "$DEST_PATH/config/$DB_CONF" "$_DB_HOST" "$_DB_PORT" "$_DB_NAME" "$_DB_USER" "$_DB_PASS" "sudo" + # Generate Cache configuration file for services + configuration_cache "$SOURCE_PATH/deploy/config/$CACHE_TEMPLATE" "$DEST_PATH/config/$CACHE_CONF" "$_CACHE_HOST" "$_CACHE_PORT" "$_CACHE_PASS" "sudo" + # Prepare DB logger configuration for services sudo cp "$DEST_PATH/config/$DB_CONF" "$DEST_PATH/config/$LOGGER_CONF" @@ -800,4 +833,4 @@ exit 0 # kthxbai # Standard deployment in a linux box would be like: -# ./deploy/provision.sh --nginx --postgres -p all --all-hostname "dev.osctrl.net" -E +# ./deploy/provision.sh -m dev -s /path/to/code --nginx --postgres --redis -p all --all-hostname "dev.osctrl.net" -E diff --git a/deploy/redis/redis.conf b/deploy/redis/redis.conf new file mode 100644 index 00000000..4051c0cf --- /dev/null +++ b/deploy/redis/redis.conf @@ -0,0 +1,2 @@ +bind 127.0.0.1 +requirepass REDIS_PASSWORD diff --git a/go.mod b/go.mod index adb7bc3f..3e7a0caa 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,8 @@ replace github.com/jmpsec/osctrl/api/handlers => ./api/handlers replace github.com/jmpsec/osctrl/backend => ./backend +replace github.com/jmpsec/osctrl/cache => ./cache + replace github.com/jmpsec/osctrl/carves => ./carves replace github.com/jmpsec/osctrl/environments => ./environments @@ -45,6 +47,7 @@ require ( github.com/jmpsec/osctrl/admin/handlers v0.2.7 github.com/jmpsec/osctrl/admin/sessions v0.2.7 github.com/jmpsec/osctrl/backend v0.2.7 + github.com/jmpsec/osctrl/cache v0.2.7 github.com/jmpsec/osctrl/carves v0.2.7 github.com/jmpsec/osctrl/environments v0.2.7 github.com/jmpsec/osctrl/logging v0.2.7 @@ -67,9 +70,12 @@ require ( require ( github.com/aws/aws-sdk-go v1.42.44 // indirect github.com/beevik/etree v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/crewjam/httperr v0.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-redis/redis/v8 v8.11.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/securecookie v1.1.1 // indirect github.com/gorilla/sessions v1.2.1 // indirect diff --git a/go.sum b/go.sum index 8d467bd5..4f2ac057 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -116,6 +117,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -131,6 +134,8 @@ github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -143,8 +148,11 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -265,6 +273,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -400,8 +409,16 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= @@ -582,6 +599,7 @@ golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -606,6 +624,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -620,6 +639,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -657,6 +677,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -670,11 +691,14 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -697,6 +721,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -791,6 +816,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -957,18 +983,21 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.3 h1:jRskFVxYaMGAMUbN0UZ7niA9gzL9B49DOqE78vg0k3w= gopkg.in/ini.v1 v1.66.3/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/logging/db.go b/logging/db.go index 3e9d1f8e..31abff96 100644 --- a/logging/db.go +++ b/logging/db.go @@ -104,57 +104,6 @@ func CreateLoggerDB(backend *backend.DBManager) (*LoggerDB, error) { // Settings - Function to prepare settings for the logger func (logDB *LoggerDB) Settings(mgr *settings.Settings) { log.Printf("Setting DB logging settings\n") - // Setting link for on-demand queries - if !mgr.IsValue(settings.ServiceAdmin, settings.QueryResultLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.QueryResultLink, settings.QueryLink); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.QueryResultLink, err) - } - } - // Setting link for status logs - if !mgr.IsValue(settings.ServiceAdmin, settings.StatusLogsLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.StatusLogsLink, settings.StatusLink); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.StatusLogsLink, err) - } - } - // Setting link for result logs - if !mgr.IsValue(settings.ServiceAdmin, settings.ResultLogsLink) { - if err := mgr.NewStringValue(settings.ServiceAdmin, settings.ResultLogsLink, settings.ResultsLink); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.ResultLogsLink, err) - } - } - // Setting values to enable log cleanup for status logs - if !mgr.IsValue(settings.ServiceAdmin, settings.CleanStatusLogs) { - if err := mgr.NewBooleanValue(settings.ServiceAdmin, settings.CleanStatusLogs, false); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.CleanStatusLogs, err) - } - } - if !mgr.IsValue(settings.ServiceAdmin, settings.CleanStatusInterval) { - if err := mgr.NewIntegerValue(settings.ServiceAdmin, settings.CleanStatusInterval, defaultCleanupInterval); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.CleanStatusInterval, err) - } - } - // Setting values to enable log cleanup for result logs - if !mgr.IsValue(settings.ServiceAdmin, settings.CleanResultLogs) { - if err := mgr.NewBooleanValue(settings.ServiceAdmin, settings.CleanResultLogs, false); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.CleanResultLogs, err) - } - } - if !mgr.IsValue(settings.ServiceAdmin, settings.CleanResultInterval) { - if err := mgr.NewIntegerValue(settings.ServiceAdmin, settings.CleanResultInterval, defaultCleanupInterval); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.CleanResultInterval, err) - } - } - // Setting values to enable log cleanup for query logs - if !mgr.IsValue(settings.ServiceAdmin, settings.CleanQueryLogs) { - if err := mgr.NewBooleanValue(settings.ServiceAdmin, settings.CleanQueryLogs, false); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.CleanQueryLogs, err) - } - } - if !mgr.IsValue(settings.ServiceAdmin, settings.CleanQueryEntries) { - if err := mgr.NewIntegerValue(settings.ServiceAdmin, settings.CleanQueryEntries, 100); err != nil { - log.Fatalf("Failed to add %s to settings: %v", settings.CleanQueryEntries, err) - } - } } // Log - Function that sends JSON result/status/query logs to the configured DB diff --git a/logging/dispatch.go b/logging/dispatch.go index b12b7ccb..c8c5c53b 100644 --- a/logging/dispatch.go +++ b/logging/dispatch.go @@ -22,11 +22,13 @@ func (l *LoggerTLS) DispatchLogs(data []byte, uuid, logType, environment string, l.Log(logType, data, environment, uuid, debug) // Refresh last logging request if logType == types.StatusLog { + // Update metadata for node if err := l.Nodes.RefreshLastStatus(uuid); err != nil { log.Printf("error refreshing last status %v", err) } } if logType == types.ResultLog { + // Update metadata for node if err := l.Nodes.RefreshLastResult(uuid); err != nil { log.Printf("error refreshing last result %v", err) } diff --git a/logging/logging.go b/logging/logging.go index c53309fc..77868004 100644 --- a/logging/logging.go +++ b/logging/logging.go @@ -3,6 +3,7 @@ package logging import ( "log" + "github.com/jmpsec/osctrl/cache" "github.com/jmpsec/osctrl/nodes" "github.com/jmpsec/osctrl/queries" "github.com/jmpsec/osctrl/settings" @@ -10,18 +11,20 @@ import ( // LoggerTLS will be used to handle logging for the TLS endpoint type LoggerTLS struct { - Logging string - Logger interface{} - Nodes *nodes.NodeManager - Queries *queries.Queries + Logging string + Logger interface{} + RedisCache *cache.RedisManager + Nodes *nodes.NodeManager + Queries *queries.Queries } // CreateLoggerTLS to instantiate a new logger for the TLS endpoint -func CreateLoggerTLS(logging, loggingFile string, mgr *settings.Settings, nodes *nodes.NodeManager, queries *queries.Queries) (*LoggerTLS, error) { +func CreateLoggerTLS(logging, loggingFile string, mgr *settings.Settings, nodes *nodes.NodeManager, queries *queries.Queries, redis *cache.RedisManager) (*LoggerTLS, error) { l := &LoggerTLS{ - Logging: logging, - Nodes: nodes, - Queries: queries, + Logging: logging, + Nodes: nodes, + Queries: queries, + RedisCache: redis, } switch logging { case settings.LoggingSplunk: @@ -159,6 +162,10 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri l.Send(logType, data, environment, uuid, debug) } } + // Add logs to cache + if err := logTLS.RedisCache.SetLogs(logType, uuid, environment, data); err != nil { + log.Printf("error sending %s logs to cache %s", logType, err) + } } // QueryLog will send query result logs via the configured method of logging @@ -229,4 +236,8 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid l.Send(logType, data, environment, uuid, debug) } } + // Add logs to cache + if err := logTLS.RedisCache.SetQueryLogs(uuid, name, data); err != nil { + log.Printf("error sending %s logs to cache %s", logType, err) + } } diff --git a/logging/splunk.go b/logging/splunk.go index 2e1d850b..e6ebbc6f 100644 --- a/logging/splunk.go +++ b/logging/splunk.go @@ -84,22 +84,6 @@ type SplunkMessage struct { // Settings - Function to prepare settings for the logger func (logSP *LoggerSplunk) Settings(mgr *settings.Settings) { log.Printf("Setting Splunk logging settings\n") - // Setting link for on-demand queries - var _v string - _v = logSP.Configuration.Queries - if err := mgr.SetString(_v, settings.ServiceAdmin, settings.QueryResultLink, false); err != nil { - log.Printf("Error setting %s with %s - %v", _v, settings.QueryResultLink, err) - } - _v = logSP.Configuration.Status - // Setting link for status logs - if err := mgr.SetString(_v, settings.ServiceAdmin, settings.StatusLogsLink, false); err != nil { - log.Printf("Error setting %s with %s - %v", _v, settings.StatusLogsLink, err) - } - _v = logSP.Configuration.Results - // Setting link for result logs - if err := mgr.SetString(_v, settings.ServiceAdmin, settings.ResultLogsLink, false); err != nil { - log.Printf("Error setting %s with %s - %v", _v, settings.ResultLogsLink, err) - } } // Send - Function that sends JSON logs to Splunk HTTP Event Collector diff --git a/settings/settings.go b/settings/settings.go index 8a5dac68..1b66b1db 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -62,26 +62,6 @@ const ( NodeDashboard string = "node_dashboard" ) -// Names for setting values for logging -const ( - QueryResultLink string = "query_result_link" - StatusLogsLink string = "status_logs_link" - ResultLogsLink string = "result_logs_link" - CleanStatusLogs string = "clean_status_logs" - CleanStatusInterval string = "clean_status_interval" - CleanResultLogs string = "clean_result_logs" - CleanResultInterval string = "clean_result_interval" - CleanQueryLogs string = "clean_query_logs" - CleanQueryEntries string = "clean_query_entries" -) - -// Default values for the setting values for logging -const ( - QueryLink string = "/query/logs/{{NAME}}" - StatusLink string = "#status-logs" - ResultsLink string = "#result-logs" -) - // Names for the values that are read from the JSON config file const ( JSONListener string = "json_listener" @@ -492,87 +472,6 @@ func (conf *Settings) InactiveHours() int64 { return value.Integer } -// QueryResultLink gets the value to be used to generate links for on-demand queries results -func (conf *Settings) QueryResultLink() string { - value, err := conf.RetrieveValue(ServiceAdmin, QueryResultLink) - if err != nil { - return "" - } - return value.String -} - -// StatusLogsLink gets the value to be used to generate links for status logs -func (conf *Settings) StatusLogsLink() string { - value, err := conf.RetrieveValue(ServiceAdmin, StatusLogsLink) - if err != nil { - return "" - } - return value.String -} - -// ResultLogsLink gets the value to be used to generate links for result logs -func (conf *Settings) ResultLogsLink() string { - value, err := conf.RetrieveValue(ServiceAdmin, ResultLogsLink) - if err != nil { - return "" - } - return value.String -} - -// CleanStatusLogs checks if status logs cleanup is enabled -func (conf *Settings) CleanStatusLogs() bool { - value, err := conf.RetrieveValue(ServiceAdmin, CleanStatusLogs) - if err != nil { - return false - } - return value.Boolean -} - -// CleanStatusInterval gets the interval in seconds to cleanup status logs -func (conf *Settings) CleanStatusInterval() int64 { - value, err := conf.RetrieveValue(ServiceAdmin, CleanStatusInterval) - if err != nil { - return 0 - } - return value.Integer -} - -// CleanResultLogs checks if result logs cleanup is enabled -func (conf *Settings) CleanResultLogs() bool { - value, err := conf.RetrieveValue(ServiceAdmin, CleanResultLogs) - if err != nil { - return false - } - return value.Boolean -} - -// CleanResultInterval gets the interval in seconds to cleanup result logs -func (conf *Settings) CleanResultInterval() int64 { - value, err := conf.RetrieveValue(ServiceAdmin, CleanResultInterval) - if err != nil { - return 0 - } - return value.Integer -} - -// CleanQueryLogs checks if query logs cleanup is enabled -func (conf *Settings) CleanQueryLogs() bool { - value, err := conf.RetrieveValue(ServiceAdmin, CleanQueryLogs) - if err != nil { - return false - } - return value.Boolean -} - -// CleanQueryEntries gets the number of entries to cleanup in query logs -func (conf *Settings) CleanQueryEntries() int64 { - value, err := conf.RetrieveValue(ServiceAdmin, CleanQueryEntries) - if err != nil { - return 0 - } - return value.Integer -} - // DefaultEnv gets the default environment // FIXME customize the fallover one func (conf *Settings) DefaultEnv(service string) string { diff --git a/tls/main.go b/tls/main.go index 2e6957c7..e5673f18 100644 --- a/tls/main.go +++ b/tls/main.go @@ -9,6 +9,7 @@ import ( "time" "github.com/jmpsec/osctrl/backend" + "github.com/jmpsec/osctrl/cache" "github.com/jmpsec/osctrl/carves" "github.com/jmpsec/osctrl/environments" "github.com/jmpsec/osctrl/logging" @@ -45,6 +46,8 @@ const ( defConfigurationFile string = "config/" + settings.ServiceTLS + ".json" // Default DB configuration file defDBConfigurationFile string = "config/db.json" + // Default redis configuration file + defRedisConfigurationFile string = "config/redis.json" // Default Logger configuration file defLoggerConfigurationFile string = "config/logger.json" // Default TLS certificate file @@ -67,7 +70,9 @@ var ( err error tlsConfig types.JSONConfigurationService dbConfig backend.JSONConfigurationDB + redisConfig cache.JSONConfigurationRedis db *backend.DBManager + redis *cache.RedisManager settingsmgr *settings.Settings envs *environments.Environment envsmap environments.MapEnvironments @@ -85,14 +90,16 @@ var ( // Variables for flags var ( - configFlag bool - configFile string - dbFlag bool - dbConfigFile string - tlsServer bool - tlsCertFile string - tlsKeyFile string - loggerFile string + configFlag bool + serviceConfigFile string + redisConfigFile string + dbFlag bool + redisFlag bool + dbConfigFile string + tlsServer bool + tlsCertFile string + tlsKeyFile string + loggerFile string ) // Valid values for auth and logging in configuration @@ -148,7 +155,7 @@ func init() { Value: defConfigurationFile, Usage: "Load service configuration from `FILE`", EnvVars: []string{"SERVICE_CONFIG_FILE"}, - Destination: &dbConfigFile, + Destination: &serviceConfigFile, }, &cli.StringFlag{ Name: "listener", @@ -190,6 +197,71 @@ func init() { EnvVars: []string{"SERVICE_LOGGER"}, Destination: &tlsConfig.Logger, }, + &cli.BoolFlag{ + Name: "redis", + Aliases: []string{"r"}, + Value: false, + Usage: "Provide redis configuration via JSON file", + EnvVars: []string{"REDIS_CONFIG"}, + Destination: &redisFlag, + }, + &cli.StringFlag{ + Name: "redis-file", + Aliases: []string{"R"}, + Value: defRedisConfigurationFile, + Usage: "Load redis configuration from `FILE`", + EnvVars: []string{"REDIS_CONFIG_FILE"}, + Destination: &redisConfigFile, + }, + &cli.StringFlag{ + Name: "redis-host", + Value: "127.0.0.1", + Usage: "Redis host to be connected to", + EnvVars: []string{"REDIS_HOST"}, + Destination: &redisConfig.Host, + }, + &cli.StringFlag{ + Name: "redis-port", + Value: "6379", + Usage: "Redis port to be connected to", + EnvVars: []string{"REDIS_PORT"}, + Destination: &redisConfig.Port, + }, + &cli.StringFlag{ + Name: "redis-pass", + Value: "redis", + Usage: "Password to be used for redis", + EnvVars: []string{"REDIS_PASS"}, + Destination: &redisConfig.Password, + }, + &cli.IntFlag{ + Name: "redis-db", + Value: 0, + Usage: "Redis database to be selected after connecting", + EnvVars: []string{"REDIS_DB"}, + Destination: &redisConfig.DB, + }, + &cli.IntFlag{ + Name: "redis-status-exp", + Value: cache.StatusExpiration, + Usage: "Redis expiration in hours for status logs", + EnvVars: []string{"REDIS_STATUS_EXP"}, + Destination: &redisConfig.StatusExpirationHours, + }, + &cli.IntFlag{ + Name: "redis-result-exp", + Value: cache.ResultExpiration, + Usage: "Redis expiration in hours for result logs", + EnvVars: []string{"REDIS_RESULT_EXP"}, + Destination: &redisConfig.ResultExpirationHours, + }, + &cli.IntFlag{ + Name: "redis-query-exp", + Value: cache.QueryExpiration, + Usage: "Redis expiration in hours for query logs", + EnvVars: []string{"REDIS_QUERY_EXP"}, + Destination: &redisConfig.QueryExpirationHours, + }, &cli.BoolFlag{ Name: "db", Aliases: []string{"d"}, @@ -204,7 +276,7 @@ func init() { Value: defDBConfigurationFile, Usage: "Load DB configuration from `FILE`", EnvVars: []string{"DB_CONFIG_FILE"}, - Destination: &configFile, + Destination: &serviceConfigFile, }, &cli.StringFlag{ Name: "db-host", @@ -315,6 +387,12 @@ func osctrlService() { log.Println("Backend NOT ready! waiting...") time.Sleep(backendWait) } + log.Println("Initializing cache...") + redis, err = cache.CreateRedisManager(redisConfig) + if err != nil { + log.Fatalf("Failed to connect to redis - %v", err) + } + log.Println("Connection to cache successful!") log.Println("Initialize environment") envs = environments.CreateEnvironment(db.Conn) log.Println("Initialize settings") @@ -339,7 +417,7 @@ func osctrlService() { } // Initialize TLS logger log.Println("Loading TLS logger") - loggerTLS, err = logging.CreateLoggerTLS(tlsConfig.Logger, loggerFile, settingsmgr, nodesmgr, queriesmgr) + loggerTLS, err = logging.CreateLoggerTLS(tlsConfig.Logger, loggerFile, settingsmgr, nodesmgr, queriesmgr, redis) if err != nil { log.Fatalf("Error loading logger - %s: %v", tlsConfig.Logger, err) } @@ -447,20 +525,27 @@ func osctrlService() { // Action to run when no flags are provided to run checks and prepare data func cliAction(c *cli.Context) error { - // Load configuration if external service JSON config file is used + // Load configuration if external JSON config file is used if configFlag { - tlsConfig, err = loadConfiguration(configFile) + tlsConfig, err = loadConfiguration(serviceConfigFile) if err != nil { - return fmt.Errorf("Error loading %s - %s", configFile, err) + return fmt.Errorf("Error loading %s - %s", serviceConfigFile, err) } } - // Load db configuration if external db JSON config file is used + // Load db configuration if external JSON config file is used if dbFlag { dbConfig, err = backend.LoadConfiguration(dbConfigFile, backend.DBKey) if err != nil { return fmt.Errorf("Failed to load DB configuration - %v", err) } } + // Load redis configuration if external JSON config file is used + if redisFlag { + redisConfig, err = cache.LoadConfiguration(redisConfigFile, cache.RedisKey) + if err != nil { + return fmt.Errorf("Failed to load redis configuration - %v", err) + } + } return nil } diff --git a/types/osquery.go b/types/osquery.go index b55604bb..69f22e54 100644 --- a/types/osquery.go +++ b/types/osquery.go @@ -2,6 +2,7 @@ package types import ( "encoding/json" + "strconv" "github.com/jmpsec/osctrl/queries" ) @@ -127,6 +128,27 @@ type LogDecorations struct { DaemonHash string `json:"osquery_md5"` } +// StringInt to parse numbers that could be strings +type StringInt int + +// UnmarshalJSON implements the json.Unmarshaler interface, which +// allows us to ingest values of any json type as an int and run our custom conversion +func (si *StringInt) UnmarshalJSON(b []byte) error { + if b[0] != '"' { + return json.Unmarshal(b, (*int)(si)) + } + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + i, err := strconv.Atoi(s) + if err != nil { + return err + } + *si = StringInt(i) + return nil +} + // LogResultData to be used processing result logs from nodes type LogResultData struct { Name string `json:"name"` @@ -134,7 +156,7 @@ type LogResultData struct { Action string `json:"action"` Columns json.RawMessage `json:"columns"` Counter int `json:"counter"` - UnixTime int `json:"unixTime"` + UnixTime StringInt `json:"unixTime"` Decorations LogDecorations `json:"decorations"` CalendarTime string `json:"calendarTime"` HostIdentifier string `json:"hostIdentifier"` @@ -147,7 +169,7 @@ type LogStatusData struct { Version string `json:"version"` Filename string `json:"filename"` Severity string `json:"severity"` - UnixTime string `json:"unixTime"` + UnixTime StringInt `json:"unixTime"` Decorations LogDecorations `json:"decorations"` CalendarTime string `json:"calendarTime"` HostIdentifier string `json:"hostIdentifier"` diff --git a/utils/time-utils.go b/utils/time-utils.go index 29e49f2d..ace5d85e 100644 --- a/utils/time-utils.go +++ b/utils/time-utils.go @@ -59,6 +59,11 @@ func PastFutureTimes(t time.Time) string { return InFutureTime(t) } +// PastFutureTimesEpoch - Helper to format past or future times +func PastFutureTimesEpoch(ts int64) string { + return PastFutureTimes(time.Unix(ts, 0)) +} + // PastTimeAgo - Helper to format past times only returning one value (minute, hour, day) func PastTimeAgo(t time.Time) string { if t.IsZero() {