diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index da63b4565b6..cacbf33577c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -282,6 +282,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix handling of infinite rate values in CEL rate limit handling logic. {pull}39940[39940] - Allow elision of set and append failure logging. {issue}34544[34544] {pull}39929[39929] - Add ability to remove request trace logs from CEL input. {pull}39969[39969] +- Add ability to remove request trace logs from entityanalytics input. {pull}[] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc b/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc index 41c01576123..4d58359d7d9 100644 --- a/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc @@ -510,17 +510,21 @@ This is a list of optional query parameters. The default is `["accountEnabled", "alternativeSecurityIds"]`. [float] -==== `tracer.filename` +==== `tracer.enabled` It is possible to log HTTP requests and responses to the EntraID API to a local file-system for debugging configurations. -This option is enabled by setting the `tracer.filename` value. Additional options are available to -tune log rotation behavior. - -To differentiate the trace files generated from different input instances, a placeholder `*` can be added to the filename and will be replaced with the input instance id. -For Example, `http-request-trace-*.ndjson`. +This option is enabled by setting `tracer.enabled` to true and setting the `tracer.filename` value. +Additional options are available to tune log rotation behavior. To delete existing logs, set `tracer.enabled` +to false without unsetting the filename option. Enabling this option compromises security and should only be used for debugging. +[float] +==== `tracer.filename` + +To differentiate the trace files generated from different input instances, a placeholder `*` can be added to the +filename and will be replaced with the input instance id. For Example, `http-request-trace-*.ndjson`. + [id="provider-okta"] ==== Okta User Identities (`okta`) @@ -810,17 +814,21 @@ shorter than the full synchronization interval (`sync_interval`). Expressed as a duration string (e.g., 1m, 3h, 24h). Defaults to `15m` (15 minutes). [float] -==== `tracer.filename` +==== `tracer.enabled` It is possible to log HTTP requests and responses to the Okta API to a local file-system for debugging configurations. -This option is enabled by setting the `tracer.filename` value. Additional options are available to -tune log rotation behavior. - -To differentiate the trace files generated from different input instances, a placeholder `*` can be added to the filename and will be replaced with the input instance id. -For Example, `http-request-trace-*.ndjson`. +This option is enabled by setting `tracer.enabled` to true and setting the `tracer.filename` value. +Additional options are available to tune log rotation behavior. To delete existing logs, set `tracer.enabled` +to false without unsetting the filename option. Enabling this option compromises security and should only be used for debugging. +[float] +==== `tracer.filename` + +To differentiate the trace files generated from different input instances, a placeholder `*` can be added to the +filename and will be replaced with the input instance id. For Example, `http-request-trace-*.ndjson`. + [float] ==== Metrics diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go index a3104ce0d00..4b98bdb05be 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go @@ -13,8 +13,10 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "net/url" + "os" "path/filepath" "strings" @@ -112,7 +114,16 @@ type graphConf struct { Transport httpcommon.HTTPTransportSettings `config:",inline"` // Tracer allows configuration of request trace logging. - Tracer *lumberjack.Logger `config:"tracer"` + Tracer *tracerConfig `config:"tracer"` +} + +type tracerConfig struct { + Enabled *bool `config:"enabled"` + lumberjack.Logger `config:",inline"` +} + +func (t *tracerConfig) enabled() bool { + return t != nil && (t.Enabled == nil || *t.Enabled) } type selection struct { @@ -398,12 +409,39 @@ func New(ctx context.Context, id string, cfg *config.C, logger *logp.Logger, aut return &f, nil } +// lumberjackTimestamp is a glob expression matching the time format string used +// by lumberjack when rolling over logs, "2006-01-02T15-04-05.000". +// https://github.com/natefinch/lumberjack/blob/4cb27fcfbb0f35cb48c542c5ea80b7c1d18933d0/lumberjack.go#L39 +const lumberjackTimestamp = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]-[0-9][0-9]-[0-9][0-9].[0-9][0-9][0-9]" + // requestTrace decorates cli with an httplog.LoggingRoundTripper if cfg.Tracer // is non-nil. func requestTrace(ctx context.Context, cli *http.Client, cfg graphConf, log *logp.Logger) *http.Client { if cfg.Tracer == nil { return cli } + if !cfg.Tracer.enabled() { + // We have a trace log name, but we are not enabled, + // so remove all trace logs we own. + err := os.Remove(cfg.Tracer.Filename) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Errorw("failed to remove request trace log", "path", cfg.Tracer.Filename, "error", err) + } + ext := filepath.Ext(cfg.Tracer.Filename) + base := strings.TrimSuffix(cfg.Tracer.Filename, ext) + paths, err := filepath.Glob(base + "-" + lumberjackTimestamp + ext) + if err != nil { + log.Errorw("failed to collect request trace log path names", "error", err) + } + for _, p := range paths { + err = os.Remove(p) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Errorw("failed to remove request trace log", "path", p, "error", err) + } + } + return cli + } + w := zapcore.AddSync(cfg.Tracer) go func() { // Close the logger when we are done. diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph_test.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph_test.go index f2fc2effe29..bafddc20a00 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph_test.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph_test.go @@ -318,9 +318,10 @@ func TestGraph_Groups(t *testing.T) { APIEndpoint: "http://" + testSrv.addr, } if *trace { - rawConf.Tracer = &lumberjack.Logger{ + // Use legacy behaviour; nil enabled setting. + rawConf.Tracer = &tracerConfig{Logger: lumberjack.Logger{ Filename: "test_trace-*.ndjson", - } + }} } c, err := config.NewConfigFrom(&rawConf) require.NoError(t, err) @@ -382,9 +383,9 @@ func TestGraph_Users(t *testing.T) { APIEndpoint: "http://" + testSrv.addr, } if *trace { - rawConf.Tracer = &lumberjack.Logger{ + rawConf.Tracer = &tracerConfig{Logger: lumberjack.Logger{ Filename: "test_trace-*.ndjson", - } + }} } c, err := config.NewConfigFrom(&rawConf) require.NoError(t, err) @@ -492,9 +493,9 @@ func TestGraph_Devices(t *testing.T) { Select: test.selection, } if *trace { - rawConf.Tracer = &lumberjack.Logger{ + rawConf.Tracer = &tracerConfig{Logger: lumberjack.Logger{ Filename: "test_trace-*.ndjson", - } + }} } c, err := config.NewConfigFrom(&rawConf) require.NoError(t, err) diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go b/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go index 873a6195d47..2bab4c9e67d 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go @@ -66,7 +66,16 @@ type conf struct { Request *requestConfig `config:"request"` // Tracer allows configuration of request trace logging. - Tracer *lumberjack.Logger `config:"tracer"` + Tracer *tracerConfig `config:"tracer"` +} + +type tracerConfig struct { + Enabled *bool `config:"enabled"` + lumberjack.Logger `config:",inline"` +} + +func (t *tracerConfig) enabled() bool { + return t != nil && (t.Enabled == nil || *t.Enabled) } type requestConfig struct { diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 0980575df3a..988f44d0840 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -10,8 +10,10 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "net/url" + "os" "path/filepath" "strings" "time" @@ -184,12 +186,39 @@ func newClient(ctx context.Context, cfg conf, log *logp.Logger) (*http.Client, e return client.StandardClient(), nil } +// lumberjackTimestamp is a glob expression matching the time format string used +// by lumberjack when rolling over logs, "2006-01-02T15-04-05.000". +// https://github.com/natefinch/lumberjack/blob/4cb27fcfbb0f35cb48c542c5ea80b7c1d18933d0/lumberjack.go#L39 +const lumberjackTimestamp = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]-[0-9][0-9]-[0-9][0-9].[0-9][0-9][0-9]" + // requestTrace decorates cli with an httplog.LoggingRoundTripper if cfg.Tracer // is non-nil. func requestTrace(ctx context.Context, cli *http.Client, cfg conf, log *logp.Logger) *http.Client { if cfg.Tracer == nil { return cli } + if !cfg.Tracer.enabled() { + // We have a trace log name, but we are not enabled, + // so remove all trace logs we own. + err := os.Remove(cfg.Tracer.Filename) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Errorw("failed to remove request trace log", "path", cfg.Tracer.Filename, "error", err) + } + ext := filepath.Ext(cfg.Tracer.Filename) + base := strings.TrimSuffix(cfg.Tracer.Filename, ext) + paths, err := filepath.Glob(base + "-" + lumberjackTimestamp + ext) + if err != nil { + log.Errorw("failed to collect request trace log path names", "error", err) + } + for _, p := range paths { + err = os.Remove(p) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Errorw("failed to remove request trace log", "path", p, "error", err) + } + } + return cli + } + w := zapcore.AddSync(cfg.Tracer) go func() { // Close the logger when we are done. diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go index 51701775ca5..6bbd71e9b3e 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go @@ -162,9 +162,10 @@ func TestOktaDoFetch(t *testing.T) { if name == "" { name = "default" } - a.cfg.Tracer = &lumberjack.Logger{ + // Use legacy behaviour; nil enabled setting. + a.cfg.Tracer = &tracerConfig{Logger: lumberjack.Logger{ Filename: fmt.Sprintf("test_trace_%s.ndjson", name), - } + }} } a.client = requestTrace(context.Background(), a.client, a.cfg, a.logger)