New option timestamp.precision to control timestamp precision (#31682)

CHANGELOG.next.asciidoc

@@ -94,6 +94,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 
 - Update to Go 1.17.10 {issue}31636[31636]
 - Add support for nanosecond precision timestamps. {issue}15871[15871] {pull}31553[31553]
+- Add new config option `timestamp.precision` to configure timestamps. {pull}31682[31682]
 
 
 *Auditbeat*

auditbeat/auditbeat.reference.yml

@@ -141,6 +141,10 @@ auditbeat.modules:
 # sub-dictionary. Default is false.
 #fields_under_root: false
 
+# Configure the precision of all timestamps in Auditbeat.
+# Available options: millisecond, microsecond, nanosecond
+#timestamp.precision: millisecond
+
 # Internal queue configuration for buffering events to be published.
 #queue:
   # Queue type by name (default 'mem')

filebeat/filebeat.reference.yml

@@ -1260,6 +1260,10 @@ filebeat.inputs:
 # sub-dictionary. Default is false.
 #fields_under_root: false
 
+# Configure the precision of all timestamps in Filebeat.
+# Available options: millisecond, microsecond, nanosecond
+#timestamp.precision: millisecond
+
 # Internal queue configuration for buffering events to be published.
 #queue:
   # Queue type by name (default 'mem')

filebeat/tests/system/config/filebeat_modules.yml.j2

@@ -38,3 +38,4 @@ setup.ilm:
   {% endif %}
 {% endif %}
 
+timestamp.precision: nanosecond

heartbeat/heartbeat.reference.yml

@@ -287,6 +287,10 @@ heartbeat.jobs:
 # sub-dictionary. Default is false.
 #fields_under_root: false
 
+# Configure the precision of all timestamps in Heartbeat.
+# Available options: millisecond, microsecond, nanosecond
+#timestamp.precision: millisecond
+
 # Internal queue configuration for buffering events to be published.
 #queue:
   # Queue type by name (default 'mem')

libbeat/_meta/config.yml.tmpl

@@ -14,6 +14,10 @@
 #fields:
 #  env: staging
 
+# Configure the precision of all timestamps in {{ .BeatName | title }}.
+# Available options: millisecond, microsecond, nanosecond
+#timestamp.precision: millisecond
+
 {{if not .ExcludeDashboards }}
 #============================== Dashboards =====================================
 # These settings control loading the sample dashboards to the Kibana index. Loading

libbeat/_meta/config/general.reference.yml.tmpl

@@ -21,6 +21,10 @@
 # sub-dictionary. Default is false.
 #fields_under_root: false
 
+# Configure the precision of all timestamps in {{ .BeatName | title }}.
+# Available options: millisecond, microsecond, nanosecond
+#timestamp.precision: millisecond
+
 # Internal queue configuration for buffering events to be published.
 #queue:
   # Queue type by name (default 'mem')

libbeat/cmd/instance/beat.go

@@ -131,6 +131,8 @@ type beatConfig struct {
 
 	// Migration config to migration from 6 to 7
 	Migration *config.C `config:"migration.6_to_7"`
+	// TimestampPrecision sets the precision of all timestamps in the Beat.
+	TimestampPrecision *config.C `config:"timestamp"`
 }
 
 var debugf = logp.MakeDebug("beat")
@@ -684,6 +686,10 @@ func (b *Beat) configure(settings Settings) error {
 		b.Info.Name = name
 	}
 
+	if err := common.SetTimestampPrecision(b.Config.TimestampPrecision); err != nil {
+		return fmt.Errorf("error setting timestamp precision: %w", err)
+	}
+
 	if err := configure.Logging(b.Info.Beat, b.Config.Logging); err != nil {
 		return fmt.Errorf("error initializing logging: %w", err)
 	}
@@ -916,7 +922,7 @@ func (b *Beat) registerESIndexManagement() error {
 
 	_, err := elasticsearch.RegisterConnectCallback(b.indexSetupCallback())
 	if err != nil {
-		return fmt.Errorf("failed to register index management with elasticsearch: %+v", err)
+		return fmt.Errorf("failed to register index management with elasticsearch: %w", err)
 	}
 	return nil
 }
@@ -1184,11 +1190,11 @@ func initPaths(cfg *config.C) error {
 	}{}
 
 	if err := cfg.Unpack(&partialConfig); err != nil {
-		return fmt.Errorf("error extracting default paths: %+v", err)
+		return fmt.Errorf("error extracting default paths: %w", err)
 	}
 
 	if err := paths.InitPaths(&partialConfig.Path); err != nil {
-		return fmt.Errorf("error setting default paths: %+v", err)
+		return fmt.Errorf("error setting default paths: %w", err)
 	}
 	return nil
 }

libbeat/common/datetime.go

@@ -21,37 +21,121 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"hash"
 	"time"
 
 	"github.com/elastic/beats/v7/libbeat/common/dtfmt"
+	conf "github.com/elastic/elastic-agent-libs/config"
 )
 
 const (
-	// TsLayout is the seconds layout to be used in the timestamp marshaling/unmarshaling everywhere.
-	// The timezone must always be UTC.
-	TsLayout = "2006-01-02T15:04:05.000Z"
+	millisecPrecision TimestampPrecision = iota + 1
+	microsecPrecision
+	nanosecPrecision
+
+	DefaultTimestampPrecision = millisecPrecision
 
 	tsLayoutMillis = "2006-01-02T15:04:05.000Z"
 	tsLayoutMicros = "2006-01-02T15:04:05.000000Z"
 	tsLayoutNanos  = "2006-01-02T15:04:05.000000000Z"
+
+	millisecPrecisionFmt = "yyyy-MM-dd'T'HH:mm:ss.fff'Z'"
+	microsecPrecisionFmt = "yyyy-MM-dd'T'HH:mm:ss.ffffff'Z'"
+	nanosecPrecisionFmt  = "yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'"
+
+	localMillisecPrecisionFmt = "yyyy-MM-dd'T'HH:mm:ss.fffz"
+	localMicrosecPrecisionFmt = "yyyy-MM-dd'T'HH:mm:ss.ffffffz"
+	localNanosecPrecisionFmt  = "yyyy-MM-dd'T'HH:mm:ss.fffffffffz"
+)
+
+// timestampFmt stores the format strings for both UTC and local
+// form of a specific precision.
+type timestampFmt struct {
+	utc   string
+	local string
+}
+
+var (
+	defaultParseFormats = []string{
+		tsLayoutMillis,
+		tsLayoutMicros,
+		tsLayoutNanos,
+	}
+
+	precisions = map[TimestampPrecision]timestampFmt{
+		millisecPrecision: timestampFmt{utc: millisecPrecisionFmt, local: localMillisecPrecisionFmt},
+		microsecPrecision: timestampFmt{utc: microsecPrecisionFmt, local: localMicrosecPrecisionFmt},
+		nanosecPrecision:  timestampFmt{utc: nanosecPrecisionFmt, local: localNanosecPrecisionFmt},
+	}
+
+	// tsFmt is the selected timestamp format
+	tsFmt = precisions[DefaultTimestampPrecision]
+	// timeFormatter is a datettime formatter with a selected timestamp precision in UTC.
+	timeFormatter = dtfmt.MustNewFormatter(tsFmt.utc)
 )
 
 // Time is an abstraction for the time.Time type
 type Time time.Time
 
-var defaultTimeFormatter = dtfmt.MustNewFormatter("yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'")
+type TimestampPrecision uint8
 
-var defaultParseFormats = []string{
-	tsLayoutMillis,
-	tsLayoutMicros,
-	tsLayoutNanos,
+type TimestampConfig struct {
+	Precision TimestampPrecision `config:"precision"`
+}
+
+func defaultTimestampConfig() TimestampConfig {
+	return TimestampConfig{Precision: DefaultTimestampPrecision}
+}
+
+func (p *TimestampPrecision) Unpack(v string) error {
+	switch v {
+	case "millisecond", "":
+		*p = millisecPrecision
+	case "microsecond":
+		*p = microsecPrecision
+	case "nanosecond":
+		*p = nanosecPrecision
+	default:
+		return fmt.Errorf("invalid timestamp precision %s, available options: millisecond, microsecond, nanosecond", v)
+	}
+	return nil
+}
+
+// SetTimestampPrecision sets the precision of timestamps in the Beat.
+// It is only supposed to be called during init because it changes
+// the format of the timestamps globally.
+func SetTimestampPrecision(c *conf.C) error {
+	if c == nil {
+		return nil
+	}
+
+	p := defaultTimestampConfig()
+	err := c.Unpack(&p)
+	if err != nil {
+		return fmt.Errorf("failed to set timestamp precision: %w", err)
+	}
+
+	tsFmt = precisions[p.Precision]
+	timeFormatter = dtfmt.MustNewFormatter(precisions[p.Precision].utc)
+
+	return nil
+}
+
+// TimestampFormat returns the datettime format string
+// with the configured timestamp precision. It can return
+// either the UTC format or the local one.
+func TimestampFormat(local bool) string {
+	if local {
+		return tsFmt.local
+	}
+	return tsFmt.utc
 }
 
 // MarshalJSON implements json.Marshaler interface.
 // The time is a quoted string in the JsTsLayout format.
 func (t Time) MarshalJSON() ([]byte, error) {
-	str, _ := defaultTimeFormatter.Format(time.Time(t).UTC())
+	str, _ := timeFormatter.Format(time.Time(t).UTC())
 	return json.Marshal(str)
 }
 
@@ -87,7 +171,7 @@ func ParseTime(timespec string) (Time, error) {
 }
 
 func (t Time) String() string {
-	str, _ := defaultTimeFormatter.Format(time.Time(t).UTC())
+	str, _ := timeFormatter.Format(time.Time(t).UTC())
 	return str
 }
 

libbeat/common/datetime_test.go

@@ -26,7 +26,9 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
+	conf "github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
@@ -49,6 +51,16 @@ func TestParseTime(t *testing.T) {
 			Input:  "2015-02-28T11:19:05.112Z",
 			Output: time.Date(2015, time.February, 28, 11, 19, 05, 112*1e6, time.UTC),
 		},
+		// ParseTime must be able to parse microsecond precision timestamps
+		{
+			Input:  "2015-02-28T11:19:05.000001Z",
+			Output: time.Date(2015, time.February, 28, 11, 19, 05, 1000, time.UTC),
+		},
+		// ParseTime must be able to parse nanosecond precision timestamps
+		{
+			Input:  "2015-02-28T11:19:05.000001122Z",
+			Output: time.Date(2015, time.February, 28, 11, 19, 05, 1122, time.UTC),
+		},
 	}
 
 	for _, test := range tests {
@@ -106,3 +118,44 @@ func TestTimeMarshal(t *testing.T) {
 		assert.Equal(t, test.Output, string(result))
 	}
 }
+
+func TestTimeString(t *testing.T) {
+	tests := map[string]struct {
+		precisionCfg *conf.C
+		ts           string
+	}{
+		"empty config": {
+			nil,
+			"2015-03-01T11:19:05.000Z",
+		},
+		"nanosecond precision": {
+			conf.MustNewConfigFrom(mapstr.M{
+				"precision": "nanosecond",
+			}),
+			"2015-03-01T11:19:05.000001112Z",
+		},
+		"millisecond precision": {
+			conf.MustNewConfigFrom(mapstr.M{
+				"precision": "millisecond",
+			}),
+			"2015-03-01T11:19:05.000Z",
+		},
+		"microsecond precision": {
+			conf.MustNewConfigFrom(mapstr.M{
+				"precision": "microsecond",
+			}),
+			"2015-03-01T11:19:05.000001Z",
+		},
+	}
+
+	ts := Time(time.Date(2015, time.March, 01, 11, 19, 05, 1112, time.UTC))
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			err := SetTimestampPrecision(test.precisionCfg)
+			require.NoError(t, err, "precision must be set")
+
+			require.Equal(t, test.ts, ts.String())
+		})
+	}
+}

libbeat/docs/generalconfig.asciidoc

@@ -107,3 +107,9 @@ processors in your config.
 
 Sets the maximum number of CPUs that can be executing simultaneously. The
 default is the number of logical CPUs available in the system.
+
+[float]
+==== `timestamp.precision`
+
+Configure the precision of all timestamps. By default it is set to microsecond.
+Available options: millisecond, microsecond, nanosecond

libbeat/outputs/codec/common.go

@@ -34,14 +34,7 @@ func MakeTimestampEncoder() func(*time.Time, structform.ExtVisitor) error {
 // MakeUTCOrLocalTimestampEncoder creates encoder function that formats time into RFC3339 representation
 // with UTC or local timezone in the output (based on localTime boolean parameter).
 func MakeUTCOrLocalTimestampEncoder(localTime bool) func(*time.Time, structform.ExtVisitor) error {
-	var dtPattern string
-	if localTime {
-		dtPattern = "yyyy-MM-dd'T'HH:mm:ss.fffffffffz"
-	} else {
-		dtPattern = "yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'"
-	}
-
-	formatter, err := dtfmt.NewFormatter(dtPattern)
+	formatter, err := dtfmt.NewFormatter(common.TimestampFormat(localTime))
 	if err != nil {
 		panic(err)
 	}

metricbeat/metricbeat.reference.yml

@@ -1014,6 +1014,10 @@ metricbeat.modules:
 # sub-dictionary. Default is false.
 #fields_under_root: false
 
+# Configure the precision of all timestamps in Metricbeat.
+# Available options: millisecond, microsecond, nanosecond
+#timestamp.precision: millisecond
+
 # Internal queue configuration for buffering events to be published.
 #queue:
   # Queue type by name (default 'mem')

packetbeat/packetbeat.reference.yml

@@ -636,6 +636,10 @@ packetbeat.ignore_outgoing: false
 # sub-dictionary. Default is false.
 #fields_under_root: false
 
+# Configure the precision of all timestamps in Packetbeat.
+# Available options: millisecond, microsecond, nanosecond
+#timestamp.precision: millisecond
+
 # Internal queue configuration for buffering events to be published.
 #queue:
   # Queue type by name (default 'mem')

packetbeat/tests/system/config/packetbeat.yml.j2

@@ -248,3 +248,7 @@ output.file:
   rotate_every_kb: {{ rotate_every_kb | default(1000) }}
   #number_of_files: 7
 {%- endif %}
+
+{% if timestamp_precision %}
+timestamp.precision: {{ timestamp_precision }}
+{%- endif %}

packetbeat/tests/system/test_0032_dns.py

@@ -13,6 +13,7 @@ def test_A(self):
         """
         self.render_config_template(
             dns_ports=[53],
+            timestamp_precision="nanosecond",
         )
         self.run_packetbeat(pcap="dns_google_com.pcap")