Add support for nanosecond precision timestamps (#31553)

CHANGELOG.next.asciidoc

@@ -87,6 +87,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 
 *Affecting all Beats*
 
+- Add support for nanosecond precision timestamps. {issue}15871[15871] {pull}31553[31553]
+
 
 *Auditbeat*
 

libbeat/common/datetime.go

@@ -23,45 +23,72 @@ import (
 	"errors"
 	"hash"
 	"time"
+
+	"github.com/elastic/beats/v7/libbeat/common/dtfmt"
 )
 
-// TsLayout is the layout to be used in the timestamp marshaling/unmarshaling everywhere.
-// The timezone must always be UTC.
-const TsLayout = "2006-01-02T15:04:05.000Z"
+const (
+	// TsLayout is the seconds layout to be used in the timestamp marshaling/unmarshaling everywhere.
+	// The timezone must always be UTC.
+	TsLayout = "2006-01-02T15:04:05.000Z"
+
+	tsLayoutMillis = "2006-01-02T15:04:05.000Z"
+	tsLayoutMicros = "2006-01-02T15:04:05.000000Z"
+	tsLayoutNanos  = "2006-01-02T15:04:05.000000000Z"
+)
 
 // Time is an abstraction for the time.Time type
 type Time time.Time
 
+var defaultTimeFormatter = dtfmt.MustNewFormatter("yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'")
+
+var defaultParseFormats = []string{
+	tsLayoutMillis,
+	tsLayoutMicros,
+	tsLayoutNanos,
+}
+
 // MarshalJSON implements json.Marshaler interface.
 // The time is a quoted string in the JsTsLayout format.
 func (t Time) MarshalJSON() ([]byte, error) {
-	return json.Marshal(time.Time(t).UTC().Format(TsLayout))
+	str, _ := defaultTimeFormatter.Format(time.Time(t).UTC())
+	return json.Marshal(str)
 }
 
 // UnmarshalJSON implements js.Unmarshaler interface.
 // The time is expected to be a quoted string in TsLayout
 // format.
 func (t *Time) UnmarshalJSON(data []byte) (err error) {
 	if data[0] != []byte(`"`)[0] || data[len(data)-1] != []byte(`"`)[0] {
-		return errors.New("Not quoted")
+		return errors.New("not quoted")
 	}
 	*t, err = ParseTime(string(data[1 : len(data)-1]))
-	return
+	return err
 }
 
 func (t Time) Hash32(h hash.Hash32) error {
 	err := binary.Write(h, binary.LittleEndian, time.Time(t).UnixNano())
 	return err
 }
 
-// ParseTime parses a time in the TsLayout format.
+// ParseTime parses a time in the MillisTsLayout, then micros and finally nanos.
 func ParseTime(timespec string) (Time, error) {
-	t, err := time.Parse(TsLayout, timespec)
+	var err error
+	var t time.Time
+
+	for _, layout := range defaultParseFormats {
+		t, err = time.Parse(layout, timespec)
+		if err == nil {
+			break
+		}
+	}
+
 	return Time(t), err
 }
 
 func (t Time) String() string {
-	return time.Time(t).UTC().Format(TsLayout)
+	str, _ := defaultTimeFormatter.Format(time.Time(t).UTC())
+	return str
 }
 
 // MustParseTime is a convenience equivalent of the ParseTime function

libbeat/common/datetime_test.go

@@ -67,7 +67,7 @@ func TestParseTimeNegative(t *testing.T) {
 	tests := []inputOutput{
 		{
 			Input: "2015-02-29TT14:06:05.071Z",
-			Err:   "parsing time \"2015-02-29TT14:06:05.071Z\" as \"2006-01-02T15:04:05.000Z\": cannot parse \"T14:06:05.071Z\" as \"15\"",
+			Err:   "parsing time \"2015-02-29TT14:06:05.071Z\" as \"2006-01-02T15:04:05.000000000Z\": cannot parse \"T14:06:05.071Z\" as \"15\"",
 		},
 	}
 

libbeat/common/dtfmt/builder.go

@@ -97,44 +97,47 @@ func (b *builder) add(e element) {
 	b.elements = append(b.elements, e)
 }
 
-func (b *builder) millisOfSecond(digits int) {
+func (b *builder) nanoOfSecond(digits int) {
 	if digits <= 0 {
 		return
 	}
 
-	switch digits {
-	case 1:
-		b.appendExtDecimal(ftMillisOfSecond, 100, 1, 1)
-	case 2:
-		b.appendExtDecimal(ftMillisOfSecond, 10, 2, 2)
-	case 3:
-		b.appendExtDecimal(ftMillisOfSecond, 0, 3, 3)
-	default:
-		b.appendExtDecimal(ftMillisOfSecond, 0, 3, 3)
-		b.appendZeros(digits - 3)
+	if digits <= 9 {
+		b.appendExtDecimal(ftNanoOfSecond, 9-digits, digits, digits)
+	} else {
+		b.appendExtDecimal(ftNanoOfSecond, 0, 9, 9)
+		b.appendZeros(digits - 9)
 	}
 }
 
-func (b *builder) millisOfDay(digits int) {
-	b.appendDecimal(ftMillisOfDay, digits, 8)
+func (b *builder) fractNanoOfSecond(digits int) {
+	const fractDigits = 3
+
+	if digits <= 0 {
+		return
+	}
+
+	// cap number of digits at 9, as we do not support higher precision and
+	// would remove trailing zeroes anyway.
+	if digits > 9 {
+		digits = 9
+	}
+
+	minDigits := fractDigits
+	if digits < minDigits {
+		minDigits = digits
+	}
+	b.add(paddedNumber{ftNanoOfSecond, 9 - digits, minDigits, digits, fractDigits, false})
 }
 
 func (b *builder) secondOfMinute(digits int) {
 	b.appendDecimal(ftSecondOfMinute, digits, 2)
 }
 
-func (b *builder) secondOfDay(digits int) {
-	b.appendDecimal(ftSecondOfDay, digits, 5)
-}
-
 func (b *builder) minuteOfHour(digits int) {
 	b.appendDecimal(ftMinuteOfHour, digits, 2)
 }
 
-func (b *builder) minuteOfDay(digits int) {
-	b.appendDecimal(ftMinuteOfDay, digits, 4)
-}
-
 func (b *builder) hourOfDay(digits int) {
 	b.appendDecimal(ftHourOfDay, digits, 2)
 }
@@ -233,12 +236,12 @@ func (b *builder) appendDecimalValue(ft fieldType, minDigits, maxDigits int, sig
 	if minDigits <= 1 {
 		b.add(unpaddedNumber{ft, maxDigits, signed})
 	} else {
-		b.add(paddedNumber{ft, 0, minDigits, maxDigits, signed})
+		b.add(paddedNumber{ft, 0, minDigits, maxDigits, 0, signed})
 	}
 }
 
-func (b *builder) appendExtDecimal(ft fieldType, div, minDigits, maxDigits int) {
-	b.add(paddedNumber{ft, div, minDigits, maxDigits, false})
+func (b *builder) appendExtDecimal(ft fieldType, divExp, minDigits, maxDigits int) {
+	b.add(paddedNumber{ft, divExp, minDigits, maxDigits, 0, false})
 }
 
 func (b *builder) appendDecimal(ft fieldType, minDigits, maxDigits int) {

libbeat/common/dtfmt/ctx.go

@@ -31,7 +31,7 @@ type ctx struct {
 	isoWeek, isoYear int
 
 	hour, min, sec int
-	millis         int
+	nano           int
 
 	tzOffset int
 
@@ -43,7 +43,7 @@ type ctxConfig struct {
 	clock    bool
 	weekday  bool
 	yearday  bool
-	millis   bool
+	nano     bool
 	iso      bool
 	tzOffset bool
 }
@@ -59,8 +59,8 @@ func (c *ctx) initTime(config *ctxConfig, t time.Time) {
 		c.isoYear, c.isoWeek = t.ISOWeek()
 	}
 
-	if config.millis {
-		c.millis = t.Nanosecond() / 1000000
+	if config.nano {
+		c.nano = t.Nanosecond()
 	}
 
 	if config.yearday {
@@ -84,8 +84,8 @@ func (c *ctxConfig) enableClock() {
 	c.clock = true
 }
 
-func (c *ctxConfig) enableMillis() {
-	c.millis = true
+func (c *ctxConfig) enableNano() {
+	c.nano = true
 }
 
 func (c *ctxConfig) enableWeekday() {
@@ -103,7 +103,3 @@ func (c *ctxConfig) enableISO() {
 func (c *ctxConfig) enableTimeZoneOffset() {
 	c.tzOffset = true
 }
-
-func isLeap(year int) bool {
-	return year%4 == 0 && (year%100 != 0 || year%400 == 0)
-}

libbeat/common/dtfmt/doc.go

@@ -22,30 +22,31 @@
 //
 //  Symbol  Meaning                      Type     Supported Examples
 //  ------  -------                      -------  --------- -------
-//  G       era                          text      no       AD
-//  C       century of era (>=0)      number    no       20
-//  Y       year of era (>=0)         year      yes      1996
-//
-//  x       weekyear                     year      yes      1996
-//  w       week of weekyear             number    yes      27
-//  e       day of week                  number    yes      2
-//  E       day of week                  text      yes      Tuesday; Tue
-//
-//  y       year                         year      yes      1996
-//  D       day of year                  number    yes      189
-//  M       month of year                month     yes      July; Jul; 07
-//  d       day of month                 number    yes      10
-//
-//  a       halfday of day               text      yes      PM
-//  K       hour of halfday (0~11)       number    yes      0
-//  h       clockhour of halfday (1~12)  number    yes      12
-//
-//  H       hour of day (0~23)           number    yes      0
-//  k       clockhour of day (1~24)      number    yes      24
-//  m       minute of hour               number    yes      30
-//  s       second of minute             number    yes      55
-//  S       fraction of second           millis    no       978
-//
+//  G       era                          text           no       AD
+//  C       century of era (>=0)      number         no       20
+//  Y       year of era (>=0)         year           yes      1996
+//
+//  x       weekyear                     year           yes      1996
+//  w       week of weekyear             number         yes      27
+//  e       day of week                  number         yes      2
+//  E       day of week                  text           yes      Tuesday; Tue
+//
+//  y       year                         year           yes      1996
+//  D       day of year                  number         yes      189
+//  M       month of year                month          yes      July; Jul; 07
+//  d       day of month                 number         yes      10
+//
+//  a       halfday of day               text           yes      PM
+//  K       hour of halfday (0~11)       number         yes      0
+//  h       clockhour of halfday (1~12)  number         yes      12
+//
+//  H       hour of day (0~23)           number         yes      0
+//  k       clockhour of day (1~24)      number         yes      24
+//  m       minute of hour               number         yes      30
+//  s       second of minute             number         yes      55
+//  S       fraction of second           nanoseconds    yes     978000
+//  f       fraction of seconds          nanoseconds    yes     123456789
+//          multiple of 3
 //  z       time zone                    text      no       Pacific Standard Time; PST
 //  Z       time zone offset/id          zone      no       -0800; -08:00; America/Los_Angeles
 //

libbeat/common/dtfmt/dtfmt_test.go

@@ -82,10 +82,37 @@ func TestFormat(t *testing.T) {
 		{mkTime(8, 5, 24, 0), "kk:mm:ss aa", "09:05:24 AM"},
 		{mkTime(20, 5, 24, 0), "k:m:s a", "21:5:24 PM"},
 		{mkTime(20, 5, 24, 0), "kk:mm:ss aa", "21:05:24 PM"},
-		{mkTime(1, 2, 3, 123), "S", "1"},
-		{mkTime(1, 2, 3, 123), "SS", "12"},
-		{mkTime(1, 2, 3, 123), "SSS", "123"},
-		{mkTime(1, 2, 3, 123), "SSSS", "1230"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "S", "1"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "SS", "12"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "SSS", "123"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "SSSS", "1230"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "f", "1"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "ff", "12"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "fff", "123"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "ffff", "123"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "fffff", "123"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "ffffff", "123"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "fffffff", "123"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "ffffffff", "123"},
+		{mkTime(1, 2, 3, 123*time.Millisecond), "fffffffff", "123"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "f", "0"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "ff", "00"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "fff", "000"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "ffff", "0001"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "fffff", "00012"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "ffffff", "000123"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "fffffff", "000123"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "ffffffff", "000123"},
+		{mkTime(1, 2, 3, 123*time.Microsecond), "fffffffff", "000123"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "f", "0"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "ff", "00"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "fff", "000"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "ffff", "000"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "fffff", "000"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "ffffff", "000"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "fffffff", "0000001"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "ffffffff", "00000012"},
+		{mkTime(1, 2, 3, 123*time.Nanosecond), "fffffffff", "000000123"},
 
 		// literals
 		{time.Now(), "--=++,_!/?\\[]{}@#$%^&*()", "--=++,_!/?\\[]{}@#$%^&*()"},
@@ -94,14 +121,26 @@ func TestFormat(t *testing.T) {
 		{time.Now(), "'plain' '' 'text'", "plain ' text"},
 		{time.Now(), "'plain '' text'", "plain ' text"},
 
-		// beats timestamp
-		{mkDateTime(2017, 1, 2, 4, 6, 7, 123),
+		// timestamps with microseconds precision only
+		{mkDateTime(2017, 1, 2, 4, 6, 7, 123*time.Millisecond),
+			"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
+			"2017-01-02T04:06:07.123Z"},
+		{mkDateTime(2017, 1, 2, 4, 6, 7, 123456*time.Microsecond),
 			"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
 			"2017-01-02T04:06:07.123Z"},
 
 		// beats timestamp
-		{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
-			"yyyy-MM-dd'T'HH:mm:ss.SSSz",
+		{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123*time.Millisecond, time.FixedZone("PST", -8*60*60)),
+			"yyyy-MM-dd'T'HH:mm:ss.fffffffffz",
+			"2017-01-02T04:06:07.123-08:00"},
+
+		// beats nanoseconds timestamp
+		{mkDateTime(2017, 1, 2, 4, 6, 7, 123*time.Nanosecond),
+			"yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'",
+			"2017-01-02T04:06:07.000000123Z"},
+
+		{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123*time.Millisecond, time.FixedZone("PST", -8*60*60)),
+			"yyyy-MM-dd'T'HH:mm:ss.fffffffffz",
 			"2017-01-02T04:06:07.123-08:00"},
 	}
 
@@ -123,14 +162,14 @@ func mkDate(y, m, d int) time.Time {
 	return mkDateTime(y, m, d, 0, 0, 0, 0)
 }
 
-func mkTime(h, m, s, S int) time.Time {
+func mkTime(h, m, s int, S time.Duration) time.Time {
 	return mkDateTime(2000, 1, 1, h, m, s, S)
 }
 
-func mkDateTime(y, M, d, h, m, s, S int) time.Time {
+func mkDateTime(y, M, d, h, m, s int, S time.Duration) time.Time {
 	return mkDateTimeWithLocation(y, M, d, h, m, s, S, time.UTC)
 }
 
-func mkDateTimeWithLocation(y, M, d, h, m, s, S int, l *time.Location) time.Time {
-	return time.Date(y, time.Month(M), d, h, m, s, S*1000000, l)
+func mkDateTimeWithLocation(y, M, d, h, m, s int, S time.Duration, l *time.Location) time.Time {
+	return time.Date(y, time.Month(M), d, h, m, s, int(S), l)
 }