From 3fa10ff910953b55547c019ddcd6ff96ab2079ee Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Thu, 5 Sep 2024 15:53:33 +0100 Subject: [PATCH] [pkg/ottl]: Add Sort converter (#34283) **Description:** Add `Sort` function to sort array to ascending order or descending order `Sort(target, Optional[order])` The Sort Converter preserves the data type of the original elements while sorting. The behavior varies based on the types of elements in the target slice: | Element Types | Sorting Behavior | Return Value | |---------------|-------------------------------------|--------------| | Integers | Sorts as integers | Sorted array of integers | | Doubles | Sorts as doubles | Sorted array of doubles | | Integers and doubles | Converts all to doubles, then sorts | Sorted array of integers and doubles | | Strings | Sorts as strings | Sorted array of strings | | Booleans | Converts all to strings, then sorts | Sorted array of booleans | | Mix of integers, doubles, booleans, and strings | Converts all to strings, then sorts | Sorted array of mixed types | | Any other types | N/A | Returns an error | Examples: - `Sort(attributes["device.tags"])` - `Sort(attributes["device.tags"], "desc")` **Link to tracking Issue:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/34200 **Testing:** - Unit tests - E2E tests **Documentation:** readme for Sort function --------- Co-authored-by: Evan Bradley <11745660+evan-bradley@users.noreply.github.com> --- .chloggen/ottl_sort_func.yaml | 27 +++ pkg/ottl/e2e/e2e_test.go | 56 ++++++ pkg/ottl/ottlfuncs/README.md | 30 ++- pkg/ottl/ottlfuncs/func_sort.go | 253 ++++++++++++++++++++++++ pkg/ottl/ottlfuncs/func_sort_test.go | 280 +++++++++++++++++++++++++++ pkg/ottl/ottlfuncs/functions.go | 1 + 6 files changed, 646 insertions(+), 1 deletion(-) create mode 100644 .chloggen/ottl_sort_func.yaml create mode 100644 pkg/ottl/ottlfuncs/func_sort.go create mode 100644 pkg/ottl/ottlfuncs/func_sort_test.go diff --git a/.chloggen/ottl_sort_func.yaml b/.chloggen/ottl_sort_func.yaml new file mode 100644 index 000000000000..7b9d32749d9c --- /dev/null +++ b/.chloggen/ottl_sort_func.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add `Sort` function to sort array to ascending order or descending order + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34200] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/pkg/ottl/e2e/e2e_test.go b/pkg/ottl/e2e/e2e_test.go index 97eda4f67e8c..f1c0d882d021 100644 --- a/pkg/ottl/e2e/e2e_test.go +++ b/pkg/ottl/e2e/e2e_test.go @@ -643,6 +643,62 @@ func Test_e2e_converters(t *testing.T) { tCtx.GetLogRecord().Attributes().PutStr("test", "5b722b307fce6c944905d132691d5e4a2214b7fe92b738920eb3fce3a90420a19511c3010a0e7712b054daef5b57bad59ecbd93b3280f210578f547f4aed4d25") }, }, + { + statement: `set(attributes["test"], Sort(Split(attributes["flags"], "|"), "desc"))`, + want: func(tCtx ottllog.TransformContext) { + s := tCtx.GetLogRecord().Attributes().PutEmptySlice("test") + s.AppendEmpty().SetStr("C") + s.AppendEmpty().SetStr("B") + s.AppendEmpty().SetStr("A") + }, + }, + { + statement: `set(attributes["test"], Sort([true, false, false]))`, + want: func(tCtx ottllog.TransformContext) { + s := tCtx.GetLogRecord().Attributes().PutEmptySlice("test") + s.AppendEmpty().SetBool(false) + s.AppendEmpty().SetBool(false) + s.AppendEmpty().SetBool(true) + }, + }, + { + statement: `set(attributes["test"], Sort([3, 6, 9], "desc"))`, + want: func(tCtx ottllog.TransformContext) { + s := tCtx.GetLogRecord().Attributes().PutEmptySlice("test") + s.AppendEmpty().SetInt(9) + s.AppendEmpty().SetInt(6) + s.AppendEmpty().SetInt(3) + }, + }, + { + statement: `set(attributes["test"], Sort([Double(1.5), Double(10.2), Double(2.3), Double(0.5)]))`, + want: func(tCtx ottllog.TransformContext) { + s := tCtx.GetLogRecord().Attributes().PutEmptySlice("test") + s.AppendEmpty().SetDouble(0.5) + s.AppendEmpty().SetDouble(1.5) + s.AppendEmpty().SetDouble(2.3) + s.AppendEmpty().SetDouble(10.2) + }, + }, + { + statement: `set(attributes["test"], Sort([Int(11), Double(2.2), Double(-1)]))`, + want: func(tCtx ottllog.TransformContext) { + s := tCtx.GetLogRecord().Attributes().PutEmptySlice("test") + s.AppendEmpty().SetDouble(-1) + s.AppendEmpty().SetDouble(2.2) + s.AppendEmpty().SetInt(11) + }, + }, + { + statement: `set(attributes["test"], Sort([false, Int(11), Double(2.2), "three"]))`, + want: func(tCtx ottllog.TransformContext) { + s := tCtx.GetLogRecord().Attributes().PutEmptySlice("test") + s.AppendEmpty().SetInt(11) + s.AppendEmpty().SetDouble(2.2) + s.AppendEmpty().SetBool(false) + s.AppendEmpty().SetStr("three") + }, + }, { statement: `set(span_id, SpanID(0x0000000000000000))`, want: func(tCtx ottllog.TransformContext) { diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 129104fe65fb..42d60d6dc89e 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -449,6 +449,7 @@ Available Converters: - [SHA1](#sha1) - [SHA256](#sha256) - [SHA512](#sha512) +- [Sort](#sort) - [SpanID](#spanid) - [Split](#split) - [String](#string) @@ -1318,7 +1319,6 @@ Examples: - `SHA256(attributes["device.name"])` - - `SHA256("name")` ### SHA512 @@ -1338,6 +1338,34 @@ Examples: - `SHA512("name")` +### Sort + +`Sort(target, Optional[order])` + +The `Sort` Converter sorts the `target` array in either ascending or descending order. + +`target` is an array or `pcommon.Slice` typed field containing the elements to be sorted. + +`order` is a string specifying the sort order. Must be either `asc` or `desc`. The default value is `asc`. + +The Sort Converter preserves the data type of the original elements while sorting. +The behavior varies based on the types of elements in the target slice: + +| Element Types | Sorting Behavior | Return Value | +|---------------|-------------------------------------|--------------| +| Integers | Sorts as integers | Sorted array of integers | +| Doubles | Sorts as doubles | Sorted array of doubles | +| Integers and doubles | Converts all to doubles, then sorts | Sorted array of integers and doubles | +| Strings | Sorts as strings | Sorted array of strings | +| Booleans | Converts all to strings, then sorts | Sorted array of booleans | +| Mix of integers, doubles, booleans, and strings | Converts all to strings, then sorts | Sorted array of mixed types | +| Any other types | N/A | Returns an error | + +Examples: + +- `Sort(attributes["device.tags"])` +- `Sort(attributes["device.tags"], "desc")` + ### SpanID `SpanID(bytes)` diff --git a/pkg/ottl/ottlfuncs/func_sort.go b/pkg/ottl/ottlfuncs/func_sort.go new file mode 100644 index 000000000000..4c9f56c820ce --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_sort.go @@ -0,0 +1,253 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs" + +import ( + "cmp" + "context" + "fmt" + "slices" + "strconv" + + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +const ( + sortAsc = "asc" + sortDesc = "desc" +) + +type SortArguments[K any] struct { + Target ottl.Getter[K] + Order ottl.Optional[string] +} + +func NewSortFactory[K any]() ottl.Factory[K] { + return ottl.NewFactory("Sort", &SortArguments[K]{}, createSortFunction[K]) +} + +func createSortFunction[K any](_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { + args, ok := oArgs.(*SortArguments[K]) + + if !ok { + return nil, fmt.Errorf("SortFactory args must be of type *SortArguments[K]") + } + + order := sortAsc + if !args.Order.IsEmpty() { + o := args.Order.Get() + switch o { + case sortAsc, sortDesc: + order = o + default: + return nil, fmt.Errorf("invalid arguments: %s. Order should be either \"%s\" or \"%s\"", o, sortAsc, sortDesc) + } + } + + return sort(args.Target, order), nil +} + +func sort[K any](target ottl.Getter[K], order string) ottl.ExprFunc[K] { + return func(ctx context.Context, tCtx K) (any, error) { + val, err := target.Get(ctx, tCtx) + if err != nil { + return nil, err + } + + switch v := val.(type) { + case pcommon.Slice: + return sortSlice(v, order) + case pcommon.Value: + if v.Type() == pcommon.ValueTypeSlice { + return sortSlice(v.Slice(), order) + } + return nil, fmt.Errorf("sort with unsupported type: '%s'. Target is not a list", v.Type().String()) + case []any: + // handle Sort([1,2,3]) + slice := pcommon.NewValueSlice().SetEmptySlice() + if err := slice.FromRaw(v); err != nil { + return nil, fmt.Errorf("sort with unsupported type: '%T'. Target is not a list of primitive types; %w", v, err) + } + return sortSlice(slice, order) + case []string: + dup := makeCopy(v) + return sortTypedSlice(dup, order), nil + case []int64: + dup := makeCopy(v) + return sortTypedSlice(dup, order), nil + case []float64: + dup := makeCopy(v) + return sortTypedSlice(dup, order), nil + case []bool: + var strings []string + for _, b := range v { + strings = append(strings, strconv.FormatBool(b)) + } + + sortTypedSlice(strings, order) + + bools := make([]bool, len(strings)) + for i, s := range strings { + boolValue, _ := strconv.ParseBool(s) + bools[i] = boolValue + } + return bools, nil + default: + return nil, fmt.Errorf("sort with unsupported type: '%T'. Target is not a list", v) + } + } +} + +// sortSlice sorts a pcommon.Slice based on the specified order. +// It gets the common type for all elements in the slice and converts all elements to this common type, creating a new copy +// Parameters: +// - slice: The pcommon.Slice to be sorted +// - order: The sort order. "asc" for ascending, "desc" for descending +// +// Returns: +// - A sorted slice as []any or the original pcommon.Slice +// - An error if an unsupported type is encountered +func sortSlice(slice pcommon.Slice, order string) (any, error) { + length := slice.Len() + if length == 0 { + return slice, nil + } + + commonType, ok := findCommonValueType(slice) + if !ok { + return slice, nil + } + + switch commonType { + case pcommon.ValueTypeInt: + arr := makeConvertedCopy(slice, func(idx int) int64 { + return slice.At(idx).Int() + }) + return sortConvertedSlice(arr, order), nil + case pcommon.ValueTypeDouble: + arr := makeConvertedCopy(slice, func(idx int) float64 { + s := slice.At(idx) + if s.Type() == pcommon.ValueTypeInt { + return float64(s.Int()) + } + + return s.Double() + }) + return sortConvertedSlice(arr, order), nil + case pcommon.ValueTypeStr: + arr := makeConvertedCopy(slice, func(idx int) string { + return slice.At(idx).AsString() + }) + return sortConvertedSlice(arr, order), nil + default: + return nil, fmt.Errorf("sort with unsupported type: '%T'", commonType) + } +} + +type targetType interface { + ~int64 | ~float64 | ~string +} + +// findCommonValueType determines the most appropriate common type for all elements in a pcommon.Slice. +// It returns two values: +// - A pcommon.ValueType representing the desired common type for all elements. +// Mixed Numeric types return ValueTypeDouble. Integer type returns ValueTypeInt. Double type returns ValueTypeDouble. +// String, Bool, Empty and mixed of the mentioned types return ValueTypeStr, as they require string conversion for comparison. +// - A boolean indicating whether a common type could be determined (true) or not (false). +// returns false for ValueTypeMap, ValueTypeSlice and ValueTypeBytes. They are unsupported types for sort. +func findCommonValueType(slice pcommon.Slice) (pcommon.ValueType, bool) { + length := slice.Len() + if length == 0 { + return pcommon.ValueTypeEmpty, false + } + + wantType := slice.At(0).Type() + wantStr := false + wantDouble := false + + for i := 0; i < length; i++ { + value := slice.At(i) + currType := value.Type() + + switch currType { + case pcommon.ValueTypeInt: + if wantType == pcommon.ValueTypeDouble { + wantDouble = true + } + case pcommon.ValueTypeDouble: + if wantType == pcommon.ValueTypeInt { + wantDouble = true + } + case pcommon.ValueTypeStr, pcommon.ValueTypeBool, pcommon.ValueTypeEmpty: + wantStr = true + default: + return pcommon.ValueTypeEmpty, false + } + } + + if wantStr { + wantType = pcommon.ValueTypeStr + } else if wantDouble { + wantType = pcommon.ValueTypeDouble + } + + return wantType, true +} + +func makeCopy[T targetType](src []T) []T { + dup := make([]T, len(src)) + copy(dup, src) + return dup +} + +func sortTypedSlice[T targetType](arr []T, order string) []T { + if len(arr) == 0 { + return arr + } + + slices.SortFunc(arr, func(a, b T) int { + if order == sortDesc { + return cmp.Compare(b, a) + } + return cmp.Compare(a, b) + }) + + return arr +} + +type convertedValue[T targetType] struct { + value T + originalValue any +} + +func makeConvertedCopy[T targetType](slice pcommon.Slice, converter func(idx int) T) []convertedValue[T] { + length := slice.Len() + var out []convertedValue[T] + for i := 0; i < length; i++ { + cv := convertedValue[T]{ + value: converter(i), + originalValue: slice.At(i).AsRaw(), + } + out = append(out, cv) + } + return out +} + +func sortConvertedSlice[T targetType](cvs []convertedValue[T], order string) []any { + slices.SortFunc(cvs, func(a, b convertedValue[T]) int { + if order == sortDesc { + return cmp.Compare(b.value, a.value) + } + return cmp.Compare(a.value, b.value) + }) + + var out []any + for _, cv := range cvs { + out = append(out, cv.originalValue) + } + + return out +} diff --git a/pkg/ottl/ottlfuncs/func_sort_test.go b/pkg/ottl/ottlfuncs/func_sort_test.go new file mode 100644 index 000000000000..48dede0a2fa9 --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_sort_test.go @@ -0,0 +1,280 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ottlfuncs + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func Test_Sort(t *testing.T) { + + pMap := pcommon.NewValueMap().SetEmptyMap() + pMap.PutStr("k", "v") + emptySlice := pcommon.NewValueSlice().SetEmptySlice() + + tests := []struct { + name string + getter ottl.Getter[any] + order string + expected any + err bool + }{ + { + name: "int slice", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{9, 6, 3}) + return s, nil + }, + }, + order: sortAsc, + expected: []any{int64(3), int64(6), int64(9)}, + }, + { + name: "int slice desc", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{3, 6, 9}) + return s, nil + }, + }, + order: sortDesc, + expected: []any{int64(9), int64(6), int64(3)}, + }, + { + name: "string slice", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{"i", "am", "awesome", "slice"}) + return s, nil + }, + }, + order: sortAsc, + expected: []any{"am", "awesome", "i", "slice"}, + }, + { + name: "double slice", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{1.5, 10.2, 2.3, 0.5}) + return s, nil + }, + }, + order: sortAsc, + expected: []any{0.5, 1.5, 2.3, 10.2}, + }, + { + name: "empty slice", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + return s, nil + }, + }, + order: sortAsc, + expected: emptySlice, + }, + { + name: "bool slice compares as string", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{true, false, true, false}) + return s, nil + }, + }, + order: sortAsc, + expected: []any{false, false, true, true}, + }, + { + name: "mixed types slice compares as string", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{1, "two", 3.33, false}) + return s, nil + }, + }, + order: sortAsc, + expected: []any{int64(1), 3.33, false, "two"}, + }, + { + name: "double and string slice compares as string", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{1.5, "10.2", 2.3, 0.5}) + return s, nil + }, + }, + order: sortAsc, + expected: []any{0.5, 1.5, "10.2", 2.3}, + }, + { + name: "mixed numeric types slice compares as double", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{0, 2, 3.33, 0}) + return s, nil + }, + }, + order: sortAsc, + expected: []any{int64(0), int64(0), int64(2), 3.33}, + }, + { + name: "mixed numeric types slice compares as double desc", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + s := pcommon.NewValueSlice().SetEmptySlice() + _ = s.FromRaw([]any{3.14, 2, 3.33, 0}) + return s, nil + }, + }, + order: sortDesc, + expected: []any{3.33, 3.14, int64(2), int64(0)}, + }, + { + name: "[]any compares as string", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return []any{1, "two", 3.33, false}, nil + }, + }, + order: sortAsc, + expected: []any{int64(1), 3.33, false, "two"}, + }, + { + name: "[]string", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return []string{"A", "a", "aa"}, nil + }, + }, + order: sortAsc, + expected: []string{"A", "a", "aa"}, + }, + { + name: "[]bool compares as string", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return []bool{true, false}, nil + }, + }, + order: sortAsc, + expected: []bool{false, true}, + }, + { + name: "[]int64", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return []int64{6, 3, 9}, nil + }, + }, + order: sortAsc, + expected: []int64{3, 6, 9}, + }, + { + name: "[]float64", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return []float64{1.5, 10.2, 2.3, 0.5}, nil + }, + }, + order: sortAsc, + expected: []float64{0.5, 1.5, 2.3, 10.2}, + }, + { + name: "pcommon.Value is a slice", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + pv := pcommon.NewValueEmpty() + s := pv.SetEmptySlice() + _ = s.FromRaw([]any{"a", "slice", "a"}) + return pv, nil + }, + }, + order: sortAsc, + expected: []any{"a", "a", "slice"}, + }, + { + name: "pcommon.Value is empty", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + pv := pcommon.NewValueEmpty() + return pv, nil + }, + }, + order: sortAsc, + expected: nil, + err: true, + }, + { + name: "unsupported ValueTypeMap", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return pMap, nil + }, + }, + order: sortAsc, + expected: nil, + err: true, + }, + { + name: "unsupported bytes", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return []byte("still fine"), nil + }, + }, + order: sortAsc, + expected: nil, + err: true, + }, + { + name: "unsupported string", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "no change", nil + }, + }, + order: sortAsc, + expected: nil, + err: true, + }, + { + name: "[]any with a map", + getter: ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return []any{map[string]string{"some": "invalid kv"}}, nil + }, + }, + order: sortAsc, + expected: nil, + err: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + exprFunc := sort(tt.getter, tt.order) + result, err := exprFunc(nil, nil) + if tt.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/ottl/ottlfuncs/functions.go b/pkg/ottl/ottlfuncs/functions.go index 1f0dfd68644a..764aa5bbb297 100644 --- a/pkg/ottl/ottlfuncs/functions.go +++ b/pkg/ottl/ottlfuncs/functions.go @@ -73,6 +73,7 @@ func converters[K any]() []ottl.Factory[K] { NewSHA1Factory[K](), NewSHA256Factory[K](), NewSHA512Factory[K](), + NewSortFactory[K](), NewSpanIDFactory[K](), NewSplitFactory[K](), NewFormatFactory[K](),