From 188e3ac6fd64ddd6f7b8b249f4ca6f3eec75f5f3 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Thu, 9 Dec 2021 11:14:33 -0500 Subject: [PATCH 1/4] Reuse identical KeyValues across traces in a SearchPage to reduce final binary size --- pkg/tempofb/SearchBlockHeader_util.go | 2 +- pkg/tempofb/search_entry_mutable.go | 60 +++++++++++++ pkg/tempofb/search_page_builder.go | 66 ++++++++++++++ pkg/tempofb/searchdata_util.go | 115 ------------------------ pkg/tempofb/searchdatamap.go | 125 +++++++++++++------------- 5 files changed, 192 insertions(+), 176 deletions(-) create mode 100644 pkg/tempofb/search_entry_mutable.go create mode 100644 pkg/tempofb/search_page_builder.go diff --git a/pkg/tempofb/SearchBlockHeader_util.go b/pkg/tempofb/SearchBlockHeader_util.go index 02507c26b20..3b46c709a70 100644 --- a/pkg/tempofb/SearchBlockHeader_util.go +++ b/pkg/tempofb/SearchBlockHeader_util.go @@ -63,7 +63,7 @@ func (s *SearchBlockHeaderMutable) Contains(k []byte, v []byte, _ *KeyValues) bo func (s *SearchBlockHeaderMutable) ToBytes() []byte { b := flatbuffers.NewBuilder(1024) - tags := s.Tags.WriteToBuilder(b) + tags := WriteSearchDataMap(b, s.Tags, nil) SearchBlockHeaderStart(b) SearchBlockHeaderAddMinDurationNanos(b, s.MinDur) diff --git a/pkg/tempofb/search_entry_mutable.go b/pkg/tempofb/search_entry_mutable.go new file mode 100644 index 00000000000..222a2f4a768 --- /dev/null +++ b/pkg/tempofb/search_entry_mutable.go @@ -0,0 +1,60 @@ +package tempofb + +import ( + flatbuffers "github.com/google/flatbuffers/go" + "github.com/grafana/tempo/tempodb/encoding/common" +) + +// SearchEntryMutable is a mutable form of the flatbuffer-compiled SearchEntry struct to make building and transporting easier. +type SearchEntryMutable struct { + TraceID common.ID + Tags SearchDataMap + StartTimeUnixNano uint64 + EndTimeUnixNano uint64 +} + +// AddTag adds the unique tag name and value to the search data. No effect if the pair is already present. +func (s *SearchEntryMutable) AddTag(k string, v string) { + if s.Tags == nil { + s.Tags = NewSearchDataMap() + } + s.Tags.Add(k, v) +} + +// SetStartTimeUnixNano records the earliest of all timestamps passed to this function. +func (s *SearchEntryMutable) SetStartTimeUnixNano(t uint64) { + if t > 0 && (s.StartTimeUnixNano == 0 || s.StartTimeUnixNano > t) { + s.StartTimeUnixNano = t + } +} + +// SetEndTimeUnixNano records the latest of all timestamps passed to this function. +func (s *SearchEntryMutable) SetEndTimeUnixNano(t uint64) { + if t > 0 && t > s.EndTimeUnixNano { + s.EndTimeUnixNano = t + } +} + +func (s *SearchEntryMutable) ToBytes() []byte { + b := flatbuffers.NewBuilder(2048) + offset := s.WriteToBuilder(b, nil) + b.Finish(offset) + return b.FinishedBytes() +} + +func (s *SearchEntryMutable) WriteToBuilder(b *flatbuffers.Builder, kvCache map[uint64]flatbuffers.UOffsetT) flatbuffers.UOffsetT { + if s.Tags == nil { + s.Tags = NewSearchDataMap() + } + + idOffset := b.CreateByteString(s.TraceID) + + tagOffset := WriteSearchDataMap(b, s.Tags, kvCache) + + SearchEntryStart(b) + SearchEntryAddId(b, idOffset) + SearchEntryAddStartTimeUnixNano(b, s.StartTimeUnixNano) + SearchEntryAddEndTimeUnixNano(b, s.EndTimeUnixNano) + SearchEntryAddTags(b, tagOffset) + return SearchEntryEnd(b) +} diff --git a/pkg/tempofb/search_page_builder.go b/pkg/tempofb/search_page_builder.go new file mode 100644 index 00000000000..50c92d3ff51 --- /dev/null +++ b/pkg/tempofb/search_page_builder.go @@ -0,0 +1,66 @@ +package tempofb + +import flatbuffers "github.com/google/flatbuffers/go" + +type SearchPageBuilder struct { + builder *flatbuffers.Builder + allTags SearchDataMap + pageEntries []flatbuffers.UOffsetT + kvcache map[uint64]flatbuffers.UOffsetT +} + +func NewSearchPageBuilder() *SearchPageBuilder { + return &SearchPageBuilder{ + builder: flatbuffers.NewBuilder(1024), + allTags: NewSearchDataMap(), + kvcache: map[uint64]flatbuffers.UOffsetT{}, + } +} + +func (b *SearchPageBuilder) AddData(data *SearchEntryMutable) int { + if data.Tags != nil { + data.Tags.Range(func(k, v string) { + b.allTags.Add(k, v) + }) + } + + oldOffset := b.builder.Offset() + offset := data.WriteToBuilder(b.builder, b.kvcache) + b.pageEntries = append(b.pageEntries, offset) + + // bytes written + return int(offset - oldOffset) +} + +func (b *SearchPageBuilder) Finish() []byte { + // At this point all individual entries have been written + // to the fb builder. Now we need to wrap them up in the final + // batch object. + + // Create vector + SearchPageStartEntriesVector(b.builder, len(b.pageEntries)) + for _, entry := range b.pageEntries { + b.builder.PrependUOffsetT(entry) + } + entryVector := b.builder.EndVector(len(b.pageEntries)) + + // Create batch-level tags + tagOffset := WriteSearchDataMap(b.builder, b.allTags, b.kvcache) + + // Write final batch object + SearchPageStart(b.builder) + SearchPageAddEntries(b.builder, entryVector) + SearchPageAddTags(b.builder, tagOffset) + batch := SearchPageEnd(b.builder) + b.builder.Finish(batch) + buf := b.builder.FinishedBytes() + + return buf +} + +func (b *SearchPageBuilder) Reset() { + b.builder.Reset() + b.pageEntries = b.pageEntries[:0] + b.allTags = NewSearchDataMap() + b.kvcache = map[uint64]flatbuffers.UOffsetT{} +} diff --git a/pkg/tempofb/searchdata_util.go b/pkg/tempofb/searchdata_util.go index 0591c74a162..2564ab3b43c 100644 --- a/pkg/tempofb/searchdata_util.go +++ b/pkg/tempofb/searchdata_util.go @@ -4,123 +4,8 @@ import ( "bytes" flatbuffers "github.com/google/flatbuffers/go" - "github.com/grafana/tempo/tempodb/encoding/common" ) -// SearchEntryMutable is a mutable form of the flatbuffer-compiled SearchEntry struct to make building and transporting easier. -type SearchEntryMutable struct { - TraceID common.ID - Tags SearchDataMap - StartTimeUnixNano uint64 - EndTimeUnixNano uint64 -} - -// AddTag adds the unique tag name and value to the search data. No effect if the pair is already present. -func (s *SearchEntryMutable) AddTag(k string, v string) { - if s.Tags == nil { - s.Tags = NewSearchDataMap() - } - s.Tags.Add(k, v) -} - -// SetStartTimeUnixNano records the earliest of all timestamps passed to this function. -func (s *SearchEntryMutable) SetStartTimeUnixNano(t uint64) { - if t > 0 && (s.StartTimeUnixNano == 0 || s.StartTimeUnixNano > t) { - s.StartTimeUnixNano = t - } -} - -// SetEndTimeUnixNano records the latest of all timestamps passed to this function. -func (s *SearchEntryMutable) SetEndTimeUnixNano(t uint64) { - if t > 0 && t > s.EndTimeUnixNano { - s.EndTimeUnixNano = t - } -} - -func (s *SearchEntryMutable) ToBytes() []byte { - b := flatbuffers.NewBuilder(2048) - offset := s.WriteToBuilder(b) - b.Finish(offset) - return b.FinishedBytes() -} - -func (s *SearchEntryMutable) WriteToBuilder(b *flatbuffers.Builder) flatbuffers.UOffsetT { - if s.Tags == nil { - s.Tags = NewSearchDataMap() - } - - idOffset := b.CreateByteString(s.TraceID) - - tagOffset := s.Tags.WriteToBuilder(b) - - SearchEntryStart(b) - SearchEntryAddId(b, idOffset) - SearchEntryAddStartTimeUnixNano(b, s.StartTimeUnixNano) - SearchEntryAddEndTimeUnixNano(b, s.EndTimeUnixNano) - SearchEntryAddTags(b, tagOffset) - return SearchEntryEnd(b) -} - -type SearchPageBuilder struct { - builder *flatbuffers.Builder - allTags SearchDataMap - pageEntries []flatbuffers.UOffsetT -} - -func NewSearchPageBuilder() *SearchPageBuilder { - return &SearchPageBuilder{ - builder: flatbuffers.NewBuilder(1024), - allTags: NewSearchDataMap(), - } -} - -func (b *SearchPageBuilder) AddData(data *SearchEntryMutable) int { - if data.Tags != nil { - data.Tags.Range(func(k, v string) { - b.allTags.Add(k, v) - }) - } - - oldOffset := b.builder.Offset() - offset := data.WriteToBuilder(b.builder) - b.pageEntries = append(b.pageEntries, offset) - - // bytes written - return int(offset - oldOffset) -} - -func (b *SearchPageBuilder) Finish() []byte { - // At this point all individual entries have been written - // to the fb builder. Now we need to wrap them up in the final - // batch object. - - // Create vector - SearchPageStartEntriesVector(b.builder, len(b.pageEntries)) - for _, entry := range b.pageEntries { - b.builder.PrependUOffsetT(entry) - } - entryVector := b.builder.EndVector(len(b.pageEntries)) - - // Create batch-level tags - tagOffset := b.allTags.WriteToBuilder(b.builder) - - // Write final batch object - SearchPageStart(b.builder) - SearchPageAddEntries(b.builder, entryVector) - SearchPageAddTags(b.builder, tagOffset) - batch := SearchPageEnd(b.builder) - b.builder.Finish(batch) - buf := b.builder.FinishedBytes() - - return buf -} - -func (b *SearchPageBuilder) Reset() { - b.builder.Reset() - b.pageEntries = b.pageEntries[:0] - b.allTags = NewSearchDataMap() -} - // Get searches the entry and returns the first value found for the given key. func (s *SearchEntry) Get(k string) string { kv := &KeyValues{} diff --git a/pkg/tempofb/searchdatamap.go b/pkg/tempofb/searchdatamap.go index cff823b2376..6f39546d66a 100644 --- a/pkg/tempofb/searchdatamap.go +++ b/pkg/tempofb/searchdatamap.go @@ -1,16 +1,17 @@ package tempofb import ( + "hash" "sort" "strings" + "github.com/cespare/xxhash" flatbuffers "github.com/google/flatbuffers/go" ) type SearchDataMap interface { Add(k, v string) Contains(k, v string) bool - WriteToBuilder(b *flatbuffers.Builder) flatbuffers.UOffsetT Range(f func(k, v string)) RangeKeys(f func(k string)) RangeKeyValues(k string, f func(v string)) @@ -84,19 +85,6 @@ func (s SearchDataMapSmall) RangeKeyValues(k string, f func(v string)) { } } -func (s SearchDataMapSmall) WriteToBuilder(b *flatbuffers.Builder) flatbuffers.UOffsetT { - keys := make([]string, 0, len(s)) - for k := range s { - keys = append(keys, k) - } - - valuesf := func(k string, _ []string) []string { - return s[k] - } - - return writeToBuilder(b, keys, valuesf) -} - type SearchDataMapLarge map[string]map[string]struct{} func (s SearchDataMapLarge) Add(k, v string) { @@ -141,66 +129,83 @@ func (s SearchDataMapLarge) RangeKeyValues(k string, f func(v string)) { } } -func (s SearchDataMapLarge) WriteToBuilder(b *flatbuffers.Builder) flatbuffers.UOffsetT { - keys := make([]string, 0, len(s)) - for k := range s { +func WriteSearchDataMap(b *flatbuffers.Builder, d SearchDataMap, cache map[uint64]flatbuffers.UOffsetT) flatbuffers.UOffsetT { + h := xxhash.New() + + var keys []string + d.RangeKeys(func(k string) { keys = append(keys, k) - } + }) + sort.Strings(keys) - valuesf := func(k string, buffer []string) []string { - buffer = buffer[:0] - for v := range s[k] { - buffer = append(buffer, v) - } + offsets := make([]flatbuffers.UOffsetT, 0, len(keys)) + var values []string + for _, k := range keys { - return buffer + values = values[:0] + d.RangeKeyValues(k, func(v string) { + values = append(values, v) + }) + + offsets = append(offsets, writeKeyValues(b, k, values, h, cache)) } - return writeToBuilder(b, keys, valuesf) + SearchEntryStartTagsVector(b, len(offsets)) + for _, kvo := range offsets { + b.PrependUOffsetT(kvo) + } + vector := b.EndVector((len(offsets))) + return vector } -func writeToBuilder(b *flatbuffers.Builder, keys []string, valuesf func(k string, buffer []string) []string) flatbuffers.UOffsetT { - - var values []string - - offsets := make([]flatbuffers.UOffsetT, 0, len(keys)) - sort.Strings(keys) - - for _, k := range keys { +func writeKeyValues(b *flatbuffers.Builder, key string, values []string, h hash.Hash64, cache map[uint64]flatbuffers.UOffsetT) flatbuffers.UOffsetT { + // Skip empty keys + if len(values) <= 0 { + return 0 + } - values := valuesf(k, values) + // Prep + key = strings.ToLower(key) + for i := range values { + values[i] = strings.ToLower(values[i]) + } + sort.Strings(values) - // Skip empty keys - if len(values) <= 0 { - continue + // Hash, cache + var ce uint64 + if cache != nil { + h.Reset() + h.Write([]byte(key)) + for _, v := range values { + h.Write([]byte{0}) // separator + h.Write([]byte(v)) } - - ko := b.CreateSharedString(strings.ToLower(k)) - - // Sort values - sort.Strings(values) - - valueStrings := make([]flatbuffers.UOffsetT, len(values)) - for i := range values { - valueStrings[i] = b.CreateSharedString(strings.ToLower(values[i])) + ce = h.Sum64() + if offset, ok := cache[ce]; ok { + return offset } + } - KeyValuesStartValueVector(b, len(valueStrings)) - for _, vs := range valueStrings { - b.PrependUOffsetT(vs) - } - valueVector := b.EndVector(len(valueStrings)) + ko := b.CreateSharedString(key) + valueStrings := make([]flatbuffers.UOffsetT, len(values)) + for i := range values { + valueStrings[i] = b.CreateSharedString(strings.ToLower(values[i])) + } - KeyValuesStart(b) - KeyValuesAddKey(b, ko) - KeyValuesAddValue(b, valueVector) - offsets = append(offsets, KeyValuesEnd(b)) + KeyValuesStartValueVector(b, len(valueStrings)) + for _, vs := range valueStrings { + b.PrependUOffsetT(vs) } + valueVector := b.EndVector(len(valueStrings)) - SearchEntryStartTagsVector(b, len(offsets)) - for _, kvo := range offsets { - b.PrependUOffsetT(kvo) + KeyValuesStart(b) + KeyValuesAddKey(b, ko) + KeyValuesAddValue(b, valueVector) + offset := KeyValuesEnd(b) + + if cache != nil { + cache[ce] = offset } - keyValueVector := b.EndVector((len(offsets))) - return keyValueVector + + return offset } From 4cedd5272a886494401b04b7efc1db53a1ea86fa Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Thu, 9 Dec 2021 15:28:49 -0500 Subject: [PATCH 2/4] comment --- pkg/tempofb/searchdatamap.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/tempofb/searchdatamap.go b/pkg/tempofb/searchdatamap.go index 6f39546d66a..a21237b5809 100644 --- a/pkg/tempofb/searchdatamap.go +++ b/pkg/tempofb/searchdatamap.go @@ -158,20 +158,22 @@ func WriteSearchDataMap(b *flatbuffers.Builder, d SearchDataMap, cache map[uint6 return vector } +// writeKeyValues saves the key->values entry to the builder. Results are optionally cached and +// existing identical key->values entries reused. func writeKeyValues(b *flatbuffers.Builder, key string, values []string, h hash.Hash64, cache map[uint64]flatbuffers.UOffsetT) flatbuffers.UOffsetT { // Skip empty keys if len(values) <= 0 { return 0 } - // Prep + // Preparation, must be done before hashing/caching. key = strings.ToLower(key) for i := range values { values[i] = strings.ToLower(values[i]) } sort.Strings(values) - // Hash, cache + // Hash, cache (optional) var ce uint64 if cache != nil { h.Reset() From aedee627eb52273c8a68fd206c5b6b71cbf7aea1 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Thu, 9 Dec 2021 16:27:22 -0500 Subject: [PATCH 3/4] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 353ff74f92a..546bd5bebe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ * [ENHANCEMENT] Make `TempoIngesterFlushesFailing` alert more actionable [#1157](https://github.com/grafana/tempo/pull/1157) (@dannykopping) * [ENHANCEMENT] Switch open-telemetry/opentelemetry-collector to grafana/opentelemetry-collectorl fork, update it to 0.40.0 and add missing dependencies due to the change [#1142](https://github.com/grafana/tempo/pull/1142) (@tete17) * [ENHANCEMENT] Allow environment variables for Azure storage credentials [#1147](https://github.com/grafana/tempo/pull/1147) (@zalegrala) +* [ENHANCEMENT] Reduce search data file sizes by optimizing contents [#1165](https://github.com/grafana/tempo/pull/1165) (@mdisibio) * [BUGFIX] Fix defaults for MaxBytesPerTrace (ingester.max-bytes-per-trace) and MaxSearchBytesPerTrace (ingester.max-search-bytes-per-trace) (@bitprocessor) * [BUGFIX] Ignore empty objects during compaction [#1113](https://github.com/grafana/tempo/pull/1113) (@mdisibio) * [BUGFIX] Add process name to vulture traces to work around display issues [#1127](https://github.com/grafana/tempo/pull/1127) (@mdisibio) From bcb4518b8055e270117cac10cfbd7e15861b3560 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Fri, 10 Dec 2021 08:31:40 -0500 Subject: [PATCH 4/4] cleanup --- pkg/tempofb/searchdatamap.go | 75 ++-------------- pkg/tempofb/searchdatamap_test.go | 142 +++++++++++++----------------- 2 files changed, 67 insertions(+), 150 deletions(-) diff --git a/pkg/tempofb/searchdatamap.go b/pkg/tempofb/searchdatamap.go index a21237b5809..dca047a34ca 100644 --- a/pkg/tempofb/searchdatamap.go +++ b/pkg/tempofb/searchdatamap.go @@ -9,16 +9,10 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -type SearchDataMap interface { - Add(k, v string) - Contains(k, v string) bool - Range(f func(k, v string)) - RangeKeys(f func(k string)) - RangeKeyValues(k string, f func(v string)) -} +type SearchDataMap map[string]map[string]struct{} func NewSearchDataMap() SearchDataMap { - return make(SearchDataMapLarge, 10) // 10 for luck + return make(SearchDataMap, 10) // 10 for luck } func NewSearchDataMapWithData(m map[string][]string) SearchDataMap { @@ -32,62 +26,7 @@ func NewSearchDataMapWithData(m map[string][]string) SearchDataMap { return s } -type SearchDataMapSmall map[string][]string - -func (s SearchDataMapSmall) Add(k, v string) { - vs, ok := s[k] - if !ok { - // First entry for key - s[k] = []string{v} - return - } - - // Key already present, now check for value - for i := range vs { - if vs[i] == v { - // Already present, nothing to do - return - } - } - - // Not found, append - s[k] = append(vs, v) -} - -func (s SearchDataMapSmall) Contains(k, v string) bool { - e := s[k] - for _, vvv := range e { - if strings.Contains(vvv, v) { - return true - } - } - - return false -} - -func (s SearchDataMapSmall) Range(f func(k, v string)) { - for k, vv := range s { - for _, v := range vv { - f(k, v) - } - } -} - -func (s SearchDataMapSmall) RangeKeys(f func(k string)) { - for k := range s { - f(k) - } -} - -func (s SearchDataMapSmall) RangeKeyValues(k string, f func(v string)) { - for _, v := range s[k] { - f(v) - } -} - -type SearchDataMapLarge map[string]map[string]struct{} - -func (s SearchDataMapLarge) Add(k, v string) { +func (s SearchDataMap) Add(k, v string) { values, ok := s[k] if !ok { // first entry @@ -101,7 +40,7 @@ func (s SearchDataMapLarge) Add(k, v string) { } } -func (s SearchDataMapLarge) Contains(k, v string) bool { +func (s SearchDataMap) Contains(k, v string) bool { if values, ok := s[k]; ok { _, ok := values[v] return ok @@ -109,7 +48,7 @@ func (s SearchDataMapLarge) Contains(k, v string) bool { return false } -func (s SearchDataMapLarge) Range(f func(k, v string)) { +func (s SearchDataMap) Range(f func(k, v string)) { for k, values := range s { for v := range values { f(k, v) @@ -117,13 +56,13 @@ func (s SearchDataMapLarge) Range(f func(k, v string)) { } } -func (s SearchDataMapLarge) RangeKeys(f func(k string)) { +func (s SearchDataMap) RangeKeys(f func(k string)) { for k := range s { f(k) } } -func (s SearchDataMapLarge) RangeKeyValues(k string, f func(v string)) { +func (s SearchDataMap) RangeKeyValues(k string, f func(v string)) { for v := range s[k] { f(v) } diff --git a/pkg/tempofb/searchdatamap_test.go b/pkg/tempofb/searchdatamap_test.go index c6a3d23323d..8ac36425d5e 100644 --- a/pkg/tempofb/searchdatamap_test.go +++ b/pkg/tempofb/searchdatamap_test.go @@ -8,75 +8,55 @@ import ( ) func TestSearchDataMap(t *testing.T) { - testCases := []struct { - name string - impl SearchDataMap - }{ - {"SearchDataMapSmall", &SearchDataMapSmall{}}, - {"SearchDataMapLarge", &SearchDataMapLarge{}}, - } + searchDataMap := NewSearchDataMap() - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - searchDataMap := tc.impl + assert.False(t, searchDataMap.Contains("key-1", "value-1-2")) - assert.False(t, searchDataMap.Contains("key-1", "value-1-2")) + searchDataMap.Add("key-1", "value-1-1") - searchDataMap.Add("key-1", "value-1-1") + assert.False(t, searchDataMap.Contains("key-1", "value-1-2")) - assert.False(t, searchDataMap.Contains("key-1", "value-1-2")) + searchDataMap.Add("key-1", "value-1-2") + searchDataMap.Add("key-2", "value-2-1") - searchDataMap.Add("key-1", "value-1-2") - searchDataMap.Add("key-2", "value-2-1") + assert.True(t, searchDataMap.Contains("key-1", "value-1-2")) + assert.False(t, searchDataMap.Contains("key-2", "value-1-2")) - assert.True(t, searchDataMap.Contains("key-1", "value-1-2")) - assert.False(t, searchDataMap.Contains("key-2", "value-1-2")) - - type Pair struct { - k string - v string - } - var pairs []Pair - capturePairFn := func(k, v string) { - pairs = append(pairs, Pair{k, v}) - } + type Pair struct { + k string + v string + } + var pairs []Pair + capturePairFn := func(k, v string) { + pairs = append(pairs, Pair{k, v}) + } - searchDataMap.Range(capturePairFn) - assert.ElementsMatch(t, []Pair{{"key-1", "value-1-1"}, {"key-1", "value-1-2"}, {"key-2", "value-2-1"}}, pairs) + searchDataMap.Range(capturePairFn) + assert.ElementsMatch(t, []Pair{{"key-1", "value-1-1"}, {"key-1", "value-1-2"}, {"key-2", "value-2-1"}}, pairs) - var strs []string - captureSliceFn := func(value string) { - strs = append(strs, value) - } + var strs []string + captureSliceFn := func(value string) { + strs = append(strs, value) + } - searchDataMap.RangeKeys(captureSliceFn) - assert.ElementsMatch(t, []string{"key-1", "key-2"}, strs) - strs = nil + searchDataMap.RangeKeys(captureSliceFn) + assert.ElementsMatch(t, []string{"key-1", "key-2"}, strs) + strs = nil - searchDataMap.RangeKeyValues("key-1", captureSliceFn) - assert.ElementsMatch(t, []string{"value-1-1", "value-1-2"}, strs) - strs = nil + searchDataMap.RangeKeyValues("key-1", captureSliceFn) + assert.ElementsMatch(t, []string{"value-1-1", "value-1-2"}, strs) + strs = nil - searchDataMap.RangeKeyValues("key-2", captureSliceFn) - assert.ElementsMatch(t, []string{"value-2-1"}, strs) - strs = nil + searchDataMap.RangeKeyValues("key-2", captureSliceFn) + assert.ElementsMatch(t, []string{"value-2-1"}, strs) + strs = nil - searchDataMap.RangeKeyValues("does-not-exist", captureSliceFn) - assert.ElementsMatch(t, []string{}, strs) - strs = nil - }) - } + searchDataMap.RangeKeyValues("does-not-exist", captureSliceFn) + assert.ElementsMatch(t, []string{}, strs) + strs = nil } func BenchmarkSearchDataMapAdd(b *testing.B) { - intfs := []struct { - name string - f func() SearchDataMap - }{ - {"SearchDataMapSmall", func() SearchDataMap { return make(SearchDataMapSmall, 10) }}, - {"SearchDataMapLarge", func() SearchDataMap { return make(SearchDataMapLarge, 10) }}, - } - testCases := []struct { name string values int @@ -92,41 +72,39 @@ func BenchmarkSearchDataMapAdd(b *testing.B) { } for _, tc := range testCases { - for _, intf := range intfs { - b.Run(fmt.Sprint(tc.name, "/", tc.values, "x value/", tc.repeats, "x repeat", "/", intf.name), func(b *testing.B) { - var k []string - for i := 0; i < b.N; i++ { - k = append(k, fmt.Sprintf("key%d", i)) - } + b.Run(fmt.Sprint(tc.name, "/", tc.values, "x value/", tc.repeats, "x repeat"), func(b *testing.B) { - var v []string - for i := 0; i < tc.values; i++ { - v = append(v, fmt.Sprintf("value%d", i)) - } + var k []string + for i := 0; i < b.N; i++ { + k = append(k, fmt.Sprintf("key%d", i)) + } + + var v []string + for i := 0; i < tc.values; i++ { + v = append(v, fmt.Sprintf("value%d", i)) + } - s := intf.f() - insert := func() { - for i := 0; i < len(k); i++ { - for j := 0; j < len(v); j++ { - s.Add(k[i], v[j]) - } + s := NewSearchDataMap() + insert := func() { + for i := 0; i < len(k); i++ { + for j := 0; j < len(v); j++ { + s.Add(k[i], v[j]) } } + } - // insert - b.ResetTimer() - insert() + // insert + b.ResetTimer() + insert() - // reinsert? - if tc.repeats > 0 { - b.ResetTimer() - for i := 0; i < tc.repeats; i++ { - insert() - } + // reinsert? + if tc.repeats > 0 { + b.ResetTimer() + for i := 0; i < tc.repeats; i++ { + insert() } - }) - } + } + }) } - }