Skip to content

Commit

Permalink
Consolidated OTel ES IndexNameProviders (#2458)
Browse files Browse the repository at this point in the history 
* Consolidated index name providers

Signed-off-by: Joe Elliott <[email protected]>

* moved to esutil

Signed-off-by: Joe Elliott <[email protected]>
  • Loading branch information
@joe-elliott
joe-elliott authored Sep 11, 2020
1 parent e37deb9 commit 5d0a0fa
Show file tree
Hide file tree
Showing 6 changed files with 115 additions and 154 deletions.
51 changes: 0 additions & 51 deletions cmd/opentelemetry/app/exporter/elasticsearchexporter/index_name.go
View file

This file was deleted.

68 changes: 0 additions & 68 deletions cmd/opentelemetry/app/exporter/elasticsearchexporter/index_name_test.go
View file

This file was deleted.

17 changes: 11 additions & 6 deletions cmd/opentelemetry/app/exporter/elasticsearchexporter/spanstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/jaegertracing/jaeger/cmd/opentelemetry/app/exporter/elasticsearchexporter/esmodeltranslator"
"github.com/jaegertracing/jaeger/cmd/opentelemetry/app/exporter/storagemetrics"
"github.com/jaegertracing/jaeger/cmd/opentelemetry/app/internal/esclient"
"github.com/jaegertracing/jaeger/cmd/opentelemetry/app/internal/esutil"
"github.com/jaegertracing/jaeger/model"
"github.com/jaegertracing/jaeger/pkg/cache"
"github.com/jaegertracing/jaeger/pkg/es/config"
Expand All @@ -53,8 +54,8 @@ type esSpanWriter struct {
nameTag tag.Mutator
client esclient.ElasticsearchClient
serviceCache cache.Cache
spanIndexName indexNameProvider
serviceIndexName indexNameProvider
spanIndexName esutil.IndexNameProvider
serviceIndexName esutil.IndexNameProvider
translator *esmodeltranslator.Translator
isArchive bool
}
Expand All @@ -69,12 +70,16 @@ func newEsSpanWriter(params config.Configuration, logger *zap.Logger, archive bo
if err != nil {
return nil, err
}
alias := esutil.AliasNone
if params.UseReadWriteAliases {
alias = esutil.AliasWrite
}
return &esSpanWriter{
logger: logger,
nameTag: tag.Insert(storagemetrics.TagExporterName(), name),
client: client,
spanIndexName: newIndexNameProvider(spanIndexBaseName, params.IndexPrefix, params.UseReadWriteAliases, archive),
serviceIndexName: newIndexNameProvider(serviceIndexBaseName, params.IndexPrefix, params.UseReadWriteAliases, archive),
spanIndexName: esutil.NewIndexNameProvider(spanIndexBaseName, params.IndexPrefix, alias, archive),
serviceIndexName: esutil.NewIndexNameProvider(serviceIndexBaseName, params.IndexPrefix, alias, archive),
translator: esmodeltranslator.NewTranslator(params.Tags.AllAsFields, tagsKeysAsFields, params.GetTagDotReplacement()),
isArchive: archive,
serviceCache: cache.NewLRUWithOptions(
Expand Down Expand Up @@ -122,7 +127,7 @@ func (w *esSpanWriter) writeSpans(ctx context.Context, spans []*dbmodel.Span) (i
dropped++
continue
}
indexName := w.spanIndexName.get(model.EpochMicrosecondsAsTime(span.StartTime))
indexName := w.spanIndexName.IndexName(model.EpochMicrosecondsAsTime(span.StartTime))
bulkOperations = append(bulkOperations, bulkItem{span: span, isService: false})
w.client.AddDataToBulkBuffer(buffer, data, indexName, spanTypeName)

Expand Down Expand Up @@ -202,7 +207,7 @@ func (w *esSpanWriter) writeService(span *dbmodel.Span, buffer *bytes.Buffer) (b
if err != nil {
return false, err
}
indexName := w.serviceIndexName.get(model.EpochMicrosecondsAsTime(span.StartTime))
indexName := w.serviceIndexName.IndexName(model.EpochMicrosecondsAsTime(span.StartTime))
w.client.AddDataToBulkBuffer(buffer, data, indexName, serviceTypeName)
return true, nil
}
Expand Down
48 changes: 36 additions & 12 deletions ...rnal/reader/es/esspanreader/index_name.go → ...lemetry/app/internal/esutil/index_name.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,39 +12,54 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package esspanreader
package esutil

import "time"

const indexDateFormat = "2006-01-02" // date format for index e.g. 2020-01-20

type indexNameProvider struct {
index string
noDateIndex bool
// Alias is used to configure the kind of index alias
type Alias string

const (
// AliasNone configures no aliasing
AliasNone Alias = "none"
// AliasRead configures read aliases
AliasRead = "read"
// AliasWrite configures write aliases
AliasWrite = "write"
)

// IndexNameProvider creates standard index names from dates
type IndexNameProvider struct {
index string
useSingleIndex bool
}

func newIndexNameProvider(index, prefix string, useAliases bool, archive bool) indexNameProvider {
// NewIndexNameProvider constructs a new IndexNameProvider
func NewIndexNameProvider(index, prefix string, alias Alias, archive bool) IndexNameProvider {
if prefix != "" {
index = prefix + "-" + index
}
index += "-"
if archive {
index += "archive"
}
if useAliases {
if alias != AliasNone {
if index[len(index)-1] != '-' {
index += "-"
}
index += "read"
index += string(alias)
}
return indexNameProvider{
index: index,
noDateIndex: archive || useAliases,
return IndexNameProvider{
index: index,
useSingleIndex: archive || (alias != AliasNone),
}
}

func (n indexNameProvider) get(start, end time.Time) []string {
if n.noDateIndex {
// IndexNameRange returns a slice of index names. One for every date in the range.
func (n IndexNameProvider) IndexNameRange(start, end time.Time) []string {
if n.useSingleIndex {
return []string{n.index}
}
var indices []string
Expand All @@ -58,3 +73,12 @@ func (n indexNameProvider) get(start, end time.Time) []string {
indices = append(indices, firstIndex)
return indices
}

// IndexName returns a single index name for the provided date.
func (n IndexNameProvider) IndexName(date time.Time) string {
if n.useSingleIndex {
return n.index
}
spanDate := date.UTC().Format(indexDateFormat)
return n.index + spanDate
}
64 changes: 55 additions & 9 deletions ...reader/es/esspanreader/index_name_test.go → ...ry/app/internal/esutil/index_name_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package esspanreader
package esutil

import (
"testing"
Expand All @@ -21,52 +21,98 @@ import (
"github.com/stretchr/testify/assert"
)

func TestIndexName(t *testing.T) {
func TestIndexNames(t *testing.T) {
tests := []struct {
name string
indices []string
nameProvider indexNameProvider
nameProvider IndexNameProvider
start time.Time
end time.Time
}{
{
name: "index prefix",
nameProvider: newIndexNameProvider("myindex", "production", false, false),
nameProvider: NewIndexNameProvider("myindex", "production", AliasNone, false),
indices: []string{"production-myindex-0001-01-01"},
},
{
name: "multiple dates",
nameProvider: newIndexNameProvider("myindex", "", false, false),
nameProvider: NewIndexNameProvider("myindex", "", AliasNone, false),
indices: []string{"myindex-2020-08-30", "myindex-2020-08-29", "myindex-2020-08-28"},
start: time.Date(2020, 8, 28, 0, 0, 0, 0, time.UTC),
end: time.Date(2020, 8, 30, 0, 0, 0, 0, time.UTC),
},
{
name: "use aliases",
nameProvider: newIndexNameProvider("myindex", "", true, false),
nameProvider: NewIndexNameProvider("myindex", "", AliasRead, false),
indices: []string{"myindex-read"},
start: time.Date(2020, 8, 28, 0, 0, 0, 0, time.UTC),
end: time.Date(2020, 8, 30, 0, 0, 0, 0, time.UTC),
},
{
name: "use archive",
nameProvider: newIndexNameProvider("myindex", "", false, true),
nameProvider: NewIndexNameProvider("myindex", "", AliasNone, true),
indices: []string{"myindex-archive"},
start: time.Date(2020, 8, 28, 0, 0, 0, 0, time.UTC),
end: time.Date(2020, 8, 30, 0, 0, 0, 0, time.UTC),
},
{
name: "use archive alias",
nameProvider: newIndexNameProvider("myindex", "", true, true),
nameProvider: NewIndexNameProvider("myindex", "", AliasRead, true),
indices: []string{"myindex-archive-read"},
start: time.Date(2020, 8, 28, 0, 0, 0, 0, time.UTC),
end: time.Date(2020, 8, 30, 0, 0, 0, 0, time.UTC),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
indices := test.nameProvider.get(test.start, test.end)
indices := test.nameProvider.IndexNameRange(test.start, test.end)
assert.Equal(t, test.indices, indices)
})
}
}

func TestIndexName(t *testing.T) {
tests := []struct {
name string
index string
nameProvider IndexNameProvider
date time.Time
end time.Time
}{
{
name: "index prefix",
nameProvider: NewIndexNameProvider("myindex", "production", AliasNone, false),
index: "production-myindex-0001-01-01",
},
{
name: "no prefix",
nameProvider: NewIndexNameProvider("myindex", "", AliasNone, false),
index: "myindex-2020-08-28",
date: time.Date(2020, 8, 28, 0, 0, 0, 0, time.UTC),
},
{
name: "use aliases",
nameProvider: NewIndexNameProvider("myindex", "", AliasWrite, false),
index: "myindex-write",
date: time.Date(2020, 8, 28, 0, 0, 0, 0, time.UTC),
},
{
name: "use archive",
nameProvider: NewIndexNameProvider("myindex", "", AliasNone, true),
index: "myindex-archive",
date: time.Date(2020, 8, 28, 0, 0, 0, 0, time.UTC),
},
{
name: "use archive alias",
nameProvider: NewIndexNameProvider("myindex", "", AliasWrite, true),
index: "myindex-archive-write",
date: time.Date(2020, 8, 28, 0, 0, 0, 0, time.UTC),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
indices := test.nameProvider.IndexName(test.date)
assert.Equal(t, test.index, indices)
})
}
}
Loading

0 comments on commit 5d0a0fa

Please sign in to comment.