From bbaa1c7515b33a1868a877bc35814a988c390c5d Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 24 Jun 2024 12:59:09 +0530 Subject: [PATCH 1/5] Star tree codec changes Signed-off-by: Bharathwaj G --- .../opensearch/index/codec/CodecService.java | 19 ++- .../composite/Composite90DocValuesFormat.java | 64 ++++++++++ .../composite/Composite90DocValuesReader.java | 94 ++++++++++++++ .../composite/Composite90DocValuesWriter.java | 116 ++++++++++++++++++ .../codec/composite/Composite99Codec.java | 56 +++++++++ .../codec/composite/CompositeIndexReader.java | 34 +++++ .../codec/composite/CompositeIndexValues.java | 21 ++++ .../datacube/startree/StarTreeValues.java | 35 ++++++ .../datacube/startree/package-info.java | 12 ++ .../index/codec/composite/package-info.java | 12 ++ .../mapper/CompositeMappedFieldType.java | 82 +++++++++++++ .../index/mapper/MapperService.java | 17 +++ 12 files changed, 558 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 67f38536a0d11..913a1978b52d8 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -39,6 +39,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; +import org.opensearch.index.codec.composite.Composite99Codec; import org.opensearch.index.mapper.MapperService; import java.util.Map; @@ -73,10 +74,20 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Mode.BEST_COMPRESSION)); codecs.put(ZLIB, new Lucene99Codec(Mode.BEST_COMPRESSION)); } else { - codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); - codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); - codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); - codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); + // CompositeCodec still delegates to PerFieldMappingPostingFormatCodec + // We can still support all the compression codecs when composite index is present + // hence we're defining the codecs like below + if (mapperService.isCompositeIndexPresent()) { + codecs.put(DEFAULT_CODEC, new Composite99Codec(Mode.BEST_SPEED, mapperService, logger)); + codecs.put(LZ4, new Composite99Codec(Mode.BEST_SPEED, mapperService, logger)); + codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Mode.BEST_COMPRESSION, mapperService, logger)); + codecs.put(ZLIB, new Composite99Codec(Mode.BEST_COMPRESSION, mapperService, logger)); + } else { + codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); + codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); + codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); + codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); + } } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java new file mode 100644 index 0000000000000..979d95c07f044 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.mapper.MapperService; + +import java.io.IOException; + +/** + * DocValues format to handle composite indices + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Composite90DocValuesFormat extends DocValuesFormat { + /** + * Creates a new docvalues format. + * + *

The provided name will be written into the index segment in some configurations (such as + * when using {@code PerFieldDocValuesFormat}): in such configurations, for the segment to be read + * this class should be registered with Java's SPI mechanism (registered in META-INF/ of your jar + * file, etc). + */ + private final DocValuesFormat delegate; + private final MapperService mapperService; + + // needed for SPI + public Composite90DocValuesFormat() { + this(new Lucene90DocValuesFormat(), null); + } + + public Composite90DocValuesFormat(MapperService mapperService) { + this(new Lucene90DocValuesFormat(), mapperService); + } + + public Composite90DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) { + super(delegate.getName()); + this.delegate = delegate; + this.mapperService = mapperService; + } + + @Override + public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + return new Composite90DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService); + } + + @Override + public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { + return new Composite90DocValuesReader(delegate.fieldsProducer(state), state, mapperService); + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java new file mode 100644 index 0000000000000..5804c16ced279 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.mapper.CompositeMappedFieldType; +import org.opensearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +/** + * Reader for star tree index and star tree doc values from the segments + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Composite90DocValuesReader extends DocValuesProducer implements CompositeIndexReader { + private DocValuesProducer delegate; + Set compositeMappedFieldTypes; + MapperService mapperService; + + public Composite90DocValuesReader(DocValuesProducer producer, SegmentReadState state, MapperService mapperService) throws IOException { + this.delegate = producer; + this.mapperService = mapperService; + this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); + // TODO : read star tree files + } + + @Override + public NumericDocValues getNumeric(FieldInfo field) throws IOException { + return delegate.getNumeric(field); + } + + @Override + public BinaryDocValues getBinary(FieldInfo field) throws IOException { + return delegate.getBinary(field); + } + + @Override + public SortedDocValues getSorted(FieldInfo field) throws IOException { + return delegate.getSorted(field); + } + + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + return delegate.getSortedNumeric(field); + } + + @Override + public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { + return delegate.getSortedSet(field); + } + + @Override + public void checkIntegrity() throws IOException { + delegate.checkIntegrity(); + // Todo : check integrity of composite index related [star tree] files + } + + @Override + public void close() throws IOException { + delegate.close(); + // Todo: close composite index related files [star tree] files + } + + @Override + public List getCompositeIndexFields() { + // todo : read from file formats and get the field names. + return null; + } + + @Override + public CompositeIndexValues getCompositeIndexValues(String field, CompositeMappedFieldType.CompositeFieldType fieldType) + throws IOException { + // TODO : read compositeIndexValues [starTreeValues] from star tree files + return null; + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java new file mode 100644 index 0000000000000..607084024940f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java @@ -0,0 +1,116 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.mapper.CompositeMappedFieldType; +import org.opensearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * This class write the star tree index and star tree doc values + * based on the doc values structures of the original index + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Composite90DocValuesWriter extends DocValuesConsumer { + private final DocValuesConsumer delegate; + private final SegmentWriteState state; + private final MapperService mapperService; + private MergeState mergeState = null; + private final Set compositeMappedFieldTypes; + private final Set compositeFieldSet; + + private final Map fieldProducerMap = new HashMap<>(); + private final Map fieldToFieldInfoMap = new HashMap<>(); + + public Composite90DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) + throws IOException { + + this.delegate = delegate; + this.state = segmentWriteState; + this.mapperService = mapperService; + this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); + compositeFieldSet = new HashSet<>(); + for (CompositeMappedFieldType type : compositeMappedFieldTypes) { + compositeFieldSet.add(type.name()); + } + } + + @Override + public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + delegate.addNumericField(field, valuesProducer); + } + + @Override + public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + delegate.addBinaryField(field, valuesProducer); + } + + @Override + public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + delegate.addSortedField(field, valuesProducer); + } + + @Override + public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + delegate.addSortedNumericField(field, valuesProducer); + // Perform this only during flush flow + if (mergeState == null) { + createCompositeIndicesIfPossible(valuesProducer, field); + } + } + + @Override + public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + delegate.addSortedSetField(field, valuesProducer); + } + + @Override + public void close() throws IOException { + + } + + private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, FieldInfo field) throws IOException { + if (compositeFieldSet.isEmpty()) return; + if (compositeFieldSet.contains(field.name)) { + fieldProducerMap.put(field.name, valuesProducer); + fieldToFieldInfoMap.put(field.name, field); + compositeFieldSet.remove(field.name); + } + // we have all the required fields to build composite fields + if (compositeFieldSet.isEmpty()) { + for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) { + if (mappedType.getCompositeIndexType().equals(CompositeMappedFieldType.CompositeFieldType.STAR_TREE)) { + // TODO : Call StarTree builder + } + } + } + } + + @Override + public void merge(MergeState mergeState) throws IOException { + // TODO : check if class variable will cause concurrency issues + this.mergeState = mergeState; + super.merge(mergeState); + // TODO : handle merge star tree + // mergeStarTreeFields(mergeState); + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java new file mode 100644 index 0000000000000..2c235cf8c4e79 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; + +/** + * Extends the Codec to support new file formats for composite indices eg: star tree index + * based on the mappings. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Composite99Codec extends FilterCodec { + public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite99Codec"; + private final MapperService mapperService; + + public Composite99Codec() { + this(COMPOSITE_INDEX_CODEC_NAME, new Lucene99Codec(), null); + } + + public Composite99Codec(Lucene99Codec.Mode compressionMode, MapperService mapperService, Logger logger) { + this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService); + } + + /** + * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to + * this ctor. + * + * @param name name of the codec + * @param delegate codec delegate + * @param mapperService mapper service instance + */ + protected Composite99Codec(String name, Codec delegate, MapperService mapperService) { + super(name, delegate); + this.mapperService = mapperService; + } + + @Override + public DocValuesFormat docValuesFormat() { + return new Composite90DocValuesFormat(mapperService); + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java new file mode 100644 index 0000000000000..d02438b75377d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.mapper.CompositeMappedFieldType; + +import java.io.IOException; +import java.util.List; + +/** + * Interface that abstracts the functionality to read composite index structures from the segment + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface CompositeIndexReader { + /** + * Get list of composite index fields from the segment + * + */ + List getCompositeIndexFields(); + + /** + * Get composite index values based on the field name and the field type + */ + CompositeIndexValues getCompositeIndexValues(String field, CompositeMappedFieldType.CompositeFieldType fieldType) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java new file mode 100644 index 0000000000000..def1c45efd3b2 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Abstract class for composite index values + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class CompositeIndexValues { + public abstract CompositeIndexValues getValues(); +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java new file mode 100644 index 0000000000000..7cfbd9bc98dc0 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite.datacube.startree; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.codec.composite.CompositeIndexValues; + +import java.util.List; + +/** + * Concrete class that holds the star tree associated values from the segment + * + * @opensearch.experimental + */ +@ExperimentalApi +public class StarTreeValues extends CompositeIndexValues { + private final List dimensionsOrder; + + // TODO : come up with full set of vales such as dimensions and metrics doc values + star tree + public StarTreeValues(List dimensionsOrder) { + super(); + this.dimensionsOrder = dimensionsOrder; + } + + @Override + public CompositeIndexValues getValues() { + return this; + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java new file mode 100644 index 0000000000000..67808ad51289a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * classes responsible for handling all star tree structures and operations as part of codec + */ +package org.opensearch.index.codec.composite.datacube.startree; diff --git a/server/src/main/java/org/opensearch/index/codec/composite/package-info.java b/server/src/main/java/org/opensearch/index/codec/composite/package-info.java new file mode 100644 index 0000000000000..5d15e99c00975 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * classes responsible for handling all composite index codecs and operations + */ +package org.opensearch.index.codec.composite; diff --git a/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java new file mode 100644 index 0000000000000..f4bdb19abd8cc --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Base class for composite field types + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class CompositeMappedFieldType extends MappedFieldType { + private final List fields; + private final CompositeFieldType type; + + public CompositeMappedFieldType( + String name, + boolean isIndexed, + boolean isStored, + boolean hasDocValues, + TextSearchInfo textSearchInfo, + Map meta, + List fields, + CompositeFieldType type + ) { + super(name, isIndexed, isStored, hasDocValues, textSearchInfo, meta); + this.fields = fields; + this.type = type; + } + + public CompositeMappedFieldType(String name, List fields, CompositeFieldType type) { + this(name, false, false, false, TextSearchInfo.NONE, Collections.emptyMap(), fields, type); + } + + /** + * Supported composite field types + * + * @opensearch.experimental + */ + @ExperimentalApi + public enum CompositeFieldType { + STAR_TREE("star_tree"); + + private final String name; + + CompositeFieldType(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static CompositeFieldType fromName(String name) { + for (CompositeFieldType metric : CompositeFieldType.values()) { + if (metric.getName().equalsIgnoreCase(name)) { + return metric; + } + } + throw new IllegalArgumentException("Invalid composite field type: " + name); + } + } + + public CompositeFieldType getCompositeIndexType() { + return type; + } + + public List fields() { + return fields; + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index a1f3894c9f14c..d4e0a3c11905c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -654,6 +654,23 @@ public ObjectMapper getObjectMapper(String name) { return this.mapper == null ? null : this.mapper.objectMappers().get(name); } + public boolean isCompositeIndexPresent() { + return this.mapper != null && !getCompositeFieldTypes().isEmpty(); + } + + public Set getCompositeFieldTypes() { + Set compositeMappedFieldTypes = new HashSet<>(); + if (this.mapper == null) { + return Collections.emptySet(); + } + for (MappedFieldType type : this.mapper.fieldTypes()) { + if (type instanceof CompositeMappedFieldType) { + compositeMappedFieldTypes.add((CompositeMappedFieldType) type); + } + } + return compositeMappedFieldTypes; + } + /** * Given a type (eg. long, string, ...), return an anonymous field mapper that can be used for search operations. */ From 28e895c75443d53333421f642d2431f568bb9bc1 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 27 Jun 2024 09:09:15 +0530 Subject: [PATCH 2/5] Adding tests Signed-off-by: Bharathwaj G --- .../services/org.apache.lucene.codecs.Codec | 1 + .../opensearch/index/codec/CodecTests.java | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec new file mode 100644 index 0000000000000..e030a813373c1 --- /dev/null +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -0,0 +1 @@ +org.opensearch.index.codec.composite.Composite99Codec diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index b31edd79411d0..a9ba751ed4fb3 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -48,6 +48,7 @@ import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.IndexAnalyzers; +import org.opensearch.index.codec.composite.Composite99Codec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; @@ -59,6 +60,8 @@ import java.io.IOException; import java.util.Collections; +import org.mockito.Mockito; + import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; import static org.hamcrest.Matchers.instanceOf; @@ -93,6 +96,35 @@ public void testZlib() throws Exception { assert codec instanceof PerFieldMappingPostingFormatCodec; } + public void testResolveDefaultCodecsWithCompositeIndex() throws Exception { + CodecService codecService = createCodecService(false, true); + assertThat(codecService.codec("default"), instanceOf(Composite99Codec.class)); + } + + public void testDefaultWithCompositeIndex() throws Exception { + Codec codec = createCodecService(false, true).codec("default"); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assert codec instanceof Composite99Codec; + } + + public void testBestCompressionWithCompositeIndex() throws Exception { + Codec codec = createCodecService(false, true).codec("best_compression"); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + assert codec instanceof Composite99Codec; + } + + public void testLZ4WithCompositeIndex() throws Exception { + Codec codec = createCodecService(false, true).codec("lz4"); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assert codec instanceof Composite99Codec; + } + + public void testZlibWithCompositeIndex() throws Exception { + Codec codec = createCodecService(false, true).codec("zlib"); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + assert codec instanceof Composite99Codec; + } + public void testBestCompressionWithCompressionLevel() { final Settings settings = Settings.builder() .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) @@ -150,10 +182,17 @@ private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Co } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { + return createCodecService(isMapperServiceNull, false); + } + + private CodecService createCodecService(boolean isMapperServiceNull, boolean isCompositeIndexPresent) throws IOException { Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); if (isMapperServiceNull) { return new CodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test")); } + if (isCompositeIndexPresent) { + return buildCodecServiceWithCompositeIndex(nodeSettings); + } return buildCodecService(nodeSettings); } @@ -176,6 +215,14 @@ private CodecService buildCodecService(Settings nodeSettings) throws IOException return new CodecService(service, indexSettings, LogManager.getLogger("test")); } + private CodecService buildCodecServiceWithCompositeIndex(Settings nodeSettings) throws IOException { + + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); + MapperService service = Mockito.mock(MapperService.class); + Mockito.when(service.isCompositeIndexPresent()).thenReturn(true); + return new CodecService(service, indexSettings, LogManager.getLogger("test")); + } + private SegmentReader getSegmentReader(Codec codec) throws IOException { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); From d0e325ae5ac5991e525eb33cd0f4b412c59af477 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 1 Jul 2024 16:01:47 +0530 Subject: [PATCH 3/5] Addressing comments Signed-off-by: Bharathwaj G --- .../opensearch/index/codec/CodecService.java | 8 ++-- .../composite/Composite90DocValuesFormat.java | 2 +- .../composite/Composite90DocValuesReader.java | 11 ++--- .../composite/CompositeCodecFactory.java | 42 +++++++++++++++++++ .../codec/composite/CompositeIndexValues.java | 6 +-- .../datacube/startree/StarTreeValues.java | 2 +- 6 files changed, 54 insertions(+), 17 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 913a1978b52d8..7a0bf9dad7189 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -39,7 +39,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; -import org.opensearch.index.codec.composite.Composite99Codec; +import org.opensearch.index.codec.composite.CompositeCodecFactory; import org.opensearch.index.mapper.MapperService; import java.util.Map; @@ -78,10 +78,8 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe // We can still support all the compression codecs when composite index is present // hence we're defining the codecs like below if (mapperService.isCompositeIndexPresent()) { - codecs.put(DEFAULT_CODEC, new Composite99Codec(Mode.BEST_SPEED, mapperService, logger)); - codecs.put(LZ4, new Composite99Codec(Mode.BEST_SPEED, mapperService, logger)); - codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Mode.BEST_COMPRESSION, mapperService, logger)); - codecs.put(ZLIB, new Composite99Codec(Mode.BEST_COMPRESSION, mapperService, logger)); + CompositeCodecFactory compositeCodecFactory = new CompositeCodecFactory(); + codecs.putAll(compositeCodecFactory.getCompositeCodecs(mapperService, logger)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java index 979d95c07f044..59ee4c4aa64da 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java @@ -59,6 +59,6 @@ public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOExcept @Override public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { - return new Composite90DocValuesReader(delegate.fieldsProducer(state), state, mapperService); + return new Composite90DocValuesReader(delegate.fieldsProducer(state), state); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java index 5804c16ced279..fbc69f2e4c24d 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java @@ -32,13 +32,9 @@ @ExperimentalApi public class Composite90DocValuesReader extends DocValuesProducer implements CompositeIndexReader { private DocValuesProducer delegate; - Set compositeMappedFieldTypes; - MapperService mapperService; - public Composite90DocValuesReader(DocValuesProducer producer, SegmentReadState state, MapperService mapperService) throws IOException { + public Composite90DocValuesReader(DocValuesProducer producer, SegmentReadState state) throws IOException { this.delegate = producer; - this.mapperService = mapperService; - this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); // TODO : read star tree files } @@ -82,13 +78,14 @@ public void close() throws IOException { @Override public List getCompositeIndexFields() { // todo : read from file formats and get the field names. - return null; + throw new UnsupportedOperationException(); + } @Override public CompositeIndexValues getCompositeIndexValues(String field, CompositeMappedFieldType.CompositeFieldType fieldType) throws IOException { // TODO : read compositeIndexValues [starTreeValues] from star tree files - return null; + throw new UnsupportedOperationException(); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java new file mode 100644 index 0000000000000..a5a7202a1c36b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import java.util.HashMap; +import java.util.Map; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.mapper.MapperService; + +import static org.opensearch.index.codec.CodecService.BEST_COMPRESSION_CODEC; +import static org.opensearch.index.codec.CodecService.DEFAULT_CODEC; +import static org.opensearch.index.codec.CodecService.LZ4; +import static org.opensearch.index.codec.CodecService.ZLIB; + + +/** + * Factory class to return the latest composite codec for all the modes + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeCodecFactory { + public CompositeCodecFactory() {} + + public Map getCompositeCodecs(MapperService mapperService, Logger logger) { + Map codecs = new HashMap<>(); + codecs.put(DEFAULT_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); + codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); + codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger)); + codecs.put(ZLIB, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger)); + return codecs; + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java index def1c45efd3b2..c28d6afc143ad 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java @@ -11,11 +11,11 @@ import org.opensearch.common.annotation.ExperimentalApi; /** - * Abstract class for composite index values + * Interface for composite index values * * @opensearch.experimental */ @ExperimentalApi -public abstract class CompositeIndexValues { - public abstract CompositeIndexValues getValues(); +public interface CompositeIndexValues { + CompositeIndexValues getValues(); } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java index 7cfbd9bc98dc0..8b93222f97f4c 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java @@ -19,7 +19,7 @@ * @opensearch.experimental */ @ExperimentalApi -public class StarTreeValues extends CompositeIndexValues { +public class StarTreeValues implements CompositeIndexValues { private final List dimensionsOrder; // TODO : come up with full set of vales such as dimensions and metrics doc values + star tree From d9d75be0d61a10e74c3ecb9d6ef69fef566f3b81 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 1 Jul 2024 19:42:54 +0530 Subject: [PATCH 4/5] addressing review comments Signed-off-by: Bharathwaj G --- .../opensearch/index/codec/CodecService.java | 5 +- .../composite/Composite90DocValuesWriter.java | 6 +- .../composite/CompositeCodecFactory.java | 2 +- .../index/mapper/MapperService.java | 9 ++ .../opensearch/index/codec/CodecTests.java | 44 +++---- .../StarTreeDocValuesFormatTests.java | 110 ++++++++++++++++++ 6 files changed, 146 insertions(+), 30 deletions(-) create mode 100644 server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 7a0bf9dad7189..59fafdf1ba74e 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -64,6 +64,7 @@ public class CodecService { * the raw unfiltered lucene default. useful for testing */ public static final String LUCENE_DEFAULT_CODEC = "lucene_default"; + private final CompositeCodecFactory compositeCodecFactory = new CompositeCodecFactory(); public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); @@ -76,10 +77,8 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe } else { // CompositeCodec still delegates to PerFieldMappingPostingFormatCodec // We can still support all the compression codecs when composite index is present - // hence we're defining the codecs like below if (mapperService.isCompositeIndexPresent()) { - CompositeCodecFactory compositeCodecFactory = new CompositeCodecFactory(); - codecs.putAll(compositeCodecFactory.getCompositeCodecs(mapperService, logger)); + codecs.putAll(compositeCodecFactory.getCompositeIndexCodecs(mapperService, logger)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java index e59dedf50a930..57fce7d0640a0 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java @@ -40,7 +40,6 @@ public class Composite90DocValuesWriter extends DocValuesConsumer { private final Set compositeFieldSet; private final Map fieldProducerMap = new HashMap<>(); - private final Map fieldToFieldInfoMap = new HashMap<>(); public Composite90DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) throws IOException { @@ -51,7 +50,7 @@ public Composite90DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); compositeFieldSet = new HashSet<>(); for (CompositeMappedFieldType type : compositeMappedFieldTypes) { - compositeFieldSet.add(type.name()); + compositeFieldSet.addAll(type.fields()); } } @@ -86,14 +85,13 @@ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) @Override public void close() throws IOException { - + delegate.close(); } private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, FieldInfo field) throws IOException { if (compositeFieldSet.isEmpty()) return; if (compositeFieldSet.contains(field.name)) { fieldProducerMap.put(field.name, valuesProducer); - fieldToFieldInfoMap.put(field.name, field); compositeFieldSet.remove(field.name); } // we have all the required fields to build composite fields diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java index 40b64c55c714c..3acedc6a27d7f 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java @@ -31,7 +31,7 @@ public class CompositeCodecFactory { public CompositeCodecFactory() {} - public Map getCompositeCodecs(MapperService mapperService, Logger logger) { + public Map getCompositeIndexCodecs(MapperService mapperService, Logger logger) { Map codecs = new HashMap<>(); codecs.put(DEFAULT_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index c2e7411a3b47a..530a3092a5aa7 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -226,6 +226,8 @@ public enum MergeReason { private final BooleanSupplier idFieldDataEnabled; + private volatile Set compositeMappedFieldTypes; + public MapperService( IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, @@ -542,6 +544,9 @@ private synchronized Map internalMerge(DocumentMapper ma } assert results.values().stream().allMatch(this::assertSerialization); + + // initialize composite fields post merge + this.compositeMappedFieldTypes = getCompositeFieldTypesFromMapper(); return results; } @@ -655,6 +660,10 @@ public boolean isCompositeIndexPresent() { } public Set getCompositeFieldTypes() { + return compositeMappedFieldTypes; + } + + private Set getCompositeFieldTypesFromMapper() { Set compositeMappedFieldTypes = new HashSet<>(); if (this.mapper == null) { return Collections.emptySet(); diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index a9ba751ed4fb3..7146b7dc51753 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -79,52 +79,52 @@ public void testDefault() throws Exception { assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); } - public void testBestCompression() throws Exception { - Codec codec = createCodecService(false).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); - } - - public void testLZ4() throws Exception { - Codec codec = createCodecService(false).codec("lz4"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); - assert codec instanceof PerFieldMappingPostingFormatCodec; - } - - public void testZlib() throws Exception { - Codec codec = createCodecService(false).codec("zlib"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); - assert codec instanceof PerFieldMappingPostingFormatCodec; - } - - public void testResolveDefaultCodecsWithCompositeIndex() throws Exception { - CodecService codecService = createCodecService(false, true); - assertThat(codecService.codec("default"), instanceOf(Composite99Codec.class)); - } - public void testDefaultWithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("default"); assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); assert codec instanceof Composite99Codec; } + public void testBestCompression() throws Exception { + Codec codec = createCodecService(false).codec("best_compression"); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + } + public void testBestCompressionWithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("best_compression"); assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); assert codec instanceof Composite99Codec; } + public void testLZ4() throws Exception { + Codec codec = createCodecService(false).codec("lz4"); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assert codec instanceof PerFieldMappingPostingFormatCodec; + } + public void testLZ4WithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("lz4"); assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); assert codec instanceof Composite99Codec; } + public void testZlib() throws Exception { + Codec codec = createCodecService(false).codec("zlib"); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + assert codec instanceof PerFieldMappingPostingFormatCodec; + } + public void testZlibWithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("zlib"); assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); assert codec instanceof Composite99Codec; } + public void testResolveDefaultCodecsWithCompositeIndex() throws Exception { + CodecService codecService = createCodecService(false, true); + assertThat(codecService.codec("default"), instanceOf(Composite99Codec.class)); + } + public void testBestCompressionWithCompressionLevel() { final Settings settings = Settings.builder() .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) diff --git a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java new file mode 100644 index 0000000000000..6c6d26656e4de --- /dev/null +++ b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite.datacube.startree; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.opensearch.common.Rounding; +import org.opensearch.index.codec.composite.Composite99Codec; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.StarTreeMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.mockito.Mockito; + +/** + * Star tree doc values Lucene tests + */ +@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") +public class StarTreeDocValuesFormatTests extends BaseDocValuesFormatTestCase { + @Override + protected Codec getCodec() { + MapperService service = Mockito.mock(MapperService.class); + Mockito.when(service.getCompositeFieldTypes()).thenReturn(Set.of(getStarTreeFieldType())); + final Logger testLogger = LogManager.getLogger(StarTreeDocValuesFormatTests.class); + return new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, service, testLogger); + } + + private StarTreeMapper.StarTreeFieldType getStarTreeFieldType() { + List m1 = new ArrayList<>(); + m1.add(MetricStat.MAX); + Metric metric = new Metric("sndv", m1); + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + StarTreeField starTreeField = getStarTreeField(d1CalendarIntervals, metric); + + return new StarTreeMapper.StarTreeFieldType("star_tree", starTreeField); + } + + private static StarTreeField getStarTreeField(List d1CalendarIntervals, Metric metric1) { + DateDimension d1 = new DateDimension("field", d1CalendarIntervals); + NumericDimension d2 = new NumericDimension("dv"); + + List metrics = List.of(metric1); + List dims = List.of(d1, d2); + StarTreeFieldConfiguration config = new StarTreeFieldConfiguration( + 100, + Collections.emptySet(), + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP + ); + + return new StarTreeField("starTree", dims, metrics, config); + } + + public void testStarTreeDocValues() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedNumericDocValuesField("dv", 1)); + doc.add(new SortedNumericDocValuesField("field", 1)); + iw.addDocument(doc); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedNumericDocValuesField("dv", 1)); + doc.add(new SortedNumericDocValuesField("field", 1)); + iw.addDocument(doc); + iw.forceMerge(1); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedNumericDocValuesField("dv", 2)); + doc.add(new SortedNumericDocValuesField("field", 2)); + iw.addDocument(doc); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedNumericDocValuesField("dv", 2)); + doc.add(new SortedNumericDocValuesField("field", 2)); + iw.addDocument(doc); + iw.forceMerge(1); + iw.close(); + + // TODO : validate star tree structures that got created + directory.close(); + } +} From 8430db2e0054159eb26875e9f9ce1069d1de0a10 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 8 Jul 2024 14:49:04 +0530 Subject: [PATCH 5/5] Addressing review comments Signed-off-by: Bharathwaj G --- .../index/codec/composite/Composite99Codec.java | 3 ++- ...sFormat.java => Composite99DocValuesFormat.java} | 12 ++++++------ ...sReader.java => Composite99DocValuesReader.java} | 4 ++-- ...sWriter.java => Composite99DocValuesWriter.java} | 13 ++++++------- .../composite/datacube/startree/StarTreeValues.java | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) rename server/src/main/java/org/opensearch/index/codec/composite/{Composite90DocValuesFormat.java => Composite99DocValuesFormat.java} (83%) rename server/src/main/java/org/opensearch/index/codec/composite/{Composite90DocValuesReader.java => Composite99DocValuesReader.java} (95%) rename server/src/main/java/org/opensearch/index/codec/composite/{Composite90DocValuesWriter.java => Composite99DocValuesWriter.java} (91%) diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java index 2c235cf8c4e79..de04944e67cd2 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java @@ -28,6 +28,7 @@ public class Composite99Codec extends FilterCodec { public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite99Codec"; private final MapperService mapperService; + // needed for SPI - this is used in reader path public Composite99Codec() { this(COMPOSITE_INDEX_CODEC_NAME, new Lucene99Codec(), null); } @@ -51,6 +52,6 @@ protected Composite99Codec(String name, Codec delegate, MapperService mapperServ @Override public DocValuesFormat docValuesFormat() { - return new Composite90DocValuesFormat(mapperService); + return new Composite99DocValuesFormat(mapperService); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesFormat.java similarity index 83% rename from server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java rename to server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesFormat.java index 59ee4c4aa64da..216ed4f68f333 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesFormat.java @@ -25,7 +25,7 @@ * @opensearch.experimental */ @ExperimentalApi -public class Composite90DocValuesFormat extends DocValuesFormat { +public class Composite99DocValuesFormat extends DocValuesFormat { /** * Creates a new docvalues format. * @@ -38,15 +38,15 @@ public class Composite90DocValuesFormat extends DocValuesFormat { private final MapperService mapperService; // needed for SPI - public Composite90DocValuesFormat() { + public Composite99DocValuesFormat() { this(new Lucene90DocValuesFormat(), null); } - public Composite90DocValuesFormat(MapperService mapperService) { + public Composite99DocValuesFormat(MapperService mapperService) { this(new Lucene90DocValuesFormat(), mapperService); } - public Composite90DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) { + public Composite99DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) { super(delegate.getName()); this.delegate = delegate; this.mapperService = mapperService; @@ -54,11 +54,11 @@ public Composite90DocValuesFormat(DocValuesFormat delegate, MapperService mapper @Override public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return new Composite90DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService); + return new Composite99DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService); } @Override public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { - return new Composite90DocValuesReader(delegate.fieldsProducer(state), state); + return new Composite99DocValuesReader(delegate.fieldsProducer(state), state); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesReader.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java rename to server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesReader.java index 0c5c5dbde868f..82c844088cfd4 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesReader.java @@ -28,10 +28,10 @@ * @opensearch.experimental */ @ExperimentalApi -public class Composite90DocValuesReader extends DocValuesProducer implements CompositeIndexReader { +public class Composite99DocValuesReader extends DocValuesProducer implements CompositeIndexReader { private DocValuesProducer delegate; - public Composite90DocValuesReader(DocValuesProducer producer, SegmentReadState state) throws IOException { + public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState state) throws IOException { this.delegate = producer; // TODO : read star tree files } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java similarity index 91% rename from server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java rename to server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java index 57fce7d0640a0..75bbf78dbdad2 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; /** * This class write the star tree index and star tree doc values @@ -31,18 +32,17 @@ * @opensearch.experimental */ @ExperimentalApi -public class Composite90DocValuesWriter extends DocValuesConsumer { +public class Composite99DocValuesWriter extends DocValuesConsumer { private final DocValuesConsumer delegate; private final SegmentWriteState state; private final MapperService mapperService; - private MergeState mergeState = null; + AtomicReference mergeState = new AtomicReference<>(); private final Set compositeMappedFieldTypes; private final Set compositeFieldSet; private final Map fieldProducerMap = new HashMap<>(); - public Composite90DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) - throws IOException { + public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) { this.delegate = delegate; this.state = segmentWriteState; @@ -73,7 +73,7 @@ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) th public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegate.addSortedNumericField(field, valuesProducer); // Perform this only during flush flow - if (mergeState == null) { + if (mergeState.get() == null) { createCompositeIndicesIfPossible(valuesProducer, field); } } @@ -106,8 +106,7 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, @Override public void merge(MergeState mergeState) throws IOException { - // TODO : check if class variable will cause concurrency issues - this.mergeState = mergeState; + this.mergeState.compareAndSet(null, mergeState); super.merge(mergeState); // TODO : handle merge star tree // mergeStarTreeFields(mergeState); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java index 8b93222f97f4c..2a5b96ce2620a 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java @@ -25,7 +25,7 @@ public class StarTreeValues implements CompositeIndexValues { // TODO : come up with full set of vales such as dimensions and metrics doc values + star tree public StarTreeValues(List dimensionsOrder) { super(); - this.dimensionsOrder = dimensionsOrder; + this.dimensionsOrder = List.copyOf(dimensionsOrder); } @Override