Star tree index config changes

distribution/src/config/opensearch.yml

@@ -125,3 +125,7 @@ ${path.logs}
 # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting.
 #
 #opensearch.experimental.feature.pluggable.caching.enabled: false
+#
+# Gates the functionality of star tree index, which improves the performance of search aggregations.
+#
+#opensearch.experimental.feature.composite_index.enabled: true

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -1324,6 +1324,8 @@ private static void updateIndexMappingsAndBuildSortOrder(
             // at this point. The validation will take place later in the process
             // (when all shards are copied in a single place).
             indexService.getIndexSortSupplier().get();
+            // validate composite index fields
+            indexService.getCompositeIndexConfigSupplier().get();
         }
         if (request.dataStreamName() != null) {
             MetadataCreateDataStreamService.validateTimestampFieldMapping(mapperService);

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -744,8 +744,11 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING,
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING,
                 RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS,
+                SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING,
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA,
-                SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING
+
+                // Composite index setting
+                IndicesService.COMPOSITE_INDEX_ENABLED_SETTING
             )
         )
     );

server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java

@@ -37,6 +37,7 @@ protected FeatureFlagSettings(
         FeatureFlags.TIERED_REMOTE_INDEX_SETTING,
         FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         FeatureFlags.PLUGGABLE_CACHE_SETTING,
+        FeatureFlags.COMPOSITE_INDEX_SETTING,
         FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING
     );
 }

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -51,6 +51,7 @@
 import org.opensearch.index.SearchSlowLog;
 import org.opensearch.index.TieredMergePolicyProvider;
 import org.opensearch.index.cache.bitset.BitsetFilterCache;
+import org.opensearch.index.compositeindex.CompositeIndexConfig;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.index.fielddata.IndexFieldDataService;
 import org.opensearch.index.mapper.FieldMapper;
@@ -238,6 +239,14 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 // Settings for concurrent segment search
                 IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING,
                 IndexSettings.ALLOW_DERIVED_FIELDS,
+
+                // Settings for composite index defaults
+                CompositeIndexConfig.STAR_TREE_DEFAULT_MAX_LEAF_DOCS,
+                CompositeIndexConfig.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING,
+                CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING,
+                CompositeIndexConfig.DEFAULT_METRICS_LIST,
+                CompositeIndexConfig.DEFAULT_DATE_INTERVALS,
+
                 // validate that built-in similarities don't get redefined
                 Setting.groupSetting("index.similarity.", (s) -> {
                     Map<String, Settings> groups = s.getAsGroups();
@@ -249,8 +258,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                         }
                     }
                 }, Property.IndexScope), // this allows similarity settings to be passed
-                Setting.groupSetting("index.analysis.", Property.IndexScope) // this allows analysis settings to be passed
-
+                Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed
+                Setting.groupSetting("index.composite_index.config.", Property.IndexScope) // this allows composite index settings to be
+                                                                                           // passed
             )
         )
     );

server/src/main/java/org/opensearch/common/util/FeatureFlags.java

@@ -72,6 +72,12 @@ public class FeatureFlags {
      */
     public static final String REMOTE_PUBLICATION_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.publication.enabled";
 
+    /**
+     * Gates the functionality of composite index i.e. star tree index, which improves the performance of search
+     * aggregations.
+     */
+    public static final String COMPOSITE_INDEX = "opensearch.experimental.feature.composite_index.enabled";
+
     public static final Setting<Boolean> REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting(
         REMOTE_STORE_MIGRATION_EXPERIMENTAL,
         false,
@@ -100,6 +106,8 @@ public class FeatureFlags {
         Property.NodeScope
     );
 
+    public static final Setting<Boolean> COMPOSITE_INDEX_SETTING = Setting.boolSetting(COMPOSITE_INDEX, false, Property.NodeScope);
+
     private static final List<Setting<Boolean>> ALL_FEATURE_FLAG_SETTINGS = List.of(
         REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         EXTENSIONS_SETTING,
@@ -108,7 +116,8 @@ public class FeatureFlags {
         DATETIME_FORMATTER_CACHING_SETTING,
         TIERED_REMOTE_INDEX_SETTING,
         PLUGGABLE_CACHE_SETTING,
-        REMOTE_PUBLICATION_EXPERIMENTAL_SETTING
+        REMOTE_PUBLICATION_EXPERIMENTAL_SETTING,
+        COMPOSITE_INDEX_SETTING
     );
     /**
      * Should store the settings from opensearch.yml.

server/src/main/java/org/opensearch/index/IndexModule.java

@@ -606,7 +606,8 @@ public IndexService newIndexService(
         BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier,
         Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
         RecoverySettings recoverySettings,
-        RemoteStoreSettings remoteStoreSettings
+        RemoteStoreSettings remoteStoreSettings,
+        BooleanSupplier isCompositeIndexCreationEnabled
     ) throws IOException {
         final IndexEventListener eventListener = freeze();
         Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> readerWrapperFactory = indexReaderWrapper
@@ -665,7 +666,8 @@ public IndexService newIndexService(
                 translogFactorySupplier,
                 clusterDefaultRefreshIntervalSupplier,
                 recoverySettings,
-                remoteStoreSettings
+                remoteStoreSettings,
+                isCompositeIndexCreationEnabled
             );
             success = true;
             return indexService;

server/src/main/java/org/opensearch/index/IndexService.java

@@ -72,6 +72,7 @@
 import org.opensearch.index.cache.IndexCache;
 import org.opensearch.index.cache.bitset.BitsetFilterCache;
 import org.opensearch.index.cache.query.QueryCache;
+import org.opensearch.index.compositeindex.CompositeIndexConfig;
 import org.opensearch.index.engine.Engine;
 import org.opensearch.index.engine.EngineConfigFactory;
 import org.opensearch.index.engine.EngineFactory;
@@ -183,6 +184,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final CircuitBreakerService circuitBreakerService;
     private final IndexNameExpressionResolver expressionResolver;
     private final Supplier<Sort> indexSortSupplier;
+    private final Supplier<CompositeIndexConfig> compositeIndexConfigSupplier;
     private final ValuesSourceRegistry valuesSourceRegistry;
     private final BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier;
     private final Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier;
@@ -223,7 +225,8 @@ public IndexService(
         BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier,
         Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
         RecoverySettings recoverySettings,
-        RemoteStoreSettings remoteStoreSettings
+        RemoteStoreSettings remoteStoreSettings,
+        BooleanSupplier isCompositeIndexCreationEnabled
     ) {
         super(indexSettings);
         this.allowExpensiveQueries = allowExpensiveQueries;
@@ -261,6 +264,15 @@ public IndexService(
             } else {
                 this.indexSortSupplier = () -> null;
             }
+
+            if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) {
+                // The validation is done right after the merge of the mapping later in the process ( similar to sort )
+                this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig()
+                    .validateAndGetCompositeIndexConfig(mapperService::fieldType, isCompositeIndexCreationEnabled);
+            } else {
+                this.compositeIndexConfigSupplier = () -> null;
+            }
+
             indexFieldData.setListener(new FieldDataCacheListener(this));
             this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
             this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool));
@@ -273,6 +285,7 @@ public IndexService(
             this.bitsetFilterCache = null;
             this.warmer = null;
             this.indexCache = null;
+            this.compositeIndexConfigSupplier = () -> null;
         }
 
         this.shardStoreDeleter = shardStoreDeleter;
@@ -385,6 +398,10 @@ public Supplier<Sort> getIndexSortSupplier() {
         return indexSortSupplier;
     }
 
+    public Supplier<CompositeIndexConfig> getCompositeIndexConfigSupplier() {
+        return compositeIndexConfigSupplier;
+    }
+
     public synchronized void close(final String reason, boolean delete) throws IOException {
         if (closed.compareAndSet(false, true)) {
             deleted.compareAndSet(false, delete);

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -48,6 +48,7 @@
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.Index;
+import org.opensearch.index.compositeindex.CompositeIndexConfig;
 import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.translog.Translog;
@@ -760,6 +761,8 @@ public static IndexMergePolicy fromString(String text) {
     private final LogByteSizeMergePolicyProvider logByteSizeMergePolicyProvider;
     private final IndexSortConfig indexSortConfig;
     private final IndexScopedSettings scopedSettings;
+
+    private final CompositeIndexConfig compositeIndexConfig;
     private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
     private final boolean softDeleteEnabled;
     private volatile long softDeleteRetentionOperations;
@@ -985,6 +988,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         this.tieredMergePolicyProvider = new TieredMergePolicyProvider(logger, this);
         this.logByteSizeMergePolicyProvider = new LogByteSizeMergePolicyProvider(logger, this);
         this.indexSortConfig = new IndexSortConfig(this);
+        this.compositeIndexConfig = new CompositeIndexConfig(this);
         searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER);
         defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE);
         setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING));
@@ -1740,6 +1744,10 @@ public IndexSortConfig getIndexSortConfig() {
         return indexSortConfig;
     }
 
+    public CompositeIndexConfig getCompositeIndexConfig() {
+        return compositeIndexConfig;
+    }
+
     public IndexScopedSettings getScopedSettings() {
         return scopedSettings;
     }

server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java

@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.util.List;
+
+/**
+ * Composite field which contains dimensions, metrics and index mode specific specs
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class CompositeField {
+    private final String name;
+    private final List<Dimension> dimensionsOrder;
+    private final List<Metric> metrics;
+    private final CompositeFieldSpec compositeFieldSpec;
+
+    public CompositeField(String name, List<Dimension> dimensions, List<Metric> metrics, CompositeFieldSpec compositeFieldSpec) {
+        this.name = name;
+        this.dimensionsOrder = dimensions;
+        this.metrics = metrics;
+        this.compositeFieldSpec = compositeFieldSpec;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public List<Dimension> getDimensionsOrder() {
+        return dimensionsOrder;
+    }
+
+    public List<Metric> getMetrics() {
+        return metrics;
+    }
+
+    public CompositeFieldSpec getSpec() {
+        return compositeFieldSpec;
+    }
+}

server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java

@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * CompositeFieldSpec interface.
+ *
+ * @opensearch.experimental
+ */
+
+@ExperimentalApi
+public interface CompositeFieldSpec {}