Star tree index config changes

Signed-off-by: Bharathwaj G <[email protected]>
opensearch-project · May 31, 2024 · 8808c6e · 8808c6e
1 parent 994bfae
commit 8808c6e
Show file tree

Hide file tree

Showing 28 changed files with 1,356 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Add dynamic cluster settings to set timeout for segments upload to Remote Store ([#13679](https://github.com/opensearch-project/OpenSearch/pull/13679))
 - [Remote Store] Upload translog checkpoint as object metadata to translog.tlog([#13637](https://github.com/opensearch-project/OpenSearch/pull/13637))
 - Add getMetadataFields to MapperService ([#13819](https://github.com/opensearch-project/OpenSearch/pull/13819))
+- [Star Tree Index] Star tree index config changes ([#13917](https://github.com/opensearch-project/OpenSearch/pull/13917))
 
 ### Dependencies
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))

diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
@@ -125,3 +125,7 @@ ${path.logs}
 # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting.
 #
 #opensearch.experimental.feature.pluggable.caching.enabled: false
+#
+# Gates the functionality of star tree index, which improves the performance of search aggregations.
+#
+#opensearch.experimental.feature.composite_index.enabled: true
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -1324,6 +1324,8 @@ private static void updateIndexMappingsAndBuildSortOrder(
             // at this point. The validation will take place later in the process
             // (when all shards are copied in a single place).
             indexService.getIndexSortSupplier().get();
+            // validate composite index fields
+            indexService.getCompositeIndexConfigSupplier().get();
         }
         if (request.dataStreamName() != null) {
             MetadataCreateDataStreamService.validateTimestampFieldMapping(mapperService);

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -113,6 +113,7 @@
 import org.opensearch.index.ShardIndexingPressureMemoryManager;
 import org.opensearch.index.ShardIndexingPressureSettings;
 import org.opensearch.index.ShardIndexingPressureStore;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.remote.RemoteStorePressureSettings;
 import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
 import org.opensearch.index.store.remote.filecache.FileCache;
@@ -741,7 +742,15 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING,
                 RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING,
                 RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS,
-                RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA
+                RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA,
+
+                // Composite index settings
+                CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING,
+                CompositeIndexSettings.COMPOSITE_INDEX_MAX_FIELDS_SETTING,
+                CompositeIndexSettings.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING,
+                CompositeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS,
+                CompositeIndexSettings.DEFAULT_METRICS_LIST,
+                CompositeIndexSettings.DEFAULT_DATE_INTERVALS
             )
         )
     );

diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
@@ -36,6 +36,7 @@ protected FeatureFlagSettings(
         FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING,
         FeatureFlags.TIERED_REMOTE_INDEX_SETTING,
         FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
-        FeatureFlags.PLUGGABLE_CACHE_SETTING
+        FeatureFlags.PLUGGABLE_CACHE_SETTING,
+        FeatureFlags.COMPOSITE_INDEX_SETTING
     );
 }
diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -249,8 +249,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                         }
                     }
                 }, Property.IndexScope), // this allows similarity settings to be passed
-                Setting.groupSetting("index.analysis.", Property.IndexScope) // this allows analysis settings to be passed
-
+                Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed
+                Setting.groupSetting("index.composite_index.", Property.IndexScope) // this allows composite index settings to be passed
             )
         )
     );

diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
@@ -67,6 +67,12 @@ public class FeatureFlags {
      */
     public static final String PLUGGABLE_CACHE = "opensearch.experimental.feature.pluggable.caching.enabled";
 
+    /**
+     * Gates the functionality of composite index i.e. star tree index, which improves the performance of search
+     * aggregations.
+     */
+    public static final String COMPOSITE_INDEX = "opensearch.experimental.feature.composite_index.enabled";
+
     public static final Setting<Boolean> REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting(
         REMOTE_STORE_MIGRATION_EXPERIMENTAL,
         false,
@@ -89,14 +95,17 @@ public class FeatureFlags {
 
     public static final Setting<Boolean> PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope);
 
+    public static final Setting<Boolean> COMPOSITE_INDEX_SETTING = Setting.boolSetting(COMPOSITE_INDEX, false, Property.NodeScope);
+
     private static final List<Setting<Boolean>> ALL_FEATURE_FLAG_SETTINGS = List.of(
         REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
         EXTENSIONS_SETTING,
         IDENTITY_SETTING,
         TELEMETRY_SETTING,
         DATETIME_FORMATTER_CACHING_SETTING,
         TIERED_REMOTE_INDEX_SETTING,
-        PLUGGABLE_CACHE_SETTING
+        PLUGGABLE_CACHE_SETTING,
+        COMPOSITE_INDEX_SETTING
     );
     /**
      * Should store the settings from opensearch.yml.

diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java
@@ -66,6 +66,7 @@
 import org.opensearch.index.cache.query.DisabledQueryCache;
 import org.opensearch.index.cache.query.IndexQueryCache;
 import org.opensearch.index.cache.query.QueryCache;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.engine.Engine;
 import org.opensearch.index.engine.EngineConfigFactory;
 import org.opensearch.index.engine.EngineFactory;
@@ -606,7 +607,8 @@ public IndexService newIndexService(
         BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier,
         Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
         RecoverySettings recoverySettings,
-        RemoteStoreSettings remoteStoreSettings
+        RemoteStoreSettings remoteStoreSettings,
+        CompositeIndexSettings compositeIndexSettings
     ) throws IOException {
         final IndexEventListener eventListener = freeze();
         Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> readerWrapperFactory = indexReaderWrapper
@@ -665,7 +667,8 @@ public IndexService newIndexService(
                 translogFactorySupplier,
                 clusterDefaultRefreshIntervalSupplier,
                 recoverySettings,
-                remoteStoreSettings
+                remoteStoreSettings,
+                compositeIndexSettings
             );
             success = true;
             return indexService;

diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -72,6 +72,8 @@
 import org.opensearch.index.cache.IndexCache;
 import org.opensearch.index.cache.bitset.BitsetFilterCache;
 import org.opensearch.index.cache.query.QueryCache;
+import org.opensearch.index.compositeindex.CompositeIndexConfig;
+import org.opensearch.index.compositeindex.CompositeIndexSettings;
 import org.opensearch.index.engine.Engine;
 import org.opensearch.index.engine.EngineConfigFactory;
 import org.opensearch.index.engine.EngineFactory;
@@ -183,6 +185,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final CircuitBreakerService circuitBreakerService;
     private final IndexNameExpressionResolver expressionResolver;
     private final Supplier<Sort> indexSortSupplier;
+    private final Supplier<CompositeIndexConfig> compositeIndexConfigSupplier;
     private final ValuesSourceRegistry valuesSourceRegistry;
     private final BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier;
     private final Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier;
@@ -223,7 +226,8 @@ public IndexService(
         BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier,
         Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
         RecoverySettings recoverySettings,
-        RemoteStoreSettings remoteStoreSettings
+        RemoteStoreSettings remoteStoreSettings,
+        CompositeIndexSettings compositeIndexSettings
     ) {
         super(indexSettings);
         this.allowExpensiveQueries = allowExpensiveQueries;
@@ -261,6 +265,14 @@ public IndexService(
             } else {
                 this.indexSortSupplier = () -> null;
             }
+
+            if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) {
+                this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig()
+                    .validateAndGetCompositeIndexConfig(mapperService::fieldType, compositeIndexSettings);
+            } else {
+                this.compositeIndexConfigSupplier = () -> null;
+            }
+
             indexFieldData.setListener(new FieldDataCacheListener(this));
             this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
             this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool));
@@ -273,6 +285,7 @@ public IndexService(
             this.bitsetFilterCache = null;
             this.warmer = null;
             this.indexCache = null;
+            this.compositeIndexConfigSupplier = () -> null;
         }
 
         this.shardStoreDeleter = shardStoreDeleter;
@@ -385,6 +398,10 @@ public Supplier<Sort> getIndexSortSupplier() {
         return indexSortSupplier;
     }
 
+    public Supplier<CompositeIndexConfig> getCompositeIndexConfigSupplier() {
+        return compositeIndexConfigSupplier;
+    }
+
     public synchronized void close(final String reason, boolean delete) throws IOException {
         if (closed.compareAndSet(false, true)) {
             deleted.compareAndSet(false, delete);

diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -48,6 +48,7 @@
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.Index;
+import org.opensearch.index.compositeindex.CompositeIndexConfig;
 import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.translog.Translog;
@@ -753,6 +754,8 @@ public static IndexMergePolicy fromString(String text) {
     private final LogByteSizeMergePolicyProvider logByteSizeMergePolicyProvider;
     private final IndexSortConfig indexSortConfig;
     private final IndexScopedSettings scopedSettings;
+
+    private final CompositeIndexConfig compositeIndexConfig;
     private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
     private final boolean softDeleteEnabled;
     private volatile long softDeleteRetentionOperations;
@@ -966,6 +969,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         this.tieredMergePolicyProvider = new TieredMergePolicyProvider(logger, this);
         this.logByteSizeMergePolicyProvider = new LogByteSizeMergePolicyProvider(logger, this);
         this.indexSortConfig = new IndexSortConfig(this);
+        this.compositeIndexConfig = new CompositeIndexConfig(this);
         searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER);
         defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE);
         setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING));
@@ -1720,6 +1724,10 @@ public IndexSortConfig getIndexSortConfig() {
         return indexSortConfig;
     }
 
+    public CompositeIndexConfig getCompositeIndexConfig() {
+        return compositeIndexConfig;
+    }
+
     public IndexScopedSettings getScopedSettings() {
         return scopedSettings;
     }

diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeField.java
@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.util.List;
+
+/**
+ * Composite field which contains dimensions, metrics and index mode specific specs
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class CompositeField {
+    private final String name;
+    private final List<Dimension> dimensionsOrder;
+    private final List<Metric> metrics;
+    private final CompositeFieldSpec compositeFieldSpec;
+
+    public CompositeField(String name, List<Dimension> dimensions, List<Metric> metrics, CompositeFieldSpec compositeFieldSpec) {
+        this.name = name;
+        this.dimensionsOrder = dimensions;
+        this.metrics = metrics;
+        this.compositeFieldSpec = compositeFieldSpec;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public List<Dimension> getDimensionsOrder() {
+        return dimensionsOrder;
+    }
+
+    public List<Metric> getMetrics() {
+        return metrics;
+    }
+
+    public CompositeFieldSpec getSpec() {
+        return compositeFieldSpec;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * CompositeFieldSpec interface.
+ *
+ * @opensearch.experimental
+ */
+
+@ExperimentalApi
+public interface CompositeFieldSpec {
+    void setDefaults(CompositeIndexSettings compositeIndexSettings);
+}