From 3576504996714ba3e685b4b078335a2f8aaaeff9 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 6 Jun 2024 14:48:05 +0530 Subject: [PATCH] Adding integ settings and correcting the flow to contain only enabled setting at cluster level Signed-off-by: Bharathwaj G --- .../common/settings/ClusterSettings.java | 10 +- .../common/settings/IndexScopedSettings.java | 11 +- .../org/opensearch/index/IndexModule.java | 5 +- .../org/opensearch/index/IndexService.java | 6 +- .../compositeindex/CompositeFieldSpec.java | 4 +- .../compositeindex/CompositeIndexConfig.java | 443 ++++++++++------ .../CompositeIndexSettings.java | 179 ------- .../index/compositeindex/DateDimension.java | 16 +- .../index/compositeindex/Dimension.java | 4 - .../index/compositeindex/Metric.java | 4 +- .../compositeindex/StarTreeFieldSpec.java | 7 - .../DefaultCompositeIndexSettings.java | 28 -- .../opensearch/indices/IndicesService.java | 40 +- .../main/java/org/opensearch/node/Node.java | 7 +- .../opensearch/index/IndexModuleTests.java | 3 +- .../CompositeIndexConfigIntegTests.java | 239 +++++++++ .../CompositeIndexConfigSettingsTests.java | 472 ++++++++++++++++-- .../snapshots/SnapshotResiliencyTests.java | 4 +- 18 files changed, 1025 insertions(+), 457 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java delete mode 100644 server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 88319bb595178..b208cdbad3635 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -113,7 +113,6 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.remote.RemoteStorePressureSettings; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.store.remote.filecache.FileCacheSettings; @@ -744,13 +743,8 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, - // Composite index settings - CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING, - CompositeIndexSettings.COMPOSITE_INDEX_MAX_FIELDS_SETTING, - CompositeIndexSettings.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, - CompositeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, - CompositeIndexSettings.DEFAULT_METRICS_LIST, - CompositeIndexSettings.DEFAULT_DATE_INTERVALS + // Composite index setting + IndicesService.COMPOSITE_INDEX_ENABLED_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 9b4521cc3647a..60422220d8524 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -51,6 +51,7 @@ import org.opensearch.index.SearchSlowLog; import org.opensearch.index.TieredMergePolicyProvider; import org.opensearch.index.cache.bitset.BitsetFilterCache; +import org.opensearch.index.compositeindex.CompositeIndexConfig; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.FieldMapper; @@ -238,6 +239,13 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for concurrent segment search IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING, + // Settings for composite index defaults + CompositeIndexConfig.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, + CompositeIndexConfig.COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, + CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING, + CompositeIndexConfig.DEFAULT_METRICS_LIST, + CompositeIndexConfig.DEFAULT_DATE_INTERVALS, + // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { Map groups = s.getAsGroups(); @@ -250,7 +258,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { } }, Property.IndexScope), // this allows similarity settings to be passed Setting.groupSetting("index.analysis.", Property.IndexScope), // this allows analysis settings to be passed - Setting.groupSetting("index.composite_index.", Property.IndexScope) // this allows composite index settings to be passed + Setting.groupSetting("index.composite_index.config.", Property.IndexScope) // this allows composite index settings to be + // passed ) ) ); diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index aaec6bfec2123..18122745a4218 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -66,7 +66,6 @@ import org.opensearch.index.cache.query.DisabledQueryCache; import org.opensearch.index.cache.query.IndexQueryCache; import org.opensearch.index.cache.query.QueryCache; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -608,7 +607,7 @@ public IndexService newIndexService( Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -668,7 +667,7 @@ public IndexService newIndexService( clusterDefaultRefreshIntervalSupplier, recoverySettings, remoteStoreSettings, - compositeIndexSettings + isCompositeIndexCreationEnabled ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 6f994e8f67c5b..96081388baae5 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -73,7 +73,6 @@ import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.query.QueryCache; import org.opensearch.index.compositeindex.CompositeIndexConfig; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -227,7 +226,7 @@ public IndexService( Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -267,8 +266,9 @@ public IndexService( } if (indexSettings.getCompositeIndexConfig().hasCompositeFields()) { + // The validation is done right after the merge of the mapping later in the process ( similar to sort ) this.compositeIndexConfigSupplier = () -> indexSettings.getCompositeIndexConfig() - .validateAndGetCompositeIndexConfig(mapperService::fieldType, compositeIndexSettings); + .validateAndGetCompositeIndexConfig(mapperService::fieldType, isCompositeIndexCreationEnabled); } else { this.compositeIndexConfigSupplier = () -> null; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java index 51a2b454abcd9..829084ebf8b1a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeFieldSpec.java @@ -17,6 +17,4 @@ */ @ExperimentalApi -public interface CompositeFieldSpec { - void setDefaults(CompositeIndexSettings compositeIndexSettings); -} +public interface CompositeFieldSpec {} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java index 308c4b62d3996..4576a1b70a5f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConfig.java @@ -10,23 +10,26 @@ import org.opensearch.common.Rounding; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.indices.IndicesService; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.BooleanSupplier; import java.util.function.Function; -import static org.opensearch.index.compositeindex.CompositeIndexSettings.COMPOSITE_INDEX_ENABLED_SETTING; - /** * Configuration of composite index containing list of composite fields. * Each composite field contains dimensions, metrics along with composite index (eg: star tree) specific settings. @@ -45,6 +48,76 @@ public class CompositeIndexConfig { NumberFieldMapper.NumberFieldType.class ); + /** + * This setting determines the max number of composite fields that can be part of composite index config. For each + * composite field, we will generate associated composite index. (eg : star tree index per field ) + */ + public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( + "index.composite_index.max_fields", + 1, + 1, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting determines the max number of dimensions that can be part of composite index field. Number of + * dimensions and associated cardinality has direct effect of composite index size and query performance. + */ + public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( + "index.composite_index.field.max_dimensions", + 10, + 2, + 10, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and + * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa + *

+ * We can remove this later or change it to an enum based constant setting. + * + * @opensearch.experimental + */ + public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( + "index.composite_index.startree.default.max_leaf_docs", + 10000, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * Default intervals for date dimension as part of composite fields + */ + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + "index.composite_index.field.default.date_intervals", + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + CompositeIndexConfig::getTimeUnit, + Setting.Property.IndexScope, + Setting.Property.Final + ); + public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( + "index.composite_index.field.default.metrics", + Arrays.asList( + MetricType.AVG.toString(), + MetricType.COUNT.toString(), + MetricType.SUM.toString(), + MetricType.MAX.toString(), + MetricType.MIN.toString() + ), + MetricType::fromTypeName, + Setting.Property.IndexScope, + Setting.Property.Final + ); + private volatile int maxLeafDocs; + private volatile List defaultDateIntervals; + private volatile List defaultMetrics; + private volatile int maxDimensions; + private volatile int maxFields; private static final String COMPOSITE_INDEX_CONFIG = "index.composite_index.config"; private static final String DIMENSIONS_ORDER = "dimensions_order"; private static final String DIMENSIONS_CONFIG = "dimensions_config"; @@ -58,23 +131,57 @@ public class CompositeIndexConfig { private static final String SKIP_STAR_NODE_CREATION_FOR_DIMS = "skip_star_node_creation_for_dimensions"; private static final String SPEC = "_spec"; private final List compositeFields = new ArrayList<>(); + private final IndexSettings indexSettings; public CompositeIndexConfig(IndexSettings indexSettings) { - + this.setMaxLeafDocs(indexSettings.getValue(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); + this.setDefaultDateIntervals(indexSettings.getValue(DEFAULT_DATE_INTERVALS)); + this.setDefaultMetrics(indexSettings.getValue(DEFAULT_METRICS_LIST)); + this.setMaxDimensions(indexSettings.getValue(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); + this.setMaxFields(indexSettings.getValue(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); final Map compositeIndexSettings = indexSettings.getSettings().getGroups(COMPOSITE_INDEX_CONFIG); + this.indexSettings = indexSettings; Set fields = compositeIndexSettings.keySet(); + if (!fields.isEmpty()) { + if (!FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING)) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + if (fields.size() > getMaxFields()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] fields", getMaxFields()) + ); + } + } for (String field : fields) { compositeFields.add(buildCompositeField(field, compositeIndexSettings.get(field))); } } /** - * This returns composite field after performing basic validations and doesn't do field type validations etc + * This returns composite field after performing basic validations but doesn't perform field type based validations * */ private CompositeField buildCompositeField(String field, Settings compositeFieldSettings) { + + List dimensions = buildDimensions(field, compositeFieldSettings); + List metrics = buildMetrics(field, compositeFieldSettings); + List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); + IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); + Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); + CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder, dimensions, this); + return new CompositeField(field, dimensions, metrics, compositeFieldSpec); + } + + /** + * Returns dimensions after performing validations + * + */ + private List buildDimensions(String field, Settings compositeFieldSettings) { List dimensions = new ArrayList<>(); - List metrics = new ArrayList<>(); List dimensionsOrder = compositeFieldSettings.getAsList(DIMENSIONS_ORDER); if (dimensionsOrder == null || dimensionsOrder.isEmpty()) { throw new IllegalArgumentException( @@ -87,6 +194,12 @@ private CompositeField buildCompositeField(String field, Settings compositeField ); } + if (dimensionsOrder.size() > getMaxDimensions()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", getMaxDimensions()) + ); + } + Map dimConfig = compositeFieldSettings.getGroups(DIMENSIONS_CONFIG); for (String dimension : dimConfig.keySet()) { @@ -102,25 +215,6 @@ private CompositeField buildCompositeField(String field, Settings compositeField } } - List metricFields = compositeFieldSettings.getAsList(METRICS); - if (metricFields == null || metricFields.isEmpty()) { - throw new IllegalArgumentException(String.format(Locale.ROOT, "metrics is required for composite index field [%s]", field)); - } - Map metricsConfig = compositeFieldSettings.getGroups(METRICS_CONFIG); - - for (String metricField : metricsConfig.keySet()) { - if (!metricFields.contains(metricField)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "metric field [%s] is not present in 'metrics' for composite index field [%s]", - metricField, - field - ) - ); - } - } - Set uniqueDimensions = new HashSet<>(); for (String dimension : dimensionsOrder) { if (!uniqueDimensions.add(dimension)) { @@ -133,72 +227,16 @@ private CompositeField buildCompositeField(String field, Settings compositeField ) ); } - dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension))); - } - uniqueDimensions = null; - Set uniqueMetricFields = new HashSet<>(); - for (String metricField : metricFields) { - if (!uniqueMetricFields.add(metricField)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "duplicate metric field [%s] found in 'metrics' for composite index field [%s]", - metricField, - field - ) - ); - } - Settings metricSettings = metricsConfig.get(metricField); - if (metricSettings == null) { - // fill cluster level defaults in create flow as part of CompositeIndexSupplier - metrics.add(new Metric(metricField, new ArrayList<>())); - } else { - String name = metricSettings.get(FIELD, metricField); - List metricsList = metricSettings.getAsList(METRICS); - if (metricsList.isEmpty()) { - // fill cluster level defaults in create flow as part of CompositeIndexSupplier - metrics.add(new Metric(name, new ArrayList<>())); - } else { - List metricTypes = new ArrayList<>(); - Set uniqueMetricTypes = new HashSet<>(); - for (String metric : metricsList) { - if (!uniqueMetricTypes.add(metric)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "duplicate metric type [%s] found in metrics for composite index field [%s]", - metric, - field - ) - ); - } - metricTypes.add(MetricType.fromTypeName(metric)); - } - uniqueMetricTypes = null; - metrics.add(new Metric(name, metricTypes)); - } - } + dimensions.add(DimensionFactory.create(dimension, dimConfig.get(dimension), this)); } - uniqueMetricFields = null; - - IndexMode indexMode = IndexMode.fromTypeName(compositeFieldSettings.get(INDEX_MODE, IndexMode.STARTREE.typeName)); - Settings fieldSpec = compositeFieldSettings.getAsSettings(indexMode.typeName + SPEC); - CompositeFieldSpec compositeFieldSpec = CompositeFieldSpecFactory.create(indexMode, fieldSpec, dimensionsOrder); - return new CompositeField(field, dimensions, metrics, compositeFieldSpec); - } - - public static Rounding.DateTimeUnit getTimeUnit(String expression) { - if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { - throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); - } - return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + return dimensions; } /** * Dimension factory based on field type */ private static class DimensionFactory { - static Dimension create(String dimension, Settings settings) { + static Dimension create(String dimension, Settings settings, CompositeIndexConfig compositeIndexConfig) { if (settings == null) { return new Dimension(dimension); } @@ -208,20 +246,13 @@ static Dimension create(String dimension, Settings settings) { case DEFAULT: return new Dimension(field); case DATE: - return new DateDimension(field, settings); + return new DateDimension(field, settings, compositeIndexConfig); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid dimension type [%s] in composite index config", type) ); } } - - static Dimension createEmptyMappedDimension(Dimension dimension, MappedFieldType type) { - if (type instanceof DateFieldMapper.DateFieldType) { - return new DateDimension(dimension.getField(), new ArrayList<>()); - } - return dimension; - } } /** @@ -254,17 +285,94 @@ public static DimensionType fromTypeName(String typeName) { } } + /** + * Returns metrics after performing validations + * + */ + private List buildMetrics(String field, Settings compositeFieldSettings) { + List metrics = new ArrayList<>(); + List metricFields = compositeFieldSettings.getAsList(METRICS); + if (metricFields == null || metricFields.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "metrics is required for composite index field [%s]", field)); + } + Map metricsConfig = compositeFieldSettings.getGroups(METRICS_CONFIG); + + for (String metricField : metricsConfig.keySet()) { + if (!metricFields.contains(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "metric field [%s] is not present in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + } + Set uniqueMetricFields = new HashSet<>(); + for (String metricField : metricFields) { + if (!uniqueMetricFields.add(metricField)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric field [%s] found in 'metrics' for composite index field [%s]", + metricField, + field + ) + ); + } + Settings metricSettings = metricsConfig.get(metricField); + if (metricSettings == null) { + metrics.add(new Metric(metricField, getDefaultMetrics())); + } else { + String name = metricSettings.get(FIELD, metricField); + List metricsList = metricSettings.getAsList(METRICS); + if (metricsList.isEmpty()) { + metrics.add(new Metric(name, getDefaultMetrics())); + } else { + List metricTypes = new ArrayList<>(); + Set uniqueMetricTypes = new HashSet<>(); + for (String metric : metricsList) { + if (!uniqueMetricTypes.add(metric)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "duplicate metric type [%s] found in metrics for composite index field [%s]", + metric, + field + ) + ); + } + metricTypes.add(MetricType.fromTypeName(metric)); + } + metrics.add(new Metric(name, metricTypes)); + } + } + } + return metrics; + } + /** * Composite field spec factory based on index mode */ private static class CompositeFieldSpecFactory { - static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + static CompositeFieldSpec create( + IndexMode indexMode, + Settings settings, + List dimensionsOrder, + List dimensions, + CompositeIndexConfig compositeIndexConfig + ) { if (settings == null) { - return new StarTreeFieldSpec(10000, new ArrayList<>(), StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP); + return new StarTreeFieldSpec( + compositeIndexConfig.getMaxLeafDocs(), + new ArrayList<>(), + StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP + ); } switch (indexMode) { case STARTREE: - return buildStarTreeFieldSpec(settings, dimensions); + return buildStarTreeFieldSpec(settings, dimensionsOrder, dimensions, compositeIndexConfig); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid index mode [%s] in composite index config", indexMode) @@ -273,26 +381,35 @@ static CompositeFieldSpec create(IndexMode indexMode, Settings settings, List dimensions) { + private static StarTreeFieldSpec buildStarTreeFieldSpec( + Settings settings, + List dimensionsString, + List dimensions, + CompositeIndexConfig compositeIndexConfig + ) { StarTreeFieldSpec.StarTreeBuildMode buildMode = StarTreeFieldSpec.StarTreeBuildMode.fromTypeName( settings.get(STAR_TREE_BUILD_MODE, StarTreeFieldSpec.StarTreeBuildMode.OFF_HEAP.getTypeName()) ); - // Fill default value as part of create flow as part of supplier - int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, Integer.MAX_VALUE); + int maxLeafDocs = settings.getAsInt(MAX_LEAF_DOCS, compositeIndexConfig.getMaxLeafDocs()); if (maxLeafDocs < 1) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid max_leaf_docs [%s] in composite index config", maxLeafDocs) ); } List skipStarNodeCreationInDims = settings.getAsList(SKIP_STAR_NODE_CREATION_FOR_DIMS, new ArrayList<>()); + Set skipListWithMappedFieldNames = new HashSet<>(); for (String dim : skipStarNodeCreationInDims) { - if (!dimensions.contains(dim)) { + if (!dimensionsString.contains(dim)) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Invalid dimension [%s] in skip_star_node_creation_for_dims", dim) ); } + boolean duplicate = !(skipListWithMappedFieldNames.add(dimensions.get(dimensionsString.indexOf(dim)).getField())); + if (duplicate) { + throw new IllegalArgumentException(String.format("duplicate dimension [%s] found in skipStarNodeCreationInDims", dim)); + } } - return new StarTreeFieldSpec(maxLeafDocs, skipStarNodeCreationInDims, buildMode); + return new StarTreeFieldSpec(maxLeafDocs, new ArrayList<>(skipListWithMappedFieldNames), buildMode); } /** @@ -337,71 +454,37 @@ public boolean hasCompositeFields() { } /** - * Validates the composite fields based on IndexSettingDefaults and the mappedFieldType - * Updates CompositeIndexConfig with newer, completely updated composite fields + * Validates the composite fields based on defaults and based on the mappedFieldType + * Updates CompositeIndexConfig with newer, completely updated composite fields. * */ public CompositeIndexConfig validateAndGetCompositeIndexConfig( Function fieldTypeLookup, - CompositeIndexSettings compositeIndexSettings + BooleanSupplier isCompositeIndexCreationEnabled ) { if (hasCompositeFields() == false) { return null; } - if (!compositeIndexSettings.isEnabled()) { + if (!isCompositeIndexCreationEnabled.getAsBoolean()) { throw new IllegalArgumentException( String.format( Locale.ROOT, "composite index cannot be created, enable it using [%s] setting", - COMPOSITE_INDEX_ENABLED_SETTING.getKey() + IndicesService.COMPOSITE_INDEX_ENABLED_SETTING.getKey() ) ); } - if (compositeFields.size() > compositeIndexSettings.getMaxFields()) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "composite index can have atmost [%s] fields", compositeIndexSettings.getMaxFields()) - ); - } - List validatedAndMappedCompositeFields = new ArrayList<>(); for (CompositeField compositeField : compositeFields) { - if (compositeField.getDimensionsOrder().size() > compositeIndexSettings.getMaxDimensions()) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "composite index can have atmost [%s] dimensions", compositeIndexSettings.getMaxDimensions()) - ); - } - List dimensions = new ArrayList<>(); for (Dimension dimension : compositeField.getDimensionsOrder()) { - validateCompositeDimensionField(dimension.getField(), fieldTypeLookup, compositeField.getName()); - dimension = mapDimension(dimension, fieldTypeLookup.apply(dimension.getField())); - dimension.setDefaults(compositeIndexSettings); - dimensions.add(dimension); + validateDimensionField(dimension, fieldTypeLookup, compositeField.getName()); } - List metrics = new ArrayList<>(); for (Metric metric : compositeField.getMetrics()) { - validateCompositeMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); - metric.setDefaults(compositeIndexSettings); - metrics.add(metric); + validateMetricField(metric.getField(), fieldTypeLookup, compositeField.getName()); } - compositeField.getSpec().setDefaults(compositeIndexSettings); - validatedAndMappedCompositeFields.add( - new CompositeField(compositeField.getName(), dimensions, metrics, compositeField.getSpec()) - ); } - this.compositeFields.clear(); - this.compositeFields.addAll(validatedAndMappedCompositeFields); return this; } - /** - * Maps the dimension to right dimension type based on MappedFieldType - */ - private Dimension mapDimension(Dimension dimension, MappedFieldType fieldType) { - if (!isDimensionMappedToFieldType(dimension, fieldType)) { - return DimensionFactory.createEmptyMappedDimension(dimension, fieldType); - } - return dimension; - } - /** * Checks whether dimension field type is same as the source field type */ @@ -419,15 +502,27 @@ private boolean isDimensionMappedToFieldType(Dimension dimension, MappedFieldTyp * The dimension fields should be of numberField type / dateField type * */ - private void validateCompositeDimensionField( - String field, - Function fieldTypeLookup, - String compositeFieldName - ) { - final MappedFieldType ft = fieldTypeLookup.apply(field); + private void validateDimensionField(Dimension dimension, Function fieldTypeLookup, String compositeFieldName) { + final MappedFieldType ft = fieldTypeLookup.apply(dimension.getField()); if (ft == null) { throw new IllegalArgumentException( - String.format(Locale.ROOT, "unknown dimension field [%s] as part of composite field [%s]", field, compositeFieldName) + String.format( + Locale.ROOT, + "unknown dimension field [%s] as part of composite field [%s]", + dimension.getField(), + compositeFieldName + ) + ); + } + if (!isDimensionMappedToFieldType(dimension, ft)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "dimension field [%s] with field type [%s] is not same as the source " + "field type for composite field [%s]", + dimension.getField(), + ft.typeName(), + compositeFieldName + ) ); } if (!isAllowedDimensionFieldType(ft)) { @@ -436,7 +531,7 @@ private void validateCompositeDimensionField( Locale.ROOT, "composite index is not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", - field, + dimension.getField(), ft.typeName(), compositeFieldName ) @@ -448,7 +543,7 @@ private void validateCompositeDimensionField( String.format( Locale.ROOT, "Aggregations not supported for the dimension field [%s] with field type [%s] as part of " + "composite field [%s]", - field, + dimension.getField(), ft.typeName(), compositeFieldName ) @@ -463,7 +558,7 @@ private void validateCompositeDimensionField( * The metric fields should be of numberField type * */ - private void validateCompositeMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { + private void validateMetricField(String field, Function fieldTypeLookup, String compositeFieldName) { final MappedFieldType ft = fieldTypeLookup.apply(field); if (ft == null) { throw new IllegalArgumentException( @@ -502,4 +597,52 @@ private static boolean isAllowedDimensionFieldType(MappedFieldType fieldType) { private static boolean isAllowedMetricFieldType(MappedFieldType fieldType) { return ALLOWED_METRIC_MAPPED_FIELD_TYPES.stream().anyMatch(allowedType -> allowedType.isInstance(fieldType)); } + + public static Rounding.DateTimeUnit getTimeUnit(String expression) { + if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + throw new IllegalArgumentException("unknown calendar interval specified in composite index config"); + } + return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + } + + public void setMaxLeafDocs(int maxLeafDocs) { + this.maxLeafDocs = maxLeafDocs; + } + + public void setDefaultDateIntervals(List defaultDateIntervals) { + this.defaultDateIntervals = defaultDateIntervals; + } + + public void setDefaultMetrics(List defaultMetrics) { + this.defaultMetrics = defaultMetrics; + } + + public void setMaxDimensions(int maxDimensions) { + this.maxDimensions = maxDimensions; + } + + public void setMaxFields(int maxFields) { + this.maxFields = maxFields; + } + + public int getMaxDimensions() { + return maxDimensions; + } + + public int getMaxFields() { + return maxFields; + } + + public int getMaxLeafDocs() { + return maxLeafDocs; + } + + public List getDefaultDateIntervals() { + return defaultDateIntervals; + } + + public List getDefaultMetrics() { + return defaultMetrics; + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java deleted file mode 100644 index 59bdfda5469ea..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex; - -import org.opensearch.common.Rounding; -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.util.FeatureFlags; - -import java.util.Arrays; -import java.util.List; - -/** - * Cluster level settings which configures defaults for composite index - */ -@ExperimentalApi -public class CompositeIndexSettings { - /** - * This cluster level setting determines whether composite index is enabled or not - */ - public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( - "indices.composite_index.enabled", - false, - value -> { - if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { - throw new IllegalArgumentException( - "star tree index is under an experimental feature and can be activated only by enabling " - + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() - + " feature flag in the JVM options" - ); - } - }, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting determines the max number of composite fields that can be part of composite index config. For each - * composite field, we will generate associated composite index. (eg : star tree index per field ) - */ - public static final Setting COMPOSITE_INDEX_MAX_FIELDS_SETTING = Setting.intSetting( - "indices.composite_index.max_fields", - 1, - 1, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting determines the max number of dimensions that can be part of composite index field. Number of - * dimensions and associated cardinality has direct effect of composite index size and query performance. - */ - public static final Setting COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING = Setting.intSetting( - "indices.composite_index.field.max_dimensions", - 10, - 2, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and - * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa - *

- * We can remove this later or change it to an enum based constant setting. - * - * @opensearch.experimental - */ - public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( - "indices.composite_index.startree.default.max_leaf_docs", - 10000, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - /** - * Default intervals for date dimension as part of composite fields - */ - public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( - "indices.composite_index.field.default.date_intervals", - Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), - CompositeIndexConfig::getTimeUnit, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( - "indices.composite_index.field.default.metrics", - Arrays.asList( - MetricType.AVG.toString(), - MetricType.COUNT.toString(), - MetricType.SUM.toString(), - MetricType.MAX.toString(), - MetricType.MIN.toString() - ), - MetricType::fromTypeName, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private volatile int maxLeafDocs; - - private volatile List defaultDateIntervals; - - private volatile List defaultMetrics; - private volatile int maxDimensions; - private volatile int maxFields; - private volatile boolean enabled; - - public CompositeIndexSettings(ClusterSettings clusterSettings) { - this.setMaxLeafDocs(clusterSettings.get(STAR_TREE_DEFAULT_MAX_LEAF_DOCS)); - this.setDefaultDateIntervals(clusterSettings.get(DEFAULT_DATE_INTERVALS)); - this.setDefaultMetrics(clusterSettings.get(DEFAULT_METRICS_LIST)); - this.setMaxDimensions(clusterSettings.get(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING)); - this.setMaxFields(clusterSettings.get(COMPOSITE_INDEX_MAX_FIELDS_SETTING)); - this.setEnabled(clusterSettings.get(COMPOSITE_INDEX_ENABLED_SETTING)); - - clusterSettings.addSettingsUpdateConsumer(STAR_TREE_DEFAULT_MAX_LEAF_DOCS, this::setMaxLeafDocs); - clusterSettings.addSettingsUpdateConsumer(DEFAULT_DATE_INTERVALS, this::setDefaultDateIntervals); - clusterSettings.addSettingsUpdateConsumer(DEFAULT_METRICS_LIST, this::setDefaultMetrics); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_DIMENSIONS_SETTING, this::setMaxDimensions); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_MAX_FIELDS_SETTING, this::setMaxFields); - clusterSettings.addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setEnabled); - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public void setMaxLeafDocs(int maxLeafDocs) { - this.maxLeafDocs = maxLeafDocs; - } - - public void setDefaultDateIntervals(List defaultDateIntervals) { - this.defaultDateIntervals = defaultDateIntervals; - } - - public void setDefaultMetrics(List defaultMetrics) { - this.defaultMetrics = defaultMetrics; - } - - public void setMaxDimensions(int maxDimensions) { - this.maxDimensions = maxDimensions; - } - - public void setMaxFields(int maxFields) { - this.maxFields = maxFields; - } - - public int getMaxDimensions() { - return maxDimensions; - } - - public int getMaxFields() { - return maxFields; - } - - public int getMaxLeafDocs() { - return maxLeafDocs; - } - - public boolean isEnabled() { - return enabled; - } - - public List getDefaultDateIntervals() { - return defaultDateIntervals; - } - - public List getDefaultMetrics() { - return defaultMetrics; - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java index 40145f9f80ef3..3e88759ec56b9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java @@ -24,11 +24,11 @@ public class DateDimension extends Dimension { private final List calendarIntervals; - public DateDimension(String name, Settings settings) { + public DateDimension(String name, Settings settings, CompositeIndexConfig compositeIndexConfig) { super(name); List intervalStrings = settings.getAsList("calendar_interval"); if (intervalStrings == null || intervalStrings.isEmpty()) { - this.calendarIntervals = new ArrayList<>(); + this.calendarIntervals = compositeIndexConfig.getDefaultDateIntervals(); } else { this.calendarIntervals = new ArrayList<>(); for (String interval : intervalStrings) { @@ -37,18 +37,6 @@ public DateDimension(String name, Settings settings) { } } - public DateDimension(String name, List calendarIntervals) { - super(name); - this.calendarIntervals = calendarIntervals; - } - - @Override - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - if (calendarIntervals.isEmpty()) { - this.calendarIntervals.addAll(compositeIndexSettings.getDefaultDateIntervals()); - } - } - public List getIntervals() { return calendarIntervals; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java index c06df856d5b8a..a18ffcd1df0db 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/Dimension.java @@ -26,8 +26,4 @@ public Dimension(String field) { public String getField() { return field; } - - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - // no implementation - } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java index dddc3c795078c..9467cf1176f7a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/Metric.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/Metric.java @@ -33,9 +33,9 @@ public List getMetrics() { return metrics; } - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { + public void setDefaults(CompositeIndexConfig compositeIndexConfig) { if (metrics.isEmpty()) { - metrics.addAll(compositeIndexSettings.getDefaultMetrics()); + metrics.addAll(compositeIndexConfig.getDefaultMetrics()); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java index c38f8ca8813c7..c3ac54feeb5e4 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/StarTreeFieldSpec.java @@ -63,11 +63,4 @@ public static StarTreeBuildMode fromTypeName(String typeName) { public int maxLeafDocs() { return maxLeafDocs.get(); } - - @Override - public void setDefaults(CompositeIndexSettings compositeIndexSettings) { - if (maxLeafDocs.get() == Integer.MAX_VALUE) { - maxLeafDocs.set(compositeIndexSettings.getMaxLeafDocs()); - } - } } diff --git a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java b/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java deleted file mode 100644 index be49269f0264c..0000000000000 --- a/server/src/main/java/org/opensearch/indices/DefaultCompositeIndexSettings.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.indices; - -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.compositeindex.CompositeIndexSettings; - -/** - * Utility to provide a {@link CompositeIndexSettings} instance containing all defaults - * - * @opensearch.experimental - */ -@ExperimentalApi -public final class DefaultCompositeIndexSettings { - private DefaultCompositeIndexSettings() {} - - public static final CompositeIndexSettings INSTANCE = new CompositeIndexSettings( - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ); -} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b1fb2f77e2981..4731ad07285c0 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -74,6 +74,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRefCounted; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.OpenSearchExecutors; @@ -106,7 +107,6 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.cache.request.ShardRequestCache; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineConfigFactory; @@ -307,6 +307,25 @@ public class IndicesService extends AbstractLifecycleComponent Property.Final ); + /** + * This cluster level setting determines whether composite index is enabled or not + */ + public static final Setting COMPOSITE_INDEX_ENABLED_SETTING = Setting.boolSetting( + "indices.composite_index.enabled", + false, + value -> { + if (FeatureFlags.isEnabled(FeatureFlags.COMPOSITE_INDEX_SETTING) == false && value == true) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.COMPOSITE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** * The node's settings. */ @@ -355,7 +374,7 @@ public class IndicesService extends AbstractLifecycleComponent private final BiFunction translogFactorySupplier; private volatile TimeValue clusterDefaultRefreshInterval; private final SearchRequestStats searchRequestStats; - private final CompositeIndexSettings compositeIndexSettings; + private volatile boolean compositeIndexCreationEnabled; @Override protected void doStart() { @@ -390,9 +409,9 @@ public IndicesService( @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory, RecoverySettings recoverySettings, CacheService cacheService, - RemoteStoreSettings remoteStoreSettings, - CompositeIndexSettings compositeIndexSettings + RemoteStoreSettings remoteStoreSettings ) { + // Build compositeIndexSettings (cluster level defaults) into node settings to initialize compositeIndexConfig this.settings = settings; this.threadPool = threadPool; this.pluginsService = pluginsService; @@ -443,6 +462,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon this.directoryFactories = directoryFactories; this.recoveryStateFactories = recoveryStateFactories; + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(COMPOSITE_INDEX_ENABLED_SETTING, this::setCompositeIndexCreationEnabled); // doClose() is called when shutting down a node, yet there might still be ongoing requests // that we need to wait for before closing some resources such as the caches. In order to // avoid closing these resources while ongoing requests are still being processed, we use a @@ -498,7 +519,6 @@ protected void closeInternal() { .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate); this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; - this.compositeIndexSettings = compositeIndexSettings; } /** @@ -908,7 +928,7 @@ private synchronized IndexService createIndexService( this::getClusterDefaultRefreshInterval, this.recoverySettings, this.remoteStoreSettings, - this.compositeIndexSettings + this::isCompositeIndexCreationEnabled ); } @@ -1901,6 +1921,14 @@ private void setIdFieldDataEnabled(boolean value) { this.idFieldDataEnabled = value; } + private void setCompositeIndexCreationEnabled(boolean value) { + this.compositeIndexCreationEnabled = value; + } + + public boolean isCompositeIndexCreationEnabled() { + return compositeIndexCreationEnabled; + } + private void updateDanglingIndicesInfo(Index index) { assert DiscoveryNode.isDataNode(settings) : "dangling indices information should only be persisted on data nodes"; assert nodeWriteDanglingIndicesInfo : "writing dangling indices info is not enabled"; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 38fb0c4cb625e..fb05bd22802b7 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -146,7 +146,6 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationStatsTracker; import org.opensearch.index.analysis.AnalysisRegistry; -import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; @@ -835,11 +834,10 @@ protected Node( final SearchRequestStats searchRequestStats = new SearchRequestStats(clusterService.getClusterSettings()); final SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService); - final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterService.getClusterSettings()); - remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); CacheModule cacheModule = new CacheModule(pluginsService.filterPlugins(CachePlugin.class), settings); CacheService cacheService = cacheModule.getCacheService(); + final IndicesService indicesService = new IndicesService( settings, pluginsService, @@ -867,8 +865,7 @@ protected Node( remoteStoreStatsTrackerFactory, recoverySettings, cacheService, - remoteStoreSettings, - compositeIndexSettings + remoteStoreSettings ); final IngestService ingestService = new IngestService( diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 8f45a872e752c..829e65569bc75 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -99,7 +99,6 @@ import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.TranslogFactory; -import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesQueryCache; @@ -266,7 +265,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, DefaultRemoteStoreSettings.INSTANCE, - DefaultCompositeIndexSettings.INSTANCE + () -> false ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java new file mode 100644 index 0000000000000..de3c3d033ced2 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigIntegTests.java @@ -0,0 +1,239 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexService; +import org.opensearch.indices.IndicesService; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class CompositeIndexConfigIntegTests extends OpenSearchIntegTestCase { + + private static final XContentBuilder TEST_MAPPING = createTestMapping(); + + private static XContentBuilder createTestMapping() { + try { + return jsonBuilder().startObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric") + .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") + .field("doc_values", true) + .endObject() + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.COMPOSITE_INDEX, "true").build(); + } + + @Before + public final void setupNodeSettings() { + Settings request = Settings.builder().put(IndicesService.COMPOSITE_INDEX_ENABLED_SETTING.getKey(), true).build(); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get()); + } + + @After + public final void cleanupNodeSettings() { + assertAcked( + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("*")) + .setTransientSettings(Settings.builder().putNull("*")) + ); + } + + public void testInvalidCompositeIndex() { + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "dimension field [timestamp] with field type [date] is not same as the source field type for composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals( + "Aggregations not supported for the composite index metric field [numeric] with field type [integer] as part of composite field [my_field]", + ex.getMessage() + ); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("invalid", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("unknown dimension field [invalid] as part of composite field [my_field]", ex.getMessage()); + + ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "1") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("invalid")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get() + ); + assertEquals("unknown metric field [invalid] as part of composite field [my_field]", ex.getMessage()); + + FeatureFlagSetter.set(FeatureFlags.COMPOSITE_INDEX); + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "2") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get(); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("test"); + GetSettingsResponse indexSettings = client().admin().indices().getSettings(getSettingsRequest).actionGet(); + indexSettings.getIndexToSettings().get("test"); + final Index index = resolveIndex("test"); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + CompositeIndexConfig compositeIndexConfig = indexService.getIndexSettings().getCompositeIndexConfig(); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("timestamp", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("numeric_dv", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0) instanceof DateDimension); + } + } + } + + public void testValidCompositeIndex() { + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put("index.number_of_shards", "1") + .put("index.number_of_replicas", "2") + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("timestamp", "numeric_dv")) + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("numeric_dv")) + .put("index.composite_index.config.my_field.dimensions_config.timestamp.type", "date") + ).setMapping(TEST_MAPPING).get(); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("test"); + GetSettingsResponse indexSettings = client().admin().indices().getSettings(getSettingsRequest).actionGet(); + indexSettings.getIndexToSettings().get("test"); + final Index index = resolveIndex("test"); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + CompositeIndexConfig compositeIndexConfig = indexService.getIndexSettings().getCompositeIndexConfig(); + assertTrue(compositeIndexConfig.hasCompositeFields()); + assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("timestamp", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("numeric_dv", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0) instanceof DateDimension); + List expectedMetrics = Arrays.asList( + MetricType.AVG, + MetricType.COUNT, + MetricType.SUM, + MetricType.MAX, + MetricType.MIN + ); + assertEquals(expectedMetrics, compositeIndexConfig.getCompositeFields().get(0).getMetrics().get(0).getMetrics()); + List expectedIntervals = Arrays.asList( + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY + ); + assertEquals( + expectedIntervals, + ((DateDimension) compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0)).getIntervals() + ); + } + } + + } + +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java index 98a017bf1acbd..e10cce26f70c2 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/CompositeIndexConfigSettingsTests.java @@ -9,43 +9,51 @@ package org.opensearch.index.compositeindex; import org.opensearch.common.Rounding; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.IpFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.BooleanSupplier; import java.util.function.Function; +import static org.opensearch.common.util.FeatureFlags.COMPOSITE_INDEX; import static org.opensearch.index.IndexSettingsTests.newIndexMeta; /** * Composite index config settings unit tests */ public class CompositeIndexConfigSettingsTests extends OpenSearchTestCase { - private static IndexSettings indexSettings(Settings settings) { - return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); - } public void testDefaultSettings() { Settings settings = Settings.EMPTY; - IndexSettings indexSettings = indexSettings(settings); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertFalse(compositeIndexConfig.hasCompositeFields()); } public void testMinimumMetrics() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + Settings settings = Settings.builder() .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + ; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("metrics is required for composite index field [my_field]", exception.getMessage()); + + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testMinimumDimensions() { @@ -55,8 +63,11 @@ public void testMinimumDimensions() { .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Atleast two dimensions are required to build composite index field [my_field]", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testInvalidDimensionType() { @@ -66,8 +77,11 @@ public void testInvalidDimensionType() { .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "invalid") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Invalid dimension type in composite index config: [invalid] ", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testInvalidIndexMode() { @@ -78,8 +92,31 @@ public void testInvalidIndexMode() { .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .put("index.composite_index.config.my_field.index_mode", "invalid") .build(); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); assertEquals("Invalid index mode in composite index config: [invalid] ", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDefaultNumberofCompositeFieldsValidation() { + Settings settings = Settings.builder() + .put("index.composite_index.max_fields", 1) + .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.field1.metrics", Arrays.asList("metric1")) + .putList("index.composite_index.config.field2.dimensions_order", Arrays.asList("dim3", "dim4")) + .put("index.composite_index.config.field2.dimensions_config.dim3.field", "dim3_field") + .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") + .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) + .build(); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getEnabledIndexSettings(settings)); + assertEquals("composite index can have atmost [1] fields", exception.getMessage()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testValidCompositeIndexConfig() { @@ -87,17 +124,25 @@ public void testValidCompositeIndexConfig() { .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .put("index.composite_index.config.my_field.index_mode", "startree") .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + assertEquals("dim1_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("dim2_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertTrue(compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1) instanceof DateDimension); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexMultipleFields() { Settings settings = Settings.builder() - .put("indices.composite_index.max_fields", 2) + .put("index.composite_index.max_fields", 2) .putList("index.composite_index.config.field1.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.field1.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.field1.dimensions_config.dim2.field", "dim2_field") @@ -107,15 +152,33 @@ public void testCompositeIndexMultipleFields() { .put("index.composite_index.config.field2.dimensions_config.dim4.field", "dim4_field") .putList("index.composite_index.config.field2.metrics", Arrays.asList("metric2")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + Exception ex = expectThrows(IllegalArgumentException.class, () -> getEnabledAndMultiFieldIndexSettings(settings)); + assertEquals("Failed to parse value [2] for setting [index.composite_index.max_fields] must be <= 1", ex.getMessage()); + /** + * // uncomment once we add support for multiple fields + IndexSettings indexSettings = getEnabledAndMultiFieldIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(2, compositeIndexConfig.getCompositeFields().size()); + assertEquals(2, compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().size()); + assertEquals(2, compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().size()); + assertEquals(1, compositeIndexConfig.getCompositeFields().get(0).getMetrics().size()); + assertEquals(1, compositeIndexConfig.getCompositeFields().get(1).getMetrics().size()); + assertEquals("dim1_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(0).getField()); + assertEquals("dim2_field", compositeIndexConfig.getCompositeFields().get(0).getDimensionsOrder().get(1).getField()); + assertEquals("dim3_field", compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().get(0).getField()); + assertEquals("dim4_field", compositeIndexConfig.getCompositeFields().get(1).getDimensionsOrder().get(1).getField()); + assertEquals("metric1", compositeIndexConfig.getCompositeFields().get(0).getMetrics().get(0).getField()); + assertEquals("metric2", compositeIndexConfig.getCompositeFields().get(1).getMetrics().get(0).getField()); + **/ + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexDateIntervalsSetting() { Settings settings = Settings.builder() - .putList("indices.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) + .putList("index.composite_index.field.default.date_intervals", Arrays.asList("day", "week")) .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim1.type", "date") @@ -123,7 +186,9 @@ public void testCompositeIndexDateIntervalsSetting() { .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); @@ -133,27 +198,31 @@ public void testCompositeIndexDateIntervalsSetting() { Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR ); assertEquals(expectedIntervals, ((DateDimension) compositeField.getDimensionsOrder().get(0)).getIntervals()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testCompositeIndexMetricsSetting() { Settings settings = Settings.builder() - .putList("indices.composite_index.field.default.metrics", Arrays.asList("count", "max")) + .putList("index.composite_index.field.default.metrics", Arrays.asList("count", "max")) .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) .putList("index.composite_index.config.my_field.metrics_config.metric1.metrics", Arrays.asList("count", "max")) .build(); - IndexSettings indexSettings = indexSettings(settings); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + IndexSettings indexSettings = getEnabledIndexSettings(settings); CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(indexSettings); assertTrue(compositeIndexConfig.hasCompositeFields()); assertEquals(1, compositeIndexConfig.getCompositeFields().size()); CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); List expectedMetrics = Arrays.asList(MetricType.COUNT, MetricType.MAX); assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } - public void testValidateWithoutCompositeSettingEnabled() { + public void testCompositeIndexEnabledSetting() { Settings settings = Settings.builder() .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") @@ -162,29 +231,25 @@ public void testValidateWithoutCompositeSettingEnabled() { .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) .build(); - + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); Map fieldTypes = new HashMap<>(); fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.LONG)); fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric2", NumberFieldMapper.NumberType.LONG)); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); Function fieldTypeLookup = fieldTypes::get; - - CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(createIndexSettings(settings)); - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(clusterSettings); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, compositeIndexSettings) + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getDisabledSupplier()) ); assertEquals( "composite index cannot be created, enable it using [indices.composite_index.enabled] setting", exception.getMessage() ); - - assertTrue(compositeIndexConfig.hasCompositeFields()); - assertEquals(1, compositeIndexConfig.getCompositeFields().size()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } public void testEnabledWithFFOff() { @@ -200,34 +265,363 @@ public void testEnabledWithFFOff() { fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new CompositeIndexConfig(getEnabledIndexSettings(settings)) + ); + assertEquals( + "star tree index is under an experimental feature and can be activated only by enabling " + + "opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + exception.getMessage() + ); + } + + public void testUnknownDimField() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1")) + .build(); + + Map fieldTypes = new HashMap<>(); + + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals("unknown dimension field [dim1_field] as part of composite field [my_field]", exception.getMessage()); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testUnknownMetricField() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals("unknown metric field [metric2] as part of composite field [my_field]", exception.getMessage()); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionMappedType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new IpFieldMapper.IpFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + Function fieldTypeLookup = fieldTypes::get; - CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(testWithEnabledSettings(settings)); - Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); - ClusterSettings clusterSettings = new ClusterSettings(settings1, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> new CompositeIndexSettings(clusterSettings) + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) ); assertEquals( - "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.enabled feature flag in the JVM options", + "composite index is not supported for the dimension field [dim1_field] with field type [ip] as part of composite field [my_field]", exception.getMessage() ); - assertTrue(compositeIndexConfig.hasCompositeFields()); - assertEquals(1, compositeIndexConfig.getCompositeFields().size()); - CompositeField compositeField = compositeIndexConfig.getCompositeFields().get(0); - assertTrue(compositeField.getDimensionsOrder().get(0) instanceof Dimension); - assertTrue(compositeField.getDimensionsOrder().get(1) instanceof DateDimension); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDimsWithNoDocValues() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put( + "dim1_field", + new NumberFieldMapper.NumberFieldType( + "dim1_field", + NumberFieldMapper.NumberType.LONG, + false, + false, + false, + true, + null, + Collections.emptyMap() + ) + ); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "Aggregations not supported for the dimension field [dim1_field] with field type [long] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testMetricsWithNoDocValues() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put( + "metric1", + new NumberFieldMapper.NumberFieldType( + "metric1", + NumberFieldMapper.NumberType.LONG, + false, + false, + false, + true, + null, + Collections.emptyMap() + ) + ); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "Aggregations not supported for the composite index metric field [metric1] with field type [long] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidDimensionMappedKeywordType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new KeywordFieldMapper.KeywordFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "composite index is not supported for the dimension field [dim1_field] with " + + "field type [keyword] as part of composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testInvalidMetricMappedKeywordType() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new DateFieldMapper.DateFieldType("dim1_field")); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new DateFieldMapper.DateFieldType("metric1")); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "dimension field [dim1_field] with field type [date] is not same as the source field type for composite field [my_field]", + exception.getMessage() + ); + // reset FeatureFlags to defaults + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDefaults() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.type", "date") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + CompositeIndexConfig config = compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()); + + assertTrue(config.hasCompositeFields()); + assertEquals(1, config.getCompositeFields().size()); + CompositeField compositeField = config.getCompositeFields().get(0); + List expectedMetrics = Arrays.asList(MetricType.AVG, MetricType.COUNT, MetricType.SUM, MetricType.MAX, MetricType.MIN); + assertEquals(expectedMetrics, compositeField.getMetrics().get(0).getMetrics()); + StarTreeFieldSpec spec = (StarTreeFieldSpec) compositeField.getSpec(); + assertEquals(10000, spec.maxLeafDocs()); + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testDimTypeValidation() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(COMPOSITE_INDEX, true).build()); + assertTrue(FeatureFlags.isEnabled(COMPOSITE_INDEX)); + + Settings settings = Settings.builder() + .putList("index.composite_index.config.my_field.dimensions_order", Arrays.asList("dim1", "dim2")) + .put("index.composite_index.config.my_field.dimensions_config.dim1.field", "dim1_field") + .put("index.composite_index.config.my_field.dimensions_config.dim2.field", "dim2_field") + .putList("index.composite_index.config.my_field.metrics", Arrays.asList("metric1", "metric2")) + .build(); + + Map fieldTypes = new HashMap<>(); + fieldTypes.put("dim1_field", new NumberFieldMapper.NumberFieldType("dim1_field", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("dim2_field", new DateFieldMapper.DateFieldType("dim2_field")); + fieldTypes.put("metric1", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + fieldTypes.put("metric2", new NumberFieldMapper.NumberFieldType("metric1", NumberFieldMapper.NumberType.DOUBLE)); + + Function fieldTypeLookup = fieldTypes::get; + + CompositeIndexConfig compositeIndexConfig = new CompositeIndexConfig(getEnabledIndexSettings(settings)); + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> compositeIndexConfig.validateAndGetCompositeIndexConfig(fieldTypeLookup, getEnabledSupplier()) + ); + assertEquals( + "dimension field [dim2_field] with field type [date] is not same as the source field type for composite field [my_field]", + ex.getMessage() + ); } private IndexSettings createIndexSettings(Settings settings) { return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); } - public IndexSettings testWithEnabledSettings(Settings settings) { - Settings settings1 = Settings.builder().put(settings).put("indices.composite_index.enabled", true).build(); - IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", settings1), Settings.EMPTY); + private Settings getEnabledSettings() { + return Settings.builder().put("index.composite_index.enabled", true).build(); + } + + public IndexSettings getEnabledIndexSettings(Settings settings) { + Settings enabledSettings = Settings.builder().put(settings).put("index.composite_index.enabled", true).build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", enabledSettings), getEnabledSettings()); return indexSettings; } + public IndexSettings getEnabledAndMultiFieldIndexSettings(Settings settings) { + Settings multiFieldEnabledSettings = Settings.builder() + .put(settings) + .put("index.composite_index.enabled", true) + .put(CompositeIndexConfig.COMPOSITE_INDEX_MAX_FIELDS_SETTING.getKey(), 2) + .build(); + IndexSettings indexSettings = new IndexSettings(newIndexMeta("test", multiFieldEnabledSettings), multiFieldEnabledSettings); + return indexSettings; + } + + private BooleanSupplier getEnabledSupplier() { + return new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + return true; + } + }; + } + + private BooleanSupplier getDisabledSupplier() { + return new BooleanSupplier() { + @Override + public boolean getAsBoolean() { + return false; + } + }; + } } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 0156b6294663c..5b39880930984 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -192,7 +192,6 @@ import org.opensearch.index.shard.PrimaryReplicaSyncer; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCacheStats; -import org.opensearch.indices.DefaultCompositeIndexSettings; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesService; @@ -2085,8 +2084,7 @@ public void onFailure(final Exception e) { new RemoteStoreStatsTrackerFactory(clusterService, settings), DefaultRecoverySettings.INSTANCE, new CacheModule(new ArrayList<>(), settings).getCacheService(), - DefaultRemoteStoreSettings.INSTANCE, - DefaultCompositeIndexSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE ); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); snapshotShardsService = new SnapshotShardsService(