Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport-2.x] Create Index using context #15530

Merged
merged 4 commits into from
Sep 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494))
- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527))
- [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410))
- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290))

### Dependencies
- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import org.apache.lucene.util.CollectionUtil;
import org.opensearch.cluster.metadata.AliasMetadata;
import org.opensearch.cluster.metadata.Context;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.xcontent.XContentParser;
Expand Down Expand Up @@ -61,6 +62,7 @@
private Map<String, Settings> settings;
private Map<String, Settings> defaultSettings;
private Map<String, String> dataStreams;
private Map<String, Context> contexts;
private String[] indices;

GetIndexResponse(
Expand All @@ -69,7 +71,8 @@
Map<String, List<AliasMetadata>> aliases,
Map<String, Settings> settings,
Map<String, Settings> defaultSettings,
Map<String, String> dataStreams
Map<String, String> dataStreams,
Map<String, Context> contexts
) {
this.indices = indices;
// to have deterministic order
Expand All @@ -89,6 +92,9 @@
if (dataStreams != null) {
this.dataStreams = dataStreams;
}
if (contexts != null) {
this.contexts = contexts;
}
}

public String[] getIndices() {
Expand Down Expand Up @@ -123,6 +129,10 @@
return dataStreams;
}

public Map<String, Context> contexts() {
return contexts;
}

/**
* Returns the string value for the specified index and setting. If the includeDefaults flag was not set or set to
* false on the {@link GetIndexRequest}, this method will only return a value where the setting was explicitly set
Expand Down Expand Up @@ -167,6 +177,7 @@
Settings indexSettings = null;
Settings indexDefaultSettings = null;
String dataStream = null;
Context context = null;
// We start at START_OBJECT since fromXContent ensures that
while (parser.nextToken() != Token.END_OBJECT) {
ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser);
Expand All @@ -185,6 +196,9 @@
case "defaults":
indexDefaultSettings = Settings.fromXContent(parser);
break;
case "context":
context = Context.fromXContent(parser);
break;

Check warning on line 201 in client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java

View check run for this annotation

Codecov / codecov/patch

client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java#L200-L201

Added lines #L200 - L201 were not covered by tests
default:
parser.skipChildren();
}
Expand All @@ -197,7 +211,7 @@
parser.skipChildren();
}
}
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream);
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings, dataStream, context);
}

// This is just an internal container to make stuff easier for returning
Expand All @@ -207,19 +221,22 @@
Settings indexSettings = Settings.EMPTY;
Settings indexDefaultSettings = Settings.EMPTY;
String dataStream;
Context context;

IndexEntry(
List<AliasMetadata> indexAliases,
MappingMetadata indexMappings,
Settings indexSettings,
Settings indexDefaultSettings,
String dataStream
String dataStream,
Context context
) {
if (indexAliases != null) this.indexAliases = indexAliases;
if (indexMappings != null) this.indexMappings = indexMappings;
if (indexSettings != null) this.indexSettings = indexSettings;
if (indexDefaultSettings != null) this.indexDefaultSettings = indexDefaultSettings;
if (dataStream != null) this.dataStream = dataStream;
if (context != null) this.context = context;
}
}

Expand All @@ -229,6 +246,7 @@
Map<String, Settings> settings = new HashMap<>();
Map<String, Settings> defaultSettings = new HashMap<>();
Map<String, String> dataStreams = new HashMap<>();
Map<String, Context> contexts = new HashMap<>();
List<String> indices = new ArrayList<>();

if (parser.currentToken() == null) {
Expand All @@ -254,12 +272,15 @@
if (indexEntry.dataStream != null) {
dataStreams.put(indexName, indexEntry.dataStream);
}
if (indexEntry.context != null) {
contexts.put(indexName, indexEntry.context);

Check warning on line 276 in client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java

View check run for this annotation

Codecov / codecov/patch

client/rest-high-level/src/main/java/org/opensearch/client/indices/GetIndexResponse.java#L276

Added line #L276 was not covered by tests
}
} else if (parser.currentToken() == Token.START_ARRAY) {
parser.skipChildren();
} else {
parser.nextToken();
}
}
return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams);
return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings, dataStreams, contexts);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.opensearch.client.AbstractResponseTestCase;
import org.opensearch.client.GetAliasesResponseTests;
import org.opensearch.cluster.metadata.AliasMetadata;
import org.opensearch.cluster.metadata.Context;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.common.settings.IndexScopedSettings;
import org.opensearch.common.settings.Settings;
Expand Down Expand Up @@ -66,6 +67,7 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
final Map<String, Settings> settings = new HashMap<>();
final Map<String, Settings> defaultSettings = new HashMap<>();
final Map<String, String> dataStreams = new HashMap<>();
final Map<String, Context> contexts = new HashMap<>();
IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS;
boolean includeDefaults = randomBoolean();
for (String index : indices) {
Expand All @@ -90,14 +92,19 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
if (randomBoolean()) {
dataStreams.put(index, randomAlphaOfLength(5).toLowerCase(Locale.ROOT));
}

if (randomBoolean()) {
contexts.put(index, new Context(randomAlphaOfLength(5).toLowerCase(Locale.ROOT)));
}
}
return new org.opensearch.action.admin.indices.get.GetIndexResponse(
indices,
mappings,
aliases,
settings,
defaultSettings,
dataStreams
dataStreams,
null
);
}

Expand All @@ -116,6 +123,7 @@ protected void assertInstances(
assertEquals(serverTestInstance.getSettings(), clientInstance.getSettings());
assertEquals(serverTestInstance.defaultSettings(), clientInstance.getDefaultSettings());
assertEquals(serverTestInstance.getAliases(), clientInstance.getAliases());
assertEquals(serverTestInstance.contexts(), clientInstance.contexts());
}

private static MappingMetadata createMappingsForIndex() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,23 @@
import org.opensearch.action.support.IndicesOptions;
import org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
import org.opensearch.cluster.applicationtemplates.SystemTemplate;
import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
import org.opensearch.cluster.metadata.Context;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.IndexService;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.mapper.MapperParsingException;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.index.query.RangeQueryBuilder;
Expand All @@ -58,7 +66,10 @@
import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
import org.opensearch.test.OpenSearchIntegTestCase.Scope;

import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiFunction;
Expand Down Expand Up @@ -429,4 +440,53 @@ public void testCreateIndexWithNullReplicaCountPickUpClusterReplica() {
);
}
}

public void testCreateIndexWithContextSettingsAndTemplate() throws Exception {
int numReplicas = 1;
String indexName = "test-idx-1";
Settings settings = Settings.builder()
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), (String) null)
.build();
Context context = new Context("test");

String templateContent = "{\n"
+ " \"template\": {\n"
+ " \"settings\": {\n"
+ " \"merge.policy\": \"log_byte_size\"\n"
+ " }\n"
+ " },\n"
+ " \"_meta\": {\n"
+ " \"_type\": \"@abc_template\",\n"
+ " \"_version\": 1\n"
+ " },\n"
+ " \"version\": 1\n"
+ "}\n";

ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
internalCluster().clusterManagerClient(),
() -> internalCluster().getInstance(ClusterService.class).state()
);
loader.loadTemplate(
new SystemTemplate(
BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
SystemTemplateMetadata.fromComponentTemplateInfo("test", 1L),
new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
)
);

assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(settings).setContext(context).get());

IndicesService indicesService = internalCluster().getInstance(IndicesService.class, internalCluster().getClusterManagerName());

for (IndexService indexService : indicesService) {
assertEquals(indexName, indexService.index().getName());
assertEquals(
numReplicas,
(int) indexService.getIndexSettings().getSettings().getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, null)
);
assertEquals(context, indexService.getMetadata().context());
assertEquals("log_byte_size", indexService.getMetadata().getSettings().get(IndexSettings.INDEX_MERGE_POLICY.getKey()));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,18 @@
import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader;
import org.opensearch.cluster.applicationtemplates.SystemTemplate;
import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata;
import org.opensearch.cluster.applicationtemplates.TemplateRepositoryMetadata;
import org.opensearch.cluster.metadata.Context;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.Priority;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.index.IndexModule;
import org.opensearch.index.IndexService;
import org.opensearch.index.VersionType;
Expand All @@ -50,10 +56,14 @@
import org.opensearch.test.OpenSearchIntegTestCase;
import org.opensearch.threadpool.ThreadPool;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.UUID;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_METADATA;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_READ;
Expand Down Expand Up @@ -98,6 +108,58 @@ public void testInvalidDynamicUpdate() {
assertNotEquals(indexMetadata.getSettings().get("index.dummy"), "invalid dynamic value");
}

public void testDynamicUpdateWithContextSettingOverlap() throws IOException {
String templateContent = "{\n"
+ " \"template\": {\n"
+ " \"settings\": {\n"
+ " \"index.merge.policy\": \"log_byte_size\"\n"
+ " }\n"
+ " },\n"
+ " \"_meta\": {\n"
+ " \"_type\": \"@abc_template\",\n"
+ " \"_version\": 1\n"
+ " },\n"
+ " \"version\": 1\n"
+ "}\n";

ClusterStateSystemTemplateLoader loader = new ClusterStateSystemTemplateLoader(
internalCluster().clusterManagerClient(),
() -> internalCluster().getInstance(ClusterService.class).state()
);
loader.loadTemplate(
new SystemTemplate(
BytesReference.fromByteBuffer(ByteBuffer.wrap(templateContent.getBytes(StandardCharsets.UTF_8))),
SystemTemplateMetadata.fromComponentTemplateInfo("testcontext", 1L),
new TemplateRepositoryMetadata(UUID.randomUUID().toString(), 1L)
)
);

createIndex("test", new Context("testcontext"));

IllegalArgumentException validationException = expectThrows(
IllegalArgumentException.class,
() -> client().admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(Settings.builder().put("index.merge.policy", "tiered"))
.execute()
.actionGet()
);
assertTrue(
validationException.getMessage()
.contains("Cannot apply context template as user provide settings have overlap with the included context template")
);

assertAcked(
client().admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(Settings.builder().put("index.refresh_interval", "60s"))
.execute()
.actionGet()
);
}

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(DummySettingPlugin.class, FinalSettingPlugin.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle;
import static org.opensearch.OpenSearchException.UNKNOWN_VERSION_ADDED;
import static org.opensearch.Version.V_2_10_0;
import static org.opensearch.Version.V_2_17_0;
import static org.opensearch.Version.V_2_1_0;
import static org.opensearch.Version.V_2_3_0;
import static org.opensearch.Version.V_2_4_0;
Expand Down Expand Up @@ -1179,6 +1180,14 @@ public static void registerExceptions() {
)
);
registerExceptionHandle(new OpenSearchExceptionHandle(CryptoRegistryException.class, CryptoRegistryException::new, 171, V_2_10_0));
registerExceptionHandle(
new OpenSearchExceptionHandle(
org.opensearch.indices.InvalidIndexContextException.class,
org.opensearch.indices.InvalidIndexContextException::new,
174,
V_2_17_0
)
);
registerExceptionHandle(
new OpenSearchExceptionHandle(
org.opensearch.cluster.block.IndexCreateBlockException.class,
Expand Down
Loading
Loading